diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a9bfc24f..e8d3d028 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -111,27 +111,6 @@ jobs: - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_volume_assets.py - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_rheon_cpu_training.py - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_rheon_evaluation.py - - name: Verify actual native project Run fork Study custody and cold accounting - env: - PYTHONPATH: tests - TULDOK_NATIVE_PROJECT_GO: '1' - TULDOK_NATIVE_RETENTION_DIR: ${{ github.workspace }}/build/qa/external-project/native-retained - run: | - build/workspace-training/bin/python - <<'PY' - import os, sys, unittest - import numpy, sklearn, psycopg, torch - assert os.environ['TULDOK_TRAINING_PYTHON'] == sys.executable - names = [ - 'test_project_native_run.NativeRunSaveAsTests.test_one_actual_native_run_save_as_cold_role_and_metadata_only_git_clone', - 'test_project_native_study.NativeStudyCustodyTests.test_actual_native_study_spent_budget_and_cold_history_require_fresh_admission', - 'test_project_native_study.NativeStudyCustodyTests.test_native_study_admission_canonical_ack_then_sql_failure_never_starts_training', - ] - loader = unittest.TestLoader() - suite = loader.loadTestsFromNames(names) - assert not loader.errors and suite.countTestCases() == len(names) - result = unittest.TextTestRunner(verbosity=2).run(suite) - sys.exit(0 if result.wasSuccessful() and result.testsRun == len(names) and not result.skipped else 1) - PY - name: Create isolated optional agent verification environment run: python -m venv build/workspace-agent - run: build/workspace-agent/bin/python -m pip install -r requirements.txt @@ -145,111 +124,7 @@ jobs: - run: node --check static/experiment-lineage.js - run: node tests/test_experiment_lineage_controller.cjs - run: build/workspace-agent/bin/python -m unittest discover -s tests -p test_artifact_segments.py - - name: Verify real external-project SDK CLI MCP and current authority - env: - PYTHONPATH: tests - run: | - build/workspace-agent/bin/python - <<'PY' - import mcp, sys, unittest - loader = unittest.TestLoader() - suite = loader.loadTestsFromName('test_project_files_m2_http.ProjectFilesM2HTTP.test_real_sdk_cli_mcp_roles_lost_reply_and_stale_binding') - assert not loader.errors and suite.countTestCases() == 1 - result = unittest.TextTestRunner(verbosity=2).run(suite) - sys.exit(0 if result.wasSuccessful() and result.testsRun == 1 and not result.skipped else 1) - PY - - name: Verify independent fork and extraction across actual owner transports - env: - PYTHONPATH: tests - run: | - build/workspace-agent/bin/python - <<'PY' - import mcp, sys, unittest - names = [ - 'test_project_files_m4_clients_http.ProjectFilesM4ClientsHTTP.test_exact_file_revisions_save_as_and_current_role_through_sdk_cli_mcp', - 'test_project_transfer_clients_http.ProjectTransferClientsHTTP.test_owner_handoff_and_single_extraction_through_sdk_cli_and_real_mcp', - ] - loader = unittest.TestLoader() - suite = loader.loadTestsFromNames(names) - assert not loader.errors and suite.countTestCases() == len(names) - result = unittest.TextTestRunner(verbosity=2).run(suite) - sys.exit(0 if result.wasSuccessful() and result.testsRun == len(names) and not result.skipped else 1) - PY - - name: Verify finite owner MCP modes and denial boundaries - env: - PYTHONPATH: tests - run: | - build/workspace-agent/bin/python - <<'PY' - import mcp, sys, unittest - loader = unittest.TestLoader() - suite = loader.loadTestsFromName('test_project_files_owner_mcp') - selected = suite.countTestCases() - assert not loader.errors and selected > 0 - result = unittest.TextTestRunner(verbosity=2).run(suite) - sys.exit(0 if result.wasSuccessful() and result.testsRun == selected and not result.skipped else 1) - PY - - name: Verify real native project SDK CLI MCP and zero-GO recovery - env: - PYTHONPATH: tests - TULDOK_NATIVE_PROJECT_GO: '1' - run: | - build/workspace-agent/bin/python - <<'PY' - import mcp, sys, unittest - loader = unittest.TestLoader() - suite = loader.loadTestsFromName('test_project_native_clients_http.NativePublicClientsHTTP.test_real_native_preflight_replay_zero_go_failure_and_original_phase_recovery') - assert not loader.errors and suite.countTestCases() == 1 - result = unittest.TextTestRunner(verbosity=2).run(suite) - sys.exit(0 if result.wasSuccessful() and result.testsRun == 1 and not result.skipped else 1) - PY - - name: Verify explicit note deletion history and cold fork across owner transports - env: - PYTHONPATH: tests - TULDOK_NOTE_DELETION_GO: '1' - run: | - build/workspace-agent/bin/python - <<'PY' - import mcp, sys, unittest - loader = unittest.TestLoader() - suite = loader.loadTestsFromName('test_project_file_deletion_clients_http.NoteDeletionClientsHTTP.test_real_note_deletion_restore_history_and_cold_independent_fork') - assert not loader.errors and suite.countTestCases() == 1 - result = unittest.TextTestRunner(verbosity=2).run(suite) - sys.exit(0 if result.wasSuccessful() and result.testsRun == 1 and not result.skipped else 1) - PY - run: python -m unittest discover -s tests - - name: Verify external-project authoring and cold reconstruction in Chromium - run: node tests/browser_project_files_m2.cjs - env: - BROWSER: /usr/bin/google-chrome - - name: Verify named independent fork and inert cold clone in Chromium - run: node tests/browser_project_files_m4.cjs - env: - BROWSER: /usr/bin/google-chrome - - name: Verify explicit runtime failure recovery and inert cold Open in Chromium - run: node tests/browser_project_files_m3.cjs - env: - BROWSER: /usr/bin/google-chrome - - name: Verify source stop and independent extraction handoff in Chromium - run: node tests/test_workspace_project_transfer.cjs --browser - env: - BROWSER: /usr/bin/google-chrome - TULDOK_M5_UI_BROWSER_GO: '1' - - name: Verify named native preflight and original-phase recovery in Chromium - run: node tests/browser_project_files_native.cjs - env: - BROWSER: /usr/bin/google-chrome - TULDOK_NATIVE_PROJECT_GO: '1' - - name: Verify explicit note deletion restore and inert cold fork in Chromium - run: node tests/browser_project_file_deletion.cjs - env: - BROWSER: /usr/bin/google-chrome - TULDOK_NOTE_DELETION_GO: '1' - - name: Verify external-project working file and recovery controls - run: | - node tests/test_workspace_project_files_m4.cjs - node tests/test_workspace_project_fork_text.cjs - node tests/test_workspace_project_runtime.cjs - node tests/test_workspace_project_candidate.cjs - node tests/test_workspace_project_transfer.cjs - node tests/test_workspace_project_native.cjs - node tests/test_workspace_project_native_validation.cjs - node tests/test_workspace_project_file_deletion.cjs - run: node --check static/auth.js - run: node tests/test_auth_helpers.cjs - run: node tests/test_auth_controller.cjs diff --git a/.gitignore b/.gitignore index 8a9476b6..ad6dead7 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ data/ /output/ /test-results/ *.pdf + +# The workspace mockups' baked data file holds the Rheon Surrogate Lab's own model names and notes. +/docs/plans/project-workspace-redesign/mockups/shared/data.js diff --git a/agent_api.py b/agent_api.py index 90a534c9..edd203c3 100644 --- a/agent_api.py +++ b/agent_api.py @@ -729,778 +729,28 @@ def model_operation(name,method,path,properties=None,required=None,*,output=None 'point':obj({'field_id':REF, **{axis:integer(0,127) for axis in 'xyz'}})}, output=JSON_OBJECT, description='Read exact retained validation field-point diagnostics and scalar compatibility; no scientific criterion or compute is invented.') -# External file projects have independent lifecycle and text-only contracts. -# Keep these additive operations outside the installed legacy registry projection. -_PROJECT_FILES_ACTIONS = ['new', 'open', 'save', 'close', 'query', 'get', 'import_text', 'annotate'] -_PROJECT_FILES_GROUPS = dict(GROUPS, uniqueItems=True) -_PROJECT_FILES_PARENTS = dict(array(text(120), 30), uniqueItems=True) -_PROJECT_FILES_PROVENANCE = obj({'method': const('import'), 'rights': text(1000), - 'normalization': const('NFC; LF newlines')}) -_PROJECT_FILES_RECORD_FIELDS = { - 'id': ID, 'kind': const('text'), 'name': text(200), 'text': text(200000), - 'groups': _PROJECT_FILES_GROUPS, 'parents': _PROJECT_FILES_PARENTS, - 'provenance': _PROJECT_FILES_PROVENANCE, 'review': {'enum': ['draft', 'human_reviewed']}, - 'revision': SAFE_POS, 'source_revision': const(1), - 'created_at': text(80), 'updated_at': text(80)} -_PROJECT_FILES_TASKS = ('text_classification', 'text_entities', 'text_corpus') -PROJECT_FILES_RECORD = {'oneOf': [obj(dict(_PROJECT_FILES_RECORD_FIELDS, - task=const(task), annotation={'oneOf': [{'type': 'null'}, ANNOTATIONS[task]]})) - for task in _PROJECT_FILES_TASKS]} -PROJECT_FILES_SUMMARY = obj({key: _PROJECT_FILES_RECORD_FIELDS[key] for key in - ('id', 'kind', 'name', 'review', 'revision', 'source_revision', 'created_at', 'updated_at')} | - {'task': {'enum': list(_PROJECT_FILES_TASKS)}}) -_PROJECT_FILES_PROJECT = obj({'project_id': ID, 'name': text(200), 'checkpoint': SHA}) -_PROJECT_FILES_RECOVERY = obj({'action': const('open'), 'root': text(160), 'directory': text(512), - 'reason': {'enum': ['unknown_publication', 'projection_unavailable']}, - 'request_key': {'oneOf': [KEY, {'type': 'null'}]}}) -PROJECT_FILES_RECEIPT = {'oneOf': [ - obj({'committed': const(True), 'selected': const(True), 'project_id': ID, - 'checkpoint': SHA, 'request_key': KEY}), - obj({'committed': const(True), 'selected': const(False), 'project_id': ID, - 'checkpoint': SHA, 'request_key': KEY, 'root': text(160), 'directory': text(512)})]} -_PROJECT_FILES_STATE_FIELDS = {'format': const('project_files_state_v1'), 'binding': ID, - 'roots': dict(array(text(160), 100), uniqueItems=True), - 'available_actions': dict(array({'enum': _PROJECT_FILES_ACTIONS}, 8), uniqueItems=True)} -PROJECT_FILES_STATE = {'oneOf': [ - obj(dict(_PROJECT_FILES_STATE_FIELDS, state=const('no_project_open'), project={'type': 'null'})), - obj(dict(_PROJECT_FILES_STATE_FIELDS, state=const('open'), - project=_PROJECT_FILES_PROJECT)), - obj(dict(_PROJECT_FILES_STATE_FIELDS, state=const('recovery_required'), - project=_PROJECT_FILES_PROJECT, recovery=_PROJECT_FILES_RECOVERY, - available_actions=const(['open'])))]} -_PROJECT_FILES_BOUND_RESULT = {'binding': ID, 'project_id': ID, 'checkpoint': SHA} -PROJECT_FILES_RECORD_RESULT = obj(dict(_PROJECT_FILES_BOUND_RESULT, record=PROJECT_FILES_RECORD, - history=array(PROJECT_FILES_RECORD, 4), - next_history_before_revision={'oneOf': [SAFE_POS, {'type': 'null'}]})) -_PROJECT_FILES_MUTATION = {'binding': ID, 'expected_checkpoint': SHA, 'request_key': KEY} -model_operation('project_files.status', 'GET', '/api/workspace/project-files', output=PROJECT_FILES_STATE) -for _action in ('new', 'open'): - _fields = dict(_PROJECT_FILES_MUTATION, expected_checkpoint={'oneOf': [SHA, {'type': 'null'}]}, - root=text(160), directory=text(512)) - if _action == 'new': - _fields['name'] = text(200) - model_operation('project_files.' + _action, 'POST', '/api/workspace/project-files/' + _action, - _fields, output=PROJECT_FILES_STATE, scopes=('inspect', 'edit'), mutation=True, - description='Explicit file project ' + _action + '; requires the current process binding and checkpoint. Never executes project code.') -for _action in ('save', 'close'): - model_operation('project_files.' + _action, 'POST', '/api/workspace/project-files/' + _action, - _PROJECT_FILES_MUTATION, output=PROJECT_FILES_STATE, scopes=('inspect', 'edit'), mutation=True) -model_operation('project_files.query', 'GET', '/api/workspace/project-files/records', - {'binding': ID, 'checkpoint': SHA, 'after_id': ID, 'limit': integer(1, 100)}, - required=['binding', 'checkpoint'], query=('binding', 'checkpoint', 'after_id', 'limit'), - output=obj(dict(_PROJECT_FILES_BOUND_RESULT, items=array(PROJECT_FILES_SUMMARY, 100), - next_after_id={'oneOf': [ID, {'type': 'null'}]}))) -model_operation('project_files.get', 'GET', '/api/workspace/project-files/records/{record_id}', - {'record_id': ID, 'binding': ID, 'checkpoint': SHA, 'before_revision': SAFE_POS, - 'history_limit': integer(1, 4)}, required=['record_id', 'binding', 'checkpoint'], - query=('binding', 'checkpoint', 'before_revision', 'history_limit'), output=PROJECT_FILES_RECORD_RESULT) -model_operation('project_files.import_text', 'POST', '/api/workspace/project-files/import-text', - dict(_PROJECT_FILES_MUTATION, asset=obj({'text': text(200000), 'name': text(200), - 'groups': _PROJECT_FILES_GROUPS, 'parents': _PROJECT_FILES_PARENTS, 'rights': text(1000)}, - required=['text', 'groups'])), output=PROJECT_FILES_RECORD_RESULT, - scopes=('inspect', 'edit'), mutation=True) -model_operation('project_files.annotate', 'POST', '/api/workspace/project-files/records/{record_id}', - dict(_PROJECT_FILES_MUTATION, record_id=ID, annotation={'oneOf': [obj({ - 'revision': SAFE_POS, 'source_revision': const(1), 'task': const(task), - 'annotation': ANNOTATIONS[task], 'groups': _PROJECT_FILES_GROUPS, - 'review': {'enum': ['draft', 'human_reviewed']}}) for task in _PROJECT_FILES_TASKS]}), - output=PROJECT_FILES_RECORD_RESULT, scopes=('inspect', 'edit'), mutation=True, - description='Save ordinary human text annotation or review at the exact checkpoint. Human review additionally requires the existing owner approve-review authority.') -_PROJECT_FILES_RECEIPT_IDS = frozenset('project_files.' + action - for action in ('new', 'open', 'save', 'close', 'import_text', 'annotate')) -_MODEL_RECEIPT_IDS.update(_PROJECT_FILES_RECEIPT_IDS) -_OWNER_OPERATION_IDS.update('project_files.' + action for action in ('new', 'open', 'save', 'close')) -for _op in _OPERATIONS: - if _op['operation_id'].startswith('project_files.'): - _op['outputSchema']['oneOf'][1]['properties']['result']['properties']['receipt'] = PROJECT_FILES_RECEIPT - -# M2 operations are fixed domain projections. Domain owners validate versioned -# documents; the public boundary owns selection, bounds and outcome correlation. -PROJECT_FILES_M2 = set() -PROJECT_FILES_M3 = set() -PROJECT_FILES_M4 = set() -_PROJECT_FILES_PAGE_INPUT = {'offset': integer(0, 1000000), 'limit': integer(1, 40)} -_PROJECT_FILES_PAGE = obj({'items': array(JSON_OBJECT, 40), 'offset': integer(0, 1000000), - 'total': integer(0, 1000000), 'limit': integer(1, 40)}) -_PROJECT_FILES_READ = {'binding': ID, 'checkpoint': SHA} -_PROJECT_FILES_ORIGIN = {'origin_catalog_id': ID} - - -def _project_file_operation(action, method, suffix, fields=None, *, required=None, - output=JSON_OBJECT, mutation=False, owner=False, scopes=None, - domain=None, page=False, milestone=2, receipt=True): - fields = copy.deepcopy(fields or {}) - needed = list(fields) if required is None else list(required) - if domain is not None: - original = next(op for op in _OPERATIONS if op['operation_id'] == domain) - fields.update(copy.deepcopy(original['inputSchema'].get('properties', {}))) - needed += original['inputSchema'].get('required', []) - bound = (_PROJECT_FILES_MUTATION if receipt else {"binding": ID, "expected_checkpoint": SHA}) if mutation else _PROJECT_FILES_READ - fields.update(bound) - needed = list(dict.fromkeys(needed + list(bound))) - if action == 'data_query': - fields['limit'] = integer(1, 40) - if page: - fields.update(copy.deepcopy(_PROJECT_FILES_PAGE_INPUT)) - output = copy.deepcopy(_PROJECT_FILES_PAGE) - if action in ('data_history', 'data_response_history', 'data_preference_history'): - fields['limit'] = integer(1, 4) - output['properties']['items']['maxItems'] = 4 - output['properties']['limit'] = integer(1, 4) - name = 'project_files.' + action - model_operation(name, method, '/api/workspace/project-files' + suffix, - fields, needed, output=obj(dict(_PROJECT_FILES_BOUND_RESULT, value=output)), - query=tuple(key for key in fields if '{' + key + '}' not in suffix) if method == 'GET' else (), - scopes=scopes or (('inspect', 'edit') if mutation else ('inspect',)), mutation=mutation, - description=action.replace('_', ' ') + '; exact selected project checkpoint, existing domain authorization and validation.' + (' Historical content never grants current execution permission.' if milestone == 3 else ' No execution.')) - if domain is not None: - _OPERATIONS[-1]['binary_fields'] = copy.deepcopy(original['binary_fields']) - {2:PROJECT_FILES_M2,3:PROJECT_FILES_M3,4:PROJECT_FILES_M4}[milestone].add(name) - if mutation and receipt: - _MODEL_RECEIPT_IDS.add(name) - if owner: - _OWNER_OPERATION_IDS.add(name) - _OPERATIONS[-1]['outputSchema']['oneOf'][1]['properties']['result']['properties']['receipt'] = PROJECT_FILES_RECEIPT - - -_project_file_operation('definitions', 'GET', '/definitions') - -# Actual data and its authored histories. Inline source imports retain the -# existing decoded-byte bound; compound sources use their existing parsers. -_project_file_operation('data_query', 'GET', '/data', domain='data.query') -_project_file_operation('data_scientific', 'GET', '/data/{record_id}/scientific', - {'content_hash': SHA}, domain='data.scientific') -_project_file_operation('data_get', 'GET', '/data/{record_id}', {'record_id': ID}) -_project_file_operation('data_availability', 'GET', '/data/{record_id}/availability', {'record_id': ID}) -for _action, _suffix in [('data_history', 'history'), ('data_responses', 'responses'), - ('data_response_history', 'response-history'), ('data_preferences', 'preferences'), - ('data_preference_history', 'preference-history')]: - _project_file_operation(_action, 'GET', '/data/{record_id}/' + _suffix, {'record_id': ID}, page=True) -for _kind in ('text', 'image', 'audio', 'mesh', 'pointcloud', 'volume', 'sequence'): - _project_file_operation('data_import_' + _kind, 'POST', '/data/import-' + _kind, - domain='data.import_' + _kind, mutation=True) -_project_file_operation('source_register', 'POST', '/sources/register', - {'root': text(160), 'manifest_path': text(512)}, mutation=True, owner=True) -_project_file_operation('data_import_scientific', 'POST', '/data/import-scientific', { - 'snapshot': obj({'origin_catalog_id': ID, 'project_id': ID, 'manifest_sha256': SHA}), - 'reference_id': REF, 'expected_sha256': SHA, - 'document': obj(OPTIONAL_META, required=['groups'])}, mutation=True) -# Retained scalar packets use their own fixed native owner. Definitions and -# original JSON evidence stay producer-authored data, never executable specs. -_MEASUREMENT_ID = {'type':'string','pattern':'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$'} -_MEASUREMENT_SCALAR = {'type':['string','number','boolean','null']} -_MEASUREMENT_DIMENSIONS = {'type':'object','maxProperties':32,'additionalProperties':_MEASUREMENT_SCALAR} -_MEASUREMENT_REF = obj({'source_id':_MEASUREMENT_ID,'pointer':{'oneOf':[text(512,True),_NULL]}}) -_MEASUREMENT_DEFINITION = obj(dict(_MEASUREMENT_REF['properties'],sha256=SHA)) -_MEASUREMENT_SCOPE = obj({'id':_MEASUREMENT_ID,'label':text(240), - 'population':{'enum':['training','consumed','independent','reserved','unknown']},'definition':_MEASUREMENT_DEFINITION}) -_MEASUREMENT_METRIC = obj({'id':_MEASUREMENT_ID,'label':text(240), - 'units':text(120),'definition':_MEASUREMENT_DEFINITION}) -_MEASUREMENT_SUBJECT = obj({'id':_MEASUREMENT_ID,'label':text(240), - 'dimensions':_MEASUREMENT_DIMENSIONS,'reference':_MEASUREMENT_REF}) -_MEASUREMENT_CASE = obj({'id':_MEASUREMENT_ID,'dimensions':_MEASUREMENT_DIMENSIONS,'reference':_MEASUREMENT_REF}) -_MEASUREMENT_STATE = {'enum':['measured','failed','unsupported','unavailable']} -_MEASUREMENT_OBSERVATION = obj({'id':_MEASUREMENT_ID,'subject_id':_MEASUREMENT_ID, - 'case_id':_MEASUREMENT_ID,'metric_id':_MEASUREMENT_ID,'state':_MEASUREMENT_STATE, - 'value':{'type':['number','null']},'reason':{'oneOf':[text(1000),_NULL]}, - 'dimensions':_MEASUREMENT_DIMENSIONS,'reference':_MEASUREMENT_REF}) -_MEASUREMENT_CAPTURE = obj({'origin_catalog_id':ID,'project_id':ID,'manifest_sha256':SHA, - 'reference_id':REF,'path':text(512),'measured_sha256':SHA}) -_MEASUREMENT_SOURCE = obj({'format':const('project_measurement_source_v1'),'origin_catalog_id':ID, - 'id':SHA,'title':text(240),'blob':obj({'sha256':SHA,'bytes':integer(1,32*1024*1024)}), - 'source':_MEASUREMENT_CAPTURE,'actor_id':ID,'imported_at':text(80)}) -_MEASUREMENT_AVAILABILITY = obj({'state':{'enum':['unverified','missing']}, - 'expected_sha256':SHA,'expected_bytes':SAFE_NAT,'observed_sha256':const(None), - 'observed_bytes':{'oneOf':[SAFE_NAT,_NULL]}}) -_MEASUREMENT_SOURCE_ITEM = obj(dict(_MEASUREMENT_SOURCE['properties'],availability=_MEASUREMENT_AVAILABILITY)) -_MEASUREMENT_HEADER = obj({'format':const('retained_measurements_v1'),'packet_sha256':SHA,'title':text(240), - 'scope':_MEASUREMENT_SCOPE,'scope_definition':{'type':'object'}, - 'dimensions':array(obj({'id':_MEASUREMENT_ID,'label':text(240),'type':{'enum':['string','number','integer','boolean']}, - 'units':{'oneOf':[text(120),_NULL]},'scope':{'enum':['subject','case','observation']}}),32), - 'metrics':array(_MEASUREMENT_METRIC,32),'subject_count':integer(1,100000),'case_count':integer(1,4096), - 'observation_count':integer(1,100000),'source_count':integer(1,256)}) -_MEASUREMENT_SUMMARY = obj({'subject_id':_MEASUREMENT_ID,'count':SAFE_NAT, - 'states':obj({state:SAFE_NAT for state in ('measured','failed','unsupported','unavailable')}), - 'maximum':{'oneOf':[obj({'id':_MEASUREMENT_ID,'case_id':_MEASUREMENT_ID,'value':{'type':'number'}}),_NULL]}}) -_MEASUREMENT_QUERY = obj({'format':const('retained_measurement_query_v1'),'packet_sha256':SHA, - 'scope':_MEASUREMENT_SCOPE,'metric':_MEASUREMENT_METRIC,'scope_definition':{'type':'object'},'definition':{'type':'object'}, - 'filters':_MEASUREMENT_DIMENSIONS,'subject_ids':array(_MEASUREMENT_ID,100000,1), - 'summary':array(_MEASUREMENT_SUMMARY,100000),'rows':array(_MEASUREMENT_OBSERVATION,100), - 'total':SAFE_NAT,'offset':SAFE_NAT,'limit':integer(1,100),'sampling':const('none')}) -_MEASUREMENT_EVIDENCE = obj({'reference':_MEASUREMENT_REF,'source_sha256':SHA, - 'state':{'enum':['included','opaque','exceeds_inspection_bound']},'value':{}}) -_MEASUREMENT_INSPECTION = obj({'packet_sha256':SHA,'observation':_MEASUREMENT_OBSERVATION, - 'case':_MEASUREMENT_CASE,'subject':_MEASUREMENT_SUBJECT,'metric':_MEASUREMENT_METRIC, - 'scope':_MEASUREMENT_SCOPE,'scope_definition':{'type':'object'},'case_evidence':_MEASUREMENT_EVIDENCE, - 'scalar_evidence':_MEASUREMENT_EVIDENCE,'source':obj({'id':_MEASUREMENT_ID,'label':text(240), - 'kind':{'enum':['json','opaque']},'bytes':integer(1,8*1024*1024),'sha256':SHA})}) -_MEASUREMENT_SELECTION = {'origin_catalog_id':ID,'packet_sha256':SHA} -_MEASUREMENT_BOUND = lambda value:obj({'source':_MEASUREMENT_SOURCE,'value':value}) -_project_file_operation('measurement_import','POST','/measurements/import', - {'snapshot':obj({'origin_catalog_id':ID,'project_id':ID,'manifest_sha256':SHA}), - 'reference_id':REF,'expected_sha256':SHA},mutation=True,output=_MEASUREMENT_SOURCE) -_project_file_operation('measurement_sources','GET','/measurements',_PROJECT_FILES_PAGE_INPUT,required=[], - output=obj({'items':array(_MEASUREMENT_SOURCE_ITEM,40),'offset':SAFE_NAT,'limit':integer(1,40),'total':SAFE_NAT})) -_project_file_operation('measurement_metadata','GET','/measurements/{packet_sha256}',_MEASUREMENT_SELECTION, - output=_MEASUREMENT_BOUND(_MEASUREMENT_HEADER)) -_project_file_operation('measurement_subjects','GET','/measurements/{packet_sha256}/subjects', - dict(_MEASUREMENT_SELECTION,**_PROJECT_FILES_PAGE_INPUT),required=list(_MEASUREMENT_SELECTION), - output=_MEASUREMENT_BOUND(obj({'items':array(_MEASUREMENT_SUBJECT,40),'offset':SAFE_NAT,'limit':integer(1,40),'total':SAFE_NAT}))) -_project_file_operation('measurement_query','POST','/measurements/{packet_sha256}/query', - dict(_MEASUREMENT_SELECTION,metric_id=_MEASUREMENT_ID,subject_ids=array(_MEASUREMENT_ID,100000,1), - filters=_MEASUREMENT_DIMENSIONS,offset=SAFE_NAT,limit=integer(1,100)), - required=[*list(_MEASUREMENT_SELECTION),'metric_id'],output=_MEASUREMENT_BOUND(_MEASUREMENT_QUERY)) -_project_file_operation('measurement_inspect','GET','/measurements/{packet_sha256}/observations/{observation_id}', - dict(_MEASUREMENT_SELECTION,observation_id=_MEASUREMENT_ID),output=_MEASUREMENT_BOUND(_MEASUREMENT_INSPECTION)) - -_COMPARISON_QUERY = obj(dict( - {key:_MEASUREMENT_QUERY['properties'][key] for key in ('metric','definition','filters','subject_ids','total','offset','limit','sampling')}, - subject_ids=array(_MEASUREMENT_ID,256,1), - summary=array(obj(dict(_MEASUREMENT_SUMMARY['properties'],source=obj(_MEASUREMENT_SELECTION))),256), - rows=array(obj(dict(_MEASUREMENT_OBSERVATION['properties'],source=obj(_MEASUREMENT_SELECTION))),100))) -_COMPARISON_VALUE = obj({'format':const('retained_measurement_comparison_v1'), - 'scope':_MEASUREMENT_SCOPE,'scope_definition':{'type':'object'}, - 'dimensions':_MEASUREMENT_HEADER['properties']['dimensions'],'metrics':array(_MEASUREMENT_METRIC,32,1), - 'definition_source':obj(_MEASUREMENT_SELECTION),'case_population_sha256':SHA, - 'case_count':integer(1,4096),'observation_count':integer(1,100000), - 'subjects':array(obj(dict(_MEASUREMENT_SUBJECT['properties'],source=obj(_MEASUREMENT_SELECTION))),256,1), - 'query':{'oneOf':[_COMPARISON_QUERY,_NULL]}}) -_project_file_operation('measurement_compare','POST','/measurements/compare', - dict(sources=array(obj(_MEASUREMENT_SELECTION),16,1),metric_id=_MEASUREMENT_ID, - subject_ids=array(_MEASUREMENT_ID,256,1),filters=_MEASUREMENT_DIMENSIONS,offset=SAFE_NAT,limit=integer(1,100)), - required=['sources'],output=obj({'sources':array(_MEASUREMENT_SOURCE,16,1),'value':_COMPARISON_VALUE})) -_project_file_operation('data_annotate', 'POST', '/data/{record_id}', { - 'record_id': ID, 'revision': SAFE_POS, 'source_revision': SAFE_POS, - 'task': {'enum': list(ANNOTATIONS)}, 'annotation': JSON_OBJECT, - 'groups': _PROJECT_FILES_GROUPS, 'review': {'enum': ['draft', 'human_reviewed']}}, mutation=True) -_project_file_operation('data_rights', 'POST', '/data/{record_id}/rights', { - 'record_id': ID, 'revision': SAFE_POS, 'source_revision': SAFE_POS, - 'note': text(1000, True)}, mutation=True) -for _action, _suffix in [('data_source_save', '/source'), ('data_source_delete', '/source/delete')]: - _project_file_operation(_action, 'POST', '/data/{record_id}' + _suffix, - {'record_id': ID, 'document': JSON_OBJECT}, mutation=True, - scopes=('inspect', 'edit', 'approve-review') if _action == 'data_source_save' else ('inspect', 'edit')) -_project_file_operation('data_response_save', 'POST', '/responses', { - 'document': obj({'id': ID, 'prompt_id': ID, 'revision': SAFE_NAT, - 'parent_revision': SAFE_POS, 'source_revision': SAFE_POS, 'completion': text(200000), - 'review': {'enum': ['draft', 'human_reviewed']}})}, mutation=True) -for _action, _suffix in [('data_preference_save', ''), ('data_preference_delete', '/delete')]: - _project_file_operation(_action, 'POST', '/preferences' + _suffix, - {'document': JSON_OBJECT}, mutation=True) -for _singular, _plural, _key in [('selection', 'selections', 'selection_id'), ('search', 'searches', 'search_id')]: - _project_file_operation(_plural, 'GET', '/' + _plural, page=True) - if _singular == 'selection': - _project_file_operation('selection_get', 'GET', '/selections/{selection_id}', {'selection_id': ID}) - _project_file_operation(_singular + '_create', 'POST', '/' + _plural, - {'document': JSON_OBJECT}, mutation=True) - for _verb in ('rename', 'delete'): - _project_file_operation(_singular + '_' + _verb, 'POST', '/' + _plural + '/{' + _key + '}' + ('/delete' if _verb == 'delete' else ''), - {_key: ID, 'revision': SAFE_POS, **({'name': text(120)} if _verb == 'rename' else {})}, mutation=True) -for _action in ('preview', 'freeze'): - _project_file_operation('release_' + _action, 'POST', '/releases/' + _action, - domain='releases.' + _action, mutation=_action == 'freeze', - scopes=('inspect', 'freeze') if _action == 'freeze' else ('inspect',)) - -# Current metadata uses the same Model/Pipeline validators, with canonical -# publication wrapped by the project session. Runtime capture remains absent. -for _plural, _key, _domain in [('models', 'model_id', 'models'), ('graphs', 'graph_id', 'graphs')]: - _project_file_operation(_plural, 'GET', '/' + _plural, page=True) - for _verb in ('get', 'create', 'save'): - _project_file_operation(('model_' if _plural == 'models' else 'graph_') + _verb, - 'GET' if _verb == 'get' else 'POST', '/' + _plural + ('/{' + _key + '}' if _verb != 'create' else ''), - domain=_domain + '.' + _verb, mutation=_verb != 'get') -_project_file_operation('graph_create_external', 'POST', '/graphs/external', - domain='graphs.create_external', mutation=True) -_project_file_operation('graph_clone', 'POST', '/graphs/{graph_id}/clone', domain='graphs.clone', mutation=True) -_project_file_operation('model_publish', 'POST', '/models/{model_id}/publish', domain='models.publish', mutation=True) -_project_file_operation('recipes', 'GET', '/recipes', page=True) -for _verb in ('get', 'create', 'save'): - _project_file_operation('recipe_' + _verb, 'GET' if _verb == 'get' else 'POST', - '/recipes' + ('/{recipe_id}' if _verb != 'create' else ''), domain='recipes.' + _verb, - mutation=_verb != 'get') -_project_file_operation('recipe_prepare', 'POST', '/recipes/{graph_id}/prepare', { - 'graph_id': ID, 'revision': SAFE_POS, 'model': MODEL_REF, - 'builtin': {'type': 'boolean'}, 'reconcile_capture': {'type': 'boolean'}}, - required=['graph_id', 'revision', 'model'], mutation=True) -for _kind, _ref in [('model', MODEL_REF), ('recipe', MATERIAL_REF), ('runtime', material_ref('runtime'))]: - _project_file_operation(_kind + '_material', 'POST', '/materials/' + _kind, - {'ref': _ref, 'verify': {'type': 'boolean'}}, required=['ref']) -_project_file_operation('snapshot', 'POST', '/materials/snapshot', { - 'project_id': ID, 'manifest_sha256': SHA, 'origin_catalog_id': ID, 'verify': {'type': 'boolean'}}, - required=['project_id', 'manifest_sha256', 'origin_catalog_id']) - -# Imported origin identity is an explicit selector, never a current grant. -for _action, _suffix, _key in [('study', 'studies', 'study_id'), ('trial', 'trials', 'trial_id'), - ('run', 'runs', 'run_id'), ('candidate', 'candidates', 'candidate_id')]: - _project_file_operation(_action, 'GET', '/' + _suffix + '/{' + _key + '}', - {_key: ID, **_PROJECT_FILES_ORIGIN, **({'verify': {'type': 'boolean'}} if _action == 'candidate' else {})}, - required=[_key, 'origin_catalog_id']) -# Study reservations are independently bounded while charge totals remain complete. -_study_operation = next(op for op in _OPERATIONS if op['operation_id'] == 'project_files.study') -_study_operation['inputSchema']['properties'].update(_PROJECT_FILES_PAGE_INPUT | {'revision': SAFE_POS}) -_study_operation['query'].extend(['offset', 'limit', 'revision']) -_run_operation = next(op for op in _OPERATIONS if op['operation_id'] == 'project_files.run') -_run_operation['inputSchema']['properties'].update(_PROJECT_FILES_PAGE_INPUT) -_run_operation['query'].extend(['offset', 'limit']) -_PROJECT_RUN_DOCUMENT_SEGMENT = obj({'format': const('project_run_document_segment_v1'), - 'origin_catalog_id': ID, 'run_id': ID, 'sha256': SHA, 'bytes': SAFE_NAT, - 'offset': SAFE_NAT, 'limit': integer(1, 262144), - 'base64': {'type': 'string', 'maxLength': 349528, 'pattern': '^[A-Za-z0-9+/]*={0,2}$'}, - 'next_offset': {'oneOf': [SAFE_NAT, {'type': 'null'}]}}) -_project_file_operation('run_document_segment', 'GET', '/runs/{run_id}/document-segment', - {'run_id': ID, 'origin_catalog_id': ID, 'expected_sha256': SHA, - 'offset': SAFE_NAT, 'limit': integer(1, 262144)}, - required=['run_id', 'origin_catalog_id', 'expected_sha256', 'offset', 'limit'], - output=_PROJECT_RUN_DOCUMENT_SEGMENT) -_project_file_operation('source_references', 'GET', '/sources/{project_id}/references', - {'origin_catalog_id': ID, 'project_id': ID, 'manifest_sha256': SHA}, page=True) -_project_file_operation('publication_result', 'GET', '/runs/{run_id}/publication-result', - {'run_id': ID, **_PROJECT_FILES_ORIGIN}) -_project_file_operation('package_state', 'GET', '/candidates/{candidate_id}/package-state', - {'candidate_id': ID, **_PROJECT_FILES_ORIGIN}) -_project_file_operation('dependencies', 'GET', '/runs/{run_id}/dependencies', - {'run_id': ID, **_PROJECT_FILES_ORIGIN, **_PROJECT_FILES_PAGE_INPUT}, - required=['run_id', 'origin_catalog_id']) -for _action in ('studies', 'runs', 'lineage', 'policies', 'artifacts', 'materials'): - _project_file_operation(_action, 'GET', '/' + _action, _PROJECT_FILES_ORIGIN, - required=[] if _action in ('studies', 'runs', 'artifacts', 'materials') else ['origin_catalog_id'], page=True) -# Intrinsic artifact identities span all origins in the selected project. -_artifacts_operation = next(op for op in _OPERATIONS if op['operation_id'] == 'project_files.artifacts') -_artifacts_operation['inputSchema']['properties'].pop('origin_catalog_id') -_artifacts_operation['query'].remove('origin_catalog_id') -_project_file_operation('policy', 'POST', '/policies/get', {'ref': POLICY_REF}) -for _verb in ('create', 'save'): - _project_file_operation('study_' + _verb, 'POST', '/studies' + ('/{study_id}' if _verb == 'save' else ''), - domain='studies.' + _verb, mutation=True, owner=True) -_project_file_operation('trial_prepare', 'POST', '/studies/{study_id}/trials', { - 'study_id': ID, 'revision': SAFE_POS, 'material': MATERIAL_REF, 'replicate': SAFE_NAT}, mutation=True) -_project_file_operation('lineage_record', 'POST', '/lineage', {'trial': _LINEAGE_REF, 'intent': _LINEAGE_INTENT}, mutation=True) -_PROJECT_PROGRESSION_ORIGIN = obj({'origin_catalog_id':ID,'study_count':SAFE_NAT,'trial_count':SAFE_NAT}) -_PROJECT_PROGRESSION_RUN = obj({key:value for key,value in _LINEAGE_RUN['properties'].items() if key!='validation'}) -_PROJECT_PROGRESSION_NODE = obj({'trial':_TRIAL,'lineage':{'oneOf':[_LINEAGE_RECORD,_NULL]}, - 'study':obj({'id':ID,'name':text(120)}),'attempt_count':SAFE_NAT, - 'runs':array(_PROJECT_PROGRESSION_RUN,5),'runs_truncated':{'type':'boolean'}}) -_project_file_operation('progression_origins','GET','/progression/origins',_PROJECT_FILES_PAGE_INPUT,required=[], - output=obj({'items':array(_PROJECT_PROGRESSION_ORIGIN,40),'offset':SAFE_NAT,'limit':integer(1,40),'total':SAFE_NAT})) -_project_file_operation('progression','GET','/progression', - {'origin_catalog_id':ID,'offset':SAFE_NAT,'limit':integer(1,20),'cohort':text(120)},required=['origin_catalog_id'], - output=obj({'origin_catalog_id':ID,'evidence':const('saved_metadata_only'),'historical':const(True), - 'execution_available':const(False),'nodes':array(_PROJECT_PROGRESSION_NODE,20),'related':array(_LINEAGE_REF,40), - 'offset':SAFE_NAT,'limit':integer(1,20),'total':SAFE_NAT,'cohort':{'oneOf':[text(120),_NULL]}})) -_project_file_operation('candidate_select', 'POST', '/studies/{study_id}/candidates', - domain='studies.select_candidate', mutation=True) -_project_file_operation('evidence', 'POST', '/runs/{run_id}/evidence', { - 'run_id': ID, **_PROJECT_FILES_ORIGIN, 'selectors': JSON_OBJECT}, - required=['run_id', 'origin_catalog_id'], output=_EVIDENCE_RESULT) -for _action, _suffix in [('retained_analysis', ''), ('retained_analysis_export', '/export')]: - _project_file_operation(_action, 'POST', '/runs/{run_id}/analysis' + _suffix, - {**_retained_request['properties'], **_PROJECT_FILES_ORIGIN}, - required=[*_retained_request['required'], 'origin_catalog_id'], output=_retained_result) -_project_file_operation('artifact_status', 'POST', '/artifacts/status', {'sha256': SHA}) -_project_file_operation('artifact_resolve', 'POST', '/artifacts/resolve', - {'sha256': SHA, 'root': text(160), 'path': text(512)}, mutation=True, owner=True, - output=obj({'format': const('project_artifact_resolution_v1'), 'sha256': SHA, 'bytes': SAFE_NAT, - 'actor_id': ID, 'observed_at': text(80)})) -_project_file_operation('expose_holdout', 'POST', '/runs/{run_id}/disclose', - {'run_id': ID, **_PROJECT_FILES_ORIGIN, 'reason': text(500)}, mutation=True, - scopes=('inspect', 'expose-holdout')) -_PROJECT_FILES_PRODUCER_FAMILY = {'enum': ['generation_history', 'grounded_history', 'caption_proposal_history', 'text_proposal_history']} -_project_file_operation('producers', 'GET', '/producers', - {'family': _PROJECT_FILES_PRODUCER_FAMILY, **_PROJECT_FILES_ORIGIN}, required=['family'], page=True) -_project_file_operation('producer', 'GET', '/producers/{origin_catalog_id}/{producer_id}', - {'family': _PROJECT_FILES_PRODUCER_FAMILY, 'origin_catalog_id': ID, 'producer_id': ID, - 'entries_offset': integer(0, 1000000), 'entries_limit': integer(1, 40)}, - required=['family', 'origin_catalog_id', 'producer_id']) -PROJECT_FILES_M2 = frozenset(PROJECT_FILES_M2) -_PROJECT_FILES_RECEIPT_IDS = _PROJECT_FILES_RECEIPT_IDS | frozenset( - name for name in PROJECT_FILES_M2 if next(op for op in _OPERATIONS if op['operation_id'] == name)['side_effects']) - -# Explicit current-local execution is separate from imported history. -_PROJECT_CURRENT_GRANT = obj({'id': ID, 'catalog_id': ID, 'approver_actor_id': ID, - 'approver_credential_id': {'oneOf': [ID, {'type': 'null'}]}, 'consumer_actor_id': ID, - 'material_sha256': SHA, 'created_at': SAFE_NAT, 'expires_at': SAFE_NAT, - 'revoked_at': {'oneOf': [SAFE_NAT, {'type': 'null'}]}}) -PROJECT_CURRENT_APPROVAL = obj({'format': const('project_current_execution_approval_v1'), - 'current_local': const(True), 'grant': _PROJECT_CURRENT_GRANT}) -PROJECT_RUN_ADMISSION = obj({'format': const('project_run_admission_v1'), - 'origin_catalog_id': ID, 'run_id': ID, 'request_sha256': SHA}) -_PROJECT_EXECUTION_HEALTH = obj({'format': const('project_execution_health_v1'), - 'active_run_ids': array(ID, 40), 'active_count': SAFE_NAT, 'offset': SAFE_NAT, - 'limit': integer(1, 40), 'local_owner_active': {'type': 'boolean'}, 'available': {'type': 'boolean'}, - 'drain_required': {'type': 'boolean'}}) -_project_file_operation('execution_health', 'GET', '/execution', _PROJECT_FILES_PAGE_INPUT, - required=[], output=_PROJECT_EXECUTION_HEALTH, milestone=3) -_project_file_operation('execution_preflight_external', 'POST', '/execution/preflight', - {'graph_id': ID, 'revision': SAFE_POS}, milestone=3) -_project_file_operation('execution_approve_external', 'POST', '/execution/approvals', - {'graph_id': ID, 'graph_revision': SAFE_POS, 'expected_digest': SHA, - 'consumer_actor_id': ID, 'ttl_seconds': integer(60, 3600)}, - required=['graph_id', 'graph_revision', 'expected_digest', 'consumer_actor_id'], - output=PROJECT_CURRENT_APPROVAL, mutation=True, owner=True, - scopes=('inspect', 'approve-execution'), milestone=3) -_project_file_operation('execution_submit_external', 'POST', '/execution/runs', - {'graph_id': ID, 'graph_revision': SAFE_POS, 'expected_digest': SHA, 'approval_id': ID}, - required=['graph_id', 'graph_revision', 'expected_digest'], output=PROJECT_RUN_ADMISSION, - mutation=True, scopes=('inspect', 'execute'), milestone=3) -_OPERATIONS[-1]['inputSchema']['properties']['request_key'] = copy.deepcopy(next(op for op in _OPERATIONS if op['operation_id'] == 'runs.submit_external')['inputSchema']['properties']['request_key']) -_project_file_operation('execution_cancel', 'POST', '/execution/runs/{run_id}/cancel', - {'run_id': ID}, output=PROJECT_RUN_ADMISSION, mutation=True, - scopes=('inspect', 'cancel'), milestone=3) -_project_file_operation('execution_study_continue', 'POST', '/execution/studies/{study_id}/continue', - {'study_id': ID, 'revision': SAFE_POS, 'admission_epoch': SAFE_NAT}, output=_STUDY, - mutation=True, owner=True, scopes=('inspect', 'execute'), milestone=3) -_project_file_operation('execution_study_approve', 'POST', '/execution/studies/{study_id}/approvals', - {'study_id': ID, 'trial_id': ID, 'expected_material_sha256': SHA, 'consumer_actor_id': ID, - 'ttl_seconds': integer(60, 3600)}, - required=['study_id', 'trial_id', 'expected_material_sha256', 'consumer_actor_id'], - output=PROJECT_CURRENT_APPROVAL, mutation=True, owner=True, - scopes=('inspect', 'approve-execution'), milestone=3) -_project_file_operation('execution_study_admit', 'POST', '/execution/studies/{study_id}/runs', - {'study_id': ID, 'revision': SAFE_POS, 'admission_epoch': SAFE_NAT, 'trial_id': ID, - 'approval_id': ID, 'parent_run_id': ID}, - required=['study_id', 'revision', 'admission_epoch', 'trial_id'], - output=obj({'format': const('project_study_admission_v1'), 'origin_catalog_id': ID, - 'study_id': ID, 'trial_id': ID, 'run_id': ID, 'reservation': _RESERVATION}), - mutation=True, scopes=('inspect', 'execute'), milestone=3) -_project_file_operation('execution_study_stop', 'POST', '/execution/studies/{study_id}/stop', - {'study_id': ID, 'revision': SAFE_POS}, - output=obj({'format': const('project_study_stop_v1'), 'origin_catalog_id': ID, - 'study_id': ID, 'receipt': obj({'study_id': ID, 'stop_receipt_id': ID, - 'admission_epoch': SAFE_NAT, 'run_ids': array(ID, 100000)}), - 'study_revision': SAFE_POS, 'state': {'enum': ['draft', 'open', 'stopping', 'stopped', 'unknown']}, - 'open': {'type': 'boolean'}}), mutation=True, scopes=('inspect', 'cancel'), milestone=3) -_project_file_operation('execution_reconcile_current', 'POST', '/execution/reconcile-current', - output=obj({'format': const('project_execution_reconciliation_v1'), 'origin_catalog_id': ID, - 'run_ids': array(ID, 100000), 'available': {'type': 'boolean'}}), - mutation=True, receipt=False, owner=True, scopes=('inspect', 'cancel'), milestone=3) - -# Finite native producer grammars. Each closed document remains validated by -# its domain owner; no operation name or callable can be supplied by a client. -_PROJECT_PRODUCER_MODE = {'protocol': const('pumas_typed_v1'), - 'profile': {'oneOf': [{'type': 'null'}, {'type': 'string', 'maxLength': 128, 'pattern': '^[A-Za-z0-9_.-]*$'}]}} -_PROJECT_PRODUCER_SOURCE = {'source_id': ID, 'revision': SAFE_POS, 'source_revision': SAFE_POS, - 'server_url': text(2048), 'model': text(256), 'instruction': text(2000), - 'seed': {'oneOf': [integer(0, 4294967295), {'type': 'null'}]}} -_PROJECT_PRODUCER_LABELS = array(text(80), 30, 1) -_PROJECT_PRODUCER_START_DOCUMENTS = { - 'generation_history': obj({'server_url': text(2048), 'model': text(256), 'prompt': text(4000), - 'count': integer(1, 10000), 'strategy': {'enum': ['varied', 'repeat']}, 'session_id': text(120), - 'width': SAFE_POS, 'height': SAFE_POS, 'seed': integer(0, 4294967295), - 'prompt_url': text(2048), 'prompt_model': text(256), 'book_id': text(120, True), - 'split': {'enum': ['unassigned', 'train', 'validation', 'test']}, **_PROJECT_PRODUCER_MODE}, - required=['server_url', 'model', 'prompt', 'count', 'strategy', 'session_id']), - 'grounded_history': obj(dict(_PROJECT_PRODUCER_SOURCE, count=integer(1, 10), **_PROJECT_PRODUCER_MODE), - required=[*_PROJECT_PRODUCER_SOURCE, 'count']), - 'caption_proposal_history': obj(dict(_PROJECT_PRODUCER_SOURCE, request_id=ID, **_PROJECT_PRODUCER_MODE), - required=[*_PROJECT_PRODUCER_SOURCE, 'request_id']), - 'text_proposal_history': obj(dict(_PROJECT_PRODUCER_SOURCE, request_id=ID, - labels=_PROJECT_PRODUCER_LABELS, **_PROJECT_PRODUCER_MODE), - required=[*_PROJECT_PRODUCER_SOURCE, 'request_id', 'labels'])} -for _document in _PROJECT_PRODUCER_START_DOCUMENTS.values(): - _document['dependentRequired'] = {'protocol': ['profile'], 'profile': ['protocol']} -_PROJECT_PRODUCER_ACTION = obj({'format': const('project_producer_action_v1'), - 'family': _PROJECT_FILES_PRODUCER_FAMILY, 'origin_catalog_id': ID, 'producer_id': ID, - 'action': {'enum': ['start', 'resume', 'review', 'decide', 'cancel']}}) -_project_file_operation('execution_producer_start', 'POST', '/execution/producers', - {'family': _PROJECT_FILES_PRODUCER_FAMILY, 'document': JSON_OBJECT}, - output=_PROJECT_PRODUCER_ACTION, mutation=True, owner=True, scopes=('inspect', 'execute'), milestone=3) -_OPERATIONS[-1]['inputSchema']['allOf'] = [ - {'if': {'properties': {'family': const(family)}}, - 'then': {'properties': {'document': document}}} - for family, document in _PROJECT_PRODUCER_START_DOCUMENTS.items()] -_project_file_operation('execution_generation_resume', 'POST', '/execution/generation/{producer_id}/resume', - {'producer_id': ID}, output=_PROJECT_PRODUCER_ACTION, mutation=True, owner=True, - scopes=('inspect', 'execute'), milestone=3) -_project_file_operation('execution_grounded_review', 'POST', '/execution/grounded/{producer_id}/review', - {'producer_id': ID, 'document': obj({'revision': SAFE_POS, 'candidate_id': ID, - 'decision': {'enum': ['reject', 'admit_draft']}, 'note': text(1000)})}, - output=_PROJECT_PRODUCER_ACTION, mutation=True, scopes=('inspect', 'edit'), milestone=3) -_project_file_operation('execution_producer_decide', 'POST', '/execution/producers/{producer_id}/decide', - {'family': {'enum': ['caption_proposal_history', 'text_proposal_history']}, 'producer_id': ID, - 'document': obj({'revision': SAFE_POS, 'decision': {'enum': ['reject', 'apply_draft']}, - 'labels': _PROJECT_PRODUCER_LABELS}, required=['revision', 'decision'])}, - output=_PROJECT_PRODUCER_ACTION, mutation=True, scopes=('inspect', 'edit'), milestone=3) -_project_file_operation('execution_producer_cancel', 'POST', '/execution/producers/{producer_id}/cancel', - {'family': _PROJECT_FILES_PRODUCER_FAMILY, 'producer_id': ID}, output=_PROJECT_PRODUCER_ACTION, - mutation=True, scopes=('inspect', 'cancel'), milestone=3) -_project_file_operation('execution_producer_health', 'GET', '/execution/producers/health', - _PROJECT_FILES_PAGE_INPUT, required=[], - output=obj({'format': const('project_producer_health_v1'), - 'items': array(obj({'family': _PROJECT_FILES_PRODUCER_FAMILY, 'producer_id': ID, - 'status': text(120), 'provider_outcome': {'oneOf': [{'type': 'null'}, text(120)]}, - 'worker_alive': {'type': 'boolean'}}), 40), - 'total': SAFE_NAT, 'offset': SAFE_NAT, 'limit': integer(1, 40), 'fenced': {'type': 'boolean'}}), milestone=3) -# Fixed current-host CPU capture projections. Bounds mirror runtime_snapshots -# and recipe_process; native inventories stay canonical, never transport grants. -_PROJECT_RUNTIME_SOURCE = {'type':'string','pattern':'^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$'} -_PROJECT_RUNTIME_FILE = obj({'path':text(1024),'bytes':integer(0,512*1024*1024), - 'sha256':SHA,'mode':{'enum':[0o400,0o500]},'kind':{'enum':['data','elf']}}) -_PROJECT_RUNTIME_LAUNCH = obj({**{field:text(1024) for field in ('python','loader','home','stdlib','dynload','packages')}, - 'library_dirs':dict(array(text(1024),32,1),uniqueItems=True)}) -_PROJECT_RUNTIME_REF = obj({'kind':const('runtime'),'catalog_id':ID,'id':SHA,'sha256':SHA}) -_PROJECT_RUNTIME_SUMMARY = {'project_id':ID,'manifest_sha256':SHA,'file_count':integer(1,40000), - 'total_bytes':integer(0,2*1024*1024*1024),'actor_id':ID,'observed_at':text(80),'request_key':KEY} -_PROJECT_RUNTIME_INTENT = obj(dict(_PROJECT_RUNTIME_SUMMARY,format=const('project_runtime_capture_intent_v1'))) -_PROJECT_RUNTIME_CAPTURE = obj(dict(_PROJECT_RUNTIME_SUMMARY,format=const('project_runtime_capture_v1'),runtime_ref=_PROJECT_RUNTIME_REF)) -_project_file_operation('runtime_sources','GET','/runtime/sources',_PROJECT_FILES_PAGE_INPUT,required=[], - output=obj(dict(_PROJECT_FILES_PAGE['properties'],items=array(obj({'source':_PROJECT_RUNTIME_SOURCE}),40))), - owner=True,scopes=('inspect','admin'),milestone=3) -_project_file_operation('runtime_inspect','GET','/runtime/inspect',{'source':_PROJECT_RUNTIME_SOURCE,**_PROJECT_FILES_PAGE_INPUT},required=['source'], - output=obj({'format':const('project_runtime_inspection_v1'),'source':_PROJECT_RUNTIME_SOURCE,'inventory_sha256':SHA, - 'profile':const('captured_cpu_python_v1'),'python_version':const('3.12'),'file_count':integer(1,40000), - 'total_bytes':integer(1,2*1024*1024*1024),'manifest_bytes':integer(1,16*1024*1024),'free_bytes':SAFE_NAT, - 'packages':array(obj({'name':text(128),'version':text(128)}),256,2),'launch':_PROJECT_RUNTIME_LAUNCH, - 'files':obj(dict(_PROJECT_FILES_PAGE['properties'],items=array(_PROJECT_RUNTIME_FILE,40)))}), - owner=True,scopes=('inspect','admin'),milestone=3) -_project_file_operation('runtime_capture','POST','/runtime/capture',{'source':_PROJECT_RUNTIME_SOURCE, - 'expected_inventory_sha256':SHA,'reserve_bytes':SAFE_NAT},required=['source','expected_inventory_sha256'], - output=_PROJECT_RUNTIME_CAPTURE,mutation=True,owner=True,scopes=('inspect','admin'),milestone=3) -_project_file_operation('runtime_status','GET','/runtime/status',{'capture_key':KEY}, - output=obj({'state':{'enum':['absent','intent_recorded','captured']}, - 'intent':{'oneOf':[{'type':'null'},_PROJECT_RUNTIME_INTENT]}, - 'result':{'oneOf':[{'type':'null'},_PROJECT_RUNTIME_CAPTURE]}}),milestone=3) -# Current Candidate package preparation is distinct from approval/admission. -_PROJECT_PREPARATION_KEY={'type':'string','pattern':'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$'} -_PROJECT_PREPARATION_NAME={'type':'string','pattern':'^[A-Za-z0-9][A-Za-z0-9_.-]{0,119}$'} -for _kind,_suffix in (('package','packages'),('verifier','verifiers')): - _result=dict(_PACKAGE_PREPARED_FIELDS) - if _kind=='verifier':_result.update(package_run_id=ID,package_sha256=SHA) - _project_file_operation('execution_study_prepare_'+_kind,'POST','/execution/studies/{study_id}/'+_suffix, - domain='studies.prepare_'+_kind,output=obj(_result),mutation=True,owner=True,scopes=('inspect','edit'),milestone=3) - _OPERATIONS[-1]['inputSchema']['properties']['directory']=_PROJECT_PREPARATION_NAME -_PROJECT_PREPARATION_ABANDON=obj({'format':const('project_candidate_preparation_abandon_v1'), - 'origin_catalog_id':ID,'study_id':ID,'preparation_key':_PROJECT_PREPARATION_KEY, - 'preparation_operation':{'enum':['study_execution.prepare_package','study_execution.prepare_verifier']}, - 'input_sha256':SHA,'candidate':material_ref('candidate'),'actor_id':ID,'reason':text(1000), - 'observed_checkpoint':SHA,'occurred_at':text(80),'state':const('abandoned')}) -_project_file_operation('execution_study_abandon_preparation','POST','/execution/studies/{study_id}/preparations/abandon', - {'study_id':ID,'preparation_key':_PROJECT_PREPARATION_KEY,'reason':text(1000)}, - output=_PROJECT_PREPARATION_ABANDON,mutation=True,owner=True,scopes=('inspect','cancel'),milestone=3) -_PROJECT_PREPARATION_COMPLETED={'oneOf':[obj(_PACKAGE_PREPARED_FIELDS),obj(dict(_PACKAGE_PREPARED_FIELDS,package_run_id=ID,package_sha256=SHA))]} -_project_file_operation('execution_study_reconcile_preparation','POST','/execution/studies/{study_id}/preparations/reconcile', - {'study_id':ID,'preparation_key':_PROJECT_PREPARATION_KEY},output=_PROJECT_PREPARATION_COMPLETED, - mutation=True,owner=True,scopes=('inspect','cancel'),milestone=3,receipt=False) -# Explicitly abandon only a current local stopped copy, retaining its science. -_PROJECT_RUNTIME_ABANDON=obj({'format':const('project_runtime_capture_abandon_v1'),'project_id':ID, - 'capture_key':KEY,'input_sha256':SHA,'manifest_sha256':SHA,'actor_id':ID,'reason':text(1000), - 'observed_checkpoint':SHA,'observed_at':text(80)}) -_project_file_operation('runtime_abandon','POST','/runtime/abandon',{'capture_key':KEY,'reason':text(1000)}, - output=_PROJECT_RUNTIME_ABANDON,mutation=True,owner=True,scopes=('inspect','admin'),milestone=3) -_runtime_status=next(op for op in _OPERATIONS if op['operation_id']=='project_files.runtime_status') -_runtime_status_value=_runtime_status['outputSchema']['oneOf'][0]['properties']['result']['properties']['value'] -_runtime_status['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']={'oneOf':[ - copy.deepcopy(_runtime_status_value),obj({'state':const('abandoned'),'intent':_PROJECT_RUNTIME_INTENT, - 'result':const(None),'abandonment':_PROJECT_RUNTIME_ABANDON})]} -# Explicit current native probes and Runs. Current operator runtime is selected -# by the native owner; no imported descriptor or serialized probe is authority. -_PROJECT_NATIVE_PHASE = obj({'format':const('project_native_phase_v1'),'origin_catalog_id':ID, - 'phase_id':ID,'phase':{'enum':['probe','train','verify']},'run_id':{'oneOf':[const(None),ID]}, - 'state':{'enum':['prepared','ready','go_intent','completed','failed','unknown']}, - 'spec_sha256':SHA,'receipt_sha256':{'oneOf':[const(None),SHA]}}) -_project_file_operation('native_preflight','POST','/execution/native/preflight', - {'graph_id':ID,'revision':SAFE_POS},mutation=True,owner=True,scopes=('inspect','execute'),milestone=3) -_project_file_operation('native_submit','POST','/execution/native/runs', - {'graph_id':ID,'graph_revision':SAFE_POS,'validation_token':SHA},output=PROJECT_RUN_ADMISSION, - mutation=True,owner=True,scopes=('inspect','execute'),milestone=3) -_project_file_operation('native_retry','POST','/execution/native/runs/{run_id}/retry', - {'run_id':ID},output=PROJECT_RUN_ADMISSION,mutation=True,owner=True,scopes=('inspect','execute'),milestone=3) -_project_file_operation('native_reconcile','POST','/execution/native/phases/{phase_id}/reconcile', - {'phase_id':ID},output=_PROJECT_NATIVE_PHASE,mutation=True,owner=True, - scopes=('inspect','cancel'),milestone=3,receipt=False) -PROJECT_FILES_M3 = frozenset(PROJECT_FILES_M3) -_PROJECT_FILES_RECEIPT_IDS |= frozenset(name for name in PROJECT_FILES_M3 - if next(op for op in _OPERATIONS if op['operation_id'] == name)['side_effects'] and name not in ('project_files.execution_reconcile_current','project_files.execution_study_reconcile_preparation','project_files.native_reconcile')) - -# M4 adds independent Save As and explicit inert working-file adoption without -# changing the frozen lifecycle state/actions or earlier per-operation pins. -_PROJECT_FILE_KIND={'enum':['script','note']} -_PROJECT_FILE_VERSION_META={'revision':SAFE_POS,'sha256':SHA,'bytes':integer(0,1048576), - 'adopted_at':text(80),'actor_id':ID} -_PROJECT_FILE_DELETED_META={'revision':integer(2,9007199254740991),'deleted':const(True),'sha256':{'type':'null'}, - 'bytes':const(0),'adopted_at':text(80),'actor_id':ID} -_PROJECT_FILE_RESULT=obj({'id':SHA,'project_id':ID,'kind':_PROJECT_FILE_KIND,'path':text(512), - 'revision':SAFE_POS,'sha256':SHA,'bytes':integer(0,1048576),'adopted_at':text(80),'changed':{'type':'boolean'}}) -_PROJECT_FORK_RESULT=obj({'source_project_id':ID,'source_checkpoint':SHA,'project_id':ID,'checkpoint':SHA, - 'root':text(160),'directory':text(512),'request_key':KEY,'reconciled':{'type':'boolean'}, - 'mapping_count':SAFE_NAT,'copied_count':SAFE_NAT,'missing_regenerable_count':SAFE_NAT}) -model_operation('project_files.save_as','POST','/api/workspace/project-files/save-as', - dict(_PROJECT_FILES_MUTATION,root=text(160),directory=text(512),name=text(200)), - output=obj({'state':PROJECT_FILES_STATE,'fork':_PROJECT_FORK_RESULT}), - scopes=('inspect','edit'),mutation=True,description='Create an independent idle project fork at an explicit configured local destination; retain inert origins and exact copy completeness.') -_OPERATIONS[-1]['outputSchema']['oneOf'][1]['properties']['result']['properties']['receipt']=PROJECT_FILES_RECEIPT -PROJECT_FILES_M4.add('project_files.save_as');_OWNER_OPERATION_IDS.add('project_files.save_as');_MODEL_RECEIPT_IDS.add('project_files.save_as') -# Composition is a new owner-only operation; existing Save As pins/state stay frozen. -_PROJECT_COMPOSITION_SOURCE=obj({'root':text(160),'directory':text(512),'project_id':ID,'checkpoint':SHA}) -model_operation('project_files.compose','POST','/api/workspace/project-files/compose', - dict(_PROJECT_FILES_MUTATION,root=text(160),directory=text(512),name=text(200), - sources=array(_PROJECT_COMPOSITION_SOURCE,15,1)), - output=obj({'state':PROJECT_FILES_STATE,'fork':_PROJECT_FORK_RESULT, - 'sources':array(obj({'project_id':ID,'checkpoint':SHA}),16,2)}), - scopes=('inspect','edit'),mutation=True, - description='Compose explicit retained project checkpoints with the current editable primary into one independent inert project.') -_OPERATIONS[-1]['outputSchema']['oneOf'][1]['properties']['result']['properties']['receipt']=PROJECT_FILES_RECEIPT -PROJECT_FILES_M4.add('project_files.compose');_OWNER_OPERATION_IDS.add('project_files.compose');_MODEL_RECEIPT_IDS.add('project_files.compose') -_PROJECT_FILE_INSPECTION=obj({'id':SHA,'kind':_PROJECT_FILE_KIND,'path':text(512),'revision':SAFE_POS, - 'expected_sha256':{'oneOf':[SHA,{'type':'null'}]},'expected_bytes':integer(0,1048576), - 'observed_sha256':{'oneOf':[SHA,{'type':'null'}]},'observed_bytes':{'oneOf':[integer(0,1048576),{'type':'null'}]}, - 'state':{'enum':['unchanged','changed','missing','invalid','deleted']}, - 'reason':{'oneOf':[{'type':'null'},{'enum':['missing','unsafe_path','too_large','invalid_utf8','unreadable','unstable']}]}}) -_project_file_operation('files_inspect','GET','/files',_PROJECT_FILES_PAGE_INPUT,required=[], - output=obj(dict(_PROJECT_FILES_PAGE['properties'],items=array(_PROJECT_FILE_INSPECTION,40))),milestone=4) -_project_file_operation('file_get','GET','/files/{file_id}',{'file_id':SHA,'revision':SAFE_POS},required=['file_id'], - output=obj({'id':SHA,'project_id':ID,'kind':_PROJECT_FILE_KIND,'path':text(512), - 'version':{'oneOf':[obj(dict(_PROJECT_FILE_VERSION_META,text=text(1048576,empty=True))), - obj(dict(_PROJECT_FILE_DELETED_META,text={'type':'null'}))]}}),milestone=4) -_project_file_operation('file_history','GET','/files/{file_id}/history',{'file_id':SHA,**_PROJECT_FILES_PAGE_INPUT},required=['file_id'], - output=obj(dict(_PROJECT_FILES_PAGE['properties'],id=SHA,items=array({'oneOf':[obj(_PROJECT_FILE_VERSION_META),obj(_PROJECT_FILE_DELETED_META)]},40))),milestone=4) -_project_file_operation('file_adopt','POST','/files/adopt',{'kind':_PROJECT_FILE_KIND,'path':text(512), - 'expected_revision':SAFE_NAT,'expected_sha256':SHA},output=_PROJECT_FILE_RESULT,mutation=True,milestone=4) -_project_file_operation('files_adopt','POST','/files/adopt-many',{'items':array(obj({'kind':_PROJECT_FILE_KIND, - 'path':text(512),'expected_revision':SAFE_NAT,'expected_sha256':SHA}),40,1)}, - output=obj({'items':array(_PROJECT_FILE_RESULT,40,1),'changed_count':integer(0,40)}),mutation=True,milestone=4) -_project_file_operation('file_adopt_deletion','POST','/files/adopt-deletion',{'path':text(512), - 'expected_revision':SAFE_POS,'expected_sha256':SHA},output=obj(dict(_PROJECT_FILE_RESULT['properties'], - kind=const('note'),revision=integer(2,9007199254740991),deleted=const(True),sha256={'type':'null'},bytes=const(0),changed=const(True))), - mutation=True,milestone=4) -_PROJECT_INTENT_FAMILY={'enum':['model_definition','pipeline_graph','recipe_definition']} -_PROJECT_INTENT_FIELDS={'enum':['task','specification','source_selector','preprocessing_policy','output_policy', - 'recipe_source','inputs','runtime','release','nodes','data','evaluation_profile','model','trainer']} -_PROJECT_INTENT_RESELECT=dict(array(_PROJECT_INTENT_FIELDS,14),uniqueItems=True) -_PROJECT_FORK_ORIGIN=obj({'project_id':ID,'checkpoint':SHA,'family':_PROJECT_INTENT_FAMILY, - 'id':text(512),'revision':SAFE_POS,'created_at':text(80)}) -_PROJECT_INTENT_CURRENT=obj({'format':const('project_material_fork_intent_v1'),'catalog_id':ID,'id':ID, - 'revision':const(1),'name':text(200),'definition_family':_PROJECT_INTENT_FAMILY, - 'definition':JSON_OBJECT,'required_reselection':_PROJECT_INTENT_RESELECT}) -_project_file_operation('material_intents','GET','/material-intents',_PROJECT_FILES_PAGE_INPUT,required=[], - output=obj(dict(_PROJECT_FILES_PAGE['properties'],items=array(obj({'family':_PROJECT_INTENT_FAMILY, - 'id':text(65),'catalog_id':ID,'definition_id':ID,'name':text(200),'revision':const(1), - 'required_reselection':_PROJECT_INTENT_RESELECT,'fork_origin':_PROJECT_FORK_ORIGIN}),40))),milestone=4) -_project_file_operation('material_intent','GET','/material-intents/{family}/{definition_id}', - {'family':_PROJECT_INTENT_FAMILY,'definition_id':ID},output=obj({'family':_PROJECT_INTENT_FAMILY, - 'id':text(65),'current':_PROJECT_INTENT_CURRENT,'fork_origin':_PROJECT_FORK_ORIGIN}),milestone=4) -_project_file_operation('material_intent_capabilities','GET','/material-intents/capabilities',_PROJECT_FILES_PAGE_INPUT,required=[], - output=obj(dict(_PROJECT_FILES_PAGE['properties'],items=array(obj({'family':_PROJECT_INTENT_FAMILY, - 'id':text(65),'requires_reselection':_PROJECT_INTENT_RESELECT}),40))),milestone=4) -_PROJECT_FORK_MAPPING=obj({'family':text(80),'source_id':text(512),'source_revision':{'oneOf':[SAFE_POS,{'type':'null'}]},'destination_id':text(512)}) -_project_file_operation('fork_info','GET','/fork',_PROJECT_FILES_PAGE_INPUT,required=[],output=obj({ - 'source':{'oneOf':[{'type':'null'},obj({'project_id':ID,'checkpoint':SHA})]},'project_id':ID, - 'mapping':obj(dict(_PROJECT_FILES_PAGE['properties'],items=array(_PROJECT_FORK_MAPPING,40))), - 'archived_count':SAFE_NAT,'copied_count':SAFE_NAT,'missing_regenerable_count':SAFE_NAT}),milestone=4) -PROJECT_FILES_M4=frozenset(PROJECT_FILES_M4) -_PROJECT_FILES_RECEIPT_IDS |= frozenset(name for name in PROJECT_FILES_M4 if next(op for op in _OPERATIONS if op['operation_id']==name)['side_effects']) - -# M5 is a separate current-owner controller, never a project-open authority. -PROJECT_TRANSFER_OPERATIONS=frozenset('project_transfer.'+action for action in ('status','preview','report_page','stop','extract')) -_PROJECT_TRANSFER_KEY={'type':'string','pattern':'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$'} -_PROJECT_TRANSFER_ALIAS={'type':'string','pattern':'^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$'} -_PROJECT_TRANSFER_DIRECTORY={'type':'string','minLength':1,'maxLength':200,'pattern':r'^[^/\\\x00-\x1f]+$','not':{'enum':['.','..']}} -_PROJECT_TRANSFER_ROOTS={ - 'records':ID,'responses':ID,'preferences':ID,'selections':ID,'searches':ID,'graphs':ID, - 'recipe_drafts':ID,'model_definitions':ID,'studies':ID,'runs':ID,'releases':SHA,'exposures':ID, - 'model_materials':SHA,'recipe_materials':SHA,'runtime_materials':SHA} -_PROJECT_TRANSFER_FAMILIES=['workbench_record','workbench_response','workbench_preference','saved_selection','saved_search', - 'pipeline_graph','recipe_definition','model_definition','study_history','run_history','dataset_release','dataset_exposure', - 'model_material','recipe_material','runtime_material','trial_history','candidate_history','project_snapshot', - 'generation_history','grounded_history','caption_proposal_history','text_proposal_history','project_artifact','project_file'] -_PROJECT_TRANSFER_SECTION={'enum':['included','required','unassigned','excluded','blockers','warnings','artifacts','working_files']} -_PROJECT_TRANSFER_LOCATOR={'family':{'enum':_PROJECT_TRANSFER_FAMILIES},'id':text(128)} -_PROJECT_TRANSFER_SELECTION=obj({'project_snapshot':obj({'id':ID,'manifest_sha256':SHA}), - 'roots':obj({**{name:dict(array(schema,10000),uniqueItems=True) for name,schema in _PROJECT_TRANSFER_ROOTS.items()}, - 'producer_jobs':dict(array(obj({'kind':{'enum':['generation','grounded','caption','text_proposal']},'id':ID}),10000),uniqueItems=True), - 'working_files':dict(array(obj({'kind':{'enum':['script','note']},'path':text(512)}),1000),uniqueItems=True)},required=[]), - 'decisions':array(obj(dict(_PROJECT_TRANSFER_LOCATOR,include={'type':'boolean'},reason=text(1000))),10000)}) -_PROJECT_TRANSFER_SOURCE=obj({'catalog_id':ID,'schema_version':const(10),'catalog_signature':SHA, - 'snapshot_sha256':SHA,'backup_receipt_sha256':{'oneOf':[SHA,{'type':'null'}]},'external_inventory_sha256':SHA}) -_PROJECT_TRANSFER_COUNTS=obj({name:SAFE_NAT for name in _PROJECT_TRANSFER_SECTION['enum']}) -_PROJECT_TRANSFER_REPORT=obj({'report_sha256':SHA,'selection_sha256':SHA,'source':_PROJECT_TRANSFER_SOURCE, - 'can_extract':{'type':'boolean'},'counts':_PROJECT_TRANSFER_COUNTS}) -_PROJECT_TRANSFER_RECEIPT=obj({'format':const('project_transfer_receipt_v1'),'committed':const(True), - 'project_id':ID,'checkpoint':SHA,'request_key':_PROJECT_TRANSFER_KEY,'input_sha256':SHA,'report_sha256':SHA, - 'source':_PROJECT_TRANSFER_SOURCE,'artifacts':obj({'copied_count':SAFE_NAT,'missing_regenerable_count':SAFE_NAT}), - 'verification':obj({'checkpoint':SHA,'active_data_records':SAFE_NAT}),'replayed':{'type':'boolean'}}) -_PROJECT_TRANSFER_PARTIAL=obj({'committed':const(True),'selected':const(False),'project_id':ID,'checkpoint':SHA, - 'request_key':_PROJECT_TRANSFER_KEY,'directory':_PROJECT_TRANSFER_DIRECTORY}) -_PROJECT_TRANSFER_FAILURE=obj({'action':{'enum':['stop','extract']},'code':text(120),'detail':text(1000,empty=True), - 'staging_directory':text(200)},required=['action','code','detail']) -PROJECT_TRANSFER_STATE=obj({'format':const('project_transfer_state_v1'),'binding':ID, - 'phase':{'enum':['source_live','source_stopped','source_fenced']},'source_catalog_id':ID,'control_catalog_id':ID, - 'roots':dict(array(_PROJECT_TRANSFER_ALIAS,100),uniqueItems=True), - 'available_actions':dict(array({'enum':['status','preview','report_page','stop','extract']},5,1),uniqueItems=True), - 'report':{'oneOf':[{'type':'null'},_PROJECT_TRANSFER_REPORT]}, - 'backup':{'oneOf':[{'type':'null'},obj({'root':_PROJECT_TRANSFER_ALIAS,'directory':_PROJECT_TRANSFER_DIRECTORY,'receipt_sha256':SHA})]}, - 'destination':{'oneOf':[{'type':'null'},obj({'root':_PROJECT_TRANSFER_ALIAS,'directory':_PROJECT_TRANSFER_DIRECTORY})]}, - 'failure':{'oneOf':[{'type':'null'},_PROJECT_TRANSFER_FAILURE]}, - 'last_receipt':{'oneOf':[{'type':'null'},_PROJECT_TRANSFER_RECEIPT,_PROJECT_TRANSFER_PARTIAL]}}) -_PROJECT_TRANSFER_ITEMS={ - 'included':obj(dict(_PROJECT_TRANSFER_LOCATOR,revision={'oneOf':[SAFE_POS,{'type':'null'}]})), - 'unassigned':obj(dict(_PROJECT_TRANSFER_LOCATOR,revision={'oneOf':[SAFE_POS,{'type':'null'}]})), - 'required':obj(_PROJECT_TRANSFER_LOCATOR),'excluded':obj(_PROJECT_TRANSFER_LOCATOR), - 'blockers':obj({'family':{'enum':_PROJECT_TRANSFER_FAMILIES+['source','closure']},'id':text(128),'code':text(120),'detail':text(1000,empty=True)}), - 'warnings':obj({'family':{'enum':_PROJECT_TRANSFER_FAMILIES+['source','closure']},'id':text(128),'code':text(120),'detail':text(1000,empty=True)}), - 'artifacts':obj({'sha256':SHA,'bytes':integer(0,2**63-1),'preservation':{'enum':['unique','regenerable']},'state':{'enum':['verified','missing','corrupt']}}), - 'working_files':obj({'kind':{'enum':['script','note']},'path':text(512),'sha256':SHA,'bytes':integer(0,1048576)})} -_PROJECT_TRANSFER_PAGE=obj({'format':const('project_transfer_report_page_v1'),'binding':ID,'report_sha256':SHA, - 'section':_PROJECT_TRANSFER_SECTION,'items':array({'oneOf':list(_PROJECT_TRANSFER_ITEMS.values())},40), - 'offset':integer(0,1000000),'limit':integer(1,40),'total':SAFE_NAT}) -# Some sections share item schemas; section-specific validation below selects one. -_PROJECT_TRANSFER_PAGE['properties']['items']['items']={} -_PROJECT_TRANSFER_PAGE['allOf']=[{'if':{'properties':{'section':const(section)}},'then':{'properties':{'items':array(schema,40)}}} - for section,schema in _PROJECT_TRANSFER_ITEMS.items()] -_PROJECT_TRANSFER_RESULT=obj({'format':const('project_transfer_extraction_result_v1'),'binding':ID, - 'receipt':_PROJECT_TRANSFER_RECEIPT,'selection':obj({'selected':{'type':'boolean'}, - 'state':{'oneOf':[{'type':'null'},PROJECT_FILES_STATE]}, - 'error':{'oneOf':[{'type':'null'},obj({'code':text(120),'detail':text(1000,empty=True)})]}})}) -for _action,_method,_suffix,_fields,_needed,_output,_mutation in ( - ('status','GET','',{},[],PROJECT_TRANSFER_STATE,False), - ('preview','POST','/preview',{'binding':ID,'selection':_PROJECT_TRANSFER_SELECTION},None,PROJECT_TRANSFER_STATE,True), - ('report_page','GET','/report',{'binding':ID,'report_sha256':SHA,'section':_PROJECT_TRANSFER_SECTION, - 'offset':integer(0,1000000),'limit':integer(1,40)},['binding','report_sha256','section'],_PROJECT_TRANSFER_PAGE,False), - ('stop','POST','/stop',{'binding':ID,'report_sha256':SHA,'backup_root':_PROJECT_TRANSFER_ALIAS, - 'backup_directory':_PROJECT_TRANSFER_DIRECTORY,'destination_root':_PROJECT_TRANSFER_ALIAS,'directory':_PROJECT_TRANSFER_DIRECTORY},None,PROJECT_TRANSFER_STATE,True), - ('extract','POST','/extract',{'binding':ID,'report_sha256':SHA,'destination_root':_PROJECT_TRANSFER_ALIAS, - 'directory':_PROJECT_TRANSFER_DIRECTORY,'name':text(200),'request_key':_PROJECT_TRANSFER_KEY},None,_PROJECT_TRANSFER_RESULT,True)): - _name='project_transfer.'+_action - model_operation(_name,_method,'/api/workspace/project-transfer'+_suffix,_fields,_needed, - output=_output,scopes=('inspect','admin'),mutation=_mutation,query=tuple(_fields) if _method=='GET' else (), - description='Explicit current-owner legacy extraction '+_action.replace('_',' ')+'. Source and control authority remain separate; no execution or credential transfer.') - _OPERATIONS[-1]['outputSchema']['oneOf'][1]['properties']['result']['properties']['receipt']=_PROJECT_TRANSFER_PARTIAL - _OWNER_OPERATION_IDS.add(_name) -_MODEL_RECEIPT_IDS.add('project_transfer.extract') - _OWNER_SNAPSHOT_HTTP_IDS=frozenset(('project_snapshots.list','project_snapshots.get','project_snapshots.open')) +# A served definition-first project folder (spec-m7.md, section 7): one contract +# per row of tuldok_project.operations, offered only by the project_folder profile. +from tuldok_project import operations as _project_folder_table +_PROJECT_FOLDER_SCOPES = {'inspect': ('inspect',), 'edit': ('inspect', 'edit'), 'cancel': ('inspect', 'cancel'), + 'request': ('inspect', 'execute'), 'owner': ('inspect', 'approve-execution')} +_PROJECT_FOLDER_REFUSAL = obj({'error': text(1000), 'code': text(120), + 'problems': array({'type': 'object'}, 50), 'references': {'type': 'array', 'items': {'type': 'string'}}, + 'files': {'type': 'array', 'items': {'type': 'string'}}}, required=['error']) +for _row in _project_folder_table.OPERATIONS: + _name = 'project.' + _row.name + register(_name, 'POST', '/api/project/' + _row.name, schema=copy.deepcopy(_row.arguments), + scopes=_PROJECT_FOLDER_SCOPES[_row.who], mutation=_row.mutation, description=_row.summary) + _OPERATIONS[-1]['outputSchema'] = {'type': 'object', 'oneOf': [ + obj({'status': integer(200, 299), 'result': {'type': 'object', 'required': list(_row.result)}}), + obj({'status': integer(400, 599), 'result': copy.deepcopy(_PROJECT_FOLDER_REFUSAL)})]} + if _row.who == 'owner': + _OWNER_OPERATION_IDS.add(_name) +PROJECT_FOLDER_OPERATIONS = frozenset('project.' + _row.name for _row in _project_folder_table.OPERATIONS) _LOOKUP = {key: operation for operation in _OPERATIONS for key in (operation['operation_id'], operation['mcp_name'])} # Explicit locally implemented versions. No remotely supplied code or routes. _CONTRACTS = {op['operation_id']: {1: copy.deepcopy(op)} for op in _OPERATIONS} -# Keep the installed v1 bounds/hash exact. Version 2 admits complete larger -# comparisons; each individual packet and every response page retain their caps. -_comparison_capacity=copy.deepcopy(_CONTRACTS['project_files.measurement_compare'][1]) -_comparison_value=_comparison_capacity['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties'] -_comparison_value['observation_count']=integer(1,250000) -_comparison_value['query']['oneOf'][0]['properties']['total']=integer(0,250000) -_CONTRACTS['project_files.measurement_compare'][2]=_comparison_capacity -# v1 remains the closed Run-only current custody view. v2 additionally exposes -# bounded original local native phases; imported history never populates it. -_native_health=copy.deepcopy(_CONTRACTS['project_files.execution_health'][1]) -_native_health_value=_native_health['outputSchema']['oneOf'][0]['properties']['result']['properties']['value'] -_native_health_value['properties']['native_phases']=obj({'items':array(_PROJECT_NATIVE_PHASE,40), - 'total':SAFE_NAT,'offset':SAFE_NAT,'limit':integer(1,40)}) -_native_health_value['required'].append('native_phases') -_CONTRACTS['project_files.execution_health'][2]=_native_health - -# Fork-origin text requires an additive result contract; import-only v1 remains -# exact. Both record and each historical revision retain the actual origin. -_PROJECT_FILES_FORK_PROVENANCE=obj({'method':const('project_fork'),'rights':text(1000), - 'origin':obj({'project_id':ID,'checkpoint':SHA,'family':{'enum':['workbench_text','workbench_record']}, - 'id':ID,'revision':SAFE_POS,'source_revision':SAFE_POS})}) -for _name in ('project_files.get','project_files.annotate'): - _fork_text=copy.deepcopy(_CONTRACTS[_name][1]) - _fork_text['description']+=' Version 2 retains exact independent-fork provenance.' - _result=_fork_text['outputSchema']['oneOf'][0]['properties']['result']['properties'] - for _record_schema in (_result['record'],_result['history']['items']): - for _variant in _record_schema['oneOf']: - _variant['properties']['provenance']={'oneOf':[copy.deepcopy(_PROJECT_FILES_PROVENANCE),copy.deepcopy(_PROJECT_FILES_FORK_PROVENANCE)]} - _CONTRACTS[_name][2]=_fork_text - # Activity changes the closed output schema, so installed snapshot v1 contracts # retain their exact identity and wire projection. for _name in ('project_snapshots.get','project_snapshots.open'): @@ -1728,19 +978,6 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non obj({'actor_id': ID, 'request_key': KEY, 'request_sha256': SHA})]} _publication_status_output['required'].append('continuation_request') _CONTRACTS['runs.publication_status'][2] = _publication_status_request -# Project mode is an additive consumer of the current fixed graph grammars. -# Copy only inputs after the domain versions have been composed; installed -# legacy graph contracts and all project binding/outcome fields retain owners. -for _project_name, _domain_name in ( - ('project_files.graph_create', 'graphs.create'), - ('project_files.graph_create_external', 'graphs.create_external'), - ('project_files.graph_save', 'graphs.save')): - _current_graph_input = copy.deepcopy(_CONTRACTS[_domain_name][max(_CONTRACTS[_domain_name])]['inputSchema']) - _current_graph_input['properties'].update(copy.deepcopy(_PROJECT_FILES_MUTATION)) - _current_graph_input['required'] = list(dict.fromkeys( - _current_graph_input['required'] + list(_PROJECT_FILES_MUTATION))) - _CONTRACTS[_project_name][1]['inputSchema'] = _current_graph_input - CONTRACT_HEADER = 'Tuldok-Operation-Contract' CAPABILITIES_LIMIT = 1024 * 1024 MAX_OPERATIONS = 256 @@ -1784,32 +1021,44 @@ def contract(name, version=1): 'authority': 'owner' if name in _OWNER_OPERATION_IDS else 'agent', 'effects': 'mutation' if op['side_effects'] else 'read', 'retry': retry, 'bounds': {'request_bytes': REQUEST_LIMIT, 'response_bytes': RESPONSE_LIMIT, 'inline_decoded_bytes': INLINE_LIMIT, 'json_depth': DEPTH_LIMIT}} - if name in PROJECT_TRANSFER_OPERATIONS:public['bounds']['response_bytes']=REQUEST_LIMIT identity = dict(public, id=name, method=op['method'], path=op['path'], query=op['query'], injected=op['injected'], binary_fields=op['binary_fields'], mcp_name=op['mcp_name']) public['sha256'] = hashlib.sha256(encoded(identity)).hexdigest() return public +def _project_folder_capabilities(catalog_id): + """The project_folder profile: every project.* operation, context.get and identity.get, all available.""" + names = ['context.get', 'identity.get'] + ['project.' + row.name for row in _project_folder_table.OPERATIONS] + operations = [{'id': name, 'contracts': [contract(name, version) for version in sorted(_CONTRACTS[name])], + 'availability': 'available'} for name in names] + value = {'format': 'agent_capabilities_v1', 'envelope_version': 1, 'catalog_id': catalog_id, + 'operations': operations, 'limits': {'request_bytes': REQUEST_LIMIT, + 'response_bytes': RESPONSE_LIMIT, 'max_operations': MAX_OPERATIONS}} + if len(encoded(value)) > CAPABILITIES_LIMIT: + raise AgentAPIError('Capabilities exceed the manifest byte bound.', 'registry_invalid', 503) + return value + + def capabilities(catalog_id, profile='catalog'): if type(catalog_id) is not str or not re.fullmatch('[a-f0-9]{32}', catalog_id): raise AgentAPIError('Invalid capability catalog identity.', 'registry_invalid', 503) - if profile not in ('catalog','project'): + if profile == 'project_folder': + return _project_folder_capabilities(catalog_id) + if profile != 'catalog': raise AgentAPIError('Unsupported capability profile.','registry_invalid',503) operations = [] for name, versions in _CONTRACTS.items(): - if (profile=='catalog' and name.startswith('project_files.') - or profile=='project' and not name.startswith(('project_files.','project_transfer.')) and name not in ('context.get','identity.get')): + if name in PROJECT_FOLDER_OPERATIONS: continue # The protocol bound applies to this advertised profile, just as the - # receiver checks its one manifest. Disjoint catalog/project contracts - # are not all transmitted together. Keep the installed 256-op limit. + # receiver checks its one manifest. Keep the installed 256-op limit. if len(operations) >= MAX_OPERATIONS: raise AgentAPIError('Operation profile exceeds its capability bound.', 'registry_invalid', 503) if not 1 <= len(versions) <= MAX_CONTRACTS: raise AgentAPIError('Contract version count exceeds its bound.', 'registry_invalid', 503) operations.append({'id': name, 'contracts': [contract(name, version) for version in sorted(versions)], - 'availability': 'unavailable' if name in PROJECT_TRANSFER_OPERATIONS else 'available'}) + 'availability': 'available'}) value = {'format': 'agent_capabilities_v1', 'envelope_version': 1, 'catalog_id': catalog_id, 'operations': operations, 'limits': {'request_bytes': REQUEST_LIMIT, 'response_bytes': RESPONSE_LIMIT, 'max_operations': MAX_OPERATIONS}} @@ -1981,39 +1230,6 @@ def operation(name, version=None): return copy.deepcopy(versions[max(versions) if version is None else version]) -def measurement_comparison_row_limit(version=None): - """Return the selected immutable public contract's comparison budget.""" - selected=operation('project_files.measurement_compare',version) - return selected['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties']['observation_count']['maximum'] - - -def _native_validation_result_ok(value, revision): - """Check the finite public truth fields without interpreting native runtime data. - - The native owner validates its scientific release/runtime observation. This - transport check prevents contradictory success and coercion into an ACK. - """ - fields={'status','valid','errors','validation_token','graph_revision','execution_sha256','release','runtime'} - def sha(raw): - return type(raw) is str and re.fullmatch('[a-f0-9]{64}',raw) is not None - if (type(value) is not dict or not fields.issubset(value) or set(value)-fields-{'selection_policy'} - or type(value['graph_revision']) is not int or value['graph_revision']!=revision - or type(value['valid']) is not bool or type(value['status']) is not str - or not value['status'].strip() or len(value['status'])>120 - or not sha(value['execution_sha256']) or type(value['errors']) is not list or len(value['errors'])>100 - or any(type(value[field]) is not dict and value[field] is not None for field in ('release','runtime'))): - return False - errors=value['errors'] - if any(type(error) is not dict or set(error)!={'path','message','code'} - or any(type(part) is not str or len(part)>2000 for part in error.values()) for error in errors): - return False - if value['valid']: - return (value['status']=='valid' and not errors and sha(value['validation_token']) - and value['release'] is not None and value['runtime'] is not None - and value['runtime'].get('available') is True) - return bool(errors) and value['status']==errors[0]['code'] and value['validation_token'] is None - - def _finite(value, depth=0, *, nul_path=None, _path=()): if depth > DEPTH_LIMIT: raise AgentAPIError('JSON nesting exceeds the transport bound.') @@ -2124,54 +1340,9 @@ def validate_registry(*, checkpoint=None): def validate_output(name, envelope, version=None): op = operation(name, version) - # A captured UTF-8 file is an exact inert byte witness, including NUL. This - # exception never admits NUL in input, metadata, error or other operation. - nul_path = ('result','value','version','text') if (op['operation_id']=='project_files.file_get' - and type(envelope) is dict and type(envelope.get('status')) is int and envelope['status']<400) else None - _finite(envelope, nul_path=nul_path) + _finite(envelope) if next(_validator(op['outputSchema']).iter_errors(envelope), None) is not None: raise AgentAPIError('HTTP result does not match its operation envelope.', 'transport', 502) - if op['operation_id'] in PROJECT_TRANSFER_OPERATIONS and envelope['status']<400: - value=envelope['result'] - if op['operation_id'] in ('project_transfer.status','project_transfer.preview','project_transfer.stop'): - phase,report=value['phase'],value['report'] - actions=['status'] - if phase in ('source_live','source_stopped'):actions+=['preview','report_page'] - if phase=='source_live' and report is not None:actions.append('stop') - if phase=='source_stopped' and report is not None:actions.append('extract') - if (value['source_catalog_id']==value['control_catalog_id'] or value['roots']!=sorted(set(value['roots'])) - or value['available_actions']!=actions - or phase=='source_stopped' and (value['backup'] is None or value['destination'] is None) - or phase=='source_fenced' and report is not None - or report is not None and (report['source']['catalog_id']!=value['source_catalog_id'] - or report['can_extract']!=(report['counts']['blockers']==0) - or phase=='source_live' and report['can_extract'] - or phase=='source_stopped' and report['source']['backup_receipt_sha256']!=value['backup']['receipt_sha256'])): - raise AgentAPIError('Extraction state differs from its source, phase or stopped backup.','transport',502) - if op['operation_id']=='project_transfer.extract': - receipt,selection=value['receipt'],value['selection'] - if (receipt['project_id']==receipt['source']['catalog_id'] or receipt['verification']['checkpoint']!=receipt['checkpoint'] - or selection['selected'] and (selection['error'] is not None or selection['state'] is None - or selection['state']['state']!='open' or selection['state']['project']['project_id']!=receipt['project_id'] - or selection['state']['project']['checkpoint']!=receipt['checkpoint']) - or not selection['selected'] and selection['error'] is None): - raise AgentAPIError('Extraction receipt differs from its independent project or selected state.','transport',502) - if op['operation_id'].startswith('project_files.') and envelope['status'] < 400: - value = envelope['result'] - if 'record' in value: - record, history = value['record'], value['history'] - revisions = [item['revision'] for item in history] - cursor = value['next_history_before_revision'] - if (any(item['id'] != record['id'] or item['revision'] > record['revision'] for item in history) - or revisions != sorted(set(revisions), reverse=True) - or cursor is not None and (not history or cursor != history[-1]['revision'])): - raise AgentAPIError('Project annotation history differs from its record.', 'transport', 502) - if 'items' in value: - identities = [item['id'] for item in value['items']] - cursor = value['next_after_id'] - if (identities != sorted(set(identities)) - or cursor is not None and (not identities or cursor != identities[-1])): - raise AgentAPIError('Project record page identity or continuation differs.', 'transport', 502) if op['operation_id'] == 'runs.artifact_segments' and envelope['status'] < 400: validate_artifact_segments(envelope['result']) if op['operation_id'] in ('runs.phase_timings', 'graphs.preparation_timings') and envelope['status'] < 400: @@ -2317,7 +1488,7 @@ def content_sha(value): def validate_arguments(name, arguments, version=None): op = operation(name, version) _finite(arguments) - if len(encoded(arguments, ensure_ascii=not op['operation_id'].startswith('project_files.'))) > REQUEST_LIMIT: + if len(encoded(arguments)) > REQUEST_LIMIT: raise AgentAPIError('Request exceeds the agent transport bound.', 'too_large', 413) if (op['operation_id'] == 'data.import_sequence' and type(arguments) is dict and type(arguments.get('files')) is dict @@ -2490,7 +1661,7 @@ def watch_read(): timer.daemon = True timer.start() try: - raw = encoded(body, ensure_ascii=not (selection is not None and selection[0].startswith('project_files.'))) if body is not None else None + raw = encoded(body) if body is not None else None if raw is not None and len(raw) > REQUEST_LIMIT: raise AgentAPIError('Request exceeds the agent transport bound.', 'too_large', 413) headers = {'Authorization': 'Bearer '+self._token, 'Accept': 'application/json', 'Accept-Encoding': 'identity', 'Connection': 'close'} @@ -2520,7 +1691,7 @@ def watch_read(): or response.getheader('Content-Type', '').split(';')[0].strip() != 'application/json'): raise AgentAPIError('HTTP response framing or content type is unsupported.', 'transport', 502) length = int(lengths[0]) - receive_limit=CAPABILITIES_LIMIT if path=='/api/agent/capabilities' or selection is not None and selection[0] in PROJECT_TRANSFER_OPERATIONS else RESPONSE_LIMIT + receive_limit=CAPABILITIES_LIMIT if path=='/api/agent/capabilities' else RESPONSE_LIMIT if length > receive_limit: raise AgentAPIError('HTTP response exceeds the agent receive bound.', 'response_too_large', 413) data = bytearray() @@ -2537,13 +1708,18 @@ def watch_read(): data.extend(chunk) if time.monotonic() > deadline: raise TimeoutError() - nul_path = ('value','version','text') if (selection is not None - and selection[0]=='project_files.file_get' and response.status<400) else None - value = decode(data, nul_path=nul_path) + value = decode(data) self._check_read_budget() if response.status >= 400: if not isinstance(value, dict) or not isinstance(value.get('error'), str): raise AgentAPIError('Malformed HTTP error response.', 'transport', 502) + if selection is not None and selection[0] in PROJECT_FOLDER_OPERATIONS: + # A project folder's refusal codes have hyphens (not-found) and carry its problems. + if (type(value.get('code')) is not str or not re.fullmatch('[a-z][a-z0-9_-]{0,79}', value['code']) + or self._token in json.dumps(value, ensure_ascii=False)): + raise AgentAPIError('Malformed project refusal.', 'transport', 502) + return {'status': response.status, 'result': {key: value[key] for key in + ('error', 'code', 'problems', 'references', 'files') if key in value}} code = value.get('code', 'http_error') if (type(code) is not str or not re.fullmatch('[a-z][a-z0-9_]{0,79}', code) or self._token in code): @@ -2552,20 +1728,7 @@ def watch_read(): report = value['preflight_report'] if has_report else None if has_report and self._token in json.dumps(report, ensure_ascii=False): raise AgentAPIError('Preflight report contains credential material.', 'transport', 502) - receipt = None - if selection is not None and selection[0].startswith('project_files.') and 'receipt' in value: - receipt = value['receipt'] - if (next(_validator(PROJECT_FILES_RECEIPT).iter_errors(receipt), None) is not None - or self._token in json.dumps(receipt, ensure_ascii=False)): - raise AgentAPIError('Malformed project outcome receipt.', 'transport', 502) - if selection is not None and selection[0] in PROJECT_TRANSFER_OPERATIONS and 'receipt' in value: - receipt=value['receipt'] - if (next(_validator(_PROJECT_TRANSFER_PARTIAL).iter_errors(receipt),None) is not None - or self._token in json.dumps(receipt,ensure_ascii=False)): - raise AgentAPIError('Malformed extraction outcome receipt.','transport',502) value = {'code': code, 'error': value['error'].replace(self._token, '[redacted]')[:1000]} - if receipt is not None: - value['receipt'] = copy.deepcopy(receipt) if has_report: value['preflight_report'] = report return {'status': response.status, 'result': value} @@ -2671,253 +1834,6 @@ def call(self, name, arguments, *, observer=None, observation=None): envelope = self._request(method, path, body, selection=selection) result = validate_output(name, envelope, version) self._check_read_budget() - if op['operation_id'] in PROJECT_TRANSFER_OPERATIONS: - value=result['result'];action=op['operation_id'].split('.')[1] - if result['status']>=400: - if value.get('receipt') is not None and value['receipt']['request_key']!=arguments.get('request_key'): - raise AgentAPIError('Extraction outcome belongs to a different request.','transport',502) - elif action in ('status','preview','stop'): - if action=='stop': - if (value['source_catalog_id']!=self.catalog_id or value['phase']!='source_stopped' - or value['binding']==arguments['binding'] or value['report'] is not None - or value['backup']['root']!=arguments['backup_root'] or value['backup']['directory']!=arguments['backup_directory'] - or value['destination']!={'root':arguments['destination_root'],'directory':arguments['directory']}): - raise AgentAPIError('Stopped-source result differs from the explicit backup or destination.','transport',502) - else: - current=value['source_catalog_id'] if value['phase']=='source_live' else value['control_catalog_id'] - if current!=self.catalog_id or action=='preview' and (value['binding']!=arguments['binding'] or value['report'] is None): - raise AgentAPIError('Extraction state belongs to another current host or process binding.','transport',502) - elif action=='report_page': - offset,limit=arguments.get('offset',0),arguments.get('limit',40) - if (any(value[field]!=arguments[field] for field in ('binding','report_sha256','section')) - or value['offset']!=offset or value['limit']!=limit - or len(value['items'])!=min(limit,max(0,value['total']-offset))): - raise AgentAPIError('Extraction report page differs from its reviewed report or continuation.','transport',502) - elif (value['binding']!=arguments['binding'] or value['receipt']['request_key']!=arguments['request_key'] - or value['receipt']['report_sha256']!=arguments['report_sha256']): - raise AgentAPIError('Extraction receipt belongs to another controller report or request.','transport',502) - if op['operation_id'] in ('project_files.save_as','project_files.compose') and result['status']<400: - value=result['result'];fork=value['fork'];state=value['state'] - if (state['state']!='open' or state['project']['project_id']!=fork['project_id'] - or state['project']['checkpoint']!=fork['checkpoint'] or state['project']['name']!=arguments['name'] or state['binding']==arguments['binding'] - or fork['project_id']==fork['source_project_id'] - or fork['source_checkpoint']!=arguments['expected_checkpoint'] - or any(fork[field]!=arguments[field] for field in ('root','directory','request_key'))): - raise AgentAPIError('Save As result differs from its source, destination or selected state.','transport',502) - if op['operation_id']=='project_files.compose': - expected=[dict(project_id=fork['source_project_id'],checkpoint=arguments['expected_checkpoint'])] - expected.extend({key:item[key] for key in ('project_id','checkpoint')} for item in arguments['sources']) - if value['sources']!=expected: - raise AgentAPIError('Composition result differs from its exact ordered sources.','transport',502) - elif op['operation_id'] in PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4: - value = result['result'] - if result['status'] >= 400: - receipt = value.get('receipt') - if receipt is not None and receipt['request_key'] != arguments.get('request_key'): - raise AgentAPIError('Project receipt belongs to a different request.', 'transport', 502) - else: - if value['binding'] != arguments['binding']: - raise AgentAPIError('Project response belongs to a different session binding.', 'transport', 502) - if not op['side_effects'] and value['checkpoint'] != arguments['checkpoint']: - raise AgentAPIError('Project read response belongs to a different checkpoint.', 'transport', 502) - page = value['value'] - action=op['operation_id'].removeprefix('project_files.') - if action=='files_adopt': - if (len(page['items'])!=len(arguments['items']) - or page['changed_count']!=sum(item['changed'] for item in page['items']) - or sum(item['bytes'] for item in page['items'])>8*1024*1024): - raise AgentAPIError('Batch adoption count or capture bound differs.','transport',502) - for item,reviewed in zip(page['items'],arguments['items']): - identity=hashlib.sha256((value['project_id']+'\0'+reviewed['kind']+'\0'+reviewed['path']).encode()).hexdigest() - if (item['id']!=identity or item['project_id']!=value['project_id'] - or item['kind']!=reviewed['kind'] or item['path']!=reviewed['path'] - or item['sha256']!=reviewed['expected_sha256'] - or item['revision']!=reviewed['expected_revision']+int(item['changed'])): - raise AgentAPIError('Batch adoption differs from its reviewed files.','transport',502) - if action in ('file_get','file_adopt','file_adopt_deletion'): - kind,path=page['kind'],page['path'] - expected_id=hashlib.sha256((value['project_id']+'\0'+kind+'\0'+path).encode()).hexdigest() - if page['id']!=expected_id or page['project_id']!=value['project_id']: - raise AgentAPIError('Working file belongs to a different project or path.','transport',502) - if action=='file_get': - version=page['version'];deleted=version.get('deleted') is True - raw=None if deleted else version['text'].encode('utf-8') - if (deleted and kind!='note' or page['id']!=arguments['file_id'] or arguments.get('revision',version['revision'])!=version['revision'] - or not deleted and (len(raw)!=version['bytes'] or len(raw)>1048576 or hashlib.sha256(raw).hexdigest()!=version['sha256'])): - raise AgentAPIError('Captured working file differs from its exact revision or bytes.','transport',502) - elif action=='file_adopt_deletion': - if (kind!='note' or path!=arguments['path'] or page['revision']!=arguments['expected_revision']+1): - raise AgentAPIError('Note deletion differs from the reviewed path or revision.','transport',502) - elif (kind!=arguments['kind'] or path!=arguments['path'] or page['sha256']!=arguments['expected_sha256'] - or page['revision']!=arguments['expected_revision']+int(page['changed'])): - raise AgentAPIError('Working-file adoption differs from the reviewed current bytes.','transport',502) - if action=='file_history' and page['id']!=arguments['file_id']: - raise AgentAPIError('Working-file history belongs to another file.','transport',502) - if action=='files_inspect': - for item in page['items']: - identity=hashlib.sha256((value['project_id']+'\0'+item['kind']+'\0'+item['path']).encode()).hexdigest() - if (item['id']!=identity or item['state']=='unchanged' and (item['observed_sha256']!=item['expected_sha256'] - or item['observed_bytes']!=item['expected_bytes'] or item['reason'] is not None) - or item['expected_sha256'] is None and (item['kind']!='note' or item['expected_bytes']!=0 or item['state'] in ('unchanged','missing')) - or item['state']=='deleted' and (item['expected_sha256'] is not None or item['observed_sha256'] is not None or item['observed_bytes'] is not None or item['reason'] is not None)): - raise AgentAPIError('Working-file observation differs from its selected identity or declared bytes.','transport',502) - if action=='material_intents': - for item in page['items']: - if (item['catalog_id']!=value['project_id'] or item['id']!=value['project_id']+'.'+item['definition_id'] - or item['fork_origin']['family']!=item['family']): - raise AgentAPIError('Material intent summary belongs to another definition.','transport',502) - if action=='material_intent_capabilities' and any(not item['id'].startswith(value['project_id']+'.') - or not re.fullmatch('[a-f0-9]{32}',item['id'][33:]) for item in page['items']): - raise AgentAPIError('Material readiness belongs to another selected project.','transport',502) - if action=='material_intent' and (page['family']!=arguments['family'] or page['current']['id']!=arguments['definition_id'] - or page['current']['definition_family']!=arguments['family'] or page['fork_origin']['family']!=arguments['family'] - or page['current']['catalog_id']!=value['project_id'] or page['id']!=value['project_id']+'.'+arguments['definition_id']): - raise AgentAPIError('Material intent belongs to a different definition.','transport',502) - if action=='fork_info': - if page['project_id']!=value['project_id']: - raise AgentAPIError('Fork mapping belongs to a different selected project.','transport',502) - mapping=page['mapping'];offset=arguments.get('offset',0);limit=arguments.get('limit',40) - if (mapping['offset']!=offset or mapping['limit']!=limit - or len(mapping['items'])!=min(limit,max(0,mapping['total']-offset))): - raise AgentAPIError('Fork mapping differs from its requested continuation.','transport',502) - if isinstance(page, dict) and {'items', 'offset', 'limit', 'total'} <= page.keys(): - requested = arguments.get('limit', 1 if op['operation_id'] in ('project_files.data_history', 'project_files.data_response_history', 'project_files.data_preference_history') else 40) - if (page['offset'] != arguments.get('offset', 0) or page['limit'] != requested - or len(page['items']) > requested or (len(page['items']) > 0 and page['total'] < page['offset'] + len(page['items'])) - or len(page['items']) != min(requested, max(0, page['total'] - page['offset']))): - raise AgentAPIError('Project page differs from its requested continuation.', 'transport', 502) - if 'origin_catalog_id' in arguments and isinstance(page, dict) and 'origin_catalog_id' in page: - if page['origin_catalog_id'] != arguments['origin_catalog_id']: - raise AgentAPIError('Historical response belongs to a different origin.', 'transport', 502) - action = op['operation_id'].split('.')[1] - if action == 'execution_health': - requested = arguments.get('limit', 40) - if (page['offset'] != arguments.get('offset', 0) or page['limit'] != requested - or page['active_run_ids'] != sorted(set(page['active_run_ids'])) - or len(page['active_run_ids']) != min(requested, max(0, page['active_count'] - page['offset']))): - raise AgentAPIError('Current custody page differs from its requested continuation.', 'transport', 502) - if version>=2: - phases=page['native_phases'];identities=[item['phase_id'] for item in phases['items']] - if (phases['offset']!=arguments.get('offset',0) or phases['limit']!=requested - or len(phases['items'])!=min(requested,max(0,phases['total']-phases['offset'])) - or identities!=sorted(set(identities)) - or any(item['origin_catalog_id']!=value['project_id'] for item in phases['items'])): - raise AgentAPIError('Current native phase page differs from its selected project or continuation.','transport',502) - if action in ('execution_study_prepare_package','execution_study_prepare_verifier','execution_study_reconcile_preparation'): - if (page['trial']['study_id']!=arguments['study_id'] or page['trial']['material']['catalog_id']!=value['project_id'] - or page['model']['catalog_id']!=value['project_id'] - or action!='execution_study_reconcile_preparation' and (page['candidate_id']!=arguments['candidate_id'] - or page['runtime_snapshot_sha256']!=arguments['runtime_snapshot_sha256']) - or action=='execution_study_prepare_verifier' and any(page[field]!=arguments[field] for field in ('package_run_id','package_sha256'))): - raise AgentAPIError('Candidate preparation belongs to a different Study, Candidate or reviewed runtime.','transport',502) - if action=='execution_study_abandon_preparation' and (page['origin_catalog_id']!=value['project_id'] - or page['study_id']!=arguments['study_id'] or page['preparation_key']!=arguments['preparation_key'] - or page['reason']!=arguments['reason'] or page['candidate']['catalog_id']!=value['project_id']): - raise AgentAPIError('Preparation abandonment differs from its current exact intent and reason.','transport',502) - if action=='runtime_inspect': - inventory=page['files'];offset=arguments.get('offset',0);limit=arguments.get('limit',40) - if (page['source']!=arguments['source'] or inventory['offset']!=offset or inventory['limit']!=limit - or inventory['total']!=page['file_count'] - or len(inventory['items'])!=min(limit,max(0,inventory['total']-offset)) - or [item['path'] for item in inventory['items']]!=sorted(set(item['path'] for item in inventory['items'])) - or [item['name'] for item in page['packages']]!=sorted(set(item['name'] for item in page['packages']))): - raise AgentAPIError('Runtime inventory differs from its configured source or continuation.','transport',502) - if action=='runtime_capture': - ref=page['runtime_ref'] - if (page['project_id']!=value['project_id'] or page['manifest_sha256']!=arguments['expected_inventory_sha256'] - or page['request_key']!=arguments['request_key'] or ref['catalog_id']!=value['project_id'] or ref['id']!=ref['sha256']): - raise AgentAPIError('Runtime capture differs from its exact reviewed inventory or request.','transport',502) - if action=='runtime_abandon' and (page['project_id']!=value['project_id'] - or page['capture_key']!=arguments['capture_key'] or page['reason']!=arguments['reason']): - raise AgentAPIError('Runtime abandonment differs from its current exact capture and reason.','transport',502) - if action=='runtime_status': - intent,complete=page['intent'],page['result'] - abandonment=page.get('abandonment') - expected='abandoned' if abandonment is not None else 'captured' if complete is not None else 'intent_recorded' if intent is not None else 'absent' - if (page['state']!=expected or complete is not None and intent is None - or abandonment is not None and (intent is None or complete is not None - or abandonment['project_id']!=value['project_id'] or abandonment['capture_key']!=arguments['capture_key'] - or abandonment['manifest_sha256']!=intent['manifest_sha256']) - or any(item['project_id']!=value['project_id'] or item['request_key']!=arguments['capture_key'] - for item in (intent,complete) if item is not None) - or complete is not None and (any(intent[field]!=complete[field] for field in - ('project_id','manifest_sha256','file_count','total_bytes','request_key')) - or complete['runtime_ref']['catalog_id']!=value['project_id'] - or complete['runtime_ref']['id']!=complete['runtime_ref']['sha256'])): - raise AgentAPIError('Runtime status differs from its retained capture request.','transport',502) - if action=='native_preflight' and not _native_validation_result_ok(page,arguments['revision']): - raise AgentAPIError('Native validation differs from the saved graph revision or result.','transport',502) - if action=='native_reconcile' and (page['origin_catalog_id']!=value['project_id'] or page['phase_id']!=arguments['phase_id']): - raise AgentAPIError('Native phase observation differs from the selected current phase.','transport',502) - if action in ('native_submit','native_retry') and (page['origin_catalog_id']!=value['project_id'] - or action=='native_retry' and page['run_id']==arguments['run_id']): - raise AgentAPIError('Native admission differs from the selected project.','transport',502) - if action in ('execution_approve_external', 'execution_study_approve'): - grant = page['grant'] - expected = arguments['expected_digest'] if action == 'execution_approve_external' else arguments['expected_material_sha256'] - if (grant['catalog_id'] != value['project_id'] or grant['material_sha256'] != expected - or grant['consumer_actor_id'] != arguments['consumer_actor_id'] - or grant['revoked_at'] is not None or grant['expires_at'] <= grant['created_at']): - raise AgentAPIError('Current local approval differs from its reviewed material or actor.', 'transport', 502) - if action in ('execution_submit_external', 'execution_cancel', 'execution_reconcile_current', - 'execution_study_admit', 'execution_study_stop') or action.startswith('execution_producer_') or action in ('execution_generation_resume', 'execution_grounded_review'): - if 'origin_catalog_id' in page and page['origin_catalog_id'] != value['project_id']: - raise AgentAPIError('Current execution acknowledgement belongs to another project.', 'transport', 502) - if action == 'execution_cancel' and page['run_id'] != arguments['run_id']: - raise AgentAPIError('Cancellation acknowledgement belongs to another Run.', 'transport', 502) - if action == 'execution_study_continue' and page['id'] != arguments['study_id']: - raise AgentAPIError('Study continuation belongs to another Study.', 'transport', 502) - if action in ('execution_study_admit', 'execution_study_stop') and page['study_id'] != arguments['study_id']: - raise AgentAPIError('Study acknowledgement belongs to another Study.', 'transport', 502) - if action == 'execution_study_admit' and (page['trial_id'] != arguments['trial_id'] - or page['reservation']['trial_id'] != page['trial_id'] or page['reservation']['run_id'] != page['run_id']): - raise AgentAPIError('Study admission differs from its selected Trial reservation.', 'transport', 502) - if action == 'execution_study_stop' and (page['receipt']['study_id'] != page['study_id'] - or len(page['receipt']['run_ids']) != len(set(page['receipt']['run_ids']))): - raise AgentAPIError('Study epoch closure receipt is inconsistent.', 'transport', 502) - producer_actions = {'execution_producer_start': 'start', 'execution_generation_resume': 'resume', - 'execution_grounded_review': 'review', 'execution_producer_decide': 'decide', 'execution_producer_cancel': 'cancel'} - if action in producer_actions: - family = arguments.get('family', 'generation_history' if action == 'execution_generation_resume' else 'grounded_history') - if (page['family'] != family or page['action'] != producer_actions[action] - or 'producer_id' in arguments and page['producer_id'] != arguments['producer_id'] - or action == 'execution_producer_start' and 'request_id' in arguments['document'] and page['producer_id'] != arguments['document']['request_id']): - raise AgentAPIError('Producer acknowledgement differs from its exact current action.', 'transport', 502) - if op['operation_id'] == 'project_files.run_document_segment': - try: - segment = base64.b64decode(page['base64'], validate=True) - except (ValueError, binascii.Error): - raise AgentAPIError('Invalid retained Run document segment bytes.', 'transport', 502) from None - end = arguments['offset'] + len(segment) - if (page['origin_catalog_id'] != arguments['origin_catalog_id'] or page['run_id'] != arguments['run_id'] - or page['sha256'] != arguments['expected_sha256'] or page['offset'] != arguments['offset'] - or page['limit'] != arguments['limit'] or page['offset'] > page['bytes'] - or len(segment) != min(page['limit'], page['bytes'] - page['offset']) - or page['next_offset'] != (end if end < page['bytes'] else None)): - raise AgentAPIError('Retained Run document segment differs from its exact selection.', 'transport', 502) - if action.startswith('measurement_'): - _validate_measurement_selection(action,page,arguments,value['project_id']) - target_key = {'data_get': 'record_id', 'model_get': 'model_id', 'graph_get': 'graph_id', - 'recipe_get': 'recipe_id', 'trial': 'trial_id'}.get(op['operation_id'].split('.')[1]) - if target_key and page.get('id') != arguments[target_key]: - raise AgentAPIError('Project response belongs to a different record.', 'transport', 502) - if op['operation_id'].startswith('project_files.') and result['status'] < 400: - value = result['result']; action = op['operation_id'].split('.')[1] - if action in ('query', 'get', 'import_text', 'annotate', 'save') and value['binding'] != arguments['binding']: - raise AgentAPIError('Project response belongs to a different session binding.', 'transport', 502) - if action in ('query', 'get') and value['checkpoint'] != arguments['checkpoint']: - raise AgentAPIError('Project read response belongs to a different checkpoint.', 'transport', 502) - if action == 'query' and (len(value['items']) > arguments.get('limit', 40) - or any(item['id'] <= arguments.get('after_id', '') for item in value['items'])): - raise AgentAPIError('Project record page differs from its requested continuation.', 'transport', 502) - if action in ('get', 'import_text', 'annotate'): - if (len(value['history']) > arguments.get('history_limit', 1) - or 'before_revision' in arguments and any(item['revision'] >= arguments['before_revision'] for item in value['history'])): - raise AgentAPIError('Project history page differs from its requested continuation.', 'transport', 502) - if action in ('get', 'annotate') and value['record']['id'] != arguments['record_id']: - raise AgentAPIError('Project response belongs to a different record.', 'transport', 502) - if action == 'save' and (value['state'] != 'open' or value['project']['checkpoint'] != arguments['expected_checkpoint']): - raise AgentAPIError('Project Save response differs from its selected checkpoint.', 'transport', 502) if op['operation_id'] in ('runs.coordinator_health', 'runs.recover_coordinator') and result['status'] < 400: value = result['result']; request = value['recovery_request'] if value['catalog_id'] != self.catalog_id: @@ -3029,14 +1945,10 @@ def call(self, name, arguments, *, observer=None, observation=None): return result -class ProjectFilesOwnerClient(AgentClient): - """Explicit owner file-project access; no unrelated owner operations.""" - # Large retained projects may outlast the generic transport's five minutes. - # Only an explicit owner configuration opts into a longer absolute wait. - MAX_TIMEOUT_SECONDS = 7200 - - OPERATIONS = frozenset('project_files.' + action for action in - ('status', 'new', 'open', 'save', 'close', 'query', 'get', 'import_text', 'annotate')) | PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4 +class PublicationOwnerClient(AgentClient): + """Explicit owner publication actions only; ordinary AgentClient stays agent-only.""" + OPERATIONS = frozenset(('runs.publication_status', 'runs.recover_publication', 'runs.cancel_publication', + 'runs.publication_result', 'runs.publication_artifact_chunk')) def bootstrap(self): result = self._bootstrap('owner') @@ -3046,134 +1958,13 @@ def bootstrap(self): def call(self, name, arguments, **kwargs): if name not in self.OPERATIONS: - raise AgentAPIError('Owner project client only supports explicit file-project operations.', 'operation_unsupported', 409) + raise AgentAPIError('Owner publication client only supports explicit publication operations.', 'operation_unsupported', 409) return super().call(name, arguments, **kwargs) -class ProjectTransferOwnerClient(AgentClient): - """Five current-owner extraction calls, with explicit authority handoff. - - After a dispatched Stop, discard this client. Construct another instance - with the independently obtained current host credential and catalog ID. - """ - OPERATIONS=PROJECT_TRANSFER_OPERATIONS - - def __init__(self,*args,**kwargs): - super().__init__(*args,**kwargs) - self._handoff_required=False - - def bootstrap(self): - if self._handoff_required: - raise AgentAPIError('Stop requires an independently authenticated current-host client.','authentication_handoff',409) - result=self._bootstrap('owner') - self._selected={name:version for name,version in self._selected.items() if name in self.OPERATIONS} - result['operations']=sorted(self._selected) - return result - - def call(self,name,arguments,**kwargs): - if name not in self.OPERATIONS: - raise AgentAPIError('Transfer owner client only supports five explicit extraction operations.','operation_unsupported',409) - if self._handoff_required: - raise AgentAPIError('Discard the source client and authenticate the current host independently.','authentication_handoff',409) - try: - result=super().call(name,arguments,**kwargs) - except AgentAPIError as error: - if name=='project_transfer.stop' and (error.status>=500 or error.code in ('transport','unknown','outcome_unknown')): - self._handoff_required=True - raise - if name=='project_transfer.stop': - # Even a failed stop can rotate binding or close source authority. - # No response causes an automatic credential/catalog substitution. - self._handoff_required=True - return result - - -def _validate_measurement_selection(action, page, arguments, project_id): - """Correlate fixed scalar replies; scientific validity remains upstream.""" - def refuse(): - raise AgentAPIError('Retained measurements differ from the selected source or continuation.', 'transport', 502) - if action=='measurement_compare': - return _validate_comparison_selection(page, arguments) - if action=='measurement_sources': - offset,limit=arguments.get('offset',0),arguments.get('limit',40) - keys=[(d['origin_catalog_id'],d['id']) for d in page['items']] - if (page['offset']!=offset or page['limit']!=limit or keys!=sorted(set(keys)) - or len(keys)!=min(limit,max(0,page['total']-offset))):refuse() - return - if action=='measurement_import': - source=page['source'] - if (page['origin_catalog_id']!=project_id or page['id']!=arguments['expected_sha256'] - or page['blob']['sha256']!=page['id'] or source['measured_sha256']!=page['id'] - or source['reference_id']!=arguments['reference_id'] - or any(source[k]!=v for k,v in arguments['snapshot'].items())):refuse() - return - source,result=page['source'],page['value'] - if (source['origin_catalog_id']!=arguments['origin_catalog_id'] or source['id']!=arguments['packet_sha256'] - or source['blob']['sha256']!=source['id'] or source['source']['measured_sha256']!=source['id']):refuse() - if action!='measurement_subjects' and result['packet_sha256']!=source['id']:refuse() - if action=='measurement_inspect' and result['observation']['id']!=arguments['observation_id']:refuse() - if action=='measurement_metadata' and result['title']!=source['title']:refuse() - if action in ('measurement_query','measurement_subjects'): - offset,limit=arguments.get('offset',0),arguments.get('limit',50 if action=='measurement_query' else 40) - rows=result['rows' if action=='measurement_query' else 'items'] - if (result['offset']!=offset or result['limit']!=limit - or len(rows)!=min(limit,max(0,result['total']-offset))):refuse() - if action=='measurement_query': - subjects=result['subject_ids'] - if (result['metric']['id']!=arguments['metric_id'] or result['filters']!=arguments.get('filters',{}) - or 'subject_ids' in arguments and subjects!=arguments['subject_ids'] - or len(subjects)!=len(set(subjects)) or [s['subject_id'] for s in result['summary']]!=subjects - or any(r['metric_id']!=arguments['metric_id'] or r['subject_id'] not in subjects for r in result['rows'])):refuse() - - -def _validate_comparison_selection(page, arguments): - """Bind aggregate replies to reviewed packets, subjects and exact page intent.""" - def refuse(): - raise AgentAPIError('Measurement comparison differs from its selected packets or continuation.', 'transport', 502) - def key(locator): - return locator['origin_catalog_id'], locator['packet_sha256'] - selected = arguments['sources'] - keys = [key(item) for item in selected] - documents, value = page['sources'], page['value'] - if (len(keys)!=len(set(keys)) or len(documents)!=len(keys) - or [(doc['origin_catalog_id'],doc['id']) for doc in documents]!=keys - or any(doc['blob']['sha256']!=doc['id'] or doc['source']['measured_sha256']!=doc['id'] for doc in documents) - or sum(doc['blob']['bytes'] for doc in documents)>128*1024*1024 - or value['definition_source']!=selected[0]):refuse() - subjects = value['subjects'] - identities = [subject['id'] for subject in subjects] - if (len(identities)!=len(set(identities)) or any(key(subject['source']) not in keys for subject in subjects) - or value['observation_count']!=len(subjects)*value['case_count']*len(value['metrics'])):refuse() - owners = {subject['id']:subject['source'] for subject in subjects} - query = value['query'] - if 'metric_id' not in arguments: - if query is not None:refuse() - return - if query is None:refuse() - wanted = arguments.get('subject_ids', identities) - offset,limit = arguments.get('offset',0),arguments.get('limit',50) - metrics = [metric for metric in value['metrics'] if metric['id']==arguments['metric_id']] - if (len(metrics)!=1 or query['metric']!=metrics[0] or query['subject_ids']!=wanted - or len(wanted)!=len(set(wanted)) or any(identity not in owners for identity in wanted) - or query['filters']!=arguments.get('filters',{}) - or query['offset']!=offset or query['limit']!=limit - or len(query['rows'])!=min(limit,max(0,query['total']-offset)) - or [summary['subject_id'] for summary in query['summary']]!=wanted - or sum(summary['count'] for summary in query['summary'])!=query['total']):refuse() - for summary in query['summary']: - if (summary['source']!=owners[summary['subject_id']] - or sum(summary['states'].values())!=summary['count'] - or (summary['maximum'] is None)!=(summary['states']['measured']==0)):refuse() - for row in query['rows']: - if (row['subject_id'] not in wanted or row['metric_id']!=arguments['metric_id'] - or row['source']!=owners[row['subject_id']] - or (row['value'] is None)!=(row['state']!='measured')):refuse() - - -class PublicationOwnerClient(AgentClient): - """Explicit owner publication actions only; ordinary AgentClient stays agent-only.""" - OPERATIONS = frozenset(('runs.publication_status', 'runs.recover_publication', 'runs.cancel_publication', - 'runs.publication_result', 'runs.publication_artifact_chunk')) +class ProjectFolderOwnerClient(AgentClient): + """The owner of a served project folder: the project.* operations only.""" + OPERATIONS = PROJECT_FOLDER_OPERATIONS def bootstrap(self): result = self._bootstrap('owner') @@ -3182,8 +1973,9 @@ def bootstrap(self): return result def call(self, name, arguments, **kwargs): - if name not in self.OPERATIONS: - raise AgentAPIError('Owner publication client only supports explicit publication operations.', 'operation_unsupported', 409) + # MCP names (project_summary) are aliases of the operation IDs (project.summary). + if name not in _LOOKUP or _LOOKUP[name]['operation_id'] not in self.OPERATIONS: + raise AgentAPIError('Owner project folder client only supports project folder operations.', 'operation_unsupported', 409) return super().call(name, arguments, **kwargs) diff --git a/ai_http.py b/ai_http.py index c1ea21bf..933a0b2b 100644 --- a/ai_http.py +++ b/ai_http.py @@ -88,6 +88,29 @@ def collect(value, depth=0): return 'Provider detail: ' + ' | '.join(messages)[:340] if messages else 'The provider returned no readable error message.' +def pumas_error_detail(data, payload=None, headers=None): + """Pumas' OpenAI-compatible errors carry its (redacted) reason in `error.message`. + + Returns '' for any body that is not a bounded `pumas_error` or typed + operation error; that reason is + still passed through the same redaction as other provider diagnostics. + """ + if not data or len(data) > MAX_ERROR_BODY: + return '' + try: + value = json.loads(data) + except (ValueError, UnicodeError): + return '' + error = value.get('error') if isinstance(value, dict) else None + # Typed operation errors (contract version 1) carry `code`, `outcome` and `message` instead of `type`. + typed = isinstance(error, dict) and value.get('contract_version') == 1 and isinstance(error.get('outcome'), str) + if (not isinstance(error, dict) or (error.get('type') != 'pumas_error' and not typed) + or not isinstance(error.get('message'), str) or not error['message'].strip()): + return '' + detail = provider_error_detail(json.dumps({'message': error['message']}).encode(), payload, headers) + return 'Pumas reported: ' + detail.removeprefix('Provider detail: ') if detail.startswith('Provider detail: ') else '' + + def request_error_hint(data): """Classify bounded provider errors without exposing arbitrary response text.""" try: @@ -184,18 +207,24 @@ def watch_cancel(): 403: 'Check account permissions and provider restrictions.', 429: 'Rate limit reached; wait before trying again.'}.get(response.status, 'Check the selected model and structured-output support.') - if response.status == 400 or response.status >= 500: - try: - transport.settimeout(max(.001, min(2, deadline - time.monotonic()))) - error_data = response.read(MAX_ERROR_BODY + 1) - specific = request_error_hint(error_data) if len(error_data) <= MAX_ERROR_BODY else '' + try: + transport.settimeout(max(.001, min(2, deadline - time.monotonic()))) + error_data = response.read(MAX_ERROR_BODY + 1) + except (OSError, http.client.HTTPException): + error_data = None + if label == 'OpenRouter' and (response.status == 400 or response.status >= 500): + hint = 'Could not read the provider error body within the diagnostic limit.' + if error_data is not None: + specific = request_error_hint(error_data) if len(error_data) <= MAX_ERROR_BODY else '' + pumas_detail = pumas_error_detail(error_data, payload, headers) + if pumas_detail: + # Pumas' own (redacted) reason is shown for every refusal status. + hint = (specific + ' ' if specific else '') + pumas_detail + elif response.status == 400 or response.status >= 500: if label == 'OpenRouter' or response.status >= 500: hint = provider_error_detail(error_data, payload, headers) elif specific: hint = specific - except (OSError, http.client.HTTPException): - if label == 'OpenRouter': - hint = 'Could not read the provider error body within the diagnostic limit.' raise CodexError(f'{label} returned HTTP {response.status}. {hint}', retryable=response.status in (502, 503, 504)) yield response, transport, deadline diff --git a/app.py b/app.py index f818cc0b..b999f9cf 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,5 @@ """Tuldok: local image capture and corner-label dataset workspace.""" import argparse -from contextlib import contextmanager import psycopg import ai import ai_codex @@ -42,22 +41,21 @@ import io import ipaddress import json -import math import re import shutil from metadata import Metadata, StorageError, CommitOutcomeUnknown, CatalogBusy -from project_store import ProjectStoreError from image_sources import ImageSources, Conflict, now, metadata, validate_annotation, CORNERS, VISIBILITY, SPLITS import tempfile import threading import time -import uuid import zipfile from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from urllib.parse import urlsplit, parse_qs +from tuldok_project import serve_protocol + from PIL import Image, ImageOps ROOT = Path(__file__).resolve().parent @@ -68,13 +66,6 @@ class Dataset(ImageSources): def __init__(self, path, *, dsn=None, project_roots=None): self._close_lock = threading.RLock() self._closed = False - self._extraction_condition = threading.Condition() - self._extraction_requests = {} - self._extraction_stopping = False - self._extraction_binding = uuid.uuid4().hex - self._extraction_guard = None - self._extraction_closer = None - self._extraction_close_error = None self.path = Path(path).resolve() self.path.mkdir(parents=True, exist_ok=True) (self.path / 'images').mkdir(exist_ok=True) @@ -123,76 +114,6 @@ def close(self): self.db.close() self._closed = True - def close_for_project_extraction(self, *, deadline): - """Bound this explicit stop without treating a timeout as cessation.""" - guard = self._extraction_guard - if guard is None: - raise workbench.WorkbenchError('A drained original controller is required.', 'source_busy', 409) - guard.require(self) - if type(deadline) not in (int, float) or not math.isfinite(deadline): - raise workbench.WorkbenchError('An exact stop deadline is required.') - with self._extraction_condition: - if self._extraction_closer is None: - def close_owned(): - try: - self.close() - except BaseException as error: - self._extraction_close_error = error - self._extraction_closer = threading.Thread( - target=close_owned, name='tuldok-legacy-extraction-close', daemon=True) - try: - self._extraction_closer.start() - except BaseException as error: - self._extraction_close_error = error - raise workbench.WorkbenchError('Owned legacy closure could not start; source remains fenced.', 'unknown', 503) from error - closer = self._extraction_closer - closer.join(max(0, deadline - time.monotonic())) - if closer.is_alive(): - raise workbench.WorkbenchError('Owned legacy workers have not drained; source remains fenced.', 'source_busy', 409) - if self._extraction_close_error is not None: - raise workbench.WorkbenchError('Owned legacy closure failed; source remains fenced.', 'unknown', 503) from self._extraction_close_error - if not self._closed: - raise workbench.WorkbenchError('Owned legacy closure is unconfirmed; source remains fenced.', 'unknown', 503) - guard.require(self) - - @contextmanager - def _project_request(self): - """Fence new legacy handlers before an explicit extraction stop.""" - identity = threading.get_ident() - with self._extraction_condition: - if self._extraction_stopping: - raise workbench.WorkbenchError('This catalog is stopped for project extraction.', 'source_stopping', 409) - self._extraction_requests[identity] = self._extraction_requests.get(identity, 0) + 1 - try: - yield - finally: - with self._extraction_condition: - self._extraction_requests[identity] -= 1 - if not self._extraction_requests[identity]: - del self._extraction_requests[identity] - self._extraction_condition.notify_all() - - def stop_for_project_extraction(self, backup_parent_fd, backup_directory, external_roots=None): - """Explicit owner action; ordinary preview/Open never enters this path.""" - deadline = time.monotonic() + 60 - identity = threading.get_ident() - with self._extraction_condition: - if self._extraction_stopping: - raise workbench.WorkbenchError('Catalog extraction stop already has an owner.', 'source_stopping', 409) - self.access.require('admin', 'project_transfer.stop_source', owner_only=True) - self._extraction_stopping = True - guard = _LegacyExtractionGuard(self, self._extraction_binding, identity) - self._extraction_guard = guard - while any(thread != identity and count for thread, count in self._extraction_requests.items()): - remaining = deadline - time.monotonic() - if remaining <= 0: - raise workbench.WorkbenchError('Existing requests have not drained; catalog remains fenced.', 'source_busy', 409) - self._extraction_condition.wait(min(.1, remaining)) - from project_transfer import stop_source - return stop_source(self, backup_parent_fd=backup_parent_fd, backup_directory=backup_directory, - external_roots=external_roots, controller_guard=guard) - - @authorized('execute', 'dataset.suggest', owner_only=True) def suggest(self, body): sample_id = body.get('sample_id') @@ -248,27 +169,12 @@ def _export(self): raise -class _LegacyExtractionGuard: - """An in-process controller proof, never a transport or project record.""" - def __init__(self, dataset, binding, owner_thread): - self.dataset, self.binding, self.owner_thread = dataset, binding, owner_thread - - def require(self, dataset): - with dataset._extraction_condition: - if (dataset is not self.dataset or dataset._extraction_guard is not self - or dataset._extraction_binding != self.binding or not dataset._extraction_stopping - or threading.get_ident() != self.owner_thread - or any(thread != self.owner_thread and count for thread, count in dataset._extraction_requests.items())): - raise workbench.WorkbenchError('Extraction requires the drained original catalog controller.', 'source_busy', 409) - - -def make_handler(dataset, observer=None, *, transfer_controller=None): +def make_handler(dataset, observer=None): class Handler(BaseHTTPRequestHandler): def log_message(self, *_): pass def _observe_request(self, handler): - self._handoff_stopped_source() name, version = (agent_api.observation_operation(self.command, self.path, self.headers) if observer is not None else (None, None)) self._observation_operation, self._observation_version = name, version @@ -276,13 +182,9 @@ def _observe_request(self, handler): self._observation_finished = False self._contract_selection = None try: - if (urlsplit(self.path).path == '/api/workspace/project-transfer' - or urlsplit(self.path).path.startswith('/api/workspace/project-transfer/')): - return self._handle_project_transfer() if isinstance(dataset, Dataset): try: - with dataset._project_request(): - return handler() + return handler() except workbench.WorkbenchError as error: return self.reply({'error': str(error), 'code': error.code}, error.status) return handler() @@ -291,71 +193,6 @@ def _observe_request(self, handler): observe_stage(observer, self._observation, 'terminal', outcome='unknown', delivery='incomplete') self._observation_finished = True - def _handoff_stopped_source(self): - nonlocal dataset - if transfer_controller is None or transfer_controller.phase == 'source_live': - return - source = transfer_controller.dataset - # A failed stop may still have old handlers draining. Keep their - # original Dataset closure until the source's request gate proves - # them absent; new ordinary requests remain fenced meanwhile. - with source._extraction_condition: - if source._extraction_stopping and not any(source._extraction_requests.values()): - dataset = transfer_controller.control_workspace - - def _handle_project_transfer(self): - """One explicit source-to-control authority transition, never Open.""" - nonlocal dataset - routes = {('GET', '/api/workspace/project-transfer'): 'status', - ('POST', '/api/workspace/project-transfer/preview'): 'preview', - ('GET', '/api/workspace/project-transfer/report'): 'report_page', - ('POST', '/api/workspace/project-transfer/stop'): 'stop', - ('POST', '/api/workspace/project-transfer/extract'): 'extract'} - try: - if transfer_controller is None: - raise workbench.WorkbenchError('Configure an explicit existing legacy extraction controller first.', 'unavailable', 409) - action = routes.get((self.command, urlsplit(self.path).path)) - if action is None: - raise workbench.WorkbenchError('Unknown extraction operation.', 'unsupported', 404) - name = 'project_transfer.' + action - token, bearer = self._credential() - authority = transfer_controller.access - with authority.bind(token): - self._same_origin(self.command == 'POST' and not bearer) - selected = self._select_contract() - agent_api.require_selection(selected, name) - body = None - if self.command == 'POST': - if self.headers.get_content_type() != 'application/json': - raise agent_api.AgentAPIError('Send a bounded JSON request.', 'invalid', 400) - length = int(self.headers.get('Content-Length', '0')) - if not 0 < length <= agent_api.REQUEST_LIMIT: - raise agent_api.AgentAPIError('Extraction request exceeds its bound.', 'too_large', 413) - raw = self.rfile.read(length) - observe_request_bytes(observer, self._observation, len(raw)) - body = agent_api.decode(raw) - arguments = agent_api.validate_selected(selected, self.command, self.path, body) - self._observe_dispatch(body) - try: - if action == 'status': value = transfer_controller.status() - elif action == 'preview': value = transfer_controller.preview(**arguments) - elif action == 'report_page': value = transfer_controller.report_page(**arguments) - elif action == 'stop': value = transfer_controller.stop(**arguments) - else: value = transfer_controller.extract(**arguments) - finally: - self._handoff_stopped_source() - return self.reply(value) - except (workbench.WorkbenchError, ProjectStoreError, agent_api.AgentAPIError, access.AccessError) as error: - result = {'error': str(error), 'code': error.code} - if hasattr(error, 'receipt'): - result['receipt'] = error.receipt - return self.reply(result, error.status) - except (CommitOutcomeUnknown, psycopg.Error): - return self.reply({'error': 'Extraction outcome requires explicit inspection.', 'code': 'storage'}, 503) - except OSError: - return self.reply({'error': 'Extraction storage outcome requires explicit status inspection; retained staging was not removed.', - 'code': 'storage'}, 503) - def _observe_dispatch(self, body=None): observe_selection(observer, getattr(self, '_observation', None), self._observation_operation, body, self._observation_version) @@ -416,12 +253,7 @@ def reply(self, data, status=200, content_type='application/json', headers=()): self._write_observed((data,), status) def project_error(self, error): - value = {'error': str(error), 'code': error.code} - # Only the typed project boundary can attach a non-secret durable - # receipt. Selected output validation checks its closed schema. - if getattr(dataset, 'project_files_mode', False) and hasattr(error, 'receipt'): - value['receipt'] = error.receipt - return self.reply(value, error.status) + return self.reply({'error': str(error), 'code': error.code}, error.status) def reply_stream(self, stream, metadata): # The owner has already verified a private spool before HTTP success. @@ -561,6 +393,80 @@ def _graph_snapshot_contract_access(self, graph_id, revision, operation, version configuration, _parameters = pipeline.recipe_contract.graph_parameters(graph) self._snapshot_contract_access(configuration['project_id'], operation, version, scope) + def _project_folder_post(self, path): + # A served project folder (spec-m7.md, section 6): POST /api/project/ and nothing else. + if not getattr(dataset, 'project_folder_mode', False): + return self.reply({'error': 'Start Tuldok with --project-folder to use this.', 'code': 'unavailable'}, 409) + selected = self._select_contract() + if not path.startswith('/api/project/'): + return self.reply({'error': 'This Tuldok serves one project folder; use /api/project/.', + 'code': 'unavailable'}, 409) + name = path.removeprefix('/api/project/') + agent_api.require_selection(selected, 'project.' + name) + if urlsplit(self.path).query or self.headers.get_content_type() != 'application/json': + raise agent_api.AgentAPIError('Send a JSON object body and no query.', 'contract_mismatch', 409) + raw = self.rfile.read(int(self.headers['Content-Length'])) # framing checked by _select_contract + observe_request_bytes(observer, getattr(self, '_observation', None), len(raw)) + body = agent_api.decode(raw) + if not isinstance(body, dict): + raise agent_api.AgentAPIError('Send a JSON object body and no query.', 'contract_mismatch', 409) + self._observe_dispatch(body) + try: + status, result = dataset.dispatch(name, body) + except (access.AccessError, CatalogBusy, CommitOutcomeUnknown, psycopg.Error): + raise + except Exception as error: + result, status = {'error': 'The operation failed: ' + type(error).__name__ + '.', 'code': 'internal'}, 500 + return self.reply(result, status) + + def _serve_post(self, path): + # A served model (docs/plans/training-and-curation/spec-t6.md, section 3): POST + # /api/serve//predict, a JSON object of inputs, behind the bound credential; no contract header. + parts = path.split('/') + if (len(parts) != 5 or parts[4] != 'predict' or not re.fullmatch('[a-z0-9][a-z0-9._-]{0,119}', parts[3]) + or urlsplit(self.path).query): + return self.reply({'error': 'Use POST /api/serve//predict.', 'code': 'not-found'}, 404) + lengths = self.headers.get_all('Content-Length', []) + if (self.headers.get_all('Transfer-Encoding', []) or len(lengths) != 1 + or not re.fullmatch('[0-9]{1,10}', lengths[0]) or int(lengths[0]) == 0): + return self.reply({'error': 'Send the body with one Content-Length.', 'code': 'invalid'}, 400) + if int(lengths[0]) > serve_protocol.REQUEST_LIMIT: + return self.reply({'error': 'A request to a served model may be at most 8 MiB.', + 'code': 'too-large'}, 413) + if self.headers.get_content_type() != 'application/json': + return self.reply({'error': 'Send a JSON object {"inputs": {...}}.', 'code': 'invalid'}, 400) + raw = self.rfile.read(int(lengths[0])) + observe_request_bytes(observer, getattr(self, '_observation', None), len(raw)) + try: + body = agent_api.decode(raw) + except agent_api.AgentAPIError: + return self.reply({'error': 'The body is not JSON.', 'code': 'invalid'}, 400) + self._observe_dispatch() + received = time.perf_counter() + status, answer = dataset.predict(parts[3], body, received=received, size=len(raw)) + return self.reply(answer, status) + + def _project_folder_file(self): + # The bytes of a product's file (spec-d1.md, section 5), for whoever may call an inspect operation. + try: + query = parse_qs(urlsplit(self.path).query, keep_blank_values=True, strict_parsing=True, + max_num_fields=4) + except ValueError: + query = {} + if set(query) != {'run', 'path'} or any(len(values) != 1 for values in query.values()): + return self.reply({'error': 'Give run and path, once each, and nothing else.', 'code': 'invalid'}, 400) + status, answer = dataset.product_file(query['run'][0], query['path'][0], self.headers.get('Range')) + if status != 200: + return self.reply(answer, status) + observe_stage(observer, getattr(self, '_observation', None), 'returned', + outcome='success', envelope_status=answer.status, http_status=answer.status) + self.send_response(answer.status) + for key, value in answer.headers: + self.send_header(key, value) + self.send_header('Content-Length', str(answer.length)) + self.end_headers() + self._write_observed(answer.chunks, answer.status) + def do_GET(self): return self._observe_request(self._handle_GET) @@ -619,6 +525,11 @@ def _handle_POST(self): except access.AccessError as error: dataset.access.denial(error, 'http.post', path) raise + if path.startswith('/api/serve/') and getattr(dataset, 'project_folder_mode', False): + return self._serve_post(path) + if path.startswith('/api/project/') or (getattr(dataset, 'project_folder_mode', False) + and not path.startswith('/api/access/')): + return self._project_folder_post(path) self._select_contract() if path == '/api/access/logout': self._access_body() @@ -629,10 +540,6 @@ def _handle_POST(self): if path.startswith('/api/access/credentials/') and path.endswith('/revoke'): self._access_body() return self.reply(dataset.access.revoke(path.split('/')[-2])) - if (getattr(dataset, 'project_files_mode', False) and - not path.startswith('/api/workspace/project-files/')): - return self.reply({'error': 'This domain is unavailable until its canonical project publication is implemented.', - 'code': 'unavailable'}, 409) if path.startswith('/api/access/execution-grants/') and path.endswith('/revoke'): projects.closed(self._access_body(), ()) return self.reply(dataset.access.revoke_execution(path.split('/')[-2])) @@ -651,34 +558,22 @@ def _handle_POST(self): def _get(self): path = urlsplit(self.path).path try: + if getattr(dataset, 'project_folder_mode', False) and path.startswith('/api/'): + # A served project folder answers its capabilities and context; its operations are POSTs. + if path == '/api/agent/capabilities': + return self.reply(dataset.capabilities()) + if path == '/api/workspace/context': + return self.reply({'version': 1, 'mode': 'real', 'dataset_id': dataset.db.dataset_id, + 'project_folder': True}) + if path == '/api/project/file': + return self._project_folder_file() + if path != '/api/agent/operations': + return self.reply({'error': 'This Tuldok serves one project folder; use /api/project/.', + 'code': 'unavailable'}, 409) if path == '/api/agent/operations': return self.reply(agent_api.discovery()) if path == '/api/agent/capabilities': - capabilities = (dataset.capabilities() if getattr(dataset, 'project_files_mode', False) - else agent_api.capabilities(dataset.db.dataset_id)) - if not getattr(dataset, 'project_files_mode', False): - for operation in capabilities['operations']: - if operation['id'].startswith('project_files.'): - operation['availability'] = 'unavailable' - if operation['id'].startswith('project_transfer.'): - operation['availability'] = ('available' if transfer_controller is not None - else 'unavailable') - return self.reply(capabilities) - if path == '/api/workspace/context' and getattr(dataset, 'project_files_mode', False): - # Bootstrap identifies the installation that owns this - # credential, without constructing a legacy domain owner. - return self.reply({'version': 1, 'mode': 'real', 'dataset_id': dataset.db.dataset_id, - 'project_files': True}) - if path == '/api/workspace/project-files' or path.startswith('/api/workspace/project-files/'): - if not getattr(dataset, 'project_files_mode', False): - raise workbench.WorkbenchError('Start explicit project-file mode to use this capability.', 'unavailable', 409) - from project_workspace import http_arguments - name, arguments = http_arguments('GET', self.path) - return self.reply(dataset.dispatch(name, arguments, - selected_version=self._contract_selection[1] if self._contract_selection else None)) - if getattr(dataset, 'project_files_mode', False) and path.startswith('/api/'): - raise workbench.WorkbenchError('This domain is unavailable until its canonical project publication is implemented.', - 'unavailable', 409) + return self.reply(agent_api.capabilities(dataset.db.dataset_id)) if (path == '/api/workspace/projects' or path.startswith('/api/workspace/projects/') or path == '/api/workspace/project-snapshots' or path.startswith('/api/workspace/project-snapshots/')): self.project_access() @@ -894,12 +789,6 @@ def _get(self): assets = {'/native-detection-import.js': ('native_detection_import.js', 'text/javascript'), '/saved-searches.js': ('saved-searches.js', 'text/javascript'), '/sequence-batch.js': ('sequence-batch.js', 'text/javascript'), '/pumas-typed.js': ('pumas-typed.js', 'text/javascript'), '/pumas-gateways.js': ('pumas-gateways.js', 'text/javascript'), '/meshes.js': ('meshes.js', 'text/javascript'), '/sequences.js': ('sequences.js', 'text/javascript'), '/text-classification-proposals.js': ('text-classification-proposals.js', 'text/javascript'), '/caption-proposals.js': ('caption-proposals.js', 'text/javascript'), '/curation.js': ('curation.js', 'text/javascript'), '/caption-import.js': ('caption_import.js', 'text/javascript'), '/bulk-import.js': ('bulk_import.js', 'text/javascript'), '/saved-selections.js': ('saved-selections.js', 'text/javascript'), '/workbench': ('workbench.html', 'text/html'), '/workbench.js': ('workbench.js', 'text/javascript'), '/workbench.css': ('workbench.css', 'text/css'), '/': ('index.html', 'text/html'), '/app.js': ('app.js', 'text/javascript'), '/style.css': ('style.css', 'text/css'), '/rights-note.js': ('rights-note.js', 'text/javascript'), '/preferences.js': ('preferences.js', 'text/javascript'), '/instruction-responses.js': ('instruction-responses.js', 'text/javascript'), '/native-text-import.js': ('native_text_import.js', 'text/javascript')} # Opt-in workspace shell keeps fixture and real service boundaries distinct. assets.update({ - '/project-files': ('project-files.html', 'text/html'), - '/project-files.js': ('project-files.js', 'text/javascript'), - '/project-files.css': ('project-files.css', 'text/css'), - '/project-transfer': ('project-transfer.html', 'text/html'), - '/project-transfer.js': ('project-transfer.js', 'text/javascript'), - '/project-transfer.css': ('project-transfer.css', 'text/css'), '/auth.js': ('auth.js', 'text/javascript'), '/auth.css': ('auth.css', 'text/css'), '/workspace-project-activity.js': ('workspace-project-activity.js', 'text/javascript'), @@ -918,10 +807,6 @@ def _get(self): '/workspaces.css': ('workspaces.css', 'text/css'), '/workspaces.js': ('workspaces.js', 'text/javascript'), '/experiment-graph.js': ('experiment-graph.js', 'text/javascript'), - '/project-progression.js': ('project-progression.js', 'text/javascript'), - '/project-model-portfolio.js': ('project-model-portfolio.js', 'text/javascript'), - '/project-measurements.js': ('project-measurements.js', 'text/javascript'), - '/project-measurements.css': ('project-measurements.css', 'text/css'), '/experiment-lineage.js': ('experiment-lineage.js', 'text/javascript'), '/experiment-lineage.css': ('experiment-lineage.css', 'text/css'), '/workspace-real.js': ('workspace-real.js', 'text/javascript'), @@ -931,7 +816,6 @@ def _get(self): '/static/retained-analysis.js': ('retained-analysis.js', 'text/javascript'), '/static/retained-chart.js': ('retained-chart.js', 'text/javascript'), '/static/retained-checkpoint-curve.vl.json': ('retained-checkpoint-curve.vl.json', 'application/json'), - '/static/retained-measurement-maxima.vl.json': ('retained-measurement-maxima.vl.json', 'application/json'), '/static/vendor/retained-analysis/vega.min.js': ('vendor/retained-analysis/vega.min.js', 'text/javascript'), '/static/vendor/retained-analysis/vega-lite.min.js': ('vendor/retained-analysis/vega-lite.min.js', 'text/javascript'), '/static/vendor/retained-analysis/vega-embed.min.js': ('vendor/retained-analysis/vega-embed.min.js', 'text/javascript'), @@ -943,8 +827,41 @@ def _get(self): '/workspaces-data.js': ('workspaces-data.js', 'text/javascript'), '/workspaces-data.css': ('workspaces-data.css', 'text/css'), }) - if getattr(dataset, 'project_files_mode', False) and path in ('/', '/workspaces', '/workbench'): - path = '/project-files' + assets.update({'/project-workspace': ('project-workspace.html', 'text/html'), + '/project-workspace/base.css': ('project-workspace/base.css', 'text/css'), + '/project-workspace/compare.css': ('project-workspace/compare.css', 'text/css'), + '/project-workspace/compare.js': ('project-workspace/compare.js', 'text/javascript'), + '/project-workspace/parallel.js': ('project-workspace/parallel.js', 'text/javascript'), + '/project-workspace/sweeps.css': ('project-workspace/sweeps.css', 'text/css'), + '/project-workspace/sweeps.js': ('project-workspace/sweeps.js', 'text/javascript'), + '/project-workspace/review.css': ('project-workspace/review.css', 'text/css'), + '/project-workspace/review.js': ('project-workspace/review.js', 'text/javascript'), + '/project-workspace/adapter.js': ('project-workspace/adapter.js', 'text/javascript'), + '/project-workspace/charts.js': ('project-workspace/charts.js', 'text/javascript'), + '/project-workspace/markdown.js': ('project-workspace/markdown.js', 'text/javascript'), + '/project-workspace/tree.js': ('project-workspace/tree.js', 'text/javascript'), + '/project-workspace/screens.js': ('project-workspace/screens.js', 'text/javascript'), + '/project-workspace/actions.js': ('project-workspace/actions.js', 'text/javascript'), + '/project-workspace/editor.js': ('project-workspace/editor.js', 'text/javascript'), + '/project-workspace/tabs.js': ('project-workspace/tabs.js', 'text/javascript'), + '/project-workspace/inside.js': ('project-workspace/inside.js', 'text/javascript'), + '/project-workspace/weights.js': ('project-workspace/weights.js', 'text/javascript'), + '/project-workspace/structure.js': ('project-workspace/structure.js', 'text/javascript'), + '/project-workspace/fields.js': ('project-workspace/fields.js', 'text/javascript'), + '/project-workspace/training.js': ('project-workspace/training.js', 'text/javascript'), + '/project-workspace/annotate.js': ('project-workspace/annotate.js', 'text/javascript'), + '/project-workspace/generate.js': ('project-workspace/generate.js', 'text/javascript'), + '/project-workspace/curation.js': ('project-workspace/curation.js', 'text/javascript'), + '/project-workspace/serving.js': ('project-workspace/serving.js', 'text/javascript'), + '/project-workspace/serving.css': ('project-workspace/serving.css', 'text/css'), + '/project-workspace/app.js': ('project-workspace/app.js', 'text/javascript'), + '/static/vendor/ubuntu-sans/UbuntuSans[wdth,wght].ttf': ('vendor/ubuntu-sans/UbuntuSans[wdth,wght].ttf', 'font/ttf'), + '/static/vendor/ubuntu-sans/UbuntuSans-Italic[wdth,wght].ttf': ('vendor/ubuntu-sans/UbuntuSans-Italic[wdth,wght].ttf', 'font/ttf'), + '/static/vendor/ubuntu-sans/UbuntuSansMono[wght].ttf': ('vendor/ubuntu-sans/UbuntuSansMono[wght].ttf', 'font/ttf'), + '/static/vendor/ubuntu-sans/UbuntuSansMono-Italic[wght].ttf': ('vendor/ubuntu-sans/UbuntuSansMono-Italic[wght].ttf', 'font/ttf'), + '/static/vendor/ubuntu-sans/LICENCE.txt': ('vendor/ubuntu-sans/LICENCE.txt', 'text/plain')}) + if getattr(dataset, 'project_folder_mode', False) and path in ('/', '/workspaces', '/workbench'): + path = '/project-workspace' if path in assets: name, kind = assets[path] return self.reply((ROOT / 'static' / name).read_bytes(), content_type=kind + '; charset=utf-8') @@ -959,8 +876,6 @@ def _get(self): return self.reply({'error': str(error), 'code': error.code}, error.status) except workbench.WorkbenchError as error: return self.project_error(error) - except ProjectStoreError as error: - return self.project_error(error) except (ValueError, OSError) as error: return self.reply({'error': str(error)}, 400) @@ -983,8 +898,6 @@ def _post(self): limit = spatial_assets.MAX_REQUEST if path.startswith('/api/workspace/'): limit = 256 * 1024 - if path == '/api/workspace/project-files' or path.startswith('/api/workspace/project-files/'): - limit = agent_api.REQUEST_LIMIT if not 0 < length <= limit: raise ValueError('Request is too large or empty.') raw_body = self.rfile.read(length) @@ -994,22 +907,10 @@ def _post(self): # Strict JSON at the selected boundary, preserving ordinary HTTP semantics. body = agent_api.decode(raw_body) agent_api.validate_selected(self._contract_selection, self.command, self.path, body) - elif path == '/api/workspace/project-files' or path.startswith('/api/workspace/project-files/'): - body = agent_api.decode(raw_body) if not isinstance(body, dict): raise ValueError('Expected an object.') self._observe_dispatch(body) path = urlsplit(self.path).path - if path == '/api/workspace/project-files' or path.startswith('/api/workspace/project-files/'): - if not getattr(dataset, 'project_files_mode', False): - raise workbench.WorkbenchError('Start explicit project-file mode to use this capability.', 'unavailable', 409) - from project_workspace import http_arguments - name, body = http_arguments('POST', self.path, body) - return self.reply(dataset.dispatch(name, body, - selected_version=self._contract_selection[1] if self._contract_selection else None)) - if getattr(dataset, 'project_files_mode', False): - raise workbench.WorkbenchError('This domain is unavailable until its canonical project publication is implemented.', - 'unavailable', 409) if path.startswith('/api/workspace/'): parts = path.removeprefix('/api/workspace/').split('/') if parts == ['scientific-comparison']: @@ -1343,8 +1244,6 @@ def graph_guard(graph): return self.reply({'error': str(error), 'code': error.code}, error.status) except workbench.WorkbenchError as error: return self.project_error(error) - except ProjectStoreError as error: - return self.project_error(error) except Conflict as error: return self.reply({'error': str(error)}, 409) except FileNotFoundError: @@ -1362,14 +1261,8 @@ def main(): parser.add_argument('--port', type=int, default=8091) parser.add_argument('--project-root', action='append', default=[], metavar='ALIAS=PATH', help='Allow read-only project inspection beneath a local directory; repeat for multiple roots.') - parser.add_argument('--project-files-root', action='append', default=[], metavar='ALIAS=PATH', - help='Use inert external text-project mode beneath explicit local roots; other domain writes are unavailable.') - parser.add_argument('--project-runtime-source', action='append', default=[], metavar='ALIAS=CONFIG_FILE', - help='Allow explicit offline CPU runtime capture from an operator-owned local configuration; never capture on Open.') - parser.add_argument('--project-transfer-control', type=Path, - help='Enable explicit stopped extraction from this existing legacy controller into a separate local authority host.') - parser.add_argument('--project-transfer-root', action='append', default=[], metavar='ALIAS=PATH', - help='Configure backup and destination roots for explicit legacy extraction; requires --project-transfer-control.') + parser.add_argument('--project-folder', type=Path, metavar='PATH', + help='Serve this definition-first project folder and nothing else.') args = parser.parse_args() roots = {} for binding in args.project_root: @@ -1377,49 +1270,24 @@ def main(): if not separator or not location or alias in roots: parser.error('Each project root needs a unique ALIAS=PATH.') roots[alias] = location - transfer_controller = control_workspace = None try: - if args.project_files_root: - if roots or args.project_transfer_control or args.project_transfer_root: - parser.error('Choose project-file mode or legacy snapshot inspection roots, not both.') - file_roots = {} - for binding in args.project_files_root: - alias, separator, location = binding.partition('=') - if not separator or not location or alias in file_roots: - parser.error('Each project-file root needs a unique ALIAS=PATH.') - file_roots[alias] = location - from project_workspace import ProjectWorkspace - runtime_sources = {} - from recipe_process import read_regular, decode - for binding in args.project_runtime_source: - alias, separator, location = binding.partition('=') - if not separator or not location or alias in runtime_sources: - parser.error('Each runtime source needs a unique ALIAS=CONFIG_FILE.') - runtime_sources[alias] = decode(read_regular(Path(location).expanduser(), 65536)) - dataset = ProjectWorkspace(args.data, file_roots, runtime_sources=runtime_sources) + if args.project_folder is not None: + if roots: + parser.error('--project-folder serves one folder; it cannot be combined with other project options.') + from tuldok_project.host import ProjectHost, refusal + from tuldok_project.store import ProjectError + try: + dataset = ProjectHost(args.data, args.project_folder) + except ProjectError as error: + problems = ''.join(' ' + ' '.join(part for part in (item['path'], item['pointer']) if part) + + ': ' + item['message'] + '\n' for item in refusal(error)[1].get('problems', [])[:10]) + parser.exit(2, str(error) + '\n' + problems) else: - if args.project_runtime_source: - parser.error('Project runtime sources require external project-file mode.') dataset = Dataset(args.data, project_roots=roots) - if args.project_transfer_control: - transfer_roots = {} - for binding in args.project_transfer_root: - alias, separator, location = binding.partition('=') - if not separator or not location or alias in transfer_roots: - parser.error('Each extraction root needs a unique ALIAS=PATH.') - transfer_roots[alias] = location - from project_workspace import ProjectWorkspace - from project_transfer_controller import ProjectTransferController - control_workspace = ProjectWorkspace(args.project_transfer_control, transfer_roots) - transfer_controller = ProjectTransferController(dataset, control_workspace) - control_workspace.transfer_controller = transfer_controller - elif args.project_transfer_root: - parser.error('Extraction roots require --project-transfer-control.') except (StorageError, workbench.WorkbenchError) as error: parser.exit(2, str(error) + "\n") observer = configured_interface(args, 'http') - server = ThreadingHTTPServer((args.host, args.port), make_handler(dataset, observer, - transfer_controller=transfer_controller)) + server = ThreadingHTTPServer((args.host, args.port), make_handler(dataset, observer)) print('Tuldok: http://' + args.host + ':' + str(server.server_port), flush=True) try: server.serve_forever() @@ -1427,19 +1295,14 @@ def main(): pass finally: server.server_close() - if transfer_controller is not None: - transfer_controller.close() - if control_workspace is not None: - control_workspace.close() dataset.close() if observer is not None: observe_close(observer) if __name__ == '__main__': - # Trusted controller capabilities use exact app-owned class identities. # Executing this file must not create a second imported app module with - # distinct Dataset/guard classes during the explicit extraction workflow. + # distinct classes when another module later imports app. import sys sys.modules['app'] = sys.modules[__name__] main() diff --git a/candidate_packages.py b/candidate_packages.py index 2c9413c5..74e0a4b7 100644 --- a/candidate_packages.py +++ b/candidate_packages.py @@ -1233,18 +1233,6 @@ def authorize_transitive_inputs(runs, run, *, seen=None): def authorize_material_inputs(runs,material,*,seen=None): """Reauthorize an exact retained input material without inventing a Run.""" from workbench import WorkbenchError - # Project producers remain inert observations. Resolve their exact input - # proof through the fixed history owner, never through Runs._load recovery - # rows or a general historical fallback on the live Run owner. - from project_history import ProjectHistory - if isinstance(runs,ProjectHistory): - return runs.authorize_material_inputs(material) - from project_execution import ProjectExecution - execution=getattr(runs,'project_execution',None) - if isinstance(execution,ProjectExecution): - if execution._runs is not runs or not isinstance(execution.projection.history,ProjectHistory): - raise WorkbenchError('Current project Run input owner differs.','unavailable',409) - return execution.projection.history.authorize_material_inputs(material) seen=set() if seen is None else seen for item in material['inputs'].values(): if item.get('kind') != 'run_artifact': diff --git a/docs/README.md b/docs/README.md index 679739f8..7be1667a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,8 +10,8 @@ Start with the [root README](../README.md) for installation and the Data → Pip - [Agent operations and MCP](contracts/agent-operations.md) - [Optional MCP observations and privacy](contracts/mcp-observer.md) - [Project manifests and read-only inspection](contracts/projects.md) -- [External text project files and current milestone limits](contracts/project-files.md) - [Project command line](contracts/project-cli.md) +- [Project folder format and checker](contracts/project-folder.md) ## Pipelines and execution diff --git a/docs/contracts/project-cli.md b/docs/contracts/project-cli.md index 4cb3d3f4..d788385b 100644 --- a/docs/contracts/project-cli.md +++ b/docs/contracts/project-cli.md @@ -1,10 +1,5 @@ # Project execution CLI -The separate `file-*` commands operate on the explicit external text-project -mode. See [project files](project-files.md) for its canonical authority, owner -selection, checkpoint binding and current milestone limits. Registration `open` -below keeps its existing meaning. - `project_cli.py` uses the existing authenticated agent operations. Saved graphs, execution grants, Run custody, progress, cancellation and artifacts keep their existing owners and remain visible in `/workspaces` → Current local catalog → diff --git a/docs/contracts/project-files.md b/docs/contracts/project-files.md deleted file mode 100644 index 387c9bc6..00000000 --- a/docs/contracts/project-files.md +++ /dev/null @@ -1,292 +0,0 @@ -# External project files - -The [external project plan](../plans/external-project-save-load/plan.md) owns current -milestone acceptance. This contract covers the external-directory format and -its finite owner operations. Execution/recovery, independent Save As and -extraction have separate acceptance evidence; an unavailable capability must -refuse before effects. An in-progress draft is not final feature acceptance. -The [authority decision](../decisions/external-project-authority.md) explains the -canonical boundary. - -Start an explicit project-file service with a separate installation control -directory and one or more existing trusted root directories: - -```sh -python app.py --data /private/tuldok-control \ - --project-files-root research=/home/me/research -``` - -Use the service's normal local owner bootstrap, then visit `/project-files`. -New creates a previously absent relative directory under a configured alias. -Open chooses an existing project directory. Root aliases and credentials stay -local. Historical snapshots may retain inert root identities or runtime path -strings as evidence; they never establish a current filesystem binding. Tokens, -token hashes and live credential records are excluded. This mode cannot be -combined with legacy `--project-root`. - -## Canonical format - -`.tuldok/project.json` has format `tuldok_project_v1`. It is the sole current -checkpoint: project UUID, display name, membership roots, sorted record inventory, -request receipts and a SHA256 checkpoint. Its digest covers canonical bytes of -the document with `checkpoint` omitted. The store supports only its declared -format and family; malformed and future versions fail before projection writes. -The canonical checkpoint index is limited to 64 MiB, separately from the -100,000-record, 40 MiB-per-record and 2 GiB-total-record limits. The aggregate -budget covers canonical record bytes, excluding artifact payloads; scientific -catalogs retain original runtime/file observations across many Runs. Commit -and cold Open enforce the same aggregate budget. Commit checks -the index size before staging immutable records. This format is unchanged; -older installations with the former 16 MiB index or 256 MiB aggregate limit cannot open larger -checkpoints and must be upgraded before consuming them. - -Verified stopped-source backups and ordinary catalog backups share identical -file contents only inside their fresh private backup tree, and only when byte -count, SHA256 and POSIX permission mode all match. The first copy has an -independent inode; no original source file is hard-linked into the backup. -Both source and copied content are verified before a duplicate link is made, -and the complete tree/hash/mode checks still run before backup publication. -The existing backup format remains a complete ordinary-file tree. Restore -copies every path into an independent file, preserving bytes and modes without -introducing writable aliases into the restored catalog. Backup trees are -immutable custody and must not be edited as working catalogs. - -Each inventory entry identifies a rooted immutable file -`.tuldok/records///.json`, its format, namespace, -byte count and SHA256. Original `workbench_text_v1` preserves original source, current -Workbench record, every annotation revision, non-secret audit facts and operation -receipts. New operation receipts use `project_operation_v2`, whose identity -includes the current project UUID. Old v1 receipts remain exact historical -facts; a fork cannot reactivate their request authority. Canonical JSON is -UTF-8, sorted keys, two-space indentation, LF ending, -finite bounded values, and no duplicate keys. The namespace equals the logical -project UUID. Historical domain catalog IDs, record identities and hash inputs -remain exact within those envelopes. No generic recursive ID rewriting is permitted. - -The finite additional families cover Workbench data/source/annotation/response/ -preference histories and tombstones; saved selections/searches; frozen releases -and exposures; model/graph/recipe revisions and immutable materials; project and -runtime snapshots; Study/Trial/Run/candidate/lineage facts; and four producer -history families. Owner decoders validate their complete reference closure. -Unknown families, future formats and unclassified authority-bearing fields fail -before reconstruction. Older text is not silently upgraded on Open. - -New stages and fsyncs a complete project in a sibling directory, then publishes -it with atomic no-replace rename. An edit publishes and fsyncs new immutable -records before atomic replacement of the checkpoint. Unreferenced staged records -are not current authority. A failure with uncertain checkpoint publication fences -further use until explicit recovery. No cleanup deletes published history. - -Save verifies/fsyncs the full selected closure and returns a durability barrier. -A no-op Save leaves canonical bytes unchanged. Close drains synchronous project -requests, performs the barrier, releases the projection and file lock, and rotates -the binding. Active or unknown local Run, producer, runtime-copy and package -writers block Close, replacement and fork. A drained thread alone does not prove -that an external provider has ceased. Open/New replacement -validates an incoming inert projection before releasing healthy current custody. - -The generated `.gitignore` excludes `/artifacts/`, `/.tuldok-local/`, `/.venv/`, -`/__pycache__/`, `/PGDATA/` and `/output/`. New does not initialize Git. Authored -records and the checkpoint are suitable for Git; database binaries, model weights, -caches and large generated data stay outside tracked content. Intrinsic -`project_artifact_v1` records contain SHA256 and byte count; domain edges classify -role, media type, unique/regenerable preservation and required operations. -Present bytes live at `artifacts/sha256/` or an explicitly supplied local -resolution. Git transports canonical text and manifests; an independent payload -transfer supplies untracked bytes. Missing payloads remain visible. Consumers -verify exact captured bytes before use; they never regenerate historical evidence. - -## Consumers and outcomes - -The shared operation registry owns `project_files.*` contracts and exact contract -hashes. Browser, SDK, CLI and MCP negotiate those contracts. Lifecycle selection -requires current owner authority. Scoped agents may inspect data and submit draft -annotations; human review still requires the owner. Every project action is bound -to an opaque workspace generation and its expected canonical checkpoint. - -Use `ProjectFilesOwnerClient` only for the narrow owner project-file operations. -Its explicit `timeout` configuration may be up to 7,200 seconds for large retained -projects; the default remains 120 seconds. The absolute request watchdog still -applies and a transport timeout never authorizes an automatic mutation retry. -Generic clients retain their 300-second maximum. This wait budget is not an -execution deadline or a cancellation request. - -Browser contract tables derive from the local `agent_api` registry. Regenerate -them with `python scripts/generate_project_files_contracts.py`; add `--check` -to verify that the installed v1 pins, all supported version pins and fixed -routes match that registry. Remote schemas remain opaque discovery data. -`ProjectFilesClient` wraps a negotiated client. CLI domain `file-*` commands take -`--arguments` JSON files; lifecycle commands use explicit flags and select -`--owner-project-files` with a -private owner configuration. Generic AgentClient remains agent-only. Keep request -keys across lost replies; the same key with changed input conflicts. Canonical -receipts survive SQL loss and return the originally committed domain result. -Current permissions and any stronger owner/disclosure requirement are checked -before replay. SQL failure after canonical publication returns a durable receipt -and fences the projection until explicit recovery. - -M2 domain operations return `{binding, project_id, checkpoint, value}`. Ordinary -list pages use offset/limit (default 40, maximum 40); heavy data-history pages -default to one and permit at most four. The original M1 text paging below stays -compatible. Historical Study metadata separates membership/reservation pages -from complete charge totals. Run summaries omit frozen source content. Exact -Run documents use digest-bound bounded byte segments, with current explicit -disclosure required before protected document bytes or evidence are read. -Imported disclosures remain historical observations and grant no current access. - -Explicit source registration captures the original manifest and establishes a -current installation-local rooted binding. Open restores no such binding. A -fresh explicit registration may renew a moved, unchanged manifest. Reference -inspection is bounded and reports observed SHA256 separately from a declared -expected digest. An unpinned reference is labeled unverified; a declared mismatch -is refused. Scientific import requires the selected exact observed bytes and the -existing scientific owner interprets them. Project scripts provide integrations; -opening or registering a manifest does not execute them. - -Owner-only artifact resolution selects an existing file relative to a configured -root. It verifies and durably copies the exact declared bytes. The receipt is a -past resolution observation; `artifact_status` reports current availability. -No network fetch, runtime installation, dependency activation or regeneration -is performed. For supported built-in graphs, explicit recipe preparation performs -a guarded current native probe before retaining the immutable material. Reading -retained native materials or historical evidence does not launch that probe. - -## Explicit execution and recovery - -Generic project scripts own domain integrations. Current external-script Run and -Study approval/admission use current local policy and grants. Admission and GO -intent are canonical before worker acknowledgement or GO. Measured terminal -evidence, spent budget and lineage remain project facts. Imported history supplies -no approval, admission epoch, cancellation right or resumable process. Producer -requests and candidate preparation follow the same intent-before-effects rule. - -For supported native graphs, the current owner explicitly calls -`project_files.native_preflight` with the saved graph ID and revision. A valid -result binds its validation token to that exact graph and measured runtime. -`native_submit` takes the graph ID, graph revision and token; admission rechecks -current inputs and authority before launching a native Run. `native_retry` -requires an actual current local Run and creates a new attempt. These actions -require current owner execute permission, the workspace binding, expected -checkpoint and a request key. `native_reconcile` instead selects an original -current local phase with the binding and expected checkpoint, requires current -owner cancel permission, and never sends GO. Its phase ID is available through -the current native-phase page of execution health v2; imported phases are history, -not current custody. - -Native execution uses the current operator's `TULDOK_TRAINING_PYTHON`, or the -service interpreter when unset. Its installed packages remain an operator trust -assumption; a runtime observation does not prove a complete dependency closure or -model quality. Open and ordinary inspection do not probe, install, train, resume -or activate a runtime. Retained validation, admission and phase facts do not grant -execution authority after a cold Open. - -An optional current operator binding selects runtime sources: - -```sh -python app.py --data /private/tuldok-control \ - --project-files-root research=/home/me/research \ - --project-runtime-source cpu=/private/cpu-runtime.json -``` - -The configuration uses the existing runtime-snapshot configuration schema. -`runtime_sources` lists aliases. Explicit Inspect measures that selected source; -Capture requires the exact inspected inventory and a stable request key, records -an intent, and copies bytes without launching them. Status distinguishes absent, -intent recorded, captured and abandoned. A cold Open has no configured alias or -cache merely because a historical descriptor contains a path. - -Recovery preserves the last acknowledged binding/checkpoint. Generic Run/producer -reconciliation requires exact original local custody and proof of drain. Runtime -retry and package reconciliation additionally require the exact original physical -writer journal and canonical intent; imported intent alone is insufficient. -Explicit abandonment records the owner reason with a separate key and retains -partial bytes and scientific facts. A completed package is reconciled rather than -silently rewritten. Recovery never sends GO or executes project code. - -The status endpoint is available before any project is open. It reports current -binding, local root aliases, project identity/checkpoint when selected, and -available actions. A stale binding or checkpoint is a conflict. Corrupt canonical -bytes cannot be blessed by Save. Missing unsupported families are never silently -omitted or auto-generated. - -Query returns summaries sorted by ID, with `limit` (1–100, default 40), exclusive -`after_id`, and `next_after_id`. Get returns the current full record and newest -history first, with `history_limit` (1–4, default 1), exclusive `before_revision`, -and `next_history_before_revision`. The same checkpoint binds all pages. Mutation -results include their committed revision and its first history page; replay never -mixes that historical result with newer revisions. The UI exposes explicit controls -to load more records and older history. - -Status may report `recovery_required`, its last acknowledged project identity, -and the selected root/directory to reopen. Explicit Open of the selected -directory is guarded by the current binding and last checkpoint. Ordinary -recovery validates canonical bytes, builds a fresh inert SQL projection and -rotates the binding. If original local custody remains, the named execution/file -recovery action must settle it first. That same-selection recovery preserves the -binding so its reply remains correlated with the caller. A failed recovery keeps the prior -custody and the recovery action available. A creation that committed but could not -be selected reports `selected:false` with the destination identity and request -receipt. Exact New retry reconciles that receipt without overwriting a different -existing destination. - -Errors distinguish invalid/unsupported format, stale selection, -writer contention, unavailable projection and unknown publication. A definite -canonical commit followed by projection failure returns its durable receipt. -Inspect/reconcile that receipt instead of minting another request key. Recovery -reopens the selected directory and reconstructs the projection without scripts. - -When changing Git revisions, close the project first. Open revalidates the complete -checkpoint; edits to canonical immutable files, unresolved conflict text and -inconsistent inventory fail. A changed checkout fences an already-open projection; -explicit reload reconstructs it without scripts. Git conflict resolution is manual. - -## Working files and independent Save As - -Adopt explicitly captures the exact UTF-8 bytes of a selected script or note, -including its history and measured hash. Merely editing a working file does not -replace an immutable scientific capture. Open never executes, repairs or restores -working files. Save As refuses an adopted file that is missing or changed until -its intended current bytes or an eligible note deletion are explicitly adopted. -A missing script remains protected: note-deletion adoption cannot remove its -captured script or make a missing script eligible for Save As. - -`project_files.file_adopt_deletion` is -`POST /api/workspace/project-files/files/adopt-deletion`. It requires current edit -permission and the usual `binding`, `expected_checkpoint` and `request_key`, plus -`path`, `expected_revision` and `expected_sha256`. The path must identify an -already adopted note that is safely absent. The revision and digest must match -its current positive captured revision; an existing file, changed canonical -revision/digest or unsafe path is refused. This action records absence; it never -deletes a filesystem entry. Inspection and Open never adopt a deletion. - -The first adopted deletion changes that note's document to `project_file_v2`. -All earlier positive revisions retain their exact original fields and text. A -closed deletion revision adds `deleted:true` and contains `revision`, -`sha256:null`, `bytes:0`, `text:null`, `adopted_at` and `actor_id`. This is distinct -from an empty positive file, whose text is `""` and whose SHA256 is the empty-byte -digest. `file_get` can select either revision; paged `file_history` returns -metadata, including the deletion marker, without embedding full texts. Current -inspection reports `deleted` only while that note remains safely absent, with -null expected/observed digests and zero expected bytes. - -Save As retains the complete note history and deletion marker and leaves its -current note path absent in the new destination. If a note is later recreated, -inspection reports its changed current bytes. Explicit `file_adopt` with the -tombstone's expected revision and the recreated file's measured SHA256 appends a -positive revision; it preserves the deletion and earlier text in history. Open -does not recreate the note or adopt its restoration. - -Save As publishes into a new absent directory, creates a new logical project and -editable identities, retains original hashed envelopes and records origin maps. -It copies required unique payloads and materializes captured working files only -into that new destination. Historical catalog IDs and material hashes remain -exact; unresolved bound drafts become explicit re-selection intents. Known -regenerable runtime binaries may remain missing with an explicit inventory; -missing unique evidence blocks a complete fork. Source bytes are unchanged. -Publication followed by selection failure reports `committed:true, selected:false`; -retry the exact request to select the already-published destination. - -The [extraction runbook](../runbooks/external-project-extraction.md) covers explicit -legacy selection, stopped backup, independent control credentials and the five -transfer operations. A whole-catalog backup is private recovery custody, not this -portable project format. diff --git a/docs/contracts/project-folder.md b/docs/contracts/project-folder.md new file mode 100644 index 00000000..b6e0551d --- /dev/null +++ b/docs/contracts/project-folder.md @@ -0,0 +1,3540 @@ +# Project folder format + +Owner: the package `tuldok_project/` (`folder.py` reads the folder, `documents.py` +and `schemas/*.json` check single files, `check.py` applies every rule, +`results.py` owns summary files, `annotations.py` the files of annotations, +`store.py` and `result_rows.py` the database copy and its results, `charts.py` charts and the report page, `plan.py` +plans, `environments.py` environments, and `builds.py`, `launcher.py` and +`snapshot.py` building). The format names are `tuldok_project_v1` for +the folder and `tuldok_definition_v1` for each definition. + +A project folder describes a piece of research as a set of small JSON files that +people and agents write and keep in Git. Each file is a **definition**: it says +what a dataset, model, training recipe, benchmark, build step, environment, +chart or annotation task is and, where something has to be produced, which +command produces it. +What a command writes is a **product**. Products stay on the machine that made +them; the folder holds only what someone wrote, plus benchmark summaries. + +`python -m tuldok_project check FOLDER` tells you whether a folder is valid. This +document says exactly what the checker accepts. + +## What belongs in the folder + +Put in the folder: the definitions, the result summary of each benchmark +(`results/`), the annotations of each annotation task (`annotations/`), and +cards and notes written by people (`notes/`). + +Keep out of the folder: generated datasets, weights, logs, run directories, +installed environments, caches, checksums of other files, and history. Git is +the history of every file in the folder. The scripts that commands run stay in +your code repository; definitions name them by path. + +A rule of thumb: if a person or an agent wrote it, it belongs in the folder. If a +command produced it, it does not. Result summaries are the one exception, so that +results travel with the project. + +## Layout + +``` +tuldok.json +README.md optional, not read +.gitignore optional, not read +datasets/.json +models/.json +recipes/.json +benchmarks/.json +steps/.json +environments/.json +charts/.json +annotations/.json +annotations/.jsonl the annotations of task +annotations// their sidecars: masks and arrays too large for a line +sweeps/.json a search over a recipe's settings +results/.csv +notes/... +``` + +- `tuldok.json` is required. Every other entry is optional. +- Any other top-level entry is an error. Move such files into `notes/` or out of + the folder. +- The nine **definition directories** (`datasets/` to `annotations/`, and `sweeps/`) hold only + `.json` files, one definition each, and no subdirectories. + `annotations/` also holds `.jsonl`, the annotations of the task in + `.json`, which must exist. +- `results/` holds only `.csv` files. +- `notes/` holds plain files and directories to any depth. Their contents are not + read. +- Every file is a regular file and every directory a real directory. A symbolic + link anywhere in the folder, including under `notes/`, is an error. +- Size limits: `tuldok.json` 64 KiB, a definition 1 MiB, a results file 16 MiB, + a file of annotations 64 MiB. + +A **name** is 1 to 120 characters of lowercase letters, digits, `.`, `_` and `-`, +starting with a letter or digit (`^[a-z0-9][a-z0-9._-]{0,119}$`). The file +`recipes/pilot-dense-k1.json` defines the recipe named `pilot-dense-k1`. + +## JSON rules + +`tuldok.json` and every definition are UTF-8 JSON, without a byte order mark, +holding one JSON object. The checker refuses: + +- a key that appears twice in one object; +- `NaN`, `Infinity` and `-Infinity`, and numbers too large to represent; +- any field this document does not define. The contents of `settings`, `config` + and the keys of `labels` are free. + +## `tuldok.json` + +| Field | Required | Rule | +| --- | --- | --- | +| `format` | yes | exactly `tuldok_project_v1` | +| `id` | yes | 32 lowercase hexadecimal characters; make a new one per project | +| `name` | yes | 1 to 120 characters | +| `code_root` | no | the path from the project folder to the root of the code that commands run in; default `.`; not empty and not absolute; may contain `..` | +| `code_paths` | no | 1 to 64 paths relative to the code root, in normal form (no `..`, no empty or `.` parts, no trailing `/`), each listed once and each existing under the code root; default: the whole code root | +| `groups` | no | 0 to 200 objects, in the order their sections are shown on the map. Each: `id` (1 to 64 characters, `[A-Za-z0-9][A-Za-z0-9_.-]*`, each id once), `title` (1 to 200 characters), `parent` (optional: the `id` of another group; a group is never its own ancestor), `description` (optional, at most 2,000 characters); no other field | +| `map` | no | `{"chart": }`, no other field: a chart of the form `progression`, whose benchmark, metric and statistic are the map's track | + +`code_paths` names the parts of the repository a run needs: a run sees a +read-only copy of only those paths at the commit it runs from, and only +changes to tracked files under them stop a run that is not a scratch run. + +The **code root** is the directory that commands run in and that code paths in +definitions are relative to. It must be an existing directory. A folder kept at +`projects/surrogate-lab/` inside the code repository uses `../..`. + +Example, `tuldok.json`: + +```json +{ + "format": "tuldok_project_v1", + "id": "0dcc3af1e6b84b89a0e70554d98e7326", + "name": "Rheon surrogate lab", + "code_root": "../.." +} +``` + +## References and inputs + +A **reference** is the string `/`, for example +`datasets/sinusoidal-v1`. It names the definition in `datasets/sinusoidal-v1.json`, +which must exist. Each field that takes a reference accepts only some +directories: + +| Field | Accepts | +| --- | --- | +| `run.inputs` | `datasets/`, `recipes/`, `steps/` | +| `run.environment` | `environments/` | +| `model` (recipe) | `models/` | +| `subjects[].recipe` (benchmark) | `recipes/` | +| `cases.dataset` (benchmark) | `datasets/` | +| `benchmark` (chart) | `benchmarks/` | +| `dataset` (annotation) | `datasets/` | +| `run.annotations` (a dataset's freeze) | `annotations/` | + +An **input** is what a command reads from another definition's product. It is +written either as a reference, meaning the whole product, or as +`{"from": , "output": }`, meaning one named output that the +other definition declares in its `run.outputs`. + +## Fields common to every definition + +| Field | Required | Rule | +| --- | --- | --- | +| `format` | yes | exactly `tuldok_definition_v1` | +| `kind` | yes | `dataset`, `model`, `recipe`, `benchmark`, `step`, `environment`, `chart` or `annotation`; must match the directory | +| `title` | yes | 1 to 200 characters | +| `description` | no | at most 4,000 characters | +| `card` | no | path of an existing file under `notes/`, written from the project folder, for example `notes/fits/early-fit.md` | +| `labels` | no | object of at most 32 entries; any keys; string values of at most 400 characters | +| `earlier_work` | no | see below; allowed on `dataset`, `model`, `recipe`, `benchmark` and `step` | + +`earlier_work` marks an item converted from work done before this format existed: + +| Field | Required | Rule | +| --- | --- | --- | +| `rebuild` | yes | `yes`, `after-inputs`, `needs-work` or `no` | +| `reason` | when `rebuild` is `needs-work` or `no` | 1 to 1,000 characters | + +The label states a fact about this item alone, never about what it uses: + +- `yes`: its command, code and settings are in Git and the command runs as written. +- `after-inputs`: the same, and one of its inputs is an earlier model's weights. +- `needs-work`: nothing it names is lost, but the recorded command does not run as + written. The reason says what has to change. +- `no`: its command, its code or its producer is lost or was never in Git. The + reason says which. + +Whether an item can in fact be rebuilt also depends on everything it uses; Tuldok +works that out from the references, so do not record it here. An item with +`rebuild: no` may leave out `run`. Code paths that are missing in an item with +`earlier_work` are warnings, not errors. + +Example, `recipes/pressure-fit-0042.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "recipe", + "title": "Pressure fit 0042", + "card": "notes/fits/pressure-fit-0042.md", + "labels": {"fit_id": "fit-0042", "trained_at": "2026-03-14"}, + "earlier_work": {"rebuild": "no", "reason": "Trained by the earlier study harness; its command and training data were not recorded."}, + "model": "models/divergence-free-stepper" +} +``` + +## The `run` block + +`dataset`, `recipe`, `benchmark` and `step` say how they are produced in a `run` +object. It is required unless `earlier_work.rebuild` is `no`. + +| Field | Required | Rule | +| --- | --- | --- | +| `command` | unless `step` is given | 1 to 64 strings, the program and its arguments; run in the code root. When the first string is exactly `python` it means the environment's interpreter | +| `step` | no | datasets only: `import`, `freeze`, `generate`, `embed` or `select`, a step Tuldok provides in place of a command; see [Steps Tuldok provides](#steps-tuldok-provides) | +| `environment` | when the first string of `command` is `python` | reference to `environments/` | +| `inputs` | no | at most 32 entries; keys `^[a-z][a-z0-9_]{0,31}$`; values are inputs from `datasets/`, `recipes/` or `steps/` | +| `settings` | no | object, free content, written to a file whose path replaces `{settings}`; not together with `settings_file` | +| `settings_file` | no | code path of a settings file whose path replaces `{settings}` | +| `env` | no | at most 32 environment variables; names `^[A-Z_][A-Z0-9_]*$`; string values | +| `outputs` | no | keys as for `inputs`; each value `{"path": , "role": }`; role `data`, `weights`, `results`, `log`, `report` or `file` | +| `resources` | no | any of `gpu` (boolean), `cpu_threads` (integer ≥ 1), `memory_gb`, `wall_hours`, `disk_gb` (numbers > 0) | +| `sources` | no | at most 64 code paths (files or directories); a change in them means the code changed | +| `estimate` | no | `{"command": [...]}`, 1 to 64 strings; the command prints one JSON object with any of `disk_bytes`, `wall_seconds`, `items` | +| `annotations` | with `step: "freeze"` or `"annotate"` only | 1 to 32 different references to `annotations/`: the tasks a freeze reads, or the one task an annotate makes a set for | + +A **code path** is a path relative to the code root, at most 1,024 characters. +`settings_file`, `sources`, a model's `config_file` and `sources`, and an +environment's `install.requirements` are code paths. A code path, and an output's +`path`, must be in normal form: not absolute, no `..`, no `.` or empty parts, no +trailing `/`. A code path must exist under the code root (as a file or a +directory) and may not lead outside it through a symbolic link. + +**Placeholders** are written in braces in `command`, in `env` values and in +`estimate.command`: + +| Placeholder | Becomes | +| --- | --- | +| `{output}` | the directory the command writes into | +| `{settings}` | the path of the settings file; requires `settings` or `settings_file` | +| `{inputs.}` | the path of that input; `` must be a key of `inputs` | +| `{subject}` | benchmarks only: the current subject's `argument` | +| `{subject.name}` | benchmarks only: the current subject's name | + +Any other placeholder is an error, and so is a brace without its partner. Write +`{{` and `}}` for literal braces. + +## The nine kinds + +The examples below come from one project. References in them point at the other +examples, except `environments/cuda-torch`, a second environment not shown. + +### `dataset` + +| Field | Required | Rule | +| --- | --- | --- | +| `run` | unless `earlier_work.rebuild` is `no` | see the `run` block | +| `splits` | no | object, split name to a number of items (integer ≥ 0) | +| `view` | no | how its product is shown as records; see [Looking inside a product](#looking-inside-a-product) | + +A dataset's run may name a `step` instead of a `command`: see +[Steps Tuldok provides](#steps-tuldok-provides). + +Example, `datasets/sinusoidal-v1.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Sinusoidal-forcing scenes v1", + "run": { + "command": ["python", "-m", "surrogate.datasets", "generate", "sinusoidal-v1", "--root", "{output}"], + "environment": "environments/lab-cpu", + "inputs": {"solver": {"from": "steps/reference-solver", "output": "probe"}}, + "env": {"SURROGATE_PROBE": "{inputs.solver}"}, + "outputs": {"bank": {"path": "sinusoidal-v1/bank.json", "role": "data"}}, + "resources": {"cpu_threads": 1}, + "sources": ["surrogate/datasets", "surrogate/scenes.py", "surrogate/solver.py"], + "estimate": {"command": ["python", "-m", "surrogate.datasets", "plan", "sinusoidal-v1", "--root", "{output}"]} + }, + "splits": {"train": 16, "development": 8, "reserved": 88} +} +``` + +#### Steps Tuldok provides + +A dataset's `run` names either a `command` or a `step`, one of `import`, +`freeze`, `generate`, `embed`, `select` and `annotate`. A run with `step` has no `command`, `environment`, `outputs`, +`settings_file`, `env` or `estimate` (an `annotate` may have `command`, `environment` and `env`; see +below): the step's outputs are fixed, its +settings are inline in `settings`, and it runs with Tuldok's own interpreter +(see [The command and its environment](#the-command-and-its-environment)). +It may have `resources` and `sources`. A plan shows it as `tuldok import`, +`tuldok freeze`, `tuldok generate`, `tuldok embed`, `tuldok select` or `tuldok annotate`. It is planned, requested, approved, run, cancelled and +recorded like any build, from a commit as builds are. `check` reports a +setting the step does not take, a required one missing and a value of the +wrong form. + +**`import`** brings files on this machine into the dataset. Its run takes no +`inputs`. Settings: + +| Setting | Required | Rule | +| --- | --- | --- | +| `source` | yes | a directory. Relative to the code root, it is a code path read from the code the run sees (the commit, or the working tree for a scratch run), and a change under it makes the product out of date; absolute, it is read on this machine only, and `check` warns | +| `pattern` | no | which files, as a pattern of [Looking inside a product](#looking-inside-a-product) without `!`, matched against paths under the source; default `**/*` | +| `kind` | no | one of the seven kinds, for every file; default by extension | +| `rights` | yes | 1 to 1,000 characters: what may be done with the files | +| `lines` | no | `true` makes each line of each text file a record of its own (a line that is empty or blank is skipped). A line of a `.jsonl` file that is an object with a string `text` gives that string; any other line gives itself | + +Its product holds the files under `files/`, each at its path under the +source, hard-linked (copied when a link cannot be made, such as across file +systems), and `records.jsonl` with one line per file, or per line with +`lines`: + +``` +{"id":"images__cat.png","kind":"image","path":"files/images/cat.png","sha256":"9f…","bytes":5120,"rights":"Photographed by us; free to use.","origin":{"path":"images/cat.png"}} +{"id":"notes__a.txt__4","kind":"text","text":"second line","sha256":"…","bytes":11,"rights":"…","origin":{"path":"notes/a.txt","line":4}} +``` + +An id is made from the path (and `/` and the line number): `/` becomes +`__`, any other character outside `A-Z a-z 0-9 . _ -` becomes `_`, it is cut +to 200 characters, and an id already taken gets `-2`, `-3`, … before its +extension. Files are taken in the order of their paths, bytewise in UTF-8. A +path holding `!` (which an address reads as an archive) is placed with `_` +in its place. `sha256` is the SHA-256 of the file, or of the UTF-8 of a +line's text; null for a file over 1 GiB. A symbolic link whose target is in +the source is taken where it stands; one that leaves the source is skipped +and counted, and so are names that are not UTF-8, entries that are not +regular files, lines that are not UTF-8 and lines too long for one line of +`records.jsonl` (64 KiB). The step never follows a link out of its source, +writes only its output and deletes nothing. + +**`freeze`** makes a dataset with splits from another dataset's records and +their annotations. Its run takes one input, `"inputs": {"records": +"datasets/"}`, and `annotations`, one or more tasks on that dataset; +`check` reports a task on another dataset. Settings: + +| Setting | Required | Rule | +| --- | --- | --- | +| `seed` | yes | an integer of at most 63 bits | +| `shares` | yes | 1 to 16 splits and their relative sizes, integers from 0 to 1,000,000, at least one above 0, for example `{"train": 8, "validation": 1, "test": 1}` | +| `include` | no | the states of the first task's annotations that take a record in; default `["accepted"]` | +| `records` | no | 1 to 20,000 different record ids of the input: only those are taken, before the first task's states decide (a selection on the map); an id the input does not have fails the step | + +It reads each task's line file from the commit the build uses, at the +project folder's place in the code's Git repository (a scratch build reads +the working tree): annotations that are not saved and committed are not +frozen, as code is not. A build from a commit refuses a project folder that +is not in the code's repository. The input's records are its product's +`records.jsonl`, or its files (an input shown through a `view` is refused). +A record is taken when the first task has an annotation of it in a state of +`include` that is not stale: its record exists and, when the annotation +carries a `sha256`, the record's hash is the same. The records taken are +ordered by the SHA-256 of `:` (`:id:` for a +record with no hash), then cut by the shares, the splits taken in the order +of their names (`test`, `train`, `validation`), each split's end being +`count × shares so far ÷ all shares`, rounded down. The output holds their +files linked under `files/` (an archive member brings its whole archive) +and `records.jsonl` with each record's keys, `split` and `annotations`: per +task by name, its `state` and `value`, for each task whose annotation of the +record is not stale. + +``` +{"id":"images__cat.png","kind":"image","path":"files/images/cat.png","sha256":"9f…","bytes":5120,"rights":"…","origin":{"path":"images/cat.png"},"split":"train","annotations":{"scene-quality":{"state":"accepted","value":"clean"}}} +``` + +Stale annotations, records left out by their state, records with no +annotation, invalid lines, stale annotations of the other tasks and records +whose file is missing are counted in the run's progress and printed in its +log. A record whose id is not one of `records.jsonl` (a file of an input +with no `records.jsonl`) gets one made as `import` makes them. + +A frozen dataset's run records the SHA-256 of each task's line file as it +read it; see [Build states](#build-states). + +Example, `datasets/scene-photos.json` and `datasets/scene-photos-v1.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Scene photos", + "run": { + "step": "import", + "settings": {"source": "data/scenes", "pattern": "**/*.png", "rights": "Photographed by us; free to use."} + } +} +``` + +```json +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Scene photos v1, frozen", + "run": { + "step": "freeze", + "inputs": {"records": "datasets/scene-photos"}, + "annotations": ["annotations/scene-quality"], + "settings": {"seed": 7, "shares": {"train": 8, "validation": 1, "test": 1}} + } +} +``` + +**`generate`** makes a dataset by asking a model that the machine's Pumas +endpoint serves (see [The Pumas endpoint](#the-pumas-endpoint)): one request +per item, over HTTP, the endpoint's address given by the build and never +written in the folder. Its run takes no input, or one, `"inputs": {"prompts": +"datasets/"}`, and no `annotations`. Settings: + +| Setting | Required | Rule | +| --- | --- | --- | +| `make` | yes | `text` or `image` | +| `model` | yes | the served model's alias: 1 to 256 bytes, no control characters, no spaces around it | +| `profile` | no | the profile it is served with, 1 to 128 of `A-Z a-z 0-9 . _ -`; needed when the endpoint serves the model with more than one | +| `prompt` | yes | 1 to 16,000 characters: the request, with `{n}` (the item's number from 1) and, with an input, `{text}` (the prompting record's text); `{{` and `}}` are literal braces | +| `count` | without an input | how many to make, 1 to 100,000; not with an input | +| `per` | no | with an input only: how many for each of its text records, 1 to 1,000, default 1 | +| `seed` | no | 0 to 4,294,967,295; item `n` is asked with `seed + n − 1` when the capability takes a seed (it advertises a `seed` bound and its options kind has one: `image_generation`) | +| `options` | no | at most 32 options of the capability, numbers: `max_tokens`, `temperature`, `top_p` for text, `width` and `height` (both required) for images; each must be advertised with a bound by the endpoint and within it, and `seed` is not one | +| `rights` | yes | 1 to 1,000 characters: what may be done with what is made | + +Before anything is asked the step reads what the endpoint serves +(`GET /v1/served-models`) and fails, saying why, when the model or profile is +not served, no capability of it is available to make the output (`text`, or +`png_base64` for an image) from a text, an option is not one the capability +takes or is outside its advertised bounds, or a prompt (`input_characters`) or +a seed is. The capability is the first of the model's available ones that +makes the output, in the endpoint's order. A request refused before the +provider took it (outcome `not_admitted`) is asked again after 1 s and after 2 +s more; a third refusal, or any other failure, fails the step with the item's +number and the endpoint's reason. What was made stays in the failed run's +directory and is no product. With an input, one request is made for each text +record of its product (inline, or a UTF-8 file of at most 64 KiB) times `per`, +in the product's order; records of other kinds are skipped and counted. An +input shown through a `view` is refused. + +Its product holds `records.jsonl` and, for images and texts too long for one +line, the files under `files/`: + +``` +{"id":"g000001","kind":"text","text":"Waves fold over.","sha256":"…","bytes":16,"rights":"…","origin":{"generated":{"model":"writer","profile":"writer-cpu","capability":"chat_generation","prompt":"Write line 1 of a short poem.","seed":null,"options":{"max_tokens":64},"input":null}}} +{"id":"g000001","kind":"image","path":"files/g000001.png","sha256":"…","bytes":5120,"rights":"…","origin":{"generated":{"model":"painter","profile":"painter-gpu","capability":"image_generation","prompt":"A picture of a red ball, number 1.","seed":100,"options":{"width":64,"height":32},"input":"subjects.txt__1"}}} +``` + +`seed` is the seed the item was made with (for an image, the one the endpoint +reports), null when the capability took none; `input` is the prompting +record's id, or null. A generated dataset is built again only when its +definition or its input changes; a model need not give the same content +again, and annotations that no longer fit are then stale. Its run keeps the +endpoint's address and reported version (`endpoint` of `run`; see +[The run directory](#the-run-directory)); +on a machine with no endpoint set, a plan shows it `blocked`. + +Example, `datasets/poem-lines.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Poem lines", + "run": { + "step": "generate", + "settings": {"make": "text", "model": "writer", "prompt": "Write line {n} of a short poem.", "count": 3, + "seed": 5, "options": {"max_tokens": 64, "temperature": 0.7}, + "rights": "Made by a model; anything may be done with it."} + } +} +``` + +**`embed`** writes embeddings of its input's records (see +[A dataset's embeddings](#a-datasets-embeddings)) by asking the embedder the +machine's Pumas endpoint serves. Its run takes one input, `"inputs": +{"records": "datasets/"}`, and no `annotations`. Every setting is +optional: + +| Setting | Rule | +| --- | --- | +| `model`, `profile` | the served alias (and its profile) to embed with. Without one, the machine's `embedding_model` (see [The project directory on this machine](#the-project-directory-on-this-machine)); without that, **EmbeddingGemma 2**: the served model whose alias or model id holds `embeddinggemma-2` (ignoring case), `unsloth/embeddinggemma-2-GGUF` first. When none is served the step fails naming the embedders the endpoint does serve; another model is never chosen in its stead | +| `prefix` | at most 1,000 characters put before each text; default the machine's `embedding_prefix`, else `task: clustering \| query: ` | +| `dimensions` | 1 to 4,096, asked as `{"kind": "embeddings", "dimensions": d}` (Pumas truncates and renormalises); checked against an advertised bound | +| `name` | the set's name, 1 to 64 of `a-z 0-9 . _ -`; default the input's name | +| `batch` | records a request carries, 1 to 256, default 32, at most the advertised `input_count` (and `image_count`, 4 when none, for images) | + +Texts (inline, or UTF-8 files of at most 64 KiB) are asked by +`text_embedding` with `{"kind": "text_batch", "texts": [...]}`; PNG and JPEG +images of at most 8 MiB by `multimodal_embedding` with `{"kind": +"content_batch", "items": [{"parts": [{"kind": "image", ...}]}]}`; other +records are not embedded and are said so in the index's `notes`. The +answer must hold one finite vector per item, all of one length. Before +anything is asked the step reads what the endpoint serves and fails when the +model is not served, cannot embed any kind the input holds, or `dimensions` +is past its bound. A request refused before the provider took it is asked +again after 1 s and 2 s; anything else fails the step with the batch's first +record. Every batch answered is kept in the run's output as it comes; when +the dataset's last run failed, the next build takes the rows that run was +answered, asked with the same input run, model, prefix and dimensions, and +asks only for the rest. Its product holds only `embeddings/.npy` and +`.json`, `source` naming the input and its run, `model` what answered +(`name`, `model_id`, `profile`, `capability`, `dimensions_asked`, +`named_by`: `definition`, `setting` or `default`, `via: "pumas"`). Like a +`generate`, its run keeps the endpoint it asked, and a plan on a machine with +no endpoint shows it `blocked`. + +**`select`** takes some of its input's records: `"inputs": {"records": +"datasets/"}` and `"settings": {"records": [ids]}` (1 to 20,000 +different ids). Its product holds exactly those records in the input's +order, each line as the input has it (its `split` and `annotations` kept), +its file linked under `files/`, its hash added when the input has none. An +id the input does not have fails the step. A selection on the map becomes +such a dataset. + +##### `annotate`: an annotation set made by an annotator + +A dataset whose run is `"step": "annotate"` makes an **annotation set**: the +annotations of one annotator (a project's script, or a model behind the +Pumas endpoint) for one task, beside the task's own annotations in Git. Its +run takes one input, `"inputs": {"records": "datasets/"}`, the task's +dataset, and `"annotations": ["annotations/"]`. Unlike the other steps +it may have `command`, `environment` and `env`: the command is the project's +annotator, run in its environment from the code snapshot. Exactly one of +`run.command` and `settings.pumas`. Settings: + +| Setting | Required | Rule | +| --- | --- | --- | +| `by` | yes | `{"kind": "model", "script" or "person", "name": 1 to 200 characters, "version"?}`: what makes the annotations | +| `pumas` | no | `{"model", "profile"?, "instruction"?}`: ask this model of the machine's Pumas endpoint instead of a command; for `text`, `class` and `tags` as a suggestion is asked, for `boxes` and `polygons` on images as a JSON list asked of a vision model | +| `limit` | no | at most this many records, 1 to 1,000,000 | +| `options` | no | an object of at most 8 KiB, given to the command as `TULDOK_ANNOTATE_OPTIONS` | + +**The command's contract.** It runs with these variables: `TULDOK_RECORDS` +(the input product), `TULDOK_TODO` (a file of the input's record lines still +to annotate, their paths made absolute), `TULDOK_TASK` (the task's +definition), `TULDOK_ANNOTATIONS` (the file to append lines to), +`TULDOK_FILES` (the directory for sidecars) and `TULDOK_ANNOTATE_OPTIONS`. +It appends one line per record, `{"record", "value", "confidence"?, +"note"?}`, in any order, writes the sidecars its values name under +`TULDOK_FILES`, and may print progress lines. When it ends with status 0 the +step keeps the last valid line of each record of the input, drops and counts +the rest, and writes the product: + +``` +annotator.json {"format": "tuldok_annotator_v1", "task", "by": {"kind", "name", "version"?, "settings"}, "records", "made", "at"} +annotations.jsonl {"record", "sha256", "value", "confidence"?, "note"?} per record, sorted by record +files/ the sidecars the lines name +``` + +`by.settings` is the run's settings without `by`. A run of the same +definition that failed or was cancelled is resumed by the next build: its +valid lines (and sidecars) whose records are unchanged are kept, and only the +other records are given to the command (or asked of the model). + +The sets of a task (the kept set, `annotations/.jsonl`, and every +annotate's product for it) are compared on the Annotate tab (Review): their +agreement per pair and per record, the most disputed records, each drawn +over the record, and a decision (keep one set's annotation, keep two sets' +together, or reject them) kept as an unsaved edit of `annotations/.jsonl` +whose `from` names the set, its run and what made it; a kept sidecar is +copied into `annotations//` by Save. + +### `model` + +A model is an architecture. It produces nothing by itself; recipes train it. + +| Field | Required | Rule | +| --- | --- | --- | +| `entry` | no | non-empty string naming the code entry point, for example `surrogate.models:DivergenceFreeStepper` | +| `config` | no | object, free content; not together with `config_file` | +| `config_file` | no | code path | +| `sources` | no | at most 64 code paths | + +Example, `models/divergence-free-stepper.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "model", + "title": "Divergence-free U-Net stepper", + "entry": "surrogate.models:DivergenceFreeStepper", + "sources": ["surrogate/models/stepper.py", "surrogate/models/unet3d.py"] +} +``` + +### `recipe` + +A recipe trains a model; its product is weights. + +| Field | Required | Rule | +| --- | --- | --- | +| `run` | unless `earlier_work.rebuild` is `no` | see the `run` block | +| `model` | no | reference to `models/` | +| `group` | no | the `id` of one of the project's `groups` in `tuldok.json`: the section of the map it is shown in | +| `change` | no | `training-policy`, `architecture`, `refinement`, `component` or `sweep-trial`: the kind of change this model makes | +| `derived_from` | no | reference to another recipe in `recipes/`: the model this one was branched from. Never the recipe itself, and following `derived_from` never comes back to where it started | +| `compared_with` | no | at most 50 objects, each `recipe` (reference to another recipe, each once in the list), `explanation` (1 to 2,000 characters) and `excerpt` (optional, at most 2,000 characters); no other field | +| `serve` | no | how Tuldok serves the model: `module_file` (a code path of a `.py` file, required), `function` (default `predict`), `load`, `backend` (`local`, the default and only one), `environment` (default `run.environment`; one of the two is required), `idle_minutes` (1 to 1,440), `start_seconds` (1 to 3,600), `request_seconds` (0.1 to 600), `gpu`, `example` (an object); see [Serving a model](#serving-a-model). Not an input: it never makes the model out of date | +| `sweep` | no | a trial of a sweep (see [`sweep`](#sweep)): `definition` (reference to `sweeps/`), `trial` (its number, 1 or more) and `values` (the settings it was given, by path, at most 16); written by Tuldok when it runs the sweep | + +The label `short`, when a recipe has it, is the text of its node on the map +(at most 16 characters are shown). `derived_from` and `compared_with` say how +recipes relate on the map; they are not inputs, so building never follows +them, and a comparison may run both ways. + +Example, `recipes/pilot-dense-k1.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "recipe", + "title": "Pilot stepper on sinusoidal scenes, one output per pass", + "model": "models/divergence-free-stepper", + "run": { + "command": ["python", "-m", "surrogate.train", "{settings}", "{output}", "--device", "cuda"], + "environment": "environments/cuda-torch", + "settings_file": "surrogate/configs/pilot-dense-k1.json", + "inputs": {"training": "datasets/sinusoidal-v1"}, + "env": {"SURROGATE_DATA": "{inputs.training}"}, + "outputs": { + "weights": {"path": "model.pt", "role": "weights"}, + "log": {"path": "log.jsonl", "role": "log"} + }, + "resources": {"gpu": true}, + "sources": ["surrogate/train.py", "surrogate/models"] + } +} +``` + +### `step` + +A step is any other command whose output something needs, such as building a +solver. + +| Field | Required | Rule | +| --- | --- | --- | +| `run` | unless `earlier_work.rebuild` is `no` | see the `run` block | + +Example, `steps/reference-solver.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "step", + "title": "Rheon reference solver probe", + "run": { + "command": ["cargo", "build", "--release", "--locked", "--manifest-path", "probes/reference/Cargo.toml"], + "env": {"CARGO_TARGET_DIR": "{output}"}, + "outputs": {"probe": {"path": "release/rheon-reference-probe", "role": "file"}}, + "sources": ["probes/reference/Cargo.toml", "probes/reference/Cargo.lock", "probes/reference/src"] + } +} +``` + +### `benchmark` + +A benchmark measures **subjects** (the things being compared, usually trained +recipes) on **cases** (usually one split of a dataset) and reports **metrics**. +Its command runs once per subject. + +| Field | Required | Rule | +| --- | --- | --- | +| `run` | unless `earlier_work.rebuild` is `no` | see the `run` block; `outputs` must declare exactly one output with role `results` | +| `metrics` | yes | 1 to 64 objects, see below | +| `subjects` | yes | 0 to 1,000 objects, see below | +| `cases` | no | `{"dataset"?: reference to datasets/, "split"?: non-empty string, "count"?: integer ≥ 0}` | +| `summary` | no | `{"group_by"?: up to 8 distinct ids, "statistics"?: distinct names from mean, median, p10, p90, min, max}` | + +A metric is `{"id", "title", "units"?, "better"}`: `id` matches +`^[a-z][a-z0-9_]{0,63}$` and is unique in the benchmark; `title` is 1 to 200 +characters; `units` is any string; `better` is `lower`, `higher` or `none`. + +A subject is `{"name", "title"?, "recipe"?, "argument"?}`: `name` follows the name +rule and is unique in the benchmark; `title` is 1 to 200 characters; `recipe` is a +reference to `recipes/`; `argument` is the string that replaces `{subject}`. The +only placeholder an argument may hold is `{recipe}`, the path of that recipe's +product, and only when the subject has a `recipe`. + +`summary.group_by` names the dimensions the summary is split by, written with the +metric id pattern. `summary.statistics` says which statistics every summary row +with measured values must fill; without it, all six. + +Example, `benchmarks/dense-development-240.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "benchmark", + "title": "Sinusoidal development scenes, 240-output rollouts", + "run": { + "command": ["python", "-m", "surrogate.bench", "run", "--suite", "dense-development-240", "--subject", "{subject}", "--output", "{output}/result.json"], + "environment": "environments/cuda-torch", + "inputs": {"cases": "datasets/sinusoidal-v1"}, + "env": {"SURROGATE_DATA": "{inputs.cases}"}, + "outputs": {"results": {"path": "result.json", "role": "results"}}, + "sources": ["surrogate/bench"] + }, + "metrics": [ + {"id": "velocity_error_final", "title": "Final velocity error", "units": "ratio", "better": "lower"}, + {"id": "time_per_output_ms", "title": "Time per output", "units": "ms", "better": "lower"} + ], + "subjects": [ + {"name": "persistence", "title": "No change: the error any model must beat", "argument": "persistence"}, + {"name": "pilot-dense-k1", "recipe": "recipes/pilot-dense-k1", "argument": "checkpoint:{recipe}/model.pt"} + ], + "cases": {"dataset": "datasets/sinusoidal-v1", "split": "development", "count": 8}, + "summary": {"group_by": ["cells_per_metre"]} +} +``` + +### `environment` + +An environment says which Python and packages a command needs. Commands whose +first word is `python` name one. + +| Field | Required | Rule | +| --- | --- | --- | +| `python` | yes | `3.`, for example `3.12` | +| `device` | yes | `cpu` or `cuda` | +| `packages` | yes | 1 to 64 entries, package name to exact version. Names match `^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$`; versions match `^[0-9A-Za-z][0-9A-Za-z.+!_-]*$`, so `2.8.0+cpu` is allowed and `>=2.8` is not | +| `install` | no | `{"requirements": , "index_urls"?: up to 4 https:// URLs}` | + +`earlier_work` is not allowed here. + +Example, `environments/lab-cpu.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "environment", + "title": "Lab CPU environment", + "python": "3.12", + "device": "cpu", + "packages": {"numpy": "2.3.5", "torch": "2.8.0+cpu"}, + "install": { + "requirements": "tuldok/requirements-cpu.lock", + "index_urls": ["https://download.pytorch.org/whl/cpu"] + } +} +``` + +### `chart` + +A chart is drawn from one benchmark's summary. + +| Field | Required | Rule | +| --- | --- | --- | +| `benchmark` | yes | reference to `benchmarks/` | +| `form` | yes | `ranking`, `tradeoff`, `progression` or `comparison` | +| `metric` | yes | a metric id of that benchmark | +| `metric_x` | when `form` is `tradeoff`; otherwise not allowed | a metric id of that benchmark | +| `statistic` | no | `mean`, `median`, `p10`, `p90`, `min` or `max`; default `median` | +| `subjects` | no | distinct subject names of that benchmark; default all | +| `group` | no | one of that benchmark's `summary.group_by` | + +`earlier_work` is not allowed here. + +Example, `charts/final-velocity-ranking.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "chart", + "title": "Final velocity error by subject", + "benchmark": "benchmarks/dense-development-240", + "form": "ranking", + "metric": "velocity_error_final", + "group": "cells_per_metre" +} +``` + +### `annotation` + +An annotation task says what is recorded about the records of one dataset +(see [Looking inside a product](#looking-inside-a-product) for records). Its +annotations are the lines of `annotations/.jsonl` beside it. A task is +never built: it has no `run`, no product, and the state `none`. Planning a task +plans its dataset in its place. + +| Field | Required | Rule | +| --- | --- | --- | +| `dataset` | yes | reference to `datasets/` | +| `task` | yes | `review`, `class`, `tags`, `text`, `boxes`, `spans`, `segments`, `polygons`, `mask`, `depth`, `keypoints`, `ranking`, `timeseries`, `track`, `points` or `box3d` | +| `vocabulary` | for every task but `review`, `text` and `depth`, which allow none | 1 to 500 different **terms**, each 1 to 200 characters without control characters; for `ranking`, the candidates ranked | +| `keypoints` | for `keypoints` only, required | 1 to 100 different names: the points of one instance, in the order its values list them | +| `skeleton` | for `keypoints` only | pairs of those names, joined when drawn | +| `sigma` | for `keypoints` only | a number above 0: the spread of the agreement measure (OKS); default 0.1 | +| `units` | for `depth` only, required | the units of the depth values, 1 to 40 characters, such as `m` | +| `mask` | for `mask` only | `semantic` (a pixel's value is its class) or `instance` (each value is one object); default `semantic` | + +`earlier_work` is not allowed here. `card` is where the guide for the people +and agents who annotate is written. + +What an annotation's `value` is, by task, and which records it may describe: + +| Task | `value` | Records of kind | +| --- | --- | --- | +| `review` | always null; the state is the decision | any but `file` | +| `class` | one term | any but `file` | +| `tags` | a list of different terms, possibly empty | any but `file` | +| `text` | a string of at most 64 KiB in UTF-8: a caption, a transcript, a summary | any but `file` | +| `boxes` | a list of `{"term", "x", "y", "w", "h"}`, numbers in pixels of the image; `x` and `y` at least 0, `w` and `h` above 0, the box inside the image | `image` | +| `spans` | a list of `{"term", "start", "end"}`, whole numbers of characters (code points) of the text; 0 ≤ `start` < `end` ≤ the text's length | `text` | +| `segments` | a list of `{"term", "start", "end"}` with an optional `"text"` (a string of at most 64 KiB), numbers of seconds; 0 ≤ `start` < `end` ≤ the sound's length | `audio` | + +| `polygons` | a list of `{"term", "points": [[x, y], …]}`: 3 to 10,000 points in pixels, inside the image | `image` | +| `mask` | `{"file": , "classes"}` or `{"rle": [[value, count], …], "classes"}`: `classes` maps a pixel value (`"1"` to `"65535"`) to a term, 0 is nothing; `rle` gives the pixels row by row from the top left and covers the image | `image` | +| `depth` | `{"file": }`: a 2-D array (height, width) of float32, float64 or uint16 in the task's units; 0 and non-finite values are no depth | `image` | +| `keypoints` | a list of `{"term", "points"}`, one point per name of the task's `keypoints`: `[x, y]`, `[x, y, v]` (v 0, 1 or 2) or null | `image` | +| `ranking` | a list of different terms, best first | any but `file` | +| `timeseries` | a list of `{"term", "start", "end"}`, 0 ≤ `start` < `end`: seconds of a sound, or positions along the first axis of an array | `audio`, `array` | +| `track` | a list of `{"track", "term", "frames": [{"frame", "x", "y", "w", "h"}, …]}`: `track` an id of 1 to 64 characters, once per annotation; frames whole numbers increasing, below the array's first axis; boxes inside its frames | `array` | +| `points` | a list of `{"term", "indices": [i, …]}` (indices increasing, below the cloud's point count), or `{"file": , "classes"}`, a 1-D array of one whole number per point (0: no object) | `pointcloud` | +| `box3d` | a list of `{"term", "center": [x, y, z], "size": [l, w, h], "yaw"}`, sizes above 0, `yaw` in radians about z | `pointcloud` | + +An item of a list value, and an object value, may also carry `"confidence"`, a +number from 0 to 1. A list holds at most 1,000 items, and an item no other key. The value may be +null in any state but `accepted`. The record's kind, an image's size, a text's +length and a sound's length are known only from the dataset's product, so the +folder check cannot apply those rules; Tuldok applies them when an annotation +is made. + +Example, `annotations/scene-quality.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "annotation", + "title": "Scene quality", + "card": "notes/annotating/scene-quality.md", + "dataset": "datasets/sinusoidal-v1", + "task": "class", + "vocabulary": ["clean", "noisy", "broken"] +} +``` + +#### The annotations file + +`annotations/.jsonl` holds the annotations of task ``, one JSON +object per line, one line per record. A record with no line is not annotated. +The keys, in this order: + +| Key | Required | Rule | +| --- | --- | --- | +| `record` | yes | the record's id: 1 to 600 characters, no control characters; each record once | +| `sha256` | yes | the SHA-256 of the record's content when it was annotated (64 lowercase hexadecimal characters), or null | +| `state` | yes | `draft`, `accepted`, `needs-work` or `rejected` | +| `value` | yes | as the task says, or null | +| `note` | no | a string of at most 4,000 characters | +| `by` | yes | `owner`, or `agent:` followed by 1 to 80 characters (the agent credential's name, cut) | +| `at` | yes | a real UTC time, `YYYY-MM-DDTHH:MM:SSZ` | +| `from` | no | an object of at most 2 KiB, written compact, saying what suggested the value; kept as it is | + +The file is written in one exact form, so that Git shows one changed line per +changed annotation: + +- lines sorted by `record`, comparing the bytes of its UTF-8; +- each line written compact (`,` and `:` with no spaces), with characters + other than ASCII written as themselves rather than as `\u` escapes, the keys + in the order above and the keys of an item in the order `track`, `frame`, + `term`, `x`, `y`, `w`, `h`, `start`, `end`, `text`, `points`, `indices`, + `center`, `size`, `yaw`, `frames`, `confidence` (an object value: `file`, + `rle`, `classes`, `confidence`; `classes` by number); +- each line ending with `\n`, the last one included. + +A file with no line left is deleted when it is saved. The JSON rules above +apply to each line: no repeated key, no `NaN` or `Infinity`, and no escaped +half of a surrogate pair. + +The check reads every line. A line that is not such an object, a value the task +does not allow, a record that an earlier line has, and a line whose record +does not sort after the one above it are errors; a term outside the +vocabulary is a warning, so that a term can be removed from a vocabulary +while annotations still use it. Each problem names its line (`line `): the +first 20 of a file, then one problem counting the rest. A file of annotations +is not checked while its task has errors. + +Example, `annotations/scene-quality.jsonl`: + +``` +{"record":"scene-0001","sha256":"9f2c0e5d1b8a7f6e5d4c3b2a19087f6e5d4c3b2a19087f6e5d4c3b2a19087f6e","state":"accepted","value":"clean","by":"owner","at":"2026-11-13T10:02:11Z"} +{"record":"scene-0002","sha256":null,"state":"rejected","value":null,"note":"blurry","by":"owner","at":"2026-11-13T10:02:40Z"} +{"record":"scene-0003","sha256":null,"state":"draft","value":"noisy","by":"agent:helper","at":"2026-11-13T10:05:02Z"} +``` + +#### Sidecars + +A `mask`, `depth` or `points` value naming a `file` names a **sidecar** in +`annotations//`, beside the task: a file named by the first 16 +hexadecimal characters of its SHA-256 and `.png` or `.npy` (so a name never +changes meaning and a file is never rewritten). The directory holds such +files only. A mask PNG is 8- or 16-bit greyscale, not interlaced, of the +image's size; an `.npy` is a C-order array of floats or whole numbers, never +pickled. The check reports a directory with no task, and a line naming a +sidecar that is not there, as errors, and a sidecar no line names as a +warning; Save removes those of a task it writes. Each is at most 64 MiB. + +### `sweep` + +A sweep searches the settings of one recipe on this machine, trial after +trial (docs/plans/training-and-curation/spec-t7.md). Each trial is a recipe +Tuldok writes, derived from the varied one; a sweep is built by asking for a +build of it alone, and one approval covers its whole budget. + +| Field | Required | Rule | +| --- | --- | --- | +| `recipe` | yes | reference to `recipes/`: the recipe whose `run.settings` it varies. That recipe has `run.command` and its settings written in it (not `settings_file`) | +| `parameters` | yes | 1 to 16 objects: `path`, a dotted path into `run.settings` (a part of digits indexes a list; every part but the last exists in the recipe as an object or a list), and either `values` (1 to 64 distinct numbers, texts, booleans or null) or a range `low` < `high` with `scale` (`linear`, the default, or `log`, then `low` > 0), `integer` (whole numbers) and `count` (2 to 64 points, for a grid). Each path once | +| `strategy` | yes | `{"kind": "grid"}` (every range has `count`; at most 100,000 points), `{"kind": "random", "seed"}`, or `{"kind": "halving", "seed", "metric", "better", "unit", "min", "factor"}`: random draws, and a trial stops early at a rung `min × factor^k` (in `step`s, the default, or seconds of `time`) where it is not among the best `1/factor` of the values of `metric` recorded there (`factor` 2 to 8, default 3) | +| `objectives` | yes | one or two objects with distinct `metric` (a series the trials log in `training/metrics.jsonl`, or `time` or `step`, the last logged), `better` (`lower` or `higher`), `take` (`last`, the default, or `best`) and an optional `title` | +| `budget` | yes | `trials` (1 to 1,000), optional `minutes` (above 0, at most 10,080) and `concurrency` (only 1: trials run one after another) | + +A grid with more points than `budget.trials` is a warning: the first in order +are run. The edge from a sweep to its recipe has the role `varies`, and a +trial's edge to its sweep the role `sweep`; like `derived`, building never +follows them. + +**A trial** is `recipes/-` (then `-2`, `-3`, … when that name +holds something else; a trial is never overwritten), with the recipe's `run` +and its values set in `run.settings`, `derived_from` the recipe, `change` +`sweep-trial`, the label `short` `T` and the `sweep` field. It is kept as +an unsaved edit and saved like any other. A trial of the same sweep with the +same text that is built already is reused, not run again. + +Example, `sweeps/lr-width.json`: + +```json +{ + "format": "tuldok_definition_v1", + "kind": "sweep", + "title": "Learning rate and width", + "recipe": "recipes/pilot-dense-k1", + "parameters": [ + {"path": "training.learning_rate", "low": 0.0001, "high": 0.01, "scale": "log"}, + {"path": "model.width", "values": [16, 32, 64]} + ], + "strategy": {"kind": "halving", "seed": 7, "metric": "val/loss", "better": "lower", "min": 200, "factor": 3}, + "objectives": [{"metric": "val/loss", "take": "best", "better": "lower"}, {"metric": "time", "better": "lower"}], + "budget": {"trials": 24, "minutes": 120} +} +``` + +## Results files + +`results/.csv` is the summary of benchmark `benchmarks/.json`, which +must exist. One row summarises the values of one metric for one subject, either +over all cases or over one group of them. + +- UTF-8, `\n` line ends (a `\r` is an error), CSV quoting as in RFC 4180, no + empty lines. The file should end with a line end; a missing one is a warning. +- The first line is exactly + `subject,metric,group,count,unavailable,mean,median,p10,p90,min,max,commit,produced` + and every other line has these 13 cells. + +| Column | Rule | +| --- | --- | +| `subject` | a subject name of the benchmark | +| `metric` | a metric id of the benchmark | +| `group` | empty for all cases, or `key=value` pairs joined by `;`, keys distinct, sorted, and each in the benchmark's `summary.group_by`; a value may not contain `;`, `=` or a line break | +| `count` | integer ≥ 0, digits only | +| `unavailable` | integer ≥ 0, digits only | +| `mean`, `median`, `p10`, `p90`, `min`, `max` | a decimal number matching `-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?`, or empty. All six are empty when `count` is 0. When `count` is above 0, each statistic in the benchmark's `summary.statistics` (all six by default) is filled; the others may be empty | +| `commit` | 40 lowercase hexadecimal characters (the Git commit of the code that measured), or empty | +| `produced` | a real date `YYYY-MM-DD` or time `YYYY-MM-DDTHH:MM:SSZ` | + +Rows are unique on (`subject`, `metric`, `group`) and sorted by those three +compared as plain strings, so `cells_per_metre=16` comes before +`cells_per_metre=8`. + +**Statistics.** For one subject, metric and group, take the measured values in +ascending order, v₀ … vₙ₋₁. `count` is n; `unavailable` is how many values could +not be measured. `mean` is the arithmetic mean, `min` is v₀ and `max` is vₙ₋₁. +`median`, `p10` and `p90` are the value at position p × (n − 1) for p = 0.5, 0.1 +and 0.9, interpolating linearly between the two closest values when the position +is not whole. For 0.12, 0.15, 0.2, 0.31, `p10` is at position 0.3: +0.12 + 0.3 × (0.15 − 0.12) = 0.129. Numbers are written as Python's +`repr(float)` writes them. `tuldok_project.results.summarise` computes them. + +Example, `results/dense-development-240.csv`: + +```csv +subject,metric,group,count,unavailable,mean,median,p10,p90,min,max,commit,produced +persistence,time_per_output_ms,,0,8,,,,,,,,2026-11-09 +persistence,velocity_error_final,cells_per_metre=8,4,0,0.9875,0.975,0.915,1.07,0.9,1.1,,2026-11-09 +pilot-dense-k1,time_per_output_ms,,8,0,43.825,43.8,40.76,46.99,40.2,47.9,4f1c0e9a7b2d3c5e6f708192a3b4c5d6e7f80912,2026-11-09T14:30:00Z +pilot-dense-k1,velocity_error_final,cells_per_metre=16,4,0,0.2625,0.235,0.192,0.355,0.18,0.4,4f1c0e9a7b2d3c5e6f708192a3b4c5d6e7f80912,2026-11-09T14:30:00Z +pilot-dense-k1,velocity_error_final,cells_per_metre=8,4,0,0.195,0.175,0.129,0.277,0.12,0.31,4f1c0e9a7b2d3c5e6f708192a3b4c5d6e7f80912,2026-11-09T14:30:00Z +``` + +## Rules across files + +1. Every reference names an existing definition in a directory its field accepts. +2. Every `{"from", "output"}` input names an output that the definition it comes + from declares in `run.outputs`. +3. Definitions may not use each other in a cycle. A definition uses another + through these fields: + + | Field | Role of the use | + | --- | --- | + | an entry of `run.inputs` | `input:` | + | `run.environment` | `environment` | + | a recipe's `model` | `model` | + | a benchmark subject's `recipe` | `subject:` | + | a benchmark's `cases.dataset` | `cases` | + | a chart's `benchmark` | `benchmark` | + | an annotation task's `dataset` | `dataset` | + + Two more fields of a recipe are edges with a role, shown in a definition's + uses and used-by, but are not uses in this sense: they may run both ways, + carry no state, and the plan and the build order do not follow them. + + | Field | Role of the edge | + | --- | --- | + | a recipe's `derived_from` | `derived` | + | a recipe's `compared_with[].recipe` | `compared` | + +4. A chart's `metric`, `metric_x`, `subjects` and `group` exist in its benchmark. +5. Code paths exist under the code root (see the `run` block). +6. Every results file has its benchmark and follows the rules above. Every + file of annotations has its task and follows the rules of its task. +7. A recipe's `group` is a group of `tuldok.json`; a group's `parent` is another + group and no group is its own ancestor; a recipe is never derived from or + compared with itself; following `derived_from` never comes back to where it + started; `map.chart` is a chart of the form `progression`. + +## Checking a folder + +From the Tuldok repository: + +```sh +.venv/bin/python -m tuldok_project check FOLDER [--no-code] [--json] +``` + +`FOLDER` is the directory that holds `tuldok.json`. `--no-code` skips looking +into the code root: the code root and code paths are not required to exist, but +paths are still checked for normal form. `--json` prints one JSON object instead +of text. + +The exit status is 0 when there are no errors (warnings allowed), 1 when there is +at least one error, and 2 for a usage error such as an unknown option. + +Every problem is reported; the check does not stop at the first. A problem has a +severity (`error` or `warning`), the file's path relative to the project folder, +a JSON pointer to the field (empty for a problem with the whole file; `line ` +in a results file or a file of annotations), and one sentence. Text output prints one line per problem, +with the parts separated by two spaces and `-` for an empty pointer, sorted by +path and pointer, then a count: + +``` +error recipes/pilot-dense-k1.json /run/inputs/training datasets/sinusoidal-v2 does not exist; add datasets/sinusoidal-v2.json or fix the reference. +warning results/dense-development-240.csv - The results file does not end with a line end. +1 error, 1 warning in /home/me/lab/projects/surrogate-lab +``` + +With `--json` the object is `{"folder", "ok", "errors", "warnings", "problems"}`, +each problem being `{"severity", "path", "pointer", "message"}`. + +These are warnings; every other problem is an error: + +- a code path that does not exist, in a definition with `earlier_work`; +- a results file without a final line end; +- a results file that is not checked because its benchmark has errors; +- in a file of annotations: a term that is not in the task's vocabulary (one + warning per term, saying how many annotations use it and on which lines), a + blank line, a line not written in the exact form below, a missing final line + end, and a file with no lines; +- a file of annotations that is not checked because its task has errors. + +One mistake is reported once. A reference into the wrong directory is reported by +the field's own rule and not again as a missing definition, and a chart is not +compared with a benchmark that has errors of its own. + +## Working with the database copy + +Tuldok keeps a working copy of a project folder in Postgres: the exact text of +every definition, the uses between them, and edits that are not yet written back +to the folder. Each project has its own schema, `project_`, named after the +`id` in `tuldok.json`. These commands work on that copy: + +```sh +.venv/bin/python -m tuldok_project COMMAND FOLDER [options] +``` + +| Command | Does | +| --- | --- | +| `open` | Imports the folder into the database, or attaches to the copy when no definition file changed. Prints the number of definitions of each kind and in each state, and how long opening took. | +| `list [--kind K]` | Prints each definition's reference, state and title, one per line. `--kind` keeps one kind. | +| `show REFERENCE` | Prints a definition's reference, title, kind and state, what it uses and what uses it (each with its role), then the definition's text. | +| `graph [--order]` | Prints every definition with its kind and state, then every use as `from role to`. `--order` instead prints the build order: each definition after everything it uses, with its state. | +| `status` | Lists definitions edited or removed in the database and not yet saved, and definition files added, changed or missing on disk since the last import; and per annotation task, how many records have unsaved annotations, and the annotation files changed on disk. | +| `set REFERENCE FILE` | Replaces or adds the definition at `REFERENCE` with the text of `FILE`, in the database only. The text is checked on its own and against the rest of the project; an edit with any error is refused and changes nothing. | +| `remove REFERENCE` | Removes the definition in the database only. It is refused while another definition uses it, or while it is the chart `tuldok.json` names in `map.chart`. | +| `revert REFERENCE` | Undoes the unsaved edit or the removal of a definition, in the database only: the file in the folder takes its place again. A definition the folder has no file for is dropped instead (refused while another definition uses it). A file that no longer checks is refused, and a definition with neither an unsaved edit nor a removal has nothing to revert (exit status 1). | +| `branch RECIPE --title TITLE [--name NAME] [--change KIND] [--description TEXT] [--keep]` | Prints a new recipe branched from `RECIPE` (a bare name or `recipes/`), as the operation `draft_branch` writes it: its reference, its problems in the form `check` prints them, and its text. Nothing is stored, unless `--keep` is given and the text has no error: then it is kept as an unsaved edit, as `set` keeps one. With `--keep` and an error, exit status 1. Here `--keep` keeps the branch; `branch` takes no `--keep` or `--discard` for a conflict. | +| `map` | Prints the project's map: per group in order, its title, how many models (recipes) it has and how many of each kind of change, then the models in no group and the totals, then how many lines (`derived_from`, and recipes used as inputs by recipes) and comparisons there are, and the track. Reads the database only, never a results file, values file or note. | +| `save` | Writes every unsaved definition to its file and deletes the files of removed ones, and writes the annotation file of each task with unsaved annotations. It writes nothing when one of those files changed on disk since it was imported, and names them. | +| `forget` | Drops the project's copy from the database, including unsaved edits. The folder is not touched. | + +`note FOLDER PATH` prints one file under the folder's `notes/` directory, +`PATH` being relative to `notes/` (for example `fits/early-fit.md`). Like +`check`, it needs no database: it reads the folder on disk, refusing a path +not in normal form, a link anywhere on the way, a file that is not regular, +larger than 1 MiB or not UTF-8 (exit status 1). + +Every command except `forget` opens the project first, which imports anything +that changed on disk, so `list`, `show`, `graph` and `status` always describe the +folder as it is now plus the unsaved edits. Opening runs the same check as +`check`; a folder with errors is not imported and the database is left as it was. + +A state is worked out from the definitions, never stored in a file: + +| State | When | +| --- | --- | +| `cannot-rebuild` | its `earlier_work.rebuild` is `no`, or something it uses through an input, directly or further up, is `cannot-rebuild` | +| `needs-work` | not `cannot-rebuild`, and its `earlier_work.rebuild` is `needs-work` or something it uses through an input is `needs-work` | +| `not-built` | any other dataset, recipe, benchmark or step | +| `running`, `built`, `out-of-date`, `failed` | a dataset, recipe, benchmark or step, by what was built on this machine (see [Build states](#build-states)) | +| `not-set-up`, `ready`, `failing` | an environment, by what this machine records for it (see [Environments](#environments)) | +| `none` | a model or a chart | + +Options: + +| Option | Commands | Meaning | +| --- | --- | --- | +| `--database-url URL` | all but `check` and `note` | The Postgres database. Without it, the `TULDOK_DATABASE_URL` environment variable is used; with neither, the command stops with exit status 2. | +| `--json` | `open`, `list`, `show`, `graph`, `map`, `status`, `revert`, `branch`, `save`, `results`, `values`, `import-values`, `chart`, `plan`, `environments`, `connect`, `check-environment`, `set-up`, `runs`, `run`, `progress`, `cancel`, `stop`, `products`, `estimate` | Print one JSON document with the same content as the text. | +| `--rebind` | all that open | The copy is bound to the folder it was first opened from. Opening the same project from another path is refused unless `--rebind` is given, which binds the copy to the new path. | +| `--keep` | all that open but `branch` | Where a file changed on disk and the same definition has unsaved edits, keep the edits; the changed file becomes the version that `save` replaces. | +| `--discard` | all that open but `branch` | Where a file changed on disk and the same definition has unsaved edits, drop the edits and take the file. | + +Without `--keep` or `--discard`, such a conflict is refused and names the +definitions concerned. + +An open project holds a lock in the database, so one command at a time works on +a project. Each command releases the lock before it exits. A command started +while the project is open elsewhere is refused. + +The exit status is 0 when the command did its work, 1 when it was refused (the +folder has errors, an edit is not valid, a definition is in use or missing, a +conflict, a file changed on disk, the project is open elsewhere) or the database +failed, and 2 for a usage error, including a missing database setting or a `set` +file that cannot be read. A refusal is printed on standard error as one sentence, +followed by any problems in the same form `check` prints them: + +``` +datasets/waves cannot be set; see the problems. +error datasets/waves.json /title Add the required field 'title'. +``` + +## Results in the database + +When a project is opened, every results file is imported into the database copy +as summary rows, one row per line of the file. Each summary row has an origin: + +| Origin | Where the row comes from | +| --- | --- | +| `file` | the line of `results/.csv` | +| `values` | computed from per-case values imported on this machine (see [Values files](#values-files)), with the statistics of [Results files](#results-files) | + +For one benchmark and subject, when the database holds per-case values, all of +that subject's summary rows are computed from them and have origin `values`; +otherwise they come from the results file. The computed rows are one row with an +empty group over all of the subject's values of a metric, and one row for each +single group key and value (`cells_per_metre=8`) over the values that carry it. +There are no rows for combinations of keys. + +Opening notices a results file that is new, changed or gone, the same way it +notices a changed definition file, and replaces that benchmark's `file` rows +with the file's lines, except for subjects that have values here. + +`save` writes `results/.csv` from the summary rows of each benchmark +whose rows differ from the file, and removes the file of a benchmark that has no +rows left. Like definitions, it writes nothing when a results file changed on +disk since it was imported. `status` lists `results unsaved` (benchmarks whose +rows differ from their file) and `results changed on disk`. Removing a benchmark +removes its summary rows and values; removing a subject from a benchmark removes +that subject's rows. + +Charts and the report read only these rows. A copy of the folder with its +`results/` files draws every chart; no run directory or values file is needed. + +## Annotations in the database + +Annotations are edited in the database copy and written to the folder by +`save`, as definitions are. The copy holds one row per task and record: the +line's fields as they now are, the line as it is in `annotations/.jsonl` +(or nothing, for an annotation made here and not yet saved), and whether the +row is unsaved. An annotation removed here and not yet saved keeps its row, +with no fields, until `save`. + +**Opening** notices an annotation file that is new, changed or gone. Its size +and time are compared with those recorded at the last import or save, and +only a file whose size or time differs is read and its SHA-256 compared; a +file with the same content and a new time is not imported again. A changed +file is read line by line: each record whose line changed takes the file's +line. Where the record also has an unsaved annotation here, the project does +not open: the refusal is `conflict` and names `annotations/.jsonl`, and +opening again with `--keep` (or `resolve_conflict` `keep`) keeps the +annotations here, the file's lines becoming the saved ones, while `--discard` +takes the file's lines. An annotation file that is gone forgets the saved +lines; an annotation with an unsaved change here stays, unsaved. A task +removed from the folder while it has unsaved annotations here is a +conflict too; it can only be discarded. A line holding the character U+0000 +cannot be kept in the database and refuses the open `invalid`. + +**`save`** writes, whole, the file of each task that has an unsaved row, from +all its annotations in the exact form of [The annotations file](#the-annotations-file) +(by a temporary file and a rename), and deletes it when the task has no +annotation left; it also deletes the file of a removed task. It writes nothing +when such a file changed on disk since it was imported, and names it +(`changed-on-disk`). After a save no annotation is unsaved, and each row's +saved line is the line written. + +**`status`** gains `annotations_unsaved`, a list of `{task, records}` (the +task's reference and how many of its records have an unsaved change), and +`annotations_changed`, the paths of the annotation files added, changed or +removed on disk since the last import or save, by size and time. Neither reads +nor writes an annotation file. + +A task with annotations cannot be removed (`in-use`, saying how many); remove +or revert its annotations first. + +**The hash of a record's content** ties an annotation to what it describes: +SHA-256 of the record's file, of an archive member's bytes once decompressed, +or of the UTF-8 of an inline `text`. A record list may give it as `sha256`. +Otherwise Tuldok computes it when the record is first annotated, or first +looked at for an annotation, and keeps it for that run and record; a run's +product never changes, so it is computed once. A record over 1 GiB is not +hashed: its annotations carry null and are tied by id alone. + +**Stale.** An annotation is stale when its dataset's current product has no +record of its id, or has one whose hash differs from the annotation's (an +annotation or a record without a hash is never stale by content). It is +worked out when asked, never stored, and only the records that have +annotations are hashed to find it. When the dataset has no product, nothing +is stale (`stale` is null). Stale annotations are kept, counted and shown; +Tuldok never drops one. + +**Invalid.** An annotation is invalid when its value is not one the task as it +now is allows (its `task` or `vocabulary` changed since it was made). Each row +remembers the task it was last checked against, so counting reads no value +until the task changes. + +## Values files + +A benchmark run writes one values file per benchmark and subject: the measured +value of every case and metric. Values files are not part of the project folder; +they stay on the machine that ran the benchmark, and `import-values` brings them +into the database copy. Format `tuldok_results_v1`, strict JSON as in +[JSON rules](#json-rules), at most 64 MiB: + +| Field | Rule | +| --- | --- | +| `format` | exactly `tuldok_results_v1` | +| `benchmark` | the name of a benchmark of the project | +| `subject` | one of that benchmark's subject names | +| `commit` | 40 lowercase hexadecimal characters, or empty | +| `produced` | a date `YYYY-MM-DD` or a time `YYYY-MM-DDTHH:MM:SSZ` | +| `values` | a list of at least one value | + +Each value: + +| Field | Required | Rule | +| --- | --- | --- | +| `case` | yes | 1 to 200 characters | +| `metric` | yes | a metric id of the benchmark | +| `value` | yes | a finite number, or `null` for a value that was not measured | +| `reason` | when `value` is `null`; otherwise not allowed | 1 to 400 characters: why there is no value | +| `group` | no | an object whose keys are in the benchmark's `summary.group_by`; each value a string, number or boolean, without `;`, `=` or a line break | + +Group values are stored as text: strings as written, `true` and `false`, and +numbers as Python's `repr` writes them (`8` stays `8`, `8.0` stays `8.0`). +(`case`, `metric`, `group`) is unique within a file. Unknown fields are refused. + +Example, `rollout-stepper-k1.json`: + +```json +{ + "format": "tuldok_results_v1", + "benchmark": "rollout", + "subject": "stepper-k1", + "commit": "0123456789abcdef0123456789abcdef01234567", + "produced": "2026-11-10", + "values": [ + {"case": "scene-01", "metric": "rmse", "value": 0.25, "group": {"horizon": 1}}, + {"case": "scene-01", "metric": "rmse", "value": 1.0, "group": {"horizon": 8}}, + {"case": "scene-01", "metric": "seconds", "value": 1.0}, + {"case": "scene-02", "metric": "rmse", "value": 0.5, "group": {"horizon": 1}}, + {"case": "scene-02", "metric": "rmse", "value": null, "reason": "diverged before step 8", "group": {"horizon": 8}}, + {"case": "scene-02", "metric": "seconds", "value": 1.5} + ] +} +``` + +## Reading results + +| Command | Does | +| --- | --- | +| `results [BENCHMARK] [--subject S] [--metric M] [--group G]` | Without a benchmark, one line per benchmark: subjects with results, subjects with values on this machine, summary rows by origin, and whether the rows differ from the results file. With a benchmark, its summary rows; `--group` keeps the rows of one group key, and `--group ''` the rows over all cases. | +| `values BENCHMARK --subject S [--metric M] [--limit N]` | The per-case values held on this machine for one subject, at most `N` (default 200), or a sentence saying there are none. | +| `import-values FILE...` | Imports values files. Every file is checked first; if any has a problem, the problems are printed and nothing is imported. Each file replaces its subject's values and summary rows in one step. | + +`BENCHMARK` is a benchmark's name (`rollout`) or its reference +(`benchmarks/rollout`). Anything else is a usage error. Example, with the values +file above: + +``` +$ .venv/bin/python -m tuldok_project import-values FOLDER rollout-stepper-k1.json +imported benchmarks/rollout stepper-k1 6 values 4 summary rows +$ .venv/bin/python -m tuldok_project results FOLDER benchmarks/rollout --subject stepper-k1 +subject metric group count unavailable mean median p10 p90 min max produced origin +stepper-k1 rmse - 3 1 0.5833 0.5 0.3 0.9 0.25 1 2026-11-10 values +stepper-k1 rmse horizon=1 2 0 0.375 0.375 0.275 0.475 0.25 0.5 2026-11-10 values +stepper-k1 rmse horizon=8 1 1 1 1 1 1 1 1 2026-11-10 values +stepper-k1 seconds - 2 0 1.25 1.25 1.05 1.45 1 1.5 2026-11-10 values +$ .venv/bin/python -m tuldok_project values FOLDER rollout --subject stepper-k1 --metric rmse +case metric group value +scene-01 rmse horizon=1 0.25 +scene-01 rmse horizon=8 1.0 +scene-02 rmse horizon=1 0.5 +scene-02 rmse horizon=8 none (diverged before step 8) +``` + +A refused import prints each problem on standard error in the form `check` +uses and exits with status 1: + +``` +error bad.json /values/0/reason A null value needs a reason of 1 to 400 characters. +Nothing was imported: 1 problem in the values files. +``` + +## Charts and the report + +| Command | Does | +| --- | --- | +| `chart CHART [--svg FILE] [--theme light\|dark]` | Prints the chart's data as text, or as JSON with `--json`. With `--svg`, also writes the drawing to `FILE`. | +| `report --output FILE [--theme light\|dark]` | Writes one HTML page of the project to `FILE`. | + +`CHART` is a chart's name (`rollout-ranking`) or its reference +(`charts/rollout-ranking`). `--theme` picks the light or the dark colours; +the default is light. + +A chart is answered from the summary rows in the database and the definitions, +never from a file. It uses the chart's `statistic` (default `median`) of each +subject's rows for its `metric`, over all cases, or with `group` set one panel +per value of that group key, ordered as numbers when every value is a number and +as text otherwise. The subjects are the chart's `subjects`, or every subject of +the benchmark. A subject with no row, with a row whose `count` is 0, or whose +row lacks the statistic is not drawn; it is listed as missing, and the drawing +says how many are missing and names them. + +"Best" follows the metric's `better`: `lower` and `none` put the lowest value +first, `higher` the highest. + +| Form | What it shows | +| --- | --- | +| `ranking` | One horizontal bar per subject, best at the top, each labelled with the subject's title and its value. Ties are ordered by subject name. The drawing grows with the number of subjects. | +| `tradeoff` | One point per subject at (`metric_x`, `metric`). A subject lacking either value is missing. The frontier is the set of subjects that no other subject is at least as good as on both metrics and better than on one; a step line joins them, and they carry labels where labels fit without overlapping. | +| `progression` | The subjects in the order they were trained: by the `trained_at` label of the subject's recipe, earliest first, then the subjects without one by name. A step line shows the best value so far, and the subjects that set a new best carry labels where they fit. | +| `comparison` | A table of numbers: one row per subject, one column per metric of the benchmark, the best value in each column marked. Rows follow the chart's `subjects`, or the ranking of `metric` when the chart has none. A subject with no value for any metric is missing. | + +An axis is logarithmic when all its values are positive and the largest is at +least 100 times the smallest; otherwise it is linear (bars always start at +zero). Axes carry the metric's title and units, and numbers show three +significant digits, very small and very large ones as `2.43e-6`. A chart with +no values says so in place of the plot. + +The drawing is one SVG document with no scripts and no links, the same bytes +for the same data. Every bar, point and number carries the subject's full title +and its value as a tooltip; long subject titles are cut with an ellipsis. + +Example: + +``` +$ .venv/bin/python -m tuldok_project chart FOLDER rollout-ranking --svg ranking.svg +chart charts/rollout-ranking ranking +title Rollout error by subject +benchmark benchmarks/rollout +metric rmse: median Rollout RMSE (m/s), lower is better +group horizon + +horizon = 1 +rank subject value count +1 stepper-k2 0.3125 4 +2 stepper-k1 0.375 2 +3 early-fit 0.625 4 + +horizon = 8 +rank subject value count +1 stepper-k1 1 1 +2 stepper-k2 1.25 4 + +1 of 3 subjects has no value: early-fit. +Wrote ranking.svg (6091 bytes). +``` + +With `--json`, `chart` prints one object: + +| Field | Meaning | +| --- | --- | +| `chart`, `title`, `description`, `form`, `benchmark`, `statistic`, `group` | from the chart definition; `group` is null without one | +| `metric`, `metric_x` | `{"id", "title", "units", "better"}`; `metric_x` is null except for a `tradeoff` | +| `subject_count` | how many subjects the chart covers | +| `series` | all forms but `comparison`: one entry per group value (one entry with group `""` without a group), each `{"group", "points", "missing"}` | +| `metrics`, `rows` | `comparison` only: every metric of the benchmark, and one row per subject `{"subject", "title", "recipe", "group", "values"}`, `values` mapping each metric id to its number or null | +| `missing` | the names of the subjects not drawn, in the chart's subject order; with a group, those missing in any panel | + +A point is `{"subject", "title", "recipe", "value", "count", "unavailable", +"produced"}`, where `recipe` is the subject's recipe reference or null. A +`tradeoff` point adds `x` and `frontier`; a `progression` point adds `position` +(from 0), `best_so_far` and `trained_at`. + +The report page, from `report`, reads top to bottom: the project's name, folder +and id; how many definitions there are of each kind and in each state, and how +many benchmarks have results; every chart with its title, a one-line note on +what it shows and how many subjects are missing, and its drawing; the results +overview; then the definitions of each kind with their states, each kind in a +closed section that opens on a click. The page is one file with its style +embedded and its charts drawn inline, no scripts and no links, so it can be +sent or opened anywhere. + +``` +$ .venv/bin/python -m tuldok_project report FOLDER --output report.html --theme dark +Wrote report.html (4 charts, 24626 bytes). +``` + +## Planning a build + +A plan says what has to be built on this machine to have one or more +definitions, in what order, and what stands in the way. Making a plan runs +nothing. It reads the database copy only: it starts no command, including a +definition's `estimate` command, and opens no file of the folder; it asks Git +what changed in the code only when something was built (see +[Build states](#build-states)). The figures in a plan are the ones each +definition declares in `run.resources`, and beside them the ones its estimate +command printed when `estimate` last ran it (see [Estimates](#estimates)). + +| Command | Does | +| --- | --- | +| `plan TARGET... [--all]` | Prints the plan for the named definitions. `--all` plans every dataset, recipe, step and benchmark, together with any `TARGET` given. | + +`TARGET` is a reference such as `recipes/stepper-k2`. A plan that cannot be +built is still a plan: the exit status is 0 whether or not it can be built. +A target that is not a definition of the project exits with status 1 and the +message `There is no definition datasets/nothing.`; no target and no `--all` +is a usage error, status 2. + +### Which definitions are in a plan + +Starting from the targets, the plan follows these uses, again and again: + +| Use | Followed | +| --- | --- | +| `input:` | yes | +| `environment` | yes | +| `subject:` | only when the subject's recipe is not blocked | +| `model`, `cases` | no; nothing is built for them | +| `benchmark` | a chart target is replaced by its benchmark | + +A model or a chart is never a step. A model target adds nothing to the plan and +a sentence saying that a model is not built, only the recipes that train it. + +Nothing is followed out of a blocked step: what it would use is not work anyone +can do. Its causes are still found over the whole graph and reported with it, +but they are not steps unless they are targets themselves. A skipped subject's +recipe is named under the benchmark's subjects and is not a step either. So a +plan lists every step that can and must be done, and each blocked target once. + +### Steps and their actions + +The steps are in build order, each after everything it uses (ties broken by +reference), numbered from 1. Each step has one action: + +| Action | For | +| --- | --- | +| `set-up` | an environment in state `not-set-up`, or `failing` (the text says its last check failed) | +| `ready` | an environment in state `ready`; it is not counted as something to set up | +| `build` | a dataset, recipe, step or benchmark in state `not-built`, `out-of-date`, `failed` or `running` (the text says why under the step; `build` refuses a step that is running) | +| `built` | a dataset, recipe, step or benchmark in state `built`; it is not counted as work | +| `blocked` | a definition in state `cannot-rebuild` or `needs-work`, a benchmark with no subject to run, and a `generate` dataset to be built on a machine with no Pumas endpoint set | + +A blocked step is explained by its causes: the definitions whose own +`earlier_work.rebuild` is `no` or `needs-work`, among the step itself and +everything it reaches through inputs, directly or further up. A definition +that is blocked only because of what it uses is not a cause. At most five +causes are listed, nearest first (fewest inputs away, then by reference), with +a count of the rest. + +A benchmark runs once per subject. A subject whose recipe is blocked is +skipped, with the reason; a subject with no recipe is run. The benchmark is +`build` when at least one subject would run. It is never blocked because one of +its subjects is; when no subject would run, it is `blocked` with no cause of +its own. + +A `generate` held by the missing endpoint is its own cause, with `rebuild` +`endpoint` and the reason; it is listed among `blocked_targets` whether it is +a target or not, and the plan cannot be built. The settings file is read for +this only when such a step is to be built. + +The plan can be built when no target is blocked. Blocked steps are listed but +not counted in the totals. + +### Totals + +The totals count `build` steps only: + +- how many steps there are to build, and how many of them declare `gpu: true`; +- the sum of the declared `disk_gb`, and how many build steps declare none; +- the sum of the declared `wall_hours`, and how many build steps declare none; +- when any build step has an estimate, the sum of the estimated bytes and + seconds, and how many build steps each sum covers. Declared and estimated + figures are never added together. + +### Text output + +The text is one opening sentence (how many steps there are to build, to set up +and blocked, how many environments are ready when any is, and whether the +targets can be built), then one line per step with +its number, action, GPU (`yes` for `gpu: true`, otherwise `-`: a step that +does not declare `gpu` needs none), disk and time as declared (`?` when not +declared) and its reference. Under a step are what it uses (each input by name, +with the named output in brackets), the environment it runs in, a benchmark's +subjects, and for a blocked step what blocks it. Then come the causes of each +blocked target with their reasons, and the totals last. + +``` +$ .venv/bin/python -m tuldok_project plan FOLDER recipes/stepper-k2 +4 steps to build, 1 to set up, 0 blocked: recipes/stepper-k2 can be built. + +# action gpu disk time reference +1 set-up - - - environments/lab-cpu +2 build - ? 0.1 h steps/solver + in environments/lab-cpu +3 build - ? ? datasets/waves + solver: steps/solver (binary) + in environments/lab-cpu +4 build - 0.1 GB 0.5 h recipes/stepper-k1 + training: datasets/waves + in environments/lab-cpu +5 build - ? ? recipes/stepper-k2 + start: recipes/stepper-k1 (weights) + training: datasets/waves + in environments/lab-cpu + +To build: 4 steps, 0 on a GPU. +Disk: 0.1 GB declared; 3 of 4 declare none. +Time: 0.6 h declared; 2 of 4 declare none. +Figures are as the definitions declare them; no command was run. +$ .venv/bin/python -m tuldok_project plan FOLDER recipes/early-fit +0 steps to build, 0 to set up, 1 blocked: recipes/early-fit cannot be built. + +# action gpu disk time reference +1 blocked - ? ? recipes/early-fit + blocked: its own label is rebuild no + +recipes/early-fit cannot be built: + no recipes/early-fit + Trained by the earlier study harness; its command and training data were + not recorded. + +Nothing would be built, so there are no totals. +``` + +### JSON output + +With `--json`, `plan` prints one object: + +| Field | Meaning | +| --- | --- | +| `targets` | the references planned, in the order given, with each chart replaced by its benchmark and repeats left out | +| `every` | true with `--all` | +| `buildable` | true when no target is blocked | +| `blocked_targets` | the targets that are blocked | +| `counts` | the number of steps of each action: `{"build", "set-up", "blocked", "ready", "built"}` | +| `gpu_steps` | how many build steps declare `gpu: true` | +| `disk_gb`, `disk_unknown` | the sum of the declared `disk_gb` over build steps, and how many build steps declare none | +| `wall_hours`, `wall_unknown` | the same for `wall_hours` | +| `estimated_disk_gb`, `estimated_disk_steps` | the sum of the estimated `disk_bytes` over build steps, in units of 10^9 bytes, and how many build steps have one | +| `estimated_wall_hours`, `estimated_wall_steps` | the same for the estimated `wall_seconds`, in hours | +| `notes` | sentences about targets that are not built themselves: a model, or a chart replaced by its benchmark | +| `steps` | the steps in order | + +Each step: + +| Field | Meaning | +| --- | --- | +| `number` | its place in the order, from 1 | +| `reference`, `kind`, `title`, `state` | from the definition; the state as `list` shows it | +| `action` | `build`, `set-up`, `blocked`, `ready` or `built` | +| `command`, `environment` | from `run`, placeholders not filled in; null when absent | +| `inputs` | each input name to `{"reference", "output"}`; `output` is null for a whole product | +| `resources` | `gpu`, `cpu_threads`, `memory_gb`, `wall_hours`, `disk_gb` as declared; null when not declared (a null `gpu` means no GPU is needed) | +| `estimate` | the stored estimate, `{"disk_bytes", "wall_seconds", "items", "measured_at"}` with null for a figure the command did not print, while the definition's text is the one it was measured for; otherwise null | +| `reasons` | why a `build` step is out of date, failed or running; empty otherwise | +| `scratch` | true when the step's current product was made by a scratch run | +| `blocked_by` | for a blocked step, up to five causes, each `{"reference", "rebuild", "reason"}`; empty otherwise | +| `blocked_by_more` | how many more causes there are | +| `subjects` | benchmarks only: `run`, the subject names that would run, and `skipped`, each `{"name", "recipe", "reason"}` | + +## Environments + +An `environment` definition says what an interpreter must be: a Python +version, a device and exact package versions. Which interpreter fulfils it is +a fact about one machine, so it is never written to the folder. Tuldok records +it in the project's database schema, in the table `environments`, one row per +environment that has an interpreter on this machine: + +| Column | Content | +| --- | --- | +| `name` | the environment's name, as in `environments/` | +| `interpreter` | the absolute path of the Python executable, exactly as it was given | +| `origin` | `connected` (an existing interpreter was named) or `installed` (Tuldok installed it) | +| `recorded_at` | when the interpreter was recorded | +| `checked_at` | when it was last checked; null before the first check | +| `ok` | whether the last check passed | +| `found` | what the last check saw: the Python version, each package's installed version (null when not installed), the interpreter's own path, and for a `cuda` check what torch says about CUDA | +| `problems` | the sentences saying why the last check failed; empty when it passed | + +Removing an environment definition, or deleting its file, drops its row. The +row never deletes an installed directory. + +Environments that Tuldok installs live in the project's local home, +`//`, where the project directory is the one +[the setting](#the-project-directory-on-this-machine) names. An installed environment is the directory +`/environments//`, with its log beside it as +`/environments/.log`. The local home is made the first time +something is installed. + +### States + +| State | When | +| --- | --- | +| `not-set-up` | no interpreter is recorded for the environment on this machine | +| `ready` | its last check passed | +| `failing` | its last check did not pass | + +The state comes from the stored result of the last check. Opening a project +never runs a check, so `list`, `show`, `graph` and `plan` show what the last +check found, however long ago it ran; `check-environment` runs it again. No +other definition's state depends on an environment's. A change to an +environment definition does not change the stored result until the next check. + +### The check + +The check runs the recorded interpreter once, as a child process: + +``` +INTERPRETER -I -c PROBE '{"device": ..., "packages": {...}}' +``` + +The child gets an environment of `PATH=/usr/bin:/bin`, `LANG=C.UTF-8` and +`HOME` set to a new empty temporary directory, which is also its working +directory and is removed afterwards. It has 180 seconds. Its output is +captured and never shown as it is. No project code is involved. + +The probe reads each package's installed version with `importlib.metadata` +and imports no package, with one exception: when the device is `cuda` and +`torch` is among the packages, it imports `torch` and reports whether CUDA is +available, the CUDA version, the first device's name and the first line of +`/proc/driver/nvidia/version` when that file exists. It works on Python 3.10 +to 3.14 and prints one JSON object. + +The check passes when all of these hold: + +| Rule | A failure reads | +| --- | --- | +| the Python version's major.minor is the definition's `python` | `Python is 3.11.9; the definition asks for 3.12.` | +| every package is installed | `numpy is not installed; the definition asks for 2.3.5.` | +| every package's version is exactly the definition's | `jsonschema is 4.26.0; the definition asks for 4.25.0.` | +| for device `cuda`, `torch` is among the packages | `The device is cuda but torch is not among the packages, so the check cannot see the GPU.` | +| for device `cuda`, torch reports CUDA as available | `CUDA is not available to torch.` | + +A probe that cannot be started, takes longer than 180 seconds, exits with an +error, or prints anything other than the expected object is a failed check +with one problem saying which, followed by the first 400 characters of its +error output. A failed check is recorded like any other result. + +### Connect + +`connect` records an existing interpreter, with origin `connected`, and then +checks it. The interpreter is named by an absolute path to an executable +file. A symbolic link is allowed, and the path is stored as given, not +resolved, so a virtual environment's `bin/python` stays that environment's. +The record is kept whether or not the check passes, so its problems can be +read with `environments`. + +### Set-up + +`set-up` installs the environment its definition describes, then records it +with origin `installed` and checks it. It needs the definition's +`install.requirements`; without it, the refusal says to connect an existing +interpreter instead. The installer is `uv`: the program named by the +`TULDOK_UV` environment variable, otherwise `uv` on `PATH`; with neither, it is +refused. + +These two commands run in order, with the code root as the working directory +and up to one hour each: + +``` +uv venv --python PYTHON TARGET +uv pip install --python TARGET/bin/python -r CODE_ROOT/REQUIREMENTS [--extra-index-url URL]... +``` + +`TARGET` is `/environments/`, `PYTHON` the definition's +`python`, `REQUIREMENTS` its `install.requirements`, and there is one +`--extra-index-url` for each of its `install.index_urls`. The output of both +goes to `TARGET.log`, which is replaced each time. + +| Option | Meaning | +| --- | --- | +| `--dry-run` | Prints each command on its own line as it would be run, then the target and the log path. Nothing is made and nothing is run. | +| `--replace` | A target that exists and is not empty is refused without this option; with it, the target is removed first. | + +When a command fails or runs out of time, the half-made target is removed, +the log is kept, nothing is recorded, and the refusal ends with the last lines +of the log. Any record the environment had before is left as it was; when +that record was the installed target that `--replace` removed, its next check +fails. + +### Disconnect + +`disconnect` removes the record, so the environment is `not-set-up` again. It +never deletes an installed directory; when the origin was `installed` it +prints that directory's path, to delete by hand when it is no longer needed. + +### Commands + +| Command | Does | +| --- | --- | +| `environments` | Prints one row per environment: reference, state, origin, when it was last checked and the interpreter; under it the versions the last check found (with what the definition asks for where they differ) and its problems, or how to set it up. A closing line says how many are ready. | +| `connect ENVIRONMENT INTERPRETER` | Records and checks an existing interpreter. Exit status 1 when the check fails. | +| `check-environment [ENVIRONMENT...]` | Checks the named environments again, or every recorded one when none is named. Exit status 1 when any fails, or when a named one is not set up. | +| `set-up ENVIRONMENT [--dry-run] [--replace]` | Installs, records and checks the environment, or with `--dry-run` prints what it would run. Exit status 1 when it is refused, a command fails, or the check fails. | +| `disconnect ENVIRONMENT` | Removes the record. | + +An environment may be named bare (`lab-cpu`) or as a reference +(`environments/lab-cpu`). `--json` prints one JSON document for +`environments`, `connect`, `check-environment` and `set-up`; each environment +in it has the definition's `python`, `device` and `packages`, whether it has +an install section (`installable`), its `state`, and the record's columns +(null when there is no record). + +In these examples `FOLDER` is a copy of the test project whose `lab-cpu` +asks for `jsonschema` and is connected to Tuldok's own `.venv/bin/python`. +Long paths are shortened: `VENV` is that `.venv`, `CODE` the code root, +`LOCAL` the project's local home and `UV` the `uv` found on `PATH`. + +``` +$ .venv/bin/python -m tuldok_project environments FOLDER +environment state origin last check interpreter +environments/lab-cpu not-set-up - - - + not set up on this machine; run set-up or connect an interpreter + +0 of 1 environment ready on this machine. +$ .venv/bin/python -m tuldok_project set-up FOLDER lab-cpu --dry-run +Would set up environments/lab-cpu by running, in CODE: +UV venv --python 3.12 LOCAL/environments/lab-cpu +UV pip install --python LOCAL/environments/lab-cpu/bin/python -r CODE/requirements.txt --extra-index-url https://pypi.org/simple +target LOCAL/environments/lab-cpu +log LOCAL/environments/lab-cpu.log +Nothing was run. +$ .venv/bin/python -m tuldok_project connect FOLDER lab-cpu VENV/bin/python +Recorded VENV/bin/python for environments/lab-cpu. +environments/lab-cpu is ready: the check passed. + python 3.12.3 (asks 3.12) + jsonschema 4.26.0 +$ .venv/bin/python -m tuldok_project environments FOLDER +environment state origin last check interpreter +environments/lab-cpu ready connected 2026-11-12 00:31 VENV/bin/python + python 3.12.3 (asks 3.12) + jsonschema 4.26.0 + +1 of 1 environment ready on this machine. +``` + +A path with spaces or other characters the shell would split is quoted in +`--dry-run`'s commands, so a line can be pasted into a shell as it is. When +the check fails, `connect` exits with status 1 and the plan sets the +environment up again: + +``` +$ .venv/bin/python -m tuldok_project connect FOLDER lab-cpu VENV/bin/python +Recorded VENV/bin/python for environments/lab-cpu. +environments/lab-cpu is failing: the check found 1 problem. + python 3.12.3 (asks 3.12) + jsonschema 4.26.0 (asks 4.25.0) + problem: jsonschema is 4.26.0; the definition asks for 4.25.0. +$ .venv/bin/python -m tuldok_project plan FOLDER steps/solver +1 step to build, 1 to set up, 0 blocked: steps/solver can be built. + +# action gpu disk time reference +1 set-up - - - environments/lab-cpu + its last check failed; see environments +2 build - ? 0.1 h steps/solver + in environments/lab-cpu +... +``` + +With the environment ready, the same plan begins +`1 step to build, 0 to set up, 0 blocked, 1 ready: steps/solver can be built.` +and the environment's line has the action `ready`. + +## Building + +`build` runs the commands of a plan on this machine and records what they +made. What a command writes is a **product**; it stays on this machine, in the +project's local home, and is never written to the folder. Each command that is +started is a **run**, recorded in the project's database schema with its +state, its command line, the code and inputs it used and how it ended. + +### What a build does + +1. It makes the plan for its targets, as `plan` does, and prints it. +2. It refuses the plan, before asking anything, when a target is blocked or + when another build is running one of its steps. +3. When something is to be built, it prepares the code the runs will see (see + below). A tree with uncommitted changes is refused here. +4. It asks `Build this? [y/N]`, unless `--yes` is given or nothing is to be + built. Any answer but `y` or `yes` builds nothing. +5. It records the plan, then takes the steps in order: + - an environment that a step to build runs in is checked again, once per + build; one that is not ready stops the build, with a message saying to + set it up or connect an interpreter; + - a step whose action is `built` is skipped; + - a step whose action is `build` is run, once for each subject of a + benchmark that would run and is not built already. +6. For each run, it fills the placeholders, records the run with state + `running`, and runs the command. The project is closed while the command + runs, so `list`, `plan`, `runs`, `run`, `progress` and the other commands + work during a build; `build` waits up to a minute when another command has + the project open. +7. When the command ends with exit status 0, every declared output must exist + under the output directory, and a benchmark's results are imported. Then + the output directory becomes the definition's current product (one per + subject for a benchmark). +8. A run that does not succeed stops the build. Later steps are not run. + +A step whose command cannot even be started (a program that is not found, an +input with no current product, a settings file that cannot be read) is +recorded as a `failed` run with the reason, and its run directory is not made. + +### The code a run sees + +The code root must be inside a Git work tree. A run uses the commit that +`HEAD` names, and the tree is **clean** when no tracked file under the code +paths (`code_paths` in `tuldok.json`, or the whole code root) differs from that +commit, staged or not. Untracked files do not count. A tree that is not clean +is refused, naming up to ten changed files. + +For a clean tree, the run sees a read-only copy of the commit, the +**snapshot**, made once with `git archive` and shared by every run at that +commit: + +| Code | Snapshot directory | +| --- | --- | +| the whole code root | `/code//` | +| `code_paths` | `/code/-/`, where the digest is the first 12 hexadecimal characters of the SHA-256 of the sorted paths, one per line | + +The snapshot keeps the repository's layout, so the command's working +directory is the code root's place inside it. A settings file named by +`settings_file` is read from the snapshot too. + +A **scratch** run, asked for with `--scratch`, uses the code root in place, +uncommitted changes included, and is allowed outside Git. It records the +commit when there is one. Its product is marked as a scratch product and never +becomes out of date because of the code. + +### The run directory + +Each run has its own directory, `/runs//`, where the run id +is 32 hexadecimal characters: + +| Entry | Content | +| --- | --- | +| `output/` | what `{output}` names, made empty before the command starts; a succeeded run's `output/` is its product | +| `settings.json` | the inline `settings` as the run saw them, when the definition has them | +| `tasks.json`, `annotations/` | a freeze's tasks and the line files it read | +| `endpoint.json` | a generate's or an embed's endpoint, `{address, version}`, as it found it; kept in the run's record as `endpoint` | +| `embed.json` | what an embed is told: its input's reference and run, the machine's embedder and prefix, and the output of the last run when it failed | +| `stdout.log`, `stderr.log` | the command's standard output and error | +| `cancel` | present when cancellation was asked for | +| `outcome.json` | how the command ended, written when every process of the run is gone | + +A product stays where its run wrote it. A newer product of the same definition +(and subject) becomes the current one and the earlier one is kept, no longer +current. Tuldok deletes no run directory and no product. Removing a definition +drops its runs, products and estimate from the database and leaves the +directories on disk. + +### The command and its environment + +The command is `run.command` with its placeholders filled. When its first word +is `python`, it is the interpreter recorded for the run's environment. A first +word containing `/` is an executable file under the code root. Any other first +word is looked up on `PATH`. + +The command's environment holds exactly these variables, later ones replacing +earlier ones: + +1. `PATH`, `HOME`, `USER`, `LOGNAME`, `LANG`, `LC_ALL` and `TZ`, from `build`'s + own environment where they are set; +2. `PYTHONDONTWRITEBYTECODE=1` and `PYTHONUNBUFFERED=1`; +3. `CUDA_VISIBLE_DEVICES` set to the empty string, unless the step declares + `gpu: true`, in which case it is passed on as `build` has it (and left + unset when `build` has none); +4. the definition's `env`, with its placeholders filled; +5. every `--env NAME=VALUE` given to `build`; +6. `TULDOK_RUN_ID`, the run id. Every process the command starts inherits it, + which is how Tuldok finds anything a run left behind. + +The working directory is the code root in the snapshot (or the code root +itself for a scratch run). Standard input is `/dev/null`; standard output and +error are appended to the two logs. + +A [step Tuldok provides](#steps-tuldok-provides) needs no environment +definition. Its command is Tuldok's own interpreter running `-m +tuldok_project.steps /settings.json /output`, followed by `--code ` for `import`, and by +`--records --tasks /tasks.json` for +`freeze`, which also finds the line files it was given in `/annotations/`. Its environment is the one above without `env`, +with `PYTHONSAFEPATH=1` and `PYTHONPATH` set to Tuldok's own directory last, +so that Tuldok's code is imported and the code root's never is. A step opens +no database; it prints progress lines like any command. + +### Placeholders at run time + +| Placeholder | Filled with | +| --- | --- | +| `{output}` | `/output` | +| `{settings}` | `/settings.json` for inline `settings`; the settings file's path in the snapshot for `settings_file` | +| `{inputs.}` | the input's current product directory; for `{"from", "output"}`, the path of that output inside it | +| `{subject.name}` | the subject's name | +| `{subject}` | the subject's `argument`, with `{recipe}` replaced by the current product directory of the subject's recipe; the subject's name when it has no `argument` | + +An input or a subject's recipe with no current product fails the run before it +starts; the plan puts what it needs earlier, so this happens only when that +earlier step was not built. + +### Limits + +| Limit | From | Default | +| --- | --- | --- | +| wall time | `resources.wall_hours` | 24 hours | +| disk | `resources.disk_gb`, in units of 10^9 bytes, for the whole run directory | 50 GB | + +The wall time is looked at every half second and the size of the run +directory every five seconds, and once more when the command has ended. A +run past either limit, a run whose `cancel` file appears, and a run whose +`build` receives `SIGINT` (Ctrl-C) or `SIGTERM` is stopped: `SIGTERM` to every +process of the run, up to ten seconds of grace, then `SIGKILL`. After the +command has ended, however it ended, every process of this user that still +carries the run's `TULDOK_RUN_ID` is killed; such a process does not change a +zero exit status into a failure, but the count is recorded as the run's +strays. + +### The GPU lock + +A step that declares `gpu: true` holds an exclusive lock on +`/gpu.lock` while its command runs. The lock is shared by +every project on this machine, so two GPU steps never run at the same time; a +build that finds the lock taken waits for it. Steps without `gpu: true` take no +lock. + +### Results published by a benchmark + +A benchmark's output with role `results` must be a values file (see +[Values files](#values-files)) for that benchmark and the subject that was +run. After the command succeeds, the file is checked and imported, replacing +that subject's values and summary rows in the database. A file that fails the +check makes the run `failed`; its problem names the first five problems, and +nothing is imported. `save` writes the new summaries to `results/` like any +others. + +### Run states + +| State | Meaning | +| --- | --- | +| `queued` | recorded and not started yet | +| `running` | its command is running | +| `succeeded` | exit status 0, every declared output written, and a benchmark's results imported | +| `failed` | a non-zero exit status, a missing output, a refused results file, or a command that could not be started | +| `timed-out` | stopped at its wall-time limit | +| `over-disk` | stopped at, or ended over, its disk limit | +| `cancelled` | stopped because cancellation was asked for, with `cancel` or by a signal to `build` | +| `unknown` | how it ended is not known: its build is gone, or a process of the run could not be stopped | +| `adopted` | nothing was run: `adopt` made files made elsewhere its product (see [Adopting existing files](#adopting-existing-files)) | + +### Build states + +What was built gives datasets, recipes, steps and benchmarks four more states, +in the place of `not-built`: + +| State | When | +| --- | --- | +| `running` | it has a `queued` or `running` run | +| `built` | it has a current product and nothing makes it out of date | +| `out-of-date` | it has a current product, and one of the causes below holds | +| `failed` | it has no current product, and its latest run ended `failed`, `timed-out`, `over-disk` or `unknown` | + +A current product is **out of date** when: + +- the definition's text differs from the text its run used; +- its `settings_file` reads differently now from what the run used; +- an input, or a benchmark subject's recipe, has a different current product + than the one the run used, or none, or is itself out of date; +- the run was not a scratch run, and one of the definition's `sources` (or a + file under one) changed since the run's commit: in a later commit or as an + uncommitted change to a tracked file. When Git no longer has that commit, + the code is taken to have changed. An `import` from a relative `source` is + out of date the same way when a file under its source changed; +- for a `freeze`, the line file of one of its tasks differs from the one its + run read: in the commit `HEAD` names (the commit a build would use; an + uncommitted change does not count), or in the working tree for a scratch + product. The cause names the task and the file (`the annotations of + annotations/scene-quality (annotations/scene-quality.jsonl) changed since + run …`, or `are gone`). A change to the file of a task it does not read + changes nothing. + +A benchmark has a product per subject. It is `built` when every subject that +would run has a current product that is not out of date, and `out-of-date` +when one of them is out of date or only some have a product. When none has a +product it is `not-built`, or `failed` when its latest run ended badly. A product built by a scratch run is `built`, and +`plan` notes that it is a scratch product. A definition with a current product +is `built` or `out-of-date` even when its label would make it +`cannot-rebuild` or `needs-work`; those take precedence only when there is no +current product. A `cancelled` run never makes a definition `failed`, and a +definition with a current product keeps it. A definition whose current +product was adopted is `built` and never out of date: none of the causes above +is looked at. + +In a plan, `built` is the action `built` and is not counted as work; +`out-of-date`, `failed` and `running` are `build`, and the plan says why under +the step. A step another build is running is refused by `build`. + +### When a builder dies + +A run is watched by a small custodian process that `build` starts for each +command. When `build` dies, even by `SIGKILL`, the custodian notices, stops +the run and kills every process that carries its run id. Its row in the +database still says `running`. The next time builds are looked at (`list`, +`plan`, `runs`, `run` and the others), a `queued` or `running` run whose build +process is not running on this boot becomes `unknown`, with a problem saying +so. It is never restarted; build again to make the product. + +`stop` is for a run whose build is gone: it kills every process of this user +that carries the run's id, whatever is left after the custodian, and marks the +run `unknown` when it was still `queued` or `running`. It is refused while the +run's build is alive; use `cancel` for that. + +### Commands + +| Command | Does | +| --- | --- | +| `build TARGET... [--all] [--scratch] [--yes] [--env NAME=VALUE]...` | Prints the plan, asks to proceed unless `--yes`, and runs it. Prints a line when each run starts and ends, with the run id, and a closing line. | +| `runs [REFERENCE]` | The runs, newest first: run id, reference, subject, state, when it started and how long it took. | +| `run RUN` | One run: its row (state, plan, code, environment, command, directory, build process, times, exit status, seconds, size, strays), what it used, its product and outputs when it made one, its last progress event and its problem. | +| `progress RUN [--follow]` | The run's state, its last progress event and the last 8 KiB of each log. `--follow` looks again every two seconds and prints what is new, until the run is no longer `queued` or `running`. | +| `cancel RUN` | Asks a `queued` or `running` run to stop by creating its `cancel` file. The build running it stops it within a second or so and records it as `cancelled`; `cancel` does not wait. | +| `stop RUN` | For a run whose build is gone: kills the processes it left and marks it `unknown`. | +| `products [REFERENCE]` | The current products: reference, subject, run id, size of the declared outputs, whether a scratch run made it, and the directory. | +| `estimate TARGET... [--all] [--scratch] [--env NAME=VALUE]...` | Runs the estimate commands of the steps the plan would build and stores what they print (see [Estimates](#estimates)). | + +`RUN` is a run id or a prefix of at least six of its characters that no other +run's id starts with. A prefix that matches several runs is refused, and the +matching runs are listed. `run`, `progress`, `cancel` and `stop` open the +project only for a moment to find the run, waiting while a build has it open, +and read the logs with the project closed, so they work while another process +builds. `progress --follow` does not hold the project open between passes. + +Options: + +| Option | Commands | Meaning | +| --- | --- | --- | +| `--all` | `build`, `estimate` | every dataset, recipe, step and benchmark, together with any `TARGET` given | +| `--scratch` | `build`, `estimate` | run from the code root as it is, uncommitted changes included | +| `--yes` | `build` | do not ask for confirmation | +| `--env NAME=VALUE` | `build`, `estimate` | add a variable to every command's environment; may be repeated | +| `--follow` | `progress` | repeat until the run ends | +| `--json` | `runs`, `run`, `progress`, `cancel`, `stop`, `products`, `estimate` | print one JSON document; `progress --follow --json` prints one JSON object per line, one per pass | + +Exit statuses: + +| Command | 0 | 1 | +| --- | --- | --- | +| `build` | every step succeeded or was built already | refused (a blocked target, a step being built elsewhere, uncommitted changes, an environment that is not ready), not confirmed, or a run did not succeed | +| `run`, `progress`, `runs`, `products` | shown | no such run, or a prefix that matches several | +| `cancel` | the `cancel` file was made, or was there already | the run is not `queued` or `running`, or no such run | +| `stop` | nothing of the run is left running | its build is alive, a process could not be stopped, or no such run | +| `estimate` | every estimate that ran was measured | an estimate failed, or the code was refused | + +Every command exits with status 2 on a usage error, such as `build` or +`estimate` without a target or an `--env` that is not `NAME=VALUE`. + +In these examples `FOLDER` is a committed copy of Tuldok's test project +`tests/fixtures/tuldok_project/buildable`, whose environment is connected to +Tuldok's own `.venv/bin/python`. `LOCAL` is the project's local home and +`VENV` that `.venv`. + +``` +$ .venv/bin/python -m tuldok_project build FOLDER benchmarks/score --yes +3 steps to build, 0 to set up, 0 blocked, 1 ready: benchmarks/score can be +built. + +# action gpu disk time reference +1 ready - - - environments/test-python +2 build - 0.01 GB 0.05 h datasets/numbers + in environments/test-python +3 build - ? ? recipes/fit-a + data: datasets/numbers (data) + in environments/test-python +4 build - ? ? benchmarks/score + data: datasets/numbers (data) + in environments/test-python + runs 2 subjects + +To build: 3 steps, 0 on a GPU. +Disk: 0.01 GB declared; 2 of 3 declare none. +Time: 0.05 h declared; 2 of 3 declare none. +Figures are as the definitions declare them; no command was run. +start datasets/numbers run 8064e90ac31441bfb72ae1a892fde0b8 +end datasets/numbers run 8064e90ac31441bfb72ae1a892fde0b8 succeeded 0.0 s +start recipes/fit-a run a14415f556a645d8aa209af956504b82 +end recipes/fit-a run a14415f556a645d8aa209af956504b82 succeeded 0.0 s +start benchmarks/score fit-a run aeef824dfaad4c9eb0d95bc684738444 +end benchmarks/score fit-a run aeef824dfaad4c9eb0d95bc684738444 succeeded 0.0 s +start benchmarks/score zero run 9916ff56c61e471b840157155b4bdf83 +end benchmarks/score zero run 9916ff56c61e471b840157155b4bdf83 succeeded 0.0 s +Done: 4 runs succeeded, 0 already built. +$ .venv/bin/python -m tuldok_project runs FOLDER +run reference subject state started seconds +9916ff56c61e471b840157155b4bdf83 benchmarks/score zero succeeded 2026-11-12 01:26 0.0 s +aeef824dfaad4c9eb0d95bc684738444 benchmarks/score fit-a succeeded 2026-11-12 01:26 0.0 s +a14415f556a645d8aa209af956504b82 recipes/fit-a - succeeded 2026-11-12 01:26 0.0 s +8064e90ac31441bfb72ae1a892fde0b8 datasets/numbers - succeeded 2026-11-12 01:26 0.0 s +$ .venv/bin/python -m tuldok_project products FOLDER +reference subject run size scratch directory +benchmarks/score fit-a aeef824dfaad4c9eb0d95bc684738444 955 B - LOCAL/runs/aeef824dfaad4c9eb0d95bc684738444/output +benchmarks/score zero 9916ff56c61e471b840157155b4bdf83 910 B - LOCAL/runs/9916ff56c61e471b840157155b4bdf83/output +datasets/numbers - 8064e90ac31441bfb72ae1a892fde0b8 74 B - LOCAL/runs/8064e90ac31441bfb72ae1a892fde0b8/output +recipes/fit-a - a14415f556a645d8aa209af956504b82 66 B - LOCAL/runs/a14415f556a645d8aa209af956504b82/output +$ .venv/bin/python -m tuldok_project run FOLDER a14415 +run a14415f556a645d8aa209af956504b82 +reference recipes/fit-a +subject - +state succeeded +plan e7bf709a3f2f44b9a743401873dc34ae +code commit 51eb0263cdd0, from its snapshot +environment environments/test-python VENV/bin/python +command VENV/bin/python train.py --data LOCAL/runs/8064e90ac31441bfb72ae1a892fde0b8/output/numbers.json --settings LOCAL/code/51eb0263cdd0954fd47e700e718043887939d6c7/code/settings/fit-a.json --out LOCAL/runs/a14415f556a645d8aa209af956504b82/output +directory LOCAL/runs/a14415f556a645d8aa209af956504b82 +builder process 1362748 +queued 2026-11-12 01:26:54 +started 2026-11-12 01:26:54 +finished 2026-11-12 01:26:54 +exit status 0 +seconds 0.0 s +size 532 B +strays 0 +input data datasets/numbers (data), run 8064e90ac31441bfb72ae1a892fde0b8 +product LOCAL/runs/a14415f556a645d8aa209af956504b82/output +output weights weights.json weights 66 B +progress fit 4 of 4 (100%) guess=-0.1727 +$ .venv/bin/python -m tuldok_project progress FOLDER a14415 +run a14415f556a645d8aa209af956504b82 +reference recipes/fit-a +state succeeded +progress fit 4 of 4 (100%) guess=-0.1727 +stdout.log: 466 B: + {"format": "tuldok_progress_v1", "stage": "fit", "completed": 1, "total": 4, "metrics": {"guess": -0.09211366666666666}} + {"format": "tuldok_progress_v1", "stage": "fit", "completed": 2, "total": 4, "metrics": {"guess": -0.1381705}} + {"format": "tuldok_progress_v1", "stage": "fit", "completed": 3, "total": 4, "metrics": {"guess": -0.16119891666666666}} + {"format": "tuldok_progress_v1", "stage": "fit", "completed": 4, "total": 4, "metrics": {"guess": -0.172713125}} +stderr.log: empty +``` + +The command line in `run` is quoted so it can be pasted into a shell. The +progress event is the last complete line of standard output, within the last +8 KiB, that is exactly one JSON object of this form, at most 4,096 bytes: + +```json +{"format": "tuldok_progress_v1", "stage": "fit", "completed": 3, "total": 4, "metrics": {"guess": -0.16}} +``` + +`stage` and each metric name match `[A-Za-z0-9][A-Za-z0-9_.-]{0,63}`; +`completed` and `total` are whole numbers with `0 <= completed <= total <= +1000000`; `metrics` holds up to eight finite numbers. Any other line is just +output. Invalid bytes in a log are shown replaced, and control characters +other than tab and newline are left out. +`run --json` prints every column of the run's row, including the definition +text and the environment it ran with, plus `product`, `builder_alive` and +`event`. + +While a build runs in another terminal, `progress --follow` prints the run's +first line, then each new line of the logs as it is written (`out` for +standard output, `err` for standard error), a `progress` line when the event +changes, and a last line when the run ends: + +``` +$ .venv/bin/python -m tuldok_project progress FOLDER 3b4a4509 --follow +run 3b4a45097c0f44b3a651763d599289aa recipes/fit-a running +out {"format": "tuldok_progress_v1", "stage": "fit", "completed": 1, "total": 4, "metrics": {"guess": -0.09211366666666666}} +out {"format": "tuldok_progress_v1", "stage": "fit", "completed": 2, "total": 4, "metrics": {"guess": -0.1381705}} +out {"format": "tuldok_progress_v1", "stage": "fit", "completed": 3, "total": 4, "metrics": {"guess": -0.16119891666666666}} +out {"format": "tuldok_progress_v1", "stage": "fit", "completed": 4, "total": 4, "metrics": {"guess": -0.172713125}} +progress fit 4 of 4 (100%) guess=-0.1727 +Run 3b4a45097c0f44b3a651763d599289aa ended succeeded after 4.3 s. +$ .venv/bin/python -m tuldok_project stop FOLDER 3b4a4509 +Run 3b4a45097c0f44b3a651763d599289aa (recipes/fit-a) is still being run by its build (process 1365836); use cancel to stop it. +``` + +(`stop` above was run while the build was still going.) Cancelling a running +build, and what the build prints: + +``` +$ .venv/bin/python -m tuldok_project cancel FOLDER 4cc81737 +Asked run 4cc81737b8b04686942a24d60e39c9bc (recipes/fit-a) to stop: its build stops it within a second or so and records it as cancelled. +``` + +``` +start recipes/fit-a run 4cc81737b8b04686942a24d60e39c9bc +end recipes/fit-a run 4cc81737b8b04686942a24d60e39c9bc cancelled 1.1 s + It was cancelled. +Stopped: recipes/fit-a did not succeed; later steps were not run. +``` + +``` +$ .venv/bin/python -m tuldok_project cancel FOLDER 4cc81737 +Run 4cc81737b8b04686942a24d60e39c9bc (recipes/fit-a) is cancelled; only a queued or running run can be cancelled. +``` + +After the build running `48856079` was killed with `SIGKILL`: + +``` +$ .venv/bin/python -m tuldok_project stop FOLDER 48856079 +No process of the run was left running: run 48856079a1694a8391a8083fafe7fbb1 (recipes/fit-a). +It was running; it is now unknown, and is never restarted. +``` + +### Estimates + +A definition's `run.estimate.command` says how big and how long its build will +be without doing it. `estimate` runs it; `plan` and `build` never do, because +it is project code. For each step the plan would build: + +- a step without an estimate command is skipped; +- the estimate command runs from the same code as a build (a clean commit's + snapshot, or the code root with `--scratch`), with the child environment and + working directory a build step gets, in a throwaway directory + `/estimates//` that is removed afterwards; +- `{output}` names a path that does not exist, `{settings}` and + `{inputs.}` are filled as for a build, and `{subject}` and + `{subject.name}` cannot be used, since an estimate is made once per + definition; +- a step whose estimate command or `env` names an input with no current + product is skipped, with that reason; +- the command has five minutes and may write 100 MB (10^8 bytes) in its + directory; it takes no GPU lock. + +The command's standard output must be exactly one JSON object, at most 64 KiB, +with any of these fields; other fields are ignored: + +| Field | Meaning | +| --- | --- | +| `disk_bytes` | how many bytes the build will write | +| `wall_seconds` | how many seconds the build will take | +| `items` | how many items it will make or read, for example cases or examples | + +Each one present must be a number of at least 0, and at least one must be +present. A command that fails, runs out of time, prints anything else or a +value of the wrong kind is a failed estimate for that step, reported with the +reason; the other steps still run, and an earlier estimate of that step is +kept. A measured estimate replaces the step's earlier one. + +An estimate is stored in the database with the SHA-256 of the definition's +text it was measured for, and `plan` shows it only while the definition's text +is that one. In the text of a plan, the declared figure comes first and the +estimate follows it marked `~... est.`; the totals keep the declared sums and +add a line with the estimated sums, saying how many steps they cover. + +``` +$ .venv/bin/python -m tuldok_project estimate FOLDER recipes/fit-a +Ran from the code at commit 51eb0263cdd0. + +step estimate disk time items +datasets/numbers skipped - - - + It has no estimate command. +recipes/fit-a skipped - - - + It needs {inputs.data}, but its input data (datasets/numbers) has no current + product. + +0 measured, 2 skipped, 0 failed. plan shows each estimate while its definition is unchanged. +``` + +Once `datasets/numbers` is built and `recipes/fit-a` declares +`wall_hours: 0.1`: + +``` +$ .venv/bin/python -m tuldok_project estimate FOLDER recipes/fit-a +Ran from the code at commit 060271be67fe. + +step estimate disk time items +recipes/fit-a measured 4.8 KB 2 s 6 + +1 measured, 0 skipped, 0 failed. plan shows each estimate while its definition is unchanged. +$ .venv/bin/python -m tuldok_project plan FOLDER benchmarks/score +2 steps to build, 0 to set up, 0 blocked, 1 ready, 1 built: benchmarks/score +can be built. + +# action gpu disk time reference +1 ready - - - environments/test-python +2 built - 0.01 GB 0.05 h datasets/numbers + in environments/test-python +3 build - ? ~4.8 KB est. 0.1 h ~2 s est. recipes/fit-a + data: datasets/numbers (data) + in environments/test-python + out of date: its definition changed since run a14415f556a6 +4 build - ? ? benchmarks/score + data: datasets/numbers (data) + in environments/test-python + runs 2 subjects + out of date: subject fit-a: the recipe recipes/fit-a of its subject is out of date + +To build: 2 steps, 0 on a GPU. +Disk: no step declares it. +Time: 0.1 h declared; 1 of 2 declares none. +Estimated: disk ~4.8 KB (1 of 2 steps); time ~2 s (1 of 2 steps). +Declared figures are as the definitions declare them; figures marked est. are +what their estimate commands printed when estimate last ran them. No command +was run now. +``` + +A measured time reads in seconds below 100 s, in minutes below 100 minutes, +and in hours above that. With `--json`, `estimate` prints `{"targets", +"scratch", "commit", "steps"}`, each step `{"reference", "result"` +(`measured`, `skipped` or `failed`), `"reason", "values", "seconds"}`. + +## Serving a project + +The Tuldok app can serve one project folder to its owner and to agents: + +``` +python app.py --data CATALOG --project-folder FOLDER +``` + +`--data` names the catalog directory, which holds the credentials; the +database is the one `TULDOK_DATABASE_URL` names. One app serves one folder and +nothing else: `--project-folder` cannot be combined with `--project-root`. +A folder that does not open stops start-up with the message and its first problems, exit +status 2. The app opens the project only for each operation, as the commands +do, so files changed on disk are imported before an operation answers and +the command line keeps working beside the app. + +The page at `/` is the Map workspace: the tabs Map, Data, Annotate, Train, +Benchmarks, Analysis and Overview over the project, read through the operations +below. Annotate shows a task's records, one in its viewer, and the task's form, +which keeps an annotation with `set_annotation`; Save writes the unsaved +annotations with the unsaved definitions. Its Suggest asks `suggest_annotation` +of a model `suggestion_models` offers and fills the form, and a kept +suggestion passes its `from`. Train shows the machine's Pumas endpoint +(`endpoint`; for the owner `set_endpoint` and `find_endpoints`, and the +machine's embedder by `set_embedder`), and Data's +Generate form keeps a dataset whose run is the step `generate`. A dataset +opened on Data shows a Map beside its records and files when `embeddings` +lists a set for it (`embedding_map` and the four searches); a region chosen on +the map is kept as a dataset whose run is the step `select`, or fills the +Freeze form's `records`; Data's Embed form keeps a dataset whose run is the +step `embed`. +Each of its controls calls one operation and is drawn only for a session that +may call it, so a credential that may only inspect sees a page that changes +nothing. The report is linked from the Overview tab. The page reads again +every two seconds while a build is queued or running and every fifteen +seconds otherwise, when its window regains focus, and after a control's +answer: it asks again for `summary`, `status`, `build_requests`, +`environments` and `runs` (and `map`, except on the timer unless a run or a +request ended or `status` changed) and compares each answer with the one it +holds. When none differs nothing on the page is drawn again but the time it +was read and the progress of a running build; when one does, what is on screen is drawn again in place, keeping +the scroll and the focus, and a drawer holding the editor or the branch form +is left alone except after a control's answer, as are the forms of New task, +Import and Freeze and an annotation typed or drawn and not kept. A reading that fails leaves +the page as it is; the status strip says the project was not read, and why. + +### Signing in and issuing an agent credential + +The owner makes a one-time code and enters it in the page's sign-in bar: + +``` +python access.py login --data CATALOG --out /private/owner-code +``` + +(`bootstrap` instead of `login` the first time a catalog has no owner.) Signed +in, the owner issues an agent credential with `POST /api/access/credentials` +and the body `{"label": ..., "scopes": [...], "ttl_seconds": ...}` (60 seconds +to 30 days, one hour by default). The answer holds the token once; keep it +in a file only the agent's user can read. An agent credential must include +`inspect`, may include `edit`, `execute` and `cancel`, and can never approve. + +### Connecting an agent + +An agent uses `agent_api.AgentClient` with its credential, or the MCP bridge +on the same private configuration file (`endpoint`, `catalog_id`, `token_file` +and optional `timeout`; see [agent operations](agent-operations.md)): + +```python +import agent_api +client = agent_api.AgentClient.from_config('/private/agent.json') +client.call('project.summary', {}) +client.call('project.plan', {'targets': ['benchmarks/held-out']}) +``` + +``` +python mcp_stdio.py --config /private/agent.json +``` + +The owner uses `agent_api.ProjectFolderOwnerClient`, or `mcp_stdio.py +--project-folder-owner`, with an owner credential. `catalog_id` is the +`dataset_id` that `GET /api/workspace/context` answers. + +Every operation is `POST /api/project/` with a JSON object of its +arguments, and must name its contract in the `Tuldok-Operation-Contract` +header, as `GET /api/agent/capabilities` lists it; the clients do this. A +call answers its result object, or a refusal `{"error", "code"}` with +`problems`, `references` and `files` when there are any. The status is 404 +for `not-found`, 400 for `invalid`, 403 for `forbidden` and 409 for every +other code. Other `/api/` routes answer 409 `unavailable`, except sign-in, +credentials, `GET /api/agent/capabilities`, `GET /api/agent/operations`, +`GET /api/workspace/context` and `GET /api/project/file` (see +[Looking inside a product](#looking-inside-a-product)). + +### Requests and approval + +Nothing runs without the owner's approval. A build asked for with +`request_build` is a **build request**. The owner's own request is queued at +once. An agent's waits as `awaiting-approval` until the owner approves it +(`approve_build`, or `python -m tuldok_project approve FOLDER REQUEST`) or +declines it. The approval covers exactly what the request showed: the commit +and the text of every definition in its plan. If either changed, approving +is refused (`stale`) and the request is declined. The app runs queued +requests one at a time; an approval not started within an hour expires. An +agent may request a build only when its credential includes `execute`. + +### Operations + +**Who** is the rule for the caller: `inspect`, `edit` or `cancel` means any +credential with that scope; `request` means the owner, or an agent whose +credential includes `execute`; `owner` means the owner only. The arguments +and result keys of each operation are in its contract. + +| Operation | Who | Does | +| --- | --- | --- | +| `summary` | inspect | Read the project's name, folder, counts of definitions by kind and state, and whether its build worker runs. | +| `status` | inspect | List the unsaved and removed definitions and the files changed on disk since the last import or save. | +| `check` | inspect | Check the project folder on disk and list its errors and warnings, even when it does not open. | +| `definitions` | inspect | List the definitions, optionally of one kind, with their states. | +| `definition` | inspect | Read one definition: its text, document, state, what it uses and what uses it. | +| `graph` | inspect | Read every definition with its kind and state, and the edges between them. | +| `results_overview` | inspect | List, per benchmark, how many subjects have results and whether they differ from the results file. | +| `results` | inspect | Read one page of a benchmark's summary rows, optionally for one subject, metric or group. | +| `values` | inspect | Read one page of a benchmark subject's per-case values. | +| `chart` | inspect | Read a chart's data, and its SVG drawing when asked. | +| `report` | inspect | Read the HTML report of the project: where it stands, every chart, the results and the definitions. | +| `plan` | inspect | Plan building the targets (or every buildable definition) without running anything. | +| `estimates` | inspect | List the stored estimates of disk and time for the definitions as they now are. | +| `environments` | inspect | List the environments with the interpreter recorded on this machine and their last check. | +| `environment` | inspect | Read one environment with the interpreter recorded on this machine and its last check. | +| `runs` | inspect | List the runs, newest first, optionally of one definition. | +| `run` | inspect | Read one run's record, its product and whether its build is alive, without variable values. | +| `progress` | inspect | Read a run's latest progress event and its logs from the given offsets; to read on, pass each log's returned offset back as `stdout_offset` and `stderr_offset`. | +| `products` | inspect | List the current products, optionally of one definition. | +| `build_requests` | inspect | List the build requests, newest first, optionally in one state. | +| `build_request` | inspect | Read one build request with what it covers and the runs of its build. | +| `activity` | inspect | List the changes asked of this project through the interface, newest first, and how each ended. | +| `map` | inspect | Read the map of the project: its groups, every recipe as a model with its group, kind of change, state and track value, the lines between them and the comparisons. | +| `note` | inspect | Read one file under the notes directory of the project folder, given relative to it. | +| `recipe_results` | inspect | Read the summary rows of every benchmark subject made from one recipe, ungrouped or for one group. | +| `draft_branch` | inspect | Write the text of a new recipe branched from one of the project's, with its name and what keeping it would find wrong; nothing is stored. | +| `records` | inspect | Read one page of the records of a finished run's product, optionally of one kind or split, with the counts by kind and split. | +| `record` | inspect | Read one record of a finished run's product by its id, or any file in it by its path, with its content: text, image size, sound peaks, points, mesh or arrays. | +| `inside` | inspect | List a directory, an archive or the arrays of an .npz in a finished run's product. | +| `array_slice` | inspect | Read a slice along two axes of an array in a finished run's product, as 32-bit floats. | +| `array_summary` | inspect | Read the range, mean, deviation and histogram of an array in a finished run's product. | +| `weights` | inspect | List the weights files of a finished run's product, each with its size, the file it started from and, when Tuldok reads it, its layers and parameter count. | +| `weights_compare` | inspect | Compare, array by array, a weights file of a finished run's product with one of the same or another finished run: norms, relative change, cosine and largest change. | +| `array_tiles` | inspect | Read tiles of two axes of an array in a finished run's product, one per position along a third axis, as 32-bit floats. | +| `fields` | inspect | List the fields a finished run's product kept: each case's native and predicted files, its name, shape and whether the two can be compared, and the figures the run reported itself. | +| `field_compare` | inspect | Read a slice of one kept case's native field, predicted field and signed error as 32-bit floats, with the figures of the error over the whole field. | +| `model_graph` | inspect | Read the structure a finished run's script recorded (model-graph.json): its operations and the edges between them, folded to a depth of the module tree with modules opened or closed by path, and the module tree. | +| `model_graph_overlay` | inspect | Read a number per module and per operation of a finished run's model graph: parameters, output values, operations, or the relative change of the weights under it against the weights it started from or another run's. | +| `model_graph_compare` | inspect | Match the modules of two finished runs' model graphs by path: the same, changed (class or tensor shapes), added or missing. | +| `training` | inspect | List what a run's training script logged, while it runs or after: its scalar series, the tensors whose histograms it kept, its samples and checkpoints, and the lines it could not read. | +| `training_series` | inspect | Read scalar series a run's training script logged, in buckets of a power of two points (mean, least and greatest), from the bucket the caller holds when its bucket size is the same. | +| `training_histograms` | inspect | Read one tensor's histograms of weights or gradients over the steps of a run's training, re-binned onto one grid. | +| `training_samples` | inspect | List one page of the samples a run's training script kept, optionally of one name. | +| `training_sample` | inspect | Read one sample a run's training script kept: an image's size and address, or an array's slice as 32-bit floats. | +| `embeddings` | inspect | List a dataset's embeddings sets: those in its own products and in the products of the datasets that take it as an input, each with its model, rows and the run whose records they are. | +| `embedding_map` | inspect | Read a map of an embeddings set's records in two dimensions (the script's own projection, or principal components Tuldok computes), each record's colour by a field, the map's cells and its thin regions. | +| `embedding_neighbours` | inspect | List the records of an embeddings set most like one of them, by cosine, exactly. | +| `embedding_duplicates` | inspect | List the pairs of an embeddings set's records at or above a cosine, most alike first, and whether every record was examined. | +| `embedding_outliers` | inspect | List the records of an embeddings set farthest from their k-th nearest record, and whether every record was examined. | +| `embedding_leakage` | inspect | List the records of an embeddings set outside the training split that have a near-copy in it, at or above a cosine. | +| `set_definition` | edit | Check a definition's text and keep it as an unsaved edit. | +| `remove_definition` | edit | Mark a definition that nothing uses as removed, until the project is saved. | +| `revert_definition` | edit | Put back the file in the folder in place of a definition's unsaved edit or removal, or drop a definition the folder has no file for. | +| `save` | edit | Write the unsaved edits and changed results files to the project folder. | +| `resolve_conflict` | edit | Open a project whose files changed on disk where it has unsaved edits, keeping or discarding those edits. | +| `import_values` | edit | Import one per-case values file, replacing that subject's values and summaries. | +| `check_environment` | edit | Check an environment's recorded interpreter against its definition and keep the result. | +| `connect_environment` | owner | Record an existing interpreter for an environment on this machine and check it. | +| `disconnect_environment` | owner | Forget an environment's recorded interpreter on this machine; nothing is deleted. | +| `request_build` | request | Ask for a build of the targets: the owner's request is queued, an agent's waits for the owner's approval (read `build_request` for its state), and a queued request not started by its `expires_at` expires. | +| `approve_build` | owner | Approve a build request awaiting approval, if what it covers is unchanged. | +| `decline_build` | owner | Decline a build request awaiting approval, with a reason. | +| `cancel_build` | cancel | Cancel a build request, stopping its build when it is running. | +| `cancel_run` | cancel | Ask a queued or running run to stop. | +| `stop_run` | cancel | Kill what a run whose build is gone left running, and mark it unknown. | +| `served` | inspect | List the serving sessions on this machine, newest first, the recipes that declare serve with whether each is built, and the machine's limits. | +| `serve_session` | inspect | Read one serving session with its timing summary and its newest requests' lines. | +| `serve_start` | request | Start serving a built recipe's model from its serve entry point: the owner's start runs at once, an agent's waits for the owner's approval. | +| `serve_approve` | owner | Approve a serving session awaiting approval, which starts it. | +| `serve_stop` | cancel | Stop a served model, or decline a serving session awaiting approval. | +| `serve_predict` | inspect | Pass inputs to a served model and read its outputs and the time taken; arrays are objects of array float32, shape and base64. | +| `serve_values` | edit | Import a serving session's request times as per-case values of a benchmark subject, one case per request. | +| `annotation_tasks` | inspect | List the annotation tasks, each with its dataset, current product and counts of its records by the state of their annotations. | +| `annotated_records` | inspect | Read one page of the records of a task's dataset with their annotations, optionally in one state or of one kind, with the counts. | +| `annotation` | inspect | Read one record's annotation under a task as it now is and as it is in the file, whether it is unsaved or stale, and the hash of the record's content. | +| `set_annotation` | edit | Check annotations of a task's records, or their removal, and keep them as unsaved edits; one item that does not check refuses them all. | +| `revert_annotations` | edit | Put back the saved annotations of a task's named records, or of all its records, in place of their unsaved edits. | +| `endpoint` | inspect | Read the Pumas endpoint this machine uses: its address, whether it answered, and each model and profile it serves with its modalities, its capabilities and what Tuldok can make with it. | +| `set_endpoint` | owner | Set the Pumas endpoint this machine uses once it has answered what it serves, or clear it with null. | +| `find_endpoints` | owner | List the Pumas endpoints advertising themselves on this machine, for the owner to pick from. | +| `set_embedder` | owner | Set the model this machine embeds with when a definition names none, and the text put before each record's text; null keeps EmbeddingGemma 2 and the default prefix. | +| `suggestion_models` | inspect | List, for each kind of record in the product of a task's dataset, the models the Pumas endpoint serves that can suggest a value of the task. | +| `suggest_annotation` | edit | Ask a model the Pumas endpoint serves for a value of a task for one record; nothing is stored, and an answer that is not a value of the task is refused. | + +`map` answers `groups` (in their order, each with `id`, `title`, `parent`, +`description` and `models`, the number of recipes naming it), `models` (one per +recipe, by `trained_at` and then by name: `reference`, `title`, +`short`, `group`, `change`, `derived_from`, `trained_at`, `state`, `unsaved` +(whether the definition has an unsaved edit), `adopted`, `card`, `value` and +`count`, `benchmarks`), `edges` (`{from, to, kind}`, +`derived` or `input`), `comparisons` (`{from, to, explanation, excerpt}` as +authored) and `track` (null, or `{chart, benchmark, metric, statistic}` from +`map.chart`). A model's `value` and `count` are the track's statistic on the +ungrouped row of its subject and how many cases it summarises; when a recipe +is the subject under several names, the best by the metric's `better` is +taken. A model's `trained_at` is its label `trained_at`; without the label, +the time the run that made its current product finished (ISO 8601 in UTC), +and null when it has no product or its product was adopted. Models with a +`trained_at` come first, earliest first. `map` runs the same few queries +whatever the number of recipes and reads no file. `note` takes `path`, relative to `notes/`, and reads it from +the folder without opening the project; anything it refuses is `not-found`. +`recipe_results` takes `recipe` and an optional `group` (absent: the +ungrouped rows only) and answers `rows`, each as in `results` with +`benchmark` added, ordered by benchmark, subject and metric, and +`benchmarks`, the benchmarks with a subject made from that recipe. + +`draft_branch` takes `recipe` (the parent, a bare name or `recipes/`), +`title` (1 to 200 characters) and optionally `name`, `change` (one of the four +kinds of change) and `description` (at most 4,000 characters). It stores +nothing and answers `reference`, `text`, `parent`, `inlined` and `problems`: + +- `reference` is `recipes/`. A given `name` must follow the name rule and + be free, and a removed definition's name is not free; otherwise the call is + refused `invalid`. Without `name` it is made from the title: lowercase, each + run of characters outside `[a-z0-9]` one `-`, trimmed of `-` at both ends and + to 80 characters (`recipe` when nothing is left), then `-2`, `-3` and so on + until it is free. +- `text` is the new recipe, written from the parent's text as it is in the + working copy: `format`, `kind`, `title`, `description` when given, the + parent's `model` and `group` when it has them, `change` (the argument, else + the parent's), `derived_from` naming the parent, and the parent's `run` when + it has one. `card`, `labels`, `earlier_work` and `compared_with` are not + carried. It is JSON indented as the parent's text is (two spaces when the + parent's has no indentation) and ends with a newline. +- When the parent's `run` has `settings_file` and that file, read from the code + root as it is on disk, is a regular file holding one JSON object of at most + 256 KiB, the new `run` carries that object as `settings` in place of + `settings_file`, and `inlined` is the file's path. A file reached through a + link that leaves the code root is not read. Otherwise `inlined` is null and + `run` is carried as it is. +- `problems` is what `set_definition` would find in `text` at `reference`, + errors and warnings, each `{severity, path, pointer, message}`. A branch of a + model without `run`, or whose code paths are missing, comes back with those + errors for the caller to fix before keeping it. + +A `recipe` that is not a recipe of the project, or is removed, is refused +`not-found`. + +`revert_definition` takes `reference` and answers `{reference, dropped}`. It +applies to a definition with an unsaved edit or marked removed: + +- When the folder holds its file, the file's text takes its place as an import + of that file would put it (text, title, kind, the file's digest and the uses + it makes), and it is neither unsaved nor removed; `dropped` is false. The text + is checked as `set_definition` checks it, and a file that no longer checks, or + that cannot be read, is refused `invalid` with its problems and nothing + changes. A benchmark's summary rows are kept to what fits it, as + `set_definition` keeps them; when its results file on disk is still the one + last imported, that file is imported again, so a benchmark whose removal is + reverted has the rows of its results file back (per-case values imported + from values files went with the removal and are not). +- When the folder holds no file for it, it is dropped as `remove_definition` + drops a definition never saved, and `dropped` is true; this is refused + `in-use` while another definition uses it. + +A definition with neither an unsaved edit nor the removed mark is refused +`nothing-to-revert`, and an unknown one `not-found`. + +Installing an environment (`set-up`), measuring estimates (`estimate`), +variables passed to a build (`build --env`), rebinding and forgetting a +project stay command-line actions. + +### Looking inside a product + +A **product** is the `output` directory of a run in state `succeeded` or +`adopted`, named by its run id (or an unambiguous prefix of it). The +operations `records`, `record`, `inside`, `array_slice`, `array_summary`, +`array_tiles`, `weights`, `weights_compare`, `fields`, `field_compare`, +`model_graph`, `model_graph_overlay` and `model_graph_compare` read it; each takes `run`, and a run that does not exist, has not finished or +has no output directory is refused `not-found`. Like every operation they +open the project for a moment, to find the run and the view of its dataset; +they read the product after closing it and write nothing: no file, no row. + +An **address** is a path relative to the product, never absolute, never with +`..`. A member of a zip archive is named after `!`, as +`training-view.zip!samples/a.npz`; an address has at most two `!`. An `.npz` +is a zip of `.npy` members; one of its arrays is named by the argument +`array`, without `.npy`. An address that is not one is refused `invalid`. An +address is resolved under the product, and its real path must lie inside +it: a missing file, and a symbolic link that leaves the product, are refused +`not-found`. Archive members are read where they are, never extracted. + +**Records.** A record is one item of the product. Its kind is `text`, +`image`, `audio`, `pointcloud`, `mesh`, `array`, or `file` for anything else +(listed, never opened). The records come from the first of: + +1. `records.jsonl` at the product's root, one JSON object per line: + + | Key | | + | --- | --- | + | `id` | required; unique; 1 to 200 characters of `A-Z a-z 0-9 . _ -` | + | `kind` | required; one of the seven kinds | + | `path` | an address; required unless `kind` is `text` and `text` is given | + | `text` | the text itself, for a short text record (at most 64 KiB) | + | `array` | for kind `array` in an `.npz`: the array to show first | + | `split`, `rights` | optional strings | + | `origin`, `meta` | optional objects, kept and shown as they are | + | `sha256` | optional; the SHA-256 of the record's content, 64 lowercase hexadecimal digits, or null (see [Annotations in the database](#annotations-in-the-database)) | + | `bytes` | optional; the record's size, an integer ≥ 0 (otherwise the file's size is shown) | + | `annotations` | optional object; a frozen record's annotations, per task its `state` and `value` | + + Other keys are kept and shown in `meta` (a key of `meta` itself wins). A + line that is not such a record, or is longer than 64 KiB, is skipped and + reported in `problems` with its line number (the first 20, then one + problem with the number of such lines); blank lines are skipped. +2. `view` on the dataset's definition (datasets only): + + ```json + "view": {"records": "lists/all.jsonl"} + "view": {"files": "training-view.zip!samples/*.npz", "kind": "array", "show": "target_pressure"} + ``` + + `records` is the address of a record list in the format above, anywhere + in the product; its paths are addresses in the product. `files` is a + pattern: `*` and `?` match within one path segment, `**` across segments, + and one `!` may reach into the archives it matches. Each match is a record + whose id is its address. `kind` (by default by extension) and `show` (the + array shown first) apply to every match and only to `files`. A view has + `records` or `files`, not both. `check` checks the shape of `view` and its + addresses, and nothing about any product. +3. Otherwise every regular file under the product, sorted by address, its id + its address and its kind by extension. At most 200,000 files are listed, + and `problems` says when there are more. + +| Kind | Extensions | +| --- | --- | +| text | `.txt .md .json .jsonl .csv .tsv .yaml .yml .log` | +| image | `.png .jpg .jpeg .webp .gif .bmp` | +| audio | `.wav .flac .mp3 .ogg .opus .m4a` | +| pointcloud | `.xyz`, and `.ply` with no faces | +| mesh | `.obj .stl`, and `.ply` with faces | +| array | `.npy .npz .safetensors` | +| file | everything else | + +The index of a product's records is kept for the 8 products read last and +made again when its list (or, without a list, the product directory) changes +size or time. + +**`records`** takes `offset`, `limit` (at most 200, 50 by default), `kind` and +`split`, and answers `records` (each `id`, `kind`, `path`, `bytes`, `split`, +`rights`, `origin`, `meta`, `sha256` (null unless the list gives it), `array` +when the list gives one, and for a text +record its first 400 characters as `preview`), `total` (after the filters), +`source` (`records.jsonl`, `view` or `files`), `kinds` and `splits` (counts +over all the records; records without a split are not counted in `splits`) +and `problems` (each `{line, message}`, `line` null for a problem of the whole +list). + +**`record`** takes `id`, or `path` (an address) in its place, exactly one of the two, and answers `record` (as above, with `media_type`: +the type of the file route for an image or a sound, with `url`, the file +route's address for it; `text/plain` for text; else +`application/octet-stream`) and `content`, by kind: + +| Kind | `content` | +| --- | --- | +| text | `text` (at most 262,144 characters), `characters` (of the whole text; null when it is longer than 16 MiB), `truncated`, `replaced` (bytes that were not UTF-8, each shown as U+FFFD) | +| image | `width`, `height`, from the header of a PNG, JPEG, GIF, WebP or BMP; null when not found | +| audio | for a WAV of PCM (8, 16, 24, 32 bit) or 32-bit float: `channels`, `rate`, `frames`, `seconds`, `peaks` (at most 1,000 pairs of lowest and highest sample over all channels, scaled to -1..1); for other sounds all null | +| pointcloud | `count`, `stride`, `bounds` (`[[x, y, z] lowest, [x, y, z] highest]` of the points returned), `points_f32` (base64 of little-endian 32-bit x, y, z; at most 100,000 points, every `stride`-th above that), `colors_u8` (base64 of r, g, b, or null). From PLY (ASCII or binary little-endian, colours from `red`, `green`, `blue`), XYZ text (colours when columns 4 to 6 are whole numbers 0 to 255), or an `.npy` of shape (N, 3 or more) when the record's kind is `pointcloud` | +| mesh | `vertices`, `triangles`, `bounds`, `positions_f32`, `triangles_u32` (base64, little-endian; polygons split into triangles as a fan). From PLY (ASCII or binary little-endian), OBJ (`v` and `f`, with `/` forms and negative indexes) and STL (binary or ASCII; equal corners are one vertex) | +| array | `arrays` (each `array`, `shape`, `dtype`, `bytes`; a plain `.npy` has one array, named null) and `show` (the record's `array`, else the view's `show`, else the first array with two or more dimensions, else the first) | +| file | an empty object | + +A point cloud or mesh file of more than 256 MiB is refused `too-large`, and +so is a mesh of more than 200,000 vertices or 400,000 triangles, with the +counts. A file that cannot be read as its kind is refused `invalid`. + +**`inside`** takes `path` (an address; the product's root by default), +`offset` and `limit` (at most 500, 200 by default), and answers `entries` and +`total`. A directory lists its children, an archive the members at its top +(an address `a.zip!samples` lists the members under `samples/`), and an +`.npy`, `.npz` or `.safetensors` its arrays. Each entry has `name`, `path` (its address), +`type` (`directory`, `file`, `archive` or `array`), `bytes` (decompressed, for +a member) and `kind`; an array also has `array`, `shape` and `dtype`, and its +`path` is the address of its file. Directories come first, then by name. A +zip nested deeper than an address can name is a `file`. A plain file is +refused `invalid`. A `.safetensors` file is listed with `type` `archive`, as +an `.npz` is, though no address reaches into it with `!`. + +**Arrays.** NPY versions 1, 2 and 3 are read, in C or Fortran order, little- +or big-endian, of floats of 2, 4 and 8 bytes, signed and unsigned integers of +1, 2, 4 and 8 bytes and booleans; `dtype` is named as `float32`, `int16`, +`uint8`, `bool`. Anything else (objects, complex numbers, strings, +structured types) is listed with its type as written and refused +`unsupported` when read; an object array is never unpickled. The header is +parsed as a literal of at most 64 KiB. + +A `.safetensors` file is read like an `.npz`: its tensors are its arrays, named +by `array` as they are in its header, listed in the order of its header. The +file is 8 bytes (the header's length, little-endian), a JSON header of at most +16 MiB (each tensor's `dtype`, `shape` and `data_offsets`, and +`__metadata__`, which is not shown), then the data, read where it lies. The +types `F16 F32 F64 I8 I16 I32 I64 U8 U16 U32 U64 BOOL` are read, little-endian +and in C order, and named as an NPY array's are; a tensor of another type +(`BF16` and the rest) is listed with its type as written and refused +`unsupported` when read. A header longer than the file, data offsets outside +the file or reaching into the header, and data whose length does not match +its shape and type are refused `unsupported`. + +**`array_slice`** takes `path`, `array` (for an `.npz`), `axes` (the two axes +drawn, by default the last two), `index` (one position per axis, by default +the middle of each; ignored for the two drawn) and `most` (rows and columns +at most, 256 by default, at most 512). It answers `shape`, `dtype`, `axes`, +`index`, `rows`, `columns`, `stride` (one for both axes; 1 unless every +`stride`-th row and column is taken to fit `most`), `values_f32` (base64 of +little-endian 32-bit floats, row by row), `min` and `max` (of the finite +values) and `nonfinite`. An array of one axis is drawn as one row and one of +none as one value; `axes` is then left out. + +**`array_summary`** takes `path`, `array` and `bins` (64 by default, at most +256), and answers `shape`, `dtype`, `count`, `min`, `max`, `mean`, `std`, +`zeros`, `nonfinite`, `histogram` (`low`, `high`, `counts`: equal bins from +the lowest to the highest finite value) and `stride`. At most 4,000,000 +values are read: beyond that every n-th value in storage order, and `stride` +is n. All but `count` and `stride` are of the values read. + +**`array_tiles`** takes `path`, `array`, `across` (the axis counted over; by +default the first axis not drawn), `axes` (the two drawn in each tile; by +default the last two other than `across`), `index` (one position per axis, by +default the middle of each; used for the axes neither drawn nor counted +over), `first` (0 by default) and `most` (tiles at most, 16 by default, at most +64). The array has three axes or more; otherwise, or when `first` is outside +`across`, the call is refused `invalid`. It answers `shape`, `dtype`, +`across`, `axes`, `index` (with the position of `first` on `across`), `first`, +`tiles` (how many follow `first`, at most `most`), `rows`, `columns`, +`stride` (as for `array_slice`, one for every tile, each tile at most 64 by +64), `values_f32` (base64 of little-endian 32-bit floats, tile by tile, each +row by row; tile `t` is the slice at position `first + t` on `across`), `min`, +`max` (of the finite values of all the tiles) and `nonfinite`. + +**Reading no more than is needed.** A file on disk is mapped read only, and a +member stored uncompressed in a zip is read at its offsets in its archive's +bytes, so a slice reads only the bytes it draws and listing an archive reads +only its central directory. A compressed member (deflate or bzip2) is +decompressed whole into memory; one of more than 256 MiB, by the size the +archive declares and again while decompressing, is refused `too-large` with +its size. What was decompressed is kept for the next read, at most 512 MiB +for all products together, the least recently used dropped first. Other +compression methods and encrypted members are refused `unsupported`. + +A file that is damaged, or is not what its name says, is refused `unsupported` +and the product stays readable: the file is one record among the others. +Arguments that do not fit (an address that is not one, axes or positions +outside an array) are refused `invalid`. + +**The bytes of an image or a sound.** `GET /api/project/file?run=&path=
` +answers the bytes of a file of a product, or of an archive member of at most +256 MiB, to whoever may call an `inspect` operation, and is refused as such an +operation is otherwise (401 without a credential). `run` and `path` are given +once each and nothing else, or the request is refused 400 `invalid`; refusals +of the product and the address are as for the operations. The media type +comes from the extension, for the image and audio extensions above only, and +those are sent `inline`; everything else is `application/octet-stream` with +`Content-Disposition: attachment`, so no product file is sent as HTML, SVG or +script. Every answer carries `X-Content-Type-Options: nosniff` and +`Cache-Control: no-store`. For a file on disk one `Range: bytes=a-b` (or +`bytes=a-`) is answered 206 with `Content-Range`; any other `Range` gets the +whole file. This route is not an operation and is not in the capability +manifest: an agent reads records, arrays and text through the operations. + +### A model's weights + +`weights`, `weights_compare` and `array_tiles` (above) read a product as the +operations above do: the same addresses and confinement, nothing written, and +a damaged file refused `unsupported`. Nothing runs the model. + +**A run's weights** are the outputs its product records with role +`weights`; a declared output that is a directory gives the files directly in +it with one of the extensions below. When the product records none, they are +the files at the top of the product whose extension is `.npz`, +`.safetensors`, `.pt`, `.pth`, `.ckpt`, `.bin` or `.onnx` and whose name +contains `weight` or `model` (in any case), not beginning `initial-`. Only +`.npz` and `.safetensors` files are read. A PyTorch file (`.pt`, `.pth`, +`.ckpt`, `.bin`) is a pickle: it is listed by name and size with the reason, +and never opened, not even to look at its first bytes. The weights a file +started from are the file beside it with the same name prefixed `initial-`. + +**Layers.** A tensor's layer is its name without its last dotted part: +`encoders.0.layers.0.weight` and `encoders.0.layers.0.bias` are the tensors +`weight` and `bias` of the layer `encoders.0.layers.0`; a name with no dot is +its own layer. Layers are in the order of the file (an `.npz`'s members in the +order of its central directory, a safetensors file's tensors in the order of +its header), each where its first tensor is. + +**`weights`** takes only `run` and answers `files`, each with `path` (its +address), `output` (the output's name, or null when found by name), `bytes` +(null when a declared output is missing), `readable`, `reason` (null when +readable; otherwise a sentence: a pickle, ONNX, another format, a file that is +missing, or one that could not be read and why), `starts_from` (the address of +its `initial-` file, or null) and, when readable, `parameters` (the count of +the tensors listed), `tensors` (how many the file holds) and `layers` (each +`name`, `parameters` and `tensors`, each tensor `array`, `name`, `shape`, +`dtype` and `parameters`). At most 10,000 tensors of a file are listed. A +damaged weights file is listed as not readable with its reason; the call is +not refused for it. + +**`weights_compare`** takes `path`, `other_run` (the same run, for its +`initial-` file, or any other finished run of the project) and `other_path`, +both weights files that are read (otherwise refused `unsupported` with the +reason). For each array in both with the same shape, in the order of `path`, +`tensors` gives `array`, `parameters`, `norm`, `other_norm`, +`difference_norm` (of this minus the other), `relative_change` (the +difference's norm over the other's; null when that is zero), `cosine` (null +when either norm is zero), `largest_change` (the largest absolute difference) +and `nonfinite` (non-finite values in either). `total` gives the same over all +of them together, without `array`. `only_here` and `only_there` name the +arrays of one file only, `reshaped` gives `array`, `shape` and `other_shape` +of those whose shapes differ, and `unread` gives `array`, `dtype` and +`other_dtype` of those with the same shape that one side's type keeps from +being read; neither is compared. Norms are Euclidean, over 64-bit floats; a +quantity that includes a non-finite value is null. Arrays are compared +element by element in C order, whichever order each is stored in. At most +32,000,000 values are read in all (both files together) and at most 10,000 +tensors per file; beyond either the call is refused `too-large`, saying how +many there are. + +### The fields a run kept + +`fields` and `field_compare` read a product as the operations above do: the +same confinement, nothing written, a damaged file refused `unsupported`. +Nothing runs a model or a solver; only kept files are read. + +**A product keeps fields** when the top of its directory holds +`field-index.json`: + +```json +[{"run_id": "…", "cases": [{"member_ordinal": 0, + "predicted": {"name": "pressure-prediction.npy", "bytes": 32896, "sha256": "…"}, + "reference": {"name": "pressure-reference.npy", "bytes": 32896, "sha256": "…"}, + "mask": null, "time_units": "not_applicable"}]}] +``` + +- A list of entries, each an object with a list `cases`; anything else is a + damaged index, refused `unsupported` (an index over 4 MiB is refused + `too-large`). Each case names a `predicted` and a `reference` file by + `name`: a plain file name at the top of the same product (no `/`, `\`, `!` + or `..`), an `.npy`. +- Every other key of a case is optional and kept as given for display + (`details`). `sha256` is not checked. `bytes`, when given, is compared with + the file's size, and a mismatch is shown (`bytes_mismatch`), not refused. +- `mask`: null, or `{"name": …}` of a boolean or integer `.npy` of the same + shape; true or non-zero is inside. With a mask the figures are over the + inside only, and the outside is NaN in the slices. +- Both files are `.npy` of a number type (floats or integers, not booleans), + of the same shape, of one to four axes and at most 16,777,216 values each. + A case that is not so is listed with the reason, and `field_compare` + refuses it `unsupported` (`too-large` for one over the size). +- **Which case it is.** When the product holds `probe.json` of `format` + `probe_v1`, a case's `member_ordinal` is looked up among its `members` (by + each member's `ordinal`, else its position): the case is named by the + member's `frozen_reference.member_id`, and its split is + `frozen_reference.split`, else the member's `split`, else the probe's. + Otherwise a case is named by its position, `case N` (`entry E case N` past + the first entry), and has no split. +- **What the run reported.** Every JSON file at the top of the product (at + most 4 MiB) whose `format` ends `_diagnostic_v1` gives its top-level + numeric fields whose names end `relative_l2`, as `reported`. They are shown, + never used to compute anything. + +**`fields`** takes only `run` and answers `kept` (whether there is an index), +`cases` and `reported` (each `{file, field, value}`), and `more` (true when +more than 10,000 cases are held; the first 10,000 are listed). Each case has +`entry` and `case` (its positions), `name`, `split` (or null), `predicted`, +`reference` and `mask` (file names, or null), `shape` and `dtype` (null when +the files could not be read as far as that; `dtype` is "float32 / float64" +when the two differ), `comparable`, `details`, and `reason` when it is not +comparable and `bytes_mismatch` (the names) when a size differs. + +**`field_compare`** takes `entry`, `case`, `axis` (the axis cut across; by +default the last), `index` (one position per axis, as `array_slice` takes it; +those of the axes drawn are ignored; by default the middle) and `mean_free` +(false by default). A one-axis field is drawn as one row (`axes` `[0]`), a +two-axis field whole (`axes` `[0, 1]`); `axis` is then null. A field of three +or four axes is drawn through the last two axes other than `axis`, the +positions on the others given by `index`. Beyond 256 rows or columns the slice +is subsampled by one stride, as `array_slice` is. It answers `entry`, `case`, +`shape`, `dtype`, `axes`, `axis`, `index`, `rows`, `columns`, `stride`, +`mean_free`, `reference_f32`, `predicted_f32` and `error_f32` (base64 of +little-endian 32-bit floats, row by row; cells outside the mask NaN), `range` +(the lowest and highest finite value of the reference and the prediction in +the slice together, or null), `error_limit` (the largest absolute finite +error in the slice, or null) and `figures`. The error is predicted minus +reference; with `mean_free` each field's mean (over the inside) is taken from +it first, in the error only: the two fields are shown as they are. + +`figures` are over the whole field, not the slice: the cells inside the mask +whose two values are finite. `count` (those cells), `nonfinite` (non-finite +values inside, of either field, left out), `max_abs_error`, `mean_error`, +`rmse`, `relative_l2` (the error's norm over the reference's; null when that +is zero) and `mean_free_relative_l2` (the same with each field's mean removed +first; null when the reference is constant). They are computed in 64-bit +floats with `math.fsum` and `math.sumprod`; a figure that would overflow is +null, as all of them are when no cell counts. A case's figures are kept while +its files are unchanged, so asking for another slice reads only the slice. + +### A model's structure + +A run's **model graph** is the file `model-graph.json` at the top of its +product, or beside one of its weights files (see above). The project's own +training script writes it, once, with an example input, where the model's +code runs: Tuldok never imports or runs a project's model. The optional +helper `tuldok_log.graph.export_graph(model, example_inputs, path)` writes it +for a PyTorch model (with `torch.export`, else `torch.fx`, else the module +tree alone); a script may also write it itself. + +```json +{"format": "tuldok_model_graph_v1", "complete": true, "reason": null, + "method": "torch.export", "torch": "2.8.0+cpu", "model": "nets.Net", + "inputs": [{"name": "x", "shape": [2, 1, 8, 8], "dtype": "float32"}], + "tensors": {"stem.weight": {"shape": [4, 1, 3, 3], "dtype": "float32", "kind": "parameter"}}, + "nodes": [{"id": "conv2d", "kind": "operation", "operation": "aten.conv2d.default", + "module": "stem", "tensors": ["stem.weight", "stem.bias"], + "inputs": [{"shape": [2, 1, 8, 8], "dtype": "float32"}], + "outputs": [{"shape": [2, 4, 8, 8], "dtype": "float32"}], + "attributes": {"padding": [1, 1]}}], + "edges": [{"from": "x", "to": "conv2d", "input": 0}], + "modules": {"path": "", "class": "nets.Net", "parameters": 1234, "children": [ + {"path": "stem", "class": "torch.nn.modules.conv.Conv2d", "parameters": 40, "children": []}]}, + "notes": []} +``` + +- Required: `format` (`tuldok_model_graph_v1`), `complete`, `tensors`, + `nodes`, `edges`, `modules`. `complete` false means only the module tree + was written; `reason` says why, and `nodes` and `edges` are not read. +- `tensors` are the model's parameters and buffers by the names of its + `state_dict` (the names of its weights file), each `shape`, `dtype` and + `kind` (`parameter`, `buffer` or `constant`). +- `nodes`, each after the nodes it reads: `id` (unique, not beginning + `module:`), `kind` (`input`, `operation` or `output`), `operation`, + `module` (the path of the deepest module it ran in, `""` for the model; a + path the tree does not list is read as its nearest listed parent), + `tensors` (names in `tensors` it reads), `inputs` and `outputs` (each + `shape` and `dtype`; a size is a whole number, or the name of a symbol), + `attributes` (optional: numbers, booleans, strings, null or short lists of + those). `edges` go `from` a node `to` a node; `input` is the position among + the values of `to`. +- `modules` is the tree of the model's modules, rooted at path `""`; each + child's path extends its parent's; `parameters` counts every parameter + under a module. +- Bounds: 32 MiB (refused `too-large` beyond), 20,000 nodes, 100,000 edges, + 20,000 modules, 64 deep, 100,000 tensors, 64 tensor names and 16 inputs or + outputs per node, 16 axes, 16 attributes, 1,024 characters per string. A + file that is not UTF-8 JSON (`NaN` and `Infinity` are not JSON), not of the + format, or past a bound is refused `unsupported`, saying what. + +**`model_graph`** takes `depth` (0 to 64; by default 1 when the graph has more +than 200 operations, else 64), `open` and `close` (module paths, at most +2,000 each). A module's depth is its count of dotted parts; it is open when +named in `open`, or when its depth is at most `depth` and it is not named in +`close`. An operation inside a closed module is shown as the shallowest +closed module on its path: a node of kind `group`, id `module:`, with +`operations` (how many it holds), `tensors` (those its operations read, at +most 64, and `tensor_count`), `parameters` (the module's), and `inputs` and +`outputs` (the values that cross into and out of it). Edges between the +same two shown nodes are one, with `count`; an edge inside a group is +dropped; each carries the `shape` and `dtype` of the first value it carries. +The answer also gives the file's `path`, `complete`, `reason`, `method`, +`torch`, `model`, `inputs`, `notes`, `counts`, the `modules` tree, the +`tensors`, the `depth` used and `open` (the modules open in effect). A run +with no graph file is not refused: `path` is null, `modules` null and the +rest empty (as `fields` answers `kept` false), so a page can ask every +model's product. `model_graph_overlay` and `model_graph_compare` refuse such +a run `not-found`. + +**`model_graph_overlay`** takes `kind`: `parameters` (a module's from the +tree; an operation's, the sizes of its tensors that are parameters), +`outputs` (the values an operation gives; a module's, the sum over its +operations; null for a symbolic size), `operations` (how many run in a +module), or `change`, with `against` `start` (the `initial-` file beside the +weights file) or `run` (with `other_run`), and optionally `path` and +`other_path` (weights files; by default the first readable one). The change +of a module is computed from `weights_compare`'s rows of the tensors under +it, `sqrt(sum difference_norm^2) / sqrt(sum other_norm^2)`, null when none +is compared, when the other's norm is zero or a norm is not finite. It +answers `kind`, `unit`, `modules` and `nodes` (a number or null each), +`low` and `high`, and for `change` `against`. + +**`model_graph_compare`** takes `other_run` and matches the two trees' +modules by path: each module of either, in this tree's order then the +other's, with `status` `same`, `changed` (its class differs, or a tensor +directly in it changed shape or is on one side only), `added` (here only) or +`missing` (there only), both classes and parameter counts, `reshaped`, +`only_here` and `only_there`; and `counts` by status. +### What a training script logged + +A run's own script may log its training into `training/` of its output +directory (`{output}/training/`), by hand or through the helper `tuldok_log` +(the package of that name at the top of the Tuldok repository, standard +library only, PyTorch only for `autolog`). Every file is optional. Tuldok +reads them; it never runs, imports or unpickles anything of the project's. +Unlike the reads above, these read a run **in any state** that has a +directory, while it runs as well as after: a file being appended to is read +up to its last complete line (a partial last line is left for the next read), +and what has been read is kept with its offset, so reading again reads only +what was appended. A file that became shorter, or was replaced, is read again +from its start. JSON is UTF-8, one object per line; `NaN` and `Infinity` are +read as non-finite. + +- **`training/metrics.jsonl`**, one line per log: + `{"step": 120, "time": 14.2, "epoch": 3, "loss": 0.0213, "lr": 0.001}`. + `step` an integer of 0 or more, `time` seconds since the run started (0 or + more) are required; `epoch` is optional. Every other key whose value is a + number is a scalar series of that name (1 to 200 characters); `null` or a + non-finite number is a gap, counted as `nonfinite`. Keys whose value is not + a number are kept and listed by name (`other_keys`), not drawn. +- **`training/histograms.jsonl`**, one line per tensor per log: + `{"step": 100, "time": 12.1, "name": "encoder.0.weight", "kind": "weights", "edges": […], "counts": […]}`. + `kind` is `weights` or `gradients`; `edges` n + 1 finite numbers in order, + `counts` n numbers of 0 or more, 1 ≤ n ≤ 1,024; optional `nonfinite`. +- **`training/samples.jsonl`** and the files under `training/samples/`: one + line per sample, `{"step": 120, "time": 14.2, "name": "prediction", "path": "samples/000000120-prediction.png"}`, + `path` relative to `training/` and under `samples/`. A `.png`, `.jpg`, + `.jpeg` or `.webp` is an image; an `.npy` an array (read as `array_slice` + reads one). A sample's file is written whole before its line. +- **`training/checkpoints.jsonl`**: `{"step": 1000, "time": 120.5, "path": "checkpoints/step-1000.safetensors", "loss": 0.012}`, + `path` relative to the output directory; the other numeric keys are the + checkpoint's figures. Listed and marked on the curves, not opened. A run's + final weights are still its declared outputs (see [A model's weights](#a-models-weights)). +- **`training/training.json`**: `{"format": "tuldok_training_v1", "started_at": ""}`, shown. + +A line that is not as above is counted (`bad_lines`) and reported with its +file and number (the first 20, `problems`); the rest are read. Bounds: 64 KiB +a line (256 KiB in `histograms.jsonl`), 1,000,000 metrics lines and 1,000,000 +histogram lines (`truncated` beyond), 200 series, 2,000 tensor and kind +pairs, 100,000 samples, 10,000 checkpoints. + +| Operation | Arguments | Result | +| --- | --- | --- | +| `training` | `run` | `run` (`id`, `state`, `active`), `present`, `started_at`, `series` (`name`, `points`, `nonfinite`, `first_step`, `last_step`, `last`, `min`, `max`), `lines`, `last_step`, `last_time`, `other_keys`, `histograms` (`name`, `kind`, `rows`, `first_step`, `last_step`), `histogram_lines`, `samples` (`name`, `count`, `last_step`), `sample_lines`, `checkpoints` (the last 200), `problems`, `bad_lines`, `truncated` | +| `training_series` | `run`, `series` (1 to 16 of `{name, size, since}`), `most` (default 1,000, at most 4,000) | `series` (`name`, `points`, `nonfinite`, `size`, `first`, `buckets`), `lines` | +| `training_histograms` | `run`, `name`, `kind`, `most` (rows, default 120, at most 500), `bins` (default 48, at most 128) | `name`, `kind`, `rows` (`step`, `time`), `stride`, `low`, `high`, `bins`, `counts`, `totals`, `nonfinite` | +| `training_samples` | `run`, `name`, `offset`, `limit` (at most 200) | `samples` (`index`, `step`, `time`, `name`, `path`, `kind`, `bytes`, `problem`, and `url` for an image), `total` | +| `training_sample` | `run`, `index` | `sample` and `content`: an image's `width`, `height` and `url`; an array's slice (the last two axes, at most 128 by 128) as `array_slice` answers | + +A series of n points is sent in **buckets** of `size` consecutive points, +`size` the smallest power of two that makes at most `most` buckets (1 while +n ≤ `most`); a bucket is `[first_step, last_step, last_time, mean, min, max]` +of its finite values (null when it has none). A caller that holds buckets +sends the `size` it holds and `since`; when the size is still the same the +answer begins at bucket `since` (`first`), otherwise at 0. A tensor's +histogram rows are chosen evenly past `most` (the first and the last kept), +and each is re-binned onto `bins` equal bins between the lowest and highest +edge of the chosen rows, a bin's count shared by overlap. A run with no +directory answers `training` with `present` false and the other four +`not-found`. `GET /api/project/file` serves a file under `training/` of a run +in any state that has a directory; any other file only of a finished run. + +### A dataset's embeddings + +A product may hold **embeddings** of records: a vector per record, written by +a project's own script or by the step `embed` (see +[Steps Tuldok provides](#steps-tuldok-provides)). Tuldok computes no +embedding; it reads them to draw a dataset as a map and to search it for +near-duplicates, outliers, thin regions and leakage between splits. A set +named `` (1 to 64 of `a-z 0-9 . _ -`, starting with a letter or digit) +is two files under `embeddings/` at the top of the product: + +- **`embeddings/.npy`**: shape `(rows, dimensions)`, floats (`float32` + as written by Tuldok; `float16` and `float64` are read too), C order, every + value finite, no row all zeros. +- **`embeddings/.json`**, the index, UTF-8 JSON of at most 16 MiB: + +```json +{"format": "tuldok_embeddings_v1", + "records": ["a1", "a2", "a3"], + "dimensions": 256, + "metric": "cosine", + "model": {"name": "embeddinggemma-2", "model_id": "unsloth/embeddinggemma-2-GGUF", "profile": "embed-gpu", + "capability": "text_embedding", "via": "pumas"}, + "prefix": "task: clustering | query: ", + "source": {"dataset": "datasets/photos", "run": null}, + "projection": {"file": "photos.map.npy", "method": "umap"}, + "notes": []} +``` + +`records` are the record ids in row order (1 to 50,000 different ids of 1 to +600 characters without control characters, as products name records); `dimensions` 1 to 4,096; `metric` is `cosine`; +`model` an object with a `name` (1 to 256 characters) and other short facts; +`prefix` the text put before each record's text, or null; `source` which +records the rows are: `dataset` (null for the product's own records) and +`run` (null when the script cannot know it). `projection`, optional, names a +`(rows, 2)` `.npy` beside the index that the script made (any method); Tuldok +draws it instead of its own map. `notes`, optional, at most 64 sentences. +Other keys are ignored. At most 40,000,000 values a set and 64 sets a +product are read. + +A dataset's sets are found in its own current products and in those of every +dataset whose `run.inputs` names it (a script that embeds its input writes +`source.dataset`). The records a set describes are those of `source.run` +when given; else of the product holding it when they are its own; else of the +run that the holding run recorded for that input. + +Tuldok makes the rows unit length (cosine is their dot product) and computes, +in 64-bit floats with the standard library, the mean and up to eight leading +principal components of an evenly spaced sample of the rows (at most +`max(256, 2^19 / dimensions)` of them) by subspace power iteration on the +sample's covariance; the map is every row's coordinates on the first two. +The pair searches use each row's coordinates on all of them and the length of +what they leave out, which bound the distance of two rows from below, to rule +pairs out without reading them, and compute the rest exactly: what they list +is never approximate. Each search examines rows in a fixed shuffled order and +stops after 20,000,000 comparisons of coordinates or 400,000,000 values +multiplied exactly; a row is examined whole or not at all, and `complete` +says whether every row was. What is computed is kept while the files are +unchanged. + +| Operation | Arguments | Result | +| --- | --- | --- | +| `embeddings` | `dataset` | `dataset`, `run` (its current product, or null), `sets`: each `run` (holding it), `holder`, `name`, `rows`, `dimensions`, `model`, `prefix`, `source`, `records_run`, `current`, `projection` (its method), `notes`, `problem` | +| `embedding_map` | `run`, `name`, `records_run` (default `run`), `colour` (`split`, `kind`, `annotation:`, `field:`), `cells` (8 to 64, default 24), `thin` (0 to 1, default 0.25) | `name`, `rows`, `dimensions`, `model`, `method` (`pca` or the script's), `explained`, `sample`, `ids`, `x_f32`, `y_f32`, `bounds` (`[x0, x1, y0, y1]`), `missing` (rows whose record is not in the records' product), `colours`, `colour` (`by`, `type` `category` with `categories` and `codes_u16` (65535 for none), or `number` with `low`, `high` and `values_f32`), `grid` (`cells`, `counts`, rows from the lowest y), `thin` (cells holding some points but at most `thin` times the median count of the cells holding any: `cell`, `count`, `x`, `y`, `records`) | +| `embedding_neighbours` | `run`, `name`, `records_run`, `record`, `limit` (default 12, at most 200) | `record`, `neighbours` (`id`, `cosine`, `split`, `kind`), most alike first | +| `embedding_duplicates` | `run`, `name`, `records_run`, `threshold` (0.5 to 1, default 0.98), `limit` (default 200, at most 1,000) | `threshold`, `pairs` (`a`, `b`, `cosine`, `a_split`, `b_split`), `total`, `groups` (records in some pair), `complete`, `examined`, `rows` | +| `embedding_outliers` | `run`, `name`, `records_run`, `k` (1 to 50, default 5), `limit` (default 50) | `k`, `outliers` (`id`, `distance` (one minus the cosine to its k-th nearest), `nearest`, `split`), farthest first, `median`, `complete`, `examined`, `rows` | +| `embedding_leakage` | `run`, `name`, `records_run`, `threshold` (default 0.95), `train` (default `train`), `limit` | `threshold`, `train`, `splits`, `pairs` (`record` of another split, `split`, `near` (the training record), `cosine`), `total`, `leaked`, `complete`, `examined`, `rows` | + +A set that breaks a rule is listed by `embeddings` with its `problem` and +refused `unsupported` by the reads (`too-large` past a bound); a record not +in the set is `not-found`. + +### Annotating + +The five operations on annotations (see [Annotations in the database](#annotations-in-the-database)) +name a task by `task`, its name or its reference `annotations/`, and +answer it as the reference. A task that the project does not have (or has +removed) is refused `not-found`. The records are those of the current product +of the task's dataset, read as `records` reads them; the reads find the product +with the project open for a moment and read it after, as the reads above do. + +**`annotation_tasks`** takes nothing and answers `tasks`, by name, each with +`task`, `title`, `dataset`, `type` (the task's `task`), `vocabulary` (empty for +`review` and `text`), `run` (the dataset's current product, or null), +`records` (how many records it has, or null), `counts` and `unsaved` (how many +records have an unsaved change). `counts` has `unannotated` (records of the +product without an annotation), `draft`, `accepted`, `needs-work`, `rejected` +(annotations by state, stale ones included), `stale` and `invalid`. + +**`annotated_records`** takes `task`, `state` (one of the names of `counts`; +absent for all), `kind`, `offset` and `limit` (at most 200, 50 by default), +and answers `task`, `run`, `records` (the page), `total` (after the filters) +and `counts`. The records are those of the product in its order, each as +`records` gives it with `in_product` true, followed (unless `kind` is given) +by the annotated records the product does not have, by id, with `in_product` +false and null `kind`, `path`, `bytes`, `split`, `rights`, `origin`, `meta` +and `sha256`. Each has `annotation`: null, or `state`, `value`, `note` (null +when there is none), `by`, `at`, `stale` and `unsaved`. + +**`annotation`** takes `task` and `record` and answers `task`, `record`, +`annotation` (the line as it now is, as an object with the keys of the +annotations file, or null), `saved` (the line in the file, likewise, or +null), `unsaved`, `stale` (null when there is no annotation or no product) +and `sha256` (the hash of the record's content now, or null). + +**`set_annotation`** takes `task`, `items` (1 to 200, each +`{record, state, value?, note?}` or `{record, remove: true}`) and `from` +(optional, an object written as every item's `from`), and answers `task`, +`set`, `removed` (how many annotations went) and `unsaved` (how many of the +task's records now have an unsaved change). Every item is checked before +anything changes: the record must be in the product (refused `not-found` +when the dataset has none), named once, and its line must be one the +annotations file allows, its value checked against the task, the vocabulary +and the record's kind, and, where the product makes it cheap to know, an +image's size, a text's length (a file of at most 16 MiB) and a WAVE sound's +length. One item that does not check refuses the call `invalid`, with one +problem per such item whose pointer is `/items/`. The line takes `sha256` +from the record now, `by` from the caller (`owner`, or `agent:` and the +credential's name cut to 80 characters; never its id, token or scopes) and +`at` from the clock. Setting an annotation again takes the record's hash +again, which is how a stale annotation that is still right is put right. +Removing a record that has no annotation does nothing. + +**`revert_annotations`** takes `task` and `records` (optional, at most 10,000) +and answers `task` and `reverted`: how many of those records (all the task's, +without `records`) had an unsaved change and now have their saved line +again, or no annotation when the file has none. + +Each call of `set_annotation` and `revert_annotations` writes one activity +row whose target is the task's reference and whose detail says how many +records it set, removed or reverted. + +**`suggestion_models`** takes `task` and answers `task`, `type`, `address` +(the machine's endpoint, or null), `kinds` and `problem` (why nothing could +be asked: no endpoint, or the endpoint's reason; else null). `kinds` maps +each kind of record the product has that the task annotates to the served +models that can suggest for it now: `{model, profile, capability}`. It is +empty for a kind no model fits, and for every kind of a `review`, `boxes`, +`spans` or `segments` task and of a point cloud, mesh, array or file. + +| Task | Record | Asked as | +| --- | --- | --- | +| `text` | image | `image_to_text`, `image_messages` with the instruction and the image | +| `text` | audio | `audio_transcription`, the `audio` input (a 16-bit PCM WAVE of at most 30 s, sent as `pcm_s16le`), no options | +| `text` | text | `chat_generation`, `messages`: the instruction as system, the text as user | +| `class`, `tags` | text, image, audio | `decision` where an available one takes the record's modality: one `choice` over the vocabulary for `class`, one `noul` per term for `tags` (kept above 0.5); else, for text and images, `chat_generation` or `image_to_text` asked to answer with terms only, kept only when the answer is made of terms | + +**`suggest_annotation`** takes `task`, `record`, `model`, `profile` (needed +when the endpoint serves the model with more than one) and `instruction` +(optional, at most 4,000 characters; a caption and a summary have a default) +and answers `task`, `record` and `suggestion`: `value`, in the task's form, +and `from`, `{model, profile, capability}`. Nothing is stored and no activity +row is written; a person who keeps the value passes `from` to +`set_annotation`, so that its line in the annotations file names the model. +It is refused `not-found` for a record the product does not have, +`unavailable` when no endpoint is set, the endpoint does not answer or +refuses, or the model cannot suggest for this record now, `invalid` for a +record that cannot be sent (an image not PNG or JPEG or over the client's +bounds, a sound not such a WAVE), and `unsuitable` for an answer that is not a +value of the task, the model's words quoted. The record is read and the +endpoint asked outside the service's lock; an answer not given within 300 s +is given up. + +### Serving a model + +A recipe that declares `serve` can be served while Tuldok runs: the owner's +`serve_start` (or an agent's, once the owner approves it with `serve_approve`) +starts the project's own entry point in the environment, with the recipe's +current product, from the committed code at HEAD (or the code as it is with +`scratch`), and `POST /api/serve//predict` answers through it. The +entry point is two functions of the project: + +```python +def load(product): # once, with the product directory; what it returns is model + ... +def predict(model, inputs): # per request: inputs is the request's object, outputs any JSON + ... +``` + +Tuldok runs them in a separate process through its adapter +(`tuldok_project/serve_adapter.py`, run with `-I` by the environment's +interpreter); Tuldok's server never imports the project's code. Standard +output and standard error go to the session's log. The process gets +`TULDOK_PRODUCT`, `TULDOK_SERVE_MODEL` and `TULDOK_RUN_ID=serve-`, +and `CUDA_VISIBLE_DEVICES` empty unless `gpu`. + +**The endpoint.** `POST /api/serve//predict` on Tuldok's own address +(loopback by default; nothing new listens), with Tuldok's credential (a bearer +credential, or the page's session with a same-origin request) allowed to +`inspect`; no contract header. The body is `application/json`, +`{"inputs": {...}}`, at most 8 MiB, nested at most 64 deep. An **array** is an +object of exactly `array` (`"float32"`), `shape` (0 to 8 integers) and +`base64` (the little-endian float32 values in C order, 4 × the product of the +shape bytes, at most 2²⁴ values); `predict` receives it as a NumPy float32 +array when NumPy is in the environment, else as nested lists. Nested lists of +numbers pass as they are. In the outputs, NumPy arrays are encoded as arrays +and NumPy scalars as numbers. The answer is +`{"model", "session", "outputs", "timing": {"queue_ms", "model_ms", "total_ms"}}`; +refusals are 400 `invalid`, 401 or 403, 404 `not-served`, 409 `not-ready`, +413 `too-large`, 422 `predict-failed` (the function raised), 502 `serve-failed`, +503 `busy` (eight requests already wait), 504 `timed-out` (the session fails). +One request is in the process at a time. + +**Sessions.** States `awaiting-approval` (an agent's start; expires after an +hour), `starting`, `ready`, `failed`, `stopped`, `declined`, `expired`. A +session stops when asked (`serve_stop`), after `idle_minutes` without a +request, or when Tuldok stops; it fails when `load` raises or takes more than +`start_seconds`, when a request takes more than `request_seconds`, or when the +process ends. At most `serve_most` models are starting or ready on the +machine (settings file, default 2), one session a model. Under the local home, +`serving//` holds `session.json`, `output.log` (its first 4 MiB) and +`requests.jsonl`: per request `n`, `at` (Unix time), `ok`, `status`, +`queue_ms`, `model_ms`, `total_ms`, `bytes_in`, `bytes_out` and `error`. +`serve_session` adds the timing summary: count, answered, failed, span, +throughput, and the mean, p50, p95, p99 and max of each time. + +**As a benchmark subject.** `serve_values` writes a session's requests as a +values file (one case `request-000001`, … per request; a failed request is a +`null` value with its error as the reason) for a subject of a benchmark whose +`recipe` is the served one, filling the metrics named in `metrics` (metric id +to `total_ms`, `model_ms`, `queue_ms`, `total_s`, `model_s` or `queue_s`; +by default each metric whose id is one of these), and imports it as +`import_values` does. A benchmark of served latency may be one whose run is +the project's own load script, or one marked `earlier_work.rebuild` `no` +whose values come only from sessions. +### Sweeps + +A build of a sweep (`request_build` with the sweep as its only target) is +planned as what its recipe needs and one step of kind `sweep`, built, or +blocked when the recipe cannot be built for a reason other than its own +`earlier_work` label; what a request covers holds the recipe's text too. The +worker runs it trial after trial with one snapshot of the code and one plan +id, so Cancel on the request stops the trial running. While a trial runs +Tuldok reads its `training/metrics.jsonl` (as `training` does) every quarter +second, and stops it through its cancel file when halving prunes it or the +budget's minutes are spent; the run then ends `cancelled`, its problem saying +why. The request succeeds when the sweep finished (its schedule or a budget +spent). What each run of a sweep did is kept in the database (the tables +`sweep_runs` and `sweep_trials`), never in the folder. + +**`sweeps`** takes nothing and answers `sweeps`, each with `reference`, +`title`, `recipe`, `strategy` (its kind), `objectives`, `budget`, `state` +(`not-built`, `running`, `built` when its last run finished with the sweep and +its recipe as they are now, or `failed`) and `run`: null, or the latest run's +`id`, `state` (`running`, `finished`, `cancelled`, `failed`), `ended` +(`schedule`, `trials`, `minutes`, `cancelled`, `failed`), `started_at`, +`finished_at`, `counts` (trials by state: `running`, `completed`, `pruned`, +`stopped`, `failed`) and `best` (`number`, `reference`, `value` of the first +objective among the completed trials, or null). + +**`sweep`** takes `sweep` (its reference or name) and an optional `run` (an id; +default the latest) and answers the sweep's `reference`, `title`, +`description`, `recipe`, `parameters`, `strategy`, `objectives`, `budget` and +`state`; `changed` (the run shown ran another text of the sweep); `request` +(the newest build request naming it alone: `id`, `state`, `requested_by`, +`requested_at`, or null); `run` (as in `sweeps` with `problem`, `seconds`, +`trials` and `request_id`); `runs` (the last 20: `id`, `state`, `started_at`, +`trials`); `trials` (each `number`, `reference`, `values`, `state`, `reused`, +`run_id`, `started_at`, `finished_at`, `seconds`, `rungs` (each `rung`, `at`, +`value`, `n`, `kept`, `goes_on`), `objectives` (by metric; a running trial's +as far as it has logged), `latest` (`step`, `time`, `values`), `problem` and +`defined`); `best`; `frontier` (the numbers of the completed trials on the +frontier of two objectives, the second across); `rungs` (the rung points up to +the last reached and one more); and `axes`, the trials as `benchmark_axes` +answers a benchmark's subjects: `metrics` the objectives, `settings` the +parameters, `subjects` the trials (`subject` and `recipe` the trial's +reference, `number`, `state`, `metrics`, `settings`). + +**`map`** answers a group per sweep that has trials (`id` the sweep's +reference, `title` its title, `parent` the group of its recipe, `sweep` true), +after the folder's groups, and a trial's model gains `sweep` (the sweep's +reference) and has its sweep's group as `group` in the answer. Other groups +and models are answered as before, without `sweep`. + +## The Pumas endpoint + +Generation, suggestions and embeddings go through a Pumas endpoint, reached by HTTP +only: nothing of Pumas is part of Tuldok. The address is a setting of the +machine, `pumas_endpoint` in the settings file (see +[The project directory on this machine](#the-project-directory-on-this-machine)), +never in the folder or in a definition. Every offer and every check comes from +what the endpoint says it serves at that moment (`GET /v1/served-models` of +the Pumas 0.8.0 contract); an output Tuldok cannot keep as a record (anything +but `text` and `png_base64`) is not offered. Every call is bounded in time +and bytes as Tuldok's client bounds it and is made outside the service's lock. + +**`endpoint`** takes nothing and answers `address` (or null), `answered`, +`version` (what its `/.well-known/pumas` reports, or null), `models` and +`problem` (null when it answered). Each model is one served (model, profile): +`model`, `profile`, `profile_required`, `model_id`, `provider`, `load_state`, +`context_tokens`, `modalities` (`input`, `output`), `capabilities` (each with +`capability`, `available`, `reason`, `input_formats`, `output_formats`, +`input_modalities`, `output_modalities` and `option_bounds`, option to +`{minimum, maximum}`) and `makes`: what a `generate` can make with it now, +each `{make, capability, options (the options it takes, with their bounds), +seed (whether it takes one)}`. + +**`set_endpoint`** takes `address`, or null to clear the setting. An address +is asked once what it serves and kept only when it answered; it must be the +numeric-loopback HTTP address Pumas 0.8.0 serves on, such as +`http://127.0.0.1:8765`, without a path (refused `invalid` otherwise, and +`unavailable` when it does not answer). It answers as `endpoint` does, and +writes an activity row (`set` or `cleared`). + +`endpoint` and `set_endpoint` also answer `embedder`: the machine's +`model`, `profile` and `prefix` (the default prefix when none is set), +`named_by` (`setting` or `default`), `default` (`embeddinggemma-2`), +`chosen` (the served `model`, `profile`, `model_id` and `capabilities` it +resolves to now, or null), `problem` (why none, or null) and `embedders` +(every served model and profile with an embedding capability available: +`model`, `profile`, `model_id`, `capabilities`, `dimensions`). + +**`set_embedder`** (owner) takes `model` (an alias, or null for +EmbeddingGemma 2), `profile` and `prefix` (null for the default), keeps them +in the settings file once the endpoint serves the model with an embedding +capability (refused `not-found` otherwise), writes an activity row (`set` or +`default`), and answers `embedder`. + +**`find_endpoints`** takes nothing and answers `endpoints`, each `{address, +version, component}`, the Pumas endpoints that advertise themselves on this +machine's loopback listeners, and `rejected`, `{server_url, reason}` of those +that answered with an advertisement Tuldok refuses. + +## The project directory on this machine + +Everything a project keeps on one machine alone (installed environments, code +snapshots, run directories and products) is in its local home, +`//`. The project directory is, first that +applies: + +1. the environment variable `TULDOK_PROJECT_HOME`, when it is set and not empty; +2. `project_home` of the settings file; +3. `~/.local/share/tuldok/projects`. + +The settings file is `~/.local/share/tuldok/settings.json`, or the file the +environment variable `TULDOK_SETTINGS` names. It is strict JSON as in +[JSON rules](#json-rules), at most 64 KiB, and has no other fields: + +```json +{"format": "tuldok_settings_v1", "project_home": "/absolute/path", "pumas_endpoint": "http://127.0.0.1:8765", + "embedding_model": "nomic-embed", "embedding_prefix": "task: clustering | query: "} +``` + +`project_home` is optional and must be absolute. `pumas_endpoint` is optional, +an `http` or `https` address of at most 2,048 characters with no spaces; it is +set by `set_endpoint` (see [The Pumas endpoint](#the-pumas-endpoint)). `embedding_model` (1 to 256 +characters), `embedding_profile` (1 to 128) and `embedding_prefix` (at most +1,000) are optional and set by `set_embedder`: the embedder of an `embed` +that names none, in place of EmbeddingGemma 2, and the text put before each +record's text. `serve_most` (1 to 16, default 2) and `serve_idle_minutes` +(1 to 1,440, default 30) are optional: how many models this machine serves at +once, and the idle time of a served model whose `serve` names none. The file is read only when +the environment variable does not decide. A settings file that cannot be read +or is not valid stops the command with a message naming the file and the +fault, exit status 2, and stops the app's start-up; Tuldok never falls back to +the default then. + +| Command | Does | +| --- | --- | +| `home [--json]` | Prints the project directory, where it came from (the environment variable, the settings file or the default), the free space on its disk and how many project directories it holds. Needs no database. | +| `home --set PATH [--json]` | Saves `PATH`, which must be absolute, as `project_home`. `PATH` is made with its parents when missing and must be a directory this user can write. The settings file is written through a temporary file renamed into place, mode 0600. Says when the directory the setting or the default gave before holds project directories (they are not moved), and when `TULDOK_PROJECT_HOME` is set and so overrides the setting. | + +``` +$ .venv/bin/python -m tuldok_project home +directory /mnt/data/tuldok/projects +from the settings file /home/me/.local/share/tuldok/settings.json +free 812 GB on its disk +projects 2 +``` + +## A new project + +`new FOLDER --name NAME [--code-root PATH] [--json]` makes `FOLDER`, which must +not exist or must be an empty directory, and writes `tuldok.json` in it with +the format, a new random id, the name and `code_root` (`.` when not given). It +writes nothing else and needs no database. A name or code root that +[`tuldok.json`](#tuldokjson) does not allow is refused with exit status 2, and +nothing is made. + +## Adopting existing files + +Some files were made before Tuldok could build them and cannot be made again, +such as an earlier model's weights. `adopt` makes such files the product of a +dataset, recipe or step, so it is `built` and what uses it can be built. It is +a command-line action only, since a manifest names files anywhere on the +machine. + +The manifest is strict JSON, at most 16 MiB: + +```json +{"format": "tuldok_adoption_v1", + "products": [ + {"reference": "recipes/pilot-a", + "note": "Tuldok run acb748bc of 2026-10-21", + "files": {"weights.npz": "/absolute/path/to/the/file", + "history.json": "/absolute/path/to/another"}} + ]} +``` + +| Field | Rule | +| --- | --- | +| `products` | 1 to 5000 entries; a reference appears once | +| `reference` | a dataset, recipe or step of the project, not removed | +| `note` | optional, 1 to 400 characters; kept as the run's problem text, where a run's remark is shown | +| `files` | 1 to 200 entries. Each key is a relative path in normal form where the file goes in the product. When the definition declares outputs, the key is the `path` of one of them or a path under one; when it declares none, its product is the directory as a whole (what a step that uses it as an input is given) and any relative path is accepted. No key may have another under it. Each value is the absolute path of an existing regular file this user can read; a link is refused | + +Unknown fields are refused. Every entry is checked before anything is +adopted: one refused entry stops the command, every problem is printed, and +nothing is adopted. Declared outputs without a file are reported as missing, +and the entry is still adopted. An entry whose definition already has a +current product is skipped, unless `--replace`. + +Adopting an entry makes `/runs//output/` holding the files +at their keys, as hard links to the sources; a source on another filesystem +is copied, and `--copy` copies always. The directory is made under a temporary +name and renamed into place. Then a run is recorded in state `adopted`, with +no commit and no command, the definition's text as it is now, the note, all +three times set to now and the total size of the files; and its product, made +current, with the declared outputs that have a file and the size of each. A +product that was current before stays in the database, no longer current. No +checksum is taken, and Tuldok never writes to an adopted file. + +A definition whose current product was adopted is `built` and never out of +date: its text, settings, inputs and code are not compared. `plan` shows it as +`built` (noting `adopted, not built here`) and plans nothing it uses. A +benchmark subject whose recipe has a current product runs even when the +recipe's own label is `no` or `needs-work`. `runs` and `run` show the state +`adopted`; `products` and `run` mark the product `(adopted)`, and their JSON, +like `status` entries and the app's `runs`, `run` and `products` results, +carries `adopted: true`. A real build of the definition replaces it like any +other product. + +| Command | Does | +| --- | --- | +| `adopt FOLDER MANIFEST [--dry-run] [--copy] [--replace] [--json]` | Checks the manifest and prints what it would do: the entries to adopt, those skipped because they have a product, the declared outputs missing, the files and bytes, and whether they are linked or copied. Without `--dry-run` it then does it and prints the same, with each new run id. | + +Exit statuses: 0 done (or, with `--dry-run`, checked); 1 when an entry is +refused, the manifest is not strict JSON, or copying or linking failed (the +entries printed before the failure were adopted); 2 when the manifest cannot +be read. + +``` +$ .venv/bin/python -m tuldok_project adopt FOLDER manifest.json --dry-run +Would adopt 1 entry, skip 0 that already have a product. +1 file, 61 B: 1 to be linked, 0 copied. + +adopt datasets/numbers 1 file 61 B linked + +Nothing was changed (--dry-run). +``` + +## What the format does not do yet + +A project folder can be made with `new`, files made elsewhere can be adopted +as products, and the folder is checked, can be opened, edited and saved through the database +copy, its results can be read and drawn, its environments can be connected, +checked and installed, and its definitions can be planned, estimated and +built on this machine, one build at a time per step, from the command line +or, through build requests, by the Tuldok app that serves the folder. Tuldok +does not yet install an environment or measure estimates from the app, +delete old products, run directories or code snapshots, use `cpu_threads` or +`memory_gb` (they are shown, not enforced), or read the contents of notes. + +Background: [the design notes for this format](../plans/definition-first-projects/). diff --git a/docs/contracts/project-model-portfolio.md b/docs/contracts/project-model-portfolio.md deleted file mode 100644 index d7a0a991..00000000 --- a/docs/contracts/project-model-portfolio.md +++ /dev/null @@ -1,87 +0,0 @@ -# Authored model organization - -The project workspace can display an adopted `.portfolio.json` note as an -authored guide to model fits/components. This is separate from **Experiment -progression**, which retains genuine Study/Trial ancestry. Group edges mean -authored grouping; they do not establish derivation, admission or scientific -acceptance. Relationship notes have explicit comparison/parameter meanings and -source excerpts. Missing historical Trial ancestry stays unknown. - -## Retained document - -`project_model_portfolio_v1` is a UTF-8 JSON note of at most1MiB with exactly: - -- `format`: the fixed string above; `project_id`: the owning32hex project ID; -- `title` (1–200characters), `description` (1–8,000characters), and - `authorship`: `retrospective`; -- `groups`: at most64records with exact fields `id`, `label`, `parent`, - `description`. IDs are unique, parent is null or another group, and grouping - is acyclic; -- `models`:1–1,000records with exact fields `id`, `label`, `group_id`, `aliases`, - `review`, `card`, `summary`. Each unique ID names an authored model entry, - not necessarily a complete generator, Trial or unique parameter state; -- `relationships`: at most2,000records with exact fields `from`, `to`, `kind`, - `explanation`, `evidence_model`, `excerpt`. All model references exist, endpoints - differ, and each ordered endpoint/kind tuple is unique. - -Group/model IDs contain1–160ASCII letters/digits or `._:/-`, beginning with a -letter/digit. Group labels are at most200characters, model labels/aliases500. -Each model has at most128unique aliases. A card reference has exact fields -`file_id` (64hex), `path` (a relative `notes/` path up to512characters without -empty/dot/dot-dot components), positive integer `revision`, and64hex `sha256`. -These are exact captured note references, never source-reading capabilities. - -Review is `source_reviewed` or `review_pending`. Reviewed summaries have exactly -six nonempty strings, each up to16,384characters: `method`, `purpose`, -`experiment`, `outcome`, `lesson`, `reproduction`. Pending summaries are null. -This status is an author's source-review statement, not numerical qualification. -Relationship kind is `comparison` or `parameter_derivation`; the latter points -from derived model to parameter source and must be acyclic. Explanation is at -most8,000characters and exact excerpt at most16,384. Comparisons are authored -context, not permission to combine incompatible metrics/populations. - -## Ownership and reads - -The ordinary project-file owner captures/revises the note, authenticates reads, -verifies captured UTF-8 bytes/hashes and retains previous revisions. There is no -new canonical family, SQL table, execution grant, recipe or mutation interface. -Existing `files_inspect` and `file_get` contracts are unchanged. Save/load keeps -the original guide and referenced note history. A fork whose new project/note -identities differ requires an explicitly re-authored guide; the reader refuses -foreign identities instead of retargeting them silently. - -`ProjectModelPortfolioDocument` owns the presentation-document decoder and -cross-reference/literal checks. `ProjectModelPortfolio` owns selection, -presentation and its pending reads. Its injected `read` must be the host's -already negotiated, current-authority, byte-verified project-file reader. -The host owns transport/hash validation and context identity. The controller -checks the exact model card ID/path/revision/SHA/project, and that all six -summary strings and that card's relationship excerpts occur literally in its -captured text before displaying a selected summary. Text is inert; no markup, -scripts, parameter loading, scientific scoring or source execution occurs. - -Only explicit actions read notes/cards. Discovery visits40adopted files per -page; no hidden scan/retry/polling occurs. The guide holds at most1,000models, -shows20per graph window with their ancestor groups, and reports exact total, -reviewed and filtered counts. Group/name/alias/review filters, paging and -textual relationship controls select the same entries as graph buttons. -Paragraphs wrap; exact source text remains in a disclosure. - -The host's binding/project/checkpoint/credential/generation context owns every -read. A context change, permission loss or teardown aborts pending work, clears -old content and restores focus to the host status when needed. Late replies -cannot publish into another context. Errors show their declared diagnostic and -clear old detail instead of displaying a previous model as the new selection. -Unsupported format or foreign-project binding is `unsupported`; malformed -document/reference/summary is `invalid`; transport/availability failures retain -their owning diagnostic. No success or empty-data fallback substitutes for a -failed read. - -## Verification scope - -Pure codec tests cover contract decisions using literal fixtures. Controlled -read and real-browser tests cover selection, unavailable/obsolete responses, -repeated invocation, focus, keyboard activation and wrapping on desktop/mobile. -Those fixtures do not establish real project capture or save/load. Native -acceptance additionally requires an actual adopted guide, exact original cards, -current authority, real project reads and retained checkpoint/custody evidence. diff --git a/docs/contracts/pumas-image-to-text-consumer.md b/docs/contracts/pumas-image-to-text-consumer.md index a6fa3697..3d54eef9 100644 --- a/docs/contracts/pumas-image-to-text-consumer.md +++ b/docs/contracts/pumas-image-to-text-consumer.md @@ -1,22 +1,64 @@ # Pumas Image→Text consumer -This opt-in caption path consumes the contract at Pumas Library merged PR68, -source `36c2aab2c885b8416fd4e39cf8c37c510596d6e4`: -[Image→Text contract](https://github.com/MrScripty/Pumas-Library/blob/36c2aab2c885b8416fd4e39cf8c37c510596d6e4/docs/contracts/image-to-text-v0.8.md). -It is based on Tuldok main `d99e23d2b0d2783a7a9863232a7453cd9f2db7aa`. -The legacy compatible caption path and the previous typed text/image operations -remain available. Their existing producer receipt is not silently repinned. +This opt-in caption path, and the typed text and image operations that share +`pumas_operations`, consume the typed HTTP contract released in Pumas Library +0.8.0, tag `v0.8.0`, commit `69e32a2208c5c2d840f4adafec03350669fcb5bc`: +[model operations, contract version 1](https://github.com/MrScripty/Pumas-Library/blob/v0.8.0/docs/contracts/model-operations-v1.md). +Pumas is reached only as an HTTP endpoint; no Pumas code is imported or started. +It supersedes the pre-release pin `4c79cd8e35db566b70caaa13d8d8f92265f5976e` +(and before it PR68), which `source.json` keeps as `previous_producer`. The +legacy compatible caption path remains available. The PR54 typed text/image +capture receipt in `tests/fixtures/pumas-typed-v1` keeps its gateway harness; +its capability files were brought to the 0.8.0 descriptor shape. -`pumas_operations.validate_manifest` accepts all seven closed capability variants, -the three additional image input formats and image byte/pixel/count option names. -The existing unavailable reasons remain unchanged. Availability is a live -observation, not a model qualification receipt. Audio's existing -`max_output_tokens` descriptor option is accepted as declared in the inspected DTO. +### Additive compatibility + +Pumas responses grow additively within contract version 1. Tuldok validates +only what it uses and selects capabilities and option bounds by name, never by +position or count: + +- `GET /v1/capabilities` requires `supported_contract_versions` containing 1, + `model`, an exact `profile`, positive `max_request_bytes`/`max_response_bytes` + and a `capabilities` list. Other fields (`context_tokens`, + `max_stream_event_bytes`, future fields) are retained but not required. +- Only `chat_generation`, `image_generation` and `image_to_text` descriptors are + validated: fixed semantic task, string format lists, an availability `state` + with an optional string reason, and finite, ordered bounds for the options + Tuldok compares (`max_tokens`, `temperature`, `top_p`, `width`, `height`, + `seed`, `input_characters`, `image_bytes`, `total_image_bytes`, `image_pixels`, + `image_count`, `message_count`, `content_parts`). A used capability or + option advertised twice is refused as ambiguous. +- Other capabilities (`multimodal_embedding`, `decision`, `rerank`, audio, future + values), formats (`content_batch`, `decision`, `decisions`, `relevance_scores`, + future values), modalities, option names, + availability states and reasons, and response or result fields are ignored. + Any availability other than `available` means unavailable, and its reason is + shown. +- The capability observation is bounded at 1 MiB. `GET /v1/served-models` is + preferred for listing serving aliases with their profiles; gateways without it + (HTTP 404) fall back to `/v1/models`, which lists each ID once with `profiles`. +- `pumas_operations.served_models(address)` reads `GET /v1/served-models` in + full, independent of the catalog: each served (model, profile) with + `profile_required`, model ID, provider, load state, context size, input and + output modalities and its descriptors with availability, bounded at 8 MiB and + 4096 entries. Values outside the 0.8.0 vocabulary are dropped; a malformed + entry refuses the listing, naming the entry. `available_capabilities(entry, + input_kind, output)` answers which available capabilities make an output from + a request input kind, as the release's resolver matches input formats. + +Availability is a live observation, not a model qualification receipt. Choose **Pumas typed v1** in the image caption form, select the serving alias and optional exact profile, and inspect capabilities. On each new attempt, the owner requires `pumas.model-operations.image-to-text:1` in the gateway build advertisement -and an available Image→Text descriptor for the exact resolved model/profile. +and an available Image→Text descriptor for the exact resolved model/profile. The +gateway advertisement (`GET /.well-known/pumas`) has its own versioned decoder, +which also grows additively: unknown fields and new protocol, schema, capability or +compiled-feature entries are ignored and are not retained in the stored build +observation, which keeps only the known fields. Missing required fields, wrong +types, unsupported schema versions and malformed endpoints are refused with a +reason naming the field; a gateway scan lists such refused advertisements with +their reasons instead of hiding them. The request uses the facade's `messages`, `semantic_task:"image_to_text"`, text-generation options and `stream:false`. The seed is omitted. Named image and `image_messages` inputs are also supported by the bounded consumer request builder. @@ -25,8 +67,10 @@ messages can contain images; audio, URLs, paths and data-URL envelopes are refus The consumer applies the producer's 32 MiB whole-request, 8 MiB per-image, 16 MiB aggregate, four-image, 4096-axis, 4,194,304-pixel, 128-message and 128-total-part -bounds. Tokens default to 512 and permit 1–2048; temperature/top-p must be finite -and within 0–2/0–1. The caption application's existing preparation is smaller: +bounds. Tokens default to 512. Pumas no longer caps Image→Text `max_tokens` at +2048; the live `max_tokens` bound follows the served context window and is +checked before admission. Temperature/top-p must be finite and within 0–2/0–1. +Advertised image byte, aggregate byte, pixel, count, dimension, message and part bounds are checked when present. The caption application's existing preparation is smaller: one RGB JPEG, at most 2 MiB and 1600 pixels on its longest side. It decodes the captured, hash-verified canonical image and retains both canonical-source and prepared-input hashes. Preparation is explicit in request history. @@ -37,8 +81,8 @@ replace Pumas's production PNG/turbojpeg validation, allocation controls or admission authority, and does not claim equivalent JPEG warning detection. No local validation result is used as permission to bypass provider refusals. -The response is the closed `{contract_version, request_id, result}` envelope. -The request ID must match; `result.kind` must be text. There is no response-model +The response is the `{contract_version, request_id, result}` envelope; added +fields are ignored. The request ID must match; `result.kind` must be text. There is no response-model requirement and no private provider envelope projection. This application requires a complete `stop` finish reason and valid caption JSON within a 256 KiB response bound; partial or filtered output is not applied. Capability/build observations, @@ -47,8 +91,17 @@ on the exact job. List responses omit image-bearing canonical requests and raw bytes. Explicit application creates a draft target and records its evidence in revision history; human review remains separate. -HTTP 400/413/422/503 pre-admission refusals and 502 failures retain their typed -code, HTTP status and `outcome` in the asynchronous job. A disconnect, uncertain +Every typed error is `{contract_version, request_id, error:{code, outcome, message}}`. +A refusal made before the request reaches its handler is `{"error":{"code","message"}}` +(for example `request_limit`, HTTP 413); it is decoded as a typed `not_admitted` refusal. +`error.outcome` is authoritative: `not_admitted` records no provider effect and +anything else is `unknown`, whatever the HTTP status. 404 `model_not_found` and +409 `ambiguous_model`/`ambiguous_operation` are ordinary refusals. An error whose +`request_id` is `null` (Pumas could not read the body) is still a typed refusal. +Unknown error codes are kept verbatim. The job retains the code, HTTP status and +outcome, and its error text includes Pumas' `error.message`, bounded and on one +line. Capability refusals show the same message. On the OpenAI-compatible routes, +a `pumas_error` body's `error.message` is shown after the existing redaction. A disconnect, uncertain response or possibly dispatched interrupted attempt remains `unknown`. Repeating an existing exact application request ID returns the retained attempt; it never replays inference. A new attempt requires a new explicit request. Browser recovery @@ -64,28 +117,30 @@ and packaged application acceptance remain integration-lead/provider-owner gates ## Evidence and coordinated review `tests/fixtures/pumas-image-to-text-v1/source.json` labels the capability/wire -fixture as **source-derived**, with inspected production-source hashes. It is not -a captured native DTO schema export or an actual model reply. The tests exercise -seven-capability parsing, named/facade PNG/JPEG inputs (including progressive +fixture as **release-derived**. It is not a captured native DTO schema export or +an actual model reply. The fixtures (`controlled-capabilities.json`, +`served-models.json`) follow the 0.8.0 descriptor construction for a llama.cpp +vision serving with an 8192-token context and a Torch image serving, including +modalities and the `multimodal_embedding`, `decision` and `rerank` descriptors. The tests exercise additive capability parsing, named/facade PNG/JPEG inputs (including progressive JPEG), order, closed shapes, finite/resource bounds, typed application captions, draft/history, live selection refusal, cancellation/borrowed-owner survival, HTTP outcomes, restart and no replay. The existing typed browser suite additionally checks the actual caption form, capability inspection, application and disconnect. -The six actual production DTO schemas regenerated from the combined source -are retained in `tests/fixtures/pumas-image-to-text-v1/schemas/`, with SHA-256 -identities in `source.json`. The source-derived capability fixture and authored -named/facade image requests and typed responses validate against these exports. -The schemas describe wire shape; finite bounds and live readiness remain runtime -checks. Neither fixture output nor schema validity proves native inference. +The schemas in `tests/fixtures/pumas-image-to-text-v1/schemas/` have SHA-256 +identities and origins in `source.json`. `capability-descriptor.schema.json` is +the copy the 0.8.0 release ships; `capabilities-response` and +`served-models-response` are derived from it and from `types.rs` at the tag; the +request, response and error schemas remain the pre-release export, as the +release's changes to them are additive and Pumas is not built here. The +capability fixtures and authored named/facade image requests and typed +responses validate against these schemas in the vision tests. The schemas +describe wire shape; finite bounds and live readiness remain runtime checks. +Neither fixture output nor schema validity proves native inference. -The reviewed producer retains identical DTO definitions and schema dependencies; -all six exported schema SHA-256 identities are unchanged. `source.json` records -the export commit and refreshed handler/controlled-test source hashes. The fixture -gateway advertisement reads the same versioned source pin as its consumer -acceptance evidence. -This pin is the immutable merged Pumas main commit. Its source tree is identical -to the reviewed producer that passed CI before the coordinated merge. +`source.json` records the tag, commit, tree and contract document hash. The +fixture gateway advertisement reads +the same versioned source pin as its consumer acceptance evidence. The combined desktop contract and packaged application are provider integration gates. Real model/projector inference, semantic quality, managed native teardown and production audio qualification remain unclaimed. No models or runtimes are diff --git a/docs/contracts/scientific-fields.md b/docs/contracts/scientific-fields.md index cf31a01a..a0b61374 100644 --- a/docs/contracts/scientific-fields.md +++ b/docs/contracts/scientific-fields.md @@ -159,38 +159,6 @@ project import and slice inspection. Native producer observations and independen NumPy qualification are recorded separately; these tests are not physics, training, or model acceptance. -## Native external-project inspection - -`project_files.data_scientific/1` reads the same bounded numeric slice through -the native project session. It requires the current `binding`, canonical -`checkpoint`, `record_id` and exact `content_hash`; `frame`, `field`, `axis` and -`index` retain the existing inspector's selection grammar. ProjectData checks -current inspect authority, the project session and source identity before using -ScientificAssets. The scientific owner verifies retained artifact bytes and -metadata. Stale bindings/checkpoints, changed content, missing/corrupt artifacts -and revoked credentials refuse; no source-path or cached-array fallback runs. -The read changes no project records and grants no execution authority. - -The native project explorer mounts the same `ScientificFieldInspector` used by -the workbench. That module owns slice validation, numeric probing, display-only -grayscale rendering and cancellation. Each page supplies its authoritative -selection predicate and slice reader; the workbench separately owns its source -download and review-note editor. The native project reader uses negotiated -project operations rather than the legacy workbench URLs. Native project -annotation stays with its existing data-note owner. - -Changing the selected record, project/checkpoint or session clears the native -view and aborts pending reads. A superseded reply cannot repaint it. Page teardown -also aborts reads. Hidden or loading canvases do not display old pixels. Field -units, producer stage, original clock and grid coordinates remain explicit; -opening a scientific data record does not qualify its physical meaning or turn -an authored diagnostic into an accepted model result. - -`test_project_sources` covers exact content and cold projection reads without -the original source root. `test_project_files_m2_http` covers real selected -transport, stale/refused reads and artifact corruption. The existing -`browser_scientific_fields.cjs` exercises the shared module through the original -workbench with its browser sandbox enabled. Lab qualification additionally -inspects unchanged retained pressure fields through native project operations -and independent relocated cold Open. Scalar Run comparisons and Experiment -progression remain separately owned interfaces. +The existing `browser_scientific_fields.cjs` exercises the inspector module through +the workbench with its browser sandbox enabled. Scalar Run comparisons and +Experiment progression remain separately owned interfaces. diff --git a/docs/plans/definition-first-projects/capability-check.md b/docs/plans/definition-first-projects/capability-check.md new file mode 100644 index 00000000..e025e220 --- /dev/null +++ b/docs/plans/definition-first-projects/capability-check.md @@ -0,0 +1,115 @@ +# Capability check: what deleting the old project mode removes + +**Status: written 2026-11-12 for the owner's sign-off, and brought up to date the same evening after the Map workspace was built and the owner had looked at it. Signed in part on 2026-11-13, and the deletion landed on this branch that day: see "Sign-off" at the end. Three things the owner did not give up are requirements from here on.** + +The [plan](plan.md) deletes the old project-files mode in milestone 8 on one condition: a written check that lists every capability of that mode as covered, moved or deliberately dropped, signed off by the owner. This is that check. It is a list of what Tuldok can do today through `python app.py --project-files-root ...` and what happens to each thing when that mode is gone. + +## The short answer + +For what the lab uses a project for (definitions, runs, results, charts, environments), the new format covers it, and since 2026-11-12 it has its own interface: the Map workspace ([plan](workspace-plan.md)), with the tabs Map, Data, Train, Benchmarks, Analysis and Overview, from which builds are planned, requested and approved and definitions are edited, branched and saved. The owner looked at it on the lab project that day ("looks ok. continue") and reported one fault, a full redraw where the page should update in place, which [spec-w4.md](spec-w4.md) repairs. + +Five things are not covered and would be gone from projects: + +1. **Annotated data inside a project.** Records, imports, annotation, review, saved searches, selections and frozen releases exist in the new format nowhere. They stay available in catalog mode, which this plan does not change. A project folder cannot hold them until the annotation plan is written. +2. **Looking inside the data from the project page.** The old page lists a project's records. The Map workspace shows each dataset as a card with its state and size; the record browser, the volume viewer, field slices and the Annotate tab of the mockup are not built (the workspace plan left them out, for a later plan). +3. **Studies, trials and candidate selection** as records Tuldok keeps. A trial becomes a recipe; the grouping lives in labels and notes. +4. **Producers and the built-in training graphs inside a project** (image generation, grounded review, caption and label proposals; the image, text and audio classifier trainings). Catalog mode keeps them. +5. **Reading a run's files through the agent interface.** An agent can read a run's record, its progress and the end of its logs, but not fetch its weights or other outputs; those are files on the same machine. + +Two more follow from the deletion itself: the lab's old project folder (`projects/rheon-model-portfolio-unified-04`, still on this disk, no longer tracked) can no longer be opened by Tuldok, and forks and composing of projects go (a Git branch is a fork). The old code stays in Git history. Nothing in the database is dropped: the old project's working copy stays in PostgreSQL, unread. + +## What would be deleted + +| What | Size | +| --- | --- | +| 37 Python modules that only the old mode reaches | 20,437 lines | +| The old project page and the extraction page, with their scripts and styles under `static/` | 3,264 lines in 11 files | +| 145 `project_files.*` and 5 `project_transfer.*` operations in `agent_api.py`, the `project` capability profile, the two owner clients and their `mcp_stdio.py` flags | | +| The old mode's tests: 72 Python files and 18 JavaScript files | 22,675 lines | +| The documentation of the old mode under `docs/contracts/`, the runbooks and the READMEs. Its plans and ledgers under `docs/plans/` stay, as the record | | + +The 37 modules: `project_artifacts`, `project_composition`, `project_custody`, `project_data`, `project_data_fork`, `project_dispatch`, `project_execution`, `project_extraction`, `project_files`, `project_fork`, `project_fork_composition`, `project_history`, `project_locks`, `project_material`, `project_material_fork`, `project_measurements`, `project_native_execution`, `project_native_preparation`, `project_native_process`, `project_producer_execution`, `project_producers`, `project_projection`, `project_record_dedupe`, `project_runtime`, `project_session`, `project_sources`, `project_store`, `project_study_execution`, `project_text`, `project_transfer`, `project_transfer_controller`, `project_validation_cache`, `project_validation_delta`, `project_working_store`, `project_workspace`, `retained_measurements`, `native_process_bootstrap`. + +`project_cli.py` was listed here as a 38th module until the deletion was prepared. It stays: it is also catalog mode's command line for agents (`open`, `inspect`, `preflight`, `submit`, `status`, `download` and the rest), and tests that stay use it. It loses only the old mode's commands, `file-*` and `project-transfer`, and the flag `--owner-project-files`. + +**What stays:** every remaining operation's contract, unchanged; catalog mode and everything it uses, including `projects.py` and `project_activity.py` (catalog mode's read-only inspection of project snapshots with `--project-root`), the access layer, the agent registry and clients, and the new `tuldok_project` package. + +**Where the old code is tied to catalog mode.** Twelve import statements in three files reach into the 37 modules, so they change before anything is deleted: + +| File | What it imports | What happens | +| --- | --- | --- | +| `app.py` | `project_workspace`, `project_transfer_controller`, `project_transfer.stop_source`, `project_store.ProjectStoreError` | the old mode's start-up flags, routes and error branches are removed | +| `candidate_packages.py` | `ProjectHistory`, `ProjectExecution`, for two `isinstance` branches | the two branches are removed | +| `pipeline.py` | `preparation_owner` (twice), `native_admission_wall_seconds` | the project-only branches are removed; the one small function catalog mode also uses moves into `pipeline.py` | + +## Every capability + +The old mode has 150 operations. They are grouped here by what they do; each group says what replaces it. + +| Group | Operations | What it did | After deletion | +| --- | --- | --- | --- | +| Opening, saving and files on disk | 11: `status`, `new`, `open`, `save`, `close`, `files_inspect`, `file_get`, `file_history`, `file_adopt`, `files_adopt`, `file_adopt_deletion` | Create, open, save and close a project; see and adopt files changed on disk; a file's history | **Covered**, differently. `open`, `save`, `status`, `check`, `resolve_conflict`. Files changed on disk are imported on the next operation. Nothing stays open, so there is no close. History is Git's. A new folder is made by `new` | +| Forks and composing | 3: `save_as`, `compose`, `fork_info` | Copy a project to an independent fork; merge retained checkpoints of several projects | **Dropped**, as the plan decided: a Git branch is a fork | +| Data records and annotation | 30: `query`, `get`, `import_text`, `annotate`, the `data_*` reads and imports for text, image, audio, mesh, point cloud, volume, sequence and scientific fields, `data_annotate`, `data_rights`, responses and preferences, `source_register`, `source_references` | Hold data records in a project, annotate and review them, record rights, responses and preference judgments | **Not covered. Catalog mode only.** The plan put annotation data out of scope; it needs its own plan before a project folder can hold it | +| Selections, searches, releases | 11: `selections`, `selection_*`, `searches`, `search_*`, `release_preview`, `release_freeze` | Saved selections and searches over records; freezing a dataset release with its splits | **Not covered. Catalog mode only**, with the group above | +| Definitions | 27: `definitions`, `models`, `model_*`, `graphs`, `graph_*`, `recipes`, `recipe_*`, the material reads, `material_intent*`, `dependencies`, `policies`, `policy` | Model drafts, node graphs and recipes with revisions; immutable captured copies ("materials") of each; publishing a model | **Covered**, differently: seven kinds of plain definition file, `definitions`, `definition`, `graph`, `set_definition`, `remove_definition`, `save`. No revisions and no captured copies; Git has the history. **Not covered:** the built-in node graphs (image, text and audio classifier training), which remain in catalog mode | +| Studies, trials, candidates, lineage | 17: `studies`, `study`, `study_create`, `study_save`, `trial`, `trial_prepare`, `candidate`, `candidate_select`, `lineage`, `lineage_record`, `progression`, `progression_origins`, `snapshot`, `evidence`, `package_state`, `publication_result`, `expose_holdout` | A study of trials with a budget; choosing a candidate; recorded lineage between runs; the progression view; recording that held-out test results were looked at | **Partly covered.** Lineage is the graph's edges; progression is a chart form ordered by `trained_at`; evidence is a recipe's card and its results. **Dropped:** studies, trials and budgets as records, candidate selection, snapshots, and the record of held-out exposure. The lab's 205 earlier models keep their study and trial origin as labels and cards | +| Runs and execution | 24: `run`, `runs`, `run_document_segment`, `artifacts`, `artifact_status`, `artifact_resolve`, `execution_health`, the external preflight, approve, submit and cancel, the `execution_study_*` operations, `execution_reconcile_current`, `native_*` | Preflight, owner approval, submission, cancel, retry and recovery of runs; reading a run's documents and artifacts in pieces | **Covered:** `plan`, `request_build`, `approve_build`, `decline_build`, `cancel_build`, `build_requests`, `runs`, `run`, `progress`, `products`, `cancel_run`, `stop_run`. A retry is a new request. **Not covered:** reading a run's output files through the interface | +| Producers | 8: `producers`, `producer`, `execution_producer_*`, `execution_generation_resume`, `execution_grounded_review` | Image generation, grounded review and caption or label proposals run inside a project | **Not covered. Catalog mode only** | +| Measurements and analysis | 9: `measurement_*`, `retained_analysis`, `retained_analysis_export` | Import result sets, query and compare them, inspect one observation with its evidence, export an analysis | **Covered:** `import_values`, `results_overview`, `results`, `values`, `chart` (ranking, tradeoff, progression, comparison), `report`. Summaries are rows in the database, not documents re-read per request. On the page: the Benchmarks tab (accuracy against time with the frontier, error along the sequence, a compare set, the table of every fit) and the per-case values of the Analysis tab. **Not covered:** the pointer from one observation back to its evidence file | +| Runtimes | 5: `runtime_sources`, `runtime_inspect`, `runtime_capture`, `runtime_status`, `runtime_abandon` | Capture a copy of a Python environment with a listing of its files | **Covered**, differently: `environments`, `environment`, `connect_environment`, `check_environment`, and `set-up` on the command line. Versions are checked; nothing is copied or listed per file | +| Extraction from a catalog | 5: `project_transfer.*` | Stop a legacy catalog and extract it into a project | **Dropped.** The one project that needed converting is the lab, which its own script converts | + +Outside the operations: + +| Capability | After deletion | +| --- | --- | +| The old mode's commands in `project_cli.py` (`file-*`, `project-transfer`) | Gone. A project folder has `python -m tuldok_project`. The rest of `project_cli.py` is catalog mode's and stays | +| Opening the lab's old folder (`projects/rheon-model-portfolio-unified-04`) in Tuldok | Gone. The lab's converter reads that folder's files directly, and the old code stays in Git history | +| The old project page (`/project-files`), about twenty stacked form panels | Gone. The Map workspace is the project's interface | +| Byte-identical save, receipts, custody and checkpoint proofs | Dropped, as the plan decided | + +## Before deletion, in milestone 8 + +These come first, so the deletion removes nothing the lab still needs: + +1. **Done.** The lab's earlier weights and prepared inputs are adopted as products into Tuldok's per-project directory (owner decision, 2026-11-12: they stay on this machine and never go into Git): 203 recipes' outputs and 25 datasets' training data, 16.9 GB. +2. **Done.** The lab repository holds the new folder only, and the clean-install test of the plan's summary passed on fresh clones. +3. **Done.** A `new` command exists. +4. **Done.** The Map workspace is built in three phases and the owner has looked at each. +5. **Part of the deletion.** The twelve import statements above are cut and catalog mode's tests pass. + +## The deletion as prepared + +Prepared 2026-11-12 on the branch `chore/retire-old-project-mode`, in a working tree of its own, and brought onto this branch on 2026-11-13. What it changes file by file, and each check with its result, is section 10.6 of [spec-m8.md](spec-m8.md). In short: + +| | | +| --- | --- | +| Removed | 143 files, 47,099 lines: 37 modules, 72 Python and 18 JavaScript test files, 11 files under `static/`, one script, four documents. 2,450 more lines are cut from 17 files that stay | +| Contracts | Every operation that remains is published byte for byte as before, in the `catalog` and the `project_folder` profile. The catalog's list goes from 115 operations to 110: the five `project_transfer.*` entries, which it listed as unavailable. The `project` profile is refused. The 145 `project_files.*` and 5 `project_transfer.*` operations are offered nowhere | +| Python tests | 3,149 before; 2,397 after, which is 3,149 less the 751 tests of the deleted files and one test of a deleted client class. No test that stays is newly skipped. One test that stays had to change: it pinned the digest of the whole catalog list and of the `project` profile, so it now pins the list without the five entries and checks that the old profile is refused. 23 tests fail before and after alike, none of them from the change: this machine has no training environment (numpy, scikit-learn), the test database's user may not create databases, and three tests of the CPU snapshot refuse this machine's Python directories | +| Project folder tests | 585 before, 586 after (the new test of the refused profile), all passing | +| JavaScript | All 49 remaining `tests/test_*.cjs` pass; `node --check` passes on all 54 scripts under `static/` | +| Browser | The workspace checks pass on their fixture and on the lab folder, read only. Of the 62 catalog checks the workflow runs, 49 ran here and 45 pass. One fails in its own clean-up of a temporary directory, the same way on the unchanged code; three need packages only the workflow installs. 13 were not run: they need training, a GPU or generated fixtures | +| Start-up | `--help` shows none of the removed flags and each is an argument error; Tuldok starts in catalog mode on an empty data directory and with `--project-folder` on the tiny example | + +**Left for a later change.** About thirty lines in ten of catalog mode's files (`runs.py`, `synthetic.py`, the proposal modules and others) still ask the catalog for an attribute only the deleted modules set (`project_execution`, `project_sources`, `project_asset`, `project_availability`, `project_study_execution`). They can no longer be reached. They run through the code that starts and recovers runs, so removing them is its own reviewed change. `docs/decisions/external-project-authority.md`, the decision record of the old mode, stays as a record. + +## Sign-off + +The owner signs by saying so; the date and the words are recorded here and in the ledger. + +**Signed in part, 2026-11-13.** Asked what projects lose, and told item by item what each thing is, the owner answered: + +> We still want tools for generating datasets and annotating them using pumas library v0.8.0 in tuldok. Tuldok also needs to be able to look inside data. It would be great if tuldok also had a layer visulization tool so we can visualize a models weights. + +and then: + +> the other stuff is all ok to go ahead on + +- [x] **Accepted.** Projects lose studies and trials as records, candidate selection, the built-in training graphs, reading run files through the interface, forks and composing, and opening the lab's old folder, as listed above, and the old project mode may be deleted. +- [ ] **Not given up; requirements from here on.** Tools for generating datasets and annotating them, through Pumas Library 0.8.0, and looking inside data. The sentence first written here had the owner accept that projects lose them; the owner did not. + +**Why the deletion landed all the same.** It takes neither of those out of Tuldok. Generation (images, grounded candidates, caption and label proposals), annotation and the data viewers (volumes, scientific fields, point clouds, meshes, audio, sequences) are catalog mode's, and catalog mode is unchanged: the table above says "Catalog mode only" for each. What went is the old format's own wrapping of them, which worked only on a folder of the old format through the old page, and could not serve a project folder. Before and after the deletion alike, a project folder has none of them. So they are to be built for project folders, on the new format, under a plan of their own, together with the view of a model's weights layer by layer that the owner asked for. Pumas Library 0.8.0 is not yet spoken by this branch: the work for it is on `feat/pumas-v080-api`, four commits not merged here. + +**Landed 2026-11-13:** `feat/definition-first-projects` fast-forwarded to the side branch's one commit. Nothing is pushed. If the owner meant the old format's own tools to stay until the new ones exist, reverting that one commit brings all of it back. diff --git a/docs/plans/definition-first-projects/data-tools-plan.md b/docs/plans/definition-first-projects/data-tools-plan.md new file mode 100644 index 00000000..acd7e77b --- /dev/null +++ b/docs/plans/definition-first-projects/data-tools-plan.md @@ -0,0 +1,98 @@ +# Plan: looking inside data, weights by layer, and generating and annotating datasets, for project folders + +**Status: approved by the owner on 2026-11-13, with the changes under "The owner's answer" below.** + +**Built:** phases 1 to 5 on 2026-11-13 and phase 6 on 2026-11-18, on `feat/definition-first-projects`, not pushed; the specs `spec-d1.md` to `spec-d6.md` say what was built and what was not verified, and the [ledger](execution-ledger.md) has the record. **Not run:** anything against a real Pumas endpoint, which waits for the owner's word (decision 3). + +## The owner's answer + +> we will use the pumas endpoint. We wont be embedding pumas. What do you mean by lables? Right now we just need the basic modality support for data generation and annotation. text, image, audio, 3D (point clouds, mesh). We dont have a specific need for the surrogate lab. This is more a setup so other projects can setup what they need to generate. The rest sounds good. You can go ahead + +What that changes in the text below, which is otherwise as proposed: + +- **Pumas is an endpoint.** Tuldok calls a running Pumas gateway over HTTP and nothing else: no Pumas code is imported, bundled or started by Tuldok. +- **"Label" is replaced by "annotation":** whatever a person records about one item of a dataset. A class, tags, a caption or transcript, boxes on an image, spans in a text, timed segments in audio, or accept and reject with a note. +- **The scope is general, not the lab's.** Five kinds of record are supported from the start, each of which can be held in a dataset, looked at, annotated and imported: text, image, audio, point cloud and mesh. Arrays (the lab's fields and volumes) are a sixth kind for looking only. +- **What can be generated through the endpoint is what the endpoint declares.** Pumas 0.8.0 produces text and PNG images; its contract reserves audio output and has nothing for 3D. So text and image datasets are generated through Pumas; audio and 3D datasets come from import or from a project's own generator command, run as an ordinary build, until a served model declares that output. Tuldok offers whatever the gateway's served models declare, so nothing changes in Tuldok when Pumas gains one. +- **Decisions:** 1 as recommended (annotations are text in Git); 2 answered above; 3 and 4 as recommended. +- **The comparison of fields in the Analysis tab moves to the end** (phase 6): it serves only the lab. + +## Summary + +On 2026-11-13, signing the [capability check](capability-check.md) in part, the owner kept three things as requirements: + +> We still want tools for generating datasets and annotating them using pumas library v0.8.0 in tuldok. Tuldok also needs to be able to look inside data. It would be great if tuldok also had a layer visulization tool so we can visualize a models weights. + +Catalog mode has generation, annotation and data viewers, and keeps them. A project folder has none: its Data tab shows a card per dataset with file names and sizes, and nothing in Tuldok can open a product file. The [workspace plan](workspace-plan.md) left the record browser, the volume viewer, field slices and the Annotate tab "for a later plan". This is that plan, with a view of a model's weights added. + +**The owner's rule for what goes in Git** (same day): the folder holds enough to understand the project, see how it performed and rebuild it, and no data, binaries or anything easily calculated. Applied here: + +| Thing | Where | Why | +| --- | --- | --- | +| An annotation, a review decision, a note a person made | Git, as text | It cannot be calculated again | +| What to generate: model, prompts or their source, seed, count | Git, in a dataset definition | It is how the dataset is rebuilt | +| Generated images, text, audio; imported media | A product, outside Git | Data | +| A model's suggestion not yet accepted | Tuldok's database | Calculable again; becomes an annotation only when a person accepts it | +| Slices, thumbnails, histograms, layer statistics | Nowhere; computed when looked at | Easily calculated | +| The address of the Pumas gateway | The machine's settings | It differs per machine | + +**The test:** on the lab folder, open a dataset, page through its samples and slice through a field; open a model and see its layers and what training changed in them. On the tiny example, annotate ten records of each kind, save, and read the annotations in `git diff`; generate a small dataset through Pumas, review it, freeze it with splits, and plan a model that trains on it. + +## What exists and what is missing + +| Wanted | Exists and is reused | Missing | +| --- | --- | --- | +| Look inside data | The slice inspector of catalog mode (`static/scientific-inspector.js`), the one viewer written to be mounted on another page; catalog mode's array readers, which use the standard library only | An operation that reads inside a product. The lab's data is `.npz` files of 32- and 64-bit float arrays, some inside a 9.8 GB zip, compressed. The record grid, thumbnails and stacked-slice volume exist only in the mockup | +| Weights by layer | The same array reading. Every earlier model's `weights.npz` is plain 32-bit arrays named by layer (`encoders.0.layers.0.weight`), and each run also keeps `initial-weights.npz` | The view. Nothing describes a model's architecture, so layers are grouped by name. New recipes write `model.pt`, which cannot be opened safely without PyTorch | +| Generate with Pumas | The code that discovers a gateway and calls it (`gateway_discovery`, `pumas_operations`, `image_generation`, `pumas_vision`, the request and decode functions of the proposal modules): plain functions, usable outside catalog mode | Running one as a build that makes a dataset product. Pumas 0.8.0, released today: the client for its contract is on `feat/pumas-v080-api` (three commits of 2 November, not merged here; a trial merge is clean) and predates the release | +| Annotate | The tasks and review states of catalog mode as the model; the Annotate screen of the mockup as the layout | Annotation as files. Catalog mode keeps every annotation in PostgreSQL and writes plain files only inside a frozen release | + +## What has to be added to the format + +1. **Records.** A dataset's product is a set of records. It says so itself with a `records.jsonl` at its root: one line per record with its id, its kind (text, image, audio, point cloud, mesh, array), its file inside the product or its text, and what is known of where it came from. A product without one is read by an optional `view` on the dataset definition (which files are records, of which kind, which array to show first), and without that every file is a record of the kind its extension says. +2. **Annotation tasks.** A new kind of definition, `annotation`: the dataset it is on, the task and its vocabulary. Tasks: accept or reject with a note, one class, tags, and a caption or transcript, on every kind of record; boxes on images; spans in text; timed segments in audio. +3. **Annotations.** `annotations/.jsonl`, one line per record: the record's id and the hash of its content, the annotation, the review state (draft, reviewed, needs work, rejected), who and when, and where it came from when a model suggested it. Sorted, so a diff shows what changed. A record that no longer exists after a rebuild is reported, never dropped. +4. **Generated, imported and frozen datasets.** No new kind. A dataset's `run` names a step Tuldok itself provides: `generate` (what the table above lists), `import` (files on this machine, their kind and rights, hard-linked into a product) or `freeze` (a dataset, a task, a seed and the split shares). Both go through plan, request, approval and progress like any build, and their output is a product. + +## What has to be added to Tuldok + +- **Reading products.** Operations to list what a product holds (files, archive members, arrays with shape and type), to read one two-dimensional slice of an array, and to summarise an array (range, mean, spread, histogram). Each read is bounded in bytes and values; the page never receives a whole array. +- **Data tab.** A dataset opens to its records as a paged grid, and a record opens in a viewer for its kind: the text; the image; the audio with its waveform; the point cloud or mesh, turned by dragging; an array as a slice with field, axis and index and as the stacked-slice volume of the mockup. +- **A model's drawer: Weights.** Layers by name with shape and parameter count; for each, the distribution of its values and its kernels or matrix as small images; against the weights it started from and against the model it derives from, how much each layer changed. +- **Pumas.** The 0.8.0 client brought in and checked against the released contract; served models discovered from the gateway instead of typed in. +- **Annotate tab.** The mockup's screen: dataset and task, filter by state with progress, the record with its viewer, the form, keys for next and previous. With a gateway connected, "Suggest" asks a served model and shows its answer for a person to accept or change. +- **The two steps,** `generate` and `freeze`, and a Generate and a Release panel on the Data tab that write the definition for them. + +## Phases + +| # | Phase | Size | Done when | +| --- | --- | --- | --- | +| 1 | Reading products; the Data tab's record grid and the viewers for text, image, audio, point cloud, mesh and array | Medium | A product holding each kind is paged through and every kind is shown; on the lab folder a sample of the 9.8 GB training view opens in under two seconds | +| 2 | Weights in a model's drawer | Small | For an earlier lab model the layers, their distributions and the change from its starting weights are shown; `.npz` and `.safetensors` are read, `model.pt` says why it is not | +| 3 | The Pumas 0.8.0 client | Small | The branch is merged, its tests pass against fakes built from the released contract, and a gateway's served models are listed | +| 4 | Annotation: the format, the operations, the Annotate tab, `import` and `freeze` | Large | On the tiny example records of each kind are imported and annotated from the page, Save writes `annotations/`, a fresh clone shows the same annotations, and a frozen dataset with splits is built from them | +| 5 | `generate` through the Pumas endpoint, and suggestions in Annotate | Medium | A small text dataset and a small image dataset are generated through a fake gateway, reviewed and frozen; a suggestion is accepted and saved as an annotation that names the model | +| 6 | Native, predicted and signed error side by side in the Analysis tab, where a run kept its predictions | Small | On the lab folder one case of one earlier model is compared | + +Phases 1 and 2 come first because the Annotate screen is built on the viewer, and because they serve the lab now. Each phase is specified, built by Opus 5.5 agents at medium effort and reviewed, as before. Nothing in phases 1 to 4 trains, benchmarks or uses the GPU. + +## Out of scope + +- A model's activations for an input, and a drawn graph of its architecture: both need the model to be run or described, which nothing records yet. +- Any change to catalog mode. Its tools stay as they are. +- Moving existing catalog annotations into a project folder. +- Video. Pumas names it as a kind of content; nothing here reads or shows it yet. + +## Decisions needed + +1. **Where annotations live.** As text in the Git folder, as above. Recommended: it is the one kind of data here that cannot be made again. +2. **What to generate and annotate first.** Answered by the owner: basic support for text, image, audio, point clouds and meshes, as a setup other projects build on. +3. **A real Pumas run.** The phases are built and checked against fakes. Proving phase 5 on the real gateway means one small generation by a model Pumas serves, which uses the GPU for a moment. Recommended: on the owner's word, as the tiny training runs were. +4. **Weights of new models.** Recommended: Tuldok reads `.npz` and `.safetensors` and never opens `model.pt`; the lab's training code writes one of the two beside it, when training resumes. + +## Risks + +- **Large products.** One dataset is a 9.8 GB zip of 783 compressed files. Listing reads the index only; a sample is decompressed alone (2 to 3 MB). Anything larger than the bound is refused with its size. +- **No numpy in Tuldok's environment.** Statistics over an array are plain Python: fast enough for a slice or a weight file of a megabyte, and sampled above a set number of values, saying so. +- **Pumas 0.8.0 moved after the client was written.** The pinned commits are in the release, but the typed contract has not been compared; phase 3 does that first. +- **Annotations and rebuilt data.** A generated dataset built again may differ. Annotations are tied to content, so those that no longer match anything are listed for the person to deal with. diff --git a/docs/plans/definition-first-projects/execution-ledger.md b/docs/plans/definition-first-projects/execution-ledger.md new file mode 100644 index 00000000..bb7b048b --- /dev/null +++ b/docs/plans/definition-first-projects/execution-ledger.md @@ -0,0 +1,706 @@ +# Execution ledger: definition-first projects + +## 2026-11-10 — milestone 1, the folder + +**Branch:** `feat/definition-first-projects` from `d8d91fbf`, nothing committed. Lab work on `reorg/reproducible-lab-20261102`, nothing committed. + +**Done by:** three Opus 5.5 agents at medium effort, one per slice, from [spec-m1-m2.md](spec-m1-m2.md); each slice reviewed and re-run by the orchestrating session. + +| Slice | Files | Result | +| --- | --- | --- | +| T1 checker | `tuldok_project/{documents,folder,graph,check,__main__}.py`, `tuldok_project/schemas/*.json`, `tests/test_tuldok_project_check.py`, `tests/fixtures/tuldok_project/minimal/` | 59 tests pass. Independent check: eight hand-broken folders (symbolic link, duplicate key, stray directory, cycle, bad placeholder, unknown field, missing reference, clean) all reported correctly | +| T2 summaries | `tuldok_project/results.py`, `tests/test_tuldok_project_results.py` | 56 tests pass. Independent check: statistics and file round trip recomputed by hand | +| L1 lab folder | `lab:scripts/build_tuldok_project.py`, `lab:scripts/tests/`, `lab:projects/surrogate-lab/` | 21 tests pass; `--check` passes; folder is 12 MB in 486 files; checker reports 0 errors, 8 warnings. Independent check: all 645 ungrouped rows of `results/original-pressure-cases.csv` recomputed from the nine source result sets, none disagree | + +Command for the Tuldok tests: `.venv/bin/python -m unittest discover -s tests -p 'test_tuldok_project_*.py'` (115 tests, 2 s). + +**The lab folder holds** 206 recipes, 32 datasets, 18 benchmarks with summaries (4,589 rows), 4 steps, 2 environments, 1 model, 205 cards. 459 of 477 result subjects are linked to a model; the other 18 are solver assemblies or execution variants. + +**Format changes made during the milestone** (all in the spec): + +- `earlier_work.rebuild` gained `needs-work`: nothing is lost, but the recorded command does not run as written. Labels state a fact about the item alone; Tuldok derives from the graph whether anything upstream blocks a rebuild. +- Earlier items name tracked files, not the staged copies under `runs/`; files the old Tuldok generated for a run are not sources. +- Checker choices accepted as implemented: paths must already be in normal form; a missing field is reported at the field's own pointer; knock-on problems from one broken reference are suppressed; a missing final newline in a results file is a warning. + +**Findings:** + +- Of the 205 earlier models, 177 are labelled `no` and 28 `needs-work`; carried through inputs, 183 and 22. No earlier training command runs as written: all 203 take arguments the old Tuldok supplied. +- 173 earlier recipes name code or run material that is in Git only inside the old project folder's blob store, or not at all: 375 distinct files, 7.8 MB (1.1 MB Python, 1.9 MB JSON, one 4.0 MB zip, 0.8 MB arrays). +- Of the 26 earlier training sets and their recording groups, 13 have an exact command; the rest have none or several. +- Authored content in the old notes that the new folder does not carry: 19 task groups and 26 comparisons between models (`notes/00-models.portfolio.json`). + +**Open for the owner:** whether to bring the earlier runs' code into the lab repository; whether to carry the authored groups and comparisons as notes; commit of both repositories' new files. + +**Lab changes the new-work definitions need** (for milestones 4 and 6): a `--json` option on `surrogate.datasets plan`; the benchmark result in Tuldok's results format; `spectral-composition` cannot run on a fresh clone; no lock file yet for the GPU environment; the reference solver needs Rust, which the format's environments do not describe. + +**Remaining in milestone 1:** slice D1, the format contract under `docs/contracts/`, after the owner has read the folder. + +## 2026-11-10 — milestone 1 follow-up and milestone 2, open and save + +**Owner decisions:** bring the earlier runs' code into the lab; keep the authored comparisons; commit on branches, no push and nothing on `main`; start milestone 2. + +**Commits** (none pushed). Tuldok `feat/definition-first-projects`: plan and spec `d55903f8`, checker and summaries `3297f8dd`, format contract `3e787e06`, store `e027ed3f`, then the commands. Lab `reorg/reproducible-lab-20261102`: folder and build script `d0df92180`, earlier run code and comparisons `89cac8b4d`. + +| Slice | Files | Result | +| --- | --- | --- | +| L2 earlier code (lab) | `lab:earlier-run-code/` (184 files, 1.6 MB), `lab:projects/surrogate-lab/notes/model-groups-and-comparisons.json`, build script and tests | 26 tests pass; `--check` passes; checker 0 errors, 8 warnings (sources of two recipes that are not on this machine) | +| D1 contract | `docs/contracts/project-folder.md`, link in `docs/README.md` | every example in it passes the checker | +| T3, T4 store | `tuldok_project/store.py`, `graph.states`, `tests/test_tuldok_project_store.py` | 37 tests. Independent check: bad edit refused, removal of a used definition refused, save refused after the file changed on disk, reopen with that conflict refused until told to discard, no schema left behind | +| T5 commands | `tuldok_project/__main__.py`, `tests/test_tuldok_project_cli.py`; helpers shared with the store made public in `folder.py` and `check.py` | 16 tests | +| T6 lab check | timing below | targets met | + +Command for all tests: `.venv/bin/python -m unittest discover -s tests -p 'test_tuldok_project_*.py'` with `TULDOK_DATABASE_URL` set to a disposable cluster (168 tests, 16 s; 45 are skipped without a database). + +**Milestone 2 acceptance, on the lab folder** (263 definitions, 2,266 edges): + +| Command | Whole process | Reported by `open` | Target | +| --- | --- | --- | --- | +| first `open`, empty database | 0.93 s | 0.72 s | under 10 s | +| second `open`, nothing changed | 0.20 s | 0.01 s | under 1 s | +| `list`, `graph --order` | 0.2 s each | | | + +States: 216 cannot be rebuilt, 27 need work, 17 not built, 2 environments not set up, 1 model. + +**Earlier models after L2.** With their code in Git, 158 of the 205 are `needs-work` on their own label and 47 are `no` (25 start from weights whose producing model is not identified, 18 miss only arrays or an archive, 2 have no command, 2 name scripts not on this machine). Carried through inputs, 27 need work and 178 cannot be rebuilt, because most read a training set that cannot be rebuilt. + +**Accepted as implemented:** + +- A `Project` holds a session advisory lock while open; a second open, or `forget`, is refused as busy. +- The schema is created inside the import transaction, so a failed first open leaves nothing behind. +- The unchanged-folder fast path compares SHA-256 of definition files only. It does not see changed `results/*.csv`, notes or stray entries; milestone 3 must add results files to it. +- `keep` on a conflict keeps the database text and records the file's new digest, so the next save overwrites the file on purpose. +- `save` accepts a file that already holds the text to be written, so an interrupted save can be retried. +- Every command opens the project first, so the command-line `status` never shows pending disk changes; a conflict appears as a refusal. `Project.status()` remains meaningful for a long-lived process. +- Two settings files copied into `earlier-run-code/` (`fine-tuning.json`, `distillation.json`) carry per-tensor checksums of starting or teacher weights. They are kept as settings; the owner can reverse this in the build script's table. + +**For milestone 3:** import `results/*.csv` into summary tables; result rows from runs; saved charts; the first changes to existing Tuldok files (`app.py`, `agent_api.py`) to serve the new core, with its own start-up flag, route prefix and agent profile. + +## 2026-11-10 — milestone 3, results and charts + +**Owner instruction:** continue with milestone 3. Spec: [spec-m3.md](spec-m3.md). + +**Scope change.** Serving the new core from the Tuldok app moved from this milestone to milestone 7, where every operation of the new core is registered once and the pinned agent contract fixtures are regenerated once. In its place this milestone has a static report page. The plan's milestone table says so. + +**Commits** (none pushed). Tuldok `feat/definition-first-projects`: spec `5ce63364` and `1bc187b0`, results in the database `e297f5ee`, charts and report `f352f5ec`. Lab `reorg/reproducible-lab-20261102`: summary shape, values files and charts `06c254ee9`, subject titles `fff10a4a1`. + +| Slice | Files | Result | +| --- | --- | --- | +| T7 results | `tuldok_project/result_rows.py`, hooks in `store.py`, `results`, `values`, `import-values` | 39 new tests | +| T8 charts and report | `tuldok_project/charts.py`, `chart`, `report`, contract sections | 24 new tests, including one that makes every file read fail while each chart is answered | +| L3 lab | build script: one ungrouped row and one row per single group value; `--values-output`; eight chart definitions | 32 tests; `--check` passes | + +Command for all Tuldok tests as before: 231 tests, 30 s. + +**Acceptance, on the lab project:** + +| Check | Result | +| --- | --- | +| Open into an empty database imports every summary row | 4,716 rows from 16 results files; first open 1.1 s, second 0.01 s | +| Import of the per-case values | 477 files, 478,660 values, 12 s; 296 MB in the local database | +| `save` after that import rewrites `results/*.csv` byte for byte | Nothing to save; the lab folder unchanged in Git | +| A chart is answered without reading a result file | proven by test; the lab's progression chart of 127 models answers from the summary table | +| An export of the committed lab alone draws its charts | `check` 0 errors, `open` 0.8 s, `report` writes 8 charts (four with data, four for the new benchmarks that have no results yet) | + +The drawings were viewed as rendered images in both themes by the implementing agent and by the orchestrating session. + +**Accepted as implemented:** + +- Removing a benchmark keeps its `results_files` digest until `save` deletes the file, so the changed-on-disk check still applies. +- Setting a benchmark also drops summary rows for removed metrics and removed group keys. +- `status` gained `results_changed`; `save` returns `results_written` and `results_deleted`. +- A values file must have all six top-level fields and at least one value. +- A subject with no row, a count of 0, or no value for the chosen statistic is listed as missing and never drawn. +- Ranking bars on a linear axis start at zero; on a logarithmic axis at the decade below the smallest value. +- A metric's units of `1` are not shown. +- The report shows the folder's name, not its absolute path. + +**Follow-ups, none blocking:** + +- `charts.py` and `result_rows.py` call a few private names of `store.py` and `results.py` (`Project._execute`, `results._COMMIT`, `_produced_is_valid`, `_benchmark_names`); make them public when those files are next touched. +- The `result_values` table costs about 620 bytes a row with its two indexes; revisit if a project holds tens of millions of values. +- Text widths in the drawings are estimated without font metrics. +- The design tokens' "good" colour fails one of the palette validator's checks; it is always accompanied by a line, a label or a key. + +**Next:** milestone 4, the rebuild plan with estimates. It needs the lab's `surrogate.datasets plan` to print JSON. + +## 2026-11-10 — milestone 4, the rebuild plan + +**Owner instruction:** continue with milestone 4. Spec: [spec-m4.md](spec-m4.md). + +**Commits** (none pushed). Tuldok `feat/definition-first-projects`: spec `58cd0d14`, then the plan. Lab `reorg/reproducible-lab-20261102`: `plan --json` and declared disk `391d1e188`. + +| Slice | Files | Result | +| --- | --- | --- | +| T9 plan | `tuldok_project/plan.py`, `plan` in `__main__.py`, `tests/test_tuldok_project_plan.py`, the "Planning a build" contract section | 30 new tests, including one that makes process creation and file opening fail while a plan of every target is made | +| L4 lab | `lab:surrogate/datasets/__main__.py` (`plan --json`), its test, and `resources.disk_gb` in the two new dataset definitions (1.4 GB and 135.2 GB) | the lab's 72 `surrogate` tests pass | + +Command for all Tuldok tests as before: 261 tests. + +**Acceptance, on the lab project:** + +- `plan recipes/pilot-dense-k1`: two environments to set up, then the reference solver, `datasets/sinusoidal-v1` (1.4 GB declared) and the recipe (GPU), in that order; nothing run. +- `plan recipes/water-residual-linear-cuda-training--72953af8f73b705f`: one blocked step, with its two causes and their reasons (its own command needs adapting; its training set has no single command line). +- `plan --all`: 17 steps to build, 2 to set up, 243 of 260 targets blocked; 137 GB of disk declared. + +**Settled during the milestone** (in the spec): + +- Making a plan never runs a definition's `estimate` command. A step's figures are those its definition declares; running estimates is an explicit action that arrives with environments. +- Edges are not followed out of a blocked step, and a benchmark subject whose recipe is blocked is named as skipped without becoming a step. A plan lists work that can be done, and each blocked target once. +- A benchmark with no subject that would run is blocked, with no cause listed. +- An absent `resources.gpu` means no GPU is needed. +- Targets and `--all` can be combined; giving neither is a usage error. + +**Follow-ups, none blocking:** `plan.py` reads all documents through the private `Project._documents`; a public accessor belongs with the other private-name cleanups. No lab definition declares `wall_hours`, so plans show no time. + +**Next:** milestone 5, environments. Its acceptance installs packages (about 1 GB for the CPU environment, several GB for the GPU one), which needs the owner's go-ahead. + +## 2026-11-12 — milestone 5, environments + +**Owner decision:** code and tests only, plus connecting the two environments that already exist on the owner's machine. Nothing downloaded or installed. Spec: [spec-m5.md](spec-m5.md). + +**Commits** (none pushed). Tuldok `feat/definition-first-projects`: spec `1a788494`, then the environments slice. + +| Slice | Files | Result | +| --- | --- | --- | +| T10 environments | `tuldok_project/environments.py`, `tuldok_project/local.py`, hooks in `store.py`, `graph.states(definitions, local)`, the `ready` action in `plan.py`, five commands, `tests/test_tuldok_project_environments.py`, the "Environments" contract section | 41 new tests; every install in tests runs a stand-in for `uv` | + +Command for all Tuldok tests as before: 302 tests, 71 s. + +**Acceptance, on the lab project** (throwaway database, local home in a scratch directory): + +| Step | Result | +| --- | --- | +| `connect lab-cpu /.venv/bin/python` | ready: Python 3.12.3, numpy 2.3.5, torch 2.8.0+cpu | +| `connect cuda-torch /torch-versions/v2.14.0/venv/bin/python` | ready: Python 3.14.7, numpy 2.5.3, torch 2.14.0+cu132, CUDA 13.2 on the RTX 5090 Laptop GPU | +| `set-up lab-cpu --dry-run` | prints `uv venv --python 3.12 ` and `uv pip install ... -r tuldok/requirements-cpu.lock --extra-index-url https://download.pytorch.org/whl/cpu`; nothing run, nothing created | +| `set-up cuda-torch --dry-run` | refused: the definition has no `install.requirements` | +| `plan recipes/pilot-dense-k1` | both environments `ready`, nothing to set up | + +**Accepted as implemented:** + +- The check reads versions with `importlib.metadata` and imports nothing, except `torch` for a `cuda` environment. +- A failed check is a stored result with problems; the record is kept so they can be read. +- Editing an environment definition does not clear its stored result; the state is as of the last check. +- An environment's record is dropped when its definition is removed. +- A dry run still refuses for a missing `install` section, a missing installer or a non-empty target. +- The installer's commands inherit the caller's environment and run in their own process group with a one-hour limit; the log is replaced on each set-up. +- New `ProjectError` codes: `not-set-up`, `no-install`, `no-installer`, `target-exists`, `install-failed`. + +**Not verified:** a real `uv venv` or `uv pip install`; the probe on Python other than 3.12 and 3.14 (it ran on both in the acceptance run); a failing GPU. + +**One deviation by the implementing agent:** it ran `connect` once against `/usr/bin/python3`, outside the repository's virtual environment, during a manual demonstration. The probe is read-only and the demonstration project was removed. + +**Open:** the lab has no lock file for the GPU environment, so `cuda-torch` can be connected but not installed. Running a definition's `estimate` command moved to milestone 6. + +**Next:** milestone 6, build: runs from a commit, the launcher and queue, products, live progress, results published by the run. Its acceptance trains one tiny model on the GPU, which needs the owner's go-ahead. + +## 2026-11-12 — milestone 6, build + +**Owner decision:** spec, code and tests, and one tiny real end-to-end build on the GPU as the acceptance run. Spec: [spec-m6.md](spec-m6.md). No implementing agent used the GPU; the acceptance run was done by the orchestrating session. + +**Commits** (none pushed), Tuldok `feat/definition-first-projects`: spec `fe2dd56a` and `b9d32e9a`; snapshot `4191d46f`; launcher `e49635b9`, then its custodian `35fdafc6`; builds `f8367c17`; run commands and estimates `419d6ed2`; the example project `d3340b5f`. + +| Slice | Files | Result | +| --- | --- | --- | +| T11 launcher | `tuldok_project/launcher.py` | 37 tests | +| T12 snapshot | `tuldok_project/snapshot.py`, `code_paths` in `tuldok.json` | 30 tests | +| T13 builds | `tuldok_project/builds.py`, the states, `build`, `runs`, `products`, the `buildable` fixture | 30 tests | +| T14 run commands and estimates | `run`, `progress`, `cancel`, `stop`, `estimate`, estimates in `plan`, the "Building" contract section | 17 tests | +| T15 example | `examples/tiny-project/` | checker 0 errors; scripts smoke-tested on the CPU | + +Command for all Tuldok tests as before: 423 tests, about 2.5 minutes. + +**Design change found in review.** The first launcher made the calling process a child subreaper. An independent test (a command that double-forks a daemon and leaves a background child) showed everything was stopped, but one zombie per run was left in the caller, and the same mechanism could take the exit status of a caller's unrelated child. Each run now has its own custodian process (`python -m tuldok_project.launcher`), which is the subreaper, reaps everything it adopts and writes the outcome. Repeating the test, with the daemon also clearing its environment: nothing alive, no zombie in the caller, and the caller's unrelated child reported its true exit status. + +**Independent checks of the builds slice** (the `buildable` fixture in a scratch Git repository): first build, 4 runs; second build, nothing run; a committed settings change made the recipe and its benchmark out of date with the reason and rebuilt only those; an uncommitted change was refused without `--scratch`; with `code_paths` the snapshot held exactly the named files and a step whose script was not named failed; a failing step stopped the build; no process left carrying a run id. + +**Acceptance run, `examples/tiny-project`** (throwaway database, local home in a scratch directory, the owner's two existing interpreters connected): + +| Step | Result | +| --- | --- | +| `connect cpu`, `connect gpu` | both ready | +| `build benchmarks/held-out --yes` | exit 0 in 23.4 s: dataset 0.3 s, training 13.9 s on the GPU, benchmark 3.1 s and 1.7 s for its two subjects | +| progress read from another process while training ran | six observations, from update 1,300 to 6,000 of 6,000, loss falling from 0.0020 to 0.0006 | +| the training run's record | from the snapshot of commit `d3340b5f`, exit 0, 0 strays, outputs `model.pt` (349 KB) and `log.json` | +| `chart held-out-ranking` | `small-net-a` median error 0.0376, `predict-zero` 0.871, 400 cases each, from the results the benchmark runs published | +| `chart held-out-tradeoff`, `report` | drawn; the tradeoff drawing was viewed as a rendered image | +| second `build` | nothing run: everything built | +| an uncommitted change to the recipe's settings | the recipe and the benchmark show out of date with the reason; built again after the change was reverted | +| afterwards | no process carrying a run id; the GPU idle with no compute process of the run; the example unchanged in Git | + +**Accepted as implemented:** + +- A definition is also out of date when something it uses is out of date, so one build settles a change. +- A benchmark records its subject's recipe among the run's inputs, and rebuilds only the subjects that are not built. +- `running` takes precedence over every state; a current product over `cannot-rebuild` and `needs-work`. +- The snapshot is prepared before confirmation, so a dirty tree is refused without asking. +- An environment is checked once per build, and only when a step being built uses it. The project stays open during that check. +- `Outcome` has a `problem` text; a launcher that cannot start the command makes the step `failed`. +- Estimates: only `build` steps are measured; a benchmark's estimate command may not use `{subject}`; the disk limit is 100 MB; no GPU lock is taken; the throwaway directory is removed afterwards. +- `stop` refuses only while the run is `queued` or `running` and its builder is alive. + +**Not verified:** cancellation and the disk limit through `build` on a real long run; a real interrupted build beyond a hand check with `SIGINT`; behaviour with Git submodules (not included in snapshots); the GPU lock with two real GPU builds. + +**Known gaps:** leftover `.partial-*` snapshot directories from a crashed build are never removed; nothing is ever deleted from the local home (runs, old products, snapshots), so a `prune` command is needed before long use; the values file's `commit` is whatever the benchmark's command writes, since the command is not told its commit. + +**Next:** milestone 7, serving the new core from the Tuldok app with agent operations; then milestone 8, converting the lab for good and retiring the old project-files code, which needs the owner's two open decisions (where the product archive goes; which earlier models to drop) and sign-off of the capability check. + +## 2026-11-12 — milestone 7, the app and the agent interface + +**Owner instruction:** continue, after the milestone 6 report said the next step was this milestone's spec and its implementation in the same way. Spec: [spec-m7.md](spec-m7.md); its section 12 lists where the code differs from the sections before it. + +**Commits** (none pushed), Tuldok `feat/definition-first-projects`: spec `1b633ffb`; the operations table, the service, build requests and the worker `2e1f21d9`; the app, the agent contracts and the page `3f8f0f4b`. + +| Slice | What | Tests | +| --- | --- | --- | +| T13 service | `tuldok_project/operations.py` (36 operations as data), `service.py`, `activity.py` | 25 | +| T14 build requests | `tuldok_project/build_requests.py`, the worker in `service.py`, `expect` and `stop` on `builds.execute`, the commands `requests`, `approve`, `decline` | 18 | +| T15 app | `tuldok_project/host.py`, additive blocks in `app.py` (59 lines), `agent_api.py` (59 lines) and `mcp_stdio.py` (4 lines), the page, the "Serving a project" contract section | 16 | + +All Tuldok project tests: 482, about 3 minutes. The 14 existing agent and contract test modules named in the spec: 173 tests pass with 31 skipped, before the milestone and on its final code (run from the repository root with `PYTHONPATH=tests`). + +**Nothing published changed.** The `catalog` profile (115 operations), the old `project` profile (152 operations) and the legacy discovery document have the SHA-256 digests they had before the milestone; a test pins the three. No pinned contract fixture was regenerated: the earlier note that they would need it was wrong, because they record earlier contracts and this milestone only adds. + +**Changed in review:** + +- `ERROR_STATUS` became `error_status`. +- The worker logs a failure that repeats every pass once, not every second. +- The page's sign-in bar and dialog kept their light colours in a dark browser; they now follow the page. +- Found in the acceptance run: a command such as `requests` failed at once with "already open" when the app had the project open for a moment. The store commands now wait up to 5 seconds, as `build` and `progress` already did. +- From the acceptance agent's remarks: a request has `expires_at` while it is queued, and the descriptions of `progress` and `request_build` say how to read on and where to see the state. + +**One change outside the listed blocks:** `agent_api.AgentClient` accepted only refusal codes without hyphens, so every refusal of the new core reached an agent as a transport error. For a `project.*` selection it now passes on the code, `problems`, `references` and `files`. No existing line changed. + +**Acceptance run, `examples/tiny-project/project`** (throwaway catalog and database, local home in a scratch directory, port 8197): + +| Step | Result | +| --- | --- | +| `python app.py --project-folder ...` | serving in under a second | +| the owner signs in with a one-time code and issues an agent credential over HTTP | owner session; agent credential with `inspect`, `edit`, `execute`, `cancel` | +| the owner connects both environments with `ProjectFolderOwnerClient` | both ready; 36 operations offered | +| an agent given only the agent credential and `agent_api.AgentClient`, told to learn the operations from their descriptions | read the summary and the plan (3 steps to build), requested `benchmarks/held-out`: `awaiting-approval` | +| the owner's side: `approve` on the command line | showed who asked, the commit, the 3 steps and the 5 definitions held to; queued, then started within a second | +| the build | `succeeded` 23 s after approval: dataset 0.3 s, training 14.8 s on the GPU, benchmark 2.2 s and 1.3 s | +| the agent's polling of `progress` during training | 9 distinct events, from update 650 to 6,000 of 6,000, loss 0.0023 to 0.0006 | +| the agent's reading of the results and the ranking chart | `small-net-a` median error 0.0376, `predict-zero` 0.871, 400 cases each; the SVG saved | +| the agent's second request | refused 409 `nothing-to-build`; no request made | +| `activity` | four rows: two connects by the owner, the agent's request, the approval from the command line; then the refused second request | +| the page, in a headless browser as the signed-in owner | name, folder and the report with both charts, in light and dark, no script error, no sideways overflow | +| afterwards | the app stopped on `SIGINT` with exit status 0; no process carrying a run id; the GPU idle; the example unchanged in Git | + +**The lab project through the app** (271 definitions, 18 benchmarks, results from `results/*.csv` only): `summary`, `definitions`, `graph`, `results_overview`, one benchmark's 1,125 summary rows and each chart with its drawing answered in 0.02 to 0.09 s; `report` in 0.18 s; `plan` for everything in 0.09 s; `check` in 0.5 s. A request to build everything was refused as `blocked` and made no request. Nothing of the lab was run. + +**Not verified:** the MCP bridge with `--project-folder-owner` (the `mcp` and `anyio` packages are not installed here; the change is the flag and the client choice); the sign-in dialog typed by hand (the browser check was given the session cookie); a browser other than headless Chromium; cancelling a real long training through `cancel_build`; two apps on one project beyond the test of the worker lock. + +**Known gaps:** + +- While a build checks an environment, which takes a few seconds for the GPU one, every other operation waits. The agent saw one 4 second poll and so never observed `queued`. +- A request has no history of its states, and its runs appear in two shapes (`result` and `runs`). +- `context.get` and `identity.get` keep their generic descriptions; they are part of the pinned legacy discovery document. +- The example's benchmark reports one timing spread over its 400 cases, so `time_ms` has no spread. +- Carried over: the values file's `commit` is empty; nothing is ever deleted from the local home; installing an environment and measuring estimates are command-line only; the catalog queue does not take the GPU lock (spec section 9). + +**Next:** milestone 8, converting the lab for good and retiring the old project-files code. It needs the owner's two open decisions (where the product archive for the earlier weights goes; which earlier models to drop) and sign-off of the written capability check before any old module is deleted. + +## 2026-11-12 — milestone 8, part 1: where products live, and adopting the earlier ones + +**Owner decisions:** "The model weights can stay with tuldok on the loca system. Tuldok should have an internal proj dir to store per project data including the weights. We do not want the weights in the git repo." Then: "yes make orangcream tuldoks project location." Then: "continue". No product archive is built. Spec: [spec-m8.md](spec-m8.md); the [capability check](capability-check.md) was written the same day and is not signed. + +**Commits** (none pushed). Tuldok `feat/definition-first-projects`: the decision and the capability check `5a52bed5`; spec `1a9b9e73`; the setting, `new` and `adopt` `3af6b6a6`. Lab `reorg/reproducible-lab-20261102`: the adoption manifest `6dac9a160`. + +| Slice | What | Tests | +| --- | --- | --- | +| T16 | `tuldok_project/local.py` (the saved setting), `adoption.py`, the adopted rule in `builds.py` and `plan.py`, the commands `home`, `new`, `adopt` | 24 | +| L4 | `lab:scripts/build_tuldok_project.py --adoption-output` | 40 in the lab's converter tests | + +All Tuldok project tests: 507, about 4.5 minutes. + +**Changed in review:** + +- A definition that is built on this machine no longer carries `cannot-rebuild` or `needs-work` to what uses it. As implemented first, a new recipe could not use an adopted dataset as its input, which is what adoption is for. +- Closing a project now releases its lock by a statement. A full test run on a busy machine failed once with "already open" straight after a close; the lock had been left to the server to release when it noticed the session end. +- One existing test read the machine's settings file. The whole suite was then run with a decoy settings file in the environment: every test passed and nothing was written to the decoy directory, so no test can write into the real project directory. +- The spec was widened during the work: a definition that declares no outputs can be adopted as a whole directory. The lab's manifest showed why: the earlier weights are 0.8 GB, and 15 GB of the old folder is training data of 25 earlier datasets, which declare no outputs. + +**What the old folder holds** (`projects/rheon-model-portfolio-unified-04/artifacts/sha256/`, 8,734 files, 16.5 GB, ignored by Git), by the lab's manifest: + +| Part | Files | Size | +| --- | --- | --- | +| Outputs of 203 earlier recipes (weights, histories, receipts) | 2,264 distinct | 0.79 GB | +| Training data of 25 earlier datasets | 177 distinct, 34 of them also a recipe's output | 15.08 GB | +| Named by no entry: old-format records, staged code, custody proofs, outputs of runs with no recipe | 6,327 | 0.69 GB | + +Every declared output of the 203 recipes was found at its recorded size; no dataset path was claimed by two different files. Two earlier recipes have no run and five earlier datasets were read by no recipe, so those seven have nothing to adopt. + +**Acceptance on a throwaway database**, with a throwaway project directory on the OrangeCream disk: + +| Step | Result | +| --- | --- | +| `adopt --dry-run` | would adopt 228 entries, 3,841 files, 16.9 GB, all to be linked; nothing changed | +| `adopt` | 228 adopted in 1.6 s; free space on the disk fell by 2.8 MB | +| The files | all 3,841 are hard links to files of the old folder, 2,407 distinct | +| States | before: 216 cannot rebuild, 27 need work, 17 not built. After: 228 built, 23 cannot rebuild, 9 not built | +| `plan --all` | before: 17 to build, 243 blocked. After: 9 to build, 23 blocked, 228 built | +| A second `adopt` | skipped all 228 | +| Removing the throwaway directory | the old folder still has its 8,734 files and 16,541,094,986 bytes, each with one link again | + +**On the owner's machine** (done after the acceptance, on "continue"): + +- `home --set` saved `/media/jeremy/OrangeCream/Linux Software/workspace/runtime-data/tuldok/projects` in `~/.local/share/tuldok/settings.json`. The path follows the disk's own layout notes, which put app data under `workspace/runtime-data/`. Before, the default was on the system disk with 16.5 GB free. +- The owner's Tuldok database was started, the lab project opened in it for the first time (0.9 s), both environments connected and ready, and the same 228 entries adopted in 0.7 s by link. The database was then stopped again, as it was found. +- The lab project there: 228 built (203 recipes, 25 datasets), 23 cannot rebuild (16 earlier benchmarks, 5 datasets, 2 recipes), 9 to build (the new datasets, the pilot recipe, its two benchmarks and four steps), 2 environments ready. + +**Not verified:** adoption by copy across disks beyond its test; `home --set` when a previous directory holds projects, beyond its test. + +**Known gaps:** no command rebuilds or sets aside an adopted product; the 0.69 GB the manifest does not name exists only in the old folder; the earlier weights and training data are single copies on this machine. + +**Machine note:** the machine restarted during this work, which emptied `/tmp`: the throwaway database was made again, and the screenshots and chart from the milestone 7 acceptance, which were only there, are gone. + +**Next:** part 2, the lab repository stops tracking the old project folder; then the deletion of the old project mode, which waits for the capability check to be signed. + +## 2026-11-12 — milestone 8, part 2: the lab repository holds the new folder only + +**Owner instruction:** "continue". Spec: section 9 of [spec-m8.md](spec-m8.md). + +**Commits** (none pushed). Tuldok `feat/definition-first-projects`: spec `b202fce6`. Lab `reorg/reproducible-lab-20261102`: README, `.gitignore`, the converter's docstring and the removed equality test `076b04c98`; the old folder out of the index `fd18d8202`. + +**What changed in the lab:** + +- `projects/rheon-model-portfolio-unified-04/` is no longer tracked: 55,865 files left the index. All 63,133 files of the folder are still on disk (17 GB), now ignored. The lab tracks 6,987 files, and under `projects/` only `surrogate-lab` (495 files, 12 MB). +- `projects/surrogate-lab/` is the lab's Tuldok project from here on, edited directly or through Tuldok. The test that required it to equal the converter's output is removed; the converter's other 39 tests, all on synthetic folders, pass. +- The README says where the project is, how to check, open, report and plan it, and that weights, datasets and run logs live in Tuldok's project directory on the machine. + +**Acceptance: the plan's test on fresh clones** (a local clone of the lab branch and of the Tuldok branch into a scratch directory, the Tuldok clone's code run with the existing interpreter, an empty database, an empty project directory): + +| Step | Result | +| --- | --- | +| The lab clone | 254 MB working tree, no old project folder, no `runs/` | +| `check` | 0 errors, 81 warnings. 73 of them are earlier recipes naming source files under `runs/`, which only the original machine has; the other 8 are the two recipes already known | +| `open` | 1.4 s | +| `list` | 271 definitions: 206 recipes, 32 datasets, 18 benchmarks, 8 charts, 4 steps, 2 environments, 1 model | +| `report` | 0.5 s; 172 KiB with 8 charts drawn from `results/` alone; no script and no external reference | +| `plan --all` | 0.3 s; 17 steps to build in order, 3 on a GPU, 137 GB of disk declared | + +The clone was not built: that needs the reference solver and the dataset generation, which the owner has not approved. Rheon was not cloned; nothing before a build needs it. + +**Left as they are:** `scripts/track_project_results.py` and `scripts/extract_model_recipes.py`, which served the old format and which nothing current refers to; the historical plans and the migration scripts under `experiments/model-portfolio-migration-v1/` that name the old folder; the README's link to the older `tuldok/` folder, which a reader may confuse with the project. + +**Next:** deleting the old project mode from Tuldok. It waits for the owner to sign the [capability check](capability-check.md). + +## 2026-11-12 — the owner looks at the interface; the Map workspace plan + +**What happened.** The owner asked to see the project open in the Tuldok interface. Tuldok was started on the lab folder in the new project-folder mode, whose only page is the report. The owner: "wtf happened to tuldoks ui? It doesnt have any of the pages it used to have." Started again the way it had been run before (`--project-files-root`), it showed the form-panel project page; the owner: "that is not the GUI it had before either", and described tabs along the top, a bench page and a model page with a horizontal ancestry tree. That is the Map layout of [the workspace redesign](../project-workspace-redesign/README.md), chosen on 2026-11-03, which exists only as mockups run from disk and was never part of the app. Shown the mockup, the owner confirmed: "yes, thts the gui i remember". + +**What this changes.** Nothing had been deleted, but the new format has no real screens, and that was not made plain before the owner looked. [The Map workspace plan](workspace-plan.md) builds the Map layout into the app in three phases; the owner approved it the same day ("the plan looks good") with its three recommended decisions. The old project mode is not deleted before phase 3 is accepted, which replaces the earlier advice to sign the capability check. + +**A fault found and repaired.** Adoption had hard-linked 2,407 files of the old project folder into Tuldok's project directory. The old project code refuses a stored file that has a second name ("Artifact source must be an ordinary file without hard-link aliases"), so those weights and training data would have been refused by the old mode. Each was replaced in Tuldok's directory by a real copy, compared byte for byte, and every file of the old folder has one name again. Cost: 15.9 GB on the OrangeCream disk. The old lab project was then opened in the old mode to check: it opened, in 166 s. The files' change times moved, so the old mode re-verifies them on first read. Adoption by link stays the command's default; a source that another tool manages should be adopted with `--copy`. + +**Commits** (none pushed), Tuldok: the plan `55a5ac68`; the brief and mockups, without the baked data file, `4af5a74a`. + +## 2026-11-12 — the Map workspace, phase 1: what the map reads + +**Owner instruction:** "the plan looks good". Spec: [spec-w1.md](spec-w1.md). + +**Commits** (none pushed). Tuldok `feat/definition-first-projects`: spec `ecca58d1`; the fields and operations `a4047b68`. Lab `reorg/reproducible-lab-20261102`: the fields on the lab's models `d9e8960a0`. + +| Slice | What | Tests | +| --- | --- | --- | +| T17 | `groups` and `map.chart` in `tuldok.json`; `group`, `change`, `derived_from`, `compared_with` on a recipe; the checker's rules; graph roles `derived` and `compared`; `tuldok_project/workspace.py` with the operations `map`, `note`, `recipe_results`; the commands `map` and `note` | 47 new; 554 in all, about 4 minutes | +| L6 | the lab's converter writes the fields; `scripts/earlier-model-map.json`; a ninth chart for the track; the folder regenerated | 44 in the lab's converter tests | + +**As built, beyond the spec:** + +- The kind of change is `change` on a recipe, not a field of a group as the plan first said: in the mockup one group holds 31 architecture and 20 refinement models. +- `derived_from` was added. The plan had lines come only from recorded inputs; a model branched from another without taking its weights has no such input, and "Branch from here" needs the field. +- `tuldok.json` is now kept in the database (a new column of the state row), so `map` reads no file; a change to `tuldok.json` alone now imports the folder again. +- Removing the chart that `map.chart` names is refused. +- `note` takes a path relative to `notes/`; a recipe's `card` value starts with `notes/`. + +**Acceptance on the lab** (throwaway database): + +| Check | Result | +| --- | --- | +| `map` through the service | 0.04 s, 133 KiB: 206 models, 19 groups, 43 derived and 63 input lines, 26 comparisons | +| Kinds | 48 training policy, 95 architecture, 36 refinement, 26 component, 1 with none (the new pilot model) | +| Track | worst pressure error on the 128 consumed requests; 150 of 206 models have a value | +| C53 | 36 models derived from it; its card read by `note` (28,684 characters); `recipe_results` gives 15 rows over 3 benchmarks | +| The plan for everything, on an empty machine | unchanged: 17 to build, 243 blocked | +| The lab folder | `tuldok.json` and 205 recipes changed, one chart added, the notes file removed; every other file byte-identical; checker 0 errors and the same 8 warnings | + +**Starting values to correct.** Kinds, parents and short labels were carried from the mockup, which inferred kinds and parents from card text: 128 models have a short label, 43 a parent (36 from C53, 4 from C37, 3 from C44). The group "Pressure proposal fits" has no model of its own, only child groups. + +**Next:** phase 2, the page, read-only. + +## 2026-11-12 — the Map workspace, phase 2: the page, read-only + +Spec: [spec-w2.md](spec-w2.md). + +**Commits** (none pushed). Tuldok `feat/definition-first-projects`: spec `e7eae194`; the shell, the map, the drawer and Overview `1f66f5a0`; Benchmarks, Train, Data and Analysis `50e73ddb`. Lab `reorg/reproducible-lab-20261102`: descriptions that lead with each model's purpose `7774816a5`. + +| Slice | What | +| --- | --- | +| T18 | `static/project-workspace.html` and `static/project-workspace/` (the adapter, the shell, the map, the charts helper, a Markdown renderer, the Map and Overview screens), Ubuntu Sans vendored with its licence, `description` on each model of `map`, the stand-in report page removed, `tests/browser_project_workspace.cjs` | +| T19 | `static/project-workspace/tabs.js`: Benchmarks, Train, Data, Analysis; the charts by form; the browser check extended | +| L7 | each earlier recipe's description leads with the Purpose paragraph of its card (205 of 205 cards have one) | + +Tuldok project tests: 561, about 4 minutes. The browser check passes on its fixture and on the lab folder: every tab in light and dark at 1440 by 900, no script error, no failed request, no sideways overflow, nothing added to the activity log. + +**Reviewed against the mockup** on the lab's 206 models, screenshot by screenshot: the top bar and tabs, the status strip, the map with its sections, the lanes "from C37" and "from C44", the minimap and the track, the list with short label, title, purpose, kind and bar, and the drawer with the card rendered, in light and dark. + +**Changed in review:** + +- A product with no declared outputs reported a size of zero, so the lab's adopted dataset directories read as empty on the Data tab. `products` now gives the size the run recorded; on the owner's project the 25 adopted datasets show 15.67 GB. +- The label of the fastest end of the frontier was printed over the frontier line; it now sits on the side the line does not leave by. + +**Where the page differs from the mockup, on purpose:** no Layout switch; no control that changes anything (phase 3); the drawer's figures are the track metric, cases, benchmarks with results and state; the track scale fits the project's values; inputs are drawn only for the selected model; the Benchmarks picker chooses a benchmark where the mockup chose a population, and each chart definition is drawn by its own form; no studies, planned models, thumbnails, volume viewer or field slices. + +**Not verified:** the hairline and progress line of a run while it runs, and the two-second refresh during a build: no fixture has a running run with a progress event. They are checked in phase 3, whose acceptance runs a real build. + +**On the owner's machine.** A preview of the committed page runs from an export of the commit in `workspace/scratch/tuldok-preview` on port 8092, on the owner's database, catalog and project directory; the older page still runs on port 8091. + +**Next:** phase 3, acting from the page, after the owner has looked at phase 2. + +## 2026-11-12 — the Map workspace, phase 3: acting from the page + +Spec: [spec-w3.md](spec-w3.md). The owner looked at phase 2 and said to continue ("its ok so far, continue"). + +**Commits** (none pushed), Tuldok `feat/definition-first-projects`: spec `b15c979d`; the core additions `0864c98c`; the build controls `a7ed05ea`; the editor, branching, the idle refresh and the acceptance script `597452df`. + +| Slice | What | +| --- | --- | +| T20 | operations `draft_branch` (writes a new recipe from one of the project's, storing nothing, with the parent's settings file carried as its own settings and the problems keeping it would find) and `revert_definition` (the folder's file back in place of an unsaved edit or a removal; a definition never saved is dropped); `map` says which models are unsaved and dates a model without a `trained_at` label by the run of its current product; commands `branch` and `revert`; 41 operations | +| T21 | `static/project-workspace/actions.js`: who may use a control, one way to call a changing operation (waiting, the two-press state, typed text and the refusal survive every redraw), the plan view, the conflict panel, Needs you and the strip's Save; Train: Plan build, a request's Approve, Decline with a reason and Cancel, a run's Cancel and Stop, an environment's Check again, Connect and Disconnect; the live run (hairline, progress line, both logs read on) | +| T22 | `static/project-workspace/editor.js`: the editor (text for every kind, form for recipes), Pick on the map, Branch from here with drafts kept in the page and drawn hollow and dashed, the steps from draft to built, the Overview's list of every definition; Train's Planned section | + +Tuldok project tests: 585 (561 before), about 4 minutes; the kept agent interface tests 44 pass (4 skipped). The pinned digests did not change: none of them is of the `project_folder` profile, which is checked against the operations table. + +**Browser checks.** `tests/browser_project_workspace_actions.cjs` (new, about 1 min 45 s): on a fixture copy in a Git repository of its own with stand-in commands, every control of the spec's section 4 as the owner, fourteen refusals provoked on purpose, and with a credential that may only inspect no control drawn. `tests/browser_project_workspace.cjs` (the read-only check) now signs the browser in with an inspect-only credential, because the owner's page has controls its assertions forbid; its assertions are unchanged, and it passes on its fixture and on the lab folder. + +**Acceptance run** (`tests/acceptance_project_workspace.cjs`, on a committed copy of `examples/tiny-project`, the owner's two interpreters, a throwaway database, a headless browser; everything after start-up done on the page): + +| Step | Result | +| --- | --- | +| Connect `environments/cpu` and `environments/gpu` from Train | ready: Python 3.12.3 and 3.14.7 | +| Plan everything buildable, Request build | 4 runs succeeded; while `small-net-a` trained the hairline of the strip and of the run drawer stood between 3 % and 19 % with "train: 200 of 6000" and the like | +| Branch from here on `small-net-a`, title, kind, description | a draft on the map after its parent, hollow and dashed, its form holding the parent's settings copied from `settings/small-net-a.json` | +| Updates 6000 to 3000 in the form, Keep, Save | `recipes/small-net-3-000-updates.json` in the folder with `derived_from`, `change` and `settings.updates` 3000 as a number | +| Plan build, Request build | trained on the GPU with its progress on the page; the model drawn built with its line from the parent, without a reload, dated by its run | +| The benchmark edited in the text view (a third subject), Keep, Save from the strip | the file is the text typed, byte for byte | +| An agent asks for `benchmarks/held-out` | the idle page counted it in Needs you by itself after 13 s; Approve there; 3 runs succeeded | +| Results | median error 0.0376 (6,000 updates), 0.0407 (3,000 updates), 0.871 (predict zero); all three on the Benchmarks tab | +| The whole run | no script error, no failed request; 1 min 40 s; two small trainings and three benchmark runs on the GPU | + +Screenshots: `build/qa/project-workspace-acceptance/final/` (21) and, for the owner's view of the lab folder at full size against the throwaway database with nothing kept or saved, `build/qa/project-workspace-acceptance/lab-owner-view/`: the map opens in 0.6 s, the editor of C53 in 0.2 s, a draft branched from C53 in 0.4 s, the lab's activity log empty afterwards and its folder unchanged. + +**Found at the acceptance run and changed:** + +- An idle page read again only when the window regained focus, so a build an agent asked for while the owner was looking at the page did not reach Needs you. The page now also asks again every fifteen seconds while nothing builds (summary, runs, build requests, status, environments: 13 to 60 ms each on the lab folder). +- Between two steps of a running build the strip said the request was queued. It now says a build is between two of its steps. +- A field with a hint beside one without (Short label and Group) stood at different heights. + +**Deviations from the spec, kept:** + +- `branch` does not take the other commands' `--keep` and `--discard` for a conflict; its own `--keep` stores the draft. +- Reverting a removed benchmark imports its results file again when it is still the file last imported; otherwise the next save would have deleted the file. +- A model that names a parent and has no date is drawn in the lane of its parent's branches, right after the parent, when it is in the parent's group. +- A never-built node is outlined in its kind's colour, not the accent, so the kind stays readable; a draft is labelled "new". +- Models branched and kept in this browser tab keep their steps list after they are built. +- The form writes the whole document again with the text's indentation: values and key order are kept, compact one-line objects are spread over lines. +- Environments on Train open a drawer for their controls; Decline takes two steps because its reason field opens on the first. + +**Not verified:** Stop what a run left running (it needs a run whose builder died); Save refused `changed-on-disk` (the opening import reports a changed file as a conflict first); widths other than 1440 by 900. The read-only check once stopped early on the lab folder while the full test suite ran beside it, and passed the next four times: it presses before a view has painted when the machine is loaded. + +**Seen, not Tuldok's:** this machine's zone is America/Vancouver, which the system clock gives as UTC−7 in November; the headless browser and Node carry older zone data and show times an hour earlier. A browser with current zone data shows the right hour. + +**Next:** the owner looks at phase 3 on the lab project. Then the capability check can be signed and the old project mode deleted (milestone 8, part 3). + +## 2026-11-12 — the owner looks at phase 3; the workspace updates in place + +**Owner, on phase 3 on the lab project:** "looks ok. continue", and then: "one thing, im noticing there is a full page reset when it should just be updating info on the page instead". Phase 3 is accepted with that one fault. The capability check is not signed by these words; it was brought up to date and its losses put to the owner again. + +Spec: [spec-w4.md](spec-w4.md). **Commits** (none pushed), Tuldok `feat/definition-first-projects`: the spec, part 3 of [spec-m8.md](spec-m8.md) and the capability check `4f206bbd`; the fix and its check, the commit before this entry. + +**What was measured before** (the lab folder, owner signed in, nothing changing; elements of the main area that were new nodes afterwards): + +| Tab | On return to the window | Every 15 s while idle | +| --- | --- | --- | +| Map | none | none | +| Data | all 356 | none | +| Train | all 64 | all 64 | +| Benchmarks | all 2,109 | none | +| Analysis | all 159 | none | +| Overview | all 2,336 | all 2,336 | + +The tab bar was rebuilt with each of those, the strip on every reading, and the sideways scroll inside tables was lost. The reading every fifteen seconds had been added at phase 3's acceptance, where only its cost was measured, not what it redrew. + +**After:** none, in every cell, with the model's drawer and the editor open as well; no strip or tab-bar node added; every scroll position kept. The same measurement is now a check, `tests/browser_project_workspace_still.cjs` (about 2 min 40 s on the fixture, 1 min on the lab folder read only): nothing touched on twelve places when nothing changed; a build asked for by a second session shown within one reading with the scroll and the focus kept and no "Reading…"; a drawer brought up to date in place; a running build's progress set in place in the strip, the run's drawer and its row of Train's list while the Map gains no node; the editor with typed text left alone; and the page kept as it is when the server cannot be reached. + +**How.** One slice by an agent, then review. The page compares the text of each watched answer (`summary`, `status`, `build_requests`, `environments`, `runs`, and `map` when it can have changed) with the one it holds, and draws nothing when none differs. When one does, the tab and the drawer are built off the page and put in place whole, in one step. No element is kept from an older drawing when its content is drawn again, so a control never acts on older answers than it was drawn from. + +**Changed in review:** + +- As first built, a running run's progress line in Train's list and Overview's "Read at" went stale until a watched answer changed. Both are now set in place on every reading. +- As first specified, a watched answer that was refused counted as a change, which emptied the tab on screen whenever the server could not be reached for a moment. A reading that fails now changes nothing: the page stays, the strip says "Not read" and why, and the timer asks again. +- `tests/browser_project_workspace_actions.cjs` failed twice in a row at one assertion made the moment a control stopped being busy; the drawer is now redrawn by the reading that follows, a few milliseconds later. Its `settled()` waits for the page's own requests; no assertion changed, and it then passed twice. + +**Tests on the final code:** project suite 585; agent interface 44 (4 skipped); HTTP 17; `browser_project_workspace.cjs` on its fixture and on the lab folder; `browser_project_workspace_actions.cjs` (27 steps, twice); `browser_project_workspace_still.cjs` on its fixture (twice) and on the lab folder. The lab folder was unchanged throughout. + +**Not verified:** the new check in the dark theme; a reading that fails while a build runs; widths other than those the checks use (1280 and 1440). Each browser check asserts at one moment after waiting; on a busy machine a wait may still prove short. + +**Next:** the owner looks at the page again. The deletion of the old project mode is being prepared on the side branch `chore/retire-old-project-mode` and lands only when the capability check is signed. + +## 2026-11-12 — milestone 8, part 3: the deletion of the old project mode is prepared, not landed + +**Status: prepared and checked on the side branch `chore/retire-old-project-mode`, one commit on top of this one. Nothing is deleted on this branch. The capability check is not signed.** + +**Why it is not landed.** The plan deletes the old mode only when the owner signs the capability check, and the owner signs by saying so. The owner was told twice that the deletion lands on the word "signed"; the replies were "continue", which is taken as leave to prepare it and not as the sign-off. + +**How.** Section 10 of [spec-m8.md](spec-m8.md). First the baseline: the capability manifests of both profiles saved, and the whole Python suite run on the unchanged branch. An agent cut the import statements that tied the old code to catalog mode; the machine was restarted at that point, which lost the agent and the first baseline run and kept the working tree. A second agent checked that first step, deleted, searched for what was left and gathered the evidence. In review every changed file's diff was read, the changed Python files were linted (no undefined name; the same warnings before and after), the manifests were compared again by a script of the reviewer's own, and the workflow file was parsed. + +**Found while preparing** (each is corrected in the capability check and in section 10.6 of the spec on the side branch): + +- `project_cli.py` is also catalog mode's command line, so it stays without its old-mode commands: 37 modules go, not 38. +- Twelve import statements tied the old code to catalog mode, not eight. +- A test of the kept code, `tests/test_tuldok_project_http.py`, pinned the digest of the whole catalog manifest and of the `project` profile, which the deletion changes by its own rule. It now pins the manifest without the five `project_transfer.*` entries (shown equal, entry by entry, to the earlier manifest with those five taken out) and a new test checks that the old profile is refused. +- About thirty lines in ten of catalog mode's files read an attribute that only the deleted modules set. They cannot be reached any more and are left for a change of their own, since they run through the code that starts and recovers runs. + +**Size:** 143 files and 47,099 lines deleted; 2,450 lines cut from 17 files that stay. + +**Tests:** + +- Contracts: every remaining operation published byte for byte as before in both profiles; the catalog's list 115 to 110 (the five `project_transfer.*` entries, listed as unavailable before); the `project` profile refused. +- Whole Python suite: 3,149 tests before with 40 not passing; 2,397 after with 25 not passing, of which 23 are the same tests failing for the same reasons of this machine (no training environment, a database user that may not create databases, three CPU snapshot tests) and two lines are the pinned digest. After that test was changed: project suite 586, agent interface and command line 67 (4 skipped). The whole suite was not run again after that change. +- JavaScript: all 49 remaining `tests/test_*.cjs`; `node --check` on 54 scripts. +- Browser: the three workspace checks on the fixture, two of them on the lab folder read only (unchanged before and after); 45 of the 49 catalog checks that can run here. +- Start-up: the removed flags are argument errors; catalog mode and `--project-folder` both start. + +**Not verified:** 13 catalog browser checks that need training, a GPU or generated fixtures, and three that need packages only the workflow installs; one catalog check fails removing its own temporary directory, on the unchanged code as well. The browser checks used Playwright's headless shell. Nothing ran on GitHub: nothing is pushed. + +**Next:** the owner's sign-off of the capability check. Then section 10.5: the words and the date recorded there and here, and the side branch brought onto this one. Separately, the unreachable lines in catalog mode's run code. + +## 2026-11-13 — milestone 8, part 3: the old project mode is deleted + +**Owner's words, 2026-11-13**, after being told item by item what projects lose: "We still want tools for generating datasets and annotating them using pumas library v0.8.0 in tuldok. Tuldok also needs to be able to look inside data. It would be great if tuldok also had a layer visulization tool so we can visualize a models weights." Then: "the other stuff is all ok to go ahead on". + +**What was done on them.** `feat/definition-first-projects` was fast-forwarded to the one commit of `chore/retire-old-project-mode` (section 10.5 of [spec-m8.md](spec-m8.md)). No old-mode app was running. Nothing is pushed. The words are recorded in the [capability check](capability-check.md) as a sign-off in part. + +**How the words were read.** The owner accepted every loss on the list but two: tools for generating and annotating datasets, and looking inside data. The deletion was landed all the same because it takes neither out of Tuldok: both are catalog mode's and stay; only the old format's wrapping of them went, which could not serve a project folder. This reading is the assistant's. One revert undoes it. + +**Requirements carried from here** (not planned yet): + +1. In a project folder: generating datasets and annotating them, through Pumas Library 0.8.0. The client for 0.8.0 is on `feat/pumas-v080-api`, four commits not merged into this branch. +2. In a project folder: looking inside data. +3. A view of a model's weights layer by layer. + +**Also carried:** the unreachable lines in catalog mode's run code that read attributes only the deleted modules set. + +## 2026-11-13 — the data tools plan; reading inside a product (server); the Pumas 0.8.0 client + +**Owner, 2026-11-13,** on the [data tools plan](data-tools-plan.md): "we will use the pumas endpoint. We wont be embedding pumas. What do you mean by lables? Right now we just need the basic modality support for data generation and annotation. text, image, audio, 3D (point clouds, mesh). We dont have a specific need for the surrogate lab. This is more a setup so other projects can setup what they need to generate. The rest sounds good. You can go ahead". The plan records what that changed: Pumas as an endpoint only, five kinds of record, "annotation" for "label", the lab's field comparison last. + +**Commits** on `feat/definition-first-projects`, none pushed: the plan and specs (`c7693b0d`, `1c08b2b0`, `685be616`, `f26361dc`); reading inside a product `710a5f01`; the Pumas client `d3270202` (the earlier branch merged), `04e1ddb2`, merged here as `be7af731`. + +**Phase 1, server side** ([spec-d1.md](spec-d1.md), slice D1a). Five read operations and one route open the output of any finished run: its records, one record, its files and archives, a slice and a summary of an array, and the bytes of an image or a sound. Kinds: text, image, audio, point cloud, mesh, array, file. Standard library only; nothing is unpickled, extracted or written. + +**Changed in review:** + +- The reads were answered under the service's one lock, so a large member being decompressed held up every other call. They now find the run under the lock and read outside it. +- The readers were fed 22,000 corrupted files. Compressed data that does not decompress and a binary PLY that ends early raised errors of their own; every entry point now refuses a damaged file as `unsupported` and logs it. A malformed `.npy` was coded `invalid`, which is kept for arguments; it is `unsupported` too. + +**Measured** on the lab's largest product (9.77 GB), through the readers alone: listing 0.004 s, one sample's arrays 0.004 s, a slice 0.009 s, a summary of 645,120 values 0.12 s. + +**Phase 3, the Pumas 0.8.0 client** ([spec-d3.md](spec-d3.md)), built beside phase 1 on a branch of its own. The client written on 2 November against a pre-release was merged, compared with the contract released at the tag `v0.8.0` (read only from the Pumas Library repository; the spec's "As built" has the table) and brought up to it. `served_models()` reads what an endpoint serves, with each model's modalities and capabilities. Nothing uses it yet. No running Pumas was contacted. + +**What Pumas 0.8.0 can make:** text and PNG images. Its contract reserves audio output and has nothing for 3D. Audio and 3D datasets therefore come from import or a project's own generator command until a served model declares them. + +**Tests:** project suite 610; `tests/test_tuldok_project_inside.py` 24; agent interface 43 (4 skipped); the Pumas client and its fakes 80, and the four catalog tools that call it (61); the two Pumas browser checks and the two controller checks. The catalog profile's manifest is unchanged by phase 1. + +**Not verified:** a real Pumas endpoint; the schemas the release does not publish (four are the pre-release export, one is derived); what spec-d1's "As built" lists. The standards service the implementers' rules name answered "awaiting application publication" on 2026-11-13, so no standards text was read for these slices. + +**Next:** the page of phase 1 (the record grid and a viewer per kind), and phase 2 (weights by layer, [spec-d2.md](spec-d2.md)), both being built. + +## 2026-11-13 — data tools: looking inside a dataset (page), weights (server), annotation (server) + +**Commits** on `feat/definition-first-projects`, none pushed: the page of phase 1 `7d376b61`; the weights readers `dfbeb10e`; annotation tasks and their file of lines `eefa274e`; the working copy and its operations `68aad8fb`; the steps `import` and `freeze` `e2ae4bb2`; merged here as `7262e8b4`. Specs: [spec-d2.md](spec-d2.md), [spec-d4.md](spec-d4.md), [spec-d5.md](spec-d5.md) (phase 5, written, not built). + +**How.** Slices were built by agents in working trees of their own, several at once where they touched different files, and merged after review. Each "As built" section of the specs has what differs from the text and what was not verified. + +**Phase 1, the page** ([spec-d1.md](spec-d1.md), slice D1b). A dataset with a product opens to its records as a paged grid with a viewer per kind: text, image, sound with its waveform, point cloud and mesh turned by dragging, array as a slice with the stacked-slice drawing and histogram; and a Files view of the product. Checked by `tests/browser_project_workspace_inside.cjs`, with screenshots read in light and dark at two widths. + +**Open defect in it.** In the reviewer's twelve further runs of that check, one failed: the main area's scroll went back to its top once, in the first second after the dataset opened, with its content unchanged. The owner's one complaint about this page so far was a full reset where an update in place was wanted, so this is not accepted as a flake: it is being reproduced in a loop with every way of moving the scroll traced. Also open: the Files view cannot yet open a text, point cloud or mesh that is not one of the product's records. + +**Phase 2, server side** (slice D2a): a run's weights by layer, a comparison of two files tensor by tensor, an array as tiles, `.safetensors` read like `.npz`. A PyTorch pickle is listed with its reason and never opened. + +**Phase 4, server side** (slices D4a to D4c): the annotation task as an eighth kind of definition; its annotations as a file of lines in Git, tied to each record's content by a hash; the working copy in the database, written by Save, with conflicts handled as for definitions; five operations; and `import` and `freeze` as steps Tuldok itself runs as ordinary builds. + +**Changed in review:** + +- `array_summary` (phase 1) answered `Infinity` for 64-bit values too large to sum, which is not JSON; found by feeding the weights readers 20,000 corrupted files. An overflowing mean or deviation is null. +- A line file rewritten to the same size within one tick of the clock looked unchanged by size and time: the edit was not imported and Save would have written over it. A file's size and time are trusted only once its time is two seconds old; a test that writes such an edit fails without the rule. +- The annotation file module was checked with 2,800 random annotations of every task written and read back byte for byte, and 3,000 corrupted files; `import` on a directory holding links out of it, a loop, a named pipe and names that collide. + +**Tests on the merged branch:** project suite 689; agent interface 43 (4 skipped). The catalog profile's manifest is unchanged throughout. + +**Not verified:** what each spec's "As built" lists; a real `.safetensors` written by the library; a copy across file systems in `import`; the standards service was unavailable (see the entry above). + +**Next:** the cause of the scroll going back; the Weights block of a model's drawer; the Annotate tab with the Import and Freeze forms; then phase 5. + +## 2026-11-13 — data tools: phases 1 to 5 are built + +**Commits** on `feat/definition-first-projects`, none pushed: the Weights block `8b7ff525`; the Annotate tab with Import and Freeze `c41f360c`; the endpoint, `generate` and suggestions, server side `f6231508`; their page `fc145a48`; the dataset view's scroll and `record` by path `32b1d0d9`; merged as `636da697` and `d95aeb4f`. + +**What a project folder has now** (the plan's five phases): + +1. **Looking inside data.** A dataset opens to its records as a paged grid with a viewer per kind (text, image, sound, point cloud, mesh, array) and a Files view in which any file or archive member opens. +2. **Weights by layer.** A model's drawer shows its weights file by layer, how each layer's values are distributed, a tensor as images, and the change from the weights it started from or from the model it derives from. +3. **The Pumas 0.8.0 client,** following the released contract. +4. **Annotation.** Tasks as a kind of definition; annotations as lines in Git; the Annotate tab with a form per task (decision, class, tags, text, boxes, spans, segments); Import and Freeze as steps Tuldok runs. +5. **Through the Pumas endpoint.** The endpoint as a setting of the machine, shown on the Train tab; a Generate form and the step `generate` for text and image datasets; Suggest in Annotate. + +**The open defect of the last entry is closed.** The dataset view's jump to its top was the browser's scroll anchoring: tiles fill in after the view is drawn, and when the tile the browser had chosen as its anchor had its picture replaced, the scroll was put back to 0. No code of the page moved it. A loop with every way of moving the scroll traced reproduced it in 3 of 12 iterations; with `overflow-anchor: none` on the main area, which keeps its scroll itself, 220 iterations were clean. The Annotate tab's grid fills the same way and is covered by the same rule. One correction to how this was handled: the agent sent to find it waited on a test run of the reviewer's that had been held up, and stood idle for about an hour and a half before that was noticed. + +**Acceptance on the merged tip `d95aeb4f`:** + +| Claim | Evidence | Result | +| --- | --- | --- | +| The project folder's code behaves as its tests say | `python -m unittest discover -s tests -p 'test_tuldok_project_*.py'`, on a disposable PostgreSQL | 719 pass | +| The agent interface and the Pumas client are unchanged in what catalog mode uses | `test_agent_api`, `test_agent_negotiation`, `test_agent_http`, and the seven modules of the Pumas client and its fakes | 123 pass, 4 skipped as before | +| Each page part does what its spec says, in a real browser against a served fixture | `browser_project_workspace_inside`, `_weights`, `_annotate`, `_endpoint`, `_actions`, `_still` and `browser_project_workspace` | 8, 7, 11, 8, 27, 6 and 23 steps pass | +| The lab's folder still opens | `python -m tuldok_project check` on `projects/surrogate-lab`, read only | 0 errors, the same 8 warnings | +| Generation and suggestions work against a real endpoint | one small text dataset, one small image dataset and one suggestion through the owner's Pumas | **not run**: it needs the owner's word, and uses that machine's GPU for a moment | +| The pages are what the owner wants | the owner looks at them | **pending** | + +Everything that speaks to Pumas was checked against a fake endpoint built from the released contract. No running Pumas was contacted and this machine's listeners were not scanned. + +**Known limits,** each in its spec's "As built": + +- The endpoint must be a numeric loopback address on this machine: that is what the 0.8.0 client sends to and what Pumas 0.8.0 serves on. +- Pumas 0.8.0 makes text and images. Audio and 3D datasets come from `import` or a project's own command. +- `set_annotation` does not refuse when the annotation changed since the form was drawn: of two people keeping the same record, the later one wins without a warning. +- Cancelling a suggestion drops its answer; the request to the endpoint runs to its end. +- The endpoint's state on the Train tab is not read again on the timer. +- The whole repository suite (catalog mode included, about an hour) was not run after the data tools; the project suite, the agent interface and the Pumas client were. + +**Not built:** phase 6, native, predicted and signed error side by side in the Analysis tab. + +**Next:** the owner looks at the pages; on the owner's word, the run against a real endpoint. + +## 2026-11-18: the Benchmarks tab opens on an accuracy-against-time chart; phase 6 is built + +**Owner, 2026-11-18:** "I agree to the changes. After that complete and test phase 6." + +**The Benchmarks tab** opened on the first benchmark with results, which in the lab defines no chart, so the accuracy-against-time chart was out of sight. It now opens, when no benchmark is asked for, on the first with results and a `tradeoff` chart, then the first with any chart (`36061650`; a step in `browser_project_workspace.cjs` checks it). The lab gained the same chart for its 127 earlier pressure models (`charts/pressure-models-error-against-speed.json`, lab commit `0cd5aaf72`): 127 points, 19 on the frontier. + +**Phase 6** (`4ff0ad42`, [spec-d6.md](spec-d6.md) with its "As built"): a product that holds `field-index.json` keeps fields; `fields` lists its cases and `field_compare` gives a slice of the native, predicted and signed error fields with whole-field figures; the Analysis tab shows them as three panels under "Kept fields", with the axis, the position, "Remove the mean" and the run's own reported figures. + +| Claim | Evidence | Result | +| --- | --- | --- | +| The readers and operations | `test_tuldok_project_fields` | 16 pass | +| Nothing else in the project folder's code changed in behaviour | the project suite on a disposable PostgreSQL | 735 pass | +| Catalog mode's manifest is unchanged | `test_agent_api`, `test_agent_negotiation`, `test_agent_http` | 43 pass, 4 skipped | +| The page | `browser_project_workspace_fields.cjs` (seven steps of spec section 5) and, again, `_weights`, `_inside` and `browser_project_workspace` | all pass | +| The lab's runs | `fields` and `field_compare` through the service on the owner's database, read only | 83 runs keep fields, all comparable; for run `7a700731…` Tuldok's mean-free relative L2 is 0.00286561549674967 and the run reported 0.0028656154967496624 | + +**Not verified:** masks in the page (no lab run has one); 1-D, 2-D and 4-D fields in the page; a field near the 16,777,216-value bound. In the lab, Analysis opens on the first subject of a benchmark, which may keep no fields: choose an earlier pressure model that does (83 of the 150 fits). + +**The data tools plan is built:** phases 1 to 6. **Not run:** anything against a real Pumas endpoint, which waits for the owner's word. diff --git a/docs/plans/definition-first-projects/plan.md b/docs/plans/definition-first-projects/plan.md new file mode 100644 index 00000000..c6fe5d43 --- /dev/null +++ b/docs/plans/definition-first-projects/plan.md @@ -0,0 +1,287 @@ +# Plan: Definition-first projects + +**Plan status:** `Approved` by the owner on 2026-11-10, with the decisions recorded under [Decisions](#decisions). Implementation proceeds one milestone at a time from the specifications in this folder. + +**Written:** 2026-11-10, against Tuldok `d8d91fbf`, with the Rheon Surrogate Lab project as the test case. + +## Summary + +A project folder becomes a small set of plain files that say what to build and how. Everything built from them (datasets, weights, raw results, logs, history) stays on the local machine, in Tuldok's database and a local product store. Tuldok builds the dependency graph in its database, rebuilds products from the definitions in order, stores results as rows with summaries computed once, and draws charts from the summaries. + +The test is the lab: a fresh clone of the lab, Tuldok and Rheon opens in seconds, shows the models, recipes and result charts, and lists the ordered steps that regenerate every dataset and model defined in it. + +## Why: what the lab project shows + +The lab's project folder in Git is 682 MB in 55,865 files. What the owner needs from it is about 12 MB (the cards, the recipes and a summary of results), plus dataset and benchmark definitions that do not exist in the project today. + +| In the folder today | Size | Why it is there | +| --- | ---: | --- | +| 31 benchmark result sets | 342.5 MB | Each embeds a copy of its raw source documents (189 MB) so every value can be checked against them; the 478,660 values take 146 MB at about 300 bytes each | +| Note and card records | 77.5 MB | The full text of every revision; the actual text is about 13 MB | +| Python environment listings | 56.2 MB | Every file of the Python install with its checksum, listed twice per record (25,765 files for the GPU install) | +| Run records (460) | 54.8 MB | Complete run documents with their admission proofs | +| Fork, extraction, composition, snapshot and operation records | 44.8 MB | Bookkeeping about the project's own history | +| `project.json` | 34.1 MB | Lists every one of the 53,977 records | +| Graphs, model definitions, recipes, studies, trials | 24.3 MB | Definitions together with every past revision | +| Run observations | 14.3 MB | Exact copies of earlier run documents | +| One record per stored file | 6.9 MB | 51,244 files, mostly one per file of a Python environment | +| Stored model source files | 11.0 MB | Captured copies of each model's source code | +| Cards, recipes and notes as plain files | 15.2 MB | The part the owner asked for | + +These are not accidents. They follow from six properties of the current design: + +1. **The folder is the whole record set, by rule.** "Only the inventory in .tuldok/project.json confers authority" (`project_store.py:3`), every record must be listed, and each commit carries a digest over all of them. There is no way to leave a family or a record out of the folder. +2. **History is stored inside the records.** Every save of a note, model, graph or study appends the complete new revision (`project_files.py:342`, `project_material.py:466`). A fork keeps both the old record and the new one. +3. **Everything is proven byte by byte.** Environments are recorded as per-file checksum lists and re-checked before and after each run. Every stored file needs its own record. Every result value must match a pointer into an embedded source document (`retained_measurements.py:230`). +4. **Nothing derived is stored.** The database holds records as opaque bytes in two tables (`project_working_store.py:37`). Every chart request re-reads and re-checks the whole result-set files; the only aggregate is the maximum per model; there are two fixed chart specifications and a chart cannot be saved. +5. **Runs are chained by hand.** The run queue is first-in, first-out (`runs.py:1909`). A step that uses another step's output names it by exact checksum. There is no dataset defined by a generator, and nothing can say "this is missing, build it". +6. **Two modes that cannot be combined.** GPU environments and live run progress exist only in the older catalog mode; projects use the project-files mode, which has neither. + +## Goals + +Stated by the owner on 2026-11-10: + +- **G1.** The Git folder holds only the minimum needed to rebuild everything: recipes, definitions, the necessary scripts and the high-level benchmark results. It does not hold generated data, or the data the results were calculated from. +- **G2.** Generated data lives only in Tuldok's local storage and can be regenerated. +- **G3.** Tuldok builds the dependency graph in its database and uses it. +- **G4.** Charts are not rebuilt from raw data on each request. +- **G5.** A clean install reproduces: clone, open, rebuild. +- **G6.** New work must be fully reproducible. Earlier lab work that cannot be reproduced is an accepted exception, and not every earlier model needs to be kept. + +## Design + +### 1. Three places + +| Place | Holds | In Git | If it is lost | +| --- | --- | --- | --- | +| Project folder | Definitions, notes and cards, result summaries | Yes | It is the source; Git has it | +| Tuldok database | The imported definitions, the graph, run history, result rows and summaries, the activity log | No | Re-import the folder, then re-run or restore products | +| Product store (Tuldok's own directory for the project, on the machine) | Generated files: datasets, weights, raw result files, logs | No | Re-run the steps. Files that cannot be regenerated, such as the earlier weights, are single copies (section 9) | + +The rule for deciding where something goes: if a person or an agent wrote it, it is a definition and belongs in the folder. If a command produced it, it is a product and stays local. The one deliberate exception is the result summary, which Tuldok writes into the folder so the results travel with the project. + +### 2. Definitions + +A definition is one small JSON file. There are seven kinds: + +| Kind | Says | Produces | +| --- | --- | --- | +| `dataset` | The command that generates or prepares the data, its settings, the splits, and which other datasets or models it is built from | A dataset | +| `model` | The architecture: the code entry point and its configuration | Nothing by itself | +| `recipe` | How to train: the model, the training datasets, the settings and seeds, the environment, the resources | Weights | +| `benchmark` | The cases (a dataset and split), the metrics, the subjects, and how to group the summary | Results | +| `step` | Any other command whose output something else needs, such as building the reference solver | Files | +| `environment` | The Python version, the package versions, the device and how to install it | An installed environment on this machine | +| `chart` | Which benchmark and metric to plot, how to group, which chart form | Nothing; it is drawn from summaries | + +Underneath, `dataset`, `recipe`, `benchmark` and `step` are the same thing: a command, its named inputs and its named outputs. The kinds exist so Tuldok knows where to show each one. + +Two illustrative files for the lab (field names are settled in milestone 1; numbers are placeholders): + +```json +{ + "kind": "dataset", + "title": "Diversity scenes v1", + "command": ["python", "-m", "surrogate.datasets", "generate", "diversity-v1", "--root", "{output}"], + "estimate": ["python", "-m", "surrogate.datasets", "plan", "diversity-v1"], + "inputs": {"solver": "reference-solver"}, + "environment": "lab-cpu", + "splits": {"train": 148, "development": 24, "reserved": 52} +} +``` + +```json +{ + "kind": "recipe", + "title": "Pilot stepper on dense scenes", + "model": "divergence-free-stepper", + "inputs": {"training": "sinusoidal-v1"}, + "command": ["python", "-m", "surrogate.train", "surrogate/configs/pilot-dense-k1.json", "{output}", "--device", "cuda"], + "environment": "cuda-torch", + "resources": {"gpu": 1, "wall_hours": 12, "disk_gb": 5}, + "outputs": {"weights": "model.pt", "log": "log.jsonl"} +} +``` + +Rules: + +- **One file per definition.** The file name is its name, and other definitions refer to it by that name. +- **No checksums of other documents.** A definition never records the checksum of another definition or of a script, so changing one file never forces a change in another. +- **Checked on import.** Each kind has a schema; unknown fields are refused with the file and field named. +- **One set of files for everyone.** Tuldok's screens, an agent and a text editor all edit the same files. + +### 3. The project folder + +``` +tuldok.json project name, id, format version, where the code root is +datasets/*.json +models/*.json +recipes/*.json +benchmarks/*.json +steps/*.json +environments/*.json +charts/*.json +results/*.csv result summaries, written by Tuldok on Save +notes/ cards and notes as plain files +``` + +There is no inventory of records, no `records/` tree, no receipts and no stored-file records. Scripts stay where they are in the repository; definitions name them by path. Git is the history of every file in the folder. + +For the lab, this folder is expected to be about 12 MB: the 205 cards (8.3 MB), the recipes (1.9 MB), the summaries (0.4 MB) and the definitions. + +### 4. The graph + +On import, Tuldok reads each definition's `inputs` and stores the graph in the database: one node per definition, one edge per input. Every node has a state: + +| State | Meaning | +| --- | --- | +| Not built | No product on this machine | +| Built | A product exists and matches the current definition | +| Out of date | A product exists, but the definition, its code or one of its inputs changed since | +| Running, failed | A run is in progress or ended in error | +| Cannot be rebuilt | An earlier lab item whose inputs or command are lost (section 10) | + +The graph is what the lineage map, the rebuild plan and "what does changing this affect" all read. It is not exported; it is rebuilt from the definitions whenever they change. + +"Out of date" is found without chains of checksums. Each product keeps a copy of its definition as it was when built, the Git commit of the code, and which products it consumed. Tuldok compares those with the present state and says what changed. It is information for the owner, never a reason to refuse to run. + +### 5. Products and runs + +**What a product records.** The definition as run, the Git commit, the input products, the package versions actually found, the device, start and end times, the output files with their sizes, and the log. Checksums are computed only for files that never change once written (weights, result files), and for large datasets only when they are archived or checked on request. + +**Runs start from committed code.** Tuldok runs each step from a clean copy of the commit, so a run is tied to a commit and later edits cannot disturb it. A run on uncommitted code is possible but is marked as a scratch run, and its product is never reported as reproducible. + +**Tuldok sets up environments.** On an explicit action Tuldok installs the environment a definition describes (the Python version and the listed package versions), or connects one that already exists on the machine. It never does this on Open. Which interpreter on this machine fulfils an environment is a local setting outside Git. + +**Environments are checked by version.** At the start of a run Tuldok checks that the Python and package versions match the `environment` definition and records what it found, including the GPU and driver. There is no per-file listing and no re-hashing of the install. + +**Limits belong to the definition.** Wall time, disk and GPU come from `resources`, with defaults, in place of fixed constants such as the 30-minute GPU limit and the 64 MiB output cap. Large outputs stay where the step wrote them in the product store; they are not copied. + +**GPU runs and live progress** work for projects: the log tail and the last progress line are available while a run is executing, to the screens and to agents. + +**Plan, then build.** "Plan" lists the steps needed to build a chosen definition, in dependency order, with what is already built and each step's own estimate of disk and time. It runs nothing. "Build" queues those steps on the existing run queue (one GPU slot, several CPU slots). There is one queue and one launcher, as now; the graph only decides the order. + +**Nothing runs on Open.** Opening a project reads files and never executes project code, installs anything or regenerates data. Setting up an environment is its own explicit action. A build needs the owner's approval of the plan. An owner can give an agent a budget (GPU hours, disk) for named definitions. + +### 6. Results and charts + +**What a benchmark writes.** One plain results file: a header (metrics with units, subjects, cases, grouping labels) and one row per value (subject, case, metric, value or the reason it is unavailable). It has no embedded source documents and no pointers. Its provenance is the run that wrote it. + +**What Tuldok stores.** On completion the rows go into a database table, and a summary table is computed once: for each benchmark, subject, metric and declared group, the count, the unavailable count, mean, median, 10th and 90th percentile, minimum and maximum. A benchmark definition can name further statistics. + +**What charts read.** A chart is a saved definition. Tuldok answers it from the summary table with one query. Looking at the individual cases behind a point reads the rows table, also by query. No result file is opened to draw a chart. + +**What goes to Git.** Save writes each benchmark's summary to `results/`, with the commit and date it was produced. For the lab's existing results that is 2,541 rows by model and metric (4,407 with the per-value settings), about 0.4 MB. A benchmark definition can ask for its per-case table to be saved as well; that is off by default. + +**On a fresh clone.** Tuldok imports the summaries, so every chart works at once. Per-case detail shows as not present until the benchmark is rebuilt or its product is restored. + +The first set of chart forms covers the Benchmarks screen of the workspace redesign (accuracy against speed with the best-trade-off line, error across checkpoints, comparison of selected models) and the two existing charts. + +### 7. Notes, history and bookkeeping + +- **Notes and cards** are plain files in `notes/`. Git keeps their history. Tuldok does not store revisions in the folder. +- **Run history and the activity log** (who did what, approvals, admissions) stay in the database. +- **Forks, extractions, compositions, snapshots and operation receipts** are dropped from the project format. A Git branch is a fork. + +### 8. Agent control + +Every operation in this plan is available through the agent interface and the command line, with the same checks as the screens: read and check definitions, read the graph and node states, plan, request a build, watch a run, query results and summaries, and create charts. Because definitions are plain files, an agent with access to the repository can also write them directly and ask Tuldok to check them. + +### 9. Product archive + +**Decided by the owner on 2026-11-12: not built now.** The weights stay with Tuldok on the local system, in Tuldok's own per-project directory, and never go into the Git repository. That directory is the product store of section 1, so the question this section answered, "where do the weights live", is answered without an archive. + +What the section proposed, kept for when a second machine needs the products: a product archive is a folder Tuldok writes on request, containing chosen products with their records, which can be copied to another disk or host and restored into another install. + +### 10. The lab project: conversion and the exception + +A one-time converter, owned by the lab, turns the existing project into the new folder: + +- **Models.** Each of the 205 earlier models becomes a `recipe` with its card and its extracted training recipe, marked as earlier work. Each carries one of three labels: can be rebuilt, can be rebuilt after another model, or cannot be rebuilt with the reason. Today 115 trace only to scenes and formulas and 88 depend on earlier models. +- **Training sets.** The 26 earlier training sets become `dataset` definitions where the command is known. Only 5 have a recorded command, 10 depend on earlier models' weights, and a few have unknown or lost producers; those are recorded as they are and labelled. +- **Results.** The 31 result sets are imported once into the rows table and summarised. +- **Products.** The existing weights and prepared inputs (16 GB, of which most files exist only on the owner's laptop) are registered in the product store where they are, so nothing is recomputed. +- **The old folder** stays on disk, outside Git. + +The owner may drop earlier models that are neither the best of their line nor needed by a later one; the converter takes a list. + +New work under `surrogate/` (dataset definitions, the pilot recipe, the benchmark suites, the reference solver build) is written as definitions from the start and must pass the clean-install test. + +## What changes in Tuldok + +| Area | Today | Under this plan | +| --- | --- | --- | +| Project folder | Record inventory, receipts and record families, 18 of them in the lab's project (`project_store.py`, `project_working_store.py`) | Definition files, notes, summaries | +| Definitions | Graphs, model drafts, materials, recipe materials, each with revisions (`project_material.py`, `pipeline.py`, `recipe_contract.py`) | Seven kinds of plain file, no revisions in the file | +| Environments | Per-file listings, captured copies, re-hashing (`runtime_snapshots.py`, `torch_runtime.py`) | Version check and a record of what was found | +| Runs | Hand-chained, fixed limits, GPU and progress in catalog mode only | Planned from the graph, limits per definition, GPU and progress for projects | +| Queue and launcher | `runs.py`, `run_capacity.py`, `recipe_process.py` | Kept | +| Results | Self-proving documents re-read per request (`retained_measurements.py`, `project_measurements.py`) | Rows and summaries in the database | +| Charts | Two fixed specifications | Saved chart definitions | +| History and bookkeeping | Records in the folder (`project_history.py`, `project_fork.py`, `project_extraction.py`, `project_composition.py`) | Database only, or dropped | +| Access, approvals, agent transport | `access.py`, `agent_api.py`, MCP, CLI | Kept; operations are replaced | +| Catalog mode | The older data workspaces | Not changed by this plan | + +The current project-files code is about 20,900 lines in 38 modules. Its record families refer to each other, so none can be removed alone. The proposal is therefore to build the new project format as a new, small core beside it, reusing the queue, the launcher, the database and the access layer, convert the lab, and then delete the old project-files modules. Tuldok's standing policy since 2026-10-20 already permits breaking format changes with an owner-led one-time migration. + +## Earlier decisions this plan changes + +| Decision | Now | +| --- | --- | +| "The external project directory owns scientific records. PostgreSQL is a disposable projection" (`docs/decisions/external-project-authority.md`) | The folder owns definitions and summaries. The database and product store own what was built. They are a cache that can be rebuilt by running, or restored from a product archive, not something to discard freely | +| "Save writes byte-identical files" and records stored as exact bytes (`docs/plans/database-working-store.md`) | No longer needed: definitions are plain authored files and Tuldok writes one only when it was edited in Tuldok | +| A clone with no database must reopen with identical history (`external-project-save-load/plan.md`) | A clone reopens with identical definitions, notes and summaries. Run history is local | +| "Do not introduce a second graph engine or scheduler" (`generic-scientific-projects/plan.md`) | Still one queue and one launcher. A project-level dependency graph is added, which that plan deferred | +| "No package install" and runtime capture by file listing | Tuldok installs and configures an environment on an explicit owner action, never on Open. The file listings go | +| Out of scope: "saved view persistence", "computed-on-open analyses" (`retained-analysis/plan.md`) | Charts are saved; summaries are computed when results arrive, not on open | + +Kept as they are: the database is the working copy and Save is explicit; Open never executes project code; one run queue; owner approval before anything runs; no per-file source checksum maps. + +## Milestones + +Each milestone ends with a check against the lab. Sizes are relative. + +| # | Milestone | Size | Done when | +| --- | --- | --- | --- | +| 1 | The folder, on paper. Schemas for the seven kinds, and the lab's converter | Small | The converted lab folder exists for the owner to read, is under 20 MB, and every file passes its schema | +| 2 | Open and Save. Tuldok imports the folder into new database tables and builds the graph | Medium | A fresh clone of the lab opens with an empty database in seconds and lists every dataset, model, recipe and benchmark with its links and state | +| 3 | Results and charts. Rows, summaries, saved charts, summaries written on Save | Medium | The lab's 31 result sets import once; per-model maxima equal today's comparison; charts are answered from summaries without opening a result file; a fresh clone draws them from `results/` alone | +| 4 | Plan. The rebuild plan with estimates | Small | Planning the pilot model lists its steps in order with disk estimates and runs nothing | +| 5 | Environments. Tuldok installs or connects an environment from its definition and checks it | Medium | On a machine without the lab's environments, Tuldok sets up the CPU and the GPU environment from their definitions and reports the versions it found. The GPU install is a multi-gigabyte download and needs the owner's go-ahead | +| 6 | Build. Runs from a commit, per-definition limits, GPU, live progress, results published by the run | Large | On a fresh clone Tuldok builds one tiny dataset, trains one tiny model on the GPU with live progress, benchmarks it and charts it. This run needs the owner's go-ahead | +| 7 | Serve the new core from the Tuldok app, with agent operations for all of the above | Medium | The app opens a project folder; an agent repeats milestone 6 through the interface alone | +| 8 | Convert the lab for good, then retire the old code. Earlier weights adopted into Tuldok's per-project directory, tracked export replaced, old project-files modules removed | Medium | The lab repository holds the new folder only; the clean-install test in the summary passes. Before any old module is deleted, a written check lists every capability of the old project-files mode as covered, moved or deliberately dropped, and the owner signs it off | + +The workspace redesign (the Map variant chosen on 2026-11-03) builds on milestones 2 and 3 and gets its own plan. + +## Out of scope + +- The workspace screens themselves, and how charts are drawn (Vega as now, or plain SVG as in the mockups). +- Annotation data. The same rule applies to it (annotations are authored, so they belong in Git; source media are referenced), but its file layout is a separate plan. +- Retiring catalog mode. +- Sandboxing. Steps remain ordinary local processes with the user's access. +- Hosting for product archives. + +## Decisions + +Decided by the owner on 2026-11-10: + +1. **The new project format is built beside the old one, and the old one is then deleted.** The condition is that we know the old one is no longer needed: milestone 8 does not delete anything until the written capability check is signed off. +2. **Runs require committed code**, with a marked scratch-run escape. +3. **Tuldok installs and configures environments.** This replaces the proposal's "checked, not installed". + +Decided by the owner on 2026-11-12: + +4. **The earlier weights stay with Tuldok on the local system**, in Tuldok's own per-project directory, which holds each project's data including its weights. They are not in the Git repository. No product archive is built now (section 9). + +Still open, and needed only before milestone 8: + +5. **Which earlier models to drop**, if any. +6. **Sign-off of the [capability check](capability-check.md)** before any old module is deleted. + +## Risks + +- **Less proof in Git.** A result in Git can no longer be checked against its raw source from the folder alone. Its provenance is the run record in the local database. This is the trade the goals ask for. +- **Local products are single copies.** Losing the machine loses weights and raw results that have no archive. This is already true today for most of the lab's stored files. +- **Size of the change.** It replaces the project-files core over several milestones. Each milestone is usable by itself and is checked on the lab before the next starts. +- **Summaries can lag.** The summaries in Git are as fresh as the last Save. Tuldok shows when results are newer than the saved summaries. diff --git a/docs/plans/definition-first-projects/spec-d1.md b/docs/plans/definition-first-projects/spec-d1.md new file mode 100644 index 00000000..d7603002 --- /dev/null +++ b/docs/plans/definition-first-projects/spec-d1.md @@ -0,0 +1,198 @@ +# Implementation spec: looking inside a product + +This is phase 1 of [the data tools plan](data-tools-plan.md). Section 1 of [spec-m1-m2.md](spec-m1-m2.md) (rules for implementers) applies, with the import rule as amended by [spec-m7.md](spec-m7.md), and the page follows [spec-w4.md](spec-w4.md) (it updates in place). The old project mode named in those rules is deleted; catalog mode's modules (`scientific_fields.py`, `volume_assets.py`, `pointclouds.py`, `meshes.py`, `audio_assets.py`) may be read for how they parse a format, and nothing is imported from them. + +**After this phase** Tuldok can open the output of any finished run of a project folder: page through its records, show a text, an image, an audio clip, a point cloud, a mesh or an array, and browse its files and archives. Nothing is written anywhere: no file, no row. Nothing here annotates, generates or calls Pumas. + +## 1. Terms + +- **Product:** the `output` directory of a run in state succeeded or adopted. It is named by its run id. +- **Record:** one item of a dataset. Its **kind** is one of `text`, `image`, `audio`, `pointcloud`, `mesh`, `array`, or `file` for anything else (listed, never opened). +- **Address:** a POSIX path relative to the product, never absolute, never containing `..`. A member of a zip archive is addressed by `!`: `training-view.zip!samples/a.npz`. At most two `!`. An `.npz` is a zip of `.npy` members; one of its arrays is named separately (`array`), without the `.npy`. + +## 2. Where a product's records come from + +In this order: + +1. **`records.jsonl` at the product's root.** One JSON object per line: + + | Key | | + | --- | --- | + | `id` | required; unique; 1 to 200 characters of `A-Z a-z 0-9 . _ -` | + | `kind` | required; one of the six | + | `path` | an address; required unless `kind` is `text` and `text` is given | + | `text` | the text itself, for a short text record (at most 64 KiB) | + | `array` | for kind `array` in an `.npz`: the array to show first | + | `split`, `rights` | optional strings | + | `origin`, `meta` | optional objects, kept and shown as they are | + + Other keys are kept and shown with `meta`. A line that is not such an object is counted and its line number reported (the first 20), and the rest are read. +2. **`view` on the dataset's definition** (new, optional, datasets only): + + ```json + "view": {"records": "lists/all.jsonl"} + "view": {"files": "training-view.zip!samples/*.npz", "kind": "array", "show": "target_pressure"} + ``` + + `records` names a record list elsewhere in the product. `files` is a pattern: `*` and `?` within one path segment, `**` across segments; it may reach into one archive. Each match is a record whose id is its address. `kind` (default: by extension) and `show` (the array drawn first) apply to every match. `records` and `files` exclude each other. `check` validates the shape of `view` and nothing about the product. +3. **Otherwise every regular file** under the product is a record, sorted by address, its id its address, its kind by extension: + + | Kind | Extensions | + | --- | --- | + | text | `.txt .md .json .jsonl .csv .tsv .yaml .yml .log` | + | image | `.png .jpg .jpeg .webp .gif .bmp` | + | audio | `.wav .flac .mp3 .ogg .opus .m4a` | + | pointcloud | `.xyz`, and `.ply` with no faces | + | mesh | `.obj .stl`, and `.ply` with faces | + | array | `.npy .npz` | + | file | everything else | + +## 3. Operations + +All five are `inspect` reads in `tuldok_project/operations.py`, served and published like the others. Every one takes `run`. A run that does not exist, is not finished or has no output directory is refused `not-found`. + +| Operation | Arguments | Result | +| --- | --- | --- | +| `records` | `offset`, `limit` (at most 200), `kind`, `split` | `records` (each: `id`, `kind`, `path`, `bytes`, `split`, `rights`, `origin`, `meta`; a text record's first 400 characters as `preview`), `total`, `source` (`records.jsonl`, `view` or `files`), `kinds` and `splits` (counts), `problems` | +| `record` | `id` | `record` (as above, with `media_type` and, for image and audio, `url`) and `content`, by kind, below | +| `inside` | `path` (default the root), `offset`, `limit` (at most 500) | `entries` (each: `name`, `path`, `type` of `directory`, `file`, `archive` or `array`, `bytes`, `kind`; for an array `shape` and `dtype`), `total`. A directory lists its children, an archive its members as a tree, an `.npz` its arrays | +| `array_slice` | `path`, `array` (for an `.npz`), `axes` (the two axes drawn, default the last two), `index` (one position per axis, default the middle; ignored for the two drawn), `most` (rows and columns at most, default 256, at most 512) | `shape`, `dtype`, `axes`, `index`, `rows`, `columns`, `stride` (1 unless subsampled to fit `most`), `values_f32` (base64, little-endian 32-bit floats, row by row), `min`, `max`, `nonfinite` | +| `array_summary` | `path`, `array`, `bins` (default 64, at most 256) | `shape`, `dtype`, `count`, `min`, `max`, `mean`, `std`, `zeros`, `nonfinite`, `histogram` (`low`, `high`, `counts`), `stride` (1 unless sampled) | + +**`content` of `record`, by kind:** + +| Kind | Content | Read from | +| --- | --- | --- | +| text | `text` (at most 262,144 characters), `characters`, `truncated`, `replaced` (bytes that were not UTF-8) | the file or the record's own `text` | +| image | `width`, `height` (from the file's header; null when not found) | PNG, JPEG, GIF, WebP, BMP headers. The pixels reach the page by `url` | +| audio | for a WAV of PCM (8, 16, 24, 32 bit) or 32-bit float: `channels`, `rate`, `frames`, `seconds`, `peaks` (at most 1,000 pairs of lowest and highest sample, scaled to -1..1, over all channels). Other formats: nulls | the `wave` module and `struct`. The sound reaches the page by `url` | +| pointcloud | `count`, `bounds`, `points_f32` (base64 x, y, z; at most 100,000 points, evenly sampled above that with `stride`), `colors_u8` (base64 r, g, b, or null) | PLY (ASCII and binary little-endian), XYZ text, and an `.npy` of shape (N, 3 or more) when the record says its kind is `pointcloud` | +| mesh | `vertices`, `triangles` (counts), `bounds`, `positions_f32`, `triangles_u32` (base64; polygons split into triangles as a fan) | PLY (ASCII and binary little-endian), OBJ (`v` and `f`, with `/` forms and negative indexes), STL (binary and ASCII) | +| array | `arrays` (each: `array`, `shape`, `dtype`, `bytes`), `show` (the record's `array`, else the view's `show`, else the first with two or more dimensions) | the `.npy` header of each member | +| file | nothing | | + +**Arrays.** NPY versions 1, 2 and 3; C and Fortran order; little- and big-endian; floats of 2, 4 and 8 bytes, signed and unsigned integers of 1 to 8 bytes, booleans. Anything else (complex, strings, structured types) is refused `unsupported`. An object array is refused and **nothing is ever unpickled**. The header is parsed with `ast.literal_eval` and is at most 64 KiB. + +**Reading no more than is needed.** A file on disk, and a member stored uncompressed in a zip that is on disk, are read through `mmap` at the offsets the slice needs. A compressed member is decompressed whole into memory, so is an archive inside an archive. What was decompressed is kept for the next read of the same member, least recently used first out. + +## 4. Bounds + +| | Bound | Beyond it | +| --- | --- | --- | +| One line of a record list | 64 KiB | the line is a problem | +| Files walked when there is no list and no view | 200,000 | the rest are not listed, and `problems` says so | +| A member decompressed into memory | 256 MiB, by the size the archive declares, and enforced while reading | refused `too-large` with the size | +| Kept decompressed data, all products together | 512 MiB | least recently used dropped | +| Values read for one summary | 4,000,000 | every n-th value is read; `stride` says n | +| A point cloud or mesh file parsed | 256 MiB | refused `too-large` | +| A mesh returned | 200,000 vertices, 400,000 triangles | refused `too-large` with the counts | +| Record indexes kept in memory | 8 products | least recently used dropped; an index is rebuilt when the list's size or time changes | + +## 5. The bytes of an image or a sound + +`GET /api/project/file?run=&path=
`, for whoever may call an `inspect` operation, and refused like one otherwise. It streams the file, or the archive member within the bound above. + +- The media type comes from a closed table by extension, for the image and audio extensions of section 2 only; those are sent `inline`. Everything else is `application/octet-stream` with `Content-Disposition: attachment`. No product file is ever sent as HTML, SVG or script. +- `X-Content-Type-Options: nosniff`, `Cache-Control: no-store`. +- One `Range: bytes=a-b` is honoured for a file on disk (a sound must be seekable); anything else in `Range` gets the whole file. +- This route is not an operation and is not in the capability manifest: an agent reads records, arrays and text through the operations, and not media bytes. + +In project-folder mode `app.py` refuses every other `/api/` GET as before. + +## 6. Confinement + +- An address is resolved under the product directory; the resolved real path must lie inside it. A symbolic link that leaves it is refused `not-found`, as is a missing file. Hard links are ordinary files. +- Archive members are read, never extracted to disk. +- No operation here opens the project for writing, imports files or changes a row. + +## 7. The page + +The Data tab has two states: the cards, as now, and one dataset opened (`#data/`). + +- **A card and the dataset's drawer** gain "Look inside" when the dataset has a current product. A dataset with several products (subjects) offers them in a select. +- **The dataset opened:** its title and the product's run, date and size; the kinds and splits with counts, as filters; a pager; the records as a grid of 48 tiles; the selected record's viewer beside the grid (below it under 1,100 px). A "Files" switch shows the product as a tree instead (`inside`), from which any file or array opens in the same viewers. Left and right arrows move the selection. +- **Tiles.** Text: its first characters. Image: the image, scaled by the browser, loaded lazily. Array: a small middle slice of the array shown first, read lazily. Audio, point cloud, mesh, file: the kind's glyph. Every tile carries the record's id. +- **Viewers:** + + | Kind | Shows | + | --- | --- | + | text | the text, wrapped, with its length; "truncated" when it is | + | image | the image fitted to the panel; a click toggles actual size; width and height | + | audio | the waveform from `peaks` and the browser's own player; length, rate, channels | + | pointcloud | the points on a canvas, turned by dragging, zoomed by the wheel; coloured by their own colours or by height; the count, and "every n-th shown" when sampled | + | mesh | the triangles, flat shaded, turned and zoomed the same way; above 60,000 triangles the vertices as points, saying so; the counts | + | array | the arrays with shape and type; for the chosen one a slice with the two axes, a slider per other axis, the range, and the value under the pointer; for three or more dimensions the stacked-slice drawing of the mockup (`drawVolume` in `docs/plans/project-workspace-redesign/mockups/screens.js`); the summary's histogram on request | + | every kind | the record's `origin`, `meta`, `rights` and `split` when it has them | + +- **Drawing.** Canvas 2D, no library, no external reference, the tokens of `base.css`, light and dark. New code in `static/project-workspace/inside.js`; `adapter.js` gains the calls. Catalog mode's `static/scientific-inspector.js` may be used through an adapter or its logic rewritten in the workspace's own module; whichever is chosen is recorded under "As built". +- **Updating in place** (spec-w4): the answers of these operations are not watched; they are forgotten when a watched answer changes, and the dataset opened is then drawn again in place, keeping page, selection and scroll. A timer or a return to the window with nothing changed touches nothing. + +## 8. Slices + +| | Slice | Files | +| --- | --- | --- | +| D1a | The readers, the five operations, `view` in the dataset schema and `check`, the file route, the contract text | `tuldok_project/inside.py` (and helpers beside it), `operations.py`, `service.py`, `schemas/dataset.schema.json`, `check.py`, `app.py`, `docs/contracts/project-folder.md`, `tests/test_tuldok_project_inside.py`, and the tests that count the project-folder operations | +| D1b | The page | `static/project-workspace/inside.js`, `tabs.js`, `adapter.js`, `base.css`, `app.py` (the asset list), `tests/browser_project_workspace_inside.cjs` | + +## 9. Tests + +**D1a, `tests/test_tuldok_project_inside.py`.** Every fixture is written by the test with the standard library (arrays with `struct`, sound with `wave`, a PNG with `zlib`), into a product made the way the existing tests make one. + +- Records from a list, from each form of `view`, and from files; paging, the kind and split filters and counts; a bad line reported and skipped. +- Each kind's `content`, compared with values the test computed itself. For arrays: every supported type, both byte orders, both orders of storage, a slice on each pair of axes of a four-dimensional array, subsampling, non-finite values; stored, compressed and nested members; `.npy` versions 1 and 2. +- Each refusal: an object array, a complex array, an address with `..`, an absolute address, a symbolic link out of the product, a member over the bound (with the bound lowered for the test), a mesh over its bound, a run that is not finished. +- The file route: the media type and disposition of an image, a sound, an `.svg` and an `.html`; a range; refused without the right to inspect. +- The five operations are in the `project_folder` profile and absent from `catalog`, whose manifest digest is unchanged. + +**D1b, `tests/browser_project_workspace_inside.cjs`**, on a fixture project whose product holds at least one record of every kind and more than 48 records in all, owner signed in, no GPU: + +1. Data shows "Look inside" on that dataset and not on one without a product. The dataset opens to 48 tiles, the counts by kind are right, the pager reaches the rest, a kind filter narrows the grid. +2. Each kind's viewer shows its record: the text's content; the image decoded at its true size; a playable sound with the right length and a drawn waveform; a point cloud and a mesh whose canvases are not blank and change when dragged; an array whose value under the pointer equals the fixture's and whose slider moves the slice. +3. "Files" lists the tree, opens an archive and an array inside it. +4. With the dataset open and a record selected, a return to the window and one timer reading add and remove no node (the counting of `browser_project_workspace_still.cjs`). +5. Light and dark: no viewer is blank in either. +6. The existing workspace checks pass unchanged in what they assert. + +With `--lab-root` nothing is asserted in this check (a fresh database has no products for the lab); the lab's own products are read by the reviewer through `tuldok_project/inside.py` directly, read only. + +## 10. Out of scope + +Annotating, importing, generating; thumbnails made on the server; video; glTF; a model's weights (phase 2 uses these operations); comparing predicted with native fields (phase 6). + +## As built + +### D1a, the readers and the operations + +What differs from the text above, and what was found. + +- **Seven kinds, not six.** `file` is a kind like the others in a record list and in `view.kind`. +- **The project is opened for a moment, like any read.** Section 6 says no operation here opens the project; finding the run and the dataset's `view` needs it. Each operation opens it as the other reads do (which imports files changed on disk), reads the run's state, directory and the definition's `view`, closes it, and only then reads files. It does not run the step that clears orphaned runs, which writes a row. +- **These reads do not hold the service's lock** (changed in review). The service answers one call at a time; as first built a large member being decompressed, or a large mesh being parsed, held up every other call, the page's own readings among them. The run is found under the lock and the files are read outside it. What `inside.py` keeps in memory has its own lock. +- **A damaged file is a refusal, never a failure** (found in review, by feeding the readers 22,000 corrupted files). Compressed data that does not decompress and a binary PLY that ends early raised errors of their own, which the app would have answered as an internal failure. Each of the six entry points now answers whatever a reader raises on bad bytes as `unsupported`, and logs it. A truncated or malformed `.npy` was coded `invalid`; it is `unsupported` too. `invalid` is kept for arguments. +- **A stored archive inside an archive is not decompressed.** Section 3 says an archive inside an archive is read into memory. Only a compressed one is; a stored one stays a part of the outer file's mapping. This is what makes the lab's 9.8 GB training view quick: its 783 inner `.npz` files are stored. +- **Tuldok reads zip archives with a parser of its own** (the central directory, zip64 included), because the standard library does not give a member's offset in the file without reading its local header through a file object. +- **Choices the text left open:** `problems` is a list of `{line, message}`; the one array of a plain `.npy` is named null; a one-dimensional array is drawn as one row; `characters` is null for a text over 16 MiB; a point cloud's `bounds` are those of the points returned; a malformed address is `invalid`, a missing file or a link out of the product `not-found`; `array` is required for an `.npz`; a summary's sampling takes every n-th value in the order stored. +- **Measured on the lab's largest product** (9.77 GB, 792 members), through the readers alone, no database: listing the archive 0.004 s; the arrays of one sample 0.004 s; a middle slice of its `features` 0.009 s; a summary of its 645,120 values 0.12 s. The disk's cache may have held the file. +- **Tests:** `tests/test_tuldok_project_inside.py`, 24; project suite 610; agent interface 43 (4 skipped, as before). The catalog profile's digest is unchanged. +- **Not verified:** a big-endian machine; a PLY whose lists are in an element other than its faces; a JPEG whose size is declared beyond its first megabyte; a product on a disk that is truly cold. A file of a finished run is read through `mmap`: if something truncated it while it was being read, the process would be killed. Tuldok never changes a finished run's output. + +### D1b, the page + +What differs from section 7, and why. + +- **The opened state is `#data/inside:`, not `#data/`.** `#data/` already opens the dataset's drawer beside the cards, and the existing checks rely on it (`browser_project_workspace_actions.cjs` edits a dataset from that drawer; `browser_project_workspace_still.cjs` counts it). `inside:` is a prefix of the selection like `edit:` and `plan:`. The opened view has no drawer. +- **"Look inside" sits under the card**, in a `.ds-item` that wraps the card's button (a button cannot hold a button), and in the drawer above Edit; it is drawn only when the dataset has a product with a run. A dataset with several products offers them in a select in the opened view's head, not on the card. +- **What redraws what.** A person's selection (a click, or the left and right arrows) sets the tiles' `aria-pressed` in place and replaces only the viewer; it is not in the address. A page, a filter, the Files switch and a level of the tree draw the whole view again, built off the page and put in place (spec-w4 R8). The arrows cross to the next or previous page at the ends of one. While the next record is read the viewer keeps the one on screen for 250 ms, then says "Reading…" (it is not on screen yet). A reading that changed something draws the opened view again in place (spec-w4 R3): `tabs.js` passes the drawer's `_again` mark to `LabInside.open`. +- **Files is browsed a level at a time**, with a line of links back up (Product / bundles / training-view.zip / samples), not as an expanding tree; a directory, an archive and an `.npy` or `.npz` open in place, an array or a file opens in the viewer. +- **What opens from the tree.** Arrays are read by their place (`array_slice` and `array_summary` take a path and an array's name). Any other file, on disk or a member of an archive, is read by `record` with `path` (added after review) and shown in the same viewer as a record: its id the address, its kind by its name, an image or a sound through the file route. +- **Tiles.** An image tile reads its record to use the `url` the operation gives (two requests a tile: the record, the bytes); an array tile reads its record (an `.npz` needs `show`, which `records` gives only from a list) and one `array_slice` of `most` 48. Tiles are filled by an `IntersectionObserver` on the main area, 48 at most a page. A tile's id is cut at its start when long, so the file's name shows; the whole id is its title and label. +- **The slice viewer is written in `inside.js`; catalog mode's `scientific-inspector.js` is not used.** That module is built around the catalog's field-sequence manifests (frames, fields named by the manifest, x, y and z axes, nested arrays of values) and writes its controls with `innerHTML`; an adapter would have had to make those manifests up from an `.npy` header. Here the slice is a heat map in a canvas, two axes chosen by selects when there are three or more, a slider for every other axis, the range, the value and full index under the pointer, and the summary's histogram on request. +- **The stacked-slice drawing** (the mockup's `drawVolume`) reads up to ten positions of the first axis not drawn, at `most` 24 each, coloured on one range across them, the slider's slice outlined. A one-dimensional array is drawn as a line, not as a single row of cells. +- **Point clouds and meshes** are drawn into an `ImageData` with a depth buffer (a mesh's triangles rasterised, flat shaded from a fixed light), z up; colours by height use the diverging ramp's two ends. Dragging turns, the wheel zooms, the arrow keys and + and − do the same on the focused canvas, "Reset view" puts it back. A frame is drawn on `requestAnimationFrame` only after one of these; canvases are drawn again when their box changes size or the theme changes, and at no other time. +- **Colours.** Five tokens were added to `base.css` in both themes, `--ramp-lo`, `--ramp-hi` (sequential), `--ramp-neg`, `--ramp-mid`, `--ramp-pos` (diverging about zero, used when a range has both signs). +- **The answers.** `adapter.js` has the five reads and the file route's address as `P.product`; their answers are kept like any other and forgotten when a reading finds a change. The slice drawn as a slider moves is asked for each time and not kept. +- **Files beyond section 8's list:** `static/project-workspace.html` gains the ` + + + + + + diff --git a/docs/plans/project-workspace-redesign/mockups/b-map.html b/docs/plans/project-workspace-redesign/mockups/b-map.html new file mode 100644 index 00000000..44636983 --- /dev/null +++ b/docs/plans/project-workspace-redesign/mockups/b-map.html @@ -0,0 +1,127 @@ + + + + + +Map · Rheon Surrogate Lab · Tuldok + + + + + +Skip to content +
+ t.Tuldok +

+ +
+
+
+
+
+ + +
+ + + + + + + diff --git a/docs/plans/project-workspace-redesign/mockups/c-ledger.html b/docs/plans/project-workspace-redesign/mockups/c-ledger.html new file mode 100644 index 00000000..a2c9a85b --- /dev/null +++ b/docs/plans/project-workspace-redesign/mockups/c-ledger.html @@ -0,0 +1,112 @@ + + + + + +Ledger · Rheon Surrogate Lab · Tuldok + + + + + +Skip to content +
t.Tuldok
+
+ +
+
+ + + + + + + diff --git a/docs/plans/project-workspace-redesign/mockups/index.html b/docs/plans/project-workspace-redesign/mockups/index.html new file mode 100644 index 00000000..93b112a6 --- /dev/null +++ b/docs/plans/project-workspace-redesign/mockups/index.html @@ -0,0 +1,36 @@ + + + + + +Three layouts · Tuldok + + + + + +
+

Three layouts for the project workspace

+

Same screen, same data, three structures. Switch between them from any page.

+ +
+ + diff --git a/docs/plans/project-workspace-redesign/mockups/preview/a.png b/docs/plans/project-workspace-redesign/mockups/preview/a.png new file mode 100644 index 00000000..bc913d8e Binary files /dev/null and b/docs/plans/project-workspace-redesign/mockups/preview/a.png differ diff --git a/docs/plans/project-workspace-redesign/mockups/preview/b.png b/docs/plans/project-workspace-redesign/mockups/preview/b.png new file mode 100644 index 00000000..f436086d Binary files /dev/null and b/docs/plans/project-workspace-redesign/mockups/preview/b.png differ diff --git a/docs/plans/project-workspace-redesign/mockups/preview/c.png b/docs/plans/project-workspace-redesign/mockups/preview/c.png new file mode 100644 index 00000000..6d36c220 Binary files /dev/null and b/docs/plans/project-workspace-redesign/mockups/preview/c.png differ diff --git a/docs/plans/project-workspace-redesign/mockups/shared/base.css b/docs/plans/project-workspace-redesign/mockups/shared/base.css new file mode 100644 index 00000000..b1962125 --- /dev/null +++ b/docs/plans/project-workspace-redesign/mockups/shared/base.css @@ -0,0 +1,447 @@ +/* Tuldok project workspace redesign · shared tokens and primitives. + Local only: no remote fonts or scripts. Ubuntu Sans is installed on the + supported Linux desktop; the implementation vendors it under static/. */ +:root { + color-scheme: light; + --page: #f3f5f7; + --surface: #ffffff; + --surface-2: #eef1f4; + --line: #dfe4ea; + --line-strong: #c5cdd6; + --ink: #1b2330; + --ink-2: #596679; + --ink-3: #8a94a3; + --accent: #2c5fb8; + --accent-ink: #ffffff; + --accent-wash: #e8eef9; + --focus: #2c5fb8; + --good: #237c60; + --warn: #b4700e; + --serious: #b7452b; + --good-wash: #e4f2ec; + --warn-wash: #fbf0dd; + --serious-wash: #f9e6e1; + --k-training: #178f74; + --k-architecture: #2e6fcf; + --k-refinement: #d2652f; + --k-composition: #8a94a3; + --mark: #2e6fcf; + --grid: #e6eaef; + --shadow: none; + --font: "Ubuntu Sans", "Noto Sans", system-ui, sans-serif; + --mono: "Ubuntu Sans Mono", "Ubuntu Mono", ui-monospace, monospace; + --radius: 4px; +} +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + color-scheme: dark; + --page: #0f1317; --surface: #171c22; --surface-2: #1f262e; + --line: #2a323c; --line-strong: #3a4552; + --ink: #e6eaf0; --ink-2: #aab3c0; --ink-3: #7c8694; + --accent: #6a9ae8; --accent-ink: #0f1317; --accent-wash: #1d2a40; --focus: #8db4f5; + --good: #4fb68f; --warn: #d9a04a; --serious: #e07a5f; + --good-wash: #15302a; --warn-wash: #342a16; --serious-wash: #3a2320; + --k-training: #22a07f; --k-architecture: #4f8ae0; --k-refinement: #d96f3c; --k-composition: #7c8694; + --mark: #4f8ae0; --grid: #222a33; + } +} +:root[data-theme="dark"] { + color-scheme: dark; + --page: #0f1317; --surface: #171c22; --surface-2: #1f262e; + --line: #2a323c; --line-strong: #3a4552; + --ink: #e6eaf0; --ink-2: #aab3c0; --ink-3: #7c8694; + --accent: #6a9ae8; --accent-ink: #0f1317; --accent-wash: #1d2a40; --focus: #8db4f5; + --good: #4fb68f; --warn: #d9a04a; --serious: #e07a5f; + --good-wash: #15302a; --warn-wash: #342a16; --serious-wash: #3a2320; + --k-training: #22a07f; --k-architecture: #4f8ae0; --k-refinement: #d96f3c; --k-composition: #7c8694; + --mark: #4f8ae0; --grid: #222a33; +} + +* { box-sizing: border-box; } +html, body { height: 100%; } +body { + margin: 0; background: var(--page); color: var(--ink); + font: 14px/1.5 var(--font); -webkit-font-smoothing: antialiased; +} +h1, h2, h3, h4 { margin: 0; font-weight: 600; letter-spacing: -0.01em; line-height: 1.25; } +h1 { font-size: 20px; } +h2 { font-size: 16px; } +h3 { font-size: 14px; } +p { margin: 0; max-width: 66ch; } +a { color: var(--accent); text-decoration: none; } +a:hover { text-decoration: underline; } +button, input, select { font: inherit; color: inherit; } +button { cursor: pointer; } +.sr-only { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; } +:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; border-radius: 2px; } +.mono { font-family: var(--mono); font-size: 12.5px; } +.num { font-variant-numeric: tabular-nums; } +.muted { color: var(--ink-2); } +.faint { color: var(--ink-3); } +.small { font-size: 12.5px; } + +/* Buttons: quiet by default. The one primary action per view is filled. */ +.btn { + display: inline-flex; align-items: center; gap: 6px; min-height: 30px; padding: 4px 10px; + border: 1px solid var(--line-strong); border-radius: var(--radius); background: var(--surface); color: var(--ink); +} +.btn:hover { background: var(--surface-2); } +.btn.primary { background: var(--accent); border-color: var(--accent); color: var(--accent-ink); } +.btn.primary:hover { filter: brightness(1.05); } +.btn.ghost { border-color: transparent; background: transparent; } +.btn.ghost:hover { background: var(--surface-2); } +.btn[disabled] { opacity: .5; cursor: not-allowed; } +.btn svg { width: 14px; height: 14px; flex: none; } +.btn.sm { min-height: 26px; padding: 2px 8px; font-size: 13px; } +.btn[aria-pressed="true"] { background: var(--accent-wash); border-color: var(--accent); } +input[type="search"], input[type="text"], select { + min-height: 30px; padding: 4px 8px; border: 1px solid var(--line-strong); border-radius: var(--radius); + background: var(--surface); color: var(--ink); +} +input::placeholder { color: var(--ink-3); } + +/* Kind and state marks: identity by a dot and a word, never color alone. */ +.chip { display: inline-flex; align-items: center; gap: 6px; font-size: 12.5px; color: var(--ink-2); white-space: nowrap; } +.chip::before { content: ""; width: 8px; height: 8px; border-radius: 50%; background: var(--k-composition); flex: none; } +.chip.training::before { background: var(--k-training); } +.chip.architecture::before { background: var(--k-architecture); } +.chip.refinement::before { background: var(--k-refinement); } +.chip.composition::before { background: var(--k-composition); } +.state { display: inline-flex; align-items: center; gap: 6px; font-size: 12.5px; color: var(--ink-2); white-space: nowrap; } +.state svg { width: 14px; height: 14px; flex: none; } +.state.good { color: var(--good); } +.state.warn { color: var(--warn); } +.state.serious { color: var(--serious); } +.state.neutral { color: var(--ink-3); } + +/* Rows: the dense list primitive shared by references, studies and models. */ +.rows { list-style: none; margin: 0; padding: 0; } +.rows > li { border-top: 1px solid var(--line); } +.rows > li:last-child { border-bottom: 1px solid var(--line); } +.row { + display: grid; align-items: center; gap: 12px; width: 100%; padding: 7px 8px; text-align: left; + border: 0; background: transparent; color: inherit; border-radius: 0; +} +.row:hover { background: var(--surface-2); } +.row[aria-pressed="true"], .row[aria-selected="true"] { background: var(--accent-wash); box-shadow: inset 3px 0 0 var(--accent); } +.row > span { min-width: 0; } +.row > span > .title, .row > span > .sub { display: block; } +.row .title { font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.row .sub { color: var(--ink-2); font-size: 12.5px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + +/* Section heading with a quiet one-line summary. */ +.sect { display: flex; align-items: baseline; gap: 12px; margin: 0 0 10px; } +.sect h2 { flex: none; } +.sect .sum { color: var(--ink-2); font-size: 13px; } +.sect .spacer { flex: 1; } + +/* Definition list for facts. */ +.facts { display: grid; grid-template-columns: max-content 1fr; gap: 4px 16px; margin: 0; font-size: 13px; } +.facts dt { color: var(--ink-2); } +.facts dd { margin: 0; overflow-wrap: anywhere; } + +/* Notes that need the user, listed not shouted. */ +.needs { list-style: none; margin: 0; padding: 0; } +.needs li { display: grid; grid-template-columns: 18px 1fr auto; gap: 10px; align-items: start; padding: 8px 0; border-top: 1px solid var(--line); } +.needs li:last-child { border-bottom: 1px solid var(--line); } +.needs svg { width: 16px; height: 16px; margin-top: 2px; } +.needs .approval svg { color: var(--accent); } +.needs .changed svg { color: var(--warn); } +.needs .missing svg { color: var(--serious); } + +/* Inspector (the right-hand or drawer panel). */ +.inspector { display: flex; flex-direction: column; gap: 18px; } +.inspector header { display: flex; flex-direction: column; gap: 6px; } +.inspector header .kicker { font-family: var(--mono); font-size: 12.5px; color: var(--ink-2); } +.inspector h2 { font-size: 17px; } +.inspector h3 { font-size: 13px; color: var(--ink-2); font-weight: 600; margin-bottom: 6px; } +.inspector p { font-size: 13.5px; line-height: 1.55; max-width: 60ch; } +.inspector .actions { display: flex; flex-wrap: wrap; gap: 8px; } +.inspector .note { font-size: 12.5px; color: var(--ink-3); } +.metrics { display: grid; grid-template-columns: repeat(2, 1fr); gap: 10px 16px; margin: 0; } +.metrics > div { display: flex; flex-direction: column; } +.metrics dt { font-size: 12px; color: var(--ink-2); } +.metrics dd { margin: 0; font-size: 16px; font-weight: 500; font-variant-numeric: tabular-nums; } +.metrics dd small { font-size: 12px; color: var(--ink-3); font-weight: 400; margin-left: 4px; } +.links { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 4px; } +.links button { text-align: left; padding: 6px 8px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); width: 100%; } +.links button:hover { background: var(--surface-2); } +.links .why { display: block; font-size: 12.5px; color: var(--ink-2); margin-top: 2px; } +.empty { color: var(--ink-2); font-size: 13.5px; max-width: 40ch; } + +/* Metric bar: one neutral mark; log scale from 1 (empty) to 1e-4 (full). */ +.mbar { display: inline-block; width: 56px; height: 6px; background: var(--grid); border-radius: 3px; overflow: hidden; vertical-align: middle; } +.mbar i { display: block; height: 100%; background: var(--mark); border-radius: 3px; } +.row[aria-pressed="true"] .mbar i { background: var(--accent); } + +/* Chart chrome. */ +.chart { position: relative; } +.chart svg { display: block; width: 100%; height: auto; } +.chart .axis text { fill: var(--ink-3); font-size: 11px; font-family: var(--font); } +.chart .axis line, .chart .grid line { stroke: var(--grid); stroke-width: 1; shape-rendering: crispEdges; } +.chart .axis .base { stroke: var(--line-strong); } +.chart .label { fill: var(--ink-2); font-size: 12px; paint-order: stroke; stroke: var(--surface); stroke-width: 3px; stroke-linejoin: round; } +.chart .pt { stroke: var(--surface); stroke-width: 2; } +.chart .pt.training { fill: var(--k-training); } +.chart .pt.architecture { fill: var(--k-architecture); } +.chart .pt.refinement { fill: var(--k-refinement); } +.chart .pt.composition { fill: var(--k-composition); } +.chart .pt.dim { opacity: .18; } +.chart .hit { fill: transparent; cursor: pointer; } +.chart .hit:focus-visible { outline: none; } +.chart .hit:focus-visible + .pt, .chart .hit:hover + .pt { stroke: var(--ink); } +.chart .sel { fill: none; stroke: var(--accent); stroke-width: 2; } +.chart .frontier { fill: none; stroke: var(--ink-3); stroke-width: 1.5; stroke-linejoin: round; } +.chart .line { fill: none; stroke: var(--mark); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +.chart .cross { stroke: var(--line-strong); stroke-width: 1; } +.tip { + position: absolute; z-index: 5; pointer-events: none; min-width: 160px; max-width: 280px; padding: 8px 10px; + background: var(--surface); border: 1px solid var(--line-strong); border-radius: var(--radius); font-size: 12.5px; color: var(--ink-2); +} +.tip b { display: block; color: var(--ink); font-size: 13px; font-weight: 600; } +.tip .v { color: var(--ink); font-weight: 500; font-variant-numeric: tabular-nums; } +.legend { display: flex; flex-wrap: wrap; gap: 14px; margin: 8px 0 0; padding: 0; list-style: none; font-size: 12.5px; color: var(--ink-2); } +.filters { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin-bottom: 14px; } +.filters .grp { display: inline-flex; gap: 4px; } +.filters .pop { margin-left: auto; font-size: 12.5px; color: var(--ink-3); } +table.tv { border-collapse: collapse; width: 100%; font-size: 13px; } +table.tv th, table.tv td { text-align: left; padding: 6px 8px; border-top: 1px solid var(--line); } +table.tv th { color: var(--ink-2); font-weight: 500; } +table.tv td.n, table.tv th.n { text-align: right; font-variant-numeric: tabular-nums; } + +/* Lineage list (git-graph style). */ +.tree-list { list-style: none; margin: 0; padding: 0; } +.tree-list .tsect { position: sticky; top: 0; z-index: 2; display: grid; grid-template-columns: var(--graphw, 72px) 1fr auto; gap: 12px; align-items: center; padding: 6px 8px; background: var(--surface); border-top: 1px solid var(--line); border-bottom: 1px solid var(--line); } +.tree-list .tsect h3 { font-size: 13px; font-weight: 600; } +.tree-list .tsect .sum { font-size: 12.5px; color: var(--ink-2); } +.tree-list .tsect svg { display: block; height: 20px; } +.tree-list .trow { display: grid; grid-template-columns: var(--graphw, 72px) 44px minmax(150px, 1fr) minmax(0, 1.4fr) 120px 64px 54px; gap: 12px; align-items: center; width: 100%; padding: 0 8px; min-height: 30px; border: 0; background: transparent; color: inherit; text-align: left; } +.tree-list .trow:hover { background: var(--surface-2); } +.tree-list .trow[aria-selected="true"] { background: var(--accent-wash); box-shadow: inset 3px 0 0 var(--accent); } +.tree-list .trow svg { display: block; height: 30px; } +.tree-list .trow .ck { font-family: var(--mono); font-size: 12.5px; color: var(--ink-2); } +.tree-list .trow .nm { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.tree-list .trow .pu { font-size: 12.5px; color: var(--ink-3); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +@media (max-width: 1100px) { .tree-list .trow { grid-template-columns: var(--graphw, 72px) 44px minmax(120px, 1fr) 120px 64px 54px; } .tree-list .trow .pu { display: none; } } +.tree-list .trow .ms { font-variant-numeric: tabular-nums; font-size: 12.5px; color: var(--ink-2); text-align: right; } +.tree-list .trow .rv { font-size: 12px; color: var(--ink-3); } +.tree-list .trow.related { background: var(--surface-2); } +.tree-list .lane0 { stroke: var(--line-strong); stroke-width: 1.5; fill: none; } +.tree-list .lanek { stroke-width: 2; fill: none; } +.tree-list .dot { stroke: var(--surface); stroke-width: 2; } +.tree-list .dot.training { fill: var(--k-training); } +.tree-list .dot.architecture { fill: var(--k-architecture); } +.tree-list .dot.refinement { fill: var(--k-refinement); } +.tree-list .dot.composition { fill: var(--k-composition); } +.tree-list .dot.sel { stroke: var(--accent); } +.lk-training { stroke: var(--k-training); } .lk-architecture { stroke: var(--k-architecture); } .lk-refinement { stroke: var(--k-refinement); } .lk-composition { stroke: var(--k-composition); } + +/* Lineage map (horizontal canvas). */ +.tree-map .tline { fill: none; stroke: var(--mark); stroke-width: 1.5; opacity: .75; } +.tree-map .tgrid { stroke: var(--grid); stroke-width: 1; shape-rendering: crispEdges; } +.tree-map .tlabel { fill: var(--ink-3); font-size: 10px; font-family: var(--font); } +.tree-map .ttitle { fill: var(--ink-2); font-size: 11px; font-family: var(--font); } +.tree-map .tpt { stroke: var(--surface); stroke-width: 1.5; cursor: pointer; } +.tree-map .tpt.training { fill: var(--k-training); } .tree-map .tpt.architecture { fill: var(--k-architecture); } .tree-map .tpt.refinement { fill: var(--k-refinement); } .tree-map .tpt.composition { fill: var(--k-composition); } +.tree-map .tpt.sel { stroke: var(--accent); stroke-width: 3; } +.tree-map .tpt.dim { opacity: .2; } +.tree-map { position: relative; overflow: auto; background: var(--surface); } +.tree-map .inner { transform-origin: 0 0; } +.tree-map svg { display: block; } +.tree-map .band { fill: none; stroke: var(--line); } +.tree-map .bandlabel { fill: var(--ink-2); font-size: 12px; font-weight: 600; } +.tree-map .lanelabel { fill: var(--ink-3); font-size: 11px; } +.tree-map .seq { stroke: var(--line-strong); stroke-width: 1.5; fill: none; } +.tree-map .der { stroke-width: 2; fill: none; opacity: .9; } +.tree-map .cmp { stroke: var(--ink-2); stroke-width: 1.5; stroke-dasharray: 4 4; fill: none; } +.tree-map .node { cursor: pointer; } +.tree-map .node circle { stroke: var(--surface); stroke-width: 2; } +.tree-map .node.training circle { fill: var(--k-training); } +.tree-map .node.architecture circle { fill: var(--k-architecture); } +.tree-map .node.refinement circle { fill: var(--k-refinement); } +.tree-map .node.composition circle { fill: var(--k-composition); } +.tree-map .node text { fill: var(--ink-2); font-size: 10.5px; font-family: var(--mono); text-anchor: middle; pointer-events: none; } +.tree-map .node.sel circle { stroke: var(--accent); stroke-width: 3; } +.tree-map .node.sel text { fill: var(--ink); font-weight: 600; } +.tree-map .node:hover circle, .tree-map .node:focus-visible circle { stroke: var(--ink); } +.tree-map .node:focus-visible { outline: none; } +.tree-map .node.dim { opacity: .25; } +.tree-map .hitr { fill: transparent; } +.minimap { position: absolute; left: 12px; bottom: 12px; width: 220px; height: 56px; background: var(--surface); border: 1px solid var(--line-strong); border-radius: var(--radius); cursor: pointer; } +.minimap svg { width: 100%; height: 100%; display: block; } +.minimap .vp { fill: var(--accent-wash); stroke: var(--accent); stroke-width: 1; opacity: .8; } +.minimap .d { fill: var(--ink-3); } +.zoomctl { display: inline-flex; gap: 4px; } + +@media (prefers-reduced-motion: no-preference) { + .drawer, .rail, .aside { transition: transform .18s ease, opacity .18s ease; } +} + +/* Layout switcher and inspector key. */ +.layouts { display: inline-flex; align-items: center; gap: 2px; font-size: 12.5px; } +.layouts .lbl { color: var(--ink-3); margin-right: 6px; } +.layouts a { padding: 3px 8px; border-radius: var(--radius); color: var(--ink-2); white-space: nowrap; } +.layouts a:hover { background: var(--surface-2); text-decoration: none; } +.layouts a[aria-current="page"] { background: var(--accent-wash); color: var(--ink); } +.layouts a.cmp { margin-left: 6px; color: var(--accent); } +.keylist { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 8px; font-size: 13px; color: var(--ink-2); } +.keylist li { display: grid; grid-template-columns: 34px 1fr; gap: 10px; align-items: center; } +.keylist i { display: block; height: 0; border-top: 2px solid var(--k-refinement); } +.keylist .k-faint i { border-top: 1.5px solid var(--line-strong); } +.keylist .k-dash i { border-top: 1.5px dashed var(--ink-2); } +.keylist .k-bar i { height: 6px; border: 0; background: linear-gradient(90deg, var(--mark) 60%, var(--grid) 60%); border-radius: 3px; } +.metrics dt.chip { font-size: 12px; } +.ov-row { display: grid; grid-template-columns: minmax(0, 1.5fr) minmax(0, 1fr); gap: 32px; margin: 24px 0 8px; } +.ov-history { margin-top: 24px; } +@media (max-width: 1100px) { .ov-row { grid-template-columns: 1fr; } } + +/* Map node labels, dataset rows, documents, preview. */ +.tree-map .node text.nl { paint-order: stroke; stroke: var(--surface); stroke-width: 3px; stroke-linejoin: round; } +.tree-map .tip { position: absolute; } +.ds-actions { display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 16px; } +.row.ds { grid-template-columns: 22px minmax(0, 1fr) 90px 70px 20px; } +.row.ds .kind { width: 18px; height: 18px; color: var(--ink-3); } +.row.ds .kind svg { width: 18px; height: 18px; } +.row.ds .path { font-family: var(--mono); font-size: 12px; color: var(--ink-2); } +.row.ds .path:hover { color: var(--accent); text-decoration: underline; } +.row.ds .glyph { display: inline-flex; width: 16px; height: 16px; } +.row.ds .glyph svg { width: 16px; height: 16px; } +.row.ds .glyph.good { color: var(--good); } .row.ds .glyph.neutral { color: var(--ink-3); } .row.ds .glyph.serious { color: var(--serious); } +.preview { width: 100%; aspect-ratio: 16 / 9; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface-2); display: block; } +.preview-cap { font-size: 12px; color: var(--ink-3); margin-top: 4px; } +.docs { display: inline-flex; gap: 4px; flex-wrap: wrap; } +.docs a, .docs button { font: inherit; font-size: 12.5px; padding: 2px 8px; border: 1px solid var(--line); border-radius: 999px; background: var(--surface); color: var(--ink-2); cursor: pointer; } +.docs a:hover, .docs button:hover { background: var(--surface-2); color: var(--ink); text-decoration: none; } +.docs button[aria-pressed="true"] { background: var(--accent-wash); border-color: var(--accent); color: var(--ink); } +table.tv td .docs { margin-top: 2px; } + +/* Series slots for compare charts (identity by legend, validated hues). */ +.chart .line.s1, .chart .pt.s1 { stroke: var(--k-architecture); } .chart .pt.s1 { fill: var(--k-architecture); stroke: var(--surface); } +.chart .line.s2, .chart .pt.s2 { stroke: var(--k-refinement); } .chart .pt.s2 { fill: var(--k-refinement); stroke: var(--surface); } +.chart .line.s3, .chart .pt.s3 { stroke: var(--k-training); } .chart .pt.s3 { fill: var(--k-training); stroke: var(--surface); } +.legend .key { display: inline-flex; align-items: center; gap: 6px; } +.legend .key::before { content: ""; width: 14px; height: 0; border-top: 2px solid var(--ink-3); } +.legend .key.s1::before { border-color: var(--k-architecture); } .legend .key.s2::before { border-color: var(--k-refinement); } .legend .key.s3::before { border-color: var(--k-training); } +/* Sortable table, compare chips. */ +table.tv th button { font: inherit; color: inherit; background: none; border: 0; padding: 0; cursor: pointer; } +table.tv th button[aria-sort="ascending"]::after { content: " ↑"; color: var(--ink-3); } table.tv th button[aria-sort="descending"]::after { content: " ↓"; color: var(--ink-3); } +table.tv tr.is-sel td { background: var(--accent-wash); } +table.tv tr.rowbtn { cursor: pointer; } table.tv tr.rowbtn:hover td { background: var(--surface-2); } +table.tv td.ck { font-family: var(--mono); font-size: 12.5px; color: var(--ink-2); } +table.tv input[type="checkbox"] { margin: 0; } +.cmp-chips { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; } +.cmp-chips .c { display: inline-flex; align-items: center; gap: 6px; padding: 2px 6px 2px 10px; border: 1px solid var(--line); border-radius: 999px; font-size: 12.5px; background: var(--surface); } +.cmp-chips .c i { width: 10px; height: 10px; border-radius: 50%; display: inline-block; } +.cmp-chips .c button { border: 0; background: none; padding: 0 2px; color: var(--ink-3); font-size: 14px; line-height: 1; cursor: pointer; } +.cmp-chips .c.s1 i { background: var(--k-architecture); } .cmp-chips .c.s2 i { background: var(--k-refinement); } .cmp-chips .c.s3 i { background: var(--k-training); } +table.cmpt td.d { color: var(--ink-2); font-size: 12.5px; } +table.cmpt td.d.better { color: var(--good); } table.cmpt td.d.worse { color: var(--serious); } +/* Train setup and new-training form. */ +.setup { display: grid; grid-template-columns: max-content 1fr; gap: 6px 14px; font-size: 13px; margin: 0; } +.setup dt { color: var(--ink-2); } .setup dd { margin: 0; overflow-wrap: anywhere; } +.setup dd button.btn { vertical-align: middle; } +.form { display: grid; gap: 10px; } +.form label { display: grid; gap: 3px; font-size: 12.5px; color: var(--ink-2); } +.form select, .form input, .form textarea { width: 100%; font: inherit; color: var(--ink); background: var(--surface); border: 1px solid var(--line-strong); border-radius: var(--radius); padding: 5px 8px; } +.form .two { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; } +.form .hint { font-family: var(--mono); font-size: 11.5px; color: var(--ink-3); } +.trials { list-style: none; margin: 0; padding: 0; } .trials li { display: grid; grid-template-columns: 18px 1fr auto; gap: 8px; align-items: center; padding: 5px 0; border-top: 1px solid var(--line); font-size: 13px; } +.row.study { grid-template-columns: 90px minmax(0, 1fr) 70px 100px 90px; } +/* Field comparison viewer. */ +.viewer-bar { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin-bottom: 12px; } +.viewer-bar label { display: inline-flex; gap: 6px; align-items: center; font-size: 12.5px; color: var(--ink-2); } +.viewer-bar input[type="range"] { width: 140px; } +.tri { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 14px; } +.tri figure { margin: 0; } +.tri canvas { display: block; width: 100%; aspect-ratio: 1; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface-2); } +.tri figcaption { font-size: 12.5px; color: var(--ink-2); margin-top: 6px; display: flex; justify-content: space-between; } +.tri figcaption b { color: var(--ink); font-weight: 500; } +.scale { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--ink-3); margin-top: 8px; } +.scale i { display: inline-block; width: 120px; height: 8px; border-radius: 4px; background: linear-gradient(90deg, var(--k-architecture), var(--surface-2), var(--k-refinement)); } +.cases { margin-top: 16px; } +.cases svg { display: block; width: 100%; height: 64px; } +.cases .b { fill: var(--mark); } .cases .b.sel { fill: var(--accent); } .cases .b:hover { fill: var(--ink); cursor: pointer; } +.cases .base { stroke: var(--line-strong); } + +.tree-map .der.cold { opacity: .18; } .tree-map .der.hot { opacity: 1; stroke-width: 2.5; } .tree-map .der { opacity: .55; stroke-width: 1.5; } +table.tv td.nowrap, table.tv td.ck { white-space: nowrap; } +table.tv td .chip { white-space: nowrap; } + +/* Planned models, kin emphasis, label visibility, drawer motion. */ +.tree-map .node text.nl.off { display: none; } +.tree-map .node.kin circle { stroke: var(--ink); } +.tree-map .node.planned circle { fill: var(--surface); stroke: var(--accent); stroke-dasharray: 3 2; stroke-width: 2; } +.tree-map .der.hot { opacity: 1; stroke-width: 2.5; } +.tree-list .dot.planned { fill: var(--surface); stroke: var(--accent); stroke-dasharray: 3 2; } +.tree-list .trow.planned .nm { color: var(--accent); } +.ds-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(230px, 1fr)); gap: 14px; } +.ds-card { display: block; text-align: left; padding: 0; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); color: inherit; overflow: hidden; } +.ds-card:hover { border-color: var(--line-strong); } +.ds-card[aria-pressed="true"] { border-color: var(--accent); box-shadow: inset 0 0 0 1px var(--accent); } +.ds-card .pv { display: block; width: 100%; aspect-ratio: 16 / 9; background: var(--surface-2); border-bottom: 1px solid var(--line); } +.ds-card .pv.text { display: grid; place-items: center; color: var(--ink-3); font: 12px var(--mono); padding: 10px; white-space: pre; overflow: hidden; } +.ds-card .t { padding: 10px 12px 12px; display: grid; gap: 3px; } +.ds-card .t .title { font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.ds-card .t .meta { display: flex; gap: 8px; align-items: center; font-size: 12.5px; color: var(--ink-2); } +.ds-card .t .meta .glyph { display: inline-flex; margin-left: auto; } +.ds-card .t .meta .glyph svg { width: 14px; height: 14px; } +.ds-card .t .meta .glyph.good { color: var(--good); } .ds-card .t .meta .glyph.neutral { color: var(--ink-3); } .ds-card .t .meta .glyph.serious { color: var(--serious); } +.ds-card .t .path { font-family: var(--mono); font-size: 11.5px; color: var(--ink-3); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.ds-actions .btn[aria-pressed="true"] { background: var(--accent-wash); border-color: var(--accent); } +.form .radio { display: flex; flex-wrap: wrap; gap: 6px; } +.form .radio label { display: inline-flex; align-items: center; gap: 6px; padding: 4px 10px; border: 1px solid var(--line-strong); border-radius: 999px; color: var(--ink); cursor: pointer; } +.form .radio input { margin: 0; } +.form .radio label:has(input:checked) { background: var(--accent-wash); border-color: var(--accent); } + +/* Selection walk, comparison arcs, record browser and volume viewer. */ +.tree-list .trow.walk { background: var(--accent-wash); } +.tree-map .node.walk circle { stroke: var(--accent); stroke-width: 3; } +.tree-map .cmp { pointer-events: stroke; stroke-width: 1.5; cursor: pointer; } +.tree-map .cmp:hover, .tree-map .cmp:focus-visible { stroke: var(--accent); stroke-width: 2.5; outline: none; } +.ds-head { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; margin-bottom: 14px; } +.ds-head h2 { font-size: 17px; } +.ds-head .meta { display: flex; gap: 12px; align-items: center; font-size: 12.5px; color: var(--ink-2); } +.ds-head .meta .glyph { display: inline-flex; } .ds-head .meta .glyph svg { width: 14px; height: 14px; } +.ds-head .actions { margin-left: auto; display: flex; gap: 8px; } +.ds-body { display: grid; grid-template-columns: minmax(0, 1fr) minmax(360px, 44%); gap: 20px; align-items: start; } +.rec-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(84px, 1fr)); gap: 8px; } +.rec { padding: 0; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); overflow: hidden; text-align: left; color: inherit; } +.rec canvas { display: block; width: 100%; aspect-ratio: 1; } +.rec span { display: block; font: 11px var(--mono); color: var(--ink-3); padding: 3px 6px; } +.rec[aria-pressed="true"] { border-color: var(--accent); box-shadow: inset 0 0 0 1px var(--accent); } +.rec:hover { border-color: var(--line-strong); } +.pager { display: flex; gap: 8px; align-items: center; font-size: 12.5px; color: var(--ink-2); margin: 10px 0 0; } +.viewer { border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); padding: 12px; display: grid; gap: 10px; } +.viewer .vol { display: block; width: 100%; aspect-ratio: 4 / 3; background: var(--surface-2); border-radius: var(--radius); } +.viewer .row2 { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; } +.viewer .sl { display: block; width: 100%; aspect-ratio: 1; background: var(--surface-2); border-radius: var(--radius); } +.viewer .ctl { display: flex; flex-wrap: wrap; gap: 8px 14px; align-items: center; font-size: 12.5px; color: var(--ink-2); } +.viewer .ctl label { display: inline-flex; gap: 6px; align-items: center; } +.viewer .ctl input[type="range"] { width: 120px; } +.viewer .facts { font-size: 12.5px; } +.ds-back { margin-bottom: 10px; } +@media (max-width: 1100px) { .ds-body { grid-template-columns: 1fr; } } + +/* Annotation screen, run rows, benchmark coverage. */ +.ann-body { display: grid; grid-template-columns: minmax(280px, 1fr) minmax(360px, 1.1fr); gap: 20px; align-items: start; } +.ann-bar { display: flex; flex-wrap: wrap; gap: 8px 14px; align-items: center; margin-bottom: 14px; } +.ann-bar label { display: inline-flex; gap: 6px; align-items: center; font-size: 12.5px; color: var(--ink-2); } +.progress { display: inline-flex; align-items: center; gap: 8px; font-size: 12.5px; color: var(--ink-2); } +.progress i { display: inline-block; width: 120px; height: 6px; border-radius: 3px; background: var(--grid); overflow: hidden; } +.progress i b { display: block; height: 100%; background: var(--good); } +.rec .badge { position: absolute; right: 4px; top: 4px; width: 8px; height: 8px; border-radius: 50%; } +.rec { position: relative; } +.rec .badge.accepted { background: var(--good); } .rec .badge.needs { background: var(--warn); } .rec .badge.reject { background: var(--serious); } .rec .badge.labelled { background: var(--accent); } +.seg { display: inline-flex; border: 1px solid var(--line-strong); border-radius: var(--radius); overflow: hidden; } +.seg button { border: 0; background: var(--surface); padding: 5px 10px; font: inherit; color: var(--ink-2); cursor: pointer; } +.seg button + button { border-left: 1px solid var(--line-strong); } +.seg button[aria-pressed="true"] { background: var(--accent-wash); color: var(--ink); } +.seg button.ok[aria-pressed="true"] { background: var(--good-wash); } .seg button.warn[aria-pressed="true"] { background: var(--warn-wash); } .seg button.bad[aria-pressed="true"] { background: var(--serious-wash); } +.hist { list-style: none; margin: 0; padding: 0; font-size: 12.5px; color: var(--ink-2); } +.hist li { padding: 5px 0; border-top: 1px solid var(--line); display: flex; gap: 10px; } +.hist li b { color: var(--ink); font-weight: 500; } +.kbd { font: 11px var(--mono); border: 1px solid var(--line-strong); border-radius: 3px; padding: 0 4px; color: var(--ink-2); } +.row.run { grid-template-columns: 44px minmax(160px, 1fr) 120px 90px 50px 100px; } +.row.run .ck { font-family: var(--mono); font-size: 12.5px; color: var(--ink-2); } +.coverage { display: flex; flex-wrap: wrap; gap: 6px 18px; font-size: 12.5px; color: var(--ink-2); margin: -6px 0 14px; } +.coverage b { color: var(--ink); font-weight: 500; } diff --git a/docs/plans/project-workspace-redesign/mockups/shared/charts.js b/docs/plans/project-workspace-redesign/mockups/shared/charts.js new file mode 100644 index 00000000..45e24504 --- /dev/null +++ b/docs/plans/project-workspace-redesign/mockups/shared/charts.js @@ -0,0 +1,121 @@ +'use strict'; +// Benchmark charts in plain SVG: an accuracy/time scatter with its Pareto +// frontier, and the worst-pressure curve along the checkpoint sequence. +// Every value is also in the table view; tooltips enhance, never gate. +window.LabCharts = (() => { + const svgNS = 'http://www.w3.org/2000/svg'; + const el = (name, attrs = {}, text) => { const n = document.createElementNS(svgNS, name); for (const k in attrs) n.setAttribute(k, attrs[k]); if (text !== undefined) n.textContent = text; return n; }; + const h = (tag, cls, text) => { const n = document.createElement(tag); if (cls) n.className = cls; if (text !== undefined) n.textContent = text; return n; }; + const fmtSci = v => window.LabTree.fmtSci(v); + const logTicks = (lo, hi) => { const out = []; for (let e = Math.ceil(Math.log10(lo)); e <= Math.floor(Math.log10(hi)); e++) out.push(Math.pow(10, e)); return out; }; + const tickLabel = v => v >= 1 ? String(v) : (v >= 0.01 ? v.toString() : '1e' + Math.log10(v)); + + function tooltip(host) { + const t = h('div', 'tip'); t.hidden = true; host.appendChild(t); + return { + show(x, y, rows) { t.textContent = ''; for (const r of rows) { if (r.b) { t.appendChild(h('b', null, r.b)); continue; } const line = h('div'); const v = h('span', 'v', r.v); line.appendChild(v); line.appendChild(document.createTextNode(' ' + r.l)); t.appendChild(line); } t.hidden = false; const W = host.clientWidth; t.style.left = Math.min(x + 14, W - 200) + 'px'; t.style.top = (y + 12) + 'px'; }, + hide() { t.hidden = true; } + }; + } + + function pareto(host, models, { selected = null, onSelect = () => {}, kinds = null, frontierOnly = false, metric = 'worstP', table = false, yLabel = '', xLabel = '', unit = 'ms', xMax = 10 } = {}) { + host.textContent = ''; host.classList.add('chart'); + const pts = models.filter(m => m.ms != null && m.ms <= xMax && m[metric] != null && m[metric] > 0); + const shown = pts.filter(m => !kinds || kinds.has(m.kind)); + const sorted = shown.slice().sort((a, b) => a.ms - b.ms || a[metric] - b[metric]); + const front = []; let best = Infinity; for (const m of sorted) { if (m[metric] < best) { front.push(m); best = m[metric]; } } + const frontSet = new Set(front.map(m => m.id)); + const W = 760, H = 340, L = 56, R = 16, T = 16, B = 40; + const ylo = 1e-4, yhi = 1; + const X = ms => L + (ms / xMax) * (W - L - R), Y = v => T + (Math.log10(yhi) - Math.log10(Math.min(Math.max(v, ylo), yhi))) / (Math.log10(yhi) - Math.log10(ylo)) * (H - T - B); + const svg = el('svg', { viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': yLabel + ' on a log scale against ' + xLabel }); + const grid = el('g', { class: 'grid' }), axis = el('g', { class: 'axis' }); + for (const v of logTicks(ylo, yhi)) { grid.appendChild(el('line', { x1: L, y1: Y(v), x2: W - R, y2: Y(v) })); axis.appendChild(el('text', { x: L - 8, y: Y(v) + 4, 'text-anchor': 'end' }, tickLabel(v))); } + const step = xMax > 10 ? 4 : 2; for (let ms = 0; ms <= xMax; ms += step) { axis.appendChild(el('line', { x1: X(ms), y1: H - B, x2: X(ms), y2: H - B + 4, class: 'base' })); axis.appendChild(el('text', { x: X(ms), y: H - B + 16, 'text-anchor': 'middle' }, ms + (ms + step > xMax ? ' ' + unit : ''))); } + axis.appendChild(el('line', { x1: L, y1: H - B, x2: W - R, y2: H - B, class: 'base' })); + axis.appendChild(el('text', { x: W - R, y: H - 4, 'text-anchor': 'end', class: 'label' }, xLabel)); + axis.appendChild(el('text', { x: L, y: T - 4, class: 'label' }, yLabel)); + svg.appendChild(grid); svg.appendChild(axis); + if (front.length > 1) svg.appendChild(el('path', { class: 'frontier', d: front.map((m, i) => (i ? 'L' : 'M') + X(m.ms) + ',' + Y(m[metric])).join(' ') })); + const tip = tooltip(host); + const order = shown.slice().sort((a, b) => (a.id === selected) - (b.id === selected)); + for (const m of order) { + const dim = frontierOnly && !frontSet.has(m.id); + const g = el('g'); + const hit = el('circle', { cx: X(m.ms), cy: Y(m[metric]), r: 12, class: 'hit', tabindex: 0, role: 'button', 'aria-label': `${m.ck || m.short}, ${fmtSci(m[metric])} at ${m.ms.toFixed(2)} milliseconds${frontSet.has(m.id) ? ', on the frontier' : ''}` }); + const pt = el('circle', { cx: X(m.ms), cy: Y(m[metric]), r: 5, class: 'pt ' + m.kind + (dim ? ' dim' : '') }); + const rows = () => [{ b: (m.ck ? m.ck + ' · ' : '') + m.short }, { v: fmtSci(m[metric]), l: yLabel }, { v: m.ms.toFixed(2) + ' ' + unit, l: xLabel }, { v: window.LabTree.kindLabel(m.kind), l: frontSet.has(m.id) ? '· frontier' : '' }]; + const pos = () => { const r = host.getBoundingClientRect(); const s = svg.getBoundingClientRect(); return [s.left - r.left + X(m.ms) * s.width / W, s.top - r.top + Y(m[metric]) * s.height / H]; }; + hit.addEventListener('pointerenter', () => { const [x, y] = pos(); tip.show(x, y, rows()); }); + hit.addEventListener('focus', () => { const [x, y] = pos(); tip.show(x, y, rows()); }); + hit.addEventListener('pointerleave', () => tip.hide()); hit.addEventListener('blur', () => tip.hide()); + hit.addEventListener('click', () => onSelect(m.id)); hit.addEventListener('keydown', e => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); onSelect(m.id); } }); + g.appendChild(hit); g.appendChild(pt); + if (m.id === selected) g.appendChild(el('circle', { cx: X(m.ms), cy: Y(m[metric]), r: 9, class: 'sel' })); + svg.appendChild(g); + } + if (front.length) { const b = front[front.length - 1]; svg.appendChild(el('text', { x: X(b.ms) + 10, y: Y(b[metric]) + 4, class: 'label' }, (b.ck || b.short) + ' · best accuracy')); const f = front[0]; if (f !== b) svg.appendChild(el('text', { x: X(f.ms) + 10, y: Y(f[metric]) + 18, class: 'label' }, (f.ck || f.short) + ' · fastest')); } + host.appendChild(svg); + const legend = h('ul', 'legend'); + for (const k of ['training', 'architecture', 'refinement', 'composition']) { const li = h('li', 'chip ' + k, window.LabTree.kindLabel(k)); legend.appendChild(li); } + const fl = h('li', null, 'Line: Pareto frontier'); legend.appendChild(fl); + host.appendChild(legend); + if (table) { + const tb = h('table', 'tv'); tb.setAttribute('aria-label', 'Plotted fits as a table'); + const thead = h('thead'); const tr = h('tr'); for (const [t, c] of [['Fit', ''], ['Kind', ''], [yLabel, 'n'], [xLabel, 'n'], ['Frontier', '']]) { const th = h('th', c, t); tr.appendChild(th); } thead.appendChild(tr); tb.appendChild(thead); + const tbody = h('tbody'); for (const m of sorted) { const r = h('tr'); r.appendChild(h('td', null, (m.ck ? m.ck + ' · ' : '') + m.short)); r.appendChild(h('td', null, window.LabTree.kindLabel(m.kind))); r.appendChild(h('td', 'n', fmtSci(m[metric]))); r.appendChild(h('td', 'n', m.ms.toFixed(2))); r.appendChild(h('td', null, frontSet.has(m.id) ? 'yes' : '')); tbody.appendChild(r); } tb.appendChild(tbody); host.appendChild(tb); + } + return { count: shown.length, total: pts.length, frontier: front }; + } + + function curve(host, models, { selected = null, onSelect = () => {}, metric = 'worstP', sections = [], yLabel = '', secondary = null } = {}) { + host.textContent = ''; host.classList.add('chart'); + const pts = models.filter(m => m.n != null && m.n <= 124 && m[metric] != null && m[metric] > 0).sort((a, b) => a.n - b.n); + const W = 760, H = 220, L = 56, R = 16, T = 28, B = 34, ylo = 1e-4, yhi = 1, nlo = pts[0].n, nhi = pts[pts.length - 1].n; + const X = n => L + (n - nlo) / (nhi - nlo) * (W - L - R), Y = v => T + (Math.log10(yhi) - Math.log10(Math.min(Math.max(v, ylo), yhi))) / 4 * (H - T - B); + const svg = el('svg', { viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': yLabel + ' for each checkpoint from C' + nlo + ' to C' + nhi }); + const grid = el('g', { class: 'grid' }), axis = el('g', { class: 'axis' }); + for (const v of logTicks(ylo, yhi)) { grid.appendChild(el('line', { x1: L, y1: Y(v), x2: W - R, y2: Y(v) })); axis.appendChild(el('text', { x: L - 8, y: Y(v) + 4, 'text-anchor': 'end' }, tickLabel(v))); } + for (let n = 20; n <= nhi; n += 20) axis.appendChild(el('text', { x: X(n), y: H - B + 16, 'text-anchor': 'middle' }, 'C' + n)); + axis.appendChild(el('line', { x1: L, y1: H - B, x2: W - R, y2: H - B, class: 'base' })); + for (const s of sections) { const a = pts.find(m => m.n >= s.from); if (!a) continue; axis.appendChild(el('line', { x1: X(s.from), y1: T - 8, x2: X(s.from), y2: H - B, class: '' })); axis.appendChild(el('text', { x: X(s.from) + 4, y: T - 12, class: 'label' }, s.label)); } + svg.appendChild(grid); svg.appendChild(axis); + svg.appendChild(el('path', { class: 'line', d: pts.map((m, i) => (i ? 'L' : 'M') + X(m.n) + ',' + Y(m[metric])).join(' ') })); + const cross = el('line', { class: 'cross', x1: 0, y1: T, x2: 0, y2: H - B }); cross.style.visibility = 'hidden'; svg.appendChild(cross); + for (const m of pts) { svg.appendChild(el('circle', { cx: X(m.n), cy: Y(m[metric]), r: 4, class: 'pt ' + m.kind + (m.id === selected ? '' : '') })); if (m.id === selected) svg.appendChild(el('circle', { cx: X(m.n), cy: Y(m[metric]), r: 8, class: 'sel' })); } + const tip = tooltip(host); + const nearest = cx => { let b = pts[0], d = Infinity; for (const m of pts) { const dd = Math.abs(X(m.n) - cx); if (dd < d) { d = dd; b = m; } } return b; }; + const toSvg = e => { const r = svg.getBoundingClientRect(); return [(e.clientX - r.left) * W / r.width, (e.clientY - r.top) * H / r.height]; }; + svg.addEventListener('pointermove', e => { const [cx] = toSvg(e); const m = nearest(cx); cross.setAttribute('x1', X(m.n)); cross.setAttribute('x2', X(m.n)); cross.style.visibility = 'visible'; const r = host.getBoundingClientRect(), s = svg.getBoundingClientRect(); tip.show(s.left - r.left + X(m.n) * s.width / W, s.top - r.top + Y(m[metric]) * s.height / H, [{ b: m.ck + ' · ' + m.short }, { v: fmtSci(m[metric]), l: yLabel }].concat(secondary ? [{ v: m[secondary.id] == null ? '—' : fmtSci(m[secondary.id]), l: secondary.label }] : []).concat([{ v: window.LabTree.kindLabel(m.kind), l: '' }])); }); + svg.addEventListener('pointerleave', () => { cross.style.visibility = 'hidden'; tip.hide(); }); + svg.addEventListener('click', e => { const [cx] = toSvg(e); onSelect(nearest(cx).id); }); + host.appendChild(svg); + return { count: pts.length }; + } + // Multi-series line chart (training curves). series: [{id, label, slot, points: [[x, y]]}] + function lines(host, series, { yLabel = '', xLabel = '', logY = true } = {}) { + host.textContent = ''; host.classList.add('chart'); + const W = 760, H = 240, L = 56, R = 16, T = 16, B = 36; + const all = series.flatMap(s => s.points); if (!all.length) { host.appendChild(h('p', 'empty', 'No curves.')); return; } + const xlo = 0, xhi = Math.max(...all.map(p => p[0])), ylo = Math.min(...all.map(p => p[1])), yhi = Math.max(...all.map(p => p[1])); + const X = x => L + (x - xlo) / (xhi - xlo || 1) * (W - L - R); + const Y = logY ? v => T + (Math.log10(yhi) - Math.log10(v)) / (Math.log10(yhi) - Math.log10(ylo) || 1) * (H - T - B) : v => T + (yhi - v) / (yhi - ylo || 1) * (H - T - B); + const svg = el('svg', { viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': yLabel + ' against ' + xLabel + ' for ' + series.map(s => s.label).join(', ') }); + const grid = el('g', { class: 'grid' }), axis = el('g', { class: 'axis' }); + const ticks = logY ? logTicks(ylo, yhi).filter(v => v >= ylo && v <= yhi) : [ylo, (ylo + yhi) / 2, yhi]; + for (const v of ticks) { grid.appendChild(el('line', { x1: L, y1: Y(v), x2: W - R, y2: Y(v) })); axis.appendChild(el('text', { x: L - 8, y: Y(v) + 4, 'text-anchor': 'end' }, logY ? tickLabel(v) : v.toFixed(2))); } + for (let i = 0; i <= 4; i++) { const x = xlo + (xhi - xlo) * i / 4; axis.appendChild(el('text', { x: X(x), y: H - B + 16, 'text-anchor': 'middle' }, Math.round(x).toLocaleString('en-GB'))); } + axis.appendChild(el('line', { x1: L, y1: H - B, x2: W - R, y2: H - B, class: 'base' })); + axis.appendChild(el('text', { x: W - R, y: H - 4, 'text-anchor': 'end', class: 'label' }, xLabel)); + axis.appendChild(el('text', { x: L, y: T - 4, class: 'label' }, yLabel)); + svg.appendChild(grid); svg.appendChild(axis); + for (const s of series) { svg.appendChild(el('path', { class: 'line s' + s.slot, d: s.points.map((p, i) => (i ? 'L' : 'M') + X(p[0]) + ',' + Y(p[1])).join(' ') })); const last = s.points[s.points.length - 1]; svg.appendChild(el('circle', { cx: X(last[0]), cy: Y(last[1]), r: 4, class: 'pt s' + s.slot })); } + const cross = el('line', { class: 'cross', x1: 0, y1: T, x2: 0, y2: H - B }); cross.style.visibility = 'hidden'; svg.appendChild(cross); + const tip = tooltip(host); + svg.addEventListener('pointermove', e => { const r = svg.getBoundingClientRect(); const cx = (e.clientX - r.left) * W / r.width; const xv = xlo + (cx - L) / (W - L - R) * (xhi - xlo); cross.setAttribute('x1', cx); cross.setAttribute('x2', cx); cross.style.visibility = 'visible'; const rows = [{ b: Math.round(xv).toLocaleString('en-GB') + ' ' + xLabel }]; for (const s of series) { let b = s.points[0]; for (const p of s.points) if (Math.abs(p[0] - xv) < Math.abs(b[0] - xv)) b = p; rows.push({ v: fmtSci(b[1]), l: s.label }); } const hr = host.getBoundingClientRect(); tip.show(e.clientX - hr.left, e.clientY - hr.top - 10, rows); }); + svg.addEventListener('pointerleave', () => { cross.style.visibility = 'hidden'; tip.hide(); }); + host.appendChild(svg); + const legend = h('ul', 'legend'); for (const s of series) { const li = h('li', 'key s' + s.slot, s.label); legend.appendChild(li); } host.appendChild(legend); + } + return { pareto, curve, lines }; +})(); diff --git a/docs/plans/project-workspace-redesign/mockups/shared/screens.js b/docs/plans/project-workspace-redesign/mockups/shared/screens.js new file mode 100644 index 00000000..57b46905 --- /dev/null +++ b/docs/plans/project-workspace-redesign/mockups/shared/screens.js @@ -0,0 +1,717 @@ +'use strict'; +// Screen content shared by every layout. A shell decides where the main +// content and the inspector live; these functions fill them from window.LAB. +// Rule: the chrome holds no project words. Metric names, populations, group +// names and decisions come from the project data. +window.LabScreens = (() => { + const LAB = window.LAB, Tree = window.LabTree, Charts = window.LabCharts; + const h = (tag, cls, text) => { const n = document.createElement(tag); if (cls) n.className = cls; if (text !== undefined) n.textContent = text; return n; }; + const svgNS = 'http://www.w3.org/2000/svg'; + const ICONS = { + check: 'M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20zm-1.3 14.2-3.6-3.6 1.4-1.4 2.2 2.2 5.1-5.1 1.4 1.4z', + alert: 'M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20zm-1 5h2v6h-2zm0 8h2v2h-2z', + triangle: 'M12 3 1 21h22zm-1 7h2v5h-2zm0 7h2v2h-2z', + minus: 'M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20zm-5 9h10v2H7z', + branch: 'M7 3a3 3 0 0 0-1 5.8v6.4A3 3 0 1 0 8 15.2V9.9c1.2 1.3 2.9 2.1 5 2.1 2.8 0 4.4-1.5 5-4.1A3 3 0 1 0 16 7.6c-.5 1.6-1.4 2.4-3 2.4-2.6 0-4.3-1.6-5-3.2A3 3 0 0 0 7 3z', + clock: 'M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20zm1 5h-2v6l5 3 1-1.7-4-2.3z', + x: 'M18.3 5.7 12 12l6.3 6.3-1.4 1.4L10.6 13.4 4.3 19.7 2.9 18.3 9.2 12 2.9 5.7l1.4-1.4 6.3 6.3 6.3-6.3z', + theme: 'M12 3a9 9 0 1 0 0 18 9 9 0 0 0 0-18zm0 2v14a7 7 0 0 1 0-14z' + }; + const icon = name => { const s = document.createElementNS(svgNS, 'svg'); s.setAttribute('viewBox', '0 0 24 24'); s.setAttribute('aria-hidden', 'true'); const p = document.createElementNS(svgNS, 'path'); p.setAttribute('d', ICONS[name] || ICONS.minus); p.setAttribute('fill', 'currentColor'); p.setAttribute('fill-rule', 'evenodd'); s.appendChild(p); return s; }; + const stateEl = (kind, text) => { const map = { good: 'check', warn: 'alert', serious: 'triangle', neutral: 'minus', pending: 'clock' }; const s = h('span', 'state ' + (kind === 'pending' ? 'warn' : kind)); s.appendChild(icon(map[kind] || 'minus')); s.appendChild(document.createTextNode(text)); return s; }; + const fmtDate = iso => new Date(iso).toLocaleDateString('en-GB', { day: 'numeric', month: 'short', year: 'numeric' }); + const fmtBytes = b => b == null ? '—' : b > 1e6 ? (b / 1e6).toFixed(1) + ' MB' : b > 1e3 ? Math.round(b / 1e3) + ' KB' : b + ' B'; + let graph = Tree.build(LAB.models, LAB.relationships), byId = graph.byId, plannedVersion = 0; + function setPlanned(list) { + const extra = list.map(pl => { const p = LAB.models.find(m => m.id === pl.parent); return Object.assign({ id: pl.id, ck: null, n: p && p.n != null ? p.n + 0.5 : null, label: 'New model from ' + (p ? (p.ck || p.short) : '?'), short: pl.name || 'new model', alias: null, group: p ? p.group : 'pressure-other', groupLabel: p ? p.groupLabel : '', kind: pl.kind || 'refinement', worstP: null, eq: null, ms: null, params: null, purpose: '', outcome: '', lesson: '', parents: [pl.parent], planned: true, setup: pl.setup }); }); + graph = Tree.build(LAB.models.concat(extra), LAB.relationships); byId = graph.byId; plannedVersion++; + } + const MET = LAB.metrics, ACC = MET.accuracy, TIME = MET.time; + const accLabel = id => (ACC.find(a => a.id === id) || ACC[0]).label; + const benchModels = () => LAB.models.filter(m => m.group !== MET.excludedGroup); + const withBench = LAB.models.filter(m => m[ACC[0].id] != null && m[TIME.id] != null).length; + const KINDS = ['training', 'architecture', 'refinement', 'composition']; + // Checkpoint sections for the sequence chart, read from authored group labels of the form "C16–C25 · name". + const CURVE_SECTIONS = LAB.groups.map(g => { const m = /^C(\d+)[–-]C(\d+)\s*·\s*(.+)$/.exec(g.label); return m ? { from: +m[1], label: m[3].split(/\s+/)[0] } : null; }).filter(Boolean); + const studyState = s => s.state === 'closed' ? 'pending' : s.decision.includes('failed') ? 'serious' : s.decision.includes('no promotion') ? 'neutral' : 'good'; + const sect = (title, sum) => { const d = h('div', 'sect'); d.appendChild(h('h2', null, title)); if (sum) d.appendChild(h('span', 'sum', sum)); return d; }; + const DOC_ROLE = path => /plan\.md$/.test(path) ? 'Plan' : /ledger\.md$/.test(path) ? 'Methods' : /(results?\.md|current\.json|results\.json)$/.test(path) ? 'Results' : null; + const docsOf = plan => { const g = LAB.project.evidence.find(x => x.plan === plan); return g ? g.items.map(([id, label, path]) => ({ id, label: DOC_ROLE(path) || label, path })) : []; }; + const docChips = (plan, sel, onPick) => { const d = h('span', 'docs'); for (const doc of docsOf(plan)) { const b = h('button', null, doc.label); b.type = 'button'; b.title = doc.path; b.setAttribute('aria-pressed', String(doc.id === sel)); b.addEventListener('click', ev => { ev.stopPropagation(); onPick(doc.id); }); d.appendChild(b); } return d; }; + const fileLink = (rel, text) => { const a = h('a', 'path', text || rel); const dir = rel.includes('/') ? rel.slice(0, rel.lastIndexOf('/')) : ''; a.href = 'file://' + LAB.project.rootPath + '/' + dir; a.title = 'Open folder'; a.addEventListener('click', ev => ev.stopPropagation()); return a; }; + const KIND_ICON = { fields: 'M3 3h18v18H3zm2 2v4h4V5zm6 0v4h4V5zm6 0v4h2V5zM5 11v4h4v-4zm6 0v4h4v-4zm6 0v4h2v-4zM5 17v2h4v-2zm6 0v2h4v-2zm6 0v2h2v-2z', records: 'M4 4h16v3H4zm0 5h16v3H4zm0 5h16v3H4zm0 5h10v2H4z', manifest: 'M6 2h9l5 5v15H6zm8 1.5V8h4.5zM8 11h8v2H8zm0 4h8v2H8z', results: 'M3 21V3h2v16h16v2zm4-4 4-6 3 3 5-8 1.5 1-6.5 10-3-3-3 4.5z' }; + const kindIcon = k => { const sv = document.createElementNS(svgNS, 'svg'); sv.setAttribute('viewBox', '0 0 24 24'); sv.setAttribute('aria-hidden', 'true'); const pth = document.createElementNS(svgNS, 'path'); pth.setAttribute('d', KIND_ICON[k] || KIND_ICON.records); pth.setAttribute('fill', 'currentColor'); pth.setAttribute('fill-rule', 'evenodd'); sv.appendChild(pth); return sv; }; + function fieldPreview(canvas, seed) { + const ctx = canvas.getContext('2d'), W = canvas.width = 320, Hh = canvas.height = 180, nx = 32, ny = 18; + const dark = getComputedStyle(document.documentElement).colorScheme === 'dark'; + const mid = dark ? [56, 56, 53] : [240, 239, 236], neg = dark ? [79, 138, 224] : [46, 111, 207], pos = dark ? [217, 111, 60] : [210, 101, 47]; + for (let j = 0; j < ny; j++) for (let i = 0; i < nx; i++) { + const u = i / nx, v = j / ny, val = Math.sin(6.3 * u + seed) * Math.cos(4.7 * v - seed * 0.5) * 0.7 + Math.sin(12 * u * v + seed) * 0.3; + const c = val < 0 ? neg : pos, t = Math.min(1, Math.abs(val)); const rgb = mid.map((m, k) => Math.round(m + (c[k] - m) * t)); + ctx.fillStyle = `rgb(${rgb[0]},${rgb[1]},${rgb[2]})`; ctx.fillRect(i * W / nx, j * Hh / ny, W / nx + 1, Hh / ny + 1); + } + } + const seeded = str => { let x = 0; for (const ch of str) x = (x * 31 + ch.charCodeAt(0)) % 1000003; return () => { x = (x * 1103515245 + 12345) % 2147483648; return x / 2147483648; }; }; + // Training curve for a fit: shaped by its recorded update count and final loss (mock data for the layout). + const curveFor = m => { const U = m.updates || 2000, L = m.loss || (m.worstP ? m.worstP * 2000 : 10), rnd = seeded(m.id), pts = []; for (let i = 0; i <= 40; i++) { const u = U * i / 40; pts.push([u, L * (1 + 30 * Math.exp(-6 * u / U)) * (1 + 0.06 * (rnd() - 0.5))]); } return pts; }; + const recipeOf = id => (LAB.recipes || []).find(r => r.id === id); + const runtimeOf = id => (LAB.runtimes || []).find(r => r.id === id); + const datasetOf = id => LAB.project.data.find(r => r.id === id); + const modelName = m => (m.ck ? m.ck + ' · ' : '') + m.short; + const popOf = state => (MET.populations || []).find(p => p.id === (state.pop || 'pressure')) || { id: 'all', label: 'All' }; + const inPop = (m, pop) => pop.only ? m.group === pop.only : pop.exclude ? m.group !== pop.exclude : true; + const benchState = m => { const a = m[ACC[0].id] != null, t = m[TIME.id] != null; return m.group === MET.excludedGroup ? 'generator benchmark' : a && t ? 'accuracy and time' : a ? 'accuracy only' : t ? 'time only' : 'none retained'; }; + const runStudy = m => { for (const [sid, trials] of Object.entries(LAB.project.trials || {})) for (const t of trials) if (t.run && t.run === m.ck) return LAB.project.studies.find(x => x.id === sid); return null; }; + const cell = (k, v, u) => { const d = h('div'); d.appendChild(h('dt', null, k)); const dd = h('dd', null, v); if (u) dd.appendChild(h('small', null, u)); d.appendChild(dd); return d; }; + + function statusLine() { const a = LAB.project.activity; return { state: a.state, text: a.phase + '. ' + a.message, updated: fmtDate(a.updated), producer: a.producer }; } + function stageSummaries() { + const d = LAB.project.data, verified = d.filter(r => r.integrity === 'verified').length, missing = d.filter(r => r.state === 'missing').length; + const st = LAB.project.studies, pending = st.filter(s => s.state === 'closed').length; + return [ + { id: 'overview', label: 'Overview', sum: `${LAB.project.needs.length} need you` }, + { id: 'data', label: 'Data', sum: `${d.length} datasets` + (missing ? ` · ${missing} missing` : '') }, + { id: 'annotate', label: 'Annotate', sum: 'No records' }, + { id: 'models', label: 'Models', sum: `${LAB.models.length} fits · ${graph.families.length} branch points` }, + { id: 'train', label: 'Train', sum: `${LAB.models.length} runs · ${st.length} studies` + (pending ? ` · ${pending} waiting` : '') }, + { id: 'benchmarks', label: 'Benchmarks', sum: `${withBench} benchmarked` }, + { id: 'analysis', label: 'Analysis', sum: `${LAB.project.evidence.reduce((n, g) => n + g.items.length, 0)} notes` } + ]; + } + + // ---- Overview ------------------------------------------------------------ + function overview(main, aside, state, act) { + main.textContent = ''; aside.textContent = ''; + const s = statusLine(); + const top = h('section', 'ov-status'); + const line = h('div', 'ov-line'); + line.appendChild(stateEl(s.state === 'completed' ? 'good' : s.state === 'failed' ? 'serious' : 'pending', s.state === 'completed' ? 'Completed' : s.state)); + line.appendChild(h('p', 'ov-text', s.text)); top.appendChild(line); + top.appendChild(h('p', 'small faint', `${s.producer} · ${s.updated}`)); + main.appendChild(top); + + const row = h('div', 'ov-row'); + const needs = h('section'); needs.appendChild(sect('Needs you', String(LAB.project.needs.length))); + const ul = h('ul', 'needs'); + for (const n of LAB.project.needs) { + const li = h('li', n.kind); li.appendChild(icon(n.kind === 'approval' ? 'clock' : n.kind === 'changed' ? 'alert' : 'triangle')); + li.appendChild(h('span', null, n.text)); + const b = h('button', 'btn sm', n.kind === 'approval' ? 'Review' : 'Open'); b.type = 'button'; b.addEventListener('click', () => act.go(n.where, n.ref)); li.appendChild(b); + ul.appendChild(li); + } + needs.appendChild(ul); row.appendChild(needs); + const facts = h('section'); facts.appendChild(sect('Project')); + const dl = h('dl', 'facts'); + for (const [k, v, c] of [['Manifest', `${LAB.project.root}/${LAB.project.manifest}`, 'mono'], ['Schema', String(LAB.project.schema)], ['Snapshot', LAB.project.id, 'mono'], ['Requirements', LAB.project.requirements.map(r => r.label).join('; ')], ['Execution', 'inspection only']]) { dl.appendChild(h('dt', null, k)); dl.appendChild(h('dd', c || '', v)); } + facts.appendChild(dl); row.appendChild(facts); main.appendChild(row); + + const cols = h('div', 'ov-cols'); + const st = h('section'); st.appendChild(sect('Stages')); + const sl = h('ul', 'rows'); + for (const g of stageSummaries().filter(x => x.id !== 'overview')) { const li = h('li'); const b = h('button', 'row ov-stage'); b.type = 'button'; b.appendChild(h('span', 'title', g.label)); b.appendChild(h('span', 'sub', g.sum)); b.addEventListener('click', () => act.go(g.id)); li.appendChild(b); sl.appendChild(li); } + st.appendChild(sl); cols.appendChild(st); + const rs = h('section'); rs.appendChild(sect('Studies', 'newest first')); + const rl = h('ul', 'rows'); + for (const s2 of LAB.project.studies.slice().sort((a, b) => b.updated.localeCompare(a.updated)).slice(0, 6)) { + const li = h('li'); const b = h('button', 'row ov-study'); b.type = 'button'; + b.appendChild(stateEl(studyState(s2), '')); + const t = h('span'); t.appendChild(h('span', 'title', s2.label)); t.appendChild(h('span', 'sub', s2.decision)); b.appendChild(t); + b.appendChild(h('span', 'sub num', fmtDate(s2.updated))); + b.addEventListener('click', () => act.go('train', s2.id)); li.appendChild(b); rl.appendChild(li); + } + rs.appendChild(rl); cols.appendChild(rs); main.appendChild(cols); + + const lg = h('section', 'ov-history'); lg.appendChild(sect('Model history', `${accLabel(ACC[0].id)} per checkpoint`)); + const c = h('div'); lg.appendChild(c); + Charts.curve(c, LAB.models, { selected: null, onSelect: id => act.go('models', id), metric: ACC[0].id, sections: CURVE_SECTIONS, yLabel: accLabel(ACC[0].id), secondary: ACC[1] || null }); + const bp = h('div', 'actions'); + for (const f of graph.families) { const pp = byId.get(f.parent); const b = h('button', 'btn sm'); b.type = 'button'; b.appendChild(icon('branch')); b.appendChild(document.createTextNode(` ${pp.ck || pp.short} · ${f.children.length} fits`)); b.addEventListener('click', () => act.go('models', pp.id)); bp.appendChild(b); } + const open = h('button', 'btn sm ghost', 'All models'); open.type = 'button'; open.addEventListener('click', () => act.go('models')); bp.appendChild(open); + lg.appendChild(bp); main.appendChild(lg); + return false; + } + + // ---- Data: datasets, then the records inside one, with a volume viewer ---- + const DIV = () => { const dark = getComputedStyle(document.documentElement).colorScheme === 'dark'; return { mid: dark ? [56, 56, 53] : [240, 239, 236], neg: dark ? [79, 138, 224] : [46, 111, 207], pos: dark ? [217, 111, 60] : [210, 101, 47] }; }; + const SEQ = () => { const dark = getComputedStyle(document.documentElement).colorScheme === 'dark'; return { lo: dark ? [31, 38, 46] : [236, 240, 246], hi: dark ? [79, 138, 224] : [28, 92, 171] }; }; + // Synthetic volume field for the layout: smooth structure that varies by record, field and position. + const volumeFn = (seed, field) => (u, v, w) => { const a = seed * 0.37, b = seed * 0.11; const base = Math.sin(6.3 * u + a) * Math.cos(4.7 * v - b) * Math.cos(3.1 * w + a) * 0.7 + Math.sin(9 * u * v + 5 * w + b) * 0.3; if (/velocity/.test(field)) return Math.abs(base) * 0.9; if (/tracer/.test(field)) return Math.max(0, Math.sin(7 * (u - 0.5) * (v - 0.5) * 8 + w * 4 + a)) * 0.9; if (/error/.test(field)) return Math.sin(17 * u + 3 * a) * Math.cos(13 * v + b) * Math.sin(5 * w) * 0.6; return base; }; + const paint = (val, signed, c, q) => { if (signed) { const col = val < 0 ? c.neg : c.pos, t = Math.min(1, Math.abs(val)); return c.mid.map((m, k) => Math.round(m + (col[k] - m) * t)); } const t = Math.min(1, Math.max(0, val)); return q.lo.map((m, k) => Math.round(m + (q.hi[k] - m) * t)); }; + function drawSlice2(canvas, fn, signed, n, size, w) { + const ctx = canvas.getContext('2d'); canvas.width = size; canvas.height = size; const c = DIV(), q = SEQ(); + for (let j = 0; j < n; j++) for (let i = 0; i < n; i++) { const rgb = paint(fn(i / n, j / n, w), signed, c, q); ctx.fillStyle = `rgb(${rgb[0]},${rgb[1]},${rgb[2]})`; ctx.fillRect(i * size / n, j * size / n, size / n + 1, size / n + 1); } + } + // Isometric stack of slices: a voxel-like impression of the whole grid. + function drawVolume(canvas, fn, signed, dims, wSel) { + const W = canvas.width = 480, H = canvas.height = 360, ctx = canvas.getContext('2d'); ctx.clearRect(0, 0, W, H); const c = DIV(), q = SEQ(); + const [nx, ny, nz] = dims, sx = 16, sy = Math.max(4, Math.round(16 * ny / nx)), nsl = Math.min(nz, 10); + const cw = Math.min(14, 300 / sx), ch = cw * 0.5, dz = Math.min(22, 220 / nsl); + const ox = W / 2 - (sx * cw) / 2 + (sy * cw) / 4, oy = H - 40 - (nsl - 1) * dz - (sy * ch) / 2; + for (let k = 0; k < nsl; k++) { + const w = (k + 0.5) / nsl, hot = wSel != null && Math.abs(w - wSel) < 0.5 / nsl + 1e-6; + ctx.globalAlpha = hot ? 1 : 0.55; + for (let j = 0; j < sy; j++) for (let i = 0; i < sx; i++) { + const rgb = paint(fn(i / sx, j / sy, w), signed, c, q); ctx.fillStyle = `rgb(${rgb[0]},${rgb[1]},${rgb[2]})`; + const px = ox + i * cw - j * (cw / 2), py = oy + (nsl - 1 - k) * dz + j * ch + i * (ch / 2) * 0; + ctx.beginPath(); ctx.moveTo(px, py); ctx.lineTo(px + cw, py); ctx.lineTo(px + cw - cw / 2, py + ch); ctx.lineTo(px - cw / 2, py + ch); ctx.closePath(); ctx.fill(); + } + if (hot) { ctx.globalAlpha = 1; ctx.strokeStyle = getComputedStyle(document.documentElement).getPropertyValue('--accent').trim() || '#2c5fb8'; ctx.lineWidth = 2; const py = oy + (nsl - 1 - k) * dz; ctx.beginPath(); ctx.moveTo(ox, py); ctx.lineTo(ox + sx * cw, py); ctx.lineTo(ox + sx * cw - sy * cw / 2, py + sy * ch); ctx.lineTo(ox - sy * cw / 2, py + sy * ch); ctx.closePath(); ctx.stroke(); } + } + ctx.globalAlpha = 1; + } + function data(main, aside, state, act) { + main.textContent = ''; aside.textContent = ''; + const sel = state.sel.data, d0 = LAB.project.data, ACTIONS = ['import', 'generate', 'release']; + const ds = d0.find(x => x.id === sel); + if (ds && ds.state !== 'missing') return datasetView(main, aside, ds, state, act); + const ac = h('div', 'ds-actions'); + for (const [id, t] of [['import', 'Import'], ['generate', 'Generate'], ['annotate', 'Annotate'], ['release', 'Release']]) { const b = h('button', 'btn', t); b.type = 'button'; b.setAttribute('aria-pressed', String(sel === id)); b.addEventListener('click', () => id === 'annotate' ? act.go('annotate') : act.select(sel === id ? null : id)); ac.appendChild(b); } + main.appendChild(ac); + main.appendChild(sect('Datasets', String(d0.length))); + const grid = h('div', 'ds-grid'); + for (const r of d0) { + const b = h('button', 'ds-card'); b.type = 'button'; b.setAttribute('aria-pressed', String(r.id === sel)); + if (r.state !== 'missing') { const cv = h('canvas', 'pv'); b.appendChild(cv); const fn = volumeFn(r.id.length * 3 + 1, r.fields[0]); const ctx = cv.getContext('2d'); cv.width = 320; cv.height = 180; const c = DIV(), q = SEQ(); const n = 32, m = 18; for (let j = 0; j < m; j++) for (let i = 0; i < n; i++) { const rgb = paint(fn(i / n, j / m, 0.5), !/velocity|tracer/.test(r.fields[0]), c, q); ctx.fillStyle = `rgb(${rgb[0]},${rgb[1]},${rgb[2]})`; ctx.fillRect(i * 10, j * 10, 11, 11); } } + else b.appendChild(h('div', 'pv text', 'file missing')); + const t = h('div', 't'); t.appendChild(h('div', 'title', r.label)); + const meta = h('div', 'meta'); meta.appendChild(h('span', null, r.records.toLocaleString('en-GB') + ' ' + r.recordKind + (r.records === 1 ? '' : 's') + ' · ' + r.grid.join('×'))); meta.appendChild(h('span', null, fmtBytes(r.bytes))); + const g = h('span', 'glyph ' + (r.state === 'missing' ? 'serious' : r.integrity === 'verified' ? 'good' : 'neutral')); g.appendChild(icon(r.state === 'missing' ? 'triangle' : r.integrity === 'verified' ? 'check' : 'minus')); g.title = r.state === 'missing' ? 'Missing' : r.integrity === 'verified' ? 'Hash verified' : 'No declared hash'; meta.appendChild(g); t.appendChild(meta); + t.appendChild(fileLink(r.path)); b.appendChild(t); + b.addEventListener('click', () => act.select(r.id)); grid.appendChild(b); + } + main.appendChild(grid); + if (ACTIONS.includes(sel)) { dataPanel(aside, sel, state, act); return true; } + if (ds && ds.state === 'missing') { + const ins = h('div', 'inspector'); const hd = h('header'); hd.appendChild(h('span', 'kicker', ds.kind + ' · ' + ds.id)); hd.appendChild(h('h2', null, ds.label)); ins.appendChild(hd); + const dl = h('dl', 'metrics'); dl.appendChild(cell('Records', `${ds.records} ${ds.recordKind}s`)); dl.appendChild(cell('Grid', ds.grid.join(' × '))); dl.appendChild(cell('State', 'file missing')); ins.appendChild(dl); + const pa = h('p', 'note'); pa.appendChild(fileLink(ds.path)); ins.appendChild(pa); ins.appendChild(h('p', 'note', 'Restore the file, or open a new snapshot without it.')); aside.appendChild(ins); return true; + } + const ins = h('div', 'inspector'); const dl = h('dl', 'metrics'); + dl.appendChild(cell('Datasets', String(d0.length))); dl.appendChild(cell('Records', d0.reduce((a, x) => a + (x.records || 0), 0).toLocaleString('en-GB'))); dl.appendChild(cell('On disk', fmtBytes(d0.reduce((a, x) => a + (x.bytes || 0), 0)))); dl.appendChild(cell('Missing', String(d0.filter(x => x.state === 'missing').length))); + ins.appendChild(dl); aside.appendChild(ins); return false; + } + function datasetView(main, aside, r, state, act) { + const ui = state.ui || {}, per = 48, page = ui['page:' + r.id] || 0, rec = ui['rec:' + r.id] != null ? ui['rec:' + r.id] : 0, field = ui['field:' + r.id] || r.fields[0], w = ui['w:' + r.id] != null ? ui['w:' + r.id] : 0.5; + const back = h('button', 'btn sm ghost ds-back', '← All datasets'); back.type = 'button'; back.addEventListener('click', () => act.select(null)); main.appendChild(back); + const head = h('div', 'ds-head'); head.appendChild(h('h2', null, r.label)); + const meta = h('div', 'meta'); meta.appendChild(h('span', null, `${r.records.toLocaleString('en-GB')} ${r.recordKind}${r.records === 1 ? '' : 's'}` + (r.scenes ? ` · ${r.scenes} scenes` : ''))); meta.appendChild(h('span', null, r.grid.join(' × ') + ' cells')); meta.appendChild(h('span', null, fmtBytes(r.bytes))); + const g = h('span', 'glyph ' + (r.integrity === 'verified' ? 'good' : 'neutral')); g.appendChild(icon(r.integrity === 'verified' ? 'check' : 'minus')); g.title = r.integrity === 'verified' ? 'Hash verified' : 'No declared hash'; meta.appendChild(g); meta.appendChild(fileLink(r.path)); head.appendChild(meta); + const acts = h('div', 'actions'); const a1 = h('a', 'btn sm', 'Open folder'); a1.href = fileLink(r.path).href; acts.appendChild(a1); const b2 = h('button', 'btn sm', 'Annotate'); b2.type = 'button'; b2.disabled = r.kind !== 'records'; acts.appendChild(b2); const b3 = h('button', 'btn sm', 'Release'); b3.type = 'button'; b3.addEventListener('click', () => { act.setDataAction('dataset', r.id); act.select('release'); }); acts.appendChild(b3); head.appendChild(acts); + main.appendChild(head); + if (r.describes) main.appendChild(h('p', 'small muted', r.describes)).style.marginBottom = '12px'; + const body = h('div', 'ds-body'); + const left = h('div'); left.appendChild(sect(r.recordKind.charAt(0).toUpperCase() + r.recordKind.slice(1) + 's', `${page * per + 1}–${Math.min(r.records, (page + 1) * per)} of ${r.records}`)); + const grid = h('div', 'rec-grid'); + for (let i = page * per; i < Math.min(r.records, (page + 1) * per); i++) { + const b = h('button', 'rec'); b.type = 'button'; b.setAttribute('aria-pressed', String(i === rec)); b.setAttribute('aria-label', r.recordKind + ' ' + (i + 1)); + const cv = h('canvas'); drawSlice2(cv, volumeFn(i + 1, field), !/velocity|tracer/.test(field), 16, 64, 0.5); b.appendChild(cv); b.appendChild(h('span', null, String(i + 1))); + b.addEventListener('click', () => act.setUI('rec:' + r.id, i)); grid.appendChild(b); + } + left.appendChild(grid); + if (r.records > per) { const pg = h('div', 'pager'); const pb = h('button', 'btn sm', 'Previous'); pb.type = 'button'; pb.disabled = page === 0; pb.addEventListener('click', () => act.setUI('page:' + r.id, page - 1)); const nb = h('button', 'btn sm', 'Next'); nb.type = 'button'; nb.disabled = (page + 1) * per >= r.records; nb.addEventListener('click', () => act.setUI('page:' + r.id, page + 1)); pg.appendChild(pb); pg.appendChild(nb); pg.appendChild(h('span', null, `page ${page + 1} of ${Math.ceil(r.records / per)}`)); left.appendChild(pg); } + body.appendChild(left); + const vw = h('div', 'viewer'); + const fn = volumeFn(rec + 1, field), signed = !/velocity|tracer/.test(field); + const vol = h('canvas', 'vol'); vol.setAttribute('role', 'img'); vol.setAttribute('aria-label', `${field} volume of ${r.recordKind} ${rec + 1}, ${r.grid.join(' by ')} cells`); drawVolume(vol, fn, signed, r.grid, w); vw.appendChild(vol); + const row2 = h('div', 'row2'); const sl = h('canvas', 'sl'); sl.setAttribute('role', 'img'); sl.setAttribute('aria-label', 'Slice'); drawSlice2(sl, fn, signed, 32, 240, w); row2.appendChild(sl); + const facts = h('dl', 'facts facts-sm'); for (const [k, v] of [[r.recordKind.charAt(0).toUpperCase() + r.recordKind.slice(1), String(rec + 1) + (r.dt ? ' · t = ' + (rec * (r.dt.includes('/') ? 1 / +r.dt.split('/')[1].replace(/\D/g, '') : +r.dt)).toFixed(3) + ' s' : '')], ['Grid', r.grid.join(' × ')], ['Spacing', r.spacing || '—'], ['Fields', r.fields.join(', ')], ['Slice', 'z = ' + Math.round(w * (r.grid[2] - 1) + 1) + ' of ' + r.grid[2]]]) { facts.appendChild(h('dt', null, k)); facts.appendChild(h('dd', null, v)); } + row2.appendChild(facts); vw.appendChild(row2); + const ctl = h('div', 'ctl'); + const fl = h('label'); fl.appendChild(document.createTextNode('Field')); const fs = h('select'); for (const f of r.fields) { const o = h('option', null, f); o.selected = f === field; fs.appendChild(o); } fs.addEventListener('change', () => act.setUI('field:' + r.id, fs.value)); fl.appendChild(fs); ctl.appendChild(fl); + const zl = h('label'); zl.appendChild(document.createTextNode('Slice')); const zi = h('input'); zi.type = 'range'; zi.min = 0; zi.max = r.grid[2] - 1; zi.value = Math.round(w * (r.grid[2] - 1)); zi.addEventListener('input', () => { const ww = +zi.value / Math.max(1, r.grid[2] - 1); drawVolume(vol, fn, signed, r.grid, ww); drawSlice2(sl, fn, signed, 32, 240, ww); const dd = facts.querySelectorAll('dd'); dd[dd.length - 1].textContent = 'z = ' + (+zi.value + 1) + ' of ' + r.grid[2]; }); zi.addEventListener('change', () => { state.ui['w:' + r.id] = +zi.value / Math.max(1, r.grid[2] - 1); }); zl.appendChild(zi); ctl.appendChild(zl); + const an = h('button', 'btn sm', 'Compare in Analysis'); an.type = 'button'; an.addEventListener('click', () => { act.setCase(rec + 1); act.go('analysis'); }); ctl.appendChild(an); + vw.appendChild(ctl); body.appendChild(vw); main.appendChild(body); + return false; + } + function dataPanel(aside, kind, state, act) { + const n = state.dataAction || {}; + const ins = h('div', 'inspector'); const f = h('div', 'form'); + const field = (label, control, hint) => { const l = h('label'); l.appendChild(document.createTextNode(label)); l.appendChild(control); if (hint) l.appendChild(h('span', 'hint', hint)); return l; }; + const radio = (name, opts, cur, key) => { const r = h('div', 'radio'); for (const [v, t] of opts) { const l = h('label'); const i = h('input'); i.type = 'radio'; i.name = name; i.checked = v === cur; i.addEventListener('change', () => act.setDataAction(key, v)); l.appendChild(i); l.appendChild(document.createTextNode(t)); r.appendChild(l); } return r; }; + const hd = h('header'); + if (kind === 'import') { + hd.appendChild(h('span', 'kicker', 'data')); hd.appendChild(h('h2', null, 'Import')); ins.appendChild(hd); + f.appendChild(field('Source', radio('src', [['files', 'Files or folder'], ['release', 'Catalog release'], ['project', 'Another project']], n.src || 'files', 'src'))); + const pathI = h('input'); pathI.type = 'text'; pathI.placeholder = 'relative to ' + LAB.project.root; pathI.value = n.path || ''; f.appendChild(field('Path', pathI)); + f.appendChild(field('Kind', radio('kind', [['fields', 'Fields'], ['images', 'Images'], ['text', 'Text'], ['audio', 'Audio'], ['mesh', 'Mesh']], n.kind || 'fields', 'kind'))); + const nm = h('input'); nm.type = 'text'; nm.placeholder = 'dataset name'; f.appendChild(field('Name', nm)); + const rights = h('input'); rights.type = 'text'; rights.value = 'unknown'; f.appendChild(field('Rights', rights, 'unknown rights allow local use and deny redistribution')); + ins.appendChild(f); + const acts = h('div', 'actions'); const b = h('button', 'btn primary', 'Import'); b.type = 'button'; acts.appendChild(b); const c = h('button', 'btn ghost', 'Close'); c.type = 'button'; c.addEventListener('click', () => act.select(null)); acts.appendChild(c); ins.appendChild(acts); + ins.appendChild(h('p', 'note', 'Bytes are hashed on import; the dataset keeps the hash and the source path.')); + } else if (kind === 'generate') { + hd.appendChild(h('span', 'kicker', 'data')); hd.appendChild(h('h2', null, 'Generate')); ins.appendChild(hd); + const gens = LAB.generators || []; const gsel = h('select'); for (const g of gens) { const o = h('option', null, g.label); o.value = g.id; o.selected = g.id === (n.generator || (gens[0] && gens[0].id)); gsel.appendChild(o); } gsel.addEventListener('change', () => act.setDataAction('generator', gsel.value)); const g = gens.find(x => x.id === (n.generator || (gens[0] && gens[0].id))); f.appendChild(field('Generator', gsel, g ? g.command : '')); + if (g) for (const prm of g.params) { const i = h('input'); i.type = prm.type || 'text'; i.value = n['p_' + prm.id] != null ? n['p_' + prm.id] : prm.value; i.addEventListener('change', () => act.setDataAction('p_' + prm.id, i.value)); f.appendChild(field(prm.label, i, prm.hint || '')); } + const out = h('input'); out.type = 'text'; out.value = n.out || (g ? g.out : ''); f.appendChild(field('Output dataset', out)); + ins.appendChild(f); + const acts = h('div', 'actions'); const b = h('button', 'btn primary', 'Generate'); b.type = 'button'; acts.appendChild(b); const c = h('button', 'btn ghost', 'Close'); c.type = 'button'; c.addEventListener('click', () => act.select(null)); acts.appendChild(c); ins.appendChild(acts); + ins.appendChild(h('p', 'note', 'Runs as an ordinary recipe. The result appears here as a dataset with its receipt.')); + } else { + hd.appendChild(h('span', 'kicker', 'data')); hd.appendChild(h('h2', null, 'Release')); ins.appendChild(hd); + const dsel = h('select'); for (const r of LAB.project.data) { if (r.state === 'missing') continue; const o = h('option', null, r.label); o.value = r.id; o.selected = r.id === n.dataset; dsel.appendChild(o); } dsel.addEventListener('change', () => act.setDataAction('dataset', dsel.value)); f.appendChild(field('Dataset', dsel)); + const nm = h('input'); nm.type = 'text'; nm.placeholder = 'release name'; nm.value = n.releaseName || ''; f.appendChild(field('Name', nm)); + const three = h('div', 'two'); three.style.gridTemplateColumns = '1fr 1fr 1fr'; for (const [k, v] of [['Train %', 70], ['Validation %', 15], ['Test %', 15]]) { const i = h('input'); i.type = 'number'; i.min = 0; i.max = 100; i.value = v; three.appendChild(field(k, i)); } f.appendChild(three); + f.appendChild(field('Selection', radio('selr', [['all', 'All records'], ['saved', 'Saved selection']], n.selr || 'all', 'selr'))); + ins.appendChild(f); + const acts = h('div', 'actions'); const b = h('button', 'btn primary', 'Freeze release'); b.type = 'button'; acts.appendChild(b); const c = h('button', 'btn ghost', 'Close'); c.type = 'button'; c.addEventListener('click', () => act.select(null)); acts.appendChild(c); ins.appendChild(acts); + ins.appendChild(h('p', 'note', 'A release freezes exact revisions. Training reads releases, never live records.')); + } + aside.appendChild(ins); + } + + // ---- Annotate: records, viewer, labels, review, history -------------------- + function annotate(main, aside, state, act) { + main.textContent = ''; aside.textContent = ''; + const A = LAB.annotation || { tasks: [] }, ui = state.ui || {}, ann = state.annotations || {}; + const sets = LAB.project.data.filter(r => r.state !== 'missing'); + const dsId = ui['ann:ds'] || A.defaultDataset || (sets[0] && sets[0].id), r = sets.find(x => x.id === dsId) || sets[0]; + const task = A.tasks.find(t => t.id === (ui['ann:task'] || (A.tasks[0] && A.tasks[0].id))) || A.tasks[0]; + const per = 48, page = ui['ann:page'] || 0, rec = ui['ann:rec'] != null ? ui['ann:rec'] : 0, filter = ui['ann:filter'] || 'all'; + const store = ann[r.id] || {}; const seedOf = i => { const sd = (A.seed || {})[r.id]; return sd && i < sd.reviewed ? { label: sd.labels[i], review: sd.labels[i] === 'needs work' ? 'needs' : 'accepted', revisions: [{ at: 'imported', label: sd.labels[i], review: 'accepted' }] } : null; }; + const recOf = i => store[i] || seedOf(i) || null; + const reviewed = Array.from({ length: r.records }, (_, i) => recOf(i)).filter(Boolean).length; + const bar = h('div', 'ann-bar'); + const dl = h('label'); dl.appendChild(document.createTextNode('Dataset')); const dsel = h('select'); for (const x of sets) { const o = h('option', null, x.label); o.value = x.id; o.selected = x.id === r.id; dsel.appendChild(o); } dsel.addEventListener('change', () => { act.setUI('ann:rec', 0); act.setUI('ann:page', 0); act.setUI('ann:ds', dsel.value); }); dl.appendChild(dsel); bar.appendChild(dl); + const tl = h('label'); tl.appendChild(document.createTextNode('Task')); const tsel = h('select'); for (const t of A.tasks) { const o = h('option', null, t.label); o.value = t.id; o.selected = t.id === task.id; tsel.appendChild(o); } tsel.addEventListener('change', () => act.setUI('ann:task', tsel.value)); tl.appendChild(tsel); bar.appendChild(tl); + const seg = h('div', 'seg'); for (const [v, t] of [['all', 'All'], ['unreviewed', 'Unreviewed'], ['accepted', 'Accepted'], ['needs', 'Needs work']]) { const b = h('button', null, t); b.type = 'button'; b.setAttribute('aria-pressed', String(filter === v)); b.addEventListener('click', () => { act.setUI('ann:page', 0); act.setUI('ann:filter', v); }); seg.appendChild(b); } bar.appendChild(seg); + const pr = h('span', 'progress'); const pi = h('i'); const pb = h('b'); pb.style.width = Math.round(100 * reviewed / r.records) + '%'; pi.appendChild(pb); pr.appendChild(pi); pr.appendChild(document.createTextNode(`${reviewed} of ${r.records} reviewed`)); bar.appendChild(pr); + const kb = h('span', 'small faint'); kb.appendChild(h('span', 'kbd', '←')); kb.appendChild(document.createTextNode(' ')); kb.appendChild(h('span', 'kbd', '→')); kb.appendChild(document.createTextNode(' move · ')); kb.appendChild(h('span', 'kbd', '1')); kb.appendChild(document.createTextNode('–')); kb.appendChild(h('span', 'kbd', String((task.classes || task.options || []).length))); kb.appendChild(document.createTextNode(' label · ')); kb.appendChild(h('span', 'kbd', 'Enter')); kb.appendChild(document.createTextNode(' save and next')); bar.appendChild(kb); + main.appendChild(bar); + const all = Array.from({ length: r.records }, (_, i) => i).filter(i => { const a = recOf(i); return filter === 'all' ? true : filter === 'unreviewed' ? !a : filter === 'accepted' ? a && a.review === 'accepted' : a && a.review === 'needs'; }); + const body = h('div', 'ann-body'); + const left = h('div'); left.appendChild(sect(r.recordKind.charAt(0).toUpperCase() + r.recordKind.slice(1) + 's', all.length ? `${page * per + 1}–${Math.min(all.length, (page + 1) * per)} of ${all.length}` : '0')); + const grid = h('div', 'rec-grid'); + for (const i of all.slice(page * per, (page + 1) * per)) { + const b = h('button', 'rec'); b.type = 'button'; b.setAttribute('aria-pressed', String(i === rec)); const a = recOf(i); b.setAttribute('aria-label', r.recordKind + ' ' + (i + 1) + (a ? ', ' + (a.label || '') + ' ' + a.review : ', unreviewed')); + const cv = h('canvas'); drawSlice2(cv, volumeFn(i + 1, r.fields[0]), !/velocity|tracer/.test(r.fields[0]), 16, 64, 0.5); b.appendChild(cv); b.appendChild(h('span', null, String(i + 1))); + if (a) b.appendChild(h('i', 'badge ' + (a.review || 'labelled'))); + b.addEventListener('click', () => act.setUI('ann:rec', i)); grid.appendChild(b); + } + left.appendChild(grid); + if (all.length > per) { const pg = h('div', 'pager'); const pb2 = h('button', 'btn sm', 'Previous'); pb2.type = 'button'; pb2.disabled = page === 0; pb2.addEventListener('click', () => act.setUI('ann:page', page - 1)); const nb = h('button', 'btn sm', 'Next'); nb.type = 'button'; nb.disabled = (page + 1) * per >= all.length; nb.addEventListener('click', () => act.setUI('ann:page', page + 1)); pg.appendChild(pb2); pg.appendChild(nb); pg.appendChild(h('span', null, `page ${page + 1} of ${Math.ceil(all.length / per)}`)); left.appendChild(pg); } + body.appendChild(left); + const vw = h('div', 'viewer'); const fn = volumeFn(rec + 1, r.fields[0]), signed = !/velocity|tracer/.test(r.fields[0]); + let w = ui['ann:w'] != null ? ui['ann:w'] : 0.5; + const vol = h('canvas', 'vol'); vol.setAttribute('role', 'img'); vol.setAttribute('aria-label', `${r.fields[0]} volume of ${r.recordKind} ${rec + 1}`); drawVolume(vol, fn, signed, r.grid, w); vw.appendChild(vol); + const row2 = h('div', 'row2'); const sl = h('canvas', 'sl'); sl.setAttribute('role', 'img'); sl.setAttribute('aria-label', 'Slice'); drawSlice2(sl, fn, signed, 32, 240, w); row2.appendChild(sl); + const facts = h('dl', 'facts'); const sliceDd = h('dd', null, 'z = ' + Math.round(w * (r.grid[2] - 1) + 1) + ' of ' + r.grid[2]); + for (const [k, v] of [[r.recordKind.charAt(0).toUpperCase() + r.recordKind.slice(1), String(rec + 1) + ' of ' + r.records], ['Grid', r.grid.join(' × ')], ['Fields', r.fields.join(', ')]]) { facts.appendChild(h('dt', null, k)); facts.appendChild(h('dd', null, v)); } facts.appendChild(h('dt', null, 'Slice')); facts.appendChild(sliceDd); + row2.appendChild(facts); vw.appendChild(row2); + const ctl = h('div', 'ctl'); const zl = h('label'); zl.appendChild(document.createTextNode('Slice')); const zi = h('input'); zi.type = 'range'; zi.min = 0; zi.max = r.grid[2] - 1; zi.value = Math.round(w * (r.grid[2] - 1)); + zi.addEventListener('input', () => { w = +zi.value / Math.max(1, r.grid[2] - 1); drawVolume(vol, fn, signed, r.grid, w); drawSlice2(sl, fn, signed, 32, 240, w); sliceDd.textContent = 'z = ' + (+zi.value + 1) + ' of ' + r.grid[2]; }); + zi.addEventListener('change', () => { state.ui['ann:w'] = w; }); zl.appendChild(zi); ctl.appendChild(zl); + const prev = h('button', 'btn sm', 'Previous'); prev.type = 'button'; prev.disabled = rec === 0; prev.addEventListener('click', () => act.setUI('ann:rec', rec - 1)); const next = h('button', 'btn sm', 'Next'); next.type = 'button'; next.disabled = rec >= r.records - 1; next.addEventListener('click', () => act.setUI('ann:rec', rec + 1)); ctl.appendChild(prev); ctl.appendChild(next); + vw.appendChild(ctl); body.appendChild(vw); main.appendChild(body); + // Form for the current record. + const cur = recOf(rec) || {}, draft = Object.assign({ label: cur.label || null, tags: cur.tags || [], review: cur.review || null, notes: cur.notes || '' }, state.draft && state.draft.key === r.id + ':' + rec ? state.draft.v : {}); + const ins = h('div', 'inspector'); + const hd = h('header'); hd.appendChild(h('span', 'kicker', r.label)); hd.appendChild(h('h2', null, r.recordKind.charAt(0).toUpperCase() + r.recordKind.slice(1) + ' ' + (rec + 1))); const meta = h('div', 'ins-meta'); meta.appendChild(cur.review ? stateEl(cur.review === 'accepted' ? 'good' : cur.review === 'needs' ? 'warn' : 'serious', cur.review === 'needs' ? 'needs work' : cur.review) : stateEl('neutral', 'unreviewed')); hd.appendChild(meta); ins.appendChild(hd); + const f = h('div', 'form'); + const field = (label, control) => { const l = h('div'); l.appendChild(h('h3', null, label)); l.appendChild(control); return l; }; + const setDraft = (k, v) => { state.draft = { key: r.id + ':' + rec, v: Object.assign({}, draft, { [k]: v }) }; act.setUI('ann:tick', (ui['ann:tick'] || 0) + 1); }; + if (task.type === 'class') { const rd = h('div', 'radio'); task.classes.forEach((c, i) => { const l = h('label'); const inp = h('input'); inp.type = 'radio'; inp.name = 'lbl'; inp.checked = draft.label === c; inp.addEventListener('change', () => setDraft('label', c)); l.appendChild(inp); l.appendChild(document.createTextNode(c)); l.appendChild(h('span', 'kbd', String(i + 1))); rd.appendChild(l); }); f.appendChild(field(task.label, rd)); } + else { const rd = h('div', 'radio'); task.options.forEach((c, i) => { const l = h('label'); const inp = h('input'); inp.type = 'checkbox'; inp.checked = draft.tags.includes(c); inp.addEventListener('change', () => setDraft('tags', inp.checked ? draft.tags.concat([c]) : draft.tags.filter(x => x !== c))); l.appendChild(inp); l.appendChild(document.createTextNode(c)); l.appendChild(h('span', 'kbd', String(i + 1))); rd.appendChild(l); }); f.appendChild(field(task.label, rd)); } + const rv = h('div', 'seg'); for (const [v, t, c] of [['accepted', 'Accept', 'ok'], ['needs', 'Needs work', 'warn'], ['reject', 'Reject', 'bad']]) { const b = h('button', c, t); b.type = 'button'; b.setAttribute('aria-pressed', String(draft.review === v)); b.addEventListener('click', () => setDraft('review', v)); rv.appendChild(b); } f.appendChild(field('Review', rv)); + const notes = h('textarea'); notes.rows = 2; notes.value = draft.notes; notes.placeholder = 'notes'; notes.addEventListener('change', () => setDraft('notes', notes.value)); f.appendChild(field('Notes', notes)); + ins.appendChild(f); + const acts = h('div', 'actions'); const save = h('button', 'btn primary', 'Save and next'); save.type = 'button'; save.disabled = !draft.review && !draft.label && !draft.tags.length; save.addEventListener('click', () => act.saveAnnotation(r.id, rec, draft)); acts.appendChild(save); const skip = h('button', 'btn ghost', 'Skip'); skip.type = 'button'; skip.addEventListener('click', () => act.setUI('ann:rec', Math.min(r.records - 1, rec + 1))); acts.appendChild(skip); ins.appendChild(acts); + const hist = cur.revisions || []; if (hist.length) { const d = h('div'); d.appendChild(h('h3', null, 'History')); const ul = h('ul', 'hist'); for (const rv2 of hist.slice().reverse()) { const li = h('li'); li.appendChild(h('b', null, rv2.at)); li.appendChild(document.createTextNode((rv2.label || (rv2.tags || []).join(', ') || '—') + ' · ' + (rv2.review === 'needs' ? 'needs work' : rv2.review || ''))); ul.appendChild(li); } d.appendChild(ul); ins.appendChild(d); } + aside.appendChild(ins); + main.tabIndex = -1; + main.onkeydown = ev => { if (ev.target.tagName === 'TEXTAREA' || ev.target.tagName === 'INPUT' || ev.target.tagName === 'SELECT') return; if (ev.key === 'ArrowRight') { act.setUI('ann:rec', Math.min(r.records - 1, rec + 1)); ev.preventDefault(); } else if (ev.key === 'ArrowLeft') { act.setUI('ann:rec', Math.max(0, rec - 1)); ev.preventDefault(); } else if (/^[1-9]$/.test(ev.key)) { const opts = task.classes || task.options; const c = opts[+ev.key - 1]; if (c) { if (task.type === 'class') setDraft('label', c); else setDraft('tags', draft.tags.includes(c) ? draft.tags.filter(x => x !== c) : draft.tags.concat([c])); } } else if (ev.key === 'Enter' && !save.disabled) act.saveAnnotation(r.id, rec, draft); }; + return true; + } + + // ---- Models -------------------------------------------------------------- + function models(main, aside, state, act, opts = {}) { + aside.textContent = ''; + const sel = state.sel.models; + const buildBar = () => { + const bar = h('div', 'filters'); + const q = h('input'); q.type = 'search'; q.placeholder = 'Find a fit'; q.value = state.query || ''; q.setAttribute('aria-label', 'Find a fit'); q.addEventListener('input', () => act.setQuery(q.value)); bar.appendChild(q); + const grp = h('div', 'grp'); grp.setAttribute('role', 'group'); grp.setAttribute('aria-label', 'Kind of change'); + for (const k of KINDS) { const b = h('button', 'btn sm chip ' + k, Tree.kindLabel(k)); b.type = 'button'; b.setAttribute('aria-pressed', String(!state.kinds || state.kinds.has(k))); b.addEventListener('click', () => act.toggleKind(k)); grp.appendChild(b); } + bar.appendChild(grp); + const showMap = opts.stacked || state.view === 'map'; + if (!opts.stacked && opts.views && opts.views.length > 1) { const vg = h('div', 'grp'); vg.setAttribute('role', 'group'); vg.setAttribute('aria-label', 'View'); for (const v of opts.views) { const b = h('button', 'btn sm', v === 'list' ? 'List' : 'Map'); b.type = 'button'; b.setAttribute('aria-pressed', String(state.view === v)); b.addEventListener('click', () => act.setView(v)); vg.appendChild(b); } bar.appendChild(vg); } + if (showMap) { + const tb = h('button', 'btn sm', 'Accuracy track'); tb.type = 'button'; tb.setAttribute('aria-pressed', String(!!state.track)); tb.addEventListener('click', () => act.toggleTrack()); bar.appendChild(tb); + const z = h('div', 'zoomctl'); for (const [t, d, l] of [['−', -0.2, 'Zoom out'], ['+', 0.2, 'Zoom in']]) { const b = h('button', 'btn sm', t); b.type = 'button'; b.setAttribute('aria-label', l); b.addEventListener('click', () => act.zoom(d)); z.appendChild(b); } bar.appendChild(z); + } + if (sel && byId.get(sel) && !byId.get(sel).planned) { const bb = h('button', 'btn sm primary'); bb.type = 'button'; bb.appendChild(icon('branch')); bb.appendChild(document.createTextNode(' Branch from ' + (byId.get(sel).ck || byId.get(sel).short))); bb.addEventListener('click', () => act.branch(sel)); bar.appendChild(bb); } + bar.appendChild(h('span', 'pop', `${LAB.models.length} fits · ${withBench} benchmarked`)); + return bar; + }; + const sig = JSON.stringify([state.kinds ? [...state.kinds] : null, state.query || '', state.zoom || 1, !!state.track, !!opts.stacked, state.view, plannedVersion]); + const prev = main._tree; + const common = { onSelect: id => act.select(id), kinds: state.kinds, query: state.query, barLabel: accLabel(ACC[0].id), unit: TIME.unit, trackLabel: accLabel(ACC[0].id) + ', lower is better', metric: ACC[0].id, timeId: TIME.id }; + if (prev && prev.sig === sig && main.contains(prev.bar)) { + const bar = buildBar(); prev.bar.replaceWith(bar); prev.bar = bar; + if (prev.map) prev.map.select(sel); if (prev.list) prev.list.select(sel); + if (sel && sel !== prev.sel) { if (prev.map) prev.map.scrollTo(sel, true); if (prev.list) prev.list.scrollTo(sel, true); } + prev.sel = sel; + } else { + main.textContent = ''; const bar = buildBar(); main.appendChild(bar); + const inst = { sig, bar, sel, map: null, list: null }; + const mapOpts = Object.assign({ selected: sel, zoom: state.zoom || 1, track: !!state.track }, common); + if (opts.stacked) { const mh = h('div', 'tree-map'); main.appendChild(mh); inst.map = Tree.renderMap(mh, graph, mapOpts); const lh = h('div', 'tree-host'); main.appendChild(lh); inst.list = Tree.renderList(lh, graph, Object.assign({ selected: sel }, common)); } + else { const host = h('div', state.view === 'map' ? 'tree-map' : 'tree-host'); main.appendChild(host); if (state.view === 'map') inst.map = Tree.renderMap(host, graph, mapOpts); else inst.list = Tree.renderList(host, graph, Object.assign({ selected: sel }, common)); } + if (sel) { if (inst.map) inst.map.scrollTo(sel, false); if (inst.list) inst.list.scrollTo(sel, false); } + main._tree = inst; + } + if (sel && byId.get(sel)) { if (byId.get(sel).planned) inspectorPlanned(aside, sel, state, act); else inspectorModel(aside, sel, act); return true; } + modelsGuide(aside, act); return false; + } + function modelsGuide(aside, act) { + const ins = h('div', 'inspector'); + const key = h('ul', 'keylist'); + for (const [cls, t] of [['k-solid', 'Derived from an earlier fit'], ['k-faint', 'Checkpoint order'], ['k-dash', 'Compared with (selected fit)'], ['k-bar', accLabel(ACC[0].id) + ', log scale, longer is better']]) { const li = h('li', cls); li.appendChild(h('i')); li.appendChild(h('span', null, t)); key.appendChild(li); } + ins.appendChild(key); + const d = h('div'); d.appendChild(h('h3', null, 'Branch points')); + const ul = h('ul', 'links'); + for (const f of graph.families) { const p = byId.get(f.parent); const first = byId.get(f.children[0]), last = byId.get(f.children[f.children.length - 1]); const li = h('li'); const b = h('button'); b.type = 'button'; b.appendChild(document.createTextNode(`${p.ck || p.short} · ${f.children.length} fits`)); b.appendChild(h('span', 'why', `${first.ck || first.short} to ${last.ck || last.short} · ${Tree.kindLabel(first.kind).toLowerCase()}`)); b.addEventListener('click', () => act.select(p.id)); li.appendChild(b); ul.appendChild(li); } + d.appendChild(ul); ins.appendChild(d); + const k = h('div'); k.appendChild(h('h3', null, 'Kinds of change')); + const dl = h('dl', 'metrics'); + for (const kind of KINDS) { const c = h('div'); c.appendChild(h('dt', 'chip ' + kind, Tree.kindLabel(kind))); c.appendChild(h('dd', null, String(LAB.models.filter(m => m.kind === kind).length))); dl.appendChild(c); } + k.appendChild(dl); ins.appendChild(k); + aside.appendChild(ins); + } + // A planned model: configured on its node, trained from Train. + function inspectorPlanned(aside, id, state, act) { + aside.textContent = ''; + const m = byId.get(id), p = byId.get(m.parent), n = m.setup || {}; + const ins = h('div', 'inspector'); + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'planned · from ' + (p.ck || p.short))); hd.appendChild(h('h2', null, m.label)); + const meta = h('div', 'ins-meta'); meta.appendChild(stateEl(n.prepared ? 'good' : 'pending', n.prepared ? 'trial prepared' : 'not prepared')); hd.appendChild(meta); ins.appendChild(hd); + const lin = h('div', 'actions'); const pb = h('button', 'btn sm ghost'); pb.type = 'button'; pb.appendChild(icon('branch')); pb.appendChild(document.createTextNode(' From ' + (p.ck || p.short) + (p.worstP ? ' · ' + Tree.fmtSci(p.worstP) : ''))); pb.addEventListener('click', () => act.select(p.id)); lin.appendChild(pb); ins.appendChild(lin); + const f = h('div', 'form'); + const field = (label, control, hint) => { const l = h('label'); l.appendChild(document.createTextNode(label)); l.appendChild(control); if (hint) l.appendChild(h('span', 'hint', hint)); return l; }; + const nm = h('input'); nm.type = 'text'; nm.value = n.name || ''; nm.placeholder = 'name'; nm.addEventListener('change', () => act.setPlan(id, 'name', nm.value)); f.appendChild(field('Name', nm)); + const kr = h('div', 'radio'); for (const k of KINDS.slice(0, 3)) { const l = h('label'); const r = h('input'); r.type = 'radio'; r.name = 'kind-' + id; r.checked = (n.kind || 'refinement') === k; r.addEventListener('change', () => act.setPlan(id, 'kind', k)); l.appendChild(r); l.appendChild(document.createTextNode(Tree.kindLabel(k))); kr.appendChild(l); } f.appendChild(field('Kind of change', kr)); + const selRecipe = h('select'); for (const r of LAB.recipes || []) { if (r.kind === 'evaluation') continue; const o = h('option', null, r.label); o.value = r.id; o.selected = r.id === (n.recipe || 'run-study'); selRecipe.appendChild(o); } selRecipe.addEventListener('change', () => act.setPlan(id, 'recipe', selRecipe.value)); const rc = recipeOf(n.recipe || 'run-study'); f.appendChild(field('Recipe', selRecipe, rc ? rc.path + (rc.config ? ' · ' + rc.config : '') : '')); + const selData = h('select'); for (const r of LAB.project.data) { if (r.state === 'missing') continue; const o = h('option', null, r.label); o.value = r.id; o.selected = r.id === (n.data || 'water-capture'); selData.appendChild(o); } selData.addEventListener('change', () => act.setPlan(id, 'data', selData.value)); f.appendChild(field('Data', selData)); + const rr = h('div', 'radio'); for (const r of LAB.runtimes || []) { const l = h('label'); const i = h('input'); i.type = 'radio'; i.name = 'rt-' + id; i.checked = (n.runtime || (rc && rc.runtime)) === r.id; i.addEventListener('change', () => act.setPlan(id, 'runtime', r.id)); l.appendChild(i); l.appendChild(document.createTextNode(r.label)); rr.appendChild(l); } f.appendChild(field('Runtime', rr)); + const two = h('div', 'two'); const upd = h('input'); upd.type = 'number'; upd.min = 1; upd.value = n.updates || p.updates || 1600; upd.addEventListener('change', () => act.setPlan(id, 'updates', +upd.value)); two.appendChild(field('Updates', upd)); const lr = h('input'); lr.type = 'text'; lr.value = n.lr || p.lr || '1e-5'; lr.addEventListener('change', () => act.setPlan(id, 'lr', lr.value)); two.appendChild(field('Learning rate', lr)); f.appendChild(two); + const two2 = h('div', 'two'); const att = h('input'); att.type = 'number'; att.min = 1; att.value = n.attempts || 2; att.addEventListener('change', () => act.setPlan(id, 'attempts', +att.value)); two2.appendChild(field('Attempts', att)); const wall = h('input'); wall.type = 'number'; wall.min = 1; wall.value = n.wall || 90; wall.addEventListener('change', () => act.setPlan(id, 'wall', +wall.value)); two2.appendChild(field('Wall minutes', wall)); f.appendChild(two2); + const two3 = h('div', 'two'); const tm = h('select'); for (const a of ACC) { const o = h('option', null, a.label); o.value = a.id; o.selected = (n.target || ACC[0].id) === a.id; tm.appendChild(o); } tm.addEventListener('change', () => act.setPlan(id, 'target', tm.value)); two3.appendChild(field('Target', tm)); const th = h('input'); th.type = 'text'; th.value = n.threshold || ''; th.placeholder = p.worstP ? 'below ' + Tree.fmtSci(p.worstP) : ''; th.addEventListener('change', () => act.setPlan(id, 'threshold', th.value)); two3.appendChild(field('Threshold', th)); f.appendChild(two3); + const ch = h('textarea'); ch.rows = 2; ch.value = n.change || ''; ch.placeholder = 'What this model changes'; ch.addEventListener('change', () => act.setPlan(id, 'change', ch.value)); f.appendChild(field('Intended change', ch)); + ins.appendChild(f); + const steps = h('ol', 'steps'); + for (const [t, done] of [['Prepared', !!n.prepared], ['Reviewed', false], ['Approved', false], ['Started', false]]) { const li = h('li', done ? 'done' : 'todo'); li.appendChild(stateEl(done ? 'good' : 'neutral', '')); li.appendChild(h('span', 'title', t)); steps.appendChild(li); } + ins.appendChild(steps); + const acts = h('div', 'actions'); + if (!n.prepared) { const b = h('button', 'btn primary', 'Prepare trial'); b.type = 'button'; b.addEventListener('click', () => act.setPlan(id, 'prepared', true)); acts.appendChild(b); } + else { const b = h('button', 'btn primary', 'Open in Train'); b.type = 'button'; b.addEventListener('click', () => act.go('train', id)); acts.appendChild(b); } + const d = h('button', 'btn ghost', 'Discard'); d.type = 'button'; d.addEventListener('click', () => act.discardPlan(id)); acts.appendChild(d); + ins.appendChild(acts); + aside.appendChild(ins); + } + + function inspectorModel(aside, id, act, { compact = false } = {}) { + aside.textContent = ''; + const m = byId.get(id); if (!m) return; + const ins = h('div', 'inspector'); + const hd = h('header'); hd.appendChild(h('span', 'kicker', (m.ck ? m.ck + ' · ' : '') + m.groupLabel)); hd.appendChild(h('h2', null, m.short)); + const meta = h('div', 'ins-meta'); meta.appendChild(h('span', 'chip ' + m.kind, Tree.kindLabel(m.kind))); meta.appendChild(stateEl('good', 'source reviewed')); + if (m.alias && m.alias !== m.ck) meta.appendChild(h('span', 'small faint', m.alias)); hd.appendChild(meta); ins.appendChild(hd); + const lin = h('div', 'actions'); + if (m.parent) { const p = byId.get(m.parent); const b = h('button', 'btn sm ghost'); b.type = 'button'; b.appendChild(icon('branch')); b.appendChild(document.createTextNode(' From ' + (p.ck || p.short))); b.addEventListener('click', () => act.select(p.id)); lin.appendChild(b); } + else lin.appendChild(h('span', 'small faint', 'No recorded ancestry')); + const kids = graph.families.find(f => f.parent === m.id); if (kids) { const b = h('button', 'btn sm ghost', `${kids.children.length} fits branch here`); b.type = 'button'; b.addEventListener('click', () => act.select(kids.children[0])); lin.appendChild(b); } + ins.appendChild(lin); + const dl = h('dl', 'metrics'); + for (const a of ACC) dl.appendChild(cell(a.label, m[a.id] == null ? '—' : Tree.fmtSci(m[a.id]))); + dl.appendChild(cell(TIME.label, m[TIME.id] == null ? '—' : m[TIME.id].toFixed(2), m[TIME.id] == null ? '' : TIME.unit)); + dl.appendChild(cell('Parameters', m.params == null ? '—' : m.params.toLocaleString('en-GB'))); + ins.appendChild(dl); + ins.appendChild(h('p', 'note', MET.population)); + if (!compact) for (const [k, v] of [['Purpose', m.purpose], ['Outcome', m.outcome], ['Lesson', m.lesson]]) { if (!v) continue; const d = h('div'); d.appendChild(h('h3', null, k)); d.appendChild(h('p', null, v + (v.length >= 200 ? '…' : ''))); ins.appendChild(d); } + const cmp = Tree.comparisons(graph, m.id); + if (cmp.length) { const d = h('div'); d.appendChild(h('h3', null, 'Compared with')); const ul = h('ul', 'links'); for (const c of cmp) { const o = byId.get(c.other); const li = h('li'); const b = h('button'); b.type = 'button'; b.appendChild(document.createTextNode((o.ck ? o.ck + ' · ' : '') + o.short)); if (!compact) b.appendChild(h('span', 'why', c.excerpt.slice(0, 140) + (c.excerpt.length > 140 ? '…' : ''))); b.addEventListener('click', () => act.select(o.id)); li.appendChild(b); ul.appendChild(li); } d.appendChild(ul); ins.appendChild(d); } + const ac = h('div', 'actions'); + const b1 = h('button', 'btn primary'); b1.type = 'button'; b1.appendChild(icon('branch')); b1.appendChild(document.createTextNode(' Branch from here')); b1.addEventListener('click', () => act.branch(m.id)); ac.appendChild(b1); + const b2 = h('button', 'btn', 'Benchmark'); b2.type = 'button'; b2.addEventListener('click', () => act.go('benchmarks', m.id)); ac.appendChild(b2); + const b3 = h('button', 'btn ghost', 'Card'); b3.type = 'button'; ac.appendChild(b3); + ins.appendChild(ac); + aside.appendChild(ins); + } + + // ---- Train: planned models, every training run, then the budgeted studies -- + function train(main, aside, state, act) { + main.textContent = ''; aside.textContent = ''; + const sel = state.sel.train, studies = LAB.project.studies, planned = (state.planned || []); + if (planned.length) { + main.appendChild(sect('Planned', String(planned.length))); + const pl = h('ul', 'rows'); + for (const pn of planned) { const m = byId.get(pn.id), p = byId.get(pn.parent); if (!m) continue; const li = h('li'); const b = h('button', 'row study'); b.type = 'button'; b.setAttribute('aria-pressed', String(pn.id === sel)); b.appendChild(stateEl(pn.setup.started ? 'pending' : pn.setup.prepared ? 'pending' : 'neutral', pn.setup.started ? 'running' : pn.setup.prepared ? 'prepared' : 'planned')); const t = h('span'); t.appendChild(h('span', 'title', m.label)); t.appendChild(h('span', 'sub', pn.setup.change || 'not configured')); b.appendChild(t); b.appendChild(h('span', 'sub mono', p ? (p.ck || p.short) : '')); b.appendChild(h('span', 'sub num', 'attempts 0 of ' + (pn.setup.attempts || 2))); b.appendChild(h('span', 'sub num', 'today')); b.addEventListener('click', () => act.select(pn.id)); li.appendChild(b); pl.appendChild(li); } + main.appendChild(pl); + } + const bar = h('div', 'filters'); if (planned.length) bar.style.marginTop = '20px'; + const q = h('input'); q.type = 'search'; q.placeholder = 'Find a run'; q.value = state.query || ''; q.setAttribute('aria-label', 'Find a run'); q.addEventListener('input', () => act.setQuery(q.value)); bar.appendChild(q); + const grp = h('div', 'grp'); for (const k of KINDS) { const b = h('button', 'btn sm chip ' + k, Tree.kindLabel(k)); b.type = 'button'; b.setAttribute('aria-pressed', String(!state.kinds || state.kinds.has(k))); b.addEventListener('click', () => act.toggleKind(k)); grp.appendChild(b); } bar.appendChild(grp); + if (!planned.length) { const hint = h('span', 'pop'); hint.appendChild(document.createTextNode('To train a new model, branch from a fit on the ')); const a = h('a', null, 'map'); a.href = '#models'; hint.appendChild(a); bar.appendChild(hint); } + main.appendChild(bar); + const runs = graph.nodes.filter(m => !m.planned && (!state.kinds || state.kinds.has(m.kind)) && (!state.query || Tree.matches(m, state.query))); + main.appendChild(sect('Runs', `${runs.length} of ${LAB.models.length}`)); + const ul = h('ul', 'rows'); let lastGroup = null; + for (const m of runs) { + if (m.group !== lastGroup) { lastGroup = m.group; const gh = h('li'); const hh = h('div', 'tsect'); hh.style.gridTemplateColumns = '1fr auto'; hh.appendChild(h('h3', null, m.groupLabel)); gh.appendChild(hh); ul.appendChild(gh); } + const st = runStudy(m), li = h('li'); const b = h('button', 'row run'); b.type = 'button'; b.setAttribute('aria-pressed', String(m.id === sel)); + b.appendChild(h('span', 'ck', m.ck || '—')); + const t = h('span'); t.appendChild(h('span', 'title', m.short)); t.appendChild(h('span', 'sub', [m.updates && m.updates.toLocaleString('en-GB') + ' updates', m.lr && 'lr ' + m.lr, m.seed && 'seed ' + m.seed, st && st.label].filter(Boolean).join(' · ') || 'recorded in the card')); b.appendChild(t); + b.appendChild(h('span', 'chip ' + m.kind, Tree.kindLabel(m.kind))); + b.appendChild(h('span', 'sub num', m.loss == null ? '' : 'loss ' + Tree.fmtSci(m.loss))); + b.appendChild(h('span', 'sub', m.device || '')); + b.appendChild(stateEl('good', 'completed')); + b.addEventListener('click', () => act.select(m.id)); li.appendChild(b); ul.appendChild(li); + } + main.appendChild(ul); + const sh = sect('Studies', String(studies.length)); sh.style.marginTop = '28px'; main.appendChild(sh); + const sul = h('ul', 'rows'); + for (const s of studies) { + const li = h('li'); const b = h('button', 'row study'); b.type = 'button'; b.setAttribute('aria-pressed', String(s.id === sel)); + b.appendChild(stateEl(studyState(s), s.state === 'closed' ? 'waiting' : s.state)); + const t = h('span'); t.appendChild(h('span', 'title', s.label)); t.appendChild(h('span', 'sub', s.decision)); b.appendChild(t); + const mm = s.model && byId.get(s.model); b.appendChild(h('span', 'sub mono', mm ? (mm.ck || mm.short) : '')); + b.appendChild(h('span', 'sub num', 'attempts ' + s.attempts)); b.appendChild(h('span', 'sub num', fmtDate(s.updated))); + b.addEventListener('click', () => act.select(s.id)); li.appendChild(b); sul.appendChild(li); + } + main.appendChild(sul); + const pn = planned.find(x => x.id === sel); + if (pn) return inspectorPlannedRun(aside, pn, act); + const run = sel && byId.get(sel); + if (run && !run.planned) return inspectorRun(aside, run, act); + const s = studies.find(x => x.id === sel); + if (!s) { + const used = studies.reduce((a, x) => { const m = /^(\d+) of (\d+)/.exec(x.attempts); if (m) { a[0] += +m[1]; a[1] += +m[2]; } return a; }, [0, 0]); + const ins = h('div', 'inspector'); const dl = h('dl', 'metrics'); + dl.appendChild(cell('Runs', String(LAB.models.length))); dl.appendChild(cell('On CUDA', String(LAB.models.filter(m => m.device === 'CUDA').length))); dl.appendChild(cell('Waiting', String(studies.filter(x => x.state === 'closed').length + planned.filter(x => x.setup.prepared && !x.setup.started).length))); dl.appendChild(cell('Live', String(planned.filter(x => x.setup.started).length))); + ins.appendChild(dl); + const rl = h('div'); rl.appendChild(h('h3', null, 'Recipes')); const rul = h('ul', 'links'); + for (const r of LAB.recipes || []) { const li = h('li'); const sp = h('button'); sp.type = 'button'; sp.appendChild(document.createTextNode(r.label)); sp.appendChild(h('span', 'why', r.path + ' · ' + (runtimeOf(r.runtime) || {}).label + ' · ' + r.kind)); li.appendChild(sp); rul.appendChild(li); } + rl.appendChild(rul); ins.appendChild(rl); + aside.appendChild(ins); return false; + } + return inspectorStudy(aside, s, act); + } + function inspectorRun(aside, m, act) { + const ins = h('div', 'inspector'), st = runStudy(m), p = m.parent && byId.get(m.parent); + const hd = h('header'); hd.appendChild(h('span', 'kicker', (m.ck ? m.ck + ' · ' : '') + m.groupLabel)); hd.appendChild(h('h2', null, m.short)); const meta = h('div', 'ins-meta'); meta.appendChild(h('span', 'chip ' + m.kind, Tree.kindLabel(m.kind))); meta.appendChild(stateEl('good', 'completed')); hd.appendChild(meta); ins.appendChild(hd); + const su = h('dl', 'setup'); const put = (k, node) => { su.appendChild(h('dt', null, k)); const dd = h('dd'); if (typeof node === 'string') dd.textContent = node; else dd.appendChild(node); su.appendChild(dd); }; + if (p) { const b = h('button', 'btn sm ghost', 'From ' + (p.ck || p.short)); b.type = 'button'; b.addEventListener('click', () => act.go('models', p.id)); put('Start', b); } else put('Start', 'no recorded ancestry'); + put('Updates', m.updates ? m.updates.toLocaleString('en-GB') : '—'); put('Learning rate', m.lr ? String(m.lr) : '—'); put('Seed', m.seed ? String(m.seed) : '—'); + put('Device', m.device || '—'); put('Final loss', m.loss == null ? '—' : Tree.fmtSci(m.loss)); put('Parameters', m.params == null ? '—' : m.params.toLocaleString('en-GB')); + if (st) { const b = h('button', 'btn sm ghost', st.label); b.type = 'button'; b.addEventListener('click', () => act.select(st.id)); put('Study', b); } + ins.appendChild(su); + if (m.experiment) { const d = h('div'); d.appendChild(h('h3', null, 'As recorded')); d.appendChild(h('p', null, m.experiment + (m.experiment.length >= 300 ? '…' : ''))); ins.appendChild(d); } + const acts = h('div', 'actions'); const b1 = h('button', 'btn primary'); b1.type = 'button'; b1.appendChild(icon('branch')); b1.appendChild(document.createTextNode(' Branch from here')); b1.addEventListener('click', () => act.branch(m.id)); acts.appendChild(b1); const b2 = h('button', 'btn', 'On the map'); b2.type = 'button'; b2.addEventListener('click', () => act.go('models', m.id)); acts.appendChild(b2); const b3 = h('button', 'btn ghost', 'Benchmark'); b3.type = 'button'; b3.addEventListener('click', () => act.go('benchmarks', m.id)); acts.appendChild(b3); ins.appendChild(acts); + aside.appendChild(ins); return true; + } + function inspectorPlannedRun(aside, pn, act) { + const m = byId.get(pn.id), p = byId.get(pn.parent), n = pn.setup, rc = recipeOf(n.recipe || 'run-study'), rt = runtimeOf(n.runtime || (rc && rc.runtime)), ds = datasetOf(n.data || 'water-capture'); + const ins = h('div', 'inspector'); + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'planned · from ' + (p.ck || p.short))); hd.appendChild(h('h2', null, m.label)); const meta = h('div', 'ins-meta'); meta.appendChild(stateEl(n.started ? 'pending' : n.prepared ? 'pending' : 'neutral', n.started ? 'running' : n.prepared ? 'prepared, waiting for review' : 'not prepared')); hd.appendChild(meta); ins.appendChild(hd); + const su = h('dl', 'setup'); const put = (k, node) => { su.appendChild(h('dt', null, k)); const dd = h('dd'); if (typeof node === 'string') dd.textContent = node; else dd.appendChild(node); su.appendChild(dd); }; + const pb = h('button', 'btn sm ghost', modelName(p)); pb.type = 'button'; pb.addEventListener('click', () => act.go('models', p.id)); put('From', pb); + if (rc) { const sp = h('span'); sp.appendChild(document.createTextNode(rc.label + ' · ')); sp.appendChild(fileLink(rc.path, rc.path)); put('Recipe', sp); } + if (rt) put('Runtime', rt.label); if (ds) put('Data', ds.label); + put('Training', `${(n.updates || p.updates || 1600).toLocaleString('en-GB')} updates · lr ${n.lr || p.lr || '1e-5'}`); + put('Budget', `${n.attempts || 2} attempts · ${n.wall || 90} min wall`); + if (n.change) put('Intended change', n.change); + ins.appendChild(su); + const steps = h('ol', 'steps'); for (const [t, done] of [['Prepared', !!n.prepared], ['Reviewed', !!n.reviewed], ['Approved', !!n.approved], ['Started', !!n.started]]) { const li = h('li', done ? 'done' : 'todo'); li.appendChild(stateEl(done ? 'good' : 'pending', '')); li.appendChild(h('span', 'title', t)); steps.appendChild(li); } ins.appendChild(steps); + const acts = h('div', 'actions'); + if (!n.prepared) { const b = h('button', 'btn primary', 'Configure on the map'); b.type = 'button'; b.addEventListener('click', () => act.go('models', pn.id)); acts.appendChild(b); } + else if (!n.reviewed) { const b = h('button', 'btn primary', 'Review material'); b.type = 'button'; b.addEventListener('click', () => act.setPlan(pn.id, 'reviewed', true)); acts.appendChild(b); } + else if (!n.approved) { const b = h('button', 'btn primary', 'Approve'); b.type = 'button'; b.addEventListener('click', () => act.setPlan(pn.id, 'approved', true)); acts.appendChild(b); const b2 = h('button', 'btn', 'Start training'); b2.type = 'button'; b2.disabled = true; acts.appendChild(b2); } + else if (!n.started) { const b = h('button', 'btn primary', 'Start training'); b.type = 'button'; b.addEventListener('click', () => act.setPlan(pn.id, 'started', true)); acts.appendChild(b); } + else acts.appendChild(stateEl('good', 'running · attempt 1 of ' + (n.attempts || 2))); + const d = h('button', 'btn ghost', 'Discard'); d.type = 'button'; d.addEventListener('click', () => act.discardPlan(pn.id)); acts.appendChild(d); + ins.appendChild(acts); aside.appendChild(ins); return true; + } + function inspectorStudy(aside, s, act) { + const ins = h('div', 'inspector'); + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'study · ' + s.id)); hd.appendChild(h('h2', null, s.label)); const meta = h('div', 'ins-meta'); meta.appendChild(stateEl(studyState(s), s.state === 'closed' ? 'waiting for approval' : s.state + ' · ' + s.decision)); hd.appendChild(meta); ins.appendChild(hd); + const mm = s.model && byId.get(s.model), rc = recipeOf(s.recipe), rt = runtimeOf(s.runtime), ds = datasetOf(s.data); + const su = h('dl', 'setup'); const put = (k, node) => { su.appendChild(h('dt', null, k)); const dd = h('dd'); if (typeof node === 'string') dd.textContent = node; else dd.appendChild(node); su.appendChild(dd); }; + if (mm) { const b = h('button', 'btn sm ghost', modelName(mm)); b.type = 'button'; b.addEventListener('click', () => act.go('models', mm.id)); put('Baseline', b); } + if (rc) { const sp = h('span'); sp.appendChild(document.createTextNode(rc.label + ' · ')); sp.appendChild(fileLink(rc.path, rc.path)); if (rc.config) { sp.appendChild(document.createTextNode(' · ')); sp.appendChild(fileLink(rc.config, rc.config)); } put('Recipe', sp); } + if (rt) put('Runtime', rt.label); + if (ds) { const b = h('button', 'btn sm ghost', ds.label); b.type = 'button'; b.addEventListener('click', () => act.go('data', ds.id)); put('Data', b); } + put('Budget', `${s.attempts} attempts · ${s.wall || '—'} wall`); + if (s.change) put('Intended change', s.change); + ins.appendChild(su); + const trials = (LAB.project.trials || {})[s.id] || Array.from({ length: s.trials }, (_, i) => ({ id: 't' + (i + 1), label: 'trial ' + (i + 1), state: s.state === 'completed' ? 'completed' : 'prepared' })); + if (trials.length) { const d = h('div'); d.appendChild(h('h3', null, 'Runs in this study')); const tl = h('ul', 'trials'); for (const t of trials) { const li = h('li'); li.appendChild(stateEl(t.state === 'completed' ? 'good' : t.state === 'failed' ? 'serious' : 'pending', '')); li.appendChild(h('span', null, t.label + ' · ' + t.state)); const r = t.run && LAB.models.find(m => m.ck === t.run); if (r) { const b = h('button', 'btn sm ghost', r.ck); b.type = 'button'; b.addEventListener('click', () => act.select(r.id)); li.appendChild(b); } else li.appendChild(h('span')); tl.appendChild(li); } d.appendChild(tl); ins.appendChild(d); } + const pending = s.state === 'closed'; + const steps = h('ol', 'steps'); for (const [t, done] of [['Prepared', true], ['Reviewed', true], ['Approved', !pending], ['Started', !pending]]) { const li = h('li', done ? 'done' : 'todo'); li.appendChild(stateEl(done ? 'good' : 'pending', '')); li.appendChild(h('span', 'title', t)); steps.appendChild(li); } ins.appendChild(steps); + const acts = h('div', 'actions'); + if (pending) { const b = h('button', 'btn primary', 'Approve'); b.type = 'button'; acts.appendChild(b); const b2 = h('button', 'btn', 'Start training'); b2.type = 'button'; b2.disabled = true; b2.title = 'Approve first'; acts.appendChild(b2); } + else if (mm) { const b = h('button', 'btn primary'); b.type = 'button'; b.appendChild(icon('branch')); b.appendChild(document.createTextNode(' Branch from ' + (mm.ck || mm.short))); b.addEventListener('click', () => act.branch(mm.id)); acts.appendChild(b); const b2 = h('button', 'btn ghost', 'Results'); b2.type = 'button'; b2.addEventListener('click', () => act.go('benchmarks', s.model)); acts.appendChild(b2); } + ins.appendChild(acts); + const docs = docsOf(s.label); if (docs.length) { const dd = h('div'); dd.appendChild(h('h3', null, 'Documents')); dd.appendChild(docChips(s.label, null, id => act.go('analysis', id))); ins.appendChild(dd); } + aside.appendChild(ins); return true; + } + + // ---- Benchmarks ---------------------------------------------------------- + function benchmarks(main, aside, state, act) { + main.textContent = ''; aside.textContent = ''; + const sel = state.sel.benchmarks, metric = state.metric || ACC[0].id, pop = popOf(state), cmp = state.cmp || new Set(); + const popModels = LAB.models.filter(m => inPop(m, pop)); + const bar = h('div', 'filters'); + const ps = h('select'); ps.setAttribute('aria-label', 'Population'); for (const pp of MET.populations || []) { const o = h('option', null, pp.label); o.value = pp.id; o.selected = pp.id === pop.id; ps.appendChild(o); } ps.addEventListener('change', () => act.setPop(ps.value)); bar.appendChild(ps); + const grp = h('div', 'grp'); grp.setAttribute('role', 'group'); grp.setAttribute('aria-label', 'Kind of change'); + for (const k of KINDS) { const b = h('button', 'btn sm chip ' + k, Tree.kindLabel(k)); b.type = 'button'; b.setAttribute('aria-pressed', String(!state.kinds || state.kinds.has(k))); b.addEventListener('click', () => act.toggleKind(k)); grp.appendChild(b); } + bar.appendChild(grp); + const ms = h('select'); ms.setAttribute('aria-label', 'Metric'); for (const a of ACC) { const o = h('option', null, a.label); o.value = a.id; o.selected = metric === a.id; ms.appendChild(o); } ms.addEventListener('change', () => act.setMetric(ms.value)); bar.appendChild(ms); + const fb = h('button', 'btn sm', 'Frontier only'); fb.type = 'button'; fb.setAttribute('aria-pressed', String(!!state.frontierOnly)); fb.addEventListener('click', () => act.toggleFrontier()); bar.appendChild(fb); + const bo = h('button', 'btn sm', 'Benchmarked only'); bo.type = 'button'; bo.setAttribute('aria-pressed', String(!!state.benchOnly)); bo.addEventListener('click', () => act.setUI('benchOnly', !state.ui.benchOnly)); bar.appendChild(bo); + const q = h('input'); q.type = 'search'; q.placeholder = 'Find a fit'; q.value = state.query || ''; q.setAttribute('aria-label', 'Find a fit'); q.addEventListener('input', () => act.setQuery(q.value)); bar.appendChild(q); + main.appendChild(bar); + { const both = popModels.filter(m => benchState(m) === 'accuracy and time').length, ao = popModels.filter(m => benchState(m) === 'accuracy only').length, to = popModels.filter(m => benchState(m) === 'time only').length, none = popModels.filter(m => benchState(m) === 'none retained').length, gb = popModels.filter(m => benchState(m) === 'generator benchmark').length; + const cov = h('div', 'coverage'); const add = (n, t) => { const sp = h('span'); sp.appendChild(h('b', null, String(n))); sp.appendChild(document.createTextNode(' ' + t)); cov.appendChild(sp); }; + add(popModels.length, 'fits'); if (gb) add(gb, 'with the generator benchmark'); if (both) add(both, 'with accuracy and time'); if (ao) add(ao, 'accuracy only'); if (to) add(to, 'time only'); if (none) add(none, 'no retained benchmark'); main.appendChild(cov); } + + const s1 = h('section'); const h1 = sect('Accuracy against time'); s1.appendChild(h1); const c1 = h('div'); s1.appendChild(c1); main.appendChild(s1); + const xMax = Math.max(10, ...popModels.filter(m => m[TIME.id] != null).map(m => Math.ceil(m[TIME.id]))); + const r = Charts.pareto(c1, popModels, { selected: sel, onSelect: id => act.select(id), kinds: state.kinds, frontierOnly: state.frontierOnly, metric, yLabel: accLabel(metric), xLabel: TIME.axis, unit: TIME.unit, xMax: pop.only ? xMax : 10 }); + h1.appendChild(h('span', 'sum', `${r.count} of ${r.total} · ${r.frontier.length} on the frontier · ${pop.note || ''}`)); + if (!pop.only) { const s2 = h('section'); s2.style.marginTop = '28px'; s2.appendChild(sect('Along the checkpoint sequence', accLabel(ACC[0].id))); const c2 = h('div'); s2.appendChild(c2); main.appendChild(s2); Charts.curve(c2, LAB.models, { selected: sel, onSelect: id => act.select(id), metric: ACC[0].id, sections: CURVE_SECTIONS, yLabel: accLabel(ACC[0].id), secondary: ACC[1] || null }); } + + if (cmp.size) { + const sc = h('section'); sc.style.marginTop = '28px'; const hc = sect('Compare', `${cmp.size} of 3`); sc.appendChild(hc); + const chips = h('div', 'cmp-chips'); const list = [...cmp].map(id => byId.get(id)).filter(Boolean); + list.forEach((m, i) => { const c = h('span', 'c s' + (i + 1)); c.appendChild(h('i')); c.appendChild(document.createTextNode(modelName(m))); const x = h('button', null, '×'); x.type = 'button'; x.setAttribute('aria-label', 'Remove ' + modelName(m)); x.addEventListener('click', () => act.toggleCompare(m.id)); c.appendChild(x); chips.appendChild(c); }); + const clr = h('button', 'btn sm ghost', 'Clear'); clr.type = 'button'; clr.addEventListener('click', () => act.clearCompare()); chips.appendChild(clr); sc.appendChild(chips); + const tb = h('table', 'tv cmpt'); tb.style.marginTop = '12px'; const thead = h('thead'); const tr = h('tr'); tr.appendChild(h('th', null, '')); list.forEach(m => tr.appendChild(h('th', 'n', modelName(m)))); thead.appendChild(tr); tb.appendChild(thead); + const tbody = h('tbody'); const base = list[0]; + const rowsDef = [...ACC.map(a => [a.label, a.id, 'lower']), [TIME.label + ' (' + TIME.unit + ')', TIME.id, 'lower'], ['Parameters', 'params', null], ['Updates', 'updates', null], ['Learning rate', 'lr', null]]; + for (const [label, key, better] of rowsDef) { const row = h('tr'); row.appendChild(h('td', null, label)); list.forEach((m, i) => { const v = m[key]; const td = h('td', 'n'); td.textContent = v == null ? '—' : (key === 'params' || key === 'updates') ? v.toLocaleString('en-GB') : key === 'lr' ? String(v) : key === TIME.id ? v.toFixed(2) : Tree.fmtSci(v); if (i > 0 && better && v != null && base[key] != null) { const d = (v - base[key]) / base[key]; const sp = h('span', 'd ' + (d < 0 ? 'better' : d > 0 ? 'worse' : ''), ' ' + (d > 0 ? '+' : '') + (d * 100).toFixed(1) + '%'); td.appendChild(sp); } row.appendChild(td); }); tbody.appendChild(row); } + tb.appendChild(tbody); sc.appendChild(tb); + const cc = h('div'); cc.style.marginTop = '14px'; sc.appendChild(cc); + Charts.lines(cc, list.map((m, i) => ({ id: m.id, label: modelName(m), slot: i + 1, points: curveFor(m) })), { yLabel: 'Training loss', xLabel: 'updates', logY: true }); + main.appendChild(sc); + } + + const st = h('section'); st.style.marginTop = '28px'; + const vis = popModels.filter(m => (!state.kinds || state.kinds.has(m.kind)) && (!state.query || Tree.matches(m, state.query)) && (!state.ui.benchOnly || benchState(m) === 'accuracy and time' || benchState(m) === 'generator benchmark')); + const sort = state.sort || { key: 'n', dir: 1 }; + const cols = [['', 'cmp'], ['', 'ck'], ['Fit', 'short'], ['Kind', 'kind'], ['Group', 'groupLabel'], ...ACC.map(a => [a.label, a.id]), [TIME.label, TIME.id], ['Parameters', 'params'], ['Benchmark', 'bench']]; + vis.sort((a, b) => { const ka = a[sort.key], kb = b[sort.key]; if (ka == null && kb == null) return 0; if (ka == null) return 1; if (kb == null) return -1; return (typeof ka === 'number' ? ka - kb : String(ka).localeCompare(String(kb))) * sort.dir; }); + st.appendChild(sect('All fits', `${vis.length} of ${popModels.length}`)); + const tb = h('table', 'tv'); const thead = h('thead'); const tr = h('tr'); + for (const [label, key] of cols) { const th = h('th', ['ck', 'short', 'kind', 'groupLabel', 'cmp', 'bench'].includes(key) ? '' : 'n'); if (label && key !== 'bench') { const b = h('button', null, label); b.type = 'button'; if (sort.key === key) b.setAttribute('aria-sort', sort.dir > 0 ? 'ascending' : 'descending'); b.addEventListener('click', () => act.setSort(key)); th.appendChild(b); } else th.textContent = label; tr.appendChild(th); } + thead.appendChild(tr); tb.appendChild(thead); + const tbody = h('tbody'); + for (const m of vis) { + const row = h('tr', 'rowbtn' + (m.id === sel ? ' is-sel' : '')); row.addEventListener('click', () => act.select(m.id)); + const tdc = h('td'); const cb = h('input'); cb.type = 'checkbox'; cb.checked = cmp.has(m.id); cb.disabled = !cmp.has(m.id) && cmp.size >= 3; cb.setAttribute('aria-label', 'Compare ' + modelName(m)); cb.addEventListener('click', ev => { ev.stopPropagation(); act.toggleCompare(m.id); }); tdc.appendChild(cb); row.appendChild(tdc); + row.appendChild(h('td', 'ck', m.ck || '')); row.appendChild(h('td', null, m.short)); const kt = h('td'); kt.appendChild(h('span', 'chip ' + m.kind, Tree.kindLabel(m.kind))); row.appendChild(kt); row.appendChild(h('td', 'small faint nowrap', m.groupLabel.replace(/^C\d+[–-]C\d+\s*·\s*/, ''))); + for (const a of ACC) row.appendChild(h('td', 'n', m[a.id] == null ? '' : Tree.fmtSci(m[a.id]))); + row.appendChild(h('td', 'n', m[TIME.id] == null ? '' : m[TIME.id].toFixed(2))); row.appendChild(h('td', 'n', m.params == null ? '' : m.params.toLocaleString('en-GB'))); row.appendChild(h('td', 'small faint nowrap', benchState(m))); + tbody.appendChild(row); + } + tb.appendChild(tbody); st.appendChild(tb); main.appendChild(st); + + const decisions = LAB.project.decisions || []; + if (decisions.length) { + const sd = h('section'); sd.style.marginTop = '28px'; sd.appendChild(sect('Study results', String(decisions.length))); + const dt = h('table', 'tv'); const dh = h('thead'); const dtr = h('tr'); for (const [t, c] of [['Study', ''], ['Decision', ''], ['Reference', 'n'], ['Candidate', 'n'], ['Change', 'n']]) dtr.appendChild(h('th', c, t)); dh.appendChild(dtr); dt.appendChild(dh); + const db = h('tbody'); for (const d of decisions) { const row = h('tr', 'rowbtn'); const stu = LAB.project.studies.find(x => x.label === d.plan); if (stu) row.addEventListener('click', () => act.go('train', stu.id)); row.appendChild(h('td', null, d.plan)); const td = h('td'); td.appendChild(stateEl(d.decision.includes('failed') ? 'serious' : d.decision.includes('pass') ? 'good' : 'neutral', d.decision)); if (d.note) td.appendChild(h('span', 'small faint', ' · ' + d.note)); row.appendChild(td); const isAcc = ACC.some(a => a.id === d.metric); row.appendChild(h('td', 'n', d.ref == null ? '—' : isAcc ? Tree.fmtSci(d.ref) : String(d.ref))); row.appendChild(h('td', 'n', d.cand == null ? '—' : isAcc ? Tree.fmtSci(d.cand) : String(d.cand))); const ch = h('td', 'n'); if (d.ref != null && d.cand != null && isAcc) { const dd = (d.cand - d.ref) / d.ref; ch.appendChild(h('span', 'd ' + (dd < 0 ? 'better' : 'worse'), (dd > 0 ? '+' : '') + (Math.abs(dd) > 9 ? (dd * 100).toFixed(0) : (dd * 100).toFixed(1)) + '%')); } row.appendChild(ch); db.appendChild(row); } + dt.appendChild(db); sd.appendChild(dt); main.appendChild(sd); + } + + if (sel) { inspectorModel(aside, sel, act, { compact: true }); const bs = benchState(byId.get(sel)); if (bs === 'none retained' || bs === 'accuracy only' || bs === 'time only') { const ev = (LAB.recipes || []).find(r => r.kind === 'evaluation'); const nb = h('p', 'note', 'No comparable benchmark retained for this fit' + (bs === 'none retained' ? '' : ' (' + bs + ')') + '.'); aside.querySelector('.inspector').insertBefore(nb, aside.querySelector('.actions')); if (ev) { const rb = h('button', 'btn', 'Run ' + ev.label.toLowerCase()); rb.type = 'button'; rb.title = ev.path; aside.querySelector('.actions').appendChild(rb); } } const acts = aside.querySelector('.actions:last-child'); const b = h('button', 'btn', cmp.has(sel) ? 'Remove from compare' : 'Compare'); b.type = 'button'; b.disabled = !cmp.has(sel) && cmp.size >= 3; b.addEventListener('click', () => act.toggleCompare(sel)); acts.insertBefore(b, acts.firstChild); const b2 = h('button', 'btn ghost', 'In Models'); b2.type = 'button'; b2.addEventListener('click', () => act.go('models', sel)); acts.appendChild(b2); return true; } + const ins = h('div', 'inspector'); const hd = h('header'); hd.appendChild(h('h2', null, 'Frontier')); hd.appendChild(h('span', 'small muted', 'fastest first')); ins.appendChild(hd); + const ul = h('ul', 'links'); + for (const m of r.frontier) { const li = h('li'); const b = h('button'); b.type = 'button'; b.appendChild(document.createTextNode(modelName(m))); b.appendChild(h('span', 'why', `${Tree.fmtSci(m[metric])} · ${m[TIME.id].toFixed(2)} ${TIME.unit} · ${Tree.kindLabel(m.kind).toLowerCase()}`)); b.addEventListener('click', () => act.select(m.id)); li.appendChild(b); ul.appendChild(li); } + ins.appendChild(ul); aside.appendChild(ins); + return false; + } + + // ---- Analysis: field comparison per case, and the reports ----------------- + function drawField(canvas, fn) { + const ctx = canvas.getContext('2d'), W = canvas.width = 240, Hh = canvas.height = 240, n = 32; + const dark = getComputedStyle(document.documentElement).colorScheme === 'dark'; + const mid = dark ? [56, 56, 53] : [240, 239, 236], neg = dark ? [79, 138, 224] : [46, 111, 207], pos = dark ? [217, 111, 60] : [210, 101, 47]; + for (let j = 0; j < n; j++) for (let i = 0; i < n; i++) { const val = fn(i / n, j / n); const c = val < 0 ? neg : pos, t = Math.min(1, Math.abs(val)); const rgb = mid.map((m, k) => Math.round(m + (c[k] - m) * t)); ctx.fillStyle = `rgb(${rgb[0]},${rgb[1]},${rgb[2]})`; ctx.fillRect(i * W / n, j * Hh / n, W / n + 1, Hh / n + 1); } + } + const caseErr = (m, c) => { const rnd = seeded(m.id + ':' + c); const base = m.worstP || 0.01; return base * (0.35 + 0.65 * Math.pow(rnd(), 2.2)); }; + function analysis(main, aside, state, act) { + main.textContent = ''; aside.textContent = ''; + const sel = state.sel.analysis, cases = LAB.cases || { count: 0 }, ds = datasetOf(cases.dataset); + const cands = LAB.models.filter(m => m.worstP != null && m.group !== MET.excludedGroup); + const fit = byId.get(state.fit) || byId.get('fit-9a6432c54d9f9434') || cands[0]; + const cn = state.caseN || 1, z = state.slice || Math.round((cases.slices || 16) / 2); + const vs = h('section'); vs.appendChild(sect('Field comparison', ds ? ds.label : '')); + const vb = h('div', 'viewer-bar'); + const fsel = h('select'); fsel.setAttribute('aria-label', 'Fit'); for (const m of cands) { const o = h('option', null, modelName(m)); o.value = m.id; o.selected = m.id === fit.id; fsel.appendChild(o); } fsel.addEventListener('change', () => act.setFit(fsel.value)); vb.appendChild(fsel); + const cl = h('label'); cl.appendChild(document.createTextNode('Case')); const ci = h('input'); ci.type = 'number'; ci.min = 1; ci.max = cases.count; ci.value = cn; ci.style.width = '64px'; ci.addEventListener('change', () => act.setCase(+ci.value)); cl.appendChild(ci); cl.appendChild(document.createTextNode('of ' + cases.count)); vb.appendChild(cl); + const zl = h('label'); zl.appendChild(document.createTextNode('Slice z')); const zi = h('input'); zi.type = 'range'; zi.min = 1; zi.max = cases.slices || 16; zi.value = z; zi.addEventListener('change', () => act.setSlice(+zi.value)); zl.appendChild(zi); zl.appendChild(h('span', 'num', String(z))); vb.appendChild(zl); + const fl = h('label'); fl.appendChild(document.createTextNode('Field')); const fs = h('select'); for (const f of cases.fields || ['pressure']) fs.appendChild(h('option', null, f)); fl.appendChild(fs); vb.appendChild(fl); + vs.appendChild(vb); + const tri = h('div', 'tri'); const err = caseErr(fit, cn), eqv = fit.eq ? fit.eq * (err / (fit.worstP || err)) : null; + const seed = cn * 0.37 + z * 0.11; + const native = (u, v) => Math.sin(6.3 * u + seed) * Math.cos(4.7 * v - seed * 0.5) * 0.7 + Math.sin(12 * u * v + seed) * 0.3; + const amp = Math.min(0.9, Math.sqrt(err) * 6), pred = (u, v) => native(u, v) + amp * 0.5 * Math.sin(17 * u + 3 * seed) * Math.cos(13 * v + seed); + const canvases = []; + for (const [title, fn, cap] of [['Native', native, 'reference solve'], ['Predicted', pred, modelName(fit)], ['Signed error', (u, v) => (pred(u, v) - native(u, v)) / Math.max(amp, 0.05), 'rel. L2 ' + Tree.fmtSci(err)]]) { const fg = h('figure'); const cv = h('canvas'); cv.setAttribute('role', 'img'); cv.setAttribute('aria-label', title + ' field, case ' + cn + ', slice ' + z); fg.appendChild(cv); drawField(cv, fn); const fc = h('figcaption'); fc.appendChild(h('b', null, title)); fc.appendChild(h('span', null, cap)); fg.appendChild(fc); tri.appendChild(fg); canvases.push([cv, title]); } + const redraw = zz => { const sd = cn * 0.37 + zz * 0.11; const nat = (u, v) => Math.sin(6.3 * u + sd) * Math.cos(4.7 * v - sd * 0.5) * 0.7 + Math.sin(12 * u * v + sd) * 0.3, pr = (u, v) => nat(u, v) + amp * 0.5 * Math.sin(17 * u + 3 * sd) * Math.cos(13 * v + sd); drawField(canvases[0][0], nat); drawField(canvases[1][0], pr); drawField(canvases[2][0], (u, v) => (pr(u, v) - nat(u, v)) / Math.max(amp, 0.05)); }; + zi.addEventListener('input', () => { redraw(+zi.value); zl.querySelector('.num').textContent = zi.value; }); + vs.appendChild(tri); + const sc = h('div', 'scale'); sc.appendChild(document.createTextNode('−')); sc.appendChild(h('i')); sc.appendChild(document.createTextNode('+ · case ' + cn + ' · z ' + z + ' · ' + accLabel(ACC[0].id) + ' ' + Tree.fmtSci(err) + (eqv ? ' · ' + accLabel(ACC[1].id) + ' ' + Tree.fmtSci(eqv) : ''))); vs.appendChild(sc); + const pc = h('div', 'cases'); const ph = sect('Per case', accLabel(ACC[0].id) + ' · ' + modelName(fit)); ph.style.marginTop = '16px'; pc.appendChild(ph); + const Wc = 760, Hc = 64, errs = Array.from({ length: cases.count }, (_, i) => caseErr(fit, i + 1)), emax = Math.max(...errs); + const svg = document.createElementNS(svgNS, 'svg'); svg.setAttribute('viewBox', `0 0 ${Wc} ${Hc}`); svg.setAttribute('role', 'img'); svg.setAttribute('aria-label', 'Error per case; the selected case is highlighted'); + const bw = Math.max(2, Wc / cases.count - 2); + errs.forEach((e, i) => { const rect = document.createElementNS(svgNS, 'rect'); rect.setAttribute('x', i * Wc / cases.count); rect.setAttribute('y', Hc - 8 - (e / emax) * (Hc - 12)); rect.setAttribute('width', bw); rect.setAttribute('height', (e / emax) * (Hc - 12)); rect.setAttribute('rx', 1.5); rect.setAttribute('class', 'b' + (i + 1 === cn ? ' sel' : '')); const t = document.createElementNS(svgNS, 'title'); t.textContent = 'case ' + (i + 1) + ' · ' + Tree.fmtSci(e); rect.appendChild(t); rect.addEventListener('click', () => act.setCase(i + 1)); svg.appendChild(rect); }); + const base = document.createElementNS(svgNS, 'line'); base.setAttribute('x1', 0); base.setAttribute('x2', Wc); base.setAttribute('y1', Hc - 8); base.setAttribute('y2', Hc - 8); base.setAttribute('class', 'base'); svg.appendChild(base); + pc.appendChild(svg); vs.appendChild(pc); main.appendChild(vs); + + const rp = h('section'); rp.style.marginTop = '28px'; const plans = LAB.project.evidence.map(g => g.plan); rp.appendChild(sect('Reports', String(plans.length))); + const tb = h('table', 'tv'); const thead = h('thead'); const tr = h('tr'); for (const t of ['Plan', 'Outcome', 'Documents']) tr.appendChild(h('th', null, t)); thead.appendChild(tr); tb.appendChild(thead); + const tbody = h('tbody'); + for (const plan of plans) { const stu = LAB.project.studies.find(x => x.label === plan); const row = h('tr'); row.appendChild(h('td', null, plan)); const od = h('td'); if (stu) od.appendChild(stateEl(studyState(stu), stu.decision)); else od.appendChild(h('span', 'faint', '—')); row.appendChild(od); const dd = h('td'); dd.appendChild(docChips(plan, sel, id => act.select(id))); row.appendChild(dd); tbody.appendChild(row); } + tb.appendChild(tbody); rp.appendChild(tb); main.appendChild(rp); + + const ins = h('div', 'inspector'); + if (sel) { + const item = LAB.project.evidence.flatMap(g => g.items.map(i => [g.plan, ...i])).find(x => x[1] === sel); + const hd = h('header'); hd.appendChild(h('span', 'kicker', item ? item[0] : sel)); hd.appendChild(h('h2', null, item ? (DOC_ROLE(item[3]) || item[2]) : sel)); ins.appendChild(hd); + if (item) { const pa = h('p', 'note'); pa.appendChild(fileLink(item[3], item[3])); ins.appendChild(pa); } + ins.appendChild(h('pre', 'evtext', item ? `(text of ${item[3].split('/').pop()}, first 64 KiB)\n\nsha256: measured on open` : '')); + aside.appendChild(ins); return true; + } + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'case ' + cn)); hd.appendChild(h('h2', null, modelName(fit))); ins.appendChild(hd); + const dl = h('dl', 'metrics'); dl.appendChild(cell(accLabel(ACC[0].id), Tree.fmtSci(err))); if (eqv) dl.appendChild(cell(accLabel(ACC[1].id), Tree.fmtSci(eqv))); dl.appendChild(cell('Worst of ' + cases.count, Tree.fmtSci(Math.max(...errs)))); dl.appendChild(cell('Median', Tree.fmtSci(errs.slice().sort((a, b) => a - b)[Math.floor(errs.length / 2)]))); ins.appendChild(dl); + const acts = h('div', 'actions'); const b1 = h('button', 'btn', 'Open fit'); b1.type = 'button'; b1.addEventListener('click', () => act.go('models', fit.id)); acts.appendChild(b1); const b2 = h('button', 'btn ghost', 'Benchmark'); b2.type = 'button'; b2.addEventListener('click', () => act.go('benchmarks', fit.id)); acts.appendChild(b2); ins.appendChild(acts); + if (ds) { const pa = h('p', 'note'); pa.appendChild(fileLink(ds.path, ds.path)); ins.appendChild(pa); } + aside.appendChild(ins); return false; + } + + // ---- Layout switcher: the three pages are proposals for the same screen. + const LAYOUTS = [['a-stages.html', 'A', 'Stages'], ['b-map.html', 'B', 'Map'], ['c-ledger.html', 'C', 'Ledger']]; + function layoutSwitch(container, current) { + const nav = h('nav', 'layouts'); nav.setAttribute('aria-label', 'Layout proposal'); nav.appendChild(h('span', 'lbl', 'Layout')); + const links = LAYOUTS.map(([file, k, label]) => { const a = h('a'); a.href = file; a.textContent = k + ' ' + label; if (file === current) a.setAttribute('aria-current', 'page'); nav.appendChild(a); return [a, file]; }); + const cmp = h('a', 'cmp', 'All three'); cmp.href = 'index.html'; nav.appendChild(cmp); + const update = () => { const r = (window.mockRoute && window.mockRoute()) || {}; const screen = r.screen || 'overview'; for (const [a, file] of links) a.href = file + (file === 'c-ledger.html' ? '#s-' + screen : '#' + screen + (r.id ? '/' + encodeURIComponent(r.id) : '')); }; + container.textContent = ''; container.appendChild(nav); update(); window.addEventListener('hashchange', update); return { update }; + } + + // ---- Theme toggle -------------------------------------------------------- + const theme = { + init(btn) { + let cur = 'auto'; try { cur = localStorage.getItem('tuldok-mock-theme') || 'auto'; } catch (e) {} + const q = new URLSearchParams(location.search).get('theme'); if (q === 'light' || q === 'dark') cur = q; + const apply = () => { if (cur === 'auto') document.documentElement.removeAttribute('data-theme'); else document.documentElement.setAttribute('data-theme', cur); btn.textContent = ''; btn.appendChild(icon('theme')); btn.appendChild(document.createTextNode(' ' + { auto: 'System', light: 'Light', dark: 'Dark' }[cur])); btn.setAttribute('aria-label', 'Theme: ' + cur); }; + btn.addEventListener('click', () => { cur = { auto: 'light', light: 'dark', dark: 'auto' }[cur]; try { localStorage.setItem('tuldok-mock-theme', cur); } catch (e) {} apply(); }); + apply(); + } + }; + window.addEventListener('error', e => { const b = document.createElement('div'); b.setAttribute('role', 'alert'); b.style.cssText = 'position:fixed;left:8px;bottom:8px;z-index:99;padding:8px 12px;background:#fff;border:2px solid #b7452b;color:#1b2330;font:13px monospace;max-width:80vw'; b.textContent = 'Mockup script error: ' + e.message + ' (' + (e.filename || '').split('/').pop() + ':' + e.lineno + ')'; document.body.appendChild(b); }); + + return { overview, data, annotate, models, train, benchmarks, analysis, inspectorModel, setPlanned, statusLine, stageSummaries, icon, stateEl, theme, layoutSwitch, graph, KINDS, kindLabel: Tree.kindLabel }; +})(); diff --git a/docs/plans/project-workspace-redesign/mockups/shared/tree.js b/docs/plans/project-workspace-redesign/mockups/shared/tree.js new file mode 100644 index 00000000..d052ae7b --- /dev/null +++ b/docs/plans/project-workspace-redesign/mockups/shared/tree.js @@ -0,0 +1,223 @@ +'use strict'; +// Model lineage: one graph built from the portfolio plus any planned models, +// rendered as a vertical list (git-graph style) or a horizontal lane map. +// Solid lanes are derivations read from the model cards; the faint line is +// checkpoint order only; dashed edges are authored comparisons for the +// selected fit. Both renderers return select() and scrollTo() so a selection +// change never rebuilds the view. +window.LabTree = (() => { + const GROUP_ORDER = ['pressure-inputs','pressure-stability','pressure-multigrid','pressure-direction','pressure-variants','pressure-successors','pressure-other','full_state_generator','pressure_head_component','pressure_component','pressure_request_component','guided_pressure_component','guided_shared_component','full_state_correction_component','full_state_readout_component','recurrent_transport_component','transport_component','viscosity_component']; + const gi = id => { const i = GROUP_ORDER.indexOf(id); return i < 0 ? 99 : i; }; + const svgNS = 'http://www.w3.org/2000/svg'; + const el = (name, attrs = {}, text) => { const n = document.createElementNS(svgNS, name); for (const k in attrs) n.setAttribute(k, attrs[k]); if (text !== undefined) n.textContent = text; return n; }; + const h = (tag, cls, text) => { const n = document.createElement(tag); if (cls) n.className = cls; if (text !== undefined) n.textContent = text; return n; }; + const reduced = () => window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + // A short run of highlights from the previous selection to the new one, so + // the eye can follow the change. Never longer than ~450 ms; skipped when the + // viewer prefers reduced motion. + function walker() { + let timers = []; + return (ordered, fromId, toId, mark) => { + for (const t of timers) clearTimeout(t); timers = []; + if (reduced() || !fromId || !toId || fromId === toId) return; + const i0 = ordered.indexOf(fromId), i1 = ordered.indexOf(toId); if (i0 < 0 || i1 < 0) return; + let steps = i1 > i0 ? ordered.slice(i0 + 1, i1) : ordered.slice(i1 + 1, i0).reverse(); + if (steps.length > 22) { const out = []; for (let k = 0; k < 22; k++) out.push(steps[Math.round(k * (steps.length - 1) / 21)]); steps = out; } + const delay = Math.min(34, 420 / Math.max(1, steps.length)); + steps.forEach((id, k) => { timers.push(setTimeout(() => { mark(id, true); timers.push(setTimeout(() => mark(id, false), 170)); }, k * delay)); }); + }; + } + + function build(models, relationships) { + const nodes = models.map(m => Object.assign({}, m)).sort((a, b) => gi(a.group) - gi(b.group) || ((a.n ?? 9999) - (b.n ?? 9999)) || a.label.localeCompare(b.label)); + nodes.forEach((m, i) => { m.t = i; }); + const byId = new Map(nodes.map(m => [m.id, m])); + const families = new Map(); const laneLast = [0]; + for (const m of nodes) { + const p = m.parents.find(id => byId.has(id)); + m.parent = p || null; + if (!p) { m.lane = 0; continue; } + if (!families.has(p)) { + let lane = 1; const pt = byId.get(p).t; + while (laneLast[lane] !== undefined && laneLast[lane] >= pt) lane++; + families.set(p, { lane, parent: p, children: [], start: pt, end: pt }); + } + const f = families.get(p); m.lane = f.lane; f.children.push(m.id); f.end = m.t; laneLast[f.lane] = m.t; + } + const edges = []; let prev = null; + for (const m of nodes) { + if (m.lane !== 0) continue; + if (prev && ((prev.n != null && m.n != null) || prev.group === m.group)) edges.push({ from: prev.id, to: m.id, kind: 'sequence' }); + prev = m; + } + for (const m of nodes) if (m.parent) edges.push({ from: m.parent, to: m.id, kind: 'derivation' }); + for (const r of relationships) if (byId.has(r.from) && byId.has(r.to)) edges.push({ from: r.from, to: r.to, kind: 'comparison', explanation: r.explanation, excerpt: r.excerpt }); + const sections = []; + for (const m of nodes) { const s = sections[sections.length - 1]; if (!s || s.group !== m.group) sections.push({ group: m.group, label: m.groupLabel, start: m.t, end: m.t, count: 1 }); else { s.end = m.t; s.count++; } } + const seqSpan = new Array(nodes.length).fill(false); + for (const e of edges) if (e.kind === 'sequence') for (let t = byId.get(e.from).t + 1; t < byId.get(e.to).t; t++) seqSpan[t] = true; + const seqIn = new Set(edges.filter(e => e.kind === 'sequence').map(e => e.to)); + const seqOut = new Set(edges.filter(e => e.kind === 'sequence').map(e => e.from)); + return { nodes, byId, edges, sections, families: [...families.values()], laneCount: laneLast.length, seqIn, seqOut, seqSpan }; + } + function related(graph, id) { const out = new Set(); for (const e of graph.edges) if (e.kind !== 'sequence' && (e.from === id || e.to === id)) out.add(e.from === id ? e.to : e.from); return out; } + function comparisons(graph, id) { return graph.edges.filter(e => e.kind === 'comparison' && (e.from === id || e.to === id)).map(e => ({ other: e.from === id ? e.to : e.from, explanation: e.explanation, excerpt: e.excerpt })); } + function metricFill(v) { if (v == null || !(v > 0)) return 0; const f = (Math.log10(1) - Math.log10(Math.max(v, 1e-4))) / 4; return Math.max(0, Math.min(1, f)); } + function matches(m, q) { if (!q) return true; q = q.toLowerCase(); return m.label.toLowerCase().includes(q) || (m.alias || '').toLowerCase().includes(q) || (m.ck || '').toLowerCase().includes(q) || m.groupLabel.toLowerCase().includes(q); } + function kindLabel(k) { return { training: 'Training policy', architecture: 'Architecture', refinement: 'Refinement', composition: 'Component' }[k] || k; } + function fmtSci(v) { if (v == null) return '—'; if (v >= 0.01) return v.toFixed(3); const e = Math.floor(Math.log10(v)); return (v / Math.pow(10, e)).toFixed(2) + 'e' + e; } + + // ---- Vertical list ------------------------------------------------------- + function renderList(container, graph, { selected = null, onSelect = () => {}, kinds = null, query = '', barLabel = 'Accuracy', unit = 'ms' } = {}) { + container.textContent = ''; + const laneW = 16, pad = 10, gw = pad * 2 + laneW * Math.max(1, graph.laneCount - 1) + 8, rowH = 30; + container.style.setProperty('--graphw', gw + 'px'); + const ul = h('ul', 'tree-list'); ul.setAttribute('role', 'listbox'); ul.setAttribute('aria-label', 'Models in checkpoint order'); + const active = t => graph.families.filter(f => f.start < t && t <= f.end); + const laneX = l => pad + 4 + l * laneW; + const visible = m => (!kinds || kinds.has(m.kind)) && matches(m, query); + const rows = new Map(); let shown = 0; + for (const s of graph.sections) { + const list = graph.nodes.slice(s.start, s.end + 1).filter(visible); + if (!list.length) continue; + const li = h('li'); const head = h('div', 'tsect'); + const hs = el('svg', { width: gw, height: 20, 'aria-hidden': 'true' }); + for (const f of active(s.start)) hs.appendChild(el('line', { x1: laneX(f.lane), y1: 0, x2: laneX(f.lane), y2: 20, class: 'lanek lk-' + graph.byId.get(f.children[0]).kind })); + if (graph.seqIn.has(graph.nodes[s.start].id) || graph.seqSpan[s.start]) hs.appendChild(el('line', { x1: laneX(0), y1: 0, x2: laneX(0), y2: 20, class: 'lane0' })); + head.appendChild(hs); head.appendChild(h('h3', null, s.label)); + const fam = graph.families.filter(f => graph.byId.get(f.parent).t >= s.start && graph.byId.get(f.parent).t <= s.end); + head.appendChild(h('span', 'sum', list.length + (list.length === s.count ? '' : ' of ' + s.count) + (fam.length ? ' · ' + fam.map(f => f.children.length + ' from ' + (graph.byId.get(f.parent).ck || graph.byId.get(f.parent).short)).join(', ') : ''))); + li.appendChild(head); ul.appendChild(li); + for (const m of list) { + shown++; + const row = h('li'); const b = h('button', 'trow' + (m.planned ? ' planned' : '')); b.type = 'button'; b.setAttribute('role', 'option'); b.setAttribute('aria-selected', 'false'); b.dataset.id = m.id; b.id = 'tree-' + m.id; + const svg = el('svg', { width: gw, height: rowH, 'aria-hidden': 'true' }); + const x0 = laneX(0), mid = rowH / 2; + if (graph.seqSpan[m.t]) svg.appendChild(el('line', { x1: x0, y1: 0, x2: x0, y2: rowH, class: 'lane0' })); + if (graph.seqIn.has(m.id)) svg.appendChild(el('line', { x1: x0, y1: 0, x2: x0, y2: mid, class: 'lane0' })); + if (graph.seqOut.has(m.id)) svg.appendChild(el('line', { x1: x0, y1: mid, x2: x0, y2: rowH, class: 'lane0' })); + for (const f of active(m.t)) { + const k = 'lanek lk-' + graph.byId.get(f.children[0]).kind, x = laneX(f.lane), isChild = f.children.includes(m.id); + svg.appendChild(el('line', { x1: x, y1: 0, x2: x, y2: isChild ? mid : rowH, class: k })); + if (isChild && m.t < f.end) svg.appendChild(el('line', { x1: x, y1: mid, x2: x, y2: rowH, class: k })); + } + for (const f of graph.families.filter(f => f.parent === m.id)) { const x = laneX(f.lane), k = 'lanek lk-' + graph.byId.get(f.children[0]).kind; svg.appendChild(el('path', { d: `M${x0},${mid} C${x0},${rowH} ${x},${mid} ${x},${rowH}`, class: k })); } + const dot = el('circle', { cx: laneX(m.lane), cy: mid, r: 5, class: 'dot ' + m.kind + (m.planned ? ' planned' : '') }); + svg.appendChild(dot); b.appendChild(svg); + b.appendChild(h('span', 'ck', m.planned ? 'new' : (m.ck || '—'))); + b.appendChild(h('span', 'nm', m.short)); + b.appendChild(h('span', 'pu', m.planned ? (m.setup && m.setup.change || 'not configured') : (m.purpose || ''))); + b.appendChild(h('span', 'chip ' + m.kind, m.planned ? (m.setup && m.setup.prepared ? 'prepared' : 'planned') : kindLabel(m.kind))); + const bar = h('span', 'mbar'); const fill = h('i'); fill.style.width = Math.round(metricFill(m.worstP) * 100) + '%'; bar.appendChild(fill); bar.title = m.worstP == null ? 'No benchmark' : barLabel + ' ' + fmtSci(m.worstP); b.appendChild(bar); + b.appendChild(h('span', 'ms', m.ms == null ? '' : m.ms.toFixed(2) + ' ' + unit)); + b.addEventListener('click', () => onSelect(m.id)); + row.appendChild(b); ul.appendChild(row); rows.set(m.id, { b, dot }); + } + } + ul.addEventListener('keydown', e => { if (e.key !== 'ArrowDown' && e.key !== 'ArrowUp') return; const opts = [...ul.querySelectorAll('.trow')]; const i = opts.indexOf(document.activeElement); if (i < 0) return; const n = opts[i + (e.key === 'ArrowDown' ? 1 : -1)]; if (n) { n.focus(); e.preventDefault(); } }); + container.appendChild(ul); + if (!shown) container.appendChild(h('p', 'empty', 'No models match.')); + let current = null; const walk = walker(); const order = [...rows.keys()]; + const api = { + shown, + select(id) { const rel = id ? related(graph, id) : new Set(); for (const [mid, r] of rows) { r.b.setAttribute('aria-selected', String(mid === id)); r.b.classList.toggle('related', rel.has(mid)); r.dot.classList.toggle('sel', mid === id); } walk(order, current, id, (mid, on) => { const r = rows.get(mid); if (r) r.b.classList.toggle('walk', on); }); current = id; }, + scrollTo(id, smooth) { const r = rows.get(id); if (!r) return; r.b.scrollIntoView({ block: 'center', behavior: smooth && !reduced() ? 'smooth' : 'auto' }); } + }; + api.select(selected); return api; + } + + // ---- Horizontal lane map, with an optional accuracy track ----------------- + function renderMap(container, graph, { selected = null, onSelect = () => {}, kinds = null, query = '', zoom = 1, track = false, metric = 'worstP', trackLabel = 'Accuracy, lower is better', unit = 'ms', timeId = 'ms' } = {}) { + container.textContent = ''; + const dx = 30, left = 70, top = 44, laneH = 52, plotH = 120; + const W = left + graph.nodes.length * dx + 60; + const lanesH = top + 20 + graph.laneCount * laneH; + const trackTop = lanesH + 40, H = track ? trackTop + plotH + 34 : lanesH + 30; + const has = m => m[metric] != null && m[metric] > 0; + const ty = v => trackTop + (0 - Math.log10(Math.min(Math.max(v, 1e-4), 1))) / 4 * plotH; + const x = m => left + m.t * dx, y = m => top + 20 + m.lane * laneH; + const inner = h('div', 'inner'); inner.style.width = W + 'px'; inner.style.height = H + 'px'; inner.style.transform = 'scale(' + zoom + ')'; + const svg = el('svg', { width: W, height: H, viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': 'Model lineage: checkpoints left to right, branches in lanes' + (track ? ', with ' + trackLabel + ' beneath' : '') }); + const visible = m => (!kinds || kinds.has(m.kind)) && matches(m, query); + for (const s of graph.sections) { + const x1 = left + s.start * dx - dx / 2; + svg.appendChild(el('line', { x1, y1: 8, x2: x1, y2: H - 8, class: 'band' })); + svg.appendChild(el('text', { x: x1 + 8, y: 22, class: 'bandlabel' }, s.label)); + if (track) { svg.appendChild(el('text', { x: x1 + 8, y: trackTop - 14, class: 'ttitle' }, trackLabel)); for (const v of [1, 0.01, 0.0001]) svg.appendChild(el('text', { x: x1 + 8, y: ty(v) - 3, class: 'tlabel' }, v >= 0.01 ? String(v) : '1e' + Math.log10(v))); } + } + for (const e of graph.edges) { if (e.kind !== 'sequence') continue; const a = graph.byId.get(e.from), b = graph.byId.get(e.to); svg.appendChild(el('line', { x1: x(a), y1: y(a), x2: x(b), y2: y(b), class: 'seq' })); } + for (const f of graph.families) { + const p = graph.byId.get(f.parent), last = graph.byId.get(f.children[f.children.length - 1]), ly = y(graph.byId.get(f.children[0])), k = graph.byId.get(f.children[0]).kind; + const x0 = x(p), xb = x0 + dx * 1.6; + svg.appendChild(el('path', { d: `M${x0},${y(p)} C${x0 + dx * 0.6},${y(p)} ${xb - dx * 0.6},${ly} ${xb},${ly} L${x(last)},${ly}`, class: 'der lk-' + k, 'data-family': f.parent })); + svg.appendChild(el('text', { x: xb + 4, y: ly - 7, class: 'lanelabel' }, 'from ' + (p.ck || p.short))); + } + const gCmp = el('g', { class: 'cmp-layer' }); svg.appendChild(gCmp); + const trackPts = new Map(); + if (track) { + for (const v of [1, 0.1, 0.01, 0.001, 0.0001]) svg.appendChild(el('line', { x1: left - 16, y1: ty(v), x2: W - 20, y2: ty(v), class: 'tgrid' })); + let d = '', prevT = null; + for (const m of graph.nodes) { if (m.lane !== 0 || !has(m) || m.n == null) continue; d += (prevT !== null && m.t - prevT <= 3 ? 'L' : 'M') + x(m) + ',' + ty(m[metric]) + ' '; prevT = m.t; } + if (d) svg.appendChild(el('path', { d: d.trim(), class: 'tline' })); + for (const m of graph.nodes) { + if (!has(m)) continue; + const c = el('circle', { cx: x(m), cy: ty(m[metric]), r: 3.5, class: 'tpt ' + m.kind + (visible(m) ? '' : ' dim'), tabindex: 0, role: 'button', 'aria-label': (m.ck || m.short) + ', ' + fmtSci(m[metric]) }); + const t = el('title'); t.textContent = (m.ck ? m.ck + ' · ' : '') + m.short + ' · ' + fmtSci(m[metric]); c.appendChild(t); + c.addEventListener('click', () => onSelect(m.id)); c.addEventListener('keydown', ev => { if (ev.key === 'Enter' || ev.key === ' ') { ev.preventDefault(); onSelect(m.id); } }); + svg.appendChild(c); trackPts.set(m.id, c); + } + } + const host = container.parentElement; host.style.position = 'relative'; + host.querySelectorAll('.minimap, .tip').forEach(n => n.remove()); + const tip = h('div', 'tip'); tip.hidden = true; host.appendChild(tip); + const showTip = (m, ev) => { + tip.textContent = ''; tip.appendChild(h('b', null, (m.ck ? m.ck + ' · ' : '') + m.short)); + const rows = m.planned ? [[m.setup && m.setup.prepared ? 'prepared' : 'planned', 'from ' + (graph.byId.get(m.parent).ck || graph.byId.get(m.parent).short)]] : [[has(m) ? fmtSci(m[metric]) : '—', trackLabel.split(',')[0]], [m[timeId] == null ? '—' : m[timeId].toFixed(2) + ' ' + unit, 'per call'], [kindLabel(m.kind), m.parent ? 'from ' + (graph.byId.get(m.parent).ck || graph.byId.get(m.parent).short) : '']]; + for (const [v, l] of rows) { const d = h('div'); d.appendChild(h('span', 'v', v)); d.appendChild(document.createTextNode(' ' + l)); tip.appendChild(d); } + const r = host.getBoundingClientRect(); tip.hidden = false; tip.style.left = Math.min(ev.clientX - r.left + 14, host.clientWidth - 240) + 'px'; tip.style.top = Math.min(ev.clientY - r.top + 12, host.clientHeight - 90) + 'px'; + }; + const nodeEls = new Map(); + for (const m of graph.nodes) { + const g = el('g', { class: 'node ' + m.kind + (m.planned ? ' planned' : '') + (visible(m) ? '' : ' dim'), tabindex: 0, role: 'button', 'aria-label': (m.ck ? m.ck + ' ' : '') + m.short, 'data-id': m.id }); + g.appendChild(el('rect', { x: x(m) - 14, y: y(m) - 14, width: 28, height: 28, class: 'hitr' })); + g.appendChild(el('circle', { cx: x(m), cy: y(m), r: 5.5 })); + const lbl = el('text', { x: x(m), y: y(m) + 18, class: 'nl' }, m.planned ? 'new' : (m.ck || m.short.slice(0, 10))); if (!(m.ck ? m.n % 2 === 1 : m.t % 2 === 1)) lbl.classList.add('off'); g.appendChild(lbl); + g.addEventListener('click', () => onSelect(m.id)); + g.addEventListener('keydown', ev => { if (ev.key === 'Enter' || ev.key === ' ') { ev.preventDefault(); onSelect(m.id); } }); + g.addEventListener('pointerenter', ev => showTip(m, ev)); g.addEventListener('pointermove', ev => showTip(m, ev)); g.addEventListener('pointerleave', () => { tip.hidden = true; }); + g.addEventListener('focus', () => { const r = g.getBoundingClientRect(); showTip(m, { clientX: r.left + 14, clientY: r.top + 14 }); }); g.addEventListener('blur', () => { tip.hidden = true; }); + svg.appendChild(g); nodeEls.set(m.id, { g, lbl }); + } + inner.appendChild(svg); container.appendChild(inner); + const mini = h('div', 'minimap'); mini.setAttribute('role', 'img'); mini.setAttribute('aria-label', 'Whole map; click to move'); + const ms = el('svg', { viewBox: `0 0 ${W} ${H}`, preserveAspectRatio: 'none' }); + for (const m of graph.nodes) ms.appendChild(el('circle', { cx: x(m), cy: y(m), r: 7, class: 'd' })); + const vp = el('rect', { class: 'vp', x: 0, y: 0, width: 10, height: 10 }); ms.appendChild(vp); mini.appendChild(ms); host.appendChild(mini); + const sync = () => { vp.setAttribute('x', container.scrollLeft / zoom); vp.setAttribute('y', container.scrollTop / zoom); vp.setAttribute('width', container.clientWidth / zoom); vp.setAttribute('height', container.clientHeight / zoom); }; + container.addEventListener('scroll', sync); sync(); + mini.addEventListener('click', ev => { const r = mini.getBoundingClientRect(); container.scrollTo({ left: ((ev.clientX - r.left) / r.width) * W * zoom - container.clientWidth / 2, top: ((ev.clientY - r.top) / r.height) * H * zoom - container.clientHeight / 2, behavior: reduced() ? 'auto' : 'smooth' }); }); + let current = null; const walk = walker(); const order = graph.nodes.map(m => m.id); + const api = { + select(id) { + const sel = graph.byId.get(id); + gCmp.textContent = ''; + walk(order, current, id, (mid, on) => { const n = nodeEls.get(mid); if (n) n.g.classList.toggle('walk', on); }); current = id; + for (const [mid, n] of nodeEls) { const m = graph.byId.get(mid); const on = mid === id; n.g.classList.toggle('sel', on); n.g.querySelector('circle').setAttribute('r', on ? 7 : 5.5); const fam = sel && (m.parent === id || mid === sel.parent || (sel.parent && m.parent === sel.parent)); n.lbl.classList.toggle('off', !(on || fam || (m.ck ? m.n % 2 === 1 : m.t % 2 === 1))); n.g.classList.toggle('kin', !!fam && !on); } + for (const p of svg.querySelectorAll('.der')) p.classList.toggle('hot', !!sel && (p.dataset.family === id || p.dataset.family === sel.parent)); + for (const [mid, c] of trackPts) c.classList.toggle('sel', mid === id); + if (sel) for (const e of graph.edges) if (e.kind === 'comparison' && (e.from === id || e.to === id)) { + const a = graph.byId.get(e.from), b = graph.byId.get(e.to), other = e.from === id ? b : a; + const path = el('path', { d: `M${x(a)},${y(a)} C${x(a)},${Math.min(y(a), y(b)) - 34} ${x(b)},${Math.min(y(a), y(b)) - 34} ${x(b)},${y(b)}`, class: 'cmp', tabindex: 0, role: 'link', 'aria-label': 'Compared with ' + (other.ck || other.short) }); + const t = el('title'); t.textContent = 'Compared with ' + (other.ck || other.short) + ': ' + (e.excerpt || e.explanation || ''); path.appendChild(t); + const show = ev => { tip.textContent = ''; tip.appendChild(h('b', null, 'Compared with ' + (other.ck || other.short))); tip.appendChild(h('div', null, (e.excerpt || e.explanation || '').slice(0, 160) + ((e.excerpt || '').length > 160 ? '…' : ''))); const r = host.getBoundingClientRect(); tip.hidden = false; tip.style.left = Math.min(ev.clientX - r.left + 14, host.clientWidth - 240) + 'px'; tip.style.top = Math.min(ev.clientY - r.top + 12, host.clientHeight - 90) + 'px'; }; + path.addEventListener('pointerenter', show); path.addEventListener('pointermove', show); path.addEventListener('pointerleave', () => { tip.hidden = true; }); + path.addEventListener('click', () => onSelect(other.id)); path.addEventListener('keydown', ev => { if (ev.key === 'Enter') onSelect(other.id); }); + gCmp.appendChild(path); + } + }, + scrollTo(id, smooth) { const m = graph.byId.get(id); if (!m) return; container.scrollTo({ left: x(m) * zoom - container.clientWidth / 2, top: Math.max(0, y(m) * zoom - container.clientHeight / 2), behavior: smooth && !reduced() ? 'smooth' : 'auto' }); } + }; + api.select(selected); return api; + } + return { build, renderList, renderMap, related, comparisons, kindLabel, fmtSci, metricFill, matches }; +})(); diff --git a/docs/plans/training-and-curation/comparison.md b/docs/plans/training-and-curation/comparison.md new file mode 100644 index 00000000..41c4ecd2 --- /dev/null +++ b/docs/plans/training-and-curation/comparison.md @@ -0,0 +1,70 @@ +# Tuldok before and after the training-and-curation plan + +**Before:** the tag `tuldok-before-training-and-curation` (`58226bb5`, 2026-11-19). +**After:** `feat/definition-first-projects` at the commit that adds this file. Nothing is pushed. + +Between them: 7 phases, 50+ commits, 123 files, about 24,000 lines added. Every phase was specified first (`spec-t1.md` … `spec-t7.md`, each with an "As built" section), built by an agent on a branch of its own, reviewed, merged and tested again after the merge. + +## Seeing them side by side + +Both run against your lab project (`projects/surrogate-lab`) and your own Tuldok database: + +| Version | Address | +| --- | --- | +| Before (the tag) | http://127.0.0.1:8093/ | +| After (latest) | http://127.0.0.1:8092/ | + +Each asks for a sign-in code; one code works for both, since they share the same data folder. The newer version adds a few tables to the database, which the older one ignores, so neither disturbs the other. + +**What your lab data shows today.** The lab's earlier runs were made before these contracts existed, so they have no training logs, model graphs, embeddings, annotation sets, sweeps or `serve` declarations. On the lab the new version therefore shows: the **Compare** tab (with kept-field outputs of three earlier pressure models side by side), **parallel coordinates** over the 150 pressure fits on Benchmarks, the two new tabs (**Sweeps**, **Compare**), and empty states that say what each new block needs. The rest is shown on the test fixtures. The screenshots of each check are listed below (git-ignored, on this machine under `build/qa/`). + +## What changed, tab by tab + +| Where | Before | After | +| --- | --- | --- | +| Tabs | Map, Data, Annotate, Train, Benchmarks, Analysis, Overview | adds **Sweeps** and **Compare**; the tab bar scrolls instead of widening the page; on a phone the tabs take their own row | +| Train → a run | progress, standard output and error | **live training curves** (series, several runs overlaid, smoothing, log scale, step or time), **weight and gradient histograms over steps**, a strip of **samples**; all updated in place while the run writes | +| Train → served models | — | **serve a trained model** from its own `serve.py` (`load`, `predict`), on this machine, behind Tuldok's credentials: `POST /api/serve//predict`; start, approve, stop, "Try it"; latency p50/p95/p99 and throughput; a served session can become a benchmark subject | +| Map → a model's drawer | card, runs, weights | adds **Structure**: the model's graph from `model-graph.json` (modules folded into groups, colour by parameters or by change from the starting weights or the parent, compared with another model, a node's tensors linked to Weights, **Open wide** over the page); **Serve** | +| Data → a dataset | records, files, look inside | adds **Map**: the dataset's embeddings in two dimensions, coloured by split or annotation, box and lasso select, **more like this, duplicates, outliers, leakage across splits, thin regions**; a selection becomes a dataset (`select`) or a freeze; an **Embed** form (default embedder **EmbeddingGemma 2** through Pumas) | +| Annotate | decision, class, tags, text, boxes, spans, segments; suggestions | adds **polygons, masks, depth maps, keypoints, ranking, time series, tracks, point-cloud objects, 3D boxes**; annotation **sets** made by an annotator (a project's script or a Pumas model) via the step `annotate`; **Review**: two sets side by side or overlaid, **agreement** (kappa, IoU, OKS, Kendall's τ, track switches), the most disputed records, Keep A / Keep B / both / neither, light corrections | +| Sweeps (new) | — | **hyperparameter search** on this machine: a `sweeps/.json` (grid, random, halving with early stopping), one approval for the whole budget, trials as derived recipes on the Map, live state, the trials' frontier and parallel coordinates | +| Benchmarks | charts, table, compare set of 3 | adds **parallel coordinates** (metrics and varying recipe settings, brushing linked to the table and charts), **bootstrap 95 % intervals** where per-case values exist (whiskers, and frontier points marked when not told apart), a compare set of 4; opens on a benchmark with an accuracy-against-time chart | +| Compare (new) | — | 2 to 4 models: what differs, metrics with intervals, place on the frontier, and their **outputs side by side** on a shared case (native, predicted, signed error, one slice control for all) | + +**For project scripts:** a small helper package, `tuldok_log`, writes these files in a few lines: `log`, `log_histograms`, `log_sample`, `autolog(model, optimizer)` for PyTorch, and `export_graph(model, example_inputs, path)`. Projects may write the files themselves instead; the formats are in `docs/contracts/project-folder.md`. + +## Tests on the final code + +| Suite | Before (at the tag) | After | +| --- | --- | --- | +| Project suite (`test_tuldok_project_*`) | 735 | **898**, all pass | +| Agent interface (catalog digest unchanged) | 43, 4 skipped | 43, 4 skipped | +| Workspace browser checks | 8 files | **15 files**, all pass (actions 27, annotate 11, annotation kinds 8, workspace 24, compare 6, curation 7, endpoint 8, fields 8, inside 8, serving 6, still 6, structure 8, sweeps 7, training 7, weights 7) | +| `tuldok_log` with PyTorch 2.8 on the CPU (the lab's environment, read only) | — | 13 pass | + +**On your real lab data, read only:** the new reads answer on the 206-model project (map 0.09 s, compare of three models 0.04 s, parallel coordinates over 150 fits 0.05 s). One was slow and was fixed: listing servable models took 13 s and now takes 0.09 s. + +## Screenshots of each new part (from the browser checks) + +| Part | Directory under `build/qa/` | +| --- | --- | +| Training curves, histograms, samples, live | `project-workspace-training/` | +| Structure, folded, compared, wide | `project-workspace-structure/` (`light-1440-3c-wide.png`) | +| Compare tab, outputs, parallel coordinates, intervals | `project-workspace-compare/` | +| Annotation types, review, agreement | `project-workspace-annotation-kinds/` | +| Embedding map, curation lists, Embed form | `project-workspace-curation/` | +| Serving | `project-workspace-serving/` | +| Sweeps | `project-workspace-sweeps/` | + +## Not done, and not verified + +- Nothing was trained on the GPU, nothing was benchmarked, and no running Pumas was contacted. EmbeddingGemma 2, Pumas annotators and the Pumas endpoint were tested against a fake. +- Your lab has none of the new inputs yet (training logs, graphs, embeddings, annotation sets, sweeps, `serve`). A converter of the lab's earlier `history.json` into training logs was deferred. +- Known limits are in each spec's "As built". The larger ones: + - Curation searches finish over every record only up to a few thousand records on dense data; the page says how many were examined. + - A sweep's objectives are training metrics, not benchmark metrics. + - Serving uses the code at HEAD, not at the product's commit. + - Mask and depth agreement is computed pixel by pixel in Python. + - At phone width the closed drawer still sits 9 px past the edge. +- Pumas cannot serve these models yet. It serves only models in its library, and only for fixed task types. A Pumas backend for serving is left as a seam (`serve.backend`). diff --git a/docs/plans/training-and-curation/plan.md b/docs/plans/training-and-curation/plan.md new file mode 100644 index 00000000..be981154 --- /dev/null +++ b/docs/plans/training-and-curation/plan.md @@ -0,0 +1,58 @@ +# Plan: training views, model structure, generated annotation, curation, comparison, serving and search + +**Status:** built 2026-11-19, phases 1 to 7 (see [comparison.md](comparison.md)); approved 2026-11-19 ("I agree"), with the four decisions as recommended. The baseline before this plan is the tag `tuldok-before-training-and-curation`. Embeddings through the Pumas endpoint default to EmbeddingGemma 2 (owner, 2026-11-19: "Set embeddinggemma 2 as the default embbedder for Pumas"). + +**Owner, 2026-11-19:** "Tuldok needs live training views and model structure viwer. … Tuldok also needs a strong set of annotation types … Tuldok centers on generated annotations rather than human made ones … automated/ai segmentors, polygons, depth mapping, image masks, ranking, time series, object tracking, point cloud objects (selecting points that belong to an object), some quality control tools like agreement betwene different models … We should also have serving trained models as a REST endpoint and autologging. … it would be great if it could compare selected models deeper like also showing output results next to eachother. … Tuldok should have curarion by embeddings. … Parallel coordinates sound very usful for Tuldok. Hyperparameter search could be usful … on tiny models localy. We are not going to add a model zoo now." + +And on how Tuldok is built: "I actualy think Tuldoks expectation of projects being made with use created python scripts to be a strength … Tuldok solves larger architectural and the more difficult aspects of a complete end to end model/data workflow." + +## The rule every phase follows + +Tuldok does not grow a generator, a training framework or an annotation model of its own. Each capability is three parts: + +1. **A file contract,** written in `docs/contracts/project-folder.md`: what a project's own script writes into its run's product (a metrics log, a graph, annotations, embeddings), as plain JSON, JSONL, `.npy` or PNG. +2. **A reader and a view in Tuldok:** standard library only, nothing unpickled, nothing of the project's code run inside Tuldok's server, bounded reads, as in the data tools. +3. **A thin optional helper,** `tuldok_log`, a small Python package a project's script may import to write those files in a few lines. It depends on the standard library, and on PyTorch only where it needs a model (graph export, autologging). It lives in the Tuldok repository and is installed into a project's environment like any other dependency; a project may also write the files itself. + +Projects keep writing their own Python (by hand or by an AI). Tuldok supplies the contracts, the orchestration (builds, approval, runs), and the views. + +## Why the structure needs the training code + +The model's structure is defined, as the owner says: it is the Python class that builds the layers and its `forward` method. But that definition is code, and the files Tuldok can read today hold none of it: + +- A weights file (`.npz`, `.safetensors`) is only named arrays: `encoder.0.weight` with its shape and values. It says nothing about which layer feeds which. +- `model.pt` saved with `torch.save(model)` is a pickle that names the class by its import path; loading it runs the project's code. Tuldok never opens a pickle. +- The lab's `model-definition.json` describes the model in the lab's own words (encoder widths 8, 16, 24, 32, bottleneck 48, SiLU, two convolutions a block). That is a good card, but not a graph Tuldok can draw for any project. + +So the graph has to be produced where the code already runs: in the project's environment, by its training script, once, with an example input. PyTorch can do this: `torch.export` (or `torch.fx`) runs the model on an example input and records each operation, its inputs and outputs, their shapes, and the module each came from. `tuldok_log.export_graph(model, example_input)` writes that record as `model-graph.json` next to the weights. Models whose `forward` branches on data values may not export whole; the helper then writes the module tree (layers, nesting, parameters) without the data flow, and says so in the file. + +## Phases + +| # | Phase | Size | Done when | +| --- | --- | --- | --- | +| 1 | **Autologging and live training views.** Contract: `training/metrics.jsonl` (one line per log: step, wall time, any named scalars), `training/histograms.jsonl` (per layer, per N steps: bins of weights and gradients), `training/samples/` (an image or a field every N steps), the checkpoint index. `tuldok_log`: `log(step, **scalars)`, `log_sample(...)`, and `autolog(model, optimizer)` for PyTorch, which records loss, learning rate, gradient norm and histograms without the script naming them. Train tab: a running run's curves drawn live and updated in place, several runs overlaid, smoothing, log scale; histograms over steps per layer; the samples as a strip. A converter for the lab's earlier `history.json` is a lab script, not Tuldok code | Medium | A tiny CPU stand-in run logs through `autolog`; its curves grow on the page while it runs without the page being rebuilt; two finished runs overlay; an earlier lab run's history is shown after the lab's converter runs | +| 2 | **The model structure viewer.** Contract `model-graph.json`: nodes (operation, module path, parameter tensors, input and output shapes, attributes), edges, the module tree. `tuldok_log.export_graph`. In a model's drawer: the graph drawn left to right, modules folded into groups that open, a node's tensors linked to the Weights block; overlays coloured on the graph (parameter count, change against the parent or the starting weights, time per operation when profiled); two models' graphs side by side with matching modules lined up | Medium | A fixture model with nested modules is exported in a test environment with PyTorch and drawn; folding, the weights link and the change overlay work; two variants of one architecture are compared | +| 3 | **Comparing chosen models deeply.** The Map's compare set (two to four models) opens a Compare view: one column per model, rows of what differs (recipe settings, data, parent, kind of change), metrics with confidence intervals, where each sits on the frontier; then the outputs: one case chosen, every model's prediction in a row with the native field and each one's signed error, the same slice and axis for all, moved together. Benchmarks gains parallel coordinates (recipe settings and metrics as axes; a range dragged on an axis keeps the runs inside it) and bootstrap confidence intervals from per-case values, with frontier points marked when their difference is not significant | Medium | On the lab folder three earlier pressure models with kept fields are compared on one case; parallel coordinates over the 150 fits filters to the best errors and shows their settings | +| 4 | **Generated annotation, more kinds, and quality control.** New annotation kinds: polygons, image masks (PNG or run-length), depth maps (`.npy` per image), keypoints, ranking, time-series segments, object tracks (an object's identity across frames or time), point-cloud objects (point indices or per-point instance ids), 3D boxes. Every annotation names what made it (model, version, settings, or a person). An **annotator step**: a project's script, or a model behind the Pumas endpoint, writes annotations for a dataset; people review instead of drawing. Quality control: agreement between annotators, human or model (IoU for masks, polygons, boxes and point sets; rank correlation for rankings; agreement scores for classes), a list of the records where they disagree most, filters by confidence; two annotators shown side by side | Large | On the tiny example, two stand-in "segmentors" annotate the same images and point clouds; their masks and point objects are drawn as overlays; agreement is computed against the test's own arithmetic; the most-disputed records are listed and one is reviewed and kept | +| 5 | **Curation by embeddings.** Contract: an embeddings product (a vector per record, by record id, `.npy` plus an index), made by a project's script or by a model behind the Pumas endpoint (0.8.0 declares `embeddings_float32`). Data tab: a map of the dataset in two dimensions (projection written by the script, or a principal-component projection Tuldok computes for small sets), coloured by any field or annotation; near-duplicates; "more like this"; outliers; thin regions; near-copies across the training and validation splits (leakage). A selection becomes a dataset definition (a subset, or Freeze from it) | Medium | On a fixture with planted duplicates and one leaked validation record, both are found; a selection on the map is frozen into a dataset whose records are exactly those chosen | +| 6 | **Serving a trained model.** A recipe may declare `serve`: the project's own entry point (a script with a `predict` function). Train tab starts and stops it as a local REST endpoint behind Tuldok's credentials, loopback only, on the owner's approval like a build; each request's time is logged, so "the model as served" can be a benchmark subject | Medium | A stand-in model is served, answers a request through Tuldok's endpoint with a credential and refuses one without; its timings appear as a benchmark subject | +| 7 | **Hyperparameter search on this machine.** A `sweep` definition: the recipe it varies, parameters with ranges, a strategy (grid, random, successive halving that stops poor runs early), one or two objectives, a budget (runs, minutes, one at a time). One approval covers the whole budget. Each trial is a recipe on the Map, grouped under the sweep; two objectives give the frontier; parallel coordinates is its main view | Medium | A sweep of a tiny CPU stand-in runs within its budget, stops poor trials early, and its trials appear on the Map, the frontier and parallel coordinates | + +**Order.** Phases 1 and 2 first: they are what the owner named first, and both can be checked now against existing lab runs and tiny stand-ins. Phase 3 next, because it reuses kept fields and serves the two targets directly. Then 4 and 5 (data quality), then 6 and 7. + +**Compute.** Nothing in these phases trains a real model or benchmarks one. Tests use stand-in scripts that finish in seconds on the CPU. Phase 2's graph export needs PyTorch in a test environment (CPU only). Any GPU use waits for the owner's word, as before. + +## Out of scope + +- A model zoo. Later, perhaps, a community hub where people share models and datasets made with Tuldok. +- Distributed or cloud training. +- A built-in data-generation language; projects write scripts. +- A manual annotation editor of Label Studio's breadth: people review generated annotations and correct them, but Tuldok does not aim to be the best drawing tool. +- Video as a record kind. Object tracks work over frames stored as images, or over time in a sequence. + +## Decisions needed + +1. **This plan and its order.** Recommended as written. +2. **Where embeddings come from.** Recommended: both a project's script and the Pumas endpoint. +3. **The `tuldok_log` helper as a package in the Tuldok repository** that projects install. Recommended: yes, kept small; projects may ignore it and write the files themselves. +4. **Serving reachable only from this machine at first.** Recommended: yes; a remote address is a later choice. diff --git a/docs/plans/training-and-curation/spec-t1.md b/docs/plans/training-and-curation/spec-t1.md new file mode 100644 index 00000000..7da8c0fb --- /dev/null +++ b/docs/plans/training-and-curation/spec-t1.md @@ -0,0 +1,290 @@ +# Implementation spec: autologging and live training views + +This is phase 1 of [the training and curation plan](plan.md), and it follows "The rule every phase follows": a file contract, a reader and a view in Tuldok, and a thin optional helper, `tuldok_log`. The rules of the data tools apply to the reader ([spec-d1.md](../definition-first-projects/spec-d1.md), [spec-d2.md](../definition-first-projects/spec-d2.md), [spec-d6.md](../definition-first-projects/spec-d6.md)): standard library only, nothing unpickled, nothing of the project's code run inside Tuldok's server, nothing written, bounded reads, a damaged line or file refused or reported and never a failure, reads outside the service lock. The page follows [spec-w4.md](../definition-first-projects/spec-w4.md): it updates in place, and nothing is rebuilt by a timer or a return to the window. + +**Owner, 2026-11-19:** "Tuldok needs live training views … We should also have … autologging." And on how Tuldok is built: projects write their own Python scripts; Tuldok supplies the contracts, the orchestration and the views. + +**After this phase** a project's training script writes a few plain files into its run's output directory while it trains, by hand or through `tuldok_log` (three lines with `autolog`). On the Train tab, the drawer of a run shows its curves while it runs, growing in place every two seconds, with other runs overlaid, smoothing and a log scale; the distributions of each layer's weights and gradients over the steps as small heat strips; and the samples the script kept, as a strip. Nothing trains a real model; nothing reads the lab's runs. + +## 1. Rules + +- **The script writes, Tuldok reads.** Tuldok never runs the project's code, never imports it, never opens a pickle. `tuldok_log` is imported by the project's script in the project's environment, never by Tuldok's server. +- **A running run is readable.** The training files are read from a run's output directory whatever its state, as soon as the run has a directory: queued with none yet answers "nothing logged"; running, succeeded, adopted, failed, cancelled or timed out are read alike (a failed run's curves are often what one most wants to see). The other reads inside a product (spec-d1) still need a finished run. +- **Files are appended while they are read.** A reader takes only complete lines (ending in a newline); a partial last line is left for the next read. A file that became shorter, or another file under the same name, is read again from its start. +- **Bounded.** Every read has a bound on bytes, lines, series and points (section 4), and long series are reduced on the server before they are sent. +- **Cheap when nothing changed.** What has been read of a file is kept in memory with its offset, so reading again reads only what was appended since. + +## 2. The contract: `training/` in a run's output directory + +Everything is under `training/` at the top of the run's output (`{output}/training/`). Every file is optional; a run with no `training/` has nothing logged. JSON is UTF-8, one object per line (JSON Lines), and Python's `NaN`, `Infinity` and `-Infinity` are read as non-finite. + +### 2.1 `training/metrics.jsonl` + +One object per log: + +```json +{"step": 120, "time": 14.2, "epoch": 3, "loss": 0.0213, "lr": 0.001, "grad_norm": 0.84, "val/loss": 0.031} +``` + +- `step`: required, an integer ≥ 0. Steps normally increase; they need not (a resumed run may log a step again), and the page draws points in the order of the file. +- `time`: required, seconds since the run started (a finite number ≥ 0). +- `epoch`: optional, a number. +- **Every other key whose value is a number is a scalar series** named by the key (1 to 200 characters). `null` or a non-finite number is a gap in that series, counted as `nonfinite`. A key absent from a line is no point of that series (a validation loss logged every hundred steps is a series of its own with fewer points). +- Keys whose value is not a number (a string, an object, a boolean) are kept as the line has them and listed by name as `other_keys`; they are not series. +- A line that is not such an object (not JSON, no integer `step`, no `time`) is counted and its line number reported; the rest are read. + +### 2.2 `training/histograms.jsonl` + +One object per tensor per log: + +```json +{"step": 100, "time": 12.1, "name": "encoder.0.weight", "kind": "weights", "edges": [-0.5, -0.4, …, 0.5], "counts": [3, 11, …, 2]} +``` + +- `step` and `time` as above; `name` (1 to 1,000 characters): the tensor; `kind`: `weights` or `gradients`. +- `edges`: n + 1 finite numbers, not decreasing; `counts`: n numbers ≥ 0 (counts of values in each bin; the last bin includes its upper edge). 1 ≤ n ≤ 1,024. +- Optional `nonfinite`: values left out because they were not finite. + +### 2.3 `training/samples/` and `training/samples.jsonl` + +What the model made every N steps, so one can watch it learn: an image (PNG) or an array (`.npy`, the forms spec-d1 reads). Each file is written whole under `training/samples/` (written to a temporary name and renamed, by `tuldok_log`), named by its step and name: `samples/000000120-prediction.png`. Then one line is appended to `training/samples.jsonl`: + +```json +{"step": 120, "time": 14.2, "name": "prediction", "path": "samples/000000120-prediction.png"} +``` + +`path` is relative to `training/`, a plain address under `samples/` (no `..`, no `!`). The kind follows the extension: `.png` (also `.jpg`, `.jpeg`, `.webp`) is an image; `.npy` is an array. A line naming a file that is missing is listed with that problem. + +### 2.4 `training/checkpoints.jsonl` and how the checkpoint index relates + +The project-folder format has no checkpoint index today: a run's final weights are its declared outputs (role `weights`, or found by name, spec-d2), and the earlier CLI's `checkpoint` role names one output. A script that keeps checkpoints during the run may list them here, one line each: + +```json +{"step": 1000, "time": 120.5, "path": "checkpoints/step-1000.safetensors", "loss": 0.012} +``` + +`path` is relative to the run's output directory. The other numeric keys are figures of that checkpoint. Tuldok lists them and marks their steps on the curves; it does not open them here (the Weights block of the model's drawer reads the run's final weights as before). + +### 2.5 `training/training.json` + +Optional, written once by `tuldok_log` at the start: `{"format": "tuldok_training_v1", "started_at": "", "helper": "tuldok_log 1"}`. A reader shows `started_at`; nothing depends on it. + +### 2.6 The earlier lab runs + +The lab's earlier runs keep their history in `history.json`, in the lab's own form. Converting it into `training/metrics.jsonl` is a **lab script**, not Tuldok code, and it is deferred: this phase writes nothing into the lab repository or into any run's product. Until it exists, those runs show "nothing logged". + +## 3. `tuldok_log` + +A package at the top of the Tuldok repository, `tuldok_log/`, installed into a project's environment like any dependency (or put on its path). Standard library only; PyTorch is imported only inside `autolog`, and the helpers that take a model or a tensor use the object's own methods (`named_parameters`, `detach`, `histc`) without importing anything. + +```python +from tuldok_log import Logger, autolog + +log = Logger(output) # writes into /training/ +log.log(step, loss=0.21, lr=1e-3) # one metrics line; epoch=... optional +log.log_histograms(step, model) # weights and gradients of every parameter; or tensors={"name": values} +log.log_sample(step, "prediction", png_bytes_or_array) +log.checkpoint(step, "checkpoints/step-1000.safetensors", loss=0.012) + +track = autolog(model, optimizer, directory=output, every=10, loss=criterion) +``` + +- `Logger(directory)`: the run's output directory; creates `training/` and writes `training.json` when it is absent. Its clock starts when it is made. Every line is written with one `write` call on a file opened for appending, so a reader sees whole lines or none (a partial line at worst while the write lands). `close()` and a context manager. +- `log(step, *, epoch=None, **scalars)`: values are converted with `float()`; a value that is not finite is written as `null`. +- `log_histograms(step, model=None, *, tensors=None, kinds=('weights', 'gradients'), bins=32, most=64)`: for a model, every parameter (the first `most`) and its gradient when it has one; for `tensors`, a mapping of name to values (a nested list, an object with `tolist()`, or a tensor). Histograms are computed by the tensor's own `histc` when it has one, else in Python. +- `log_sample(step, name, value)`: `bytes` beginning with the PNG signature are written as a `.png`; an object with `__array_interface__` or `dtype`/`shape`/`tobytes` (a NumPy array), a tensor (`detach().cpu()`), or a nested list of numbers is written as an `.npy` of its numeric type (float32 for a list). Written to a temporary name, renamed, then indexed. +- `autolog(model, optimizer, *, directory=None, logger=None, every=50, histograms_every=None, loss=None, bins=32)`: registers the optimizer's step hooks (PyTorch 2.0 or later). Each optimizer step counts one step. Every `every` steps it writes one metrics line with the learning rate (`lr`, or `lr/` for several parameter groups), the gradient norm over all parameters before the step (`grad_norm`) and, when it knows it, the loss: the mean of the losses seen since the last line. It knows the loss when `loss` is the loss module (a forward hook reads its output) or when the script calls `track.note(loss=…)` (any named scalars). Every `histograms_every` steps (default: `every`) it writes the histograms of the weights and of the gradients. `track.close()` removes the hooks. The script names none of these. + +## 4. Operations + +Five reads, `inspect`, in the `project_folder` profile and absent from `catalog` (whose digest is unchanged). Each takes `run`; a run with no directory answers as having nothing logged (`training`) or `not-found` (the others). + +| Operation | Arguments | Result | +| --- | --- | --- | +| `training` | none besides `run` | `run` (`id`, `state`, `active`), `present` (whether `training/` exists), `started_at`, `series` (each: `name`, `points`, `nonfinite`, `first_step`, `last_step`, `last`, `min`, `max`), `lines` (complete metrics lines read), `last_step`, `last_time`, `other_keys`, `histograms` (each: `name`, `kind`, `rows`, `first_step`, `last_step`), `histogram_lines`, `samples` (each name with `count` and `last_step`), `sample_lines`, `checkpoints` (the last 200: `step`, `time`, `path`, figures), `problems` (the first 20: `file`, `line`, `problem`), `bad_lines`, `truncated` | +| `training_series` | `series`: up to 16 of `{name, size, since}`; `most` (default 1,000, at most 4,000) | `series`: each `name`, `points`, `nonfinite`, `size`, `first`, `buckets`; `lines` | +| `training_histograms` | `name`, `kind`, `most` (rows, default 120, at most 500), `bins` (default 48, at most 128) | `name`, `kind`, `rows` (each `step`, `time`), `stride`, `low`, `high`, `bins`, `counts` (row by row, re-binned), `nonfinite` | +| `training_samples` | `name`, `offset`, `limit` (at most 200) | `samples` (each `index`, `step`, `time`, `name`, `path`, `kind`, `bytes`, `problem`; an image's `url`), `total` (of that name, or all) | +| `training_sample` | `index` | `sample` (as above) and `content`: for an image `width`, `height`, `url`; for an array a slice as `array_slice` answers it (the last two axes, at most 128 by 128) | + +**Reduced series.** A series of `n` points is sent in buckets of `size` consecutive points, `size` the smallest power of two for which there are at most `most` buckets (1 while `n ≤ most`). A bucket is `[first_step, last_step, last_time, mean, min, max]` over its finite values (`mean`, `min`, `max` null when it has none). The bucket boundaries depend only on `size`, so the page keeps what it has: it sends the `size` it holds and `since`, the bucket it wants from (its count less one, since the last may have grown); when the server's `size` is the same it answers from `since` (`first`), otherwise from 0, and the page replaces what it held. A live run thus sends a few buckets a reading. + +**Histograms over steps.** The rows are the tensor's logs, evenly chosen when there are more than `most` (the first and the last always among them). Every row is re-binned onto `bins` equal bins over the lowest and highest edge of the chosen rows, a bin's count shared out by overlap (as if spread evenly inside it); `counts` are the shares, not rounded. + +**Images of a running run.** `GET /api/project/file` serves a file under `training/` of any run that has a directory, as it serves a finished product's files (same checks: inside the output directory, no `..`). + +## 5. Bounds + +| | Bound | Beyond it | +| --- | --- | --- | +| A line of `metrics.jsonl`, `samples.jsonl`, `checkpoints.jsonl` | 64 KiB | counted as a bad line | +| A line of `histograms.jsonl` | 256 KiB | counted as a bad line | +| Lines of `metrics.jsonl` kept | 1,000,000 | later lines not read, `truncated` | +| Series | 200 | later names not kept, reported once | +| Lines of `histograms.jsonl` indexed | 1,000,000 | `truncated` | +| Tensor and kind pairs | 2,000 | later pairs not kept, reported once | +| Bins of one histogram line | 1,024 | a bad line | +| Lines of `samples.jsonl` and of `checkpoints.jsonl` | 100,000 and 10,000 | `truncated` | +| Files kept in memory | 16, least recently used dropped | read again from the start when asked again | + +A histogram line is kept as its offset and length only; its bins are read again when a strip is drawn. + +## 6. The page + +In the run's drawer on the Train tab, a block **Training**, after "Latest progress", when the run has a directory. A run with nothing logged shows one line that says so, and how a script logs (`tuldok_log`), and nothing else. + +- **Curves.** A chart (canvas 2D, no library) of the chosen series against the step (or the time since the run started). A chooser of series (the run's series, `loss` first when there is one; up to four shown). Controls: **Smoothing** (a slider, 0 to 0.99, exponential moving average over the drawn buckets; the raw line stays faint behind), **Log scale** (values ≤ 0 left out, said under the chart), **Step / Time**. When a series is reduced, the band between each bucket's min and max is shaded. Checkpoints are ticks on the step axis. Hover shows the step, the time and the value of each line at that step. +- **Overlay.** "Compare with" lists the other runs of the project (newest first, those of the same model first); each run added is drawn in its own colour, with a legend of run, model and state; a run is removed from its chip. Up to four runs. +- **Histograms.** Under the curves, for each tensor (weights, and gradients when logged; a switch between the two), a heat strip: steps across, bins down, each column's counts as shares of its own total so a narrowing distribution shows. The first eight tensors are drawn; "Show all" draws the rest. Each strip is read when it comes on screen. +- **Samples.** A strip of the last twelve samples of the chosen name (a select when there are several), newest last, with the step under each; an array is drawn as an image with `inside.js`'s colour ramp. +- **Live, in place.** While the run is active, every reading of the page (every two seconds, spec-w4 R7) asks `training` once; when its `lines` grew, `training_series` for the shown series from where the page is (appending buckets); when `histogram_lines` grew, the strips on screen are read again; when `sample_lines` grew, the new samples. The canvases are drawn again and new samples appended; **the block is never rebuilt** by a reading, and a reading that found nothing new touches no element. A run's answers are kept by the page while it is open, so the drawer drawn again (the run ended) puts the block back without "Reading…" and without reading the series again. +- **The address** keeps the overlay and the settings: `trWith` (run ids), `trSeries`, `trSmooth`, `trLog`, `trX`. +- Light and dark from the page's tokens; at 390 wide the controls wrap and the chart fills the drawer. + +## 7. Slices + +| | Slice | Files | +| --- | --- | --- | +| T1a | The contract text, the reader, the five operations, the file route | `docs/contracts/project-folder.md`, `tuldok_project/training.py` (new), `operations.py`, `service.py`, `tests/test_tuldok_project_training.py` (new), and the tests that count the project-folder operations | +| T1b | `tuldok_log` | `tuldok_log/__init__.py` (new), `tests/test_tuldok_log.py` (new) | +| T1c | The Training block | `static/project-workspace/training.js` (new), `tabs.js` (the run's drawer), `adapter.js`, `app.js` (the address, the reading), `base.css`, `app.py` and `static/project-workspace.html` (the asset), `tests/browser_project_workspace_training.cjs` (new) | + +## 8. Tests + +**T1a, `tests/test_tuldok_project_training.py`,** files written by the test: + +- `training` on a run with no `training/`; on a metrics file with several series, a gap (`null`, `NaN`), a key that is not a number, a bad line, a line too long; on a file whose last line is partial (not counted), then completed by an append (counted); on a file replaced by a shorter one (read again). +- `training_series` against the test's arithmetic: raw points while `n ≤ most`; buckets of a power of two beyond, with mean, min and max per bucket; `since` with the same size answers only the later buckets; with a different size from 0; the bound on lines (lowered for the test). +- `training_histograms`: two rows with different edges re-binned onto one grid, the totals kept; row choice past `most`, first and last kept; a damaged line counted. +- `training_samples` and `training_sample`: a PNG (its size), an `.npy` (its slice), a missing file, a path with `..`. +- A running run (state `running`) is read; a queued run without directory answers nothing logged; the file route serves `training/` of a running run and not its other files. +- The five are in `project_folder` and absent from `catalog`; the catalog digest is unchanged. + +**T1b, `tests/test_tuldok_log.py`:** `Logger` writes lines the reader reads back (series, gaps as `null`, histograms from lists, samples PNG and `.npy` and the index); every line is whole; `autolog` with PyTorch on the CPU (a tiny model, ten steps, `every=2`): metrics lines with `loss`, `lr`, `grad_norm`; weights and gradients histograms for every parameter; the hooks removed by `close()`. Skipped when PyTorch is not importable; run with the lab's interpreter, which has it (read only, nothing installed). + +**T1c, `tests/browser_project_workspace_training.cjs`,** in a headless browser, the owner signed in, no GPU: a fixture with two finished runs of a small stand-in script that logs through `tuldok_log` (built before the server starts), and a third run of the same script for about ten seconds, asked for and run while the page watches: + +1. The running run's drawer shows Training; its curve grows across readings (more points drawn, the last step larger) while a marker set on the block survives and no node of the block is added or removed by a reading. +2. The two finished runs overlaid: three lines and a legend of three. +3. Smoothing, log scale and Step / Time change the drawing (another image), the block the same element. +4. Histograms drawn (non-blank strips, one per tensor shown) and samples drawn (images loaded, steps under them). +5. Nothing is touched by a return to the window with the finished run open. +6. Light and dark, 1440 and 390 wide: no sideways overflow, no script error, no failed request. + +## 9. Out of scope + +A lab converter of `history.json` (a lab script, deferred); TensorBoard event files or any other format; logging from inside Tuldok's server; text or audio samples; comparing histograms of two runs; alerts on a diverging run; the Analysis or Benchmarks tabs; any change to how runs are started or to the lab's runs. + +## As built + +What differs from the text above, and what was found while building it. + +### T1a, the reader and the operations + +- **`tuldok_project/training.py`** holds the reader; `service.py` gains `TRAINING_READS` beside `INSIDE_READS`. The + run is found under the service's lock (`_training_run`, any state, its `directory`) and its files read outside + it. A run of any state with an output directory is read; one without (a queued run) answers `training` with + `present` false and the other four `not-found`. +- **`training_histograms` also answers `totals`** (each row's count before re-binning), which the page uses to shade + each column by its share. It is in the contract text. +- **Damaged lines.** A line that is not JSON, not an object, has no integer `step` or finite `time` (metrics), or + fails the histogram, sample or checkpoint checks is counted in `bad_lines`; past the bound on series or tensor + pairs the dropped name is reported once in `problems` but not counted as a bad line. A line longer than the bound + is skipped up to its newline even when that arrives in a later read. A number too large for a float + (`1e400` as an integer) is a non-finite value, a gap. +- **What is kept per file:** the metrics as arrays (step, time, epoch, and per series its line positions and + values) with each series' least, greatest and last finite value kept as lines arrive, so `training` costs a + `stat` and the appended bytes, not a pass over the series; a histogram line as its offset and length; samples and + checkpoints as small dicts. 64 file states (four a run) are kept, least recently used dropped. +- **A symlink** under `training/` that leads outside the output directory is treated as absent. +- **`GET /api/project/file`** serves a `training/` address (no `!`) of a run in any state with a directory through + `service.product_file`; `app.py` is unchanged. +- **Tests:** `tests/test_tuldok_project_training.py`, 18 (15 without a database, 3 with); the read test and the + operation count (61 to 66) in `test_tuldok_project_service.py`. The catalog's digest is unchanged + (`ad5cc7ac…`, 110 operations), checked in the new file and by the agent interface tests. + +### T1b, `tuldok_log` + +- As section 3, with `Logger.elapsed()` public and `log_sample` returning the path it wrote. `autolog` logs the + first optimizer step as well as every `every`-th, so a curve starts at step 1. Gradient histograms are written in + the optimizer's pre-step hook (gradients may be zeroed after the step), weight histograms after it; both carry the + step number after the step. +- **The loss module's forward hook counts every call of the module**, so a script that also calls the same loss + module for validation mixes those values into `loss`. The docstring says to pass the module or call `note()`; + a separate validation loss module (or `note(val_loss=…)`) avoids it. Not otherwise guarded. +- `note()` scalars other than the loss are written with the next logged line and then forgotten; a note made on a + step that is not logged waits for the next one. +- **Tests:** `tests/test_tuldok_log.py`, 8: 6 with the standard library (read back through the reader), and 2 of + `autolog` that need PyTorch, skipped in Tuldok's environment. Run with the lab's interpreter + (`/.venv/bin/python -m unittest tests/test_tuldok_log.py`, PyTorch 2.8.0+cpu, `PYTHONDONTWRITEBYTECODE=1`, + read only, nothing installed): 8 OK on the CPU. The grad norm is checked against the test's own norm taken in a + pre-step hook of its own; the loss against the mean of the losses it saw. + +### T1c, the block + +- **Where.** The block is in the run's drawer, after "Latest progress", read with the drawer (`LabTraining.read` + in `runDrawer`'s load) and built with it (`LabTraining.block`). A failure to read it shows one line in the block, + not a failed drawer. `tabs.js`'s `S.tick` calls `LabTraining.tick()` first on every reading. +- **One chart a series** rather than one chart with a series chooser: series of different scales (loss and learning + rate) are never drawn on one axis. Up to four series and four runs. Runs are told apart by colour and a legend + (swatch, model's short name, start of the run id, state), and the hover tip names each run with its value, step + and time; there are no direct labels at the ends of the lines (the drawer is narrow). The four colours are new + tokens `--run-1` … `--run-4` in `base.css`, light and dark, checked with the dataviz palette validator (lightness, + chroma, colour-blind separation, contrast: all pass in both modes). +- **The overlay chooser** lists the project's other runs that are not queued (the run's own model's first), up to + fifty; runs that logged nothing draw no line but stay in the legend. +- **Smoothing on a log scale** smooths the logarithms, so a falling loss is not pulled up by its early values. A + value of 0 or less is left out on the log scale and a line under the chart says so; the same line says when a + series is reduced (each point a mean of N, the band their range) and when it is smoothed. +- **Histograms**: one strip a tensor for the kind chosen (weights or gradients), 32 bins and at most 120 steps, each + column shaded by the square root of its share of its column's total. The common range over all steps means a + distribution that narrows a lot is drawn in a few bins at the end (visible in the check's gradients); a per-column + range was not built. A strip is read when it comes on screen, and again when histograms were added since. +- **Live updates in place.** A reading asks `training` for each active run in view; new lines ask + `training_series` from the last bucket held, and the charts are redrawn in their canvases; new histogram lines + read again the strips already read; new samples are appended to the strip (the oldest removed past twelve). A + name that was not there before (a validation loss first logged at step 10, the first samples, new tensors) adds + its chooser button or draws its part again (histograms, samples); only a run that had logged nothing before draws + the block's content anew. Texts (where the run is, a chart's caption) are set as the data of their text node, so + a reading adds or removes no node. When the run ends, the page draws the drawer again (as before this phase), + and the block is built again from what is held, without "Reading…" and without reading the series again. +- **The page reads every two seconds only while it knows a build runs** (spec-w4 R1): a build asked for elsewhere + is seen on the next reading, up to fifteen seconds later, and only then is the block read live. The check makes + that reading at once by a return to the window. Opening a run's drawer does not by itself start fast reading. +- **The address** keeps `trWith`, `trSeries`, `trSmooth`, `trLog`, `trX` (`app.js`'s `ADDRESSED`). The chosen + sample name, weights or gradients and "Show all" are kept by the page for the run while it lives, not in the + address. +- **The check** (`tests/browser_project_workspace_training.cjs`, with the stand-in `tests/project_training_stand_in.py`, + plain Python, logging through `tuldok_log` put on the run's `PYTHONPATH` by the recipe's `env`): 7 PASS, on + the last two runs (once before its commit, once at the end; the earlier runs failed on the check's own timing and + on the block's first way of adding names, both fixed). In step 4 the live run (50 steps, 0.25 s each) grew 10 → 18 → + 27 → 35 points across readings with the block, its chart and its strips the same elements and no node added or + removed outside the samples strip; the marker is set once the run has logged every name it will (from its tenth + step), since a first name is drawn by adding its control. +- **Files beyond section 7's list:** `tests/project_training_stand_in.py`; `adapter.js` was not changed (the block + calls `P.call` and keeps its own answers by run, since a running run's answers must not be kept by the adapter). + +### Runs at the end + +Disposable PostgreSQL under `/tmp/claude-1000/t1`, `TULDOK_PROJECT_HOME` and `TULDOK_SETTINGS` there: + +- `PYTHONPATH=tests .venv/bin/python -m unittest discover -s tests -p 'test_tuldok_project_*.py'`: 753 OK (735 + before, plus the 18 of `test_tuldok_project_training`). +- `PYTHONPATH=tests .venv/bin/python -m unittest test_agent_api test_agent_negotiation test_agent_http`: 43 OK, 4 + skipped. +- `tests/test_tuldok_log.py`: 8 OK with the lab's interpreter (PyTorch on the CPU); 6 OK, 2 skipped with Tuldok's. +- Browser: `browser_project_workspace_training.cjs` 7 PASS; unchanged `browser_project_workspace.cjs` 24 PASS, + `browser_project_workspace_fields.cjs` 8 PASS, `browser_project_workspace_still.cjs` 6 PASS (the run drawer + changed). + +### Not verified + +A real training run (nothing was trained; `autolog` was exercised on a tiny model for ten steps on the CPU); the +lab's earlier runs (the converter of `history.json` is deferred and nothing was written into the lab); a metrics +file near the bound of 1,000,000 lines (its memory, about 16 bytes a line and 16 a value, and the first read's +time); bucket sizes above 1 in the page (the check's series are shorter than 800 points; the reader's buckets are +unit-tested); `training_histograms` past `most` in the page; a sample `.npy` of three or more axes in the page; +several parameter groups or the overlay of a run that is itself running; the hover tip in the check (drawn, not +asserted); keyboard order and focus rings beyond the screenshots; a theme changed while the drawer is open (each +theme is loaded); the coding-standards guidance (the Standards Engine answered that its guidance is awaiting +publication, so none was read). diff --git a/docs/plans/training-and-curation/spec-t2.md b/docs/plans/training-and-curation/spec-t2.md new file mode 100644 index 00000000..7707bab8 --- /dev/null +++ b/docs/plans/training-and-curation/spec-t2.md @@ -0,0 +1,135 @@ +# Implementation spec: a model's structure + +This is phase 2 of [the training and curation plan](plan.md). It follows that plan's rule: a file contract, a reader and a view in Tuldok, and a thin optional helper. The readers follow [spec-d1.md](../definition-first-projects/spec-d1.md) and [spec-d2.md](../definition-first-projects/spec-d2.md): standard library only, nothing unpickled, nothing of the project's code run, nothing written, bounded reads, a damaged file refused and not a failure, reads outside the service lock. The page follows [spec-w4.md](../definition-first-projects/spec-w4.md): nothing is rebuilt by a timer or a return to the window. + +**Owner, 2026-11-19:** "Tuldok needs live training views and model structure viwer." + +**After this phase** a model's drawer shows, beside its Weights, how the model is built: its operations from input to output drawn left to right, its modules as groups that open and close, each operation's tensors linked to the Weights block, the drawing coloured by a figure per module (parameters, output size, how much the weights changed), and the graph of another model lined up with it, module by module. Tuldok draws what the project's training script recorded; it never runs the model. + +## 1. The file: `model-graph.json` + +Written by the project's script, where the model's code already runs, once, with an example input (plan, "Why the structure needs the training code"). It lies at the top of a run's product, or beside one of its weights files (spec-d2 section 1). Plain JSON, UTF-8, at most 32 MiB. + +```json +{"format": "tuldok_model_graph_v1", + "complete": true, "reason": null, "method": "torch.export", + "torch": "2.8.0+cpu", "model": "mypackage.nets.Net", + "inputs": [{"name": "x", "shape": [2, 1, 8, 8], "dtype": "float32"}], + "tensors": {"stem.weight": {"shape": [4, 1, 3, 3], "dtype": "float32", "kind": "parameter"}}, + "nodes": [ + {"id": "x", "kind": "input", "operation": "input", "module": "", "tensors": [], + "inputs": [], "outputs": [{"shape": [2, 1, 8, 8], "dtype": "float32"}], "attributes": {}}, + {"id": "conv2d", "kind": "operation", "operation": "aten.conv2d.default", "module": "stem", + "tensors": ["stem.weight", "stem.bias"], "inputs": [{"shape": [2, 1, 8, 8], "dtype": "float32"}], + "outputs": [{"shape": [2, 4, 8, 8], "dtype": "float32"}], "attributes": {"padding": [1, 1]}}], + "edges": [{"from": "x", "to": "conv2d", "input": 0}], + "modules": {"path": "", "class": "mypackage.nets.Net", "parameters": 1234, "children": [ + {"path": "stem", "class": "torch.nn.modules.conv.Conv2d", "parameters": 40, "children": []}]}, + "notes": []} +``` + +- `format` is `tuldok_model_graph_v1`. `complete` is false when only the module tree could be written; `reason` then says why (a sentence), and `nodes` and `edges` are empty. `method` is `torch.export`, `torch.fx` or `module-tree`. `torch` is the version that wrote it, `model` the class's import path (a name only: Tuldok never imports it). +- `inputs`: the example input's tensors, each `name`, `shape` and `dtype`. +- `tensors`: the model's parameters and buffers by the names of its `state_dict`, the names its weights file uses, each `shape`, `dtype` and `kind` (`parameter` or `buffer`). +- `nodes`, in an order where each node comes after the nodes it reads (the exported graph's order): `id` (unique), `kind` (`input`, `operation` or `output`), `operation` (the operator's name, `aten.conv2d.default`, or for a module traced whole by `torch.fx` the module's class), `module` (the path of the deepest module it ran in, `""` for the model itself), `tensors` (the names in `tensors` it reads), `inputs` and `outputs` (each `shape` and `dtype`, a list since an operation can take or give several; a value that is not a tensor is left out), and `attributes` (at most 16 of its other arguments, by name, each a number, a boolean, a short string, null or a list of at most 16 of those). +- `edges`: `from` a node `to` a node; `input` is the position of the value among the arguments of `to` that are nodes. +- `modules`: the tree of the model's modules from `named_modules`, each `path`, `class` (its import path), `parameters` (every parameter under it, counted once) and `children`. +- `notes`: sentences on what was left out (a truncated graph, an attribute dropped). + +**Bounds,** which the helper keeps and the reader checks: 20,000 nodes, 100,000 edges, 20,000 modules, a module tree at most 64 deep, 100,000 tensors, 64 tensor names and 16 inputs or outputs per node, 16 axes per shape, 1,024 characters per string. A model whose exported graph has more than 20,000 nodes is written as its module tree with `complete` false and the reason. + +## 2. The helper: `tuldok_log.graph.export_graph` + +`tuldok_log` is the small package of the plan (decision 3). Phase 2 adds `tuldok_log/graph.py`; it imports the standard library and, inside `export_graph`, PyTorch. + +`export_graph(model, example_inputs, path)` writes `model-graph.json` at `path` (a file, or a directory to write it in) and returns the dictionary written. + +1. `torch.export.export(model, example_inputs)` (strict off, so a script's own Python is traced as written). Each `call_function` node becomes an operation; its module is the last entry of its `nn_module_stack`; its tensors are the parameters, buffers and constants it reads, by the names of the graph signature (which are the `state_dict` names); its shapes and types come from each node's `meta["val"]` (fake tensors: nothing is computed again). +2. If export fails: `torch.fx.symbolic_trace(model)`, the shapes from running `ShapeProp` once on the example input (a forward pass on the CPU or wherever the inputs are); a `call_module` node is one operation named by its module's class, with that module's tensors. +3. If both fail (a `forward` that branches on data values, say): the module tree only, `complete` false, `reason` naming both errors (their first line, at most 400 characters). + +Never pickles, never imports anything of Tuldok's server, writes with `json.dump` to a temporary file in the same directory and renames it, so a reader never sees half a file. + +## 3. Operations + +Three reads, `inspect`, in `INSIDE_READS` like the others, in the `project_folder` profile and absent from `catalog` (whose digest is unchanged). Each takes `run`. The file is found at the top of the product, else beside the run's weights files (in their order); none is refused `not-found`. + +| Operation | Arguments | Result | +| --- | --- | --- | +| `model_graph` | `depth` (0 to 64, default 1 when the graph has more than 200 operations, else 64), `open` and `close` (module paths, at most 2,000 each) | `path`, `format`, `complete`, `reason`, `method`, `torch`, `model`, `inputs`, `notes`, `depth`, `counts` (`nodes`, `edges`, `operations`, `modules`, `tensors` of the whole file), `modules` (the tree), `tensors`, `open` (the modules open in effect), and the graph folded: `nodes` and `edges` | +| `model_graph_overlay` | `kind`: `parameters`, `outputs`, `operations` or `change`; for `change`, `against`: `start` (the weights file's `initial-` file) or `run` (with `other_run`), and optionally `path` and `other_path` (weights files; default the first readable one) | `kind`, `unit`, `modules` ({path: number or null}), `nodes` ({id: number or null}), `low`, `high` (over the finite numbers of both), and for `change` `against` (`run`, `path`, `other_run`, `other_path`) | +| `model_graph_compare` | `other_run` | `modules`: every module path of either tree, in this tree's order then the other's, each `path`, `status` (`same`, `changed`, `added`: here only, `missing`: there only), `class`, `other_class`, `parameters`, `other_parameters`, `reshaped` (tensors directly in it whose shape differs: `name`, `shape`, `other_shape`), `only_here` and `only_there` (its own tensors on one side); `counts` by status | + +**Folding.** A module's depth is its number of dotted parts (`""` is 0, `blocks` 1, `blocks.0.conv` 3). A module is open when it is in `open`, or when its depth is at most `depth` and it is not in `close`; the model itself is always open. An operation is shown when every module on its path is open; otherwise it is folded into the shallowest module on its path that is closed, shown as one node of kind `group`, id `module:`. Inputs and outputs are always shown. A shown node carries `id`, `kind`, `label`, `module`, `class` (of its module), `operation` (operations), `tensors` (an operation's own; a group's, every tensor its operations read, at most 64, with `tensor_count`), `parameters` (the sum of the sizes of its tensors that are parameters, for a group its module's count), `inputs` and `outputs` (a group's: the values that cross into and out of it, at most 16 each), `operations` (1, or how many a group holds) and `first` (the position in the file of its first node). Shown nodes are in the order of `first`. Edges join the shown nodes of their two ends: an edge inside one group is dropped, edges between the same two shown nodes are one with `count`, and each carries `shape` and `dtype` of the first value it carries. + +**Overlays.** `parameters`: a module's count from the tree; an operation's, the parameters among its tensors. `outputs`: the number of values an operation gives (the product of its outputs' shapes), a module's the sum over the operations in it. `operations`: how many operations run in a module (an operation's is 1). `change`: from `weights_compare` of spec-d2 (its bounds and refusals), the relative change of the tensors under a module (`sqrt(Σ difference_norm²) / sqrt(Σ other_norm²)` over them, the page's layer change of D2b) and of an operation's tensors; null where none is compared, where the other's norm is zero, or where a norm is not finite. "Under a module" means its own tensors and those of every module inside it. + +**Comparing.** Modules are matched by path. `changed`: the class differs, or a tensor directly in it changed shape or is on one side only. A module whose children differ but whose own tensors and class are the same is `same`; its children say what changed. + +**Refusals.** A file that is not UTF-8 JSON, not of the format, missing a key, of a wrong type, past a bound, with a duplicate node id, with an edge naming no node, or with a module tree not rooted at `""` is refused `unsupported` with a sentence saying what (a file over 32 MiB `too-large`). The parsed file is kept (8 files, least recently used dropped) while its size, time and inode are unchanged. + +## 4. The page + +In a model's drawer, a block **Structure** right after Weights, when one of the model's current products has `model-graph.json` (`model_graph` refused `not-found` for all of them: no block at all). + +- **Head:** how it was written (`method`, `torch`), its counts, the example input's shapes; when not `complete`, the reason, and the module tree alone. +- **The drawing,** SVG, no library: left to right, ranks by the longest path from the inputs (an edge back to an earlier node is ignored in ranking and drawn as a curve), each rank's nodes ordered by the barycentre of their neighbours (four sweeps, down and up). Each node a box with its label (operation or module name), its output shape, and a mark for a group (how many operations). In a scrolling box of its own, so a wide graph never widens the drawer. +- **Folding:** a graph of more than 200 operations starts at depth 1, else open. A group pressed opens (one `model_graph`); the module outline under the drawing lists every module with its class and parameters, open or closed, as a tree whose rows open and close the module. "Open all" and "Fold to depth 1". +- **A node pressed** shows its details under the drawing: operation, module and class, inputs and outputs, attributes, and its tensors, each a link that opens that tensor in the Weights block above (its layer opened and scrolled to). +- **Colour by:** nothing, parameters, output size, change from the weights it started from (when kept), change from the model it derives from. The nodes are filled on a sequential scale (logarithmic for counts), a legend gives the ends; a node with no value is left plain. +- **Compare with:** the model it derives from and the models derived from it, when they have a graph. The other's drawing is shown under this one (the drawer is 420 px wide, so the two are stacked, one above the other, folded the same way), and both are marked from `model_graph_compare`: a module added (here only), missing (there only) or changed is outlined and named in a list of the differences. +- Every node is a focusable button (Enter or Space), with a text alternative. Light and dark from the tokens of `base.css`. Read again for the model on screen, the block resolves only once complete, and is put in place in one step with the person's place in it (folding, selection, colour, comparison) kept. + +## 5. Slices + +| | Slice | Files | +| --- | --- | --- | +| T2a | The contract, the helper, the reader and the operations | `docs/contracts/project-folder.md`, `tuldok_log/graph.py` (new), `tuldok_project/inside_graph.py` (new), `operations.py`, `service.py`, `tests/test_tuldok_project_model_graph.py` (new), `tests/test_tuldok_log_graph.py` (new), the count of the table's rows in `tests/test_tuldok_project_service.py` | +| T2b | The Structure block | `static/project-workspace/structure.js` (new), `weights.js` (opening a tensor from outside), `screens.js`, `adapter.js`, `base.css`, `static/project-workspace.html`, `app.py` (the asset list), `tests/browser_project_workspace_structure.cjs` (new), its fixture | + +## 6. Tests + +**T2a, `tests/test_tuldok_project_model_graph.py`,** on graph files the tests write by hand: a graph read whole; folding at depth 0, 1 and full, with `open` and `close`, against counts the test computes (nodes, edges merged with their count, an edge inside a group dropped, a group's crossing values and parameters); each overlay's arithmetic (parameters, outputs, operations, change against a known weights difference, against the start and another run); the comparison (same, changed by class, by shape, added, missing); every refusal (not JSON, wrong format, missing key, duplicate id, dangling edge, a bound lowered for the test, deep nesting); a module-tree-only file; the file found beside the weights; the three in `project_folder`, not in `catalog`, digest unchanged; the committed fixture graphs (written by the helper) read. + +**`tests/test_tuldok_log_graph.py`,** with PyTorch on the CPU, skipped when it is absent: a nested model (a convolution stem, residual blocks of two convolutions with an add, an MLP head) exported whole: every parameter name in a node's `tensors` is in `state_dict`, every `state_dict` name is in `tensors`, the shapes are those of a forward pass, the add has two edges in, the module tree's counts are `sum(p.numel())`; a model whose `forward` branches on a value writes the module tree, `complete` false, with the reason; the file is JSON with no pickle; the node bound (lowered) falls back to the tree. + +**T2b, `tests/browser_project_workspace_structure.cjs`,** on fixture products for a parent and a derived model (graphs written by the helper once and committed as test data, weights written with the standard library, the derived one with `initial-weights.npz`), owner signed in, no GPU, no PyTorch: + +1. The derived model's drawer shows Structure after Weights, with the file's counts and the drawing; a model without a graph has no block. +2. Pressing a group opens it in place (more nodes, one `model_graph`, nothing else in the drawer rebuilt); the outline closes it again. +3. A node pressed shows its details; its tensor link opens that layer in the Weights block. +4. Colour by parameters and by change: the nodes' fills change and differ between nodes. +5. Compare with the parent: the changed and added modules are marked, as `model_graph_compare` says. +6. Light and dark, 1440 and 390 wide: no sideways overflow of the drawer or the page, no script error, no failed request; nothing redrawn on a return to the window. + +## 7. Out of scope + +Running the model, activations, profiling time per operation (a later field `seconds` on a node; nothing writes it yet), ONNX or TorchScript files, graphs of frameworks other than PyTorch, editing a model's structure, drawing more than two graphs at once. + +## As built + +What differs from the text above, and what was found. Built on `feat/t2-model-graph` beside phase 1, which owns `tuldok_log/__init__.py`; this branch holds a one-line `tuldok_log/__init__.py` (a docstring) only so that `tuldok_log.graph` imports, and the merge takes phase 1's. + +### T2a, the file, the helper, the reader and the operations + +- **A run with no graph is not refused by `model_graph`.** Section 3 says "none is refused `not-found`". As first built it was, and the page, which asks `model_graph` for each current product of every model whose drawer opens, then drew a failed request (a 404, which the browser logs as an error) for every model without a graph; the weights check, which fails on any such error, would have failed. `model_graph` now answers `path` null, `modules` null and the rest empty, as `fields` answers `kept` false (spec-d6). `model_graph_overlay` and `model_graph_compare` still refuse it `not-found`; the page asks them only of a run that has a graph. +- **The arguments are named `open` and `close`** in the operation; the reader's keywords are `opened` and `closed`. +- **The node bound past 20,000 in the helper** is checked while the graph is walked: past it, the module tree alone is written with the reason, as section 2 says; the edge bound (100,000) the same way after the walk. A node past 64 tensors keeps the first 64 with a note. +- **Tensors.** The helper lists `state_dict` (parameters and persistent buffers, with `kind`) and adds non-persistent buffers as `buffer`. Lifted constants (`CONSTANT_TENSOR` in an exported graph) are read by name like parameters but are not in `tensors` unless the `state_dict` holds them; a node's `tensors` may then name one the reader does not know, which it accepts (it counts no parameters for it). +- **Module paths** in `nn_module_stack` are taken in the dotted form of `named_modules`; a path the tree does not list is walked up to its nearest listed parent, by the helper and by the reader alike. PyTorch 2.8 already gives dotted paths; the other spellings (`L['self'].blocks[0]`) are handled but were not met. +- **`tuldok_log` writes through a file beside the target** (`.model-graph.json...tmp`, opened exclusively so the umask applies) renamed into place; `tempfile.mkstemp` was tried first and left the file readable by its owner only. +- **The reader's group `outputs`** are the values of the first edge from the group to each shown node it feeds, at most 16; `inputs` every crossing edge's value, at most 16. +- **Tests.** `tests/test_tuldok_project_model_graph.py`, 20 (one with the database). `tests/test_tuldok_log_graph.py`, 5, run with the lab's PyTorch 2.8.0+cpu on the CPU (`PYTHONPATH=.:tests -m unittest test_tuldok_log_graph`); in Tuldok's environment, which has no PyTorch, they are skipped (5 skipped). The service test's count of the table's rows went from 61 to 64. The catalog profile's digest is unchanged. + +### T2b, the Structure block + +- **The two drawings are one above the other**, not side by side: the drawer is at most 420 px wide. Modules are matched by path by `model_graph_compare`; the other drawing is folded with the same depth, `open` and `close`. A node's mark is its module's status; a group whose own module is the same but holds a module that differs is marked "something inside it differs" (`inside`). A module only in the other model is marked in the other drawing. +- **A group pressed opens** (rather than selecting it): its parents are opened with it, past the depth when needed. An operation pressed shows its details; "Fold " in the details, and every row of the outline, close a module again. Inputs and outputs show their details too. +- **"Fit to width"** (not in section 4) shrinks the drawings to the drawer's width for an overview, never larger than their own size; pressed again, their actual size in the scrolling box. Added because the fixture's chain of 18 nodes shows two or three at a time in the drawer; even fitted, a long chain is a thin strip (doubt: a top-to-bottom layout might suit the narrow drawer better; section 4 asks for left to right). +- **The colours.** The fills are `color-mix` of the tokens `--ramp-lo` and `--ramp-hi` in CSS, so a theme change recolours them with no script; a node past 55 % of the scale writes its labels in `--surface`. Counts (parameters, output size, operations) are on a logarithmic scale from zero, the change linear from its lowest. "Operations" is offered as a colour too (section 4 names parameters, output size and the two changes). +- **"Change from the weights it started from"** is offered whether or not the run kept an `initial-` file; without one, choosing it shows the server's refusal in the block. "Change from the model it derives from" compares with the first current product of the parent. +- **`weights.js` gained `reveal(model, array)`:** it opens the tensor's layer with that tensor in the Weights block (the same rebuild in place as a person's click), scrolls the layer's row to the top of the drawer's view and focuses its button. +- **Files beyond section 5's list:** `tests/project_structure_browser_fixture.py` (the weights, standard library) and `tests/project_structure_fixture_export.py` (writes the committed graphs `tests/fixtures/model_graph/parent.json` and `derived.json`; run once with the lab's PyTorch). +- **Found at 390 px:** the page's own header (tabs and session bar), strip, stage and map are wider than the window on every tab before this block draws (the page is 553 to 554 px wider than 390). The check asserts that the drawer does not pass its stage and that nothing in the drawer overflows, and records the page's overflow; it does not assert the page's, which is not this phase's. The drawer itself sits 9 px past the window because the stage does. +- **Runs** (a disposable PostgreSQL, no GPU): project suite 755 OK (735 before this phase, and the 20 of `test_tuldok_project_model_graph.py`); agent interface 43 OK (4 skipped); `test_tuldok_log_graph.py` 5 OK with the lab's PyTorch, 5 skipped without it; `browser_project_workspace_structure.cjs` 8 PASS, and unchanged `_weights` (7 PASS) and `browser_project_workspace.cjs` (24 PASS). +- **Not verified:** a graph near the bounds (20,000 nodes: the reader's time and the page's drawing of thousands of SVG nodes; folding to depth 1 by default past 200 operations is untested in the page); a graph written by `torch.fx` drawn in the page (tested in the helper only); a module-tree-only file in the page (tested in the reader only); a model with several products of which more than one holds a graph (the first is drawn); the overlay's and comparison's refusals shown in the block; a theme changed while the drawer is open (the check loads each theme); Tab order and the focus ring of the SVG nodes, by eye in the screenshots only; the lab's own models (none has a `model-graph.json` yet; nothing was exported from the lab's code). diff --git a/docs/plans/training-and-curation/spec-t3.md b/docs/plans/training-and-curation/spec-t3.md new file mode 100644 index 00000000..9ca56a91 --- /dev/null +++ b/docs/plans/training-and-curation/spec-t3.md @@ -0,0 +1,129 @@ +# Implementation spec: comparing chosen models deeply + +This is phase 3 of [the training and curation plan](plan.md). It follows that plan's rule (Tuldok reads what a project's scripts wrote; nothing of the project's code runs in Tuldok), and it adds no file contract: everything here is read from what Tuldok already holds, the definitions, the per-case values of [the values files](../../contracts/project-folder.md#values-files) and the fields a run kept ([spec-d6.md](../definition-first-projects/spec-d6.md)). The readers follow spec-d1 and spec-d6: standard library only, nothing written, bounded work, an unusable input listed with its reason rather than failing the whole answer. The page follows [spec-w4.md](../definition-first-projects/spec-w4.md): nothing is rebuilt by a timer or a return to the window, and a person's own moves are drawn in place. + +**Owner, 2026-11-19:** "Tuldok already kinda has comparison view for models in the ancestory tree/list, but it would be great if it could compare selected models deeper like also showing output results next to eachother. Im not sure how the GUI would work for that." And: "Parallel coordinates sound very usful for Tuldok." + +**After this phase** the models a person chose (two to four) open a **Compare** tab: one column per model, rows of who each is and what differs between their recipes, their metrics on every benchmark that measures them with a 95 % interval where this machine holds per-case values, where each sits on the frontier, and then their outputs on one case they all kept, the native field, each prediction and each signed error, moved together by one slice control. The Benchmarks tab gains **parallel coordinates** over the benchmark's metrics and the recipe settings that vary between its subjects (a range dragged on an axis keeps the subjects inside it, in the table and in the other charts), and **intervals**: whiskers on the ranking charts and in the table, and a mark on the frontier where two neighbours are not told apart by their per-case values. + +## 1. What differs between recipes + +A recipe's definition is a JSON document. To compare two to four of them, and to put their settings on axes, each document is **flattened** to its leaves: + +- A leaf is `null`, a boolean, a number, a string, or an empty object or list (shown as `{}` and `[]`). An object's keys are joined with `.`; a list's items are `[0]`, `[1]`, …; a key that is not a plain name (holding `.`, `[`, `]`, `"` or a space, or empty) is written `["key"]`. So the lab's learning rate is `run.settings.captures[0].experiment.training.learning_rate`. +- The keys of who a model is and how it is described are not settings: `format`, `kind`, `title`, `description`, `card`, `labels`, `earlier_work` and `compared_with` are left out. `group`, `change` and `derived_from` are compared on their own rows (kinds `group`, `change`, `parent`); the leaves under `run.inputs` are of kind `data`; every other leaf is of kind `setting`. +- **Bounds:** at most 5,000 leaves a document and 32 levels deep (past either, the rest is left out and the document marked `truncated`); a string is compared whole but shown to 200 characters. +- A settings file a recipe names (`run.settings_file`) is not opened: only what the definition holds is compared (doubt: a recipe whose settings live in its file shows none here). + +Two leaves are **the same** when both models have the path and the values are equal as JSON values (`1` and `1.0` are equal; `true` and `1` are not). A path **differs** when any model lacks it or holds another value. The comparison lists every differing path, `group`, `change` and `parent` first, then `data`, then the settings in path order, at most 400 rows (`more` says how many past them), and counts the paths that are the same. + +## 2. Intervals from per-case values + +A summary row of a benchmark (count, mean, median, p10, p90, min, max) made from per-case values (`origin` `values`) can carry an interval; one read from a results file cannot, as the cases are not on this machine. + +- **The cases** of a subject and metric are its held values that are numbers, over every group (the cases the ungrouped summary row is made from), each keyed by its case and group. +- **The resampling** (a percentile bootstrap): the values sorted ascending; a fresh `random.Random(20261119)`; 2,000 resamples, each `rng.choices(values, k=n)` sorted; the six statistics of each resample computed as `results.summarise` computes them (the mean with `math.fsum`, the percentiles by its linear interpolation); the interval of each statistic is the 2.5th and 97.5th percentiles of its 2,000 values, by linear interpolation at `q × 1999`. A subject with fewer than 2 values has no interval. The same seed for every interval makes an answer the same on every reading and lets a test do the arithmetic itself. +- **The difference between two subjects** on a metric and a statistic: when they share at least 2 case keys, *paired*: the shared keys in sorted order, each resample `rng.choices(range(m), k=m)`, the statistic of each side over the drawn keys and their difference (first minus second); otherwise *independent*: each resample draws the first's values then the second's. The interval is taken as above. The difference is **significant** when its interval does not hold zero. +- **The frontier.** For a tradeoff chart that is not grouped, its frontier points (chart_data's, in its order by x) are taken in pairs of neighbours; a pair is compared on both of the chart's metrics with the chart's statistic. A pair is **separated** when both differences are significant; otherwise each of its points is **not separated from** the other. A pair where either side lacks values on either metric is left unknown (null). Grouped tradeoff charts are listed with no pairs (doubt: the cases of one group value would do; not built). +- **Cost.** 2,000 resamples of 128 values take about 25 ms in CPython; a benchmark of 150 subjects, five metrics and 128 cases each takes about 20 s. The intervals of a benchmark are therefore computed outside the service's lock and kept (8 benchmarks, least recently used dropped) while its held values are the same; the page draws the tab first and the intervals when they arrive (section 5.2). + +## 3. Parallel coordinates + +For one benchmark and a statistic, every subject is a line through: + +- **the metric axes,** one per metric of the benchmark: the subject's statistic from its summary row (null when it has none). `positive` says whether every value is above zero, so the page may offer a logarithmic scale. +- **the setting axes:** the subjects' recipes flattened as in section 1 (the identity rows `group` and `change` and the `data` leaves included as settings may vary there too). A path is a candidate when at least two subjects have it and their values are not all the same. It is a **number** axis when every value is a number (not a boolean), and a **category** axis when its values are texts, booleans or a mix, with 2 to 12 distinct values; categories are ordered numbers first (by value), then `false` and `true`, then texts, then the rest, and a subject's value on the axis is the position of its category. A path with more than 12 distinct non-numeric values is left out. +- **Which settings first.** The candidates are ordered by how strongly they go with the first metric that has at least three values: the absolute Spearman rank correlation (ties given their mean rank) over the subjects that have both, null under three or when either side is constant; then by how many subjects have the path, then by path. At most 48 are answered (`left_out` counts the others). + +The page shows the metrics and the first six settings by default, at most twelve axes at once, with a chooser for the rest. + +## 4. Operations + +Three reads, `inspect`, in the `project_folder` profile and absent from `catalog` (whose digest is unchanged). Nothing new is written to the contract file: these read the definitions and results it already describes. + +| Operation | Arguments | Result | +| --- | --- | --- | +| `model_compare` | `recipes` (2 to 4 distinct recipe references), `statistic` (one of the six, default `median`) | `statistic`, `resamples`, `level`, `seed`; `models`: per recipe `reference`, `title`, `group`, `change`, `derived_from`, `truncated`; `differs`: each `kind`, `path`, `values` (one per model, JSON as held, null where absent), `present` (booleans); `same` (count of paths the same), `more`; `benchmarks`: each benchmark with a subject made from one of the recipes: `benchmark`, `title`, `metrics` (`id`, `title`, `units`, `better`), `models` (one per recipe: null, or its `subject`, `title`, `per_case`, and `values` by metric: `value`, `count`, `low`, `high`), `frontiers` (per tradeoff chart: `chart`, `title`, `statistic`, `metric`, `metric_x`, `grouped`, `models`: one per recipe, null when not drawn, else `on` (on the frontier), `x`, `value`, `not_separated_from` (subject names)) | +| `benchmark_axes` | `benchmark`, `statistic` (default `median`) | `benchmark`, `statistic`, `metrics` (`id`, `title`, `units`, `better`, `positive`, `low`, `high`), `settings` (`path`, `kind` `number` or `category`, `categories`, `low`, `high`, `count`, `distinct`, `association`), `left_out`, `subjects` (`subject`, `title`, `recipe`, `metrics` by id, `settings` by path) | +| `intervals` | `benchmark` | `benchmark`, `resamples`, `level`, `seed`, `intervals`: per subject and metric with values, `subject`, `metric`, `count`, and for each statistic `[low, high]` (or null); `frontiers`: per tradeoff chart `chart`, `title`, `statistic`, `metric`, `metric_x`, `grouped`, `pairs` (`subjects` [a, b], `metric` and `metric_x` each `low`, `high`, `paired`, `count`, `significant`, or null; `separated`), `not_separated` (subject names) | + +`model_compare` refuses `not-found` for a recipe that is not defined, `invalid` for repeated ones; the values of `model_compare` are those of `intervals` for the subjects of its recipes (the frontier pairs only those that touch one of them). The three are reads of the database and the definitions: their answer is computed from what the project holds, under the service's lock for the reading and, for `intervals` and `model_compare`, outside it for the resampling. + +## 5. The page + +### 5.1 The Compare tab + +The compare set (the page's `cmp`, kept in the address as before) may now hold **four** models; choosing another benchmark no longer empties it, since it is no longer only of one benchmark. It is filled from three places: the boxes of the Benchmarks table, **Compare** in a model's drawer on the Benchmarks tab, and **Compare** in a model's drawer on the **Map** (new). Each drawer with Compare also has **Open Compare** once the set holds two. + +**Why a tab, not a view inside Benchmarks or a drawer:** the set is the page's, not one benchmark's; one column a model needs the full width (the drawer is 420 px); and a tab has an address (`#compare`), so a comparison can be reloaded and shared. The tab is after Benchmarks. + +The tab, for the models of the set that are recipes on the map (a benchmark subject made from no recipe is named and left out): + +- **Head:** the set as chips with × and Clear, the statistic (shared with Benchmarks), and, under two models, a sentence on how to choose them. +- **The table,** one column per model (its name opens its drawer, with Compare in it): *Who* (group, kind of change, derives from, state, trained); *What differs* (section 1: each differing path with each model's value; absent written as "—"; the count of the same); then per benchmark, its metrics, each cell the statistic, its interval as `low – high` and the number of cases, the best of a row marked; and for each tradeoff chart, the frontier row: "on the frontier", "on the frontier, not separated from …", "off the frontier", or "—". The table scrolls sideways in its own box when narrow. +- **Outputs.** The cases the models' runs kept (spec-d6 `fields` of every current product of each model), matched by name (the probe's member id, or "case N"); a case is offered when at least two models kept it, comparable and of the same shape (a model of another shape is said so in its column). A select chooses it (the most shared first). One control for all columns: the axis cut across, the position (and the fourth axis), "Remove the mean". Then a grid, a column per model and three rows, Native, Predicted and Signed error; Native and Predicted of every column on one sequential scale (over every column's `range`), the errors on one diverging scale (±the largest `error_limit`), a colour bar for each row with its ends; under each column the whole-field figures (relative L2, mean-free relative L2, RMSE). Hover shows the cell's position and the three values of that column. A move asks `field_compare` of every column at once (one request each, at most one round in flight, the latest wins) and draws the canvases again in place. The case, axis, position and mean option are kept in the address (`cmpCase`, `cmpAxis`, `cmpIndex`, `cmpMeanFree`). +- Read with the tab and put in place in one step (tabs.js's `paint`); the outputs' first slices are read with it. + +### 5.2 Benchmarks: parallel coordinates and intervals + +- **Settings and metrics** (parallel coordinates), after the charts, when the benchmark has two subjects or more: an SVG of one vertical axis per chosen axis, one line per subject (in the kind of change's colour, as the charts' points), each axis named with its ends (a category axis with its categories). Under it, the chooser: the metric axes and the 48 settings (each with its association), as toggles, at most twelve on. Each axis has buttons to move it left and right, and a metric axis whose values are all positive a "log" toggle. **Brushing:** a range dragged on an axis keeps the lines whose value on it lies inside (a line with no value there is out); ranges on several axes all apply; a range is cleared by a click on the axis without a drag, and all by "Clear ranges". The lines out of range are faint; the subjects table hides their rows and says how many are shown; the charts above draw their points faint. All of this is set in place (no reading, no rebuild of the tab); the order, the axes on, the log scales and the ranges are the tab's settings, kept when the tab is drawn again. +- **Intervals:** read after the tab is put in place (section 2's cost); when they arrive the tab is drawn again in place with them (its scroll, focus and brushing kept). The ranking charts get a whisker from `low` to `high` of the chart's statistic for each subject with an interval; the table's metric cells a second line `low – high` of the tab's statistic; the tradeoff chart a dashed ring round each frontier point not separated from a neighbour, a tip line naming that neighbour, and a line of the legend. A line under the charts says how the intervals are made (2,000 resamples, seed, percentile 95 %). + +## 6. Slices + +| | Slice | Files | +| --- | --- | --- | +| T3a | Flattening, the differences, the intervals, the axes; the three operations | `tuldok_project/comparing.py` (new), `operations.py`, `service.py`, `tests/test_tuldok_project_comparing.py` (new), the count of the table's rows in `tests/test_tuldok_project_service.py` and its read test | +| T3b | The Compare tab | `static/project-workspace/compare.js` (new), `compare.css` (new), `fields.js` (its helpers shared), `tabs.js` (the limit of four, Compare in drawers), `screens.js` (Compare on the Map's drawer), `app.js` (the tab, the limit, the address), `adapter.js` (a kept answer peeked), `static/project-workspace.html`, `app.py` (the asset list) | +| T3c | Parallel coordinates and intervals on Benchmarks | `static/project-workspace/parallel.js` (new), `charts.js` (whiskers, the ring), `tabs.js`, `compare.css`; `tests/browser_project_workspace_compare.cjs` (new) and its fixture `tests/project_compare_browser_fixture.py` (new) | + +## 7. Tests + +**T3a, `tests/test_tuldok_project_comparing.py`:** flattening (nested objects and lists, quoted keys, empty containers, the identity keys left out, the bounds lowered for the test, `1` equal to `1.0` and not to `true`); the differences of three documents (each kind, an absent path, the order, `more`); the bootstrap against the test's own arithmetic (its own seeded `random.Random`, choices and percentiles: every statistic's interval equal), fewer than 2 values (none), the paired and the independent difference, significance; the axes on documents the test writes (a number axis, a category axis with its positions, a constant path left out, a path of 13 texts left out, the association against a Spearman correlation the test computes, the order, `left_out` with the bound lowered); the three operations on a project with per-case values imported (`model_compare` of three recipes: differences, values with intervals equal to the module's, frontier rows; `intervals` with frontier pairs; `benchmark_axes`); refusals (an unknown recipe, a repeated one, an unknown benchmark); the three in `project_folder` and not in `catalog`, the digest unchanged. + +**T3b and T3c, `tests/browser_project_workspace_compare.cjs`,** on the minimal fixture with three recipes of earlier work whose adopted products keep fields of one shared case (and one more case kept by two of them), a benchmark whose subjects they are with per-case values imported for each and a tradeoff and a ranking chart, and recipes whose settings vary (a number, a category): + +1. The Map drawer's Compare adds models; Open Compare shows the tab with three columns, the differing settings (and not the same ones), the metrics with intervals equal to `intervals`' answer, and the frontier rows. +2. Outputs: the shared case chosen, nine panels drawn; one slider move asks `field_compare` three times and redraws every canvas in place (the rest of the tab the same elements); the Native row of every column on one scale (the same colour where the fields are equal). +3. Benchmarks: parallel coordinates with the metric and setting axes; a range dragged on the metric axis hides the table rows outside it and dims the charts' points, with no request; reordering and the log toggle redraw in place; clearing shows every row again. +4. Intervals: the ranking chart has whiskers, the table has `low – high` lines, the tradeoff chart marks the frontier points the answer says are not separated. +5. Light and dark, 1440 and 390 wide: no sideways overflow of the page's main area outside the tables' own scrolling boxes, no blank canvas, no script error, no failed request. + +**Regressions:** the project suite, the agent interface tests, `browser_project_workspace.cjs`, `_fields.cjs` and `_structure.cjs`. + +**On the lab folder (acceptance, read only):** parallel coordinates over the 150-pressure-fits benchmark's 150 subjects (definitions and the results file only) filter to the best errors and show their settings; three earlier pressure models with kept fields compared on one case (the owner, on his database). + +## 8. Out of scope + +Running anything to make a prediction; comparing more than four models; intervals for grouped rows (by horizon, say) and for results read from a file; multiple-comparison corrections; settings read from a recipe's settings file; brushing on the Map; a hyperparameter search's own view (phase 7 reuses this). + +## As built + +What differs from the text above, and what was found. Built on `feat/t3-compare` beside phase 1, which owns the Train tab and `tuldok_log/`; neither is touched here. Edits to shared files are small: `app.js` (the tab, the bound of four, the address), `tabs.js` (Compare in drawers, intervals, the parallel-coordinates hook), `screens.js` (Compare in the Map's drawer), `adapter.js` (`peek`), `charts.js` (whiskers, ring), `operations.py`, `service.py`, the service test's count (64 to 67). `base.css` and `docs/contracts/project-folder.md` are not changed: the styles are in a new `compare.css`, and no file contract is added. + +### T3a, the reader and the operations + +- **The setting axes are ordered by the association weighed by its coverage**: `|ρ| × (subjects with both) / (subjects with a value of the first metric)`, then by count and path. Section 3 said the plain `|ρ|`. On the lab's 150 pressure fits the plain order led with list positions of `run.sources` that six subjects hold, at |ρ| 0.88 to 0.95 by chance. For the same reason **`run.command` and `run.sources` are of a sixth kind, `code`**: listed in what differs (last), never an axis. `association` in the answer is still the plain ρ. +- **Categories are ordered** numbers first by value, then `false`, `true`, then texts, then the rest (written into section 3 before the build: ordering by JSON text put `"wide"` before `16` and `8` after `40`). +- **`model_compare` resamples only for its statistic.** Its intervals equal those of `intervals` for that statistic (the draws are the same sequence; the browser check compares them). +- **The intervals of a benchmark are kept** by comparing what was gathered with what was kept (no hash), 8 benchmarks. +- **Tests:** `tests/test_tuldok_project_comparing.py`, 13 (three with the database, through `Service`): flattening and its bounds, the differences, every statistic's interval and the paired and independent differences against the test's own seeded resampling and percentiles, the frontier pairs (separated, not separated, unknown, grouped, `touching`), the axes (number, category with mixed values, constant and 13-text paths left out, ρ against the test's own Spearman, the weighed order, code never an axis, `left_out`), the three operations on the minimal fixture with its values imported, refusals, and the catalog digest unchanged. In the minimal fixture's tradeoff chart only one point is on the frontier, so its pairs are tested on hand-made chart data. + +### T3b and T3c, the page + +- **The tab is `#compare`, after Benchmarks.** Its drawer is the model's drawer with Compare. Choosing another benchmark keeps the set (it used to empty it). +- **The intervals arrive after the tab:** the Benchmarks tab is put in place without them; when `intervals` answers, the tab is drawn again in place (`act.setUI('ivFor', …)`, the whole tab built off the page with its scroll, focus and ranges kept); afterwards the kept answer is peeked (`P.peek`) and the tab draws them at once. A line under the charts says how they are made. +- **The frontier mark** is a dashed ring round each point (not a dashed segment of the line). +- **Brushing is by pointer only**; the axes' move, log and remove buttons and the chooser are buttons and checkboxes, but a range cannot be set from the keyboard (doubt; a pair of number fields per axis would do). The lines are SVG paths with a ``, not focusable. +- **The outputs** take the shape and the first slice from the first model that kept the chosen case; a model that kept it in another shape says so in its cells. The colour bars sit with each row's title; under each column, relative L2, mean-free relative L2, RMSE and the largest error over the whole field. +- **Files beyond section 6's list:** `compare.css`, and the fixture writes recipes, a benchmark, two charts, values files and three products. + +### Runs (a disposable PostgreSQL, no GPU) + +Project suite 768 OK (755 before, and the 13 new); agent interface 43 OK (4 skipped); `browser_project_workspace_compare.cjs` 6 PASS (twice); unchanged `_fields.cjs` 8 PASS, `_structure.cjs` 8 PASS, `browser_project_workspace.cjs` 24 PASS. The page did not overflow sideways at 390 px on the Compare and Benchmarks tabs in the check. + +### On the lab folder (read only, no database) + +`benchmark_axes`' computation over `150-pressure-fits-complete-gpu-requests-128-consumed-cases`, from the folder's definitions and its results file only (a script giving `comparing.benchmark_axes` what `gather_axes` would read), in 0.03 s: 150 subjects, all 150 recipes found, 5 metric axes, 48 settings offered (39 more left out). The first settings: `run.resources.wall_hours`, `run.settings.training.learning_rate` (144 subjects, ρ 0.72 with the equation residual), `run.settings.resources.wall_seconds`, `run.settings.training.schedule[0].learning_rate`, the seeds, `group`, the datasets. Keeping the best tenth by mean-free pressure error (≤ 1.05e-4: 16 subjects) shows `wall_hours` 0.5 for all 16, and for the 10 of them that have it learning rate 1e-5, schedule learning rate 1e-6, `wall_seconds` 1800; 6 have no `run.settings.training` (their settings are elsewhere in their recipes). Not run in the browser on the lab folder. + +**Not verified:** the kept-fields comparison of three lab models (the owner, on his database); the time of `intervals` on the lab's real per-case values (estimated 20 s the first time for 150 subjects, five metrics and 128 cases: 25 ms a subject and metric measured); four models in the browser (three were checked); a case kept in two shapes; a refused `field_compare` in a column; the slider dragged quickly; a grouped tradeoff chart (no pairs, by design); the chooser past twelve axes and many axes at 390 px (titles are cut to the space between axes); Tab order and focus rings, by eye in the screenshots only. diff --git a/docs/plans/training-and-curation/spec-t4.md b/docs/plans/training-and-curation/spec-t4.md new file mode 100644 index 00000000..69b7023a --- /dev/null +++ b/docs/plans/training-and-curation/spec-t4.md @@ -0,0 +1,226 @@ +# Implementation spec: generated annotation, more kinds, and quality control + +This is phase 4 of [the training and curation plan](plan.md). It follows that plan's rule: Tuldok supplies a file contract, a reader and a view; a project's own script (or a model behind the Pumas endpoint) makes the annotations. It builds on [spec-d4.md](../definition-first-projects/spec-d4.md) (tasks, the line file, the working copy, the steps, the Annotate tab) and [spec-d5.md](../definition-first-projects/spec-d5.md) (the Pumas endpoint and suggestions); the viewers are spec-d1's (`inside.js`), and the page follows [spec-w4.md](../definition-first-projects/spec-w4.md). Section 1 of [spec-m1-m2.md](../definition-first-projects/spec-m1-m2.md) applies. + +**Owner, 2026-11-19:** "Tuldok also needs a strong set of annotation types, however i dont nessisarly want everything that Label Studio has beacuse Tuldok centers on generated annotations rather than human made ones. That means we can do things like automated/ai segmentors, polygons, depth mapping, image masks, ranking, time series, object tracking, point cloud objects (selecting points that belong to an object), some quality control tools like agreement betwene different models, etc." + +**After this phase** a task may be of nine more types. A project's annotator (its own script, or a model behind the Pumas endpoint) is built as a dataset with the step `annotate`; its product is an **annotation set** for one task. A task can have several sets side by side (one per annotator) and the **kept set**: the annotations in `annotations/<task>.jsonl` in Git, which people decide. Tuldok measures how far the sets agree, lists the records where they disagree most, draws each set over the record, and a person keeps one set's annotation, both, or neither; that decision is written by the existing working copy and Save. People review instead of drawing. + +**Words.** As spec-d4: record, annotation, task, term. An **annotator** is what made a set: a model, a script or a person. A **set** is the annotations of one annotator for one task. The word "label" is not used. + +## 1. The new types of task + +A task's `task` may now also be one of these. Coordinates on an image are pixels with the origin at the top left, x right, y down; a point may lie on the far edge (`x ≤ width`). + +| Task | An annotation's `value` | On records of kind | Vocabulary | +| --- | --- | --- | --- | +| `polygons` | a list of `{"term", "points": [[x, y], …]}`, 3 to 10,000 points each, inside the image | image | required | +| `mask` | an object: `{"file": <sidecar .png>}` or `{"rle": [[v, n], …]}`, and `"classes": {"<v>": term, …}` | image | required | +| `depth` | an object `{"file": <sidecar .npy>}` | image | none | +| `keypoints` | a list of `{"term", "points": [p, …]}`, one `p` per name of the task's `keypoints`, each `[x, y]`, `[x, y, v]` (v 0, 1 or 2 as COCO's visibility) or `null` | image | required | +| `ranking` | a list of different terms, best first, at least one | any but `file` | required: the candidates | +| `timeseries` | a list of `{"term", "start", "end"}`, `start` < `end`: seconds of a sound, or positions along the first axis of an array | audio, array | required | +| `track` | a list of `{"track", "term", "frames": [{"frame", "x", "y", "w", "h"}, …]}`, frames increasing; `track` an id of 1 to 64 characters, different in one annotation | array (frames on its first axis) | required | +| `points` | a list of `{"term", "indices": [i, …]}`, indices increasing, each below the cloud's point count; or an object `{"file": <sidecar .npy>, "classes": {"<id>": term}}` of one whole number per point (0: no object) | pointcloud | required | +| `box3d` | a list of `{"term", "center": [x, y, z], "size": [l, w, h], "yaw"}`, sizes above zero, `yaw` in radians about z | pointcloud | required | + +- **Fields of the task** for these types: `keypoints` (1 to 100 different names, required for and only for `keypoints`), `skeleton` (pairs of those names, optional), `units` (a string of 1 to 40 characters for `depth`, such as `m`, required for it), `mask` (`semantic` or `instance`, default `semantic`, only for `mask`). +- **Confidence.** Any item of a list value, and the object of a `mask`, `depth` or `points` file value, may carry `"confidence"`: a number from 0 to 1. A set's line may carry one too (section 3). +- **A mask.** `classes` maps a pixel value (1 to 65,535, written as its decimal text) to a term; 0 is nothing. In a `semantic` mask a pixel's value is its class; in an `instance` mask each value is one object and `classes` names its term. `rle` is the mask row by row from the top left as runs `[value, count]`, counts above zero, summing to width × height. A sidecar PNG is 8- or 16-bit greyscale, not interlaced, of the image's size. +- **A depth map** is a 2-D `.npy` of float32, float64 or uint16 of shape (height, width); 0 and values that are not finite mean no depth; its units are the task's. +- **A point-id file** is a 1-D `.npy` of whole numbers (8 to 64 bits), one per point of the cloud. +- **Checked against the record** where cheap: an image's size (polygons, keypoints, masks, depth maps), a cloud's point count (points), an array's shape (track: frames below its first axis's length, boxes inside its second and third; timeseries on an array: ends at most its first axis's length), a sound's length (timeseries, from WAV only, as segments). +- **Bounds:** 1,000 items in a list value (as spec-d4), 10,000 points in a polygon, 1,000,000 indices in a `points` value, 10,000 frames in a track, 4,000 runs of `rle` per image row on average and 16 Mpx for a mask or depth map, 256 classes. +- **Key order** of an item: `track`, `frame`, `term`, `x`, `y`, `w`, `h`, `start`, `end`, `text`, `points`, `indices`, `center`, `size`, `yaw`, `frames`, `confidence`; of an object value: `file`, `rle`, `classes`, `confidence`; `classes` by number. Inside `frames` the same order. + +`spans`, `segments`, `boxes`, `class`, `tags`, `text` and `review` are unchanged, but their list items may carry `confidence` too. + +## 2. Sidecars: payloads too large for a line + +A value that names a `file` names a **sidecar**: a file beside the annotations it belongs to. + +- **In the kept set** (Git): `annotations/<task>/<name>`, where `<name>` is the first 16 hexadecimal characters of the file's SHA-256 and `.png` or `.npy`. Content-addressed: the same mask kept twice is one file, a name never changes meaning, and a file is never rewritten. The directory holds such files only (no links, no subdirectories). +- **In a set made by an annotator:** `files/<path>` of its product, any relative path. +- `check`: a sidecar directory with no task of its name is an error; a line naming a sidecar that is not there is an error; a file of the directory no line names is a warning ("Save removes it"). +- **Kept from a set** (section 6), a sidecar is copied into the working copy (the database), and written to `annotations/<task>/` by Save, with the line that names it; Save also removes the files of that task's directory that no saved line names. A sidecar is at most 64 MiB. + +## 3. The annotation set: what an annotator writes + +The product of a dataset whose `run.step` is `annotate`: + +``` +annotator.json {"format": "tuldok_annotator_v1", "task": "annotations/<task>", "by": {...}, "records": n, "made": m, "at": "…Z"} +annotations.jsonl one line per record annotated, sorted by record +files/ the sidecars its lines name +``` + +A line is `{"record", "sha256", "value", "confidence"?, "note"?}`: the record's id, its hash as the input's `records.jsonl` gives it (or null), a value as section 1 and spec-d4 say for the task (never null), an optional confidence for the whole annotation, and an optional note of at most 4,000 characters. + +**Provenance.** `by` is `{"kind": "model" | "script" | "person", "name", "version"?, "settings"?}`: names of 1 to 200 characters, `version` up to 200, `settings` an object of at most 8 KiB. It holds for every line of the set. A line kept from a set into Git carries it in `from` (section 6), so the kept annotation still says what made it. + +## 4. The step `annotate` + +A dataset whose `run` has `step: "annotate"`, `inputs: {"records": <the task's dataset>}`, `annotations: [<one task>]`, and settings: + +| | | +| --- | --- | +| `by` | required: `kind`, `name`, `version` as section 3 (Tuldok adds `settings`: the run's settings without `by`) | +| `pumas` | `{"model", "profile"?, "instruction"?}`: ask a model the endpoint serves. Excludes `run.command` | +| `limit` | at most this many records, 1 to 1,000,000 (default all) | +| `options` | any JSON object of at most 8 KiB, passed to the command (in `TULDOK_ANNOTATE_OPTIONS`) and recorded in `by.settings` | + +Exactly one of `run.command` and `settings.pumas`. Unlike `import`, `freeze` and `generate`, an `annotate` with a command also takes `run.environment` (and `env`), since the command is the project's: **the project's code runs in its environment, not in Tuldok.** + +**With a command.** Tuldok's step process (as for every step) reads the input's records, writes the records still to do to `<run>/todo.jsonl` (lines of the input's `records.jsonl`, the files' paths made absolute), then runs the command, from the code snapshot, with the environment's interpreter, with: + +| Variable | | +| --- | --- | +| `TULDOK_RECORDS` | the input product's directory | +| `TULDOK_TODO` | the records to annotate | +| `TULDOK_TASK` | the task's definition, as a file | +| `TULDOK_ANNOTATIONS` | the file to append lines to: `<output>/annotations.jsonl` | +| `TULDOK_FILES` | the directory for sidecars: `<output>/files` | +| `TULDOK_ANNOTATE_OPTIONS` | `settings.options` as JSON, or `{}` | + +The command appends one line per record, in any order, and may print `tuldok_progress_v1` lines (they pass through). When it ends with status 0, the step reads what it wrote, keeps the last line of each record of the input with a valid value, counts and drops the rest (an unknown record, an invalid value, a sidecar that is missing or outside `files/`), writes the lines sorted, and `annotator.json`. A command that fails fails the step. + +**With the Pumas endpoint.** Only what a served model can produce, from what spec-d5 already asks: + +| Task | Record | Asked as | +| --- | --- | --- | +| `text`, `class`, `tags` | text, image, audio | as `suggest_annotation` (spec-d5 section 4, as built) | +| `boxes`, `polygons` | image | `image_to_text` with an instruction asking for a JSON list of `{"term", "x", "y", "w", "h"}` (or `points`) in pixels of the image of the size given, terms from the vocabulary; an answer that is not such a list is a refusal for that record | + +A record the model's answer does not fit is counted as refused and skipped; the step fails only when the endpoint cannot be reached or refuses every record of the first ten. `by` gets `kind: model`, the model, profile and capability. + +**Bounded, resumable, with progress.** Progress is records done of records to do. Lines are appended as they are made. A run of the same definition that failed or was cancelled leaves its lines: the next build starts from them (`--resume`), copying the valid lines (and their sidecars) of records whose hash is unchanged, and annotates only the others. A run never reads or writes outside its output and run directory. + +**`by.settings`** is the run's `settings` without `by`, so a set's provenance is what its definition says; the definition, in Git, is what made it. + +## 5. Quality control: agreement between sets + +The **sets of a task** are its kept set (`kept`: the working copy's annotations with a value, in any state but rejected) and the current product of every dataset that annotates it (`annotate` with `annotations` naming it), each named by that dataset's reference. + +**Pairs.** For two sets, a record is compared when either has an annotation for it. One without an annotation (or with one emptied by the confidence filter, below) scores 0 on that record and is listed as missing. Each record gets an **agreement** from 0 to 1; a pair's agreement is the mean over its records. With more than two sets, a record's agreement is the mean over the pairs that compare it, and its **disagreement** is 1 minus that. The **most disputed** records are those of greatest disagreement, ties broken by record id. + +**Per type** (what one record scores for one pair, and what the pair also reports): + +| Task | A record's agreement | The pair also reports | +| --- | --- | --- | +| `class` | 1 when the terms are equal, else 0 | Cohen's kappa over the records both annotated; with three or more sets, Fleiss' kappa over the records every set annotated | +| `tags` | Jaccard of the two term sets (1 when both are empty) | | +| `text` | `difflib.SequenceMatcher` ratio of the first 4,000 characters | exact matches | +| `ranking` | (τ + 1) / 2, Kendall's τ over the candidates both rank (1 when fewer than two are shared and those agree) | mean τ | +| `boxes`, `polygons`, `box3d`, `points` (both forms), `mask` (`instance`) | matched IoU: objects of the same term matched one to one maximising the summed IoU; the sum of the matched IoUs over the larger of the two counts (1 when both are empty) | mean IoU of matches, matches, F1 at IoU 0.5 | +| `mask` (`semantic`) | mean over the terms either uses of their pixel IoU | | +| `keypoints` | as matched IoU with OKS in place of IoU | mean OKS of matches | +| `depth` | 1 − min(1, mean ‖a − b‖ / mean (‖a‖ + ‖b‖) / 2) over pixels with depth in both | mean absolute difference in the task's units, δ < 1.25 | +| `spans`, `segments`, `timeseries` | mean over the terms either uses of the interval IoU (length of the intersection of the unions over that of the union) | | +| `track` | tracks of the same term matched as above by their track IoU (the mean box IoU over the frames either has, a frame one lacks scoring 0) | identity switches, fragments | +| `review` | not compared | | + +- **IoU.** Boxes: exact. Polygons: both drawn on a grid over their joint bounding box at most 128 × 128 cells, pixel centres, even-odd rule (so IoU is exact to the grid). Masks: by pixel, decoded. Point sets: of the index sets (a point-id file becomes one object per id). 3-D boxes: the area of the intersection of the two rotated rectangles seen from above (convex clipping) times the overlap of their heights, over the union of volumes. +- **Matching.** The Hungarian method (minimising 1 − IoU) when both sides have at most 64 objects of a term; otherwise greedy, the pair of greatest IoU first. Pairs with IoU 0 are not matches. +- **OKS** of two instances: the mean over the keypoints both have (visibility above 0 where given) of exp(−d² / (2 s² κ²)), s² the mean of the areas of the two instances' bounding boxes of their points (at least 1 px²), κ = 0.1 (the task may say `sigma`). A keypoint one has and the other lacks scores 0. +- **Tracks.** An identity switch: for a track of the first set, its box in each frame is matched to the box of greatest IoU (above 0.5) of the second set; a change of the matched track from one matched frame to the next counts one. A fragment: a gap in a track's frames (its frames are not consecutive). Both are counted per set and per pair. +- **Kappa.** Cohen's: (p_o − p_e) / (1 − p_e) with p_e from each side's term frequencies; 1 when p_e is 1. Fleiss': with n sets per record, P̄ from the agreeing pairs per record and P̄_e from the overall term shares; 1 when P̄_e is 1. +- **Confidence filter** (`min_confidence`): items below it are dropped from list values; a line (or a mask, depth or point-id object) below it is treated as no annotation. A value with no confidence passes. +- **Bounds.** At most 8 sets; the agreement reads at most 20,000 records per set and stops there, saying so; masks and depth maps over 16 Mpx are not compared (listed with the reason). + +## 6. Operations + +All in the `project_folder` profile, absent from `catalog` (its digest unchanged). + +| Operation | Who | Arguments | Result | +| --- | --- | --- | --- | +| `annotation_sets` | inspect | `task` | `task`, `type`, `sets`: each `set` (`kept` or a dataset reference), `title`, `by`, `run`, `annotated`, `skipped` (lines not valid), `problem` (a set that cannot be read, with why) | +| `annotation_agreement` | inspect | `task`; `sets` (2 to 8; default every set that can be read); `min_confidence`; `offset`, `limit` (of the disputed list, at most 200) | `task`, `type`, `sets`, `measure` (what the score is, in words), `pairs` (`a`, `b`, `records`, `agreement`, and the type's figures), `fleiss`, `records` (compared), `disputed` (`record`, `disagreement`, `pairs` {`a|b`: agreement}, `missing`), `total`, `offset`, `limit`, `truncated` | +| `annotation_compare` | inspect | `task`, `record`, `sets` (1 to 8), `min_confidence` | `task`, `type`, `record`, `sets`: each `set`, `value` (null when none), `confidence`, `drawn` (masks as `{width, height, rle}` by term, depth as `{width, height, low, high, values_u8}` downsampled to at most 256 px a side, point-id files as `points` lists), `problem`; `pairs` (this record's agreement per pair and the type's figures); `frames` (for `track`: the array's frames as 8-bit grey, at most 64 frames of at most 128 px a side) or `series` (for `timeseries` on an array: at most 1,024 positions of at most 8 channels) | +| `keep_annotation` | edit, changes | `task`, `record`, `keep` (`one`, `both` or `none`), `sets` (one set for `one`, two for `both`, any for `none`), `note` | `task`, `record`, `state`, `unsaved` | + +`keep_annotation` writes the kept set through the same path as `set_annotation` (checked against the task and the record, unsaved until Save, one activity row): `one` keeps that set's value as `accepted` with `from` `{"set", "run", "by"}` (`by` without `settings` when it would pass 2 KiB); `both` keeps the union of two list values (refused `invalid` for a type whose value is not a list) with `from` naming both; `none` keeps `rejected` with a null value and `from` naming the sets rejected. A kept value's sidecars are copied (section 2). `kept` cannot be named for `one` or `both` (it is already kept). + +## 7. The page + +**The Annotate tab gains a second mode, Review**, chosen in the bar when the task has at least one set made by an annotator. Annotate mode stays as spec-d4 built it; for the new types its form shows the kept value drawn over the record and the decision, with no drawing tools. + +Review mode, in the tab's three columns: + +- **Left: the agreement and the most disputed records.** Two set choosers (A and B; any two of the task's sets), the confidence filter, the measure in words, A–B's agreement and the type's figures (with Fleiss' kappa when there are three or more sets), then the disputed records in order, each with its disagreement as a number and a bar; selecting one shows it. +- **Middle: the record with the sets drawn over it.** Side by side (A left, B right) or overlaid (A and B in two colours, what only one has marked), switched by a toggle. On an image: boxes, polygons, keypoints (with the skeleton), masks (as colour by term or instance, half transparent), depth (as a colour map, with its scale). On a point cloud: each point object's points coloured by its term, the 3-D boxes as wireframes. On a sound or an array of one or two axes: timeseries, segments as bands under the waveform or the series. On an array of frames: the frame with each track's box and id, a frame slider and the arrow keys. Text: spans marked. +- **Right: the decision.** Each set's value in words (count of objects, terms, confidences), this record's agreement, **Keep A**, **Keep B**, **Keep both** (for list values), **Reject both**, a note; then **Save** (the strip's). A record already kept shows what was kept and from which set. +- **Light correction**, only on the kept value and only where cheap: a kept polygon's vertex can be dragged or deleted (select it, then Delete); a kept point object's points can be removed by dragging a rectangle on the cloud's view while "Remove points" is on. Each correction is kept with `set_annotation`, as an unsaved edit. + +Updating in place (spec-w4): the disputed list, the record and the decision are drawn in parts, each put in place only when what it shows differs; a reading that changed nothing redraws nothing; the choice of sets, the filter, the mode and the frame are the page's and survive readings. + +## 8. Slices + +| | Slice | Mainly | +| --- | --- | --- | +| T4a | The nine types, their fields, values and checks against the record; sidecars in the folder, `check` | `annotations.py`, `schemas/annotation.schema.json`, `annotation_records.py` (`facts`), `folder.py`, `store.py` (listing), `check.py` | +| T4b | The set format and its reader; the step `annotate` with a command and with Pumas; resume | new `annotation_sets.py`, `steps.py`, `builds.py`, `check.py`, `schemas/` (run) | +| T4c | Agreement: the measures, matching, the disputed list | new `annotation_agreement.py` | +| T4d | The four operations; the sidecar copy and Save | `operations.py`, `service.py`, `annotation_rows.py`, `store.py` | +| T4e | Review mode and the overlays, light correction | new `static/project-workspace/review.js`, `annotate.js`, `inside.js` (exports), `base.css`, `app.py`, `project-workspace.html` | +| T4f | The example's small data and stand-in annotators; the browser check | `examples/tiny-project`, `tests/` | + +`docs/contracts/project-folder.md` is brought up to date for the types, sidecars, sets and the step. + +## 9. Tests + +- **T4a,** `tests/test_tuldok_project_annotation_kinds.py`: a valid value of each type checks and each rule of section 1 broken once gives its error; checks against an image's size, a cloud's count, an array's shape and a sound's length; a sidecar PNG and `.npy` read and refused when of another size or type; RLE that does not sum; a line file of each type written and read back byte for byte, key order included; a sidecar directory without its task, a missing sidecar and an unreferenced one in `check`. +- **T4b,** `tests/test_tuldok_project_annotate_step.py`: two stand-in segmentors (scripts in the test, run in a test environment) annotate four small images and two small clouds of a dataset built by `import`; each set has the lines, sidecars and `annotator.json` expected; a line with a bad value or unknown record is dropped and counted; a failed run is resumed and annotates only the rest; the Pumas route against the fake endpoint for `class` and for `boxes`, an answer that is not a list refused; nothing contacts a real Pumas. +- **T4c,** `tests/test_tuldok_project_annotation_agreement.py`: every measure of section 5 against the test's own arithmetic on hand-made values: kappa and Fleiss on a known table, Kendall's τ, interval IoU, box, polygon (a square in a square), mask (instance and semantic), point-set and 3-D box IoU (a rotated pair), OKS, track switches and fragments, the Hungarian matching against brute force, the confidence filter, the disputed order. +- **T4d,** in the same file and `test_tuldok_project_service.py`: the four operations on the stand-ins' sets; `keep_annotation` one, both and none, Save writing the line and the sidecar, a second Save removing an unreferenced sidecar; refusals. +- **T4e and T4f,** `tests/browser_project_workspace_annotation_kinds.cjs`: on a project whose stand-ins have run, overlays drawn for each type of the fixture (masks, polygons, depth, keypoints, point objects, 3-D boxes, timeseries, tracks across frames), two sets compared side by side and overlaid, the agreement shown, the disputed list in the computed order, one record reviewed and kept (Save writes `annotations/`), a polygon vertex moved, nothing rebuilt by a reading that changed nothing; light and dark at 1440 and 390 px, nothing overflowing sideways, no script error. + +## 10. Out of scope + +A drawing editor (drawing masks, polygons, keypoints or 3-D boxes by hand); video files as records (frames are an array); agreement for `review` tasks; exporting sets to COCO or other formats; per-term confusion matrices; a set made by a person outside the kept set (a second person's set is an annotator's product like any other); weighting annotators. + +## As built + +What differs from the text above, and what was chosen where it was silent. + +### T4a, the types and the sidecars + +- **The fields a task's values are checked against** (`keypoints`, `skeleton`, `units`, `mask`, `sigma`) are passed as `annotations.task_fields(document)`; `value_problems`, `validate` and `read` take them, and the working copy's fingerprint of a task now includes them (the fingerprint of a task without them is unchanged, so no row is judged again on upgrade). `check` also refuses a skeleton naming a point that is not in `keypoints`. +- **Sidecar names** are the first 16 hexadecimal characters of the SHA-256 and `.png` or `.npy`. The folder reader lists `annotations/<task>/` (names and sizes, nothing read); a file not named that way is an error, as a link or a subdirectory is. The store's listing skips such a directory when its task exists, and never compares it for changes on disk: content-addressed files are never rewritten. +- **The byte formats** are in a new pure module, `annotation_payloads.py` (the step process imports it): a PNG reader for 8- and 16-bit greyscale with every row filter, a PNG writer, runs (`[value, count]`, row by row), and an `.npy` reader and writer (C order, floats and whole numbers; never unpickled). `rle` in a value is checked for its shape in a line, and against the image's size only where the size is known (an edit, a set's line in the step). +- **A test changed:** `test_tuldok_project_annotations.py` used `polygons` as its example of an unknown task; it now uses `video`. +- **Not checked against the record:** a JPEG's size in the step (PNG only); a sound's length in the step (the set reader in Tuldok checks it as before for `segments`; `timeseries` on a sound is checked when kept); `rle` in the folder check (the image's size is not known there). + +### T4b, the step `annotate` + +- **`run` in all four run schemas:** `step` gains `annotate`; `command`, `environment` and `env` are refused only for `import`, `freeze` and `generate`; `annotations` belongs to `freeze` or `annotate`. `check` asks for one input `records`, one task, exactly one of `run.command` and `settings.pumas`, `by` as section 3, `limit` and `options` in their bounds, and that the task annotates the input. +- **The command's process.** Tuldok's step process (`python -m tuldok_project.steps annotate … --command-json …`) runs the command with `subprocess.call`, from the code snapshot, its argv filled and its program found as a build's are (`python` is the environment's interpreter). The command's environment is the run's without `PYTHONSAFEPATH` and `PYTHONPATH` (which make Tuldok's own code importable to the step); a `PYTHONPATH` the run's `env` set is given back to it. Its output goes to the run's log, so its progress lines are the run's. The task file is `<run>/tasks.json` (`{reference, document}`) and `<run>/task.json` (the document) for the command. +- **Resume** takes the latest run of the same reference whose state is neither succeeded, running nor queued and whose definition text is the same, when its output holds `annotations.jsonl`. +- **What is dropped and counted:** a line that is not JSON, not a line of section 3 (unknown keys, no record, no value), a record the input lacks, a value the task does not allow (null included), a sidecar missing, outside `files/` or not of the record's size or point count; a record annotated twice keeps its last valid line. +- **The Pumas route** asks `pumas.suggest` record by record (`by.model` records model, profile and capability); `boxes` and `polygons` on an image are asked of `image_to_text` with an instruction naming the image's size and the terms, and the first JSON list in the answer (a fenced block too) is kept when it is a value of the task. The step fails when each of the first `min(10, records)` is refused. `pumas.routes` now offers this route to `suggest_annotation` and `suggestion_models` as well; the page still asks only for `text`, `class` and `tags` (`test_tuldok_project_pumas.py`'s table of routes was brought up to date). A plan marks an `annotate` with `pumas` blocked when no endpoint is set, as a `generate`. +- **Not verified:** a real Pumas; polygons asked of a model (only boxes against the fake); cancelling a run while its command runs (the launcher stops the process group, as for any run); a set of many records. + +### T4c, agreement + +- As section 5. **Polygon IoU** is exact to the grid (1 px cells up to 128 px across, coarser above). **3-D IoU** clips the footprints with Sutherland–Hodgman. **Greedy matching** above 64 objects a term per side. A pair's figures are pooled over its records: mean IoU weighted by matches, F1 averaged per record. +- **Not measured:** time on large sets. Masks and depth maps are compared pixel by pixel in Python (a second or so per megapixel, by estimate). + +### T4d, the operations + +- **`keep_annotation`** reads the sets outside the service's lock, stages a kept sidecar in a new table `annotation_sidecars` (task, name, bytes), then goes through `_prepare_annotations` and `_set_annotation` as `set_annotation` does: the same checks against the record, one activity row (`"<record> accepted"` or `rejected`). `from` is `{"set", "run", "by"}` for one set, `{"both": [...]}` for two, `{"rejected": [...]}` for none, each cut to the 2 KiB `from` allows (first `by.settings`, then `by`). Keeping both tracks renames the second set's repeated track ids (`-2`, …). +- **Save** writes a task's staged sidecars that its saved lines name, removes from `annotations/<task>/` the content-named files no saved line names (and the directory when empty), and drops the task's staged rows. A revert leaves staged rows until that task's next Save. +- **The kept set** for agreement and review is the working copy's annotations with a value and not rejected, unsaved ones included, stale ones not left out. +- **Operations:** 76 (four new, `keep_annotation` a mutation); the catalog profile is unchanged. + +### T4e, the page + +- **Review** is a mode of the Annotate tab (`Annotate | Review` in the bar, offered when a task has a set made by an annotator); `review.js` (`window.LabReview`) draws it into the tab's own three columns, and `review.css` styles it (base.css is untouched). `annotate.js` gained the nine types (New task asks for keypoints, units and the mask's kind), the mode, and, for the new types in Annotate mode, the kept value in words and drawn read-only over an image or a point cloud. +- **Colours:** A is `--ramp-neg`, B `--ramp-pos`, both `--good`; one set alone is coloured by term. Overlaid, an object with no counterpart of the same term at a box IoU of 0.5 is drawn dashed (a quick greedy test in the page, for marking only); masks are coloured A only, B only and both by pixel; depth maps show A − B on the diverging ramp. +- **inside.js** exports `space` and `turned`; `space` draws `m.lines` (wireframes) and, with `m.select` and `st.selecting`, a dragged rectangle whose shown points it reports (`stride` is accounted for by the caller). +- Each panel carries `data-drawn` (what it drew per set), read by the browser check. +- **Not covered:** keyboard-only correction of polygons (the vertex is chosen and dragged by pointer; Delete needs the canvas focused); a session that may only inspect (the decision buttons are hidden by the usual rule); more than 200 disputed records listed (the list says how many are not); screen readers. + +### T4f, the example and the checks + +- `examples/tiny-project` gained `code/make_scenes.py` (writing `code/scenes`, 3,897 bytes: three 24 × 16 images, two point clouds of 20 and 30 points, 6 frames of 16 × 20, a series of 48 × 2), `code/annotators/standin.py` (one stand-in for mask, polygons, depth, keypoints, points, box3d, timeseries and track), `datasets/scenes`, `annotations/scene-masks` and `scene-objects`, and four annotators. `check` reports nothing. The example's annotators were not built here: its environment `cpu` needs numpy, which this checkout's interpreter lacks; the same stand-in is built by the browser check in an environment of its own. +- **Tests:** `test_tuldok_project_annotation_kinds.py` 15, `test_tuldok_project_annotate_step.py` 6, `test_tuldok_project_annotation_agreement.py` 21; project suite 828 (786 before), agent interface 43 (4 skipped). `tests/browser_project_workspace_annotation_kinds.cjs` passes (steps 1 to 6 of its header, 4b being the point correction; 4b's rectangle happens to take all nine points of the kept object, so a partial removal is not shown). +- **Regression checks:** `browser_project_workspace.cjs` passes (24). `browser_project_workspace_inside.cjs`, `_annotate.cjs` and `_endpoint.cjs` fail here exactly as they do on the base of this branch (`7a0b3665`, run from an archive of it): at 1000 px the page overflows sideways by 65 px (`page` 65, the main area 0), and `_annotate.cjs` also still expects the tab list without phase 3's Compare. With that one expectation updated (in a throwaway copy), `_annotate.cjs` passes the same nine steps before the same overflow on both trees; `_inside.cjs` and `_endpoint.cjs` pass six steps on both. The overflow is not this phase's and was not fixed here. diff --git a/docs/plans/training-and-curation/spec-t5.md b/docs/plans/training-and-curation/spec-t5.md new file mode 100644 index 00000000..07e07fca --- /dev/null +++ b/docs/plans/training-and-curation/spec-t5.md @@ -0,0 +1,164 @@ +# Implementation spec: curation by embeddings + +This is phase 5 of [the training and curation plan](plan.md). It follows that plan's rule: a file contract, a reader and a view in Tuldok, and no embedder of Tuldok's own. The readers follow [spec-d1.md](../definition-first-projects/spec-d1.md) and [spec-d6.md](../definition-first-projects/spec-d6.md): standard library only, nothing unpickled, nothing of the project's code run, nothing written, bounded reads, a damaged file refused and not a failure, reads outside the service lock. The step follows [spec-d4.md](../definition-first-projects/spec-d4.md) section 6 and [spec-d5.md](../definition-first-projects/spec-d5.md) section 3 (`generate`): Pumas is reached only through `tuldok_project/pumas.py` and the 0.8.0 client of [spec-d3.md](../definition-first-projects/spec-d3.md). The page follows [spec-w4.md](../definition-first-projects/spec-w4.md): nothing is rebuilt by a timer or a return to the window. + +**Owner, 2026-11-19:** "Tuldok should have curarion by embeddings. Tuldok needs to have great curaiton tools for datasets in order to build high quality training data." And: "Set embeddinggemma 2 as the default embbedder for Pumas." + +**After this phase** a dataset that has embeddings opens on the Data tab as a map: its records as points in two dimensions, coloured by split, kind, an annotation the records carry or any field of theirs; beside it the near-duplicates, the outliers, the thin parts of the map, the near-copies between the training split and the others (leakage), and, for the record opened, the records most like it. A region chosen on the map becomes a new dataset of exactly those records, or opens Freeze with them. Embeddings are made by a project's own script, or by a new step `embed` that asks a model the Pumas endpoint serves, EmbeddingGemma 2 unless the definition or the machine names another. Tuldok never computes an embedding. + +## 1. The file: an embeddings product + +A product holds a set of embeddings `<name>` (1 to 64 characters of `a-z 0-9 . _ -`, starting with a letter or digit) as two files under `embeddings/` at its top: + +- **`embeddings/<name>.npy`:** one row per record, shape `(rows, dimensions)`, `float32` (little- or big-endian; `float16` and `float64` are read too), C order, every value finite, no row all zeros. +- **`embeddings/<name>.json`,** the index, UTF-8 JSON of at most 16 MiB: + +```json +{"format": "tuldok_embeddings_v1", + "records": ["a1", "a2", "a3"], + "dimensions": 256, + "metric": "cosine", + "model": {"name": "embeddinggemma-2", "model_id": "unsloth/embeddinggemma-2-GGUF", "profile": "embed-gpu", + "capability": "text_embedding", "via": "pumas"}, + "prefix": "task: clustering | query: ", + "source": {"dataset": "datasets/photos", "run": "4f1c…"}, + "projection": {"file": "photos.map.npy", "method": "umap"}, + "notes": ["2 point clouds were not embedded: no model the endpoint serves takes them."]} +``` + +- `records`: the record ids in row order, 1 to 50,000 different ids of 1 to 600 characters without control characters (a product read from its files names its records by their addresses, spec-d1). +- `dimensions`: 1 to 4,096, the second axis of the `.npy`. `metric`: `cosine` (the only one). +- `model`: an object with `name` (1 to 256 characters) and any other short facts (strings, numbers, booleans or null), shown as given. +- `prefix`: the text put before each record's text, or null. EmbeddingGemma expects a task prefix (Pumas: "task prefixes … are part of the caller's text"). +- `source`: which records the rows are: `dataset` (a reference, or null for the product's own records) and `run` (the run whose product holds them, or null when the script cannot know it; section 3 finds it). +- `projection` (optional): a two-dimensional map the script made (UMAP, t-SNE, anything), a `.npy` of shape `(rows, 2)` beside the index, and its `method` (1 to 64 characters). Tuldok draws it instead of its own. +- `notes` (optional): at most 64 sentences of at most 1,000 characters. + +Other keys are kept and ignored. Bounds: rows × dimensions at most 40,000,000 values (160 MB of `float32`); at most 64 sets in one product. A set breaking a rule is listed with the reason and refused `unsupported` when read; a `.npy` over the bound `too-large`. + +**Who writes it.** A project's own script, in the product of a dataset (its own records' embeddings, `source.dataset` null) or of a dataset whose run takes the records' dataset as an input (`source.dataset` names it). Or the step `embed` (section 5). The contract goes into `docs/contracts/project-folder.md`. + +## 2. The default embedder + +When nothing names an embedding model, Tuldok uses **EmbeddingGemma 2**: the served (model, profile) whose alias or `model_id` contains `embeddinggemma-2`, ignoring case, preferring the `model_id` `unsloth/embeddinggemma-2-GGUF`, then the endpoint's order, among those with an embedding capability available. When none is served the call is refused `not-found`, naming what the endpoint serves with an embedding capability; another model is never chosen silently. + +The machine's settings file (beside `pumas_endpoint`, `local.py`) may hold `embedding_model` (an alias, and optionally `embedding_profile`) to use instead of EmbeddingGemma 2, and `embedding_prefix` (default `task: clustering | query: `). A step's own `model` and `prefix` come first. The order is: the definition, the machine's setting, EmbeddingGemma 2. + +- **`endpoint`** answers, besides what it did, `embedder`: `{model, profile, prefix, named_by` (`setting` or `default`)`, chosen` (the served entry it resolves to now, or null)`, problem` (why none, or null)`, embedders` (every served (model, profile) with an embedding capability available: `model`, `profile`, `model_id`, `capabilities`, `dimensions` (the advertised bound or null)`)}`. +- **`set_embedder`** (owner, changes): `model` (an alias, or null for EmbeddingGemma 2), `profile`, `prefix` (null for the default prefix). A model that the endpoint does not serve with an embedding capability is refused; the prefix is at most 1,000 characters. Answers `endpoint`'s `embedder`. + +## 3. Finding a dataset's embeddings + +**`embeddings`** (inspect), argument `dataset` (a reference). The dataset's current products, and the current products of every dataset whose `run.inputs` names it, are looked through for `embeddings/*.json`. A set belongs to the dataset when its `source.dataset` names it, or when it lies in the dataset's own product with `source.dataset` null. Its **records run** is `source.run` when given; else the run holding it when the records are its own; else the run the holding run recorded for that input. The answer: `dataset`, `run` (its first current product's run, or null), and `sets`, each `run` (holding it), `holder` (the dataset holding it), `name`, `rows`, `dimensions`, `model`, `prefix`, `source`, `records_run`, `current` (the records run is a current product of the dataset), `projection` (its method or null), `notes`, `problem` (null, or why it cannot be read). + +## 4. Reads + +Five reads, `inspect`, in `INSIDE_READS` like the others, in the `project_folder` profile and absent from `catalog` (whose digest is unchanged). Each takes `run` (holding the set), `name`, and `records_run` (whose records the rows are; default `run`), which joins the product arguments the service finds under its lock. + +| Operation | Arguments | Result | +| --- | --- | --- | +| `embedding_map` | `colour` (`split`, `kind`, `annotation:<task>`, `field:<key>`; optional), `cells` (8 to 64, default 24), `thin` (0 to 1, default 0.25) | `name`, `rows`, `dimensions`, `model`, `method` (`pca`, or the script's), `explained` (the share of the variance on each axis, for `pca`), `sample` (rows the components were computed from), `ids`, `x_f32`, `y_f32` (base64), `bounds`, `missing` (rows whose record is not in the records product), `colours` (what can be chosen: `split`, `kind`, `annotation:<task>` for each task the records carry, `field:<key>` for each key of `meta` found), `colour` (`by`, `type` `category` or `number`, `categories` with counts, `codes_u16` or `values_f32` (base64, per row), or null), `grid` (`cells`, `counts` row by row), `thin` (cells) | +| `embedding_neighbours` | `record`, `limit` (1 to 200, default 12) | `record`, `neighbours` (`id`, `cosine`, `split`, `kind`), most alike first; exact | +| `embedding_duplicates` | `threshold` (0.5 to 1, default 0.98), `limit` (1 to 1,000, default 200) | `threshold`, `pairs` (`a`, `b`, `cosine`, `a_split`, `b_split`), most alike first, `total`, `groups` (how many records are in some pair), `complete`, `examined`, `rows` | +| `embedding_outliers` | `k` (1 to 50, default 5), `limit` (1 to 1,000, default 50) | `k`, `outliers` (`id`, `distance` (one minus the cosine to its k-th nearest record), `nearest`, `split`), farthest first, `median` (of every examined row's distance), `complete`, `examined`, `rows` | +| `embedding_leakage` | `threshold` (default 0.95), `train` (the split trained on, default `train`), `limit` | `threshold`, `train`, `splits` (each split's count), `pairs` (`record`, `split`, `near` (the training record), `cosine`), most alike first, `total`, `leaked` (records of other splits with a near-copy in training), `complete`, `examined`, `rows`; a product with no splits answers `splits` {} and no pairs | + +**Arithmetic,** in 64-bit floats with `math.sumprod`, `math.fsum` and `math.dist`. Rows are made unit length; cosine is their dot product. + +- **The map (`pca`).** The mean and the leading eight principal components (fewer when there are fewer dimensions or rows) of a sample of the rows, evenly spaced, of at most `max(256, 2^19 / dimensions)` rows: by power iteration on the sample's covariance (computed whole when there are at most 1,024 dimensions, else applied without forming it), eight vectors together, orthonormalised each turn and rotated to the covariance's own axes by a small Jacobi solve (Rayleigh–Ritz), until they move less than 1e-9 or after 200 turns. Every row is projected on the first two. `explained` is each component's variance over the total. Signs are fixed so that each component's largest entry is positive. +- **Bounding the pair searches.** Each row also gets its coordinates on all the components and the length of what they leave out (the residual). For two unit rows, the distance in those coordinates, with the residual lengths as one more, is never more than their true distance, so a pair whose coordinates are farther apart than the threshold allows is ruled out without reading its rows. The rows are sorted by their first coordinate and each is compared with those within the threshold's distance along it; what is not ruled out is computed exactly. Nothing found is approximate. +- **What bounds it.** A search examines rows in a fixed shuffled order and stops after 20,000,000 comparisons of coordinates or 400,000 exact ones; a row is examined whole or not at all, so what is listed is exactly right for the rows examined, and `complete` says whether every row was. Duplicates and outliers examine every row; leakage the rows of the splits other than `train`. +- **Outliers** are the rows farthest from their k-th nearest neighbour (exact, by the same sorting: the search around a row stops when the first coordinate alone is farther than its k-th nearest so far). +- **Thin regions** are the map's cells (a `cells` by `cells` grid over its bounds) that hold some points but at most `thin` times the median count of the cells that hold any; each with its place, count and its records (at most 20), fewest first, at most 64 cells. +- **Colour.** `split` and `kind` are the records' own; `annotation:<task>` the value a frozen record carries for that task (a term, the terms joined, or its state when it has no value); `field:<key>` a key of a record's `meta` (a number is a number, anything else a category). At most 24 categories, the rest one `other`; a row with no value has the code 65535 or NaN. +- **Kept.** What is computed for a set (its unit rows, components, coordinates, sorted order, the searches' answers) is kept while its two files are unchanged, at most two sets and 400 MB, least recently used dropped. + +## 5. The steps `embed` and `select`, and freezing a selection + +**`embed`.** A dataset whose `run` has `step: "embed"`, `inputs: {"records": <a dataset>}`, and `settings`, every one optional: `model` and `profile` (the served alias; section 2's order otherwise), `prefix` (at most 1,000 characters), `dimensions` (1 to 4,096; Pumas truncates and renormalises), `name` (of the set; default the input's name), `batch` (1 to 256, default 32). Output: `embeddings/<name>.npy` and `.json`, with `source` naming the input and its run, `model` naming what answered (`name`, `model_id`, `profile`, `capability`, `via: "pumas"`), and nothing else in the product. + +- **What is asked.** Text records (inline, or a UTF-8 file of at most 64 KiB) by `text_embedding` with `text_batch`, the prefix put before each text; PNG and JPEG images of at most 8 MiB by `multimodal_embedding` with `content_batch`, one image part an item. A batch is at most `batch` items, at most the advertised `input_count`, and for images at most the advertised `image_count` (4 without one) and 16 MiB. Options `{"kind": "embeddings", "dimensions": d}` only when `dimensions` is set; it is checked against an advertised `dimensions` bound when there is one. Records of other kinds, or that cannot be sent, are not embedded and are counted in `notes`. +- **Before anything is asked,** the step reads what the endpoint serves and fails, saying why, when the model is not served, the capability a kind of record needs is not available, or `dimensions` is past its bound. A dataset with records of a kind the model cannot embed fails before anything is asked unless it also has records it can; then those are noted. +- **The answer** must hold one finite vector per item, all of one length (`dimensions` when set). A refusal before the provider took it (`not_admitted`) is asked again twice, after 1 s and 2 s; anything else fails the step with the batch's first record. +- **Resumable.** Every batch answered is appended to `embeddings/.<name>.partial` (rows) and its ids to `.<name>.partial.json` (with the model, the input run, the prefix and the dimensions). When the dataset's last run failed, the build passes its output to the step, which takes the rows already answered from it when the input run, model, prefix and dimensions are the same, and asks only for the rest; the step says how many it took. +- **Progress** is the count of records embedded, in the existing form; the endpoint's address and version are kept in the run's record as for `generate`, never in the folder. Planning: without an endpoint the step is blocked as `generate` is. + +**`select`.** A dataset whose `run` has `step: "select"`, `inputs: {"records": <a dataset>}`, `settings: {"records": [ids]}` (1 to 20,000 different ids): its product holds exactly those records of the input's product, in the input's order, each line as the input has it (its split and annotations kept), its file linked under `files/`, its hash computed when the input has none. An id the input does not have fails the step, naming the first. This is what a selection on the map becomes. + +**Freeze** gains an optional setting `records` (1 to 20,000 ids): only those records of the input are taken, before the first task's states decide. "Freeze these" on the map opens the Freeze form with the dataset and the ids filled in. + +`check` checks the settings of both steps; the schemas list them; both are planned, requested, approved, run and recorded like any build. + +## 6. The page + +**The dataset opened** (`#data/inside:<reference>`, spec-d1) gains a third mode beside Records and Files, **Map**, when `embeddings` lists a set for it. With several sets, a select names each (`holder · name · model`). A set whose records run is not the dataset's current product says so. + +- **The map,** a canvas: the points, coloured by the choice (`Colour by`: none, split, kind, each annotation, each field), a legend with each category's count or the number scale's ends; the thin cells shaded; the selected record ringed, its most alike joined to it by lines. The wheel zooms about the pointer, dragging pans; "Reset view". +- **Hovering** a point shows its id and colour value in a tip; resting on it 300 ms shows the record in the viewer beside the map, with inside.js's viewers. A click opens it (pinned) and asks for its neighbours. +- **Choosing a region:** Box or Lasso (a toggle), then dragging. The chosen points are marked and counted. "New dataset from these" (title, name; Keep) keeps a `select` definition as an unsaved edit, as the other forms do, staying on the map; "Freeze these" opens the Freeze form filled in; "Clear". +- **Beside the map,** lists, each row a button that opens its record: **More like this** (the opened record's neighbours), **Duplicates** (threshold), **Outliers**, **Leakage** (when the records have splits; threshold and the training split), **Thin regions** (a cell's row selects its records). Each list is asked for when first shown and when its controls change. +- **The Embed form,** on the Data tab beside Import, Freeze and Generate: title, name, the dataset, the model (EmbeddingGemma 2 or the machine's setting, shown as what it resolves to now; or any served embedder), prefix (the default shown), dimensions (with the advertised bound), batch. It keeps an `embed` definition as an unsaved edit. With no endpoint, or none that embeds, it says so. +- **The endpoint's drawer** on Train shows the embedder (what the machine's setting or the default resolves to), and for the owner a select of the served embedders ("EmbeddingGemma 2 (default)" first) and the prefix, with Set. +- Every control is a button, select or field; the canvas has a text alternative and is reachable by Tab, with the arrow keys moving through the points nearest in the map. Light and dark from the tokens of `base.css`. Updating in place: the map's answers are not watched; a reading that changed something draws the view again in place keeping the set, colour, zoom, selection and the lists' controls; a timer or a return to the window with nothing changed touches nothing. + +## 7. Slices + +| | Slice | Files | +| --- | --- | --- | +| T5a | The contract, the reader and the six reads | `docs/contracts/project-folder.md`, `tuldok_project/embeddings.py` (new), `operations.py`, `service.py`, `tests/test_tuldok_project_curation.py` (new), the count of the table's rows in `tests/test_tuldok_project_service.py` | +| T5b | The default embedder, the settings, `embed`, `select`, freeze's `records` | `pumas_operations.py` (the `embeddings` result), `tuldok_project/pumas.py`, `local.py`, `steps.py`, `builds.py`, `plan.py`, `check.py`, `schemas/`, `operations.py`, `service.py`, `tests/fake_pumas_typed.py`, `tests/browser_fake_pumas.py`, `tests/test_tuldok_project_curation.py` | +| T5c | The page | `static/project-workspace/curation.js` (new), `inside.js`, `tabs.js` (the Embed button), `annotate.js` (Freeze filled in), `generate.js` (the embedder), `base.css`, `static/project-workspace.html`, `app.py` (the asset list), `tests/browser_project_workspace_curation.cjs` (new), its fixture | + +## 8. Tests + +**T5a and T5b, `tests/test_tuldok_project_curation.py`,** fixtures written by the test with the standard library: + +- The contract: a set read; each rule broken once refused with its sentence (format, ids repeated, rows and shape disagreeing, a non-finite value, a zero row, a projection of the wrong shape, a bound lowered for the test). +- The map's components against the test's own arithmetic on a planted structure (points spread along two known orthogonal directions with known variances, and noise): the axes and the shares of variance; a script's projection used as given. +- Neighbours, duplicates, outliers and leakage on a fixture with planted near-duplicates, one far record and one validation record that is a near-copy of a training one, against a brute force the test computes; a budget lowered for the test gives `complete` false with exactly-right pairs for the rows examined; thin cells against the test's counting. +- The default embedder: EmbeddingGemma 2 chosen when served (by alias and by model id, the preferred id first), refused with the served embedders listed when not, never another model; the machine's setting and the definition's model in that order. +- `embed` against the fake endpoint, built for real by `builds.execute`: a text and image dataset embedded in batches, the requests' inputs, prefix and options as section 5 says, a valid product read back by the reads; a point cloud noted and not embedded; a `not_admitted` asked again; a failure part way and a second build that takes the answered rows and asks only for the rest; no endpoint, the plan says so. `select` and freeze's `records` built for real, with exactly the records chosen. +- The six reads and `set_embedder` in `project_folder`, not in `catalog`, its digest unchanged. + +**T5c, `tests/browser_project_workspace_curation.cjs`,** on a fixture project (a frozen dataset with planted duplicates and one leaked validation record, embeddings written by a script), the fake endpoint, owner signed in, no GPU: the map drawn and coloured by split; a region chosen and kept as a new dataset whose definition holds exactly those records; Freeze filled in with them; the duplicates and the leaked record listed; more like this for an opened record; the Embed form showing EmbeddingGemma 2 as the default; light and dark at 1440 and 390 wide, no sideways overflow, no script error, nothing redrawn on a return to the window. + +## 9. Out of scope + +Computing embeddings in Tuldok; approximate nearest-neighbour indexes; UMAP or t-SNE in Tuldok (a script may write either as the projection); embeddings of sounds, point clouds and meshes through the endpoint (Pumas 0.8.0 embeds text and images only); colouring by an annotation that is not carried in the records (the working copy's annotations of a task); several sets on one map; clustering. + +## As built + +What differs from the text above, and what was found. Built on `feat/t5-curation` beside phase 3, which owns the Benchmarks and Analysis tabs and the Map's compare set; this phase touched `tabs.js` (the Embed button and its drawer, two lines), `base.css` (one block at its end), `operations.py`, `service.py`, `docs/contracts/project-folder.md` and the service test's count of operations (69 to 76) only where its own rows go. + +### T5a, the contract, the reader and the reads + +- **Record ids** in an index are 1 to 600 characters without control characters, not spec-d1's 1 to 200 of `A-Z a-z 0-9 . _ -`: a product read from its files names its records by their addresses (`notes/a.txt`), and those must be embeddable. Section 1 was corrected. +- **The blocking of the pair searches is a grid, not a sort.** As first built, rows were sorted by their first coordinate and each compared with those within the threshold along it. On clustered data (20 clusters, 256 dimensions) that window held most rows. The searches now bucket rows by up to five leading coordinates in cells twice the radius wide and look in the 2⁵ cells on the near side of each coordinate (exact: a row within the radius in every coordinate lies in one of them); the lower bound in all eight coordinates and the residual rules pairs out before the dot product. Outliers search the cells within the 90th percentile of the k-th nearest distance of 32 rows first, and fall back to the outward sort for a row whose k nearest are not all within it. +- **The budget is 20,000,000 coordinate comparisons or 400,000,000 values multiplied** (not 400,000 exact products: the cost of one depends on the dimensions), checked before each row; a row is examined whole. `complete` and `examined` say how far it got; a test lowers the budget and checks that the pairs listed are exactly the brute force's pairs of the rows examined (the shuffled order is `random.Random(f'{rows}:{dimensions}')`). +- **Measured** (this machine, CPU, pure Python, `math.sumprod` at about 8 to 12 ns a value), on the worst case the reader meets, clustered synthetic rows with no duplicate in them: 50,000 × 256 — reading and making unit 1.0 s, the map 2.4 s, more like this 0.18 s, duplicates 8.0 s and incomplete (634 rows examined), outliers 7.9 s (162), leakage 2.6 s (1,030 of 10,000 held-out rows); 10,000 × 768 — the map 8.8 s, duplicates 6.3 s (1,044), outliers 5.4 s (201), leakage complete in 1.0 s; 20,000 × 256 duplicates 6.8 s (1,587); 2,000 × 64 everything complete under 1 s. Each is computed once per set and kept. **The searches are complete for a few thousand rows only when the data is dense at the threshold's radius**; past that they list exact pairs for a shuffled sample of rows and say so. Without vectorised arithmetic this is the honest bound; an approximate index was left out (section 9). +- **Colour.** Numbers with at most 24 different values are drawn as categories (a split numbered 0, 1, 2 reads better so); more are a number scale. A category beyond the 24th is one `other`. +- **`embeddings`** is an operation outside the lock (it finds products under it, reads indexes after), not an `INSIDE_READS` row; `records_run` joined the service's product arguments. A set in a dataset that takes this one as an input whose index cannot be read is not listed (it cannot be told which records it describes); one in the dataset's own product is listed with its problem. +- **Tests:** `tests/test_tuldok_project_curation.py`, 13 for T5a (contract 3, components 3, searches 7). The catalog profile's digest is unchanged (110 operations). + +### T5b, the embedder, `embed`, `select` and freeze's records + +- **The default's match** is "the alias or `model_id` contains `embeddinggemma-2`" ignoring case; among those the `model_id` `unsloth/embeddinggemma-2-GGUF` (case ignored) first, then the endpoint's order; only entries with `text_embedding` or `multimodal_embedding` available and `embeddings_float32` out count. Several profiles of the default are not an error: the first is used. The refusal names every served embedder with its model id and profile. +- **The machine's setting** is `embedding_model`, `embedding_profile` and `embedding_prefix` in the settings file; `set_embedder` checks a named model against the endpoint (refused `not-found`, or `unavailable` with no endpoint) and writes an activity row `set` or `default`. A step reads the machine's setting at build time through the facts file (`<run>/embed.json`), so an `embed` that names no model is not reproducible from its definition alone; the index records what answered and `named_by` (`definition`, `setting` or `default`). +- **Requests.** `options` is always sent, `{"kind": "embeddings"}` and `dimensions` when set; `dimensions` is checked against an advertised bound when there is one and sent without one (the release accepts it on every embedder; spec-d5's rule that an option must be advertised was not applied here, since llama.cpp embedders may advertise none). Image batches stop short of 16 MiB of base64. +- **Rows are written in the order answered** (texts first, then images; on a resume, the rows taken first), and `records` follows that order; the contract allows any order. +- **A resume** takes rows only from the dataset's newest run, when it failed, and only when the input run, model, model id, profile, prefix and dimensions asked are all the same; the partial files are removed when the step ends, so a product holds only the set. +- **`pumas_operations`** gained the `embeddings` result (finite vectors of one length) beside `decisions`, additively; catalog mode never asks for it. +- **`select`** keeps a record's id when it is a list id and makes one (`id_from`) from an address otherwise, keeping the address in `meta.selected_from`. Freeze's `records` names input ids; one that is not in the input fails the step. +- **Not in the text:** `check` refuses `profile` without `model`; a step with no required setting still needs `"settings": {}`. +- **Tests:** 13 more in `tests/test_tuldok_project_curation.py` (the default embedder 3, settings 1, check 2, built for real against the fake 6, published 1). Project suite 799 OK (773 before, and these 26); agent interface 43 OK (4 skipped). + +### T5c, the page + +- **Map is a third mode of the opened dataset** (inside.js), shown when `embeddings` lists a readable set; its state is per dataset in page memory (the set, colour, view, tool, chosen, opened, each list's controls), not in the address: a reload opens on Records. The viewer beside the map is inside.js's, reading the record from the set's records run. +- **Tools** are Move, Box and Lasso (a toggle; Box first); a click that does not move opens the nearest point within 10 px. The wheel zooms about the pointer from 1 to 200 times. Keys: the arrows move a highlight to the nearest point in that direction (Enter opens it, Escape clears the choice), announced in a live region. +- **New dataset from these** is a form under the map (title, name, Keep), not a drawer, so the map stays; after Keep the line "Kept datasets/<name> as an unsaved edit" stays with the choice. **Freeze these** fills the Freeze form's slot (`annotate.js`'s new `fillFreeze`) and goes to `#data/new:freeze`, which shows "Only the N records chosen on the map" with a button to freeze every record instead; changing the dataset there drops them. +- **The embedder** is shown and set in the endpoint's drawer on Train (owner: a select with "EmbeddingGemma 2 (the default)" first, the prefix, Set the embedder); the Train section's rows were not changed. The Embed form's first model is "Default: <EmbeddingGemma 2 or the machine's> → <what it resolves to now>"; with the default chosen the definition names no model, and the prefix is written only when changed from the machine's. +- **Colours** are the dataviz reference palette's eight categorical hues as tokens `--cat-1` to `--cat-8` (light and dark steps); "other" and rows with no value are `--ink-3`; thin cells are washed with `--warn` at 14 %. Not run through the palette validator here (the palette is the validated reference instance). +- **The check,** `tests/browser_project_workspace_curation.cjs` (fixture `tests/project_curation_browser_fixture.py`, 244 texts in 16 dimensions with the plants; `browser_fake_pumas.py --embedders`): 7 PASS — the map drawn and coloured by split (pixels of the three colours counted on the canvas) and by a field; the three planted pairs, the leaked v001 alone, far01 the first outlier, more like this; a box of 97 records kept as a select of exactly the ids the test computes from `embedding_map`, and Freeze filled in and kept with them; the Embed form's default "EmbeddingGemma 2 → embeddinggemma-2 · embeddinggemma-2-gpu (unsloth/embeddinggemma-2-GGUF)"; nothing touched by a focus and a timer reading; light and dark at 1440 and 390, nothing in the map, its lists or the drawer overflowing. Regressions unchanged: `browser_project_workspace.cjs` (24 PASS), `_inside` (8), `_annotate` (11), `_endpoint` (8). +- **Found at 390 px:** the page is 553 to 554 px wider than the window on every tab, as phase 2 found (its header and strip); the check records it and asserts only the map, its lists and the drawer. +- **Not verified:** a real Pumas or a real EmbeddingGemma 2 (the owner's word is needed; no GPU was used); an embed built from the page (built in the unit tests against the fake); thin regions in the page (the fixture has none; tested in the reader); Lasso, Move and the wheel by a person (drawn and wired, not driven by the check); 50,000 points drawn in the browser (the map draws one arc a point, unmeasured past 244); a set whose records run is not current, and several sets of one dataset (the select is drawn, not exercised); colouring by an annotation the records carry in the page (tested in the reader); screen readers. diff --git a/docs/plans/training-and-curation/spec-t6.md b/docs/plans/training-and-curation/spec-t6.md new file mode 100644 index 00000000..58e5d90d --- /dev/null +++ b/docs/plans/training-and-curation/spec-t6.md @@ -0,0 +1,177 @@ +# Implementation spec: serving a trained model + +This is phase 6 of [the training and curation plan](plan.md). It follows that plan's rule: Tuldok supplies a contract (the `serve` declaration and the protocol), the orchestration (start, approval, supervision, stop) and the views; the project writes its own Python. Tuldok's server never imports the project's code: the entry point runs in a separate process, in the recipe's environment, as a build's command does ([spec-m6.md](../definition-first-projects/spec-m6.md) section 4). Who may do what follows [spec-w3.md](../definition-first-projects/spec-w3.md) and `service._allowed`; credentials and sessions are those of [actors.md](../../contracts/actors.md). The page follows [spec-w4.md](../definition-first-projects/spec-w4.md): what changes on its own (a model becoming ready) is set in place, nothing is rebuilt by a timer. Decision 4 of the plan holds: **a served model is reachable only from this machine**; Tuldok opens no new listener, and the endpoint is a route of the server Tuldok already runs on the loopback address. + +**Owner, 2026-11-19:** "We should also have serving trained models as a REST endpoint and autologging." + +**After this phase** a recipe may declare `serve`: the file and functions of the project's own entry point. The owner starts serving a built model from the Train tab or from the model's drawer (an agent asks, and the owner approves, as with a build); Tuldok starts the entry point in the recipe's environment with the model's product, waits for it to load, and answers `POST /api/serve/<model>/predict` behind Tuldok's credentials. Every request's time is logged; a session's timings can be written into a benchmark as per-case values, so the model as served is a benchmark subject like any other. A served model stops when asked, after an idle time, or when Tuldok stops. + +## 1. The declaration + +A recipe may have a `serve` object: + +| Field | Required | Rule | +| --- | --- | --- | +| `module_file` | yes | a code path (as `settings_file`) of a `.py` file: the project's entry point | +| `function` | no | a Python name (`^[A-Za-z_][A-Za-z0-9_]{0,63}$`), default `predict`: called for every request as `function(model, inputs)` | +| `load` | no | a Python name: called once at start as `load(product)` with the product directory; what it returns is `model`. Without `load`, `model` is the product directory (a string) | +| `environment` | no | a reference to `environments/`; default the recipe's `run.environment`. One of the two is required | +| `idle_minutes` | no | integer 1 to 1,440: stop after this long without a request; default the machine's (section 4) | +| `start_seconds` | no | number 1 to 3,600, default 120: how long `load` may take before the start fails | +| `request_seconds` | no | number 0.1 to 600, default 60: how long one request may take before the process is stopped as failed | +| `gpu` | no | boolean, default false: without it `CUDA_VISIBLE_DEVICES` is empty, as for a run | +| `example` | no | an object: the inputs the page's "Try it" starts with | + +`check` adds: `module_file` is a code path that must exist (an error; a warning for earlier work, as other code paths), and ends in `.py`; no environment in `serve` or in `run` is an error. `serve` is not an input: it never makes a model out of date, and building never reads it. + +Example: + +```json +"serve": {"module_file": "serve.py", "load": "load", "function": "predict", + "idle_minutes": 20, "example": {"x": [1, 2, 3]}} +``` + +and the project's `serve.py`: + +```python +import json, os + +def load(product): + with open(os.path.join(product, 'weights.json')) as handle: + return json.load(handle) + +def predict(model, inputs): + return {'y': [model['constant'] + value for value in inputs['x']]} +``` + +**Why a module and two functions, not a command that speaks a protocol.** Both run the project's code in its own process. With a command, every project writes the read loop, the framing, the bounds and the array encoding itself, and one stray `print` in the model's code (or a library's warning on standard output) corrupts the protocol. With a module, Tuldok's small adapter (section 2) owns all of that once, in the standard library: it moves the process's standard output to the log before importing the project's file, speaks the protocol on two private pipes, and decodes and encodes arrays. The project writes two plain functions it can also call from a test or a notebook. It is the simpler contract for projects and the one Tuldok can make safe. A project that needs a server of its own can still write one as a `run` and is outside this phase. + +There is no `tuldok_log` helper for this: the two functions above are the whole skeleton, and the contract document carries the example. + +## 2. The process and the protocol + +**Starting.** The model is the recipe; serving needs its current product (refused `not-built` without one) and the environment's recorded interpreter, checked as a build checks it (refused `not-ready`). The code is a snapshot as for a build (`snapshot.prepare`): the committed code at HEAD, refused `dirty` when tracked files under the code paths have changes, unless the start asks for `scratch` (the code root as it is on disk). The session records the commit, the product's run, the environment and the interpreter. + +Tuldok starts, in the code directory, `<interpreter> -I <Tuldok>/tuldok_project/serve_adapter.py` with the two pipe descriptors, the module file, the function names and the product directory as arguments. `-I` keeps Tuldok's own directory and `PYTHON*` variables out of the process; the adapter puts the code directory first on `sys.path`. The environment is a run's (`PATH`, `HOME`, `USER`, `LOGNAME`, `LANG`, `LC_ALL`, `TZ`), `CUDA_VISIBLE_DEVICES` empty unless `gpu`, `TULDOK_PRODUCT` (the product directory), `TULDOK_SERVE_MODEL` (the recipe) and `TULDOK_RUN_ID=serve-<session>`. Standard input is empty; standard output and standard error go to one log. The process is started in a session of its own with `PR_SET_PDEATHSIG` (it dies with Tuldok), from a thread that lives as long as the process (the signal follows the thread that started it). + +**The protocol,** one UTF-8 JSON object per line on the two private pipes: + +1. The adapter imports the module, calls `load`, and writes `{"ready": true, "load_seconds": s, "numpy": bool, "python": "3.12.3"}`, or `{"ready": false, "error": "<the exception, one line>"}` and exits. No line within `start_seconds`: the start fails and the process is stopped. +2. Per request Tuldok writes `{"id": n, "inputs": {…}}`; the adapter calls `function(model, inputs)` and writes `{"id": n, "ok": true, "outputs": …, "model_seconds": s}` or `{"id": n, "ok": false, "error": "…", "model_seconds": s}` (the model raised: the session stays ready). No answer within `request_seconds`, an answer longer than 16 MiB, one that is not JSON or has another id, or the process gone: the session fails and the process is stopped. +3. When Tuldok closes the request pipe the adapter exits. + +**Arrays.** A value in the inputs or outputs that is an object of exactly the keys `array` (`"float32"`), `shape` (0 to 8 integers ≥ 0) and `base64` (the little-endian float32 values in C order, standard base64, length exactly 4 × the product of the shape) is an array. The adapter gives `predict` such an input as a `numpy.ndarray` of float32 when NumPy is importable in the environment, otherwise as nested lists of floats. Nested lists of numbers are passed as they are. In the outputs, a NumPy array (anything with `shape`, `dtype` and `tobytes`) is cast to float32 and encoded so; a NumPy scalar becomes a number; tuples become lists; anything else that is not JSON is the request's error. Only float32 is encoded; other values are JSON numbers. + +**Bounds.** A request body is at most 8 MiB (1 MiB through the operation, which is the operation contract's bound), JSON nested at most 64 deep, an array at most 2²⁴ elements; an answer at most 16 MiB. At most 8 requests wait for a served model; the ninth is refused `busy`. One request is in the process at a time. + +**Stopping.** The request pipe is closed, the process group gets `SIGTERM`, then `SIGKILL` after 5 seconds, and every process carrying the session's `TULDOK_RUN_ID` is swept (`launcher.stop_strays`). The log keeps the first 4 MiB of output and then only says that more was dropped; the last 4 KiB are kept in memory as the tail shown with a failure. + +## 3. The endpoint and who may use it + +`POST /api/serve/<name>/predict`, where `<name>` is the recipe's name (`recipes/<name>`), on the server Tuldok already runs (`127.0.0.1` by default). The body is `application/json`, `{"inputs": {…}}` and nothing else. The credential is Tuldok's: a bearer credential, or the page's session cookie with a same-origin request, exactly as for `/api/project/<operation>`; no operation contract header is needed, so that any HTTP client can call it. It is answered only in project-folder mode. + +| Answer | When | +| --- | --- | +| 200 `{"model", "session", "outputs", "timing": {"queue_ms", "model_ms", "total_ms"}}` | answered | +| 400 `invalid` | not a JSON object of `inputs` only, a malformed array, too deep | +| 401, 403 | no credential, or one without `inspect` | +| 404 `not-served` | the model is not being served | +| 409 `not-ready` | it is starting or awaiting approval | +| 413 `too-large` | the body is over 8 MiB | +| 422 `predict-failed` | the project's function raised; `error` says what | +| 502 `serve-failed` | the process broke the protocol or died; the session is failed | +| 503 `busy` | eight requests are waiting already | +| 504 `timed-out` | no answer within `request_seconds`; the session is failed | + +**Who.** Calling `predict` needs `inspect`: it changes nothing kept, and the owner's start is the consent to run the code; least privilege lets an owner hand a client a credential that can only read and predict. Starting needs `request`, as a build: the owner's start runs at once, an agent's (with `execute`) waits for the owner's approval for up to an hour. Approving is the owner's. Stopping needs `cancel` (an agent with `cancel` may stop what it asked for or anything served, as it may cancel a build); stopping one that awaits approval declines it. Writing a session's timings into a benchmark needs `edit` (it imports values, as `import_values`). + +## 4. Sessions, limits and the log + +A **session** is one start of one model: `id` (32 hex), `model` (recipe reference), `state` (`awaiting-approval`, `starting`, `ready`, `failed`, `stopped`, `declined`, `expired`), `reason` (why it ended: `asked`, `idle`, `closing`, or the problem), who asked and decided, the times, the commit, scratch, the product's run and directory, the environment and interpreter, the pid, the idle minutes, the request counts, the last request's time and the output's tail. + +**Limits of the machine,** two optional fields of the settings file (`local.py`): `serve_most`, how many models may be starting or ready at once (1 to 16, default 2), and `serve_idle_minutes` (1 to 1,440, default 30), the idle time of a model whose `serve` names none. A start (or an approval) past `serve_most` is refused `limit`, naming the ones served. A model already awaiting approval, starting or ready is refused `already-served`. + +**The log,** under the project's local home, `serving/<session>/`: `session.json` (the session, written again at each change), `output.log` (section 2), and `requests.jsonl`, one line per request answered or refused after it reached the model: `{"n", "at", "ok", "status", "queue_ms", "model_ms", "total_ms", "bytes_in", "bytes_out", "error"?}` — the time waiting for the model, the model's own time (reported by the adapter around `function`), and the total from when Tuldok had read the request to when the answer was ready. At most 100,000 lines a session; past that requests are counted and not written. Sessions are in memory while Tuldok runs; `served` also lists the 50 newest sessions on disk, and a session left starting or ready by a Tuldok that is gone is shown `stopped` with the reason `Tuldok stopped`. + +**Timing summary** of a session (`serve_session`): the count, how many failed, the span from first to last request, the throughput (answered requests per second over that span, null under two requests), and for `queue_ms`, `model_ms` and `total_ms` the mean, p50, p95, p99 and max of the answered requests (percentiles by the linear interpolation of `results.summarise`). + +## 5. The model as served, as a benchmark subject + +`serve_values` turns a session's answered requests into a [values file](../../contracts/project-folder.md#values-files) and imports it, as `import_values` does: `benchmark` and `subject` name a subject of that benchmark (whose `recipe`, when it has one, must be the served model); `metrics` maps metric ids of the benchmark to one of `total_ms`, `model_ms`, `queue_ms`, `total_s`, `model_s`, `queue_s` (default: each metric whose id is one of those names). Each answered request is a case `request-000001`, …; a request the model failed is a value `null` with its error as the reason. The commit is the session's, `produced` the time of its last request. The file is also written to the session's directory. The benchmark then holds per-case values, its summary rows (median, p10, p90, …), the intervals of spec-t3 and its charts; p95, p99 and throughput, which the results format has no column for, stay in the session summary. + +How to use it: a benchmark of its own for serving, say `benchmarks/served-latency` with metrics `total_ms`, `model_ms` and `queue_ms` (`better: lower`, `units: ms`) and a subject per served model naming its recipe. Its `run` may be the project's own load script that calls the endpoint (built like any benchmark, writing its values), or it may be marked `earlier_work: {"rebuild": "no", …}` when its values come only from `serve_values`. The timings depend on what else ran on the machine; the session records when. + +## 6. Operations + +Seven rows of the `project_folder` profile, absent from `catalog` (its digest unchanged): + +| Operation | Who | Mutation | Arguments | Result | +| --- | --- | --- | --- | --- | +| `served` | inspect | no | — | `sessions` (newest first), `servable` (each recipe declaring `serve`: `model`, `title`, `module_file`, `environment`, `built`, `product_run`, `example`, `session`), `most`, `idle_minutes` | +| `serve_session` | inspect | no | `session` | the session, `summary` (section 4), `recent` (the 20 newest request lines) | +| `serve_start` | request | yes | `model`, `scratch` | the session | +| `serve_approve` | owner | yes | `session` | the session | +| `serve_stop` | cancel | yes | `session` | the session | +| `serve_predict` | inspect | no | `model`, `inputs` | `model`, `session`, `outputs`, `timing` | +| `serve_values` | edit | yes | `session`, `benchmark`, `subject`, `metrics` | `benchmark`, `subject`, `values`, `file` | + +They are answered outside the service's lock; those that read the project take it only for that. `serve_predict` never takes it. Refusals carry the codes of section 3 (`not-served`, `not-ready`, `busy`, …) with the statuses there. + +## 7. The page + +- **Train: Served models,** after Build requests: one row per recipe that declares `serve`, with its state (not served, awaiting approval, starting, ready, failed, stopped), its requests and its last session's line; **Serve** (`serve_start`) for a built model not served, **Stop** (two presses) for one starting or ready, **Approve** and **Decline** for one awaiting approval. A row opens the model's serving drawer (`serve:<recipe>`). +- **The serving drawer, and a Serving block in a model's drawer** (Map) for a model that declares `serve`: the state with its reason, the address of the endpoint (`POST /api/serve/<name>/predict`), the controls, the failure's output tail, the timing summary of the session, and **Try it**: a text field holding the inputs as JSON (the declaration's `example`, else `{}`), **Send**, and the answer (outputs, the three times) or the refusal in the server's words. +- **In place.** While a session shown is starting (or awaiting approval), the page asks `served` every second and sets the state mark, the line and the controls of that block in place; when it is ready the controls change from Stop-only to Stop and Try it. Nothing else is read again. Try it does not read the project again. + +## 8. Slices + +| | Slice | Files | +| --- | --- | --- | +| T6a | The declaration, the protocol, the adapter, the supervisor, the operations, the endpoint | `tuldok_project/serving.py`, `serve_protocol.py`, `serve_adapter.py` (new), `schemas/recipe.schema.json`, `check.py`, `local.py`, `operations.py`, `service.py`, `host.py`, `app.py`, `docs/contracts/project-folder.md`; tests `tests/test_tuldok_project_serving.py` (new), the service test's count | +| T6b | The page | `static/project-workspace/serving.js` (new), `tabs.js` (the section and drawer hooks), `screens.js` (the block in a model's drawer), `base.css` or a new `serving.css`, `project-workspace.html`, `app.py` (assets); `tests/browser_project_workspace_serving.cjs` and its fixture (new) | + +## 9. Tests + +**Unit:** the declaration (valid, missing environment, a bad name, a missing module file, not `.py`); the protocol's checks (a request of only `inputs`, a valid and a malformed array, its length against its shape, too deep, too large) and the adapter's own encoding and decoding with and without NumPy, run in a child process; the supervisor with a stand-in module: start to ready, a `load` that raises (failed, with its output's tail), a `load` that hangs (failed after `start_seconds`), a request that raises (422, still ready), a request that hangs (504, failed), idle stop, the limit, stop, and no process left after each test; approval (an agent's start awaits the owner; approve; stop declines); the credential checks through HTTP (none: 401; inspect only: predict yes, start no; owner: start and stop). + +**End to end:** the buildable fixture with `serve.py` and a `serve` block on `fit-a`, built, served through the owner's start, called through `POST /api/serve/fit-a/predict` with a reader credential (nested lists and a base64 array), refused without one; timings in `serve_session`; `serve_values` into a benchmark of served latency, whose summary rows the test checks against its own arithmetic. + +**Browser,** `tests/browser_project_workspace_serving.cjs`: start a served model from its drawer, its state goes to ready in place (the same elements), Try it returns a result, stop it; the Train tab's section; light and dark, 1440 and 390 wide, no sideways overflow, no script error. + +## 10. Out of scope + +A remote address (decision 4); more than one process per model, batching, streaming answers, GPU scheduling between serving and builds (a served model with `gpu` does not take the GPU lock: doubt); authentication other than Tuldok's credentials; serving a dataset or a benchmark; restarting a failed model on its own. + +## As built + +What differs from the text above, and what was found. Built on `feat/t6-serving` beside phase 4 (annotation), which is not touched. Edits to shared files are small and local: `operations.py` (seven rows at the end of the table, `SESSION`, `SERVE_MEASURES`), `service.py` (the serving calls in `OUTSIDE_LOCK`, the supervisor held and closed, the methods appended at the end, `session` as an activity target), `app.py` (the route, two asset rows), `tabs.js` (`served` read with the Train tab, the section after Build requests, the `serve:` drawer), `screens.js` (the Serving block after Structure), `project-workspace.html`, `docs/contracts/project-folder.md` (the `serve` row of the recipe, seven operation rows, a section *Serving a model* before *The Pumas endpoint*, two settings), the service test's count (79 to 86) and its lists of mutations, owner rows and reads. `base.css` is not changed (its phone header kept); the styles are in a new `serving.css`. `check.py`, `store.py` (one line: `check_serve` among the cross-file checks), `local.py` and `schemas/recipe.schema.json` take the declaration. + +### The backend seam + +`serve.backend` is a field of the declaration: `"local"` (the default and the only value the schema accepts). In `serving.py`, `BACKENDS` maps a backend's name to an object with `start(session, launch, on_ready, on_exit)`, `exchange(session, inputs, deadline)` and `stop(session)`; `LocalBackend` is the one built. The `Supervisor` owns everything that is not the backend's (states, approval, the machine's limit, idle stop, the queue of one model, timings and the log) and calls the backend only through those three. A later `"pumas"` backend (registering an exported model with the owner's Pumas endpoint and forwarding requests to it, once Pumas offers a generic tensor-in/tensor-out capability; 0.8.0 does not) would be a second entry of `BACKENDS` and a value of the schema's enum; it is not built and Pumas is not contacted. `served` and the sessions answer `backend`. + +### T6a, the core + +- **Files:** `tuldok_project/serving.py` (sessions, `Launch`, `LocalBackend`, `Supervisor`), `serve_protocol.py` (the checks of a request, an answer and the ready line, the statuses of section 3, the summary), `serve_adapter.py` (the child side, run as a file with `-I`, Python 3.8 or later). +- **The code served** is the snapshot at HEAD, as section 2 says, not the code at the product's commit: a `serve.py` added after the model was built would not be in the older snapshot. The session records the commit it serves from; the product's run is recorded beside it. A dirty tree is refused `dirty` unless `scratch`. +- **An owner's start works out the launch first** (product, environment check, snapshot, the module file in the snapshot) and refuses (`not-built`, `not-ready`, `dirty`, `invalid`, `not-servable`) before any session is made; an agent's start only checks that the recipe declares `serve` and is built, and the launch is worked out at approval (a refusal then leaves the session awaiting). Refusal codes `not-servable`, `already-served`, `limit`, `not-awaiting`, `not-running` were added to section 3's list; through the operations they are 409 (`operations.error_status`), through the endpoint the statuses of `serve_protocol.STATUS`. +- **Times.** `total_ms` runs from when the app has read and parsed the body; `queue_ms` therefore holds Tuldok's credential check (a database query) as well as the wait for the model, about 0.3 to 2 ms on this machine when nothing waits. `model_ms` is the adapter's own time around `function`. Throughput is (answered − 1) / (last − first request time). +- **A stop that comes while the process is being started** (a race the first supervisor test found: the process was spawned after the stop and left running) is handled by a `stopping` event the starting thread looks at once the process exists. `close()` also stops the processes of sessions that failed and are still being stopped. +- **Approval** is held in memory, not in the database: an agent's start awaits the owner while Tuldok runs and is gone after a restart (its `session.json` reads `stopped`, reason `Tuldok stopped`). It is not in Needs you (adapter.js is not changed); it is shown on the Train tab's row and drawer with Approve and Decline. +- **The endpoint** needs no contract header; an `Origin` is required for a cookie, as for every POST. A body over 8 MiB is refused from its `Content-Length` before it is read (413). Answers go through `app.reply`; requests are not observed by the MCP observer beyond its stages. +- **Tests:** `tests/test_tuldok_project_serving.py`, 20: the declaration (valid, each refusal, `backend` other than `local` refused, the module file as a code path), the protocol (requests, malformed arrays, depth, answers, the summary against the test's own percentiles), the adapter (decoding to lists, encoding with a stand-in for a NumPy array, an answer and its failure), the supervisor with a pure-Python stand-in and the interpreter running the tests (start to ready, predict with lists and a base64 array, a stray `print` in the log and not in the protocol, a request that raises: 422 and still ready, stop; a `load` that raises and one that hangs; a request that hangs: 504 and failed; idle stop; one session a model and the limit; approval, decline and expiry; a session left by a Tuldok that is gone), each test checking that no process and no stray of its sessions is left; and three end-to-end tests through the app on port 0 (no credential and a wrong one 401, a reader's predict 200 with lists and a base64 array, the reader refused `serve_start` and `serve_values`, 400, 404, 413 and 422 refusals, `serve_predict`, `serve_session`'s summary, `serve_values` into a `served-latency` benchmark whose summary rows equal the test's own median, minimum and maximum, the activity rows; an agent's start awaiting approval, refused approval, the owner's approval, the agent's stop; the limit from the settings file, `not-servable`, `already-served`, and the service's close stopping what it serves). + +### T6b, the page + +- `static/project-workspace/serving.js` and `serving.css`. The Train tab's **Served models** is after Build requests; its rows and the panels update in place while a session shown is starting or awaiting approval (a one-second `served`, only then). After Try it the panel reads `serve_session` and sets its summary and its line's counts in place. +- `served` and `serve_session` are called, not kept (`P.call`): neither is in `P.WATCHED`, so a reading on the timer never redraws a panel; a control's answer redraws the drawer as every control does. +- **Doubts:** the Train tab's row does not update on its own while ready (its request count changes only when the tab is drawn again); "Try it" shows the answer as JSON, not as an image or a field; an agent's start is not in Needs you; keyboard use of Try it was not checked beyond the buttons being buttons and the field a textarea. +- `tests/browser_project_workspace_serving.cjs`: Serve from fit-a's drawer on the Map, starting and then ready in place (the same element, the drawer not rebuilt, only `served` and `serve_session` asked in between), Try it answering the test's own arithmetic with its times and the summary set in place, the Train tab's row and drawer, light and dark at 1440 and 390 (the drawer, and on Train the main area, not wider than themselves; on the Map at 390 the main area is the map, which scrolls sideways before this phase too, so only the drawer is checked there), Stop in two presses, the process gone and the endpoint answering 404; no script error and no failed request. + +### Runs (a disposable PostgreSQL, no GPU) + +Project suite 832 OK (812 before, and the 20 new); agent interface 43 OK (4 skipped); `browser_project_workspace_serving.cjs` 6 PASS (three runs); unchanged `browser_project_workspace.cjs` 24 PASS, `_actions.cjs` 27 PASS, `_training.cjs` 7 PASS. After every run no `serve_adapter.py` process was left. + +### Not verified + +NumPy in the adapter (no NumPy in the test environment and nothing downloaded: the decoding to a `numpy.ndarray` and the encoding of a real array are untested; the encoding was checked with a stand-in object); a model on the GPU (`gpu` only empties `CUDA_VISIBLE_DEVICES` or not, and a served model does not take the GPU lock that builds take); a project whose `serve.py` imports PyTorch; requests near the 8 MiB bound for speed; eight requests waiting at once (`busy`); the lab folder (nothing there declares `serve`). diff --git a/docs/plans/training-and-curation/spec-t7.md b/docs/plans/training-and-curation/spec-t7.md new file mode 100644 index 00000000..baafa240 --- /dev/null +++ b/docs/plans/training-and-curation/spec-t7.md @@ -0,0 +1,140 @@ +# Implementation spec: hyperparameter search on this machine + +This is phase 7, the last, of [the training and curation plan](plan.md). It follows that plan's rule: Tuldok does not grow a training framework. A sweep varies the settings of a recipe the project already has and runs the project's own script for each trial, one after another, under the build machinery that already exists ([spec-m6.md](../definition-first-projects/spec-m6.md), [spec-m7.md](../definition-first-projects/spec-m7.md) section 4). What Tuldok adds is a definition (`sweep`), the schedule of trials, the decision to stop a poor trial early read from the metrics the script logs anyway ([spec-t1.md](spec-t1.md) section 2.1), and the views. Standard library only on the server; nothing of the project's code runs in Tuldok's server; the page follows [spec-w4.md](../definition-first-projects/spec-w4.md) and updates in place. + +**Owner, 2026-11-19:** "Hyperparameter search could be usful, Tuldok is a little bit limited for that since it is not distributed, but it could still work well on tiny models localy." + +**After this phase** a project can hold `sweeps/<name>.json`: the recipe it varies, the settings it varies and their ranges, a strategy (grid, random, or successive halving that stops poor trials early), one or two objectives read from the trials' training logs, and a budget (trials, minutes; one trial at a time). A build request for the sweep, approved once, runs the trials one after another. Each trial is a real recipe, derived from the varied one, drawn on the Map in a section of the sweep. The **Sweeps** tab shows a sweep's state live (trials done, running, pruned; best so far; budget used), its trials through parallel coordinates (its main view), its frontier when it has two objectives, and every trial in a table that opens it on the Map. + +## 1. Decisions + +- **One approval covers the whole budget.** A sweep is asked for with `request_build` naming it as the only target; the owner's request is queued, an agent's awaits approval, as for any build. What the approval covers is the sweep's text, the varied recipe's text, the texts of what that recipe needs (its inputs and environment) and the commit. Every trial is made by Tuldok from those two texts alone (section 4), so a trial is within the approval though its own text did not exist when it was approved. Changing the sweep or the recipe before the request runs makes it stale, as for any build. +- **Trials are recipes, kept in the project like any edit.** Each trial is written into Tuldok's working copy as a definition `recipes/<sweep>-<NNN>` with `derived_from` the varied recipe, `change` `sweep-trial`, and a `sweep` block naming the sweep, its number and the values it was given; it is an unsaved edit, and Save writes it into the folder like any other. Why the folder, and not a store of Tuldok's own until the owner keeps the winners: a trial's product is recorded against its definition text, and a run is only explained by a definition someone can read again; trials are small (one recipe each, a few hundred bytes); the folder is the place for definitions and high-level results, and keeping the pruned and failed trials too keeps the record of what was tried honest. The raw logs stay in the runs' directories on this machine, never in the folder. The owner who does not want a sweep's trials in Git removes them before saving (Remove on each, as for any definition); keeping only the winners by one control is not built (section 9). +- **Trials are never overwritten.** A trial named `<sweep>-007` that already exists with other values (the sweep's ranges changed since) is left alone, and the new trial takes `<sweep>-007-2` (then `-3`, …). An existing trial of this sweep with the same values and the same text is reused: built already, it is not run again. +- **Concurrency is one.** Trials run one after another on this machine; there is no distributed or parallel search. +- **Objectives are training metrics.** An objective is a series of the trial's `training/metrics.jsonl` (its last or best value), or the trial's last logged `time` or `step`. **A benchmark objective is not built:** a trial measured by a benchmark must be one of its subjects, and adding a subject changes the benchmark's definition, which makes every other subject of it out of date (builds.status compares the whole text); that would need a per-subject definition hash. The way to benchmark a sweep's winners is the existing one: add them as subjects once chosen. +- **Pruning is successive halving without resuming.** A trial cannot be paused and taken up again (there is no checkpoint contract to resume from), so the asynchronous rule of ASHA is applied as a stop rule: at each rung a trial reaches, it goes on only when it is among the best `1/factor` of the values recorded at that rung so far (section 5). + +## 2. The definition: `sweeps/<name>.json` + +A new kind, `sweep`, in a new directory `sweeps/`. Fields (unknown ones refused): + +| Field | Rule | +| --- | --- | +| `format`, `kind`, `title`, `description`, `card`, `labels` | as every definition; `kind` is `sweep` | +| `recipe` | required: `recipes/<name>`, the recipe whose settings it varies. It must have `run.settings` written in the definition (not `settings_file`); Branch from here gives a branch its own copy | +| `parameters` | required: 1 to 16 objects, each with `path` (a dotted path into `run.settings`: keys separated by `.`, a part of digits indexes a list; every part but the last must exist in the recipe's settings as an object or a list; the last may be absent) and either `values` (1 to 64 distinct numbers, texts, booleans or null) or a range: `low` < `high` (numbers), `scale` `linear` (default) or `log` (then `low` > 0), `integer` (default false; `low` and `high` then whole numbers), `count` (2 to 64: how many points of the range a grid takes). Each path once | +| `strategy` | required: `{"kind": "grid"}` (every range then needs `count`), `{"kind": "random", "seed": <0 … 2^63−1>}`, or `{"kind": "halving", "seed", "metric", "better", "unit", "min", "factor"}`: `metric` a series name (1 to 200 characters), `better` `lower` or `higher`, `unit` `step` (default) or `time` (seconds), `min` the first rung (> 0; a whole number for steps), `factor` a whole number from 2 to 8 (default 3) | +| `objectives` | required: one or two objects with distinct `metric` (a series name, or `time` or `step`: the last logged), `take` `last` (default) or `best`, `better` `lower` or `higher`, optional `title` (at most 200 characters) | +| `budget` | required: `trials` (1 to 1,000), optional `minutes` (> 0, at most 10,080), optional `concurrency` (only 1) | + +The checker reports as errors also: a `recipe` that is not a recipe (as any reference); a recipe without `run.settings`, or with `settings_file`; a path whose parents are not in the recipe's settings, or not objects or lists; a grid with a range without `count`; a grid of more than 100,000 points. As a warning: a grid with more points than `budget.trials` (the first in order are run). The edge from a sweep to its recipe has the role `varies`; it carries no state, and a plan does not follow it. + +A recipe gains two optional fields: `change` may be `sweep-trial`, and `sweep` is `{definition: "sweeps/<name>", trial: <1 or more>, values: {<path>: <value>}}` (at most 16 values). The edge from a trial to its sweep has the role `sweep`, a map role like `derived`. The flattening of spec-t3 section 1 leaves `sweep` out (its values are in `run.settings` already). + +## 3. The schedule + +The trials are numbered from 1 in the order the strategy gives them; the same definition gives the same schedule every time. A value drawn from a range is rounded to 6 significant digits (`float(f'{x:.6g}')`), and an integer range's to `floor(x + 0.5)` within `low` and `high`. + +- **Grid:** the Cartesian product in the order of `parameters`, the first varying slowest (`itertools.product`). A range's points are `low + i (high − low) / (count − 1)`, or on a log scale `exp(ln low + i (ln high − ln low) / (count − 1))`, for `i` from 0 to `count − 1`, rounded, repeats dropped. The first `budget.trials` points are run. +- **Random and halving:** `random.Random(seed)`; for each trial, each parameter in order draws: from `values`, `values[rng.randrange(len(values))]`; from a range, `u = rng.random()`, then `low + u (high − low)` or `exp(ln low + u (ln high − ln low))`, rounded. A draw equal to an earlier trial's values is dropped and drawn again; after 1,000 draws in a row with none new the schedule ends (a small space is exhausted). + +## 4. A trial's definition + +Made from the varied recipe's document as it is in the working copy: `format`, `kind`, `title` "<sweep title> · trial <n>", `description` "Trial <n> of sweeps/<name>: <path> = <value>, …", the recipe's `model` and `group` when it has them, `labels` `{"short": "T<n>"}`, `change` `sweep-trial`, `derived_from` the recipe, `sweep` as in section 2, and the recipe's `run` with each path set to the trial's value. Not carried over: `card`, `labels`, `earlier_work`, `compared_with`. JSON with two spaces of indentation and a newline at the end. It is stored with `set_definition`; a trial that does not check is recorded failed with the first problems, and the sweep goes on. + +## 5. Running a sweep + +The worker takes a request whose target is a sweep as any other (section 4 of spec-m7.md: the order, the expiry, the covers checked again), and runs it with `sweeps.execute`, which returns what `builds.execute` returns, so the request ends the same way: + +1. The code is prepared once, at the approved commit (or as it is on disk for a scratch request), and one plan id is used for every run of the sweep, so the request's runs are all of them and Cancel reaches the one running. +2. What the varied recipe needs and is not built is built first, as a build would. A failure there fails the sweep. +3. Then trial after trial in the schedule's order. Before each: the trials budget (trials recorded in this run of the sweep, reused ones included), the minutes budget (wall time since the sweep started), and a stop asked for (Cancel, or Tuldok closing). The trial's definition is made and kept (section 4); its SHA-256 is added to what the build expects, and it is run like a build step. +4. While a trial runs, a watcher reads its `training/metrics.jsonl` every quarter second (spec-t1's reader, only the lines appended) and records the trial's latest step, time and objective values. It stops the trial cleanly (the run's cancel file: SIGTERM, then SIGKILL after the launcher's grace) when the minutes budget is spent, when a stop is asked for, or when halving prunes it. +5. **Halving.** The rungs are `min × factor^k`, k = 0, 1, … (at most 32), in steps or seconds. A trial reaches rung k at the first line whose step (or time) is at least the rung; its value there is the latest finite value of the metric on that line or before (none: nothing is recorded at that rung and the trial goes on). It is then compared with the values recorded at rung k by the earlier trials of this sweep run: with n values including its own, while n < factor it goes on; otherwise it goes on when fewer than ceil(n / factor) values are strictly better than its own, and is pruned otherwise. A line crossing several rungs records each. Every value is recorded, the trial's own included, whether it goes on or not. +6. When the trial's run ends, its objectives are read from its log (a pruned or stopped trial's as far as it got), and it is recorded **completed** (the run succeeded), **pruned**, **stopped** (budget or a stop), or **failed** (the run failed, or the trial did not check). A pruned trial's run is cancelled and its problem says it was pruned and at which rung. +7. The sweep ends **finished** (the schedule or a budget is spent), **cancelled** or **failed**; its request succeeds when it finished. + +A trial reused from before (section 1) costs nothing: its objectives are read from its current product's run. A pruned trial of an earlier run of the same sweep with the same text is not reused; it runs again. + +**State.** Two tables of the project schema: `sweep_runs` (one row per run of a sweep: the sweep, the request, the plan, the state and why it ended, the texts' SHA-256 it ran with, the strategy, objectives and budget, when it started and finished, the owner process) and `sweep_trials` (per trial: number, reference, values, state, whether reused, run, start and finish, seconds, the rungs reached with their values and decisions, the objectives, the latest line seen, the problem). A run left running by a process that is gone is marked failed when sweeps are read. Nothing of these tables is written to the folder. + +**A sweep in a plan.** `plan` with a sweep as its only target answers the plan of what its recipe needs (the recipe itself left out) and one more step, of kind `sweep`: its action `build`, or `built` when its last run finished with the sweep's and the recipe's texts as they are now, or `blocked` when the recipe cannot be built for a reason other than its own `earlier_work` label; its reasons say what it will run, its resources the minutes as `wall_hours`. A sweep with other targets, or with `every`, is refused `invalid`. `request_build` with such a plan covers the recipe's text as well. + +## 6. Operations + +Two reads, `inspect`, in the `project_folder` profile only (the catalog's digest is unchanged): + +| Operation | Arguments | Result | +| --- | --- | --- | +| `sweeps` | none | `sweeps`: each `reference`, `title`, `recipe`, `strategy` (its kind), `objectives`, `budget`, `state` (as in a plan), and `run`: null or the latest run's `id`, `state`, `ended`, `started_at`, `finished_at`, `counts` (by trial state), `best` | +| `sweep` | `sweep` (a reference or a bare name), optional `run` (a run id; default the latest) | `reference`, `title`, `description`, `recipe`, `parameters`, `strategy`, `objectives`, `budget`, `state`, `changed` (the latest run ran another text), `request` (the newest build request naming it: `id`, `state`, `requested_by`), `run` (null, or `id`, `state`, `ended`, `problem`, `started_at`, `finished_at`, `seconds`, `trials`, `counts`), `runs` (the last 20: `id`, `state`, `started_at`, `trials`), `trials` (each `number`, `reference`, `values`, `state`, `reused`, `run_id`, `seconds`, `rungs`, `objectives`, `latest`, `problem`, `defined`), `best` (`number`, `reference`, `value`, or null), `frontier` (the numbers of the completed trials on the frontier of the two objectives; empty with one), `rungs` (the rung points up to the last reached and one more), `axes` (as `benchmark_axes` answers: `metrics` the objectives, `settings` the parameters, `subjects` the trials) | + +`map` gains a group per sweep that has trials (`id` the sweep's reference, `title` its title, `parent` the varied recipe's group, `sweep` true), and a trial's model a `sweep` (the reference); a trial's `group` in the answer is its sweep's group. The folder's `groups` and the other models are answered as before. + +## 7. The page: the Sweeps tab + +A tab **Sweeps** after Train (`#sweeps`; the chosen sweep in the address, `?sweep=`). + +- **Head:** the sweep chooser, its recipe (a link to it on the Map), the strategy, objectives and budget in words, and **New sweep** for a session that may edit: a recipe chosen, a sweep's text written from it (its numeric settings as ranges around their values, random, its first logged series as the objective, 8 trials) into the editor's text view, kept with Keep. +- **State, live:** the run's state; trials completed, running, pruned, stopped, failed of the budget's; the minutes used of the budget's; the running trial with its step and objective values; best so far; and the controls: **Plan sweep** (the plan view, with Request build), **Approve** and **Decline** when a request awaits approval, **Cancel** while it is queued or running. Updated in place on every reading while a sweep runs: texts and bar widths set, no node added or removed. +- **Trials, parallel coordinates** (its main view): `parallel.js` with the sweep's axes, one line per trial in the colour of its state (completed, pruned, stopped, failed, running); dragging ranges filters the table below. +- **Frontier** with two objectives: the tradeoff chart of `charts.js` over the completed trials, the frontier marked; with one objective, the progression chart (the objective by trial number, best so far as a step line). +- **The table:** every trial: number, values, state (and the rung it was pruned at), objectives, seconds; a row opens the trial's drawer (the model's drawer), and **Open on the Map** selects it there. +- **Updating in place:** a reading that finds the sweep's answer changed updates the state texts, appends rows for new trials and sets the changed cells, and draws the charts again in their hosts; the tab is never emptied, and a person's own choice (another sweep, an axis) draws it again in place (`putInPlace`). + +Trials carry the kind of change **Sweep trial** on the Map (a chip and a colour of their own, `--k-sweep-trial`, light and dark). + +## 8. Slices + +| | Slice | Files | +| --- | --- | --- | +| T7a | The kind, the checks, the schedule, the plan, the runner and its tables, the two operations, the map's groups | `tuldok_project/sweeps.py` (new), `schemas/sweep.schema.json` (new), `schemas/recipe.schema.json`, `documents.py`, `check.py`, `graph.py`, `plan.py`, `build_requests.py` (covers), `service.py` (dispatch, the reads), `store.py` (the tables), `operations.py`, `workspace.py`, `comparing.py` (`sweep` left out), `docs/contracts/project-folder.md`, `tests/test_tuldok_project_sweeps.py` (new), `tests/project_sweep_stand_in.py` (new), the service test's count | +| T7b | The Sweeps tab | `static/project-workspace/sweeps.js`, `sweeps.css` (new), `app.js` (the tab, the address), `adapter.js` (the kind), `parallel.js` (its selection and colours given by the caller), `static/project-workspace.html`, `app.py` (assets), `tests/browser_project_workspace_sweeps.cjs` (new) | + +## 9. Tests + +**T7a, `tests/test_tuldok_project_sweeps.py`:** the definition (schema and checker: each field, the paths against the recipe, a grid without count, the warning past the budget, the trial's `sweep` block and its edge); the grid and random schedules against the test's own arithmetic (product order, log points, rounding, integers, duplicates dropped, exhaustion); halving's decisions on fixture metrics logs against the test's arithmetic (rungs crossed together, values before the rung, n < factor, ties); budget enforcement (trials, minutes); the trial's text and its derivation link, reuse and the `-2` name; the plan of a sweep and its covers; end to end through a started `Service` with the stand-in `tests/project_sweep_stand_in.py` (plain Python, logging through `tuldok_log`, about a second a trial, its loss falling to a floor set by its settings): an agent's request approved once, trials run one after another, the poor ones pruned by halving, within the trials budget; a second sweep stopped by its minutes; the trials on the map in the sweep's group; the two reads; the operations in `project_folder` and not in `catalog`, the digest unchanged. + +**T7b, `tests/browser_project_workspace_sweeps.cjs`:** on a fixture with the stand-in: a sweep made with New sweep and kept; asked for by an agent and approved on the tab; while the trials run the state block's counts and the latest step change across readings with the block the same element and no node of it added or removed; the frontier and parallel coordinates show every trial; a trial opens on the Map; light and dark, 1440 and 390 wide, no sideways overflow, no script error, no failed request. + +**Regressions:** the project suite, the agent interface tests, `browser_project_workspace.cjs`, `_actions.cjs`, `_training.cjs`, `_compare.cjs`. + +## 10. Out of scope + +Running trials at once or on other machines; resuming a pruned trial (no checkpoint contract); Bayesian or other model-based strategies; benchmark objectives (section 1); a control keeping only a sweep's winners; any GPU run (the strategy is the same, the owner's word is needed first); the lab's models. + +## As built + +What differs from the text above, and what was found. Built on `feat/t7-sweeps` beside phase 6 (`tuldok-t6`, serving), which owns the Train tab's and the model drawer's new actions and a recipe's `serve`; neither is touched here. Edits to shared files are small and local: `app.js` (one tab, one addressed key), `operations.py` (two rows, the kind, a result tuple), `service.py` (two reads and three lines of dispatch in the worker), `check.py` (two lines at the end of `check_map`), `plan.py` (three lines in `build_plan`), `build_requests.py` (one line of `covers`), `store.py` (the tables, imported inside `_create_schema`), the recipe schema (`sweep-trial` and `sweep`), `docs/contracts/project-folder.md` (the layout, the recipe's two fields, a `### sweep` section, a `### Sweeps` section under the operations), the service test's operation count (83 to 85). `tabs.js` and `base.css` are not changed: the tab is a new `sweeps.js` that wraps `S.tick`, and its styles and the new kind's colour are a new `sweeps.css`. `builds.py` is not changed: the runner uses `builds._Build` (`run_one`, the snapshot, the environment check) as `builds.execute` does. + +### T7a, the core + +- **`tuldok_project/sweeps.py`** holds all of it: the checks (`check_sweeps`, called at the end of `check.check_map`, so `set_definition` reports them too), the schedule, the halving rule (`Rungs`, `goes_on`), the trial's text and name, the plan of a sweep (`plan.build_plan` hands a sweep target to `sweeps.plan`), the runner (`sweeps.execute`, which the worker calls instead of `builds.execute` when the plan names a sweep), the two tables and the two reads, and the map's groups (`map_groups`, called by `workspace.project_map`). +- **The map's shape is only extended where there are sweeps.** Section 6 first said every model and group gained `sweep`; as built, only a sweep's group (`sweep: true`) and a trial's model (`sweep`: the reference) carry it, so the answers of projects without sweeps are byte for byte what they were (the workspace tests compare whole answers). +- **A pruned decision taken after the trial ended** (the last lines are read once more when its run ends) is recorded in its rungs (`goes_on` false), and its values count for the later trials, but the trial is recorded **completed** because its run succeeded. With the watcher reading every quarter second this happens only to a trial whose last rung falls in its last quarter second. +- **The sweep's and the recipe's texts are checked against the approval once, before anything runs** (refused `stale`); a trial's text is added to what the build expects before it runs, so `_check_expected` checks it as any step's. +- **jsonb orders an object's keys** (shorter keys first), so a document read back from the database (`definition`'s `document`) lists the parameters in another order than the text; the schedule reads the document, whose list order is kept, so it is not affected. +- **Tests:** `tests/test_tuldok_project_sweeps.py`, 24 (19 without a database, 5 with): the schedule (a grid against the test's own product, log points and integer rounding; random draws against the test's own `random.Random`; exhaustion; `1` and `1.0` one draw), halving (the rung points, a line crossing two rungs, `goes_on` with ties and both directions, decisions on metrics logs the test writes, lines appended between two reads, the minutes stopping a trial through its cancel file), the trial's text and its names, the paths, the checker (the schema's pointers, each rule beyond it, the grid's count, warning and bound, a recipe whose settings are in a file, a trial's `sweep` edge and a missing sweep), the plan and its covers, and three runs end to end: **halving through a started `Service`**, an agent's request approved once by the owner, six trials of the stand-in one after another with one plan id, the test's own arithmetic of the stand-in's floors and the halving rule giving each trial's state (completed or pruned), the pruned runs `cancelled` with the problem, the best, the frontier, the axes, the trials unsaved, derived from the recipe and in the sweep's group on the map, saved into the folder, and asking again refused `nothing-to-build`; **the minutes budget** stopping a sweep mid-trial (`ended` minutes, the last trial stopped); **a grid run twice**, its built trials reused and a third trial taking `-003-2` because `-003` held another recipe; and a stale recipe refused. The stand-in is `tests/project_sweep_stand_in.py`. + +### T7b, the page + +- **The tab** is `#sweeps`, after Train; `?sweep=` keeps the chosen sweep. Its drawer holds New sweep (`#sweeps/new`), the plan view (`#sweeps/plan:sweeps/<name>`, `actions.js`'s, whose Request build goes to the request on Train as everywhere) or a trial (the model's drawer). +- **In place:** the tab is painted once per sweep and per person's choice; a reading that changed something (`aside._again`) and every reading while the sweep runs or is asked for (a wrapper of `S.tick`) ask `sweep` and set the state block's texts and bar widths, the trial rows' cells (appending a row for a new trial), the chooser's word, the controls (rebuilt in their own box only when what they offer changed), and draw the charts and parallel coordinates again in their hosts only when their data changed. The check counts no node added to or removed from the state block across a whole run. +- **`parallel.js`** gained three optional fields of its `view`: `selected`, `lead` (the sentence under the heading) and `noun` ("trials"); the Benchmarks tab passes none and is unchanged. The trials table has the Benchmarks table's class and rows keyed by subject, so `LabParallel.apply` filters it unchanged. +- **The chart** is `charts.js`'s scatter with two objectives (completed trials only, the frontier from the answer) and its progression with one. +- **New sweep** writes a sweep from the chosen recipe's numeric and boolean settings (at most four: an integer as a range from half to twice it, a positive number as a log range from a tenth to ten times it), random, `loss` as the objective and eight trials, for the person to change and keep. It reads the recipe's document from the database, so its parameters come in jsonb's key order. +- **The kind of change "Sweep trial"** is a fifth entry of `adapter.js`'s kinds; its colour `--k-sweep-trial` is the purple of the training runs (`--run-3`, already checked beside the other kinds' colours), defined with every rule that colours a kind in `sweeps.css`. The editor's list of every definition lists Sweeps after Recipes. +- **The check** (`tests/browser_project_workspace_sweeps.cjs`, 7 PASS on its last two runs; the first runs failed on the check's own expectations of key order and of the map's section title, and once on the page: a reading 0.4 s after the tab was drawn was skipped by a guard against asking twice, so a request made meanwhile was not shown; the guard now applies only to `S.tick`): New sweep from the recipe, the halving sweep of six trials kept; an agent's request shown awaiting approval and approved on the tab; the state changing in place across four readings with the block the same element and no node of it added or removed; finished with three completed and three pruned, the frontier chart of the three, parallel coordinates of six lines through loss, time, learning rate and width, a range on the loss axis hiding rows; Open on the Map selecting trial 1 in the section "Small models / Learning rate and width" with the kind Sweep trial; light and dark at 1440 and 390 with nothing overflowing sideways (the trials table scrolls in its own box). + +### Runs at the end + +Disposable PostgreSQL under `/tmp/claude-1000/t7`, `TULDOK_PROJECT_HOME` and `TULDOK_SETTINGS` there, no GPU: + +- `PYTHONPATH=tests .venv/bin/python -m unittest discover -s tests -p 'test_tuldok_project_*.py'`: 878 OK (854 before, plus the 24 of `test_tuldok_project_sweeps`; `test_tuldok_project_check`'s every-kind assertion leaves out `sweep`, and the service test reads `sweeps` and `sweep`). +- `PYTHONPATH=tests .venv/bin/python -m unittest test_agent_api test_agent_negotiation test_agent_http`: 43 OK, 4 skipped. The catalog's digest is unchanged (`ad5cc7ac…`, 110 operations), checked in the new test file too. +- Browser: `browser_project_workspace_sweeps.cjs` 7 PASS; unchanged `browser_project_workspace.cjs` 24 PASS, `_actions.cjs` 27 PASS, `_training.cjs` 7 PASS, `_compare.cjs` 6 PASS. + +### Not verified + +A sweep of a real model or on the GPU (nothing was trained; the stand-in sleeps); halving in `time` units end to end (unit-tested only); a grid larger than its budget in a run (checked as a warning only); a sweep cancelled from the page while a trial runs (cancelling a running request is the existing path, and the runner's stop is unit-reached only through the minutes; not run end to end); Tuldok stopped mid-sweep (the orphan rule is written, not exercised); two sweeps asked for at once (the worker runs requests one at a time, so they queue; not run); the agent's decline of a sweep on the tab; a trial that does not check (recorded failed; not run); a sweep whose recipe needs a dataset built first (the prerequisite path is the plan's, run with the environment step only); many trials (the table and the reads have no bound beyond the budget's 1,000); keyboard use of the trial rows beyond Enter; the coding-standards guidance (routed; core, router, implementation and verification were read, the other twelve modules it listed were not). diff --git a/docs/research/competitive-landscape/README.md b/docs/research/competitive-landscape/README.md new file mode 100644 index 00000000..c3154478 --- /dev/null +++ b/docs/research/competitive-landscape/README.md @@ -0,0 +1,230 @@ +# Tuldok and the tools that do the same jobs + +Researched 2026-11-18. Four areas were surveyed in parallel: annotation and dataset curation; experiment tracking, lifecycle and versioning; synthetic data generation, model visualisation and evaluation harnesses; physics-AI and simulation-surrogate platforms. Web facts were read on 2026-11-18 unless dated otherwise. Open-source repositories were cloned shallowly and read, never built or run, under `workspace/scratch/competitor-repos/` (about 3.9 GB in all: label-studio, cvat, argilla, doccano, mlflow, clearml, dvc, aim, zenml, distilabel, netron, WeightWatcher, BlenderProc, model-explorer, lm-evaluation-harness, physicsnemo, neuraloperator, PDEBench, the_well, deepxde). FiftyOne (2.1 GB) was grepped and deleted; Rerun was read through the GitHub tree. + +**[unverified]** marks a claim taken from vendor marketing, an aggregator or memory and not confirmed in a primary source. Tuldok's column describes the branch `feat/definition-first-projects` as of today (project-folder mode, the Map workspace, data tools phases 1 to 6) plus catalog mode. + +## 1. The short answer + +**Nobody else does the whole loop.** Tuldok's purpose is: make or import data, look inside it, annotate it, freeze it, define models and how they are trained, build them, benchmark them, and see the results. That spans what is usually four products: + +| Job | Usual tools | Tuldok | +| --- | --- | --- | +| Annotate and curate | Label Studio, CVAT, FiftyOne, Encord, Labelbox | Annotate tab, Look inside, Freeze | +| Track, version, reproduce | MLflow, W&B, ClearML, DVC, ZenML | Git folder of definitions, Map, Train, builds | +| Generate data, inspect models, evaluate | distilabel, Replicator, Netron, TensorBoard, lm-eval | Generate via Pumas, Weights, Benchmarks | +| Physics surrogates | PhysicsNeMo, NeuralOperator, SimAI, PhysicsX, Simcenter | Benchmarks, Analysis (kept fields), project's own solver step | + +**Three things only Tuldok has**, confirmed by searching the code of the open tools and the documentation of the closed ones: + +1. **An accuracy-against-time chart with the Pareto frontier.** No tracker draws one (MLflow, ClearML, DVC, ZenML, Aim: the word "pareto" appears in MLflow only inside its GEPA prompt optimiser, and in ClearML's SDK only where `HyperParameterOptimizer` sorts multi-objective trials by successive Pareto fronts, a ranking with no chart; see section 5a), no harness (lm-eval, HELM, Inspect, Promptfoo, MLPerf) and no physics benchmark (PDEBench, The Well, PhysicsNeMo-CFD) draws one. MLPerf sets an accuracy pass mark and then ranks by speed; HELM and Promptfoo add speed as a column; W&B can be made to draw one with a custom Vega panel. For a project with two targets, accuracy-first models and real-time models, this is the central view, and it is Tuldok's. +2. **A family tree of models.** Others draw how files were produced: W&B artifact lineage, ClearML parent tasks, ZenML pipeline DAGs, DVC `dvc dag`, Valohai data heritage, Ansys SimAI's 2026 R1 Build panel. None shows which model derives from which with the kind of change, groups and comparisons, as the Map does. The nearest real model tree is the Hugging Face Hub's `base_model` tree, outside these tools. +3. **A project that is a Git folder of definitions only,** with the working copy in a database, weights and products kept out of Git on the machine, and no hash lockfiles. Only DVC (`dvc.yaml`, plots declared in files, experiments as Git refs) and Valohai (`valohai.yaml`) keep definitions in Git, and DVC rests on hash lockfiles. Charts declared as files: Tuldok and DVC only. + +**Also rare:** agents may ask for a build and only the owner approves it (Valohai's `require-approval` nodes and ZenML's Slack ask-steps come close; no tool has a queue of agent requests); annotations stored as text in Git (CVAT removed its Git sync; Hugging Face datasets are Git repositories but have no annotation); one viewer for text, image, audio, point clouds, meshes and arrays (FiftyOne has no audio; Label Studio Community has no 3D; Rerun views but does not annotate); weights compared with the weights a model started from or with its parent (WeightWatcher compares base and fine-tuned models; nobody else links it to an ancestry). + +**Where Tuldok trails,** in the order that matters for its stated purpose: + +1. **Physics-specific learning:** no model zoo or pretrained physics models (PhysicsNeMo, NeuralOperator, The Well checkpoints), no mesh/CAD-native geometric learning, no physics-informed losses, no confidence or out-of-range flag per prediction (SimAI, physicsAI and Rescale all sell this). +2. **Scale:** no multi-GPU, cluster or cloud training; no hyperparameter search. +3. **Live training views:** no live loss curves, no weight histograms over training, no parallel-coordinates plot (nearly every tracker has one). +4. **Annotation breadth:** no polygons, masks, keypoints, cuboids or tracking; no video; no QA by consensus or ground truth; no curation by embeddings or similarity; no rank or rate questions; no standard export formats (COCO, YOLO, KITTI): Tuldok writes JSON and JSONL. +5. **Model structure:** no drawn architecture graph (Netron, Model Explorer, TensorBoard), no activations or attribution (Captum). +6. **Generation breadth:** Pumas 0.8.0 makes text and PNG images only; nothing renders 3D scenes with ground truth (Replicator, BlenderProc, Kubric), and generated data gets no real-against-synthetic quality report (SDV, MOSTLY AI). +7. **Ecosystem:** no hosted option, no large community, no ready task catalogues. + +**The market is unstable,** which favours Tuldok's choice of plain definition files that do not depend on one vendor: + +- Neptune shut down in March 2026 and deleted user data with no export. +- lakeFS bought DVC (2025-11) and moved lakeFS itself to BSL 1.1 (2026-09-22); DVC stays Apache-2.0. +- ClearML's server is SSPL, W&B self-hosting needs a licence, and SDV is BSL. +- Argilla is in maintenance only, HELM entered maintenance mode on 2026-06-01, Unity Perception is discontinued, and NeMo Microservices shut down on 2026-10-01. +- Gretel went to NVIDIA, Monolith to CoreWeave, Altair to Siemens, Promptfoo to OpenAI (announced 2026-03-09), and W&B is CoreWeave's. +- Determined, Pachyderm and Guild AI look dormant, and Aim's main branch has had no commits since 2025-12-31. + +## 2. Area by area + +### 2.1 Annotation, curation and looking at data + +| Tool | Licence, price | Closest overlap with Tuldok | Better than Tuldok | Tuldok has, it lacks | +| --- | --- | --- | --- | --- | +| **Label Studio** 1.23 | Apache-2.0 open core; Starter Cloud and Enterprise by quote | Its ML backend: an HTTP server with `/predict`, `/setup`, `/train`, `/health`, predictions stored with a `model_version` for a person to accept (`label_studio/ml/api_connector.py`). The nearest thing to Pumas suggestions | About 20 control tags (polygon, brush, keypoint, relations, ranker, pairwise, time series, video tracking, PDF); about 10 export formats | 3D in the open edition (LiDAR is Enterprise), meshes, arrays, annotations in Git, frozen releases, training and benchmarks | +| **CVAT** 2.83 | MIT; cloud from $23/user/month yearly | 3D point clouds with cuboids, audio (new in 2026), and a pull-based CLI agent that runs your own model | The 2D/3D vision editor, tracking with interpolation, quality control and consensus, about 30 export formats | Meshes, arrays, text; Git sync was removed (`cvat/apps/dataset_repo/README.md`); no releases; no training | +| **FiftyOne** 1.22 | Apache-2.0 open core; enterprise "Voxel51" by quote | Meshes (glTF, OBJ, PLY, STL), point clouds, Gaussian splats, MCAP/ROS/Rerun import | Curation: views, embeddings, similarity, evaluation panels, large model zoo | Audio; annotations in Git; versioning (Enterprise only: read-only snapshots) | +| **Rerun** 0.43 | Apache/MIT; Hub priced per deployment | Viewing `volume_3d`, `voxel_grid_map`, `tensor`, `mesh3d`, audio | Large time-indexed 3D and volume viewing; a reference design for solver fields | Any annotation | +| **Argilla** 2.8 | Apache-2.0, maintenance only | Suggestions with the model that made them | Rank and rate questions, vector search | 3D, audio, being maintained | +| **Doccano** 1.8 | MIT | Auto-labelling through any REST endpoint, with a response mapping | Sequence labelling, intent and slot | 3D, releases, training | +| **Prodigy** | Proprietary; $390 personal, $490 per seat | Local, scriptable, model-in-the-loop accept or reject | The fastest accept/reject flow; any API from a recipe | 3D, frozen releases, open source | +| **Hugging Face Hub** | Hosted; viewer on private data needs a paid plan | Every dataset is a Git repository | Distribution at scale, SQL console | 3D viewer, annotation | +| **Encord, Labelbox, V7, Supervisely, SuperAnnotate, Scale Nucleus, Roboflow** | Closed, mostly by quote | Encord: LiDAR to 10M points a frame, DICOM; Supervisely: the widest modality range, meshes by export apps | Team workflows, QA, active learning, hosted models | Local-first, open, Git text, links to training and benchmarks | + +### 2.2 Tracking, lifecycle, versioning + +| Tool | Licence, price | Core model | Better than Tuldok | Tuldok has, it lacks | +| --- | --- | --- | --- | --- | +| **MLflow** 3.18 | Apache-2.0 | Logged runs in SQLite by default; registry; GenAI tracing | Autologging, scale, serving; a built-in MCP server and an in-app assistant (`mlflow/mcp/server.py`, `mlflow/assistant/`); parallel coordinates and scatter | Definitions in Git, model tree, Pareto chart, approval, annotation, weights | +| **W&B** | SDK MIT, server proprietary; self-hosting licensed | Logged runs; Artifacts lineage DAG | Polish, Sweeps, Reports, collaboration; official MCP server | Free local use beyond personal, definitions in Git, approval | +| **ClearML** 2.2 | SDK Apache-2.0, server SSPL | Auto-captured runs; clone, edit, re-enqueue | Remote execution on queues, comparisons, HPO | Open server licence, Git definitions, model tree | +| **DVC** | Apache-2.0 (now owned by lakeFS) | `dvc.yaml` stages in Git, plots in files, `dvc repro` | Content-addressed remote for large files, experiments as Git refs | GUI (only Studio), roles, MCP, model semantics; Tuldok avoids its hash lockfiles | +| **ZenML** 0.98 | Apache-2.0; Pro paid | Code-defined pipelines, model control plane | Orchestration across stacks; official MCP server; ask-steps for approval | Git definitions, model tree, Pareto | +| **Valohai** | Commercial | `valohai.yaml` in Git, immutable outputs | `require-approval` pipeline nodes, managed compute | Open source, local | +| **Aim, TensorBoard, Trackio** | Apache-2.0 / Apache-2.0 / MIT | Local logged metrics | Live curves, histograms, HParams, projector (TensorBoard); W&B-compatible logging (Trackio) | Definitions, lineage, roles | +| **Kubeflow, Metaflow, Polyaxon, Determined, Pachyderm** | Apache-2.0 | Cluster pipelines | Distributed training, HPO | Local simplicity; two are dormant | +| **Neptune** | Shut down March 2026 | | | A warning about SaaS-only tracking | + +### 2.3 Generation, model inspection, evaluation + +| Tool | What it is | Better than Tuldok | Tuldok has, it lacks | +| --- | --- | --- | --- | +| **distilabel** (Apache-2.0; last commit 2025-12-15) | LLM pipelines for synthetic text: Self-Instruct, Evol-Instruct, Magpie, UltraFeedback, judges; any OpenAI-compatible `base_url`; Ray | Recipes and judges, many providers, scale-out | Review built in, frozen releases, non-text data | +| **NeMo Data Designer** (ex-Gretel, Apache-2.0) | Column-by-column datasets from samplers and LLMs, with validators | Declarative schema with validators | Review, freeze, training | +| **Omniverse Replicator / Cosmos** (NVIDIA licences) | Randomised USD scenes rendered with ground truth (boxes, segmentation, depth, point clouds) | Physically based rendering and sensor simulation | Openness, review, freeze | +| **BlenderProc** (GPL-3), **Kubric** (Apache-2.0) | Blender rendering pipelines with depth, segmentation, flow; Kubric adds PyBullet physics | 3D scenes with ground truth, video | Both could run as a Tuldok project's own command | +| **SDV** (BSL), **MOSTLY AI** (Apache-2.0 SDK) | Tabular generators with quality reports | Real-against-synthetic fidelity scores | Tuldok has no tabular generation | +| **Netron** (MIT) | Graph viewer for about 180 formats; opens `.pt` pickles with a JavaScript unpickler that runs no Python (`source/python.js`) | Architecture graph; safe reading of nearly every format | Distributions, kernels as tiles, change against parent or start, links to runs and benchmarks | +| **Model Explorer** (Apache-2.0) | GPU-drawn graphs, side-by-side graphs, per-node data overlays from JSON | Overlays and graph comparison | It loads `.pt2` with `torch.export.load`, trusting the file | +| **WeightWatcher** (Apache-2.0) | Per-layer spectral quality (alpha) without test data; base vs fine-tuned | Data-free layer quality | Ancestry, GUI; it loads `.bin` with `torch.load` and relies on torch's `weights_only` default | +| **TensorBoard** | Scalars, histograms over steps, graph, projector | Training-time views | Definitions, ancestry | +| **lm-eval, HELM, OpenCompass, Inspect, Promptfoo, MLPerf** | Language-model evaluation; MLPerf for systems | Task catalogues, standard errors, per-sample transcripts, sandboxes | Any non-language metric, the Pareto chart, ancestry | + +Tuldok never opens a pickle: `.pt` files are listed with the reason (spec-d2, decision 4). Netron's approach, a pickle reader that runs no code, is the one to copy if `.pt` files must be read. + +### 2.4 Physics AI and surrogates + +The field has four layers. Tuldok sits in a fourth that only closed enterprise products occupy, and they aim at a different buyer. + +1. **Libraries you fork:** + - **PhysicsNeMo** 2.2 (Apache-2.0; Hydra configs; FNO, MeshGraphNet, Transolver, DoMINO; multi-GPU and domain-parallel; CFD metrics including force coefficients and spectra). Its `benchmarks/` folder times library code, not models. + - **NeuralOperator** 2.0 (MIT; FNO, TFNO, UNO, SFNO, GINO and more). + - **DeepXDE** (LGPL-2.1; physics-informed networks). + - None has a project, a model tree, an accuracy-against-time view or a GUI. +2. **Benchmark datasets:** + - **PDEBench** (MIT, but `pdebench/models/metrics.py` carries an NEC non-commercial header: do not copy it). + - **The Well** (BSD-3; 15 TB across 16+ datasets; the best-specified HDF5 format; a ready true / predicted / |error| video per field). + - Both report accuracy only. +3. **Solvers that make data:** PhiFlow, JAX-Fluids, Warp and Newton, Genesis World 1.x, Taichi. Each could be a Tuldok project's own solver step. +4. **Commercial surrogate platforms, the real competitors:** + + | Platform | What it offers | Notes | + | --- | --- | --- | + | Ansys SimAI (now Synopsys) | Full 3D fields from 30–100 earlier simulations, with confidence for out-of-range designs | An automatic evaluation report; a Build panel in 2026 R1; a desktop Pro tier | + | Siemens Simcenter ROM, HEEDS AI and the former Altair physicsAI / romAI | Geometric learning on native meshes, flags for out-of-range designs | FMU export for real-time system simulation | + | PhysicsX | Data Factory, large geometry models, applications | Raised $300M in June 2026 [unverified] | + | Neural Concept | CAD-native design copilot | Raised $100M in December 2025 | + | Rescale AI Physics | Cloud HPC solvers with a dataset-creation agent, Auto Tune and confidence estimates | | + | Monolith (CoreWeave) | Engineering test data | | + | Luminary Cloud | Cloud CFD with a surrogate | | + + All are closed and priced by quote. + +**Nobody targets games.** The nearest things are FMU reduced-order models for real-time system simulation (Siemens, Altair), NVIDIA's Omniverse digital-twin blueprint (a live surrogate in a rendered scene, not a game export), and Taichi's AOT runtime. A benchmark subject that is the model *as deployed* (timed on ONNX Runtime or inside an engine) would be a feature nobody in the field has. + +**How kept fields compare** (phase 6): The Well's true / predicted / |error| rows and PhysicsNeMo's `pred_`/`true_` mesh fields are offline files for ParaView or a video. SimAI and Simcenter show predicted fields in 3D viewers (side-by-side error views [unverified]). Tuldok's Analysis tab shows native, predicted and *signed* error side by side, with whole-field figures, in the same place as the benchmark that scored the model. + +## 3. Feature matrix + +✓ yes · ~ partly or in a paid tier only · ✗ no · ? unverified. One column per family where the members agree; the area sections above have the detail. + +| Capability | Tuldok | Label Studio | CVAT | FiftyOne | MLflow | W&B | ClearML | DVC | ZenML | TensorBoard | Netron | PhysicsNeMo | The Well / PDEBench | SimAI / Simcenter / PhysicsX | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Fully open source | ✓ | ~ open core | ✓ | ~ open core | ✓ | ✗ | ~ SSPL server | ✓ | ~ | ✓ | ✓ | ✓ | ✓ | ✗ | +| Local-first, no account | ✓ | ✓ | ✓ | ✓ | ✓ | ~ personal | ~ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ~ desktop tier | +| Project = definitions in Git | ✓ | ✗ | ✗ | ✗ | ~ | ✗ | ✗ | ✓ | ~ | ✗ | – | ~ configs | ~ configs | ✗ | +| Weights kept out of Git | ✓ | – | – | – | ✓ | ✓ | ✓ | ✓ | ✓ | – | – | ✓ | ✓ | ✓ | +| Text / image / audio data | ✓ | ✓ | ~ no text | ~ no audio | – | ~ | ~ | – | – | ~ | – | ✗ | ✗ | ✗ | +| Point cloud / mesh / arrays | ✓ | ~ Ent LiDAR | ~ PCD only | ✓ | – | ✗ | ✗ | – | – | ✗ | – | ✓ (in code) | ✓ arrays | ✓ meshes | +| Video | ✗ | ✓ | ✓ | ✓ | – | ~ | ~ | – | – | ✗ | – | ✗ | ✗ | ? | +| Annotation, shapes beyond boxes | ✗ | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Annotations as Git text | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ~ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Model suggestions, human accepts | ✓ Pumas | ✓ ML backend | ✓ Nuclio, agent | ~ in-process | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Frozen release with provenance | ✓ | ~ snapshots | ✗ | ~ Ent | ~ | ✓ Artifacts | ✓ Datasets | ✓ | ✓ | ✗ | ✗ | ✗ | ✓ fixed sets | ? | +| Curation by embeddings | ✗ | ✗ | ✗ | ✓ | ✗ | ~ | ~ | ✗ | ✗ | ~ projector | ✗ | ✗ | ✗ | ✗ | +| Standard export formats | ✗ JSON/JSONL | ✓ | ✓ ~30 | ✓ | – | – | – | – | – | – | – | ✓ HDF5/Zarr | ✓ HDF5 | ? | +| Data generation step | ✓ Pumas text/PNG; own command | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ stages | ✓ steps | ✗ | ✗ | ✗ | ✓ PDEBench scripts | ~ | +| Training regime as definition | ✓ recipes | ✗ | ✗ | ✗ | ~ MLproject | ~ | ~ | ✓ | ✓ | ✗ | ✗ | ✓ Hydra | ✓ Hydra | ~ automatic | +| Rebuild from definitions | ✓ | ✗ | ✗ | ✗ | ~ | ~ | ~ | ✓ | ~ | ✗ | ✗ | ~ | ~ | ✗ | +| Live training curves | ~ progress | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ~ | ~ | ✓ | ✗ | ✓ TB | ✓ | ? | +| Hyperparameter search | ✗ | ✗ | ✗ | ✗ | ~ | ✓ | ✓ | ✗ | ~ | ✗ | ✗ | ~ | ✗ | ✓ automatic | +| Multi-GPU / cluster / cloud | ✗ | – | – | – | ✗ | ~ | ✓ | ✗ | ✓ | – | – | ✓ | ~ | ✓ | +| Model family tree | ✓ Map | ✗ | ✗ | ✗ | ✗ | ~ artifact DAG | ~ parents | ~ dag | ~ DAG | ✗ | ✗ | ✗ | ✗ | ~ Build panel | +| Benchmarks as definitions | ✓ | ✗ | ✗ | ~ | ✗ | ✗ | ✗ | ~ metrics | ✗ | ✗ | ✗ | ~ | ✓ | ✗ | +| Accuracy-vs-time Pareto chart | ✓ | ✗ | ✗ | ✗ | ✗ | ~ custom Vega | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Parallel coordinates | ✗ | ✗ | ✗ | ✗ | ✓ | ✓ | ✓ | ~ | ✗ | ✓ | ✗ | ✗ | ✗ | ? | +| Charts declared as files | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Per-case values | ✓ | – | – | ✓ | ~ | ~ | ~ | ✗ | ✗ | ✗ | ✗ | ✓ metrics.jsonl | ~ | ? | +| Native / predicted / error fields | ✓ Analysis | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ~ files | ~ video | ✓ 3D viewer | +| Confidence per prediction | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ~ UQ | ✗ | ✓ | +| Weights: layers, distributions | ✓ | ✗ | ✗ | ✗ | ✗ | ✓ histograms | ✗ | ✗ | ✗ | ✓ | ~ stats | ✗ | ✗ | ✗ | +| Weights: change vs parent / start | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | +| Architecture graph | ✗ | ✗ | ✗ | ✗ | ✗ | ~ | ✗ | ✗ | ✗ | ✓ | ✓ | ✗ | ✗ | ✗ | +| Pretrained models / zoo | ✗ | ✗ | ~ | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | – | ✓ | ✓ | ✓ | +| Agents request, owner approves | ✓ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ✗ | ~ ask step | ✗ | ✗ | ✗ | ✗ | ✗ | +| MCP server | ✓ with grants | ~ | ~ | ? | ✓ | ✓ | ~ community | ✗ | ✓ | ✗ | ✗ | ~ skill | ✗ | ~ SDK | +| Roles | ✓ owner/edit/inspect | ~ Ent | ✓ | ~ Ent | ✓ | ✓ | ~ paid | ~ Studio | ~ Pro | ✗ | ✗ | ✗ | ✗ | ✓ | +| Hosted option, large community | ✗ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Real-time / game deployment | ✗ (a stated goal) | – | – | – | ~ serving | ✗ | ~ | ✗ | ~ | – | – | ~ Omniverse twin | ✗ | ~ FMU ROMs | + +## 4. What to do with this + +### Defend (what nobody else has, and what serves both targets) + +1. The tradeoff chart and frontier. Add an MLPerf-style preset: "accuracy at least X, then fastest", and allow a benchmark subject to be the model *as deployed* (ONNX Runtime, a game engine) so the real-time target is measured where it runs. +2. The Map. Keep derivations, kinds of change and comparisons first-class; nobody else has them. +3. Definition-only Git folders with no hash lockfiles, and charts as files. +4. Owner approval of agent requests. MLflow, W&B and ZenML now have MCP servers, but none has grants or an approval queue. + +### Borrow (ideas and formats, not code, unless the licence allows) + +| Idea | From | Size | +| --- | --- | --- | +| Import The Well and PDEBench HDF5; PhysicsNeMo `metrics.jsonl` as per-case values and `pred_`/`true_` fields as kept fields | The Well, PDEBench, PhysicsNeMo | Small to medium | +| Spectral (Fourier-band) error, rollout errors over time, integrated quantities (forces) as benchmark metrics | PDEBench, The Well, PhysicsNeMo-CFD | Small | +| A confidence / out-of-range flag per prediction | SimAI, physicsAI, Rescale | Medium (needs the training code's help) | +| Parallel coordinates on Benchmarks | MLflow, W&B, TensorBoard | Small | +| Live training curves on Train from a run's progress | every tracker | Small | +| Weight distributions across checkpoints in training order | TensorBoard, W&B | Small (the lab keeps checkpoints) | +| An architecture graph from a recipe's model definition | Netron, Model Explorer | Medium | +| A per-layer spectral quality column | WeightWatcher (Apache-2.0) | Small | +| A pickle reader that runs no code, if `.pt` must be read | Netron (MIT) | Medium | +| Polygons, keypoints, masks; COCO / YOLO / KITTI export | Label Studio, CVAT | Medium each | +| Rank and rate annotation questions | Argilla, Label Studio | Small | +| Similarity and embedding views for curation | FiftyOne | Medium | +| A real-against-synthetic quality report for generated datasets | SDV, MOSTLY AI | Medium | +| Generation templates (Magpie, Evol-Instruct, judges) for `generate` | distilabel | Small | +| A read-only MCP mode | W&B | Small | + +### Call rather than rebuild (complements) + +PhysicsNeMo and NeuralOperator models as a recipe's training code; PhiFlow, Warp, Genesis or JAX-Fluids as a project's solver step; BlenderProc or Kubric as a scene generator command; Label Studio or CVAT as an external annotator for shapes Tuldok does not draw, with their exports imported. + +### Licence cautions + +- **Do not copy:** PDEBench's `pdebench/models/metrics.py` (NEC non-commercial header). +- **Copyleft:** DeepXDE is LGPL-2.1 and BlenderProc is GPL-3. +- **Source-available, not open source:** SDV is BSL, lakeFS is BSL from 2026-09-22, and the ClearML server is SSPL. +- **Fine with attribution:** PhysicsNeMo, NeuralOperator, The Well, Netron, WeightWatcher, Label Studio's open edition, CVAT, FiftyOne's open edition and MLflow are permissive. + +## 5a. Checked by hand in the clones + +The four areas were researched by separate agents. The claims this document leans on were then checked again directly in the cloned code on 2026-11-18: + +| Claim | Where | Result | +| --- | --- | --- | +| CVAT removed its Git sync | `cvat/apps/dataset_repo/README.md` | Confirmed: "…synchronizing CVAT tasks with Git repositories, which was removed." | +| No tracker draws an accuracy-vs-time Pareto chart | whole-word `pareto` in mlflow, clearml, dvc, aim, zenml | **Corrected.** Earlier wording said a search finds nothing. MLflow: only `genai/optimize/optimizers/gepa_optimizer.py`, `genai/judges/optimizers/gepa.py` and their protos and docs (prompt optimisation). ClearML: `automation/optimization.py:1207`, trials sorted by Pareto fronts. DVC, Aim, ZenML: none. No chart in any of them; a case-insensitive search also matches `compareTo…` names, which explains the agent's false positives | +| Label Studio Community has no point clouds | `web/libs/editor/src`, for `lidar`, `pointcloud`, `.pcd` | Confirmed: no file | +| PDEBench's metrics file is not MIT | `pdebench/models/metrics.py`, header | Confirmed: NEC Laboratories Europe copyright header, "All rights reserved" | +| Netron reads pickles with its own reader | `netron/source/python.js:3895` | Confirmed: `registerType('pickle.Unpickler', …)` in JavaScript | +| Argilla adds no new features | `argilla/README.md` | Confirmed: "we won't be adding new features going forward" | +| MLflow has a built-in MCP server | `mlflow/mcp/server.py` | Confirmed | +| The Well renders true / predicted / error videos | `the_well/benchmark/metrics/plottable_data.py:227` | Confirmed: `make_video` | +| PhysicsNeMo's `benchmarks/` times library code | `physicsnemo/benchmarks/` | Confirmed: an ASV-style suite (`run_benchmarks.sh`), no model accuracy | + +Claims about closed products, prices and 2026 company events were not checked again beyond the sources each area cites. They are listed below where they could not be confirmed. + +## 5. Not verified + +- **Commercial physics platforms:** most capability claims are marketing, especially whether they show error fields and whether they track lineage. +- **Prices and plans:** Labelbox, V7, Roboflow Core, Voxel51 Enterprise and the W&B tiers. +- **Capabilities:** Labelbox and V7 LiDAR support; whether FiftyOne has an MCP server; JAX-Fluids' licence; Taichi's current activity. +- **Recent events:** PhysicsX's 2026 round, the Anaconda–Outerbounds deal, and the date Warp's licence changed. diff --git a/docs/runbooks/external-project-extraction.md b/docs/runbooks/external-project-extraction.md deleted file mode 100644 index 682a689b..00000000 --- a/docs/runbooks/external-project-extraction.md +++ /dev/null @@ -1,255 +0,0 @@ -# Extracting a legacy project into a portable directory - -This runbook describes the M5 extraction contract under implementation. It is -not an instruction to migrate a personal catalog before the extraction gate is -accepted. Work on a disposable representative fixture first. Do not inspect or -migrate a user's local lab as a product acceptance test. - -An extraction keeps the original stopped source and creates a new, independent -project. It is neither a database migration nor a generic SQL export. Historical -materials, Runs, Trials, source observations and their hashes retain their -original namespace. Editable copies receive destination identities through -their domain owners. No credentials, live grants, admissions, process control or -resumable operation authority enter the destination. - -## Preview first - -Preview uses a read-only, repeatable-read PostgreSQL view and pinned filesystem -capabilities. It does not construct `Metadata`, `Dataset`, Jobs or Runs; the -legacy `Metadata` constructor can append `.metadata-init.lock`, even with -`create=False`. Preview does not repair schema, enroll images, execute scripts, -install dependencies, contact producers, or regenerate missing bytes. - -Select one exact registered project snapshot and explicit roots from the finite -families: data records, responses, preferences, saved selections and searches, -graphs, recipe drafts, model definitions, Studies, Runs, producer histories and -working files. A registration snapshot identifies captured external files; it -does not establish ownership of otherwise global data, Studies or Runs. - -Review the report's required dependency closure, shared dependencies, unassigned -global records, ambiguities, missing payloads and unsupported source facts. -Parent/baseline lineage can require another Study as historical provenance. That -expansion is visible in the report and must not be presented as inferred -ownership. A required dependency cannot be excluded while claiming a complete -extraction. A saved search preserves entered criteria; its current search -results are not implicitly selected membership. - -The report binds the exact selection, source schema and snapshot, backup and -external inventory. Extraction requires the same report digest and selection. -An unresolved or incomplete report is not successful empty-project creation. - -When the selected content and unique payloads are available, Preview validates -the captured native domain composition while the source is still live. Domain -refusals appear before an owner has to stop the source to discover them. This -does not satisfy cessation: the independent `cessation_proof_unavailable` -blocker remains until the existing trusted owner completes Stop and capture. - -## Stop through the existing owner - -The explicit stop operation belongs to the currently opened legacy controller. -It first fences new requests and drains existing request handlers. It verifies -that this instance owns all five known runtime claims, drains the four producer -workers and standalone image requests, and asks the existing Run owner to prove -local worker and external custodian drainage. A peer's runtime ownership, -unconfirmed descendant cessation, failed drain, unknown outcome or lost lock -prevents a stopped-source capability from being issued. - -An old backup receipt, `quiesced=True`, a process ID, an idle activity message or -a stored terminal state is not this proof. A newly opened application without -the original live owners can still preview, but reports -`cessation_proof_unavailable` until a trusted owner performs stop and capture. - -The controller issues an opaque, local stopped-source lease only after confirmed -drainage, peer exclusion, read-only snapshot pinning, a verified backup and an -external payload inventory. The lease owns its connection and directory -capabilities. It cannot be reconstructed from JSON or imported project facts. - -Stopping can legitimately persist cancellations or interruptions through the -existing runtime owners. The source-unchanged baseline is recorded **after** -that explicit stop and verified backup. Extraction itself must preserve that -baseline exactly. - -## Preserve original evidence - -Verify required unique payload bytes, including native BYTEA bundles, original -images/audio/scientific sources, release ZIPs, execution inputs and outputs, -runtime manifests and captured scripts. Resolve files only through current -explicit capabilities; historical absolute paths are inert evidence. - -Missing original bytes cannot be replaced by running a producer, re-exporting a -release, recreating a manifest's formatting or remeasuring a result. Declared -regenerable dependencies retain visible availability. Missing required unique -bytes block a complete extraction. - -Legacy storage has genuine history gaps. Saved selections/searches retain only -their current row, and image source revisions are not a complete source event -stream. A report must identify absent historical revisions, timestamps or -bindings. It must not invent earlier names, source revisions, audit events or -complete-history claims. An observed-history representation requires explicit -domain support; otherwise extraction remains blocked for that case. - -## Publish and verify the destination - -Use a new destination under an authorized root. Source, stopped backup, -destination and task-owned staging must not overlap. Existing destinations are -not overwritten or merged. Stage validated domain records and immutable payloads -without writable hard links, then publish atomically through the project store. -Project-owned scripts and notes remain ordinary inert UTF-8 files; copying them -does not execute them. - -The extraction receipt binds the source, selection/report, request identity, -destination project/checkpoint, identity mappings, artifact completeness and -independent cold reconstruction. The deciding test opens the new project with a -fresh physical PostgreSQL projection and checks retained identities, hashes, -history, closure and source preservation. Current permissions come from the new -installation, never the imported source's access tables. - -## Use the process-local transfer controller - -The transfer controller takes the already running legacy Dataset and a separate, -inert control workspace. Configure publication root aliases for the verified -backup and destination; the destination alias must also be a root of the control -workspace. Root paths are pinned directory capabilities, and backup and project -names are single relative directory names. Symlink roots, publication roots -inside the source catalog, overlapping targets and existing backup targets are -refused before Stop. - -The five fixed controller actions are `project_transfer.status`, `preview`, -`report_page`, `stop` and `extract`. These are controller methods; the application -transport exposes only its corresponding validated routes. An ordinary Open or -preview cannot stop a source or create a stopped-source lease. - -1. Authenticate as the current legacy owner with administration permission. Read - Status and submit the explicit snapshot, root selection and decisions to - Preview with that process binding. Preview remains read only. Its summary - gives the report digest and counts; Report Page exposes one of `included`, - `required`, `unassigned`, `excluded`, `blockers`, `warnings`, `artifacts` or - `working_files` in pages of at most 40 items. Keep the same report digest while - reviewing pages. -2. Request Stop with the reviewed live report digest, backup root/directory and - proposed destination root/directory. Stop validates both target locations, - drains the actual source owner and creates the private verified backup. A - successful Stop rotates the process binding and clears the live report. -3. Sign in explicitly using credentials issued independently by the control - workspace. Source credentials are not copied, rebound or accepted as control - credentials. The stopped source's authentication database may already be - closed. Refresh Status and Preview the same explicit selection again; the - stopped-source report now binds the verified backup and cessation capability. -4. Extract using that exact stopped report digest, configured destination, - display name and a stable request key. Publication first produces a durable - independent project. The controller then attempts an ordinary inert Open in - the control workspace. Inspect the separate `selection.selected` result; - committed publication is preserved if workspace selection fails. - -The controller binding is an opaque process-local correlation value. Neither a -request nor a stored report contains the stopped-source capability. Restarting -the process does not reconstruct that capability from JSON. A failed stop after -the native source fence keeps the source fenced and exposes the failure through -independently authenticated control Status. If existing handlers have not yet -drained, the application delays its ordinary-route authentication handoff until -they finish. A native authorization denial before installation of that fence -leaves the source phase live and requires a refreshed report. - -The bounded public receipt gives publication identity, source/report/input -digests, copied/missing artifact counts and independent verification. Artifact -declarations remain available through report pages. Its checkpoint and -verification describe the **current verified destination head** on an exact-key -retry, including legitimate later destination edits. The source, selection, -report and input digests remain immutable birth facts; retry does not extract -again. A changed body with the same key conflicts. - -Keep the exact key after a lost publication acknowledgement. A typed unknown -outcome may already carry `committed: true` and `selected: false`; controller -Status retains that partial receipt. Retry reconciles the actual destination. -If publication failed before rename, preserve the reported staging directory for -inspection. Do not substitute a new key or assume the destination is absent. - -On retry, reconcile the exact request key and input digest before attempting -another publication. Keep a published destination. If publication is uncertain, -retain task-owned staging and receipts until its outcome is checked; do not -delete a destination to make a retry appear clean. Report the exact known or -uncertain outcome and preserve the original stopped source. - -## Current owner API - -These are internal owner interfaces. They do not add a public HTTP, SDK, MCP or -CLI capability until those consumers have been qualified against the same -selection, completeness and outcome contracts. - -```python -# Use the already opened legacy controller and its authenticated current actor. -# parent_fd is a caller-owned directory capability outside the source catalog. -from project_transfer import ProjectTransfer, preview_source - -selection = { - 'project_snapshot': {'id': registered_id, 'manifest_sha256': manifest_hash}, - 'roots': {'records': [chosen_record_id], 'studies': [chosen_study_id]}, - 'decisions': [], -} -with preview_source(existing_dataset) as view: - review = ProjectTransfer(view).preview(selection) -# This first report can inspect closure but cannot claim cessation proof. - -with existing_dataset.stop_for_project_extraction(parent_fd, 'stopped-backup') as source: - transfer = ProjectTransfer(source) - report = transfer.preview(selection) - # Review can_extract/blockers, required dependencies and unassigned inventory. - receipt = transfer.extract(selection, report['report_sha256'], parent_fd, - 'new-project', 'New independent project', - 'caller-owned-extraction-key') -``` - -The selection accepts only the finite root arrays `records`, `responses`, -`preferences`, `selections`, `searches`, `graphs`, `recipe_drafts`, -`model_definitions`, `studies`, `runs`, `releases`, `exposures`, `model_materials`, -`recipe_materials`, `runtime_materials`, `producer_jobs` and `working_files`. -Omitted arrays are empty. Producer roots identify one known kind and job ID; -working files identify `script`/`note` plus an exact `scripts/...`/`notes/...` -path. Explicit decisions identify a known family and exact canonical ID, an -include/exclude choice and the entered reason. They do not authorize omitting a -required dependency. - -The stopped backup is private legacy custody and includes the original access -rows. Portable domain records exclude those rows. The canonical extraction -manifest retains only non-secret source digests, the normalized selection, -actual first-adoption observations for selected working files, fixed-family -population counts and copied/missing-regenerable artifact identities. - -A successful receipt has `committed: true` and `selected: false`: publication -creates a project without changing the application's selection. A lost final -acknowledgement returns an `unknown` error carrying its known committed identity -when that identity can be proven. Inspect that destination and repeat the exact -key/input. Before publication, failure retains `staging_directory`; it does not -claim the destination exists. A missing unique payload, absent legacy history, -unknown cessation, unsupported schema or required exclusion stays a blocker. - -Current focused evidence uses disposable PostgreSQL 17 fixtures and private -filesystem roots. It covers ordinary cold Open under a different installation, -current scoped inspection and revocation, actual handler drainage, peer -exclusion, exact source/archive preservation, malformed or missing source facts, -required closure, failed publication and lost-reply reconciliation. Historical -fixture custody is not a claim that an imported process was executed again or -that an old scientific result was remeasured. - -The extraction-only closer has a bounded deadline. If an existing owner does not -finish closing, the controller returns `source_busy`, keeps new requests fenced -and issues no stopped-source lease. Its single owned closer may finish later; -that does not turn a timed-out request into a successful extraction or revive -ordinary requests. A closure failure or unconfirmed descendant remains unknown. -The ordinary application's `close()` semantics are unchanged. - -The focused owner gate is: - -```sh -# Run from the repository root with a disposable PostgreSQL fixture DSN only. -PYTHONPATH="tests:.:${PYTHONPATH:-}" python -m unittest test_project_transfer -v -``` - -The current owner checkpoint passed **20 tests in 17.495 seconds** on the private -PostgreSQL 17.11 fixture. The cross-Study case retains two Studies, two Trials, -parent and baseline lineage, a settled reservation, a candidate, native Run -artifacts, model and recipe materials, a frozen release and shared source bytes. -A separate ordinary Open uses a different current installation and fresh -physical SQL projection, while constructor tripwires verify that Jobs and Runs -do not start. The historical case establishes custody and reconstruction only; -current-host preflight and actual execution have their own gates. diff --git a/docs/runbooks/project-composition.md b/docs/runbooks/project-composition.md deleted file mode 100644 index 76757fd6..00000000 --- a/docs/runbooks/project-composition.md +++ /dev/null @@ -1,54 +0,0 @@ -# Compose retained projects - -The owner operation `project_files.compose` creates one independent project from -an open editable primary and one or more explicit retained project checkpoints. -It preserves the original scientific evidence and forks the primary editable -notes, data and definitions through the same owners as Save As. Secondary drafts -and operation receipts remain inert historical sources. No source project is -modified and no imported program, runtime or credential becomes active. - -Negotiate the operation's version-1 contract through the native project owner -client. The route is `POST /api/workspace/project-files/compose`. In addition to -the current `binding`, `expected_checkpoint` and one fresh `request_key`, supply: - -- `root`, `directory` and `name` for a new destination under a configured root; -- `sources`, an ordered list of additional selections, each containing `root`, - `directory`, `project_id` and `checkpoint`. - -The current project is implicitly the first source. At most 15 additional -sources are accepted by the HTTP contract. All source projects must be idle and -available for native exclusive store ownership. The primary's captured working -files must be clean. The destination cannot contain, lie inside or alias any -source, and existing unrelated destinations are never overwritten. - -Equal canonical envelopes share one retained representation. Same-ID conflicts -are rejected except for Run documents that differ only in their historical -approval grant/consumption while retaining the same admission and every other -field. The complete alternate document is retained as a native Run observation; -it does not become a second Run or create a disclosure. Both documents remain -subject to the history owner's proof and payload validation. - -The persisted `project_composition_v1` record binds the ordered source identities -and exact canonical envelope hashes to the destination's ordinary fork overlay. -A source membership explicitly identifies any alternate Run observation needed -to recover that source's document. Its decoder rejects missing, altered or -contradictory memberships. Subsequent ordinary forks preserve and archive this -provenance. Older software that does not support this family must reject the -project explicitly; records must not be dropped for compatibility. - -Payloads are verified and copied as independent ordinary files. A missing copy -can be supplied by another source with the same verified declaration; a corrupt -selected source is rejected. Unique bytes cannot be omitted. Only dependencies -classified as regenerable by their domain owners can remain explicitly missing. - -On success, `state` describes the newly selected project, `fork` reports the -primary origin and copy completeness, and `sources` lists every selected source -identity/checkpoint in order. The local owner installation retains its own -credentials. Existing Save As and lifecycle-state contracts are unchanged. - -Retain a request before dispatch. An interrupted publication can leave an inert -stage or a complete unselected destination. Inspect the original request and -owner state before reconciling with the same key and arguments. Native receipts -bind reconciliation to the complete ordered source set. A timeout does not -justify another request key or deleting a retained stage. Selection failure -returns a durable unselected receipt while preserving the current primary. diff --git a/examples/tiny-project/README.md b/examples/tiny-project/README.md new file mode 100644 index 00000000..0496be1f --- /dev/null +++ b/examples/tiny-project/README.md @@ -0,0 +1,173 @@ +# Tiny project + +The smallest real Tuldok project: a dataset, a network trained on a GPU, a +benchmark and two charts, and beside them a few small files of each kind +brought in, annotated and frozen with splits. Building it end to end takes about a minute, and it +uses every part of a build: environments, a command per definition, a product +passed from one definition to the next, progress lines while training, a +benchmark's results imported into the database, and charts drawn from them. +Copy it as the starting point for a project of your own. + +The task: read 64 noisy samples of a sine wave whose amplitude, frequency and +phase are random, and predict the next 16 samples of the clean wave. + +| Folder | Holds | +| --- | --- | +| `project/` | the project folder: `tuldok.json` and the definitions | +| `code/` | the code root (`code_root` is `../code`): the scripts the commands run | + +| Definition | What it does | +| --- | --- | +| `environments/cpu` | Python 3.12 with numpy 2.3.5 | +| `environments/gpu` | Python 3.14 with torch 2.14.0+cu132 and numpy 2.5.3, with CUDA | +| `datasets/waves` | `make_waves.py` writes `waves.npz`: 4,000 training and 400 test waves from seed 7 | +| `models/small-net` | `net.py`'s `SmallNet`: fully connected, 64 to 256 to 256 to 16 | +| `recipes/small-net-a` | `train.py` trains it on the GPU for the updates in `code/settings/small-net-a.json` | +| `benchmarks/held-out` | `bench.py` scores a subject on the 400 test waves: error (RMSE) and time per wave | +| `charts/held-out-ranking` | the subjects ranked by error | +| `charts/held-out-tradeoff` | error against time per wave | +| `datasets/samples` | the step `import`: the files under `code/samples` (two texts, a PNG, a WAV, a PLY point cloud and an OBJ mesh, written by `make_samples.py`), one record each | +| `annotations/sample-quality` | a `class` task on `datasets/samples`: whether each sample is clean; its annotations are `annotations/sample-quality.jsonl` | +| `datasets/samples-v1` | the step `freeze`: the samples whose quality is accepted, split three to one from seed 7 | +| `datasets/scenes` | the step `import`: the files under `code/scenes` (three images, two point clouds, an array of frames and a series, written by `make_scenes.py`) | +| `annotations/scene-masks`, `annotations/scene-objects` | a `mask` task (instance masks of the images' objects) and a `points` task (the top of each point cloud) on `datasets/scenes` | +| `datasets/scene-masks-a`, `-b`, `datasets/scene-objects-a`, `-b` | the step `annotate`: `code/annotators/standin.py`, a stand-in segmentor, run at two thresholds; each is an annotation set for its task | + +The benchmark has two subjects: `small-net-a`, the trained network, and +`predict-zero`, which always predicts zero and shows the error any model has to +beat. + +## What it needs + +- Two Python interpreters that already exist on the machine (there is no + `install` section, so Tuldok does not install them; it connects them): + - one with **Python 3.12 and numpy 2.3.5** for `environments/cpu`; + - one with **Python 3.14, torch 2.14.0+cu132 and numpy 2.5.3**, on a machine + where torch sees a CUDA GPU, for `environments/gpu`. + + Versions are checked exactly. If yours differ, edit the two files under + `project/environments/` to match, or the check fails and says what it found. +- A Postgres database for Tuldok's working copy, named by `TULDOK_DATABASE_URL` + (or `--database-url` on every command). +- The example committed to Git. A build runs each command from a read-only copy + of the commit, so uncommitted or untracked changes under `code/` are not seen: + a changed tracked file stops the build, and a file that was never committed is + missing from the copy. `--scratch` runs from the working tree instead and marks + the products as scratch products. + +## Build it + +Run these from the Tuldok repository root. Replace the two interpreter paths +with your own; they must be absolute. + +```sh +export TULDOK_DATABASE_URL=postgresql://... +P=examples/tiny-project/project + +# Check the folder: expect "0 errors, 0 warnings". +.venv/bin/python -m tuldok_project check $P + +# Connect the two interpreters. Each is checked at once; seconds each. +.venv/bin/python -m tuldok_project connect $P cpu /path/to/python3.12-with-numpy/bin/python +.venv/bin/python -m tuldok_project connect $P gpu /path/to/python3.14-with-torch/bin/python +.venv/bin/python -m tuldok_project environments $P + +# See what would be built. Runs nothing. +.venv/bin/python -m tuldok_project plan $P --all + +# Build everything: prints the plan, asks, then runs each step in turn. +.venv/bin/python -m tuldok_project build $P --all +``` + +The plan has ten build steps after the two environments: `datasets/waves`, +`recipes/small-net-a`, then `benchmarks/held-out` once for each subject, +`datasets/samples`, `datasets/samples-v1` and `datasets/scenes`, which +Tuldok's own steps build in a moment with no environment, and the four +annotators, which run `standin.py` in `environments/cpu` in a moment each. + +| Step | Time | +| --- | --- | +| `datasets/waves` | about a second on the CPU | +| `recipes/small-net-a` | 6,000 updates; well under a minute on a GPU, much of it starting CUDA. The recipe allows 6 minutes | +| `benchmarks/held-out` | a few seconds per subject, mostly starting torch | + +### The samples, annotated + +`datasets/samples` brings the files under `code/samples` in as records; open +it on the Data tab of the page (`python app.py --data CATALOG --project-folder +$P`) to look inside. On the Annotate tab, `annotations/sample-quality` shows +each record with its annotation; change one, press Keep and next, and Save +writes `annotations/sample-quality.jsonl`. `datasets/samples-v1` takes the +records whose annotation is accepted, as committed, and splits them; build it +again after committing a change to the annotations. To write the samples +again, run `python make_samples.py` from `code/`: the bytes are the same. + +### Two segmentors, compared + +`datasets/scene-masks-a` and `-b` run the same stand-in segmentor at two +thresholds, so their masks differ by a rim of a pixel; `scene-objects-a` and +`-b` take slightly different tops of each cloud. On the Annotate tab, choose +`annotations/scene-masks` and Review: the two sets' agreement (matched IoU), +the records they disagree on most, each drawn over its image side by side or +overlaid, and Keep A, Keep B or Reject both, which Save writes to +`annotations/scene-masks.jsonl` and its mask to `annotations/scene-masks/`. To +write the scenes again, run `python make_scenes.py` from `code/`. + +### Watch the training + +While the recipe runs, in a second terminal: + +```sh +.venv/bin/python -m tuldok_project runs $P +.venv/bin/python -m tuldok_project progress $P RUN --follow +``` + +`RUN` is the run id from `runs` or from the line `build` printed when the step +started; its first six characters are enough. `progress` shows the latest +`tuldok_progress_v1` line (stage `train`, updates done of 6,000, and the loss) +and the ends of the two logs, every two seconds until the run ends. + +### Look at the results + +```sh +.venv/bin/python -m tuldok_project results $P held-out +.venv/bin/python -m tuldok_project chart $P held-out-ranking --svg ranking.svg +.venv/bin/python -m tuldok_project chart $P held-out-tradeoff --svg tradeoff.svg +.venv/bin/python -m tuldok_project report $P --output report.html +.venv/bin/python -m tuldok_project save $P +``` + +`build` imports each subject's `results.json` into the database, so the charts +and the report work straight away. Write the SVG and HTML files somewhere +outside the project folder; `save` writes the benchmark's summary to +`project/results/held-out.csv`, the one built thing that belongs in the folder. + +The trained network's median error should be far below the baseline's: on the +CPU, even 100 updates bring it to about 0.07 against 0.87 for `predict-zero`. + +### Build again + +A second `build $P --all` finds every step `built` and runs nothing. Change +`code/settings/small-net-a.json` (say, `"updates": 3000`) and `plan $P --all` +shows the recipe and the benchmark as out of date; commit the change and build +again to bring them up to date. + +## Running the scripts by hand + +The scripts take everything from their arguments and write only into +`--output`. To try them without a GPU, `train.py` and `bench.py` accept +`--device cpu`, which the definitions never pass; without it they refuse to run +when CUDA is missing. From `code/`, with an interpreter that has numpy and +torch, and an output directory outside the repository (datasets and weights are +never committed): + +```sh +OUT=$(mktemp -d) && mkdir $OUT/data $OUT/train $OUT/bench-net $OUT/bench-zero +python make_waves.py --output $OUT/data --seed 7 +python train.py --data $OUT/data/waves.npz --settings settings/small-net-a.json --output $OUT/train --device cpu +python bench.py --data $OUT/data/waves.npz --subject model:$OUT/train/model.pt --name small-net-a --output $OUT/bench-net --device cpu +python bench.py --data $OUT/data/waves.npz --subject zero --name predict-zero --output $OUT/bench-zero +``` + +On the CPU the 6,000 updates take about ten seconds. Set +`PYTHONDONTWRITEBYTECODE=1` to keep `__pycache__` out of `code/`. diff --git a/examples/tiny-project/code/annotators/standin.py b/examples/tiny-project/code/annotators/standin.py new file mode 100644 index 00000000..cc40a7d4 --- /dev/null +++ b/examples/tiny-project/code/annotators/standin.py @@ -0,0 +1,168 @@ +"""A stand-in annotator for the example: what a project's own segmentor would do, in a few lines of the standard library. + +Run by Tuldok's step annotate (docs/contracts/project-folder.md), it reads the records to annotate from +$TULDOK_TODO and the task from $TULDOK_TASK, and appends one line per record to $TULDOK_ANNOTATIONS, writing masks +and arrays under $TULDOK_FILES. Its one argument is a threshold, 0 to 255: two runs with different thresholds are +two "segmentors" that mostly agree, which is what the Annotate tab's Review compares. + + python annotators/standin.py 160 + +What it makes, by the task's type: on the scenes' images, pixels darker than the threshold are objects (one per +colour): a mask (an instance PNG), polygons (each object's outline box), keypoints (its top left and bottom right +corners) and a depth map (objects nearer than the ground); on the point clouds, the points higher than +threshold / 255 are one object (points) and its box (box3d); on an array of frames, the bright square's box in each +frame (a track; above a threshold of 180 the track is split in two at frame 3, an identity switch); on a series, +the stretches where the first channel is high (timeseries; above 180 each stretch ends one position later). +""" +import ast +import json +import os +import struct +import sys +import zlib + +THRESHOLD = float(sys.argv[1]) +TASK = json.load(open(os.environ['TULDOK_TASK'], encoding='utf-8')) +FILES = os.environ['TULDOK_FILES'] + + +def read_png(path): + """(width, height, rows of (r, g, b)) of an RGB PNG whose rows are not filtered (as make_scenes.py writes).""" + data = open(path, 'rb').read() + width, height = struct.unpack('>II', data[16:24]) + at, body = 8, b'' + while at < len(data): + length, kind = struct.unpack('>I4s', data[at:at + 8]) + if kind == b'IDAT': + body += data[at + 8:at + 8 + length] + at += 12 + length + raw = zlib.decompress(body) + stride = width * 3 + 1 + return width, height, [[tuple(raw[y * stride + 1 + x * 3:y * stride + 4 + x * 3]) for x in range(width)] + for y in range(height)] + + +def write_png(path, width, height, values): + rows = b''.join(b'\x00' + bytes(values[y * width:(y + 1) * width]) for y in range(height)) + + def chunk(kind, body): + return struct.pack('>I', len(body)) + kind + body + struct.pack('>I', zlib.crc32(kind + body)) + with open(path, 'wb') as out: + out.write(b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 0, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(rows)) + chunk(b'IEND', b'')) + + +def read_npy(path): + data = open(path, 'rb').read() + length = struct.unpack('<H', data[8:10])[0] + header = ast.literal_eval(data[10:10 + length].decode('latin-1')) + body, count = data[10 + length:], 1 + for n in header['shape']: + count *= n + values = list(body[:count]) if header['descr'] == '|u1' else list(struct.unpack(f'<{count}f', body[:4 * count])) + return list(header['shape']), values + + +def write_npy(path, shape, values): + header = "{'descr': '<f4', 'fortran_order': False, 'shape': (%s), }" % ', '.join(map(str, shape)) + header += ' ' * (64 - (10 + len(header) + 1) % 64) + '\n' + with open(path, 'wb') as out: + out.write(b'\x93NUMPY\x01\x00' + struct.pack('<H', len(header)) + header.encode('latin-1') + + struct.pack(f'<{len(values)}f', *values)) + + +def objects(path): + """(width, height, ids row by row, {colour: id}) of the image's pixels darker than the threshold.""" + width, height, pixels = read_png(path) + ids, found = [0] * (width * height), {} + for y in range(height): + for x in range(width): + r, g, b = pixels[y][x] + if (r + g + b) / 3 < THRESHOLD: + ids[y * width + x] = found.setdefault('blue' if b > r else 'red', len(found) + 1) + return width, height, ids, found + + +def corners(ids, width, number): + xs = [i % width for i, v in enumerate(ids) if v == number] + ys = [i // width for i, v in enumerate(ids) if v == number] + return min(xs), min(ys), max(xs) + 1, max(ys) + 1 + + +def image_value(record): + width, height, ids, found = objects(record['path']) + kind = TASK['task'] + if kind == 'mask': + name = record['id'] + '.png' + write_png(os.path.join(FILES, name), width, height, ids) + return {'file': name, 'classes': {str(n): TASK['vocabulary'][0] for n in found.values()} or {'1': TASK['vocabulary'][0]}, + 'confidence': round(1 - abs(THRESHOLD - 180) / 400, 2)} + if kind == 'depth': + name = record['id'] + '.npy' + write_npy(os.path.join(FILES, name), [height, width], + [2.0 + THRESHOLD / 1000 if v else 5.0 for v in ids]) + return {'file': name} + value = [] + for colour, number in sorted(found.items()): + x0, y0, x1, y1 = corners(ids, width, number) + if kind == 'polygons': + value.append({'term': colour, 'points': [[x0, y0], [x1, y0], [x1, y1], [x0, y1]], + 'confidence': round(0.95 - 0.1 * number, 2)}) + elif kind == 'keypoints': + value.append({'term': colour, 'points': [[x0, y0, 2], [x1, y1, 2]]}) + return value + + +def cloud_value(record): + rows = open(record['path'], encoding='ascii').read().split('end_header\n')[1].split('\n') + points = [[float(word) for word in row.split()] for row in rows if row.strip()] + high = [i for i, p in enumerate(points) if p[2] > THRESHOLD / 255] + if TASK['task'] == 'points': + return [{'term': TASK['vocabulary'][0], 'indices': high}] if high else [] + if not high: + return [] + low = [min(points[i][axis] for i in high) for axis in range(3)] + top = [max(points[i][axis] for i in high) for axis in range(3)] + return [{'term': TASK['vocabulary'][0], 'center': [round((a + b) / 2, 3) for a, b in zip(low, top)], + 'size': [round(max(0.05, b - a), 3) for a, b in zip(low, top)], 'yaw': round(THRESHOLD / 2550, 3)}] + + +def array_value(record): + shape, values = read_npy(record['path']) + if TASK['task'] == 'track' and len(shape) == 3: + count, height, width = shape + track = {'track': 'square', 'term': TASK['vocabulary'][0], 'frames': []} + tracks = [track] + for frame in range(count): + lit = [i for i in range(height * width) if values[frame * height * width + i] > 128] + if not lit: + continue + xs, ys = [i % width for i in lit], [i // width for i in lit] + box = {'frame': frame, 'x': min(xs), 'y': min(ys), 'w': max(xs) - min(xs) + 1, 'h': max(ys) - min(ys) + 1} + if THRESHOLD > 180 and frame == 3: + track = {'track': 'square-2', 'term': TASK['vocabulary'][0], 'frames': []} + tracks.append(track) + track['frames'].append(box) + return tracks + if TASK['task'] == 'timeseries' and len(shape) == 2: + level = values[::shape[1]] + value, start = [], None + for n, v in enumerate(level + [0.0]): + if v > 0.5 and start is None: + start = n + elif v <= 0.5 and start is not None: + value.append({'term': TASK['vocabulary'][0], 'start': start, + 'end': min(len(level), n + (1 if THRESHOLD > 180 else 0))}) + start = None + return value + return None + + +with open(os.environ['TULDOK_ANNOTATIONS'], 'a', encoding='utf-8') as out: + for line in open(os.environ['TULDOK_TODO'], encoding='utf-8'): + record = json.loads(line) + value = (image_value(record) if record['kind'] == 'image' else + cloud_value(record) if record['kind'] == 'pointcloud' else + array_value(record) if record['kind'] == 'array' else None) + if value is not None: + out.write(json.dumps({'record': record['id'], 'value': value}) + '\n') diff --git a/examples/tiny-project/code/bench.py b/examples/tiny-project/code/bench.py new file mode 100644 index 00000000..386f932f --- /dev/null +++ b/examples/tiny-project/code/bench.py @@ -0,0 +1,98 @@ +"""Score one subject on the 400 test waves and write results.json (format tuldok_results_v1). + +A subject is either `model:<path of model.pt>`, a trained SmallNet, or `zero`, +a baseline that always predicts zero: the error a model has to beat. +""" +import argparse +import datetime +import json +import os +import statistics +import sys +import time + +import numpy as np +import torch + +from net import SmallNet + +WARM_UP = 5 # untimed passes first: the first GPU calls pay for start-up, not for the model +REPEATS = 50 # timed passes; their median is the time + + +def model_predictor(path, device): + model = SmallNet() + model.load_state_dict(torch.load(path, map_location='cpu', weights_only=True)) + model = model.to(device).eval() + + def predict(inputs): + with torch.no_grad(): + outputs = model(torch.from_numpy(inputs).to(device)) + return outputs.cpu().numpy() # the copy back waits for the GPU, so the clock sees the whole pass + return predict + + +def zero_predictor(inputs): + return np.zeros((len(inputs), 16), dtype=np.float32) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--data', required=True, help='the waves.npz file') + parser.add_argument('--subject', required=True, help='model:<path of model.pt>, or zero') + parser.add_argument('--name', required=True, help="the subject's name, written into results.json") + parser.add_argument('--output', required=True, help='the directory to write results.json into') + # As in train.py: the benchmark runs models on the GPU; cpu is for smoke tests. + parser.add_argument('--device', choices=['cuda', 'cpu'], default='cuda') + arguments = parser.parse_args() + + if arguments.subject == 'zero': + predict = zero_predictor # needs no GPU, so it runs anywhere + elif arguments.subject.startswith('model:'): + if arguments.device == 'cuda' and not torch.cuda.is_available(): + sys.exit('bench.py: CUDA is not available to torch, and this benchmark runs models on the GPU. ' + 'Pass --device cpu to smoke-test the script by hand.') + if arguments.device == 'cpu': + torch.set_num_threads(1) # as in train.py: one thread is fastest for a network this small + predict = model_predictor(arguments.subject[len('model:'):], torch.device(arguments.device)) + else: + sys.exit(f'bench.py: unknown subject {arguments.subject!r}; expected model:<path> or zero.') + + with np.load(arguments.data) as waves: + inputs, targets = waves['test_inputs'], waves['test_targets'] + + # Time the whole batch, as a model would be used, and report it per wave. + # Every case of the subject gets the same time; the median of many passes + # keeps one slow pass from deciding it. + for _ in range(WARM_UP): + predict(inputs) + passes = [] + for _ in range(REPEATS): + started = time.perf_counter() + predictions = predict(inputs) + passes.append(time.perf_counter() - started) + time_ms = statistics.median(passes) * 1000.0 / len(inputs) + + errors = np.sqrt(np.mean((predictions.astype(np.float64) - targets) ** 2, axis=1)) + values = [] + for index, error in enumerate(errors): + case = f'test-{index:03d}' + values.append({'case': case, 'metric': 'error', 'value': float(error)}) + values.append({'case': case, 'metric': 'time_ms', 'value': time_ms}) + + document = { + 'format': 'tuldok_results_v1', + 'benchmark': 'held-out', + 'subject': arguments.name, + 'commit': '', # Tuldok records the commit of the run itself + 'produced': datetime.datetime.now(datetime.timezone.utc).date().isoformat(), + 'values': values, + } + with open(os.path.join(arguments.output, 'results.json'), 'w', encoding='utf-8') as handle: + json.dump(document, handle, indent=1) + handle.write('\n') + print(f'{arguments.name}: median error {float(np.median(errors)):.4f}, {time_ms:.6f} ms per wave') + + +if __name__ == '__main__': + main() diff --git a/examples/tiny-project/code/make_samples.py b/examples/tiny-project/code/make_samples.py new file mode 100644 index 00000000..9bd26169 --- /dev/null +++ b/examples/tiny-project/code/make_samples.py @@ -0,0 +1,79 @@ +"""Writes samples/, the few small files datasets/samples imports: two texts, a PNG, a WAV, a PLY point cloud and +an OBJ mesh, about thirteen kilobytes in all. Standard library only; the same bytes on every run. + + python make_samples.py (from code/; writes samples/ beside this file) +""" +import io +import math +import os +import struct +import wave +import zlib + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def png(width, height, boxes): + """An RGB PNG of a pale background with filled rectangles (x, y, w, h, colour).""" + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + + def colour(x, y): + for bx, by, bw, bh, rgb in boxes: + if bx <= x < bx + bw and by <= y < by + bh: + return rgb + return (232, 236, 240) + rows = b''.join(b'\x00' + b''.join(bytes(colour(x, y)) for x in range(width)) for y in range(height)) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(rows, 9)) + chunk(b'IEND', b'')) + + +def wav(rate=8000, seconds=1.2): + """8-bit mono at 8,000 Hz: two tones (0.20 to 0.55 s at 250 Hz, 0.80 to 1.10 s at 400 Hz) with silence around them.""" + frames = int(rate * seconds) + samples = bytearray() + for n in range(frames): + t = n / rate + tone = 250 if 0.20 <= t < 0.55 else 400 if 0.80 <= t < 1.10 else 0 + samples.append(128 + (round(90 * math.sin(2 * math.pi * tone * t)) if tone else 0)) + buffer = io.BytesIO() + with wave.open(buffer, 'wb') as sound: + sound.setnchannels(1) + sound.setsampwidth(1) + sound.setframerate(rate) + sound.writeframes(bytes(samples)) + return buffer.getvalue() + + +def ring(count=120): + lines = ['ply', 'format ascii 1.0', f'element vertex {count}', 'property float x', 'property float y', + 'property float z', 'end_header'] + for n in range(count): + t = 2 * math.pi * n / count + lines.append(f'{math.cos(t):.3f} {math.sin(t):.3f} {0.2 * math.sin(3 * t):.3f}') + return '\n'.join(lines) + '\n' + + +PYRAMID = '\n'.join(['v 0 0 0', 'v 1 0 0', 'v 1 1 0', 'v 0 1 0', 'v 0.5 0.5 0.8', + 'f 1 3 2', 'f 1 4 3', 'f 1 2 5', 'f 2 3 5', 'f 3 4 5', 'f 4 1 5']) + '\n' + +FILES = { + 'texts/greeting.txt': 'Ada Lovelace wrote to Charles Babbage from London 🌍 in 1843.\n', + 'texts/weather.txt': 'Rain over Manila this morning; clear skies in Cebu by the afternoon.\n', + 'images/shapes.png': png(64, 48, [(6, 8, 22, 18, (44, 95, 184)), (36, 20, 20, 22, (210, 101, 47))]), + 'sounds/two-tones.wav': wav(), + 'clouds/ring.ply': ring(), + 'meshes/pyramid.obj': PYRAMID, +} + + +def main(): + for name, data in FILES.items(): + path = os.path.join(HERE, 'samples', *name.split('/')) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'wb') as handle: + handle.write(data if isinstance(data, bytes) else data.encode('utf-8')) + + +if __name__ == '__main__': + main() diff --git a/examples/tiny-project/code/make_scenes.py b/examples/tiny-project/code/make_scenes.py new file mode 100644 index 00000000..0fbaedb3 --- /dev/null +++ b/examples/tiny-project/code/make_scenes.py @@ -0,0 +1,88 @@ +"""Writes scenes/, the small files datasets/scenes imports for the annotators: three images of dark rectangles +with a softer rim, two point clouds rising in height, an array of frames with a square moving across them and a +two-channel series. About five kilobytes in all. Standard library only; the same bytes on every run. + + python make_scenes.py (from code/; writes scenes/ beside this file) +""" +import math +import os +import struct +import zlib + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def png(width, height, rectangles): + """An RGB PNG of dark rectangles (x, y, w, h, 'red' or 'blue') with a one-pixel rim halfway to white.""" + def pixel(x, y): + for bx, by, bw, bh, colour in rectangles: + dark = (40, 40, 200) if colour == 'blue' else (200, 40, 40) + if bx <= x < bx + bw and by <= y < by + bh: + return dark + if bx - 1 <= x < bx + bw + 1 and by - 1 <= y < by + bh + 1: + return tuple((255 + value) // 2 for value in dark) + return (255, 255, 255) + rows = b''.join(b'\x00' + b''.join(bytes(pixel(x, y)) for x in range(width)) for y in range(height)) + + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(rows, 9)) + chunk(b'IEND', b'')) + + +def cloud(count): + """An ASCII PLY of count points along a line rising from z 0 to 1.""" + lines = ['ply', 'format ascii 1.0', f'element vertex {count}', 'property float x', 'property float y', + 'property float z', 'end_header'] + lines += [f'{n * 0.1:.2f} {0.05 * (n % 3):.2f} {n / (count - 1):.3f}' for n in range(count)] + return ('\n'.join(lines) + '\n').encode('ascii') + + +def npy(shape, values, descr): + """An NPY (version 1) array in C order.""" + header = "{'descr': '%s', 'fortran_order': False, 'shape': (%s), }" % ( + descr, ''.join(f'{n}, ' for n in shape) if len(shape) == 1 else ', '.join(map(str, shape))) + header += ' ' * (64 - (10 + len(header) + 1) % 64) + '\n' + code = {'|u1': 'B', '<f4': '<f'}[descr] + body = bytes(values) if code == 'B' else struct.pack(f'<{len(values)}f', *values) + return b'\x93NUMPY\x01\x00' + struct.pack('<H', len(header)) + header.encode('latin-1') + body + + +def frames(count=6, height=16, width=20): + """count frames of a 4 by 4 square (220) on a dark ground (20), moving two pixels right each frame.""" + values = [] + for frame in range(count): + x0 = 2 + 2 * frame + values += [220 if x0 <= x < x0 + 4 and 6 <= y < 10 else 20 for y in range(height) for x in range(width)] + return npy((count, height, width), values, '|u1') + + +def series(length=48): + """Two channels: a level that is high from 10 to 20 and 30 to 38, and a slow wave.""" + values = [] + for n in range(length): + values += [1.0 if 10 <= n < 20 or 30 <= n < 38 else 0.0, round(math.sin(n / 6), 4)] + return npy((length, 2), values, '<f4') + + +FILES = { + 'images/a.png': png(24, 16, [(2, 2, 6, 5, 'red'), (14, 6, 7, 7, 'blue')]), + 'images/b.png': png(24, 16, [(4, 4, 10, 8, 'red')]), + 'images/c.png': png(24, 16, [(1, 1, 4, 4, 'blue'), (10, 3, 5, 9, 'blue')]), + 'clouds/p.ply': cloud(20), + 'clouds/q.ply': cloud(30), + 'frames/moving.npy': frames(), + 'series/signal.npy': series(), +} + + +def main(): + for name, data in FILES.items(): + path = os.path.join(HERE, 'scenes', *name.split('/')) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'wb') as handle: + handle.write(data) + + +if __name__ == '__main__': + main() diff --git a/examples/tiny-project/code/make_waves.py b/examples/tiny-project/code/make_waves.py new file mode 100644 index 00000000..fa831a80 --- /dev/null +++ b/examples/tiny-project/code/make_waves.py @@ -0,0 +1,45 @@ +"""Write waves.npz: noisy sine waves to read, and the clean continuation to predict. + +Every wave has its own amplitude, frequency and phase, so a model has to read +them from the 64 noisy samples it is given before it can continue the wave. +""" +import argparse +import os + +import numpy as np + +SEEN = 64 # noisy samples a model reads +AHEAD = 16 # clean samples it must predict +NOISE = 0.1 # standard deviation of the noise on the samples it reads +COUNTS = {'train': 4000, 'test': 400} + + +def waves(generator, count): + amplitude = generator.uniform(0.5, 2.0, size=(count, 1)) + # Radians per sample: periods from about 16 to 125 samples, so every input + # holds at least half a period and the task is never hopeless. + frequency = generator.uniform(0.05, 0.4, size=(count, 1)) + phase = generator.uniform(0.0, 2.0 * np.pi, size=(count, 1)) + clean = amplitude * np.sin(frequency * np.arange(SEEN + AHEAD) + phase) + noisy = clean[:, :SEEN] + generator.normal(0.0, NOISE, size=(count, SEEN)) + return noisy.astype(np.float32), clean[:, SEEN:].astype(np.float32) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', required=True, help='the directory to write waves.npz into') + parser.add_argument('--seed', type=int, required=True) + arguments = parser.parse_args() + + # One generator drawn in a fixed order: the same seed gives the same file. + generator = np.random.default_rng(arguments.seed) + arrays = {} + for split, count in COUNTS.items(): + arrays[f'{split}_inputs'], arrays[f'{split}_targets'] = waves(generator, count) + path = os.path.join(arguments.output, 'waves.npz') + np.savez(path, **arrays) + print(f'wrote {path}: ' + ', '.join(f'{count} {split}' for split, count in COUNTS.items())) + + +if __name__ == '__main__': + main() diff --git a/examples/tiny-project/code/net.py b/examples/tiny-project/code/net.py new file mode 100644 index 00000000..a6bf969b --- /dev/null +++ b/examples/tiny-project/code/net.py @@ -0,0 +1,11 @@ +"""The small network that models/small-net names: 64 samples in, 16 samples out.""" +from torch import nn + + +class SmallNet(nn.Sequential): + def __init__(self, seen=64, hidden=256, ahead=16): + super().__init__( + nn.Linear(seen, hidden), nn.ReLU(), + nn.Linear(hidden, hidden), nn.ReLU(), + nn.Linear(hidden, ahead), + ) diff --git a/examples/tiny-project/code/samples/clouds/ring.ply b/examples/tiny-project/code/samples/clouds/ring.ply new file mode 100644 index 00000000..7098931e --- /dev/null +++ b/examples/tiny-project/code/samples/clouds/ring.ply @@ -0,0 +1,127 @@ +ply +format ascii 1.0 +element vertex 120 +property float x +property float y +property float z +end_header +1.000 0.000 0.000 +0.999 0.052 0.031 +0.995 0.105 0.062 +0.988 0.156 0.091 +0.978 0.208 0.118 +0.966 0.259 0.141 +0.951 0.309 0.162 +0.934 0.358 0.178 +0.914 0.407 0.190 +0.891 0.454 0.198 +0.866 0.500 0.200 +0.839 0.545 0.198 +0.809 0.588 0.190 +0.777 0.629 0.178 +0.743 0.669 0.162 +0.707 0.707 0.141 +0.669 0.743 0.118 +0.629 0.777 0.091 +0.588 0.809 0.062 +0.545 0.839 0.031 +0.500 0.866 0.000 +0.454 0.891 -0.031 +0.407 0.914 -0.062 +0.358 0.934 -0.091 +0.309 0.951 -0.118 +0.259 0.966 -0.141 +0.208 0.978 -0.162 +0.156 0.988 -0.178 +0.105 0.995 -0.190 +0.052 0.999 -0.198 +0.000 1.000 -0.200 +-0.052 0.999 -0.198 +-0.105 0.995 -0.190 +-0.156 0.988 -0.178 +-0.208 0.978 -0.162 +-0.259 0.966 -0.141 +-0.309 0.951 -0.118 +-0.358 0.934 -0.091 +-0.407 0.914 -0.062 +-0.454 0.891 -0.031 +-0.500 0.866 -0.000 +-0.545 0.839 0.031 +-0.588 0.809 0.062 +-0.629 0.777 0.091 +-0.669 0.743 0.118 +-0.707 0.707 0.141 +-0.743 0.669 0.162 +-0.777 0.629 0.178 +-0.809 0.588 0.190 +-0.839 0.545 0.198 +-0.866 0.500 0.200 +-0.891 0.454 0.198 +-0.914 0.407 0.190 +-0.934 0.358 0.178 +-0.951 0.309 0.162 +-0.966 0.259 0.141 +-0.978 0.208 0.118 +-0.988 0.156 0.091 +-0.995 0.105 0.062 +-0.999 0.052 0.031 +-1.000 0.000 0.000 +-0.999 -0.052 -0.031 +-0.995 -0.105 -0.062 +-0.988 -0.156 -0.091 +-0.978 -0.208 -0.118 +-0.966 -0.259 -0.141 +-0.951 -0.309 -0.162 +-0.934 -0.358 -0.178 +-0.914 -0.407 -0.190 +-0.891 -0.454 -0.198 +-0.866 -0.500 -0.200 +-0.839 -0.545 -0.198 +-0.809 -0.588 -0.190 +-0.777 -0.629 -0.178 +-0.743 -0.669 -0.162 +-0.707 -0.707 -0.141 +-0.669 -0.743 -0.118 +-0.629 -0.777 -0.091 +-0.588 -0.809 -0.062 +-0.545 -0.839 -0.031 +-0.500 -0.866 -0.000 +-0.454 -0.891 0.031 +-0.407 -0.914 0.062 +-0.358 -0.934 0.091 +-0.309 -0.951 0.118 +-0.259 -0.966 0.141 +-0.208 -0.978 0.162 +-0.156 -0.988 0.178 +-0.105 -0.995 0.190 +-0.052 -0.999 0.198 +-0.000 -1.000 0.200 +0.052 -0.999 0.198 +0.105 -0.995 0.190 +0.156 -0.988 0.178 +0.208 -0.978 0.162 +0.259 -0.966 0.141 +0.309 -0.951 0.118 +0.358 -0.934 0.091 +0.407 -0.914 0.062 +0.454 -0.891 0.031 +0.500 -0.866 0.000 +0.545 -0.839 -0.031 +0.588 -0.809 -0.062 +0.629 -0.777 -0.091 +0.669 -0.743 -0.118 +0.707 -0.707 -0.141 +0.743 -0.669 -0.162 +0.777 -0.629 -0.178 +0.809 -0.588 -0.190 +0.839 -0.545 -0.198 +0.866 -0.500 -0.200 +0.891 -0.454 -0.198 +0.914 -0.407 -0.190 +0.934 -0.358 -0.178 +0.951 -0.309 -0.162 +0.966 -0.259 -0.141 +0.978 -0.208 -0.118 +0.988 -0.156 -0.091 +0.995 -0.105 -0.062 +0.999 -0.052 -0.031 diff --git a/examples/tiny-project/code/samples/images/shapes.png b/examples/tiny-project/code/samples/images/shapes.png new file mode 100644 index 00000000..eba5f68e Binary files /dev/null and b/examples/tiny-project/code/samples/images/shapes.png differ diff --git a/examples/tiny-project/code/samples/meshes/pyramid.obj b/examples/tiny-project/code/samples/meshes/pyramid.obj new file mode 100644 index 00000000..cdc4552f --- /dev/null +++ b/examples/tiny-project/code/samples/meshes/pyramid.obj @@ -0,0 +1,11 @@ +v 0 0 0 +v 1 0 0 +v 1 1 0 +v 0 1 0 +v 0.5 0.5 0.8 +f 1 3 2 +f 1 4 3 +f 1 2 5 +f 2 3 5 +f 3 4 5 +f 4 1 5 diff --git a/examples/tiny-project/code/samples/sounds/two-tones.wav b/examples/tiny-project/code/samples/sounds/two-tones.wav new file mode 100644 index 00000000..fa19afba Binary files /dev/null and b/examples/tiny-project/code/samples/sounds/two-tones.wav differ diff --git a/examples/tiny-project/code/samples/texts/greeting.txt b/examples/tiny-project/code/samples/texts/greeting.txt new file mode 100644 index 00000000..f269535a --- /dev/null +++ b/examples/tiny-project/code/samples/texts/greeting.txt @@ -0,0 +1 @@ +Ada Lovelace wrote to Charles Babbage from London 🌍 in 1843. diff --git a/examples/tiny-project/code/samples/texts/weather.txt b/examples/tiny-project/code/samples/texts/weather.txt new file mode 100644 index 00000000..302ea685 --- /dev/null +++ b/examples/tiny-project/code/samples/texts/weather.txt @@ -0,0 +1 @@ +Rain over Manila this morning; clear skies in Cebu by the afternoon. diff --git a/examples/tiny-project/code/scenes/clouds/p.ply b/examples/tiny-project/code/scenes/clouds/p.ply new file mode 100644 index 00000000..296f0330 --- /dev/null +++ b/examples/tiny-project/code/scenes/clouds/p.ply @@ -0,0 +1,27 @@ +ply +format ascii 1.0 +element vertex 20 +property float x +property float y +property float z +end_header +0.00 0.00 0.000 +0.10 0.05 0.053 +0.20 0.10 0.105 +0.30 0.00 0.158 +0.40 0.05 0.211 +0.50 0.10 0.263 +0.60 0.00 0.316 +0.70 0.05 0.368 +0.80 0.10 0.421 +0.90 0.00 0.474 +1.00 0.05 0.526 +1.10 0.10 0.579 +1.20 0.00 0.632 +1.30 0.05 0.684 +1.40 0.10 0.737 +1.50 0.00 0.789 +1.60 0.05 0.842 +1.70 0.10 0.895 +1.80 0.00 0.947 +1.90 0.05 1.000 diff --git a/examples/tiny-project/code/scenes/clouds/q.ply b/examples/tiny-project/code/scenes/clouds/q.ply new file mode 100644 index 00000000..90490b4e --- /dev/null +++ b/examples/tiny-project/code/scenes/clouds/q.ply @@ -0,0 +1,37 @@ +ply +format ascii 1.0 +element vertex 30 +property float x +property float y +property float z +end_header +0.00 0.00 0.000 +0.10 0.05 0.034 +0.20 0.10 0.069 +0.30 0.00 0.103 +0.40 0.05 0.138 +0.50 0.10 0.172 +0.60 0.00 0.207 +0.70 0.05 0.241 +0.80 0.10 0.276 +0.90 0.00 0.310 +1.00 0.05 0.345 +1.10 0.10 0.379 +1.20 0.00 0.414 +1.30 0.05 0.448 +1.40 0.10 0.483 +1.50 0.00 0.517 +1.60 0.05 0.552 +1.70 0.10 0.586 +1.80 0.00 0.621 +1.90 0.05 0.655 +2.00 0.10 0.690 +2.10 0.00 0.724 +2.20 0.05 0.759 +2.30 0.10 0.793 +2.40 0.00 0.828 +2.50 0.05 0.862 +2.60 0.10 0.897 +2.70 0.00 0.931 +2.80 0.05 0.966 +2.90 0.10 1.000 diff --git a/examples/tiny-project/code/scenes/frames/moving.npy b/examples/tiny-project/code/scenes/frames/moving.npy new file mode 100644 index 00000000..2a766315 Binary files /dev/null and b/examples/tiny-project/code/scenes/frames/moving.npy differ diff --git a/examples/tiny-project/code/scenes/images/a.png b/examples/tiny-project/code/scenes/images/a.png new file mode 100644 index 00000000..678ba10b Binary files /dev/null and b/examples/tiny-project/code/scenes/images/a.png differ diff --git a/examples/tiny-project/code/scenes/images/b.png b/examples/tiny-project/code/scenes/images/b.png new file mode 100644 index 00000000..3e79b183 Binary files /dev/null and b/examples/tiny-project/code/scenes/images/b.png differ diff --git a/examples/tiny-project/code/scenes/images/c.png b/examples/tiny-project/code/scenes/images/c.png new file mode 100644 index 00000000..d35fb880 Binary files /dev/null and b/examples/tiny-project/code/scenes/images/c.png differ diff --git a/examples/tiny-project/code/scenes/series/signal.npy b/examples/tiny-project/code/scenes/series/signal.npy new file mode 100644 index 00000000..3fef6444 Binary files /dev/null and b/examples/tiny-project/code/scenes/series/signal.npy differ diff --git a/examples/tiny-project/code/settings/small-net-a.json b/examples/tiny-project/code/settings/small-net-a.json new file mode 100644 index 00000000..8f43e99b --- /dev/null +++ b/examples/tiny-project/code/settings/small-net-a.json @@ -0,0 +1,6 @@ +{ + "updates": 6000, + "batch": 256, + "learning_rate": 0.001, + "seed": 1 +} diff --git a/examples/tiny-project/code/train.py b/examples/tiny-project/code/train.py new file mode 100644 index 00000000..acbd14b7 --- /dev/null +++ b/examples/tiny-project/code/train.py @@ -0,0 +1,92 @@ +"""Train SmallNet on the training waves and write model.pt and log.json. + +Prints a tuldok_progress_v1 line every 50 updates so Tuldok can show how far +training has come while it runs. +""" +import argparse +import json +import math +import os +import sys +import time + +import numpy as np +import torch + +from net import SmallNet + +EVERY = 50 # updates between progress lines and log entries + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--data', required=True, help='the waves.npz file') + parser.add_argument('--settings', required=True, help='a JSON file: updates, batch, learning_rate, seed') + parser.add_argument('--output', required=True, help='the directory to write model.pt and log.json into') + # The recipe trains on the GPU. --device cpu exists only to smoke-test this + # script on a machine without one; the recipe never passes it. + parser.add_argument('--device', choices=['cuda', 'cpu'], default='cuda') + arguments = parser.parse_args() + + if arguments.device == 'cuda' and not torch.cuda.is_available(): + # Refuse rather than fall back: a recipe that says it trains on the GPU + # must not quietly produce a CPU model. + sys.exit('train.py: CUDA is not available to torch, and this recipe trains on the GPU. ' + 'Check the gpu environment (tuldok_project check-environment), ' + 'or pass --device cpu to smoke-test the script by hand.') + device = torch.device(arguments.device) + if arguments.device == 'cpu': + # A network this small runs fastest on one thread; on a many-core machine + # torch's default thread pool makes it a hundred times slower. + torch.set_num_threads(1) + + with open(arguments.settings, encoding='utf-8') as handle: + settings = json.load(handle) + updates, batch = int(settings['updates']), int(settings['batch']) + + # Same seed, same result: the weights are made on the CPU before moving, the + # batches are drawn by a CPU generator, and the GPU is asked for + # deterministic kernels (the recipe sets CUBLAS_WORKSPACE_CONFIG for cuBLAS). + # warn_only keeps a run going if some kernel has no deterministic version. + torch.manual_seed(settings['seed']) + torch.use_deterministic_algorithms(True, warn_only=True) + batches = torch.Generator().manual_seed(settings['seed']) + + with np.load(arguments.data) as waves: + inputs = torch.from_numpy(waves['train_inputs']).to(device) + targets = torch.from_numpy(waves['train_targets']).to(device) + + model = SmallNet().to(device) + optimiser = torch.optim.Adam(model.parameters(), lr=float(settings['learning_rate'])) + loss_function = torch.nn.MSELoss() + log = {'every': EVERY, 'updates': [], 'loss': []} + + started = time.perf_counter() + for update in range(1, updates + 1): + chosen = torch.randint(len(inputs), (batch,), generator=batches).to(device) + loss = loss_function(model(inputs[chosen]), targets[chosen]) + optimiser.zero_grad(set_to_none=True) + loss.backward() + optimiser.step() + if update % EVERY == 0 or update == updates: + value = loss.item() # waits for the GPU only here, not on every update + if not math.isfinite(value): + sys.exit(f'train.py: the loss became {value} at update {update}; stopping.') + log['updates'].append(update) + log['loss'].append(value) + print(json.dumps({'format': 'tuldok_progress_v1', 'stage': 'train', 'completed': update, + 'total': updates, 'metrics': {'loss': value}}), flush=True) + log['seconds'] = time.perf_counter() - started + log['device'] = arguments.device + + # Saved on the CPU so the weights load on any machine. + torch.save({name: tensor.cpu() for name, tensor in model.state_dict().items()}, + os.path.join(arguments.output, 'model.pt')) + with open(os.path.join(arguments.output, 'log.json'), 'w', encoding='utf-8') as handle: + json.dump(log, handle, indent=1) + handle.write('\n') + print(f'trained {updates} updates in {log["seconds"]:.1f} s; final loss {log["loss"][-1]:.5f}') + + +if __name__ == '__main__': + main() diff --git a/examples/tiny-project/project/annotations/sample-quality.json b/examples/tiny-project/project/annotations/sample-quality.json new file mode 100644 index 00000000..3385e268 --- /dev/null +++ b/examples/tiny-project/project/annotations/sample-quality.json @@ -0,0 +1,9 @@ +{ + "format": "tuldok_definition_v1", + "kind": "annotation", + "title": "Sample quality", + "description": "Whether each sample is clean enough to keep.", + "dataset": "datasets/samples", + "task": "class", + "vocabulary": ["clean", "noisy", "broken"] +} diff --git a/examples/tiny-project/project/annotations/sample-quality.jsonl b/examples/tiny-project/project/annotations/sample-quality.jsonl new file mode 100644 index 00000000..83139ec8 --- /dev/null +++ b/examples/tiny-project/project/annotations/sample-quality.jsonl @@ -0,0 +1,6 @@ +{"record":"clouds__ring.ply","sha256":"3de402a535f473fd002e7f74f88aba003dc18e7cc1d3b23195fc10b0c752dae8","state":"accepted","value":"clean","by":"owner","at":"2026-11-13T09:30:00Z"} +{"record":"images__shapes.png","sha256":"1ab3564df0659b467dc93b26be066accd4b19f5dbc0c0216831d856facec9372","state":"accepted","value":"clean","by":"owner","at":"2026-11-13T09:30:00Z"} +{"record":"meshes__pyramid.obj","sha256":"ebf7519500bc461c4856157e63bc2ab6661b56894c36f04f8da217bc2d05b593","state":"accepted","value":"clean","by":"owner","at":"2026-11-13T09:30:00Z"} +{"record":"sounds__two-tones.wav","sha256":"fd10d893427a1a9dd4a0005abc271e1c94ab22adfb7d222042b71929246880ff","state":"accepted","value":"clean","by":"owner","at":"2026-11-13T09:30:00Z"} +{"record":"texts__greeting.txt","sha256":"a9af2325f767d1c8d2e130f52d0c119e5d98831bb11d0a148e33c39b0874b15d","state":"accepted","value":"clean","by":"owner","at":"2026-11-13T09:30:00Z"} +{"record":"texts__weather.txt","sha256":"43242d49e671a79209b537a360a4a11ef3112e9e4b7a394b02ed701c3ba677a5","state":"rejected","value":null,"note":"too short to be of use","by":"owner","at":"2026-11-13T09:30:00Z"} diff --git a/examples/tiny-project/project/annotations/scene-masks.json b/examples/tiny-project/project/annotations/scene-masks.json new file mode 100644 index 00000000..9ecf4a22 --- /dev/null +++ b/examples/tiny-project/project/annotations/scene-masks.json @@ -0,0 +1,12 @@ +{ + "format": "tuldok_definition_v1", + "kind": "annotation", + "title": "Objects in the scenes, as masks", + "description": "An instance mask of each dark object of the scenes' images, made by the stand-in segmentors.", + "dataset": "datasets/scenes", + "task": "mask", + "mask": "instance", + "vocabulary": [ + "blob" + ] +} diff --git a/examples/tiny-project/project/annotations/scene-objects.json b/examples/tiny-project/project/annotations/scene-objects.json new file mode 100644 index 00000000..08c8061b --- /dev/null +++ b/examples/tiny-project/project/annotations/scene-objects.json @@ -0,0 +1,11 @@ +{ + "format": "tuldok_definition_v1", + "kind": "annotation", + "title": "The top of each point cloud", + "description": "The points of each cloud that belong to its top, made by the stand-in segmentors.", + "dataset": "datasets/scenes", + "task": "points", + "vocabulary": [ + "top" + ] +} diff --git a/examples/tiny-project/project/benchmarks/held-out.json b/examples/tiny-project/project/benchmarks/held-out.json new file mode 100644 index 00000000..44ee9929 --- /dev/null +++ b/examples/tiny-project/project/benchmarks/held-out.json @@ -0,0 +1,23 @@ +{ + "format": "tuldok_definition_v1", + "kind": "benchmark", + "title": "Held-out waves", + "description": "How well each subject predicts the next 16 samples of the 400 test waves, and how long it takes.", + "run": { + "command": ["python", "bench.py", "--data", "{inputs.cases}", "--subject", "{subject}", "--name", "{subject.name}", "--output", "{output}"], + "environment": "environments/gpu", + "inputs": {"cases": {"from": "datasets/waves", "output": "data"}}, + "outputs": {"results": {"path": "results.json", "role": "results"}}, + "resources": {"gpu": true}, + "sources": ["bench.py", "net.py"] + }, + "metrics": [ + {"id": "error", "title": "Prediction error (RMSE)", "better": "lower"}, + {"id": "time_ms", "title": "Time per wave", "units": "ms", "better": "lower"} + ], + "subjects": [ + {"name": "small-net-a", "title": "Small net, 6,000 updates", "recipe": "recipes/small-net-a", "argument": "model:{recipe}/model.pt"}, + {"name": "predict-zero", "title": "Predict zero: the error any model must beat", "argument": "zero"} + ], + "cases": {"dataset": "datasets/waves", "split": "test", "count": 400} +} diff --git a/examples/tiny-project/project/charts/held-out-ranking.json b/examples/tiny-project/project/charts/held-out-ranking.json new file mode 100644 index 00000000..7cd06fb3 --- /dev/null +++ b/examples/tiny-project/project/charts/held-out-ranking.json @@ -0,0 +1,8 @@ +{ + "format": "tuldok_definition_v1", + "kind": "chart", + "title": "Error on the held-out waves", + "benchmark": "benchmarks/held-out", + "form": "ranking", + "metric": "error" +} diff --git a/examples/tiny-project/project/charts/held-out-tradeoff.json b/examples/tiny-project/project/charts/held-out-tradeoff.json new file mode 100644 index 00000000..73e93634 --- /dev/null +++ b/examples/tiny-project/project/charts/held-out-tradeoff.json @@ -0,0 +1,9 @@ +{ + "format": "tuldok_definition_v1", + "kind": "chart", + "title": "Error against time per wave", + "benchmark": "benchmarks/held-out", + "form": "tradeoff", + "metric": "error", + "metric_x": "time_ms" +} diff --git a/examples/tiny-project/project/datasets/samples-v1.json b/examples/tiny-project/project/datasets/samples-v1.json new file mode 100644 index 00000000..a3557688 --- /dev/null +++ b/examples/tiny-project/project/datasets/samples-v1.json @@ -0,0 +1,12 @@ +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Samples v1, frozen", + "description": "The samples whose quality annotation is accepted, split three to one.", + "run": { + "step": "freeze", + "inputs": {"records": "datasets/samples"}, + "annotations": ["annotations/sample-quality"], + "settings": {"seed": 7, "shares": {"train": 3, "test": 1}} + } +} diff --git a/examples/tiny-project/project/datasets/samples.json b/examples/tiny-project/project/datasets/samples.json new file mode 100644 index 00000000..44777ab3 --- /dev/null +++ b/examples/tiny-project/project/datasets/samples.json @@ -0,0 +1,10 @@ +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Samples of each kind", + "description": "Two texts, an image, a sound, a point cloud and a mesh, imported from code/samples (written by code/make_samples.py).", + "run": { + "step": "import", + "settings": {"source": "samples", "pattern": "**/*", "rights": "Made by make_samples.py for this example; free to use."} + } +} diff --git a/examples/tiny-project/project/datasets/scene-masks-a.json b/examples/tiny-project/project/datasets/scene-masks-a.json new file mode 100644 index 00000000..82d82bda --- /dev/null +++ b/examples/tiny-project/project/datasets/scene-masks-a.json @@ -0,0 +1,28 @@ +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Segmentor A on scene-masks", + "description": "The stand-in segmentor at threshold 160: an annotation set for annotations/scene-masks.", + "run": { + "step": "annotate", + "inputs": { + "records": "datasets/scenes" + }, + "annotations": [ + "annotations/scene-masks" + ], + "command": [ + "python", + "annotators/standin.py", + "160" + ], + "environment": "environments/cpu", + "settings": { + "by": { + "kind": "script", + "name": "stand-in segmentor A", + "version": "1" + } + } + } +} diff --git a/examples/tiny-project/project/datasets/scene-masks-b.json b/examples/tiny-project/project/datasets/scene-masks-b.json new file mode 100644 index 00000000..9da26cde --- /dev/null +++ b/examples/tiny-project/project/datasets/scene-masks-b.json @@ -0,0 +1,28 @@ +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Segmentor B on scene-masks", + "description": "The stand-in segmentor at threshold 200: an annotation set for annotations/scene-masks.", + "run": { + "step": "annotate", + "inputs": { + "records": "datasets/scenes" + }, + "annotations": [ + "annotations/scene-masks" + ], + "command": [ + "python", + "annotators/standin.py", + "200" + ], + "environment": "environments/cpu", + "settings": { + "by": { + "kind": "script", + "name": "stand-in segmentor B", + "version": "1" + } + } + } +} diff --git a/examples/tiny-project/project/datasets/scene-objects-a.json b/examples/tiny-project/project/datasets/scene-objects-a.json new file mode 100644 index 00000000..12e5ee8f --- /dev/null +++ b/examples/tiny-project/project/datasets/scene-objects-a.json @@ -0,0 +1,28 @@ +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Segmentor A on scene-objects", + "description": "The stand-in segmentor at threshold 128: an annotation set for annotations/scene-objects.", + "run": { + "step": "annotate", + "inputs": { + "records": "datasets/scenes" + }, + "annotations": [ + "annotations/scene-objects" + ], + "command": [ + "python", + "annotators/standin.py", + "128" + ], + "environment": "environments/cpu", + "settings": { + "by": { + "kind": "script", + "name": "stand-in segmentor A", + "version": "1" + } + } + } +} diff --git a/examples/tiny-project/project/datasets/scene-objects-b.json b/examples/tiny-project/project/datasets/scene-objects-b.json new file mode 100644 index 00000000..423cfa09 --- /dev/null +++ b/examples/tiny-project/project/datasets/scene-objects-b.json @@ -0,0 +1,28 @@ +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Segmentor B on scene-objects", + "description": "The stand-in segmentor at threshold 140: an annotation set for annotations/scene-objects.", + "run": { + "step": "annotate", + "inputs": { + "records": "datasets/scenes" + }, + "annotations": [ + "annotations/scene-objects" + ], + "command": [ + "python", + "annotators/standin.py", + "140" + ], + "environment": "environments/cpu", + "settings": { + "by": { + "kind": "script", + "name": "stand-in segmentor B", + "version": "1" + } + } + } +} diff --git a/examples/tiny-project/project/datasets/scenes.json b/examples/tiny-project/project/datasets/scenes.json new file mode 100644 index 00000000..61330fab --- /dev/null +++ b/examples/tiny-project/project/datasets/scenes.json @@ -0,0 +1,14 @@ +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Scenes for the annotators", + "description": "Three images, two point clouds, an array of frames and a series, imported from code/scenes (written by code/make_scenes.py).", + "run": { + "step": "import", + "settings": { + "source": "scenes", + "pattern": "**/*", + "rights": "Made by make_scenes.py for this example; free to use." + } + } +} diff --git a/examples/tiny-project/project/datasets/waves.json b/examples/tiny-project/project/datasets/waves.json new file mode 100644 index 00000000..3c27c265 --- /dev/null +++ b/examples/tiny-project/project/datasets/waves.json @@ -0,0 +1,14 @@ +{ + "format": "tuldok_definition_v1", + "kind": "dataset", + "title": "Noisy sine waves", + "description": "Each item is 64 samples of a noisy sine wave with a random amplitude, frequency and phase; the target is the next 16 samples of the clean wave.", + "run": { + "command": ["python", "make_waves.py", "--output", "{output}", "--seed", "7"], + "environment": "environments/cpu", + "outputs": {"data": {"path": "waves.npz", "role": "data"}}, + "resources": {"cpu_threads": 1, "wall_hours": 0.05, "disk_gb": 0.1}, + "sources": ["make_waves.py"] + }, + "splits": {"train": 4000, "test": 400} +} diff --git a/examples/tiny-project/project/environments/cpu.json b/examples/tiny-project/project/environments/cpu.json new file mode 100644 index 00000000..604a030f --- /dev/null +++ b/examples/tiny-project/project/environments/cpu.json @@ -0,0 +1,8 @@ +{ + "format": "tuldok_definition_v1", + "kind": "environment", + "title": "Python 3.12 with numpy, on the CPU", + "python": "3.12", + "device": "cpu", + "packages": {"numpy": "2.3.5"} +} diff --git a/examples/tiny-project/project/environments/gpu.json b/examples/tiny-project/project/environments/gpu.json new file mode 100644 index 00000000..1ae0c718 --- /dev/null +++ b/examples/tiny-project/project/environments/gpu.json @@ -0,0 +1,8 @@ +{ + "format": "tuldok_definition_v1", + "kind": "environment", + "title": "Python 3.14 with torch and CUDA", + "python": "3.14", + "device": "cuda", + "packages": {"torch": "2.14.0+cu132", "numpy": "2.5.3"} +} diff --git a/examples/tiny-project/project/models/small-net.json b/examples/tiny-project/project/models/small-net.json new file mode 100644 index 00000000..ebd3f954 --- /dev/null +++ b/examples/tiny-project/project/models/small-net.json @@ -0,0 +1,8 @@ +{ + "format": "tuldok_definition_v1", + "kind": "model", + "title": "Small fully connected network", + "description": "Three fully connected layers, 64 to 256 to 256 to 16, that read 64 samples of a wave and predict the next 16.", + "entry": "net:SmallNet", + "sources": ["net.py"] +} diff --git a/examples/tiny-project/project/recipes/small-net-a.json b/examples/tiny-project/project/recipes/small-net-a.json new file mode 100644 index 00000000..8a467401 --- /dev/null +++ b/examples/tiny-project/project/recipes/small-net-a.json @@ -0,0 +1,20 @@ +{ + "format": "tuldok_definition_v1", + "kind": "recipe", + "title": "Small net, 6,000 updates", + "description": "Trains the small network on the training waves with Adam on mean squared error, on the GPU.", + "model": "models/small-net", + "run": { + "command": ["python", "train.py", "--data", "{inputs.training}", "--settings", "{settings}", "--output", "{output}"], + "environment": "environments/gpu", + "inputs": {"training": {"from": "datasets/waves", "output": "data"}}, + "settings_file": "settings/small-net-a.json", + "env": {"CUBLAS_WORKSPACE_CONFIG": ":4096:8"}, + "outputs": { + "weights": {"path": "model.pt", "role": "weights"}, + "log": {"path": "log.json", "role": "log"} + }, + "resources": {"gpu": true, "wall_hours": 0.1, "disk_gb": 0.1}, + "sources": ["train.py", "net.py"] + } +} diff --git a/examples/tiny-project/project/tuldok.json b/examples/tiny-project/project/tuldok.json new file mode 100644 index 00000000..d44c51b5 --- /dev/null +++ b/examples/tiny-project/project/tuldok.json @@ -0,0 +1,6 @@ +{ + "format": "tuldok_project_v1", + "id": "9a37848258ef4d079bb020f57c8f6dec", + "name": "Tiny project", + "code_root": "../code" +} diff --git a/gateway_discovery.py b/gateway_discovery.py index 314023d3..7528c658 100644 --- a/gateway_discovery.py +++ b/gateway_discovery.py @@ -35,7 +35,10 @@ def listening_endpoints(): return sorted(endpoints, key=lambda endpoint: (endpoint[1], endpoint[0])) -def get_json(host, port, path, deadline, payload=None, max_response=MAX_RESPONSE, decode=json.loads, require_complete=False): +def get_json(host, port, path, deadline, payload=None, max_response=MAX_RESPONSE, decode=json.loads, require_complete=False, + error=None): + """GET/POST bounded JSON. With `error`, a non-200 body is read under the same bounds + and passed to `error(status, raw)`, which raises a readable refusal.""" timeout = min(1.5, deadline - time.monotonic()) if timeout <= 0: raise TimeoutError() @@ -64,7 +67,7 @@ def expire(): expiry.daemon = True expiry.start() with connection.getresponse() as response: - if response.status != 200: + if response.status != 200 and error is None: raise ValueError('Not a gateway response') if require_complete: declared = response.getheader('Content-Length') @@ -86,6 +89,9 @@ def expire(): raise ValueError('Pumas descriptor response ended before its declared length') if require_complete and time.monotonic() >= deadline: raise TimeoutError() + if response.status != 200: + error(response.status, bytes(data)) + raise ValueError('Not a gateway response') return decode(data) finally: if expiry is not None: @@ -212,15 +218,20 @@ def scan_advertised(): def observe(item): host, port = item address = '[' + host + ']' if ':' in host else host + url = f'http://{address}:{port}' try: - return inspect_advertised(f'http://{address}:{port}', deadline) + return inspect_advertised(url, deadline), None + except descriptor.Rejected as error: + # Pumas answered but its advertisement was refused: keep the reason for the user. + return None, {'server_url': url, 'reason': str(error)} except (OSError, ValueError, TypeError, http.client.HTTPException): - return None + return None, None with ThreadPoolExecutor(max_workers=32) as pool: results = list(pool.map(observe, endpoints)) # Distinct IPv4/IPv6 services on one port must retain their library context. - found = {row['server_url']: row for row in results if row is not None} - return {'gateways': list(found.values()), 'qualification': + found = {row['server_url']: row for row, _ in results if row is not None} + rejected = [refusal for _, refusal in results if refusal is not None] + return {'gateways': list(found.values()), 'rejected': rejected, 'qualification': 'Read-only HTTP advertisements. Core ownership is not authenticated; compiled features do not prove model readiness.'} finally: SCAN_LOCK.release() diff --git a/image_generation.py b/image_generation.py index 76e37f2f..67f1a92e 100644 --- a/image_generation.py +++ b/image_generation.py @@ -11,6 +11,7 @@ import http.client import io import json +import re import select import socket import threading @@ -93,7 +94,8 @@ def decode_image(data, width, height): for key in ('seed', 'steps', 'guidance', 'duration_seconds'): if type(metadata.get(key)) in (int, float): safe_metadata[key] = metadata[key] - if metadata.get('memory_policy') in ('sequential_cpu_offload', 'model_cpu_offload', 'gpu'): + # The policy set grows with adapters (0.8.0 adds `scaled_fp8_to_bf16_sequential_cpu_offload`). + if isinstance(metadata.get('memory_policy'), str) and re.fullmatch(r'[a-z0-9_]{1,64}', metadata['memory_policy']): safe_metadata['memory_policy'] = metadata['memory_policy'] result['metadata'] = safe_metadata return result @@ -122,6 +124,8 @@ def typed_generate(base, body, operation, payload, on_admission=None): evidence = {'capability_observation': observed, 'producer_contract_source': pumas.SOURCE_COMMIT} try: pumas.require(observed, 'image_generation', 'text', 'png_base64', options) + pumas.require(observed, 'image_generation', 'text', 'png_base64', {'input_characters': len(payload['prompt'])}, + advisory=('input_characters',)) request = {'contract_version': 1, 'request_id': body['request_id'], 'model': payload['model'], 'profile': observed['capabilities']['profile'], 'capability': 'image_generation', 'input': {'kind': 'text', 'text': payload['prompt']}, 'output': 'png_base64', @@ -258,7 +262,13 @@ def watch_cancel(): 499: 'Image generation cancelled.', 503: 'The image model is unavailable. Load it in Pumas and refresh models.', 507: 'Pumas reported insufficient GPU memory. Free memory before trying again.'} - raise ValueError(messages.get(response.status, 'Pumas image generation failed. Check its runtime status. The request was not retried.')) + message = messages.get(response.status, 'Pumas image generation failed. Check its runtime status. The request was not retried.') + try: + transport.settimeout(2) + detail = ai_http.pumas_error_detail(response.read(ai_http.MAX_ERROR_BODY + 1), payload) + except (OSError, http.client.HTTPException): + detail = '' + raise ValueError(message + (' ' + detail if detail else '')) data = bytearray() while not response.isclosed(): chunk = response.read1(65536) diff --git a/mcp_stdio.py b/mcp_stdio.py index 9cf12db5..e342381d 100644 --- a/mcp_stdio.py +++ b/mcp_stdio.py @@ -236,12 +236,7 @@ async def call_tool(name, arguments): try: agent_api.validate_arguments(name, arguments, contract_version(name)) observe('dispatch', pending_observation) - # The explicit owner clients admit canonical operation IDs only; - # MCP names are transport aliases, validated by the fixed profile. - dispatch_name = (agent_api.operation(name)['operation_id'] - if isinstance(client, (agent_api.ProjectFilesOwnerClient, - agent_api.ProjectTransferOwnerClient)) else name) - envelope = await anyio.to_thread.run_sync(client.call, dispatch_name, arguments) + envelope = await anyio.to_thread.run_sync(client.call, name, arguments) pending_http_status = envelope.get('status') if isinstance(envelope, dict) else None agent_api.validate_output(name, envelope, contract_version(name)) pending_outcome = ('permission-refusal' if envelope['status'] in (401, 403) @@ -412,10 +407,8 @@ def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--config', required=True, help='Private SDK JSON configuration file') owner_profile = parser.add_mutually_exclusive_group() - owner_profile.add_argument('--project-files-owner', action='store_true', - help='Explicit current owner credential; expose only finite project-file operations') - owner_profile.add_argument('--project-transfer-owner', action='store_true', - help='Explicit current host owner; expose only the five project-transfer operations') + owner_profile.add_argument('--project-folder-owner', action='store_true', + help='Explicit owner of a served project folder; expose only the project folder operations') parser.add_argument('--observe-v2-database', help='Explicit operator opt-in to a fresh private v2 interface store') parser.add_argument('--observe-database', help='Opt in to operator-only SQLite in an existing private directory') parser.add_argument('--observe-max-calls') @@ -428,8 +421,7 @@ def main(argv=None): import mcp # noqa: F401 - explicit optional consumer boundary import jsonschema # noqa: F401 import agent_api - client_type = (agent_api.ProjectFilesOwnerClient if args.project_files_owner else - agent_api.ProjectTransferOwnerClient if args.project_transfer_owner else agent_api.AgentClient) + client_type = agent_api.ProjectFolderOwnerClient if args.project_folder_owner else agent_api.AgentClient client = client_type.from_config(args.config) client.bootstrap() if args.observe_database is not None or args.observe_v2_database is not None: diff --git a/native_process_bootstrap.py b/native_process_bootstrap.py deleted file mode 100644 index 56466b41..00000000 --- a/native_process_bootstrap.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Closed native job launch, entered only behind the existing custodian GO. - -The custodian arms parent-death kill before exec. This module rechecks and -retains that exact parent before importing a captured native worker. It creates -new execution evidence; the original job template is never modified. -""" -import ctypes -import hashlib -import json -import os -from pathlib import Path -import signal -import stat -import sys - -WORKERS = frozenset(('image_training.py', 'text_training.py', 'audio_training.py')) -TEMPLATE_LIMIT = 64 * 1024 - - -def encoded(value): - return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True, - allow_nan=False).encode('utf-8') - - -def pairs(items): - value = {} - for key, item in items: - if key in value: raise ValueError('Duplicate native template member.') - value[key] = item - return value - - -def read_regular(path): - descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) - with os.fdopen(descriptor, 'rb') as source: - before = os.fstat(source.fileno()) - if (not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 - or before.st_uid != os.getuid() or before.st_mode & 0o222 - or before.st_size > TEMPLATE_LIMIT): - raise ValueError('Native template is not an owned immutable file.') - raw = source.read(TEMPLATE_LIMIT + 1) - after = os.fstat(source.fileno()) - if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns, before.st_ctime_ns) != ( - after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns, after.st_ctime_ns): - raise ValueError('Native template changed.') - if len(raw) > TEMPLATE_LIMIT: raise ValueError('Native template exceeds its bound.') - return raw - - -def parent_identity(): - parent = os.getppid() - if parent <= 1: raise ValueError('Native custodian parent is unavailable.') - libc = ctypes.CDLL(None, use_errno=True) - if libc.prctl(1, signal.SIGKILL, 0, 0, 0): - raise ValueError('Native parent-death protection is unavailable.') - fields = Path('/proc/' + str(parent) + '/stat').read_text().rsplit(')', 1)[1].split() - value = dict(pid=parent, start_ticks=fields[19], - boot_id=Path('/proc/sys/kernel/random/boot_id').read_text().strip()) - if os.getppid() != parent: raise ValueError('Native custodian parent changed.') - return value - - -def write_once(directory, name, raw): - descriptor = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, - 0o400, dir_fd=directory) - with os.fdopen(descriptor, 'wb') as target: - target.write(raw); target.flush(); os.fsync(target.fileno()) - os.fsync(directory) - - -def main(argv=None): - arguments = sys.argv[1:] if argv is None else argv - if len(arguments) != 2 or arguments[0] not in ('train', 'verify'): - raise ValueError('Unsupported native launch phase.') - # The helper already armed this death signal before executing Python. - # Re-arm and observe it before runpy or any native worker import. - parent = parent_identity() - root = Path.cwd() - template = Path(arguments[1]) - if template != root / 'native-job-template.json': - raise ValueError('Native template path differs.') - raw = read_regular(template) - value = json.loads(raw, object_pairs_hook=pairs, - parse_constant=lambda _: (_ for _ in ()).throw(ValueError('Nonfinite native value.'))) - if (type(value) is not dict or set(value) != {'format', 'phase', 'worker', 'attempt_id', 'request_sha256', 'job'} - or value['format'] != 'native_parent_binding_v1' or value['phase'] != arguments[0] - or value['worker'] not in WORKERS or type(value['job']) is not dict - or type(value['job'].get('parent_pid')) is not int or value['job']['parent_pid'] != 1 - or encoded(value) != raw): - raise ValueError('Native job template differs from the closed contract.') - job = dict(value['job'], parent_pid=parent['pid']) - effective = encoded(job) - observation = encoded(dict(format='native_parent_observation_v1', attempt_id=value['attempt_id'], - request_sha256=value['request_sha256'], template_sha256=hashlib.sha256(raw).hexdigest(), - effective_job_sha256=hashlib.sha256(effective).hexdigest(), helper=parent)) - directory = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - try: - write_once(directory, 'effective-job.json', effective) - write_once(directory, 'native-parent-observation.json', observation) - finally: - os.close(directory) - worker = Path(__file__).resolve().parent / value['worker'] - sys.path.insert(0, str(worker.parent)) - sys.argv[:] = [str(worker), '--job' if value['phase'] == 'train' else '--verify-evidence', - str(root / 'effective-job.json')] - import runpy - runpy.run_path(str(worker), run_name='__main__') - - -if __name__ == '__main__': - try: - main() - except Exception: - print('Native launch binding failed.', file=sys.stderr, flush=True) - raise SystemExit(2) diff --git a/pipeline.py b/pipeline.py index d9c9894c..2a21bdd8 100644 --- a/pipeline.py +++ b/pipeline.py @@ -546,6 +546,23 @@ def decode_probe_observation(recipe,value,*,with_definition=False): return copy.deepcopy(value) +# Finite wall-clock allowance of each phase of a built-in recipe's native run. +NATIVE_PHASE_WALL_SECONDS = {'probe': 45, 'train': 600, 'verify': 90} +NATIVE_PARENT_DRAIN_ALLOWANCE = 5 + + +def native_admission_wall_seconds(): + """New material's finite probe/train/verifier and custody time allowance. + + This budgets child/custodian phases, not a storage availability guarantee. + Cleanup already includes the custodian's TERM grace interval. + """ + import recipe_process + overhead = (recipe_process.READY_SECONDS + recipe_process.GO_WAIT_SECONDS + recipe_process.CLEANUP_SECONDS + + NATIVE_PARENT_DRAIN_ALLOWANCE) + return sum(NATIVE_PHASE_WALL_SECONDS.values()) + len(NATIVE_PHASE_WALL_SECONDS) * overhead + + class Pipelines: def __init__(self, dataset): self.access = dataset.access @@ -937,10 +954,6 @@ def probe(self, recipe='image_classification'): # A native interpreter launch is execution even when its command only # reports runtime capability. Pure integrity recipes above never spawn. self.access.require('execute', 'pipeline.probe_native', recipe) - from project_native_preparation import preparation_owner - current=preparation_owner(self) - if current is not None: - return current.probe(self,recipe) worker_path = Path(adapter.__file__).resolve() executable = os.environ.get('TULDOK_TRAINING_PYTHON') or sys.executable try: @@ -1205,7 +1218,6 @@ def prepare_builtin_material(self, graph_id, revision, model_ref, *, reconcile_c capture_bytes=sum(map(len,payloads.values())) model_bytes=sum(item['blob']['bytes'] for item in model_body['source_closure'])+sum(model_body[key]['bytes'] for key in ('specification','preprocessing_policy','output_policy')) output_bytes=64*1024**2 - from project_native_execution import native_admission_wall_seconds envelope={'attempt_slots':1,'admitted_wall_seconds':native_admission_wall_seconds(),'retained_bytes':capture_bytes+model_bytes+output_bytes+256*1024, 'peak_staging_bytes':2*capture_bytes+model_bytes+2*output_bytes+256*1024} material={'format':'recipe_material_v1','catalog_id':self.db.dataset_id,'recipe_id':graph_id,'recipe_revision':revision, @@ -1227,12 +1239,6 @@ def prepare_builtin_material(self, graph_id, revision, model_ref, *, reconcile_c shutil.rmtree(staging,ignore_errors=True);raise def _prepare_builtin_recipe_material(self, value): - from project_native_preparation import preparation_owner - current=preparation_owner(self) - if current is not None:return current.current(self).capture_builtin(self,value) - return self._prepare_builtin_recipe_material_current(value) - - def _prepare_builtin_recipe_material_current(self, value): from model_definitions import read_owned_blob, retained_json material=value['material'];sha=value['ref']['sha256'] snapshot=retained_json(read_owned_blob(self.material_path,sha,material['recipe_document'],2*1024**2).decode(),2*1024**2) diff --git a/project_artifacts.py b/project_artifacts.py deleted file mode 100644 index fe2a0ee0..00000000 --- a/project_artifacts.py +++ /dev/null @@ -1,463 +0,0 @@ -"""Exact project payload custody, separate from domain role and retention policy. - -Declarations contain intrinsic byte identity only. Domain records own origin, -media type, unique/regenerable policy and permission. Publishing never deletes a -final payload on SQL rollback: unreferenced immutable bytes remain inert. Local -resolution capabilities never enter portable records, and no source is executed. -""" -from contextlib import contextmanager -import copy -import errno -import hashlib -import os -import re -import stat -import threading -import uuid - -from project_store import (ProjectStoreError, anchored_directory, DIR_FLAGS, FILE_FLAGS, - _directory, _physical, _relative_parts, _rename_noreplace, _walk) - -FORMAT = 'project_artifact_v1' -MAX_BYTES = 2 ** 63 - 1 -DEFAULT_READ_BYTES = 40 * 1024 * 1024 -CHUNK_BYTES = 1024 * 1024 - - -class ProjectArtifactError(ProjectStoreError): - pass - - -class _Missing(Exception): - pass - - -def _fail(message, code='invalid', status=400): - raise ProjectArtifactError(message, code, status) - - -def validate_declaration(value): - if type(value) is not dict: - _fail('Artifact declaration must be an object.') - if value.get('format') != FORMAT: - if type(value.get('format')) is str and re.fullmatch(r'project_artifact_v[1-9][0-9]*', value['format']): - _fail('Unsupported artifact declaration format.', 'unsupported', 422) - _fail('Invalid artifact declaration format.') - if set(value) != {'format', 'sha256', 'bytes'}: - _fail('Artifact declarations contain only format, SHA256 and byte count.') - digest, size = value['sha256'], value['bytes'] - if type(digest) is not str or len(digest) != 64 or any(c not in '0123456789abcdef' for c in digest): - _fail('Artifact requires a SHA256 digest.') - if type(size) is not int or not 0 <= size <= MAX_BYTES: - _fail('Artifact byte count must fit a nonnegative signed 64-bit integer.') - return dict(format=FORMAT, sha256=digest, bytes=size) - - -def _maximum(declaration, maximum_bytes): - if maximum_bytes is not None: - if type(maximum_bytes) is not int or not 0 <= maximum_bytes <= MAX_BYTES: - _fail('Artifact caller byte bound must be a nonnegative integer.') - if declaration['bytes'] > maximum_bytes: - _fail('Artifact exceeds the consuming operation byte bound.', 'too_large', 413) - - -def _info(fd, *, rooted=False): - info = os.fstat(fd) - # An explicit caller fd or our unlinked private snapshot can have no name. - # Rooted selected files must have exactly one name; aliases are never accepted. - if not stat.S_ISREG(info.st_mode) or info.st_nlink > 1 or (rooted and info.st_nlink != 1): - _fail('Artifact source must be an ordinary file without hard-link aliases.') - return info - - -def _identity(info): - return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns - - -def _write(fd, raw): - view = memoryview(raw) - while view: - written = os.write(fd, view) - if not written: - raise OSError(errno.EIO, 'short artifact write') - view = view[written:] - - -def _observe(fd, declaration, *, output=None): - """Observe one bounded source and optionally capture the exact observed bytes.""" - before = _info(fd) - if before.st_size != declaration['bytes']: - return dict(state='corrupt', expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], - observed_sha256=None, observed_bytes=before.st_size) - digest, count = hashlib.sha256(), 0 - while True: - raw = os.pread(fd, min(CHUNK_BYTES, declaration['bytes'] + 1 - count), count) - if not raw: - break - count += len(raw) - digest.update(raw) - if output is not None: - _write(output, raw) - if count > declaration['bytes']: - break - after = _info(fd) - if _identity(before) != _identity(after): - _fail('Artifact source changed during observation.', 'stale', 409) - observed = digest.hexdigest() - return dict(state='verified' if count == declaration['bytes'] and observed == declaration['sha256'] else 'corrupt', - expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], - observed_sha256=observed, observed_bytes=count) - - -def _verified(observation): - if observation['state'] != 'verified': - failure = ProjectArtifactError('Artifact bytes disagree with their exact declaration.', 'invalid', 400) - failure.observation = observation - raise failure - - -def _io(error, *, published=False): - if published: - raise ProjectArtifactError('Artifact publication durability is unknown; retain bytes and retry the same identity.', - 'unknown', 503) from error - if error.errno in (errno.ELOOP, errno.ENOTDIR, errno.ENXIO): - raise ProjectArtifactError('Unsafe artifact filesystem object.') from error - if error.errno in (errno.EXDEV, errno.ENOSYS, errno.EOPNOTSUPP): - raise ProjectArtifactError('Required artifact filesystem mechanism is unsupported.', 'unsupported', 422) from error - raise ProjectArtifactError('Artifact bytes are unavailable.', 'unavailable', 503) from error - - -class ProjectArtifacts: - def __init__(self, store, *, resolutions=None): - """Bind optional explicit SHA -> {root_fd, root_path, relative} locations. - - Overrides choose one local source explicitly; a corrupt selected source - never silently falls back to another copy. The caller retains its fds; - this owner duplicates and releases only its own root capabilities. - """ - self.store = store - self._mutex = threading.RLock() - self._closed = False - self._resolutions = {} - try: - if resolutions is not None and type(resolutions) is not dict: - _fail('Artifact resolutions must be an explicit mapping.') - for digest, location in (resolutions or {}).items(): - validate_declaration(dict(format=FORMAT, sha256=digest, bytes=0)) - if type(location) is not dict or set(location) != {'root_fd', 'root_path', 'relative'}: - _fail('Invalid local artifact resolution fields.') - if type(location['root_fd']) is not int or location['root_fd'] < 0: - _fail('Local artifact resolution needs a directory capability.') - parts = _relative_parts(location['relative']) - with anchored_directory(location['root_fd'], root_path=location['root_path']): - owned = os.dup(location['root_fd']) - self._resolutions[digest] = dict(root_fd=owned, root_path=location['root_path'], parts=parts) - except OSError as error: - self.close() - _io(error) - except BaseException: - self.close() - raise - - def _ensure_open(self): - if self._closed: - _fail('Artifact owner is closed.', 'unavailable', 409) - - @contextmanager - def _source(self, root_fd, declaration): - override = self._resolutions.get(declaration['sha256']) - if override is None: - parts = ['artifacts', 'sha256', declaration['sha256']] - yield from self._source_at(root_fd, parts) - else: - with anchored_directory(override['root_fd'], root_path=override['root_path']) as external: - yield from self._source_at(external, override['parts']) - - def _source_at(self, root_fd, parts): - parent = source = None - try: - try: - parent = _walk(root_fd, parts[:-1]) - source = os.open(parts[-1], FILE_FLAGS, dir_fd=parent) - except FileNotFoundError as error: - raise _Missing() from error - _info(source, rooted=True) - yield source - # Path replacement cannot redirect a pinned fd, but it invalidates - # the selected source location and must not be called current. - current = _walk(root_fd, parts[:-1]) - try: - if _physical(current) != _physical(parent): - _fail('Artifact source directory changed.', 'stale', 409) - selected = os.stat(parts[-1], dir_fd=current, follow_symlinks=False) - if (selected.st_dev, selected.st_ino) != _physical(source): - _fail('Artifact source location changed.', 'stale', 409) - finally: - os.close(current) - finally: - if source is not None: - os.close(source) - if parent is not None: - os.close(parent) - - def inspect(self, declaration): - """Presence is unverified; it never claims a hash was checked.""" - declaration = validate_declaration(declaration) - with self._mutex: - self._ensure_open() - try: - with self.store.root_directory() as root: - return self._inspect_at(root, declaration) - except OSError as error: - _io(error) - - def _inspect_at(self, root, declaration): - try: - with self._source(root, declaration) as source: - observed = _info(source, rooted=True).st_size - state = 'unverified' - except _Missing: - observed, state = None, 'missing' - return dict(state=state, expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], - observed_sha256=None, observed_bytes=observed) - - def inspect_many(self, declarations): - """Observe a finite owner-supplied page under one verified root scope. - - Each file still receives the same anchored ordinary-file checks as - inspect. Validate the canonical closure before AND after the group; - metadata corruption or a checkpoint change cannot produce a result. - Presence remains unverified and never claims a payload hash was read. - """ - if type(declarations) is not list: - _fail('Artifact inspection pages must be explicit finite lists.') - declarations = [validate_declaration(value) for value in declarations] - with self._mutex: - self._ensure_open() - try: - with self.store.root_directory(verify_after=True) as root: - result = [self._inspect_at(root, value) for value in declarations] - return result - except OSError as error: - _io(error) - - def status(self, declaration): - declaration = validate_declaration(declaration) - with self._mutex: - self._ensure_open() - try: - with self.store.root_directory() as root: - return self._status_at(root, declaration) - except OSError as error: - _io(error) - - def _status_at(self, root, declaration): - try: - with self._source(root, declaration) as source: - return _observe(source, declaration) - except _Missing: - return dict(state='missing', expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], - observed_sha256=None, observed_bytes=None) - - def status_many(self, declarations): - """Hash each explicit artifact under one before/after-verified root. - - Unlike presence inspection, these observations verify present payload - hashes. An unchanged canonical closure is proved for the entire group; - every individual file retains its anchored source-location checks. - """ - if type(declarations) is not list: - _fail('Artifact status groups must be explicit finite lists.') - declarations = [validate_declaration(value) for value in declarations] - with self._mutex: - self._ensure_open() - try: - with self.store.root_directory(verify_after=True) as root: - result = [self._status_at(root, value) for value in declarations] - return result - except OSError as error: - _io(error) - - @contextmanager - def _payload_directory(self, root): - artifacts = payloads = None - try: - artifacts = _directory(root, 'artifacts', create=True) - payloads = _directory(artifacts, 'sha256', create=True) - yield artifacts, payloads - finally: - if payloads is not None: - os.close(payloads) - if artifacts is not None: - os.close(artifacts) - - def _publish(self, declaration, capture): - with self._mutex: - self._ensure_open() - published = False - try: - with self.store.root_directory() as root, self._payload_directory(root) as (artifacts, payloads): - temporary = '.artifact-' + uuid.uuid4().hex + '.tmp' - output = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, - 0o600, dir_fd=payloads) - try: - capture(output) - os.fchmod(output, 0o400) - os.fsync(output) - finally: - os.close(output) - try: - _rename_noreplace(payloads, temporary, declaration['sha256']) - published = True - except FileExistsError: - existing = os.open(declaration['sha256'], FILE_FLAGS, dir_fd=payloads) - try: - _info(existing, rooted=True) - _verified(_observe(existing, declaration)) - os.fsync(existing) - finally: - os.close(existing) - os.unlink(temporary, dir_fd=payloads) - published = True - os.fsync(payloads) - os.fsync(artifacts) - os.fsync(root) - return copy.deepcopy(declaration) - except OSError as error: - _io(error, published=published) - # Failed staging remains inert. No callback owns deletion of final bytes. - - def publish_bytes(self, raw, declaration, *, maximum_bytes=None): - declaration = validate_declaration(declaration) - _maximum(declaration, maximum_bytes) - if type(raw) is not bytes: - _fail('Artifact publication requires exact bytes.') - if len(raw) != declaration['bytes'] or hashlib.sha256(raw).hexdigest() != declaration['sha256']: - _fail('Artifact input differs from its declaration.') - return self._publish(declaration, lambda output: _write(output, raw)) - - def publish_fd(self, source_fd, declaration, *, maximum_bytes=None): - declaration = validate_declaration(declaration) - _maximum(declaration, maximum_bytes) - if type(source_fd) is not int or source_fd < 0: - _fail('Artifact publication requires an explicit file capability.') - def capture(output): - _verified(_observe(source_fd, declaration, output=output)) - return self._publish(declaration, capture) - - def _snapshot(self, declaration): - snapshot = None - try: - with self.store.root_directory() as root: - snapshot = self._snapshot_at(root, declaration) - return snapshot - except BaseException: - if snapshot is not None: - os.close(snapshot) - raise - - def _snapshot_at(self, root, declaration): - snapshot = None - try: - local = staging = None - try: - with self._source(root, declaration) as source: - local = _directory(root, '.tuldok-local', create=True) - staging = _directory(local, 'artifact-snapshots', create=True) - temporary = '.snapshot-' + uuid.uuid4().hex + '.tmp' - output = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, - 0o600, dir_fd=staging) - try: - _verified(_observe(source, declaration, output=output)) - os.fchmod(output, 0o400) - os.fsync(output) - finally: - os.close(output) - snapshot = os.open(temporary, FILE_FLAGS, dir_fd=staging) - _info(snapshot, rooted=True) - os.unlink(temporary, dir_fd=staging) - except _Missing as error: - raise ProjectArtifactError('Required artifact bytes are missing.', 'unavailable', 404) from error - finally: - if staging is not None: - os.close(staging) - if local is not None: - os.close(local) - return snapshot - except BaseException: - if snapshot is not None: - os.close(snapshot) - raise - - def copy_many_to(self, destination, declarations): - """Copy a finite inventory with one before/after-verified source root. - - Each payload uses its own verified unlinked snapshot and ordinary atomic - destination publication. Only one temporary payload is retained at once. - A later failure can leave earlier verified destination blobs inert; the - caller owns publication of the complete destination project. - """ - if not isinstance(destination, ProjectArtifacts) or destination is self: - _fail('Artifact copying requires a distinct destination owner.') - if type(declarations) is not list: - _fail('Artifact copy groups must be explicit finite lists.') - declarations = [validate_declaration(value) for value in declarations] - first, second = sorted((self, destination), key=id) - with first._mutex, second._mutex: - self._ensure_open() - destination._ensure_open() - try: - with self.store.root_directory(verify_after=True) as root: - for declaration in declarations: - snapshot = self._snapshot_at(root, declaration) - try: - destination.publish_fd(snapshot, declaration, maximum_bytes=declaration['bytes']) - finally: - os.close(snapshot) - except OSError as error: - _io(error) - - @contextmanager - def open_verified(self, declaration, *, maximum_bytes=None): - """Borrow a verified unlinked read-only snapshot, not a mutable source fd.""" - declaration = validate_declaration(declaration) - _maximum(declaration, maximum_bytes) - with self._mutex: - self._ensure_open() - try: - snapshot = self._snapshot(declaration) - except OSError as error: - _io(error) - # Release owner/project locks before consumer work. Source resolution - # roots can close while this independent byte snapshot remains borrowed. - try: - yield snapshot - finally: - os.close(snapshot) - - def read_blob(self, descriptor, *, maximum_bytes=DEFAULT_READ_BYTES): - if type(descriptor) is not dict or set(descriptor) != {'sha256', 'bytes'}: - _fail('Blob references contain exactly SHA256 and byte count.') - declaration = dict(format=FORMAT, **descriptor) - with self.open_verified(declaration, maximum_bytes=maximum_bytes) as source: - chunks = [] - while True: - raw = os.read(source, CHUNK_BYTES) - if not raw: - break - chunks.append(raw) - return b''.join(chunks) - - def close(self): - with self._mutex: - if not self._closed: - for location in self._resolutions.values(): - os.close(location['root_fd']) - self._resolutions.clear() - self._closed = True - - def __enter__(self): - return self - - def __exit__(self, *error): - self.close() diff --git a/project_cli.py b/project_cli.py index 1d54389b..5b8132d7 100644 --- a/project_cli.py +++ b/project_cli.py @@ -12,8 +12,8 @@ import signal import threading -from agent_api import (AgentAPIError, AgentClient, ProjectFilesOwnerClient, ProjectTransferOwnerClient, PublicationOwnerClient, CoordinatorOwnerClient, REQUEST_LIMIT, decode, - validate_arguments, verify_artifact, verify_publication_artifact, PROJECT_FILES_M2, PROJECT_FILES_M3, PROJECT_FILES_M4) +from agent_api import (AgentAPIError, AgentClient, PublicationOwnerClient, CoordinatorOwnerClient, REQUEST_LIMIT, decode, + validate_arguments, verify_artifact, verify_publication_artifact) from tuldok_client import WorkflowClient, download from mcp_observer import (add_interface_options, configured_interface, observe_begin, observe_stage, observe_selection, observe_close, outcome) @@ -51,48 +51,8 @@ def parser(): result = argparse.ArgumentParser(description=__doc__) result.add_argument('--config', type=Path, required=True, help='Private existing AgentClient endpoint/catalog/token-file configuration') - result.add_argument('--owner-project-files', action='store_true', - help='Use an existing owner credential, restricted to file-* commands') add_interface_options(result) commands = result.add_subparsers(dest='command', required=True) - commands.add_parser('file-status', help='Inspect the active external file project and process binding') - for action in ('new', 'open', 'save', 'close', 'save-as'): - command = commands.add_parser('file-' + action, help='Explicit external project ' + action) - command.add_argument('--binding', required=True) - command.add_argument('--expected-checkpoint', required=True, - help='Exact SHA256; use null only when no project is open for New/Open') - command.add_argument('--request-key', required=True) - if action in ('new', 'open', 'save-as'): - command.add_argument('--root', required=True) - command.add_argument('--directory', required=True) - if action in ('new','save-as'): - command.add_argument('--name', required=True) - for action in ('query', 'get', 'import-text', 'annotate'): - command = commands.add_parser('file-' + action, help='Checkpoint-bound project text ' + action) - command.add_argument('--arguments', type=Path, required=True, - help='Exact project_files operation JSON arguments including binding/checkpoint') - for operation in sorted((PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4)-{'project_files.save_as'}): - action = operation.split('.')[1].replace('_', '-') - command = commands.add_parser('file-' + action, help='Checkpoint-bound project ' + action) - command.add_argument('--arguments', type=Path, required=True, - help='Exact selected project operation arguments') - transfer=commands.add_parser('project-transfer',help='Explicit owner extraction; Stop requires new current-host credentials') - actions=transfer.add_subparsers(dest='transfer_action',required=True) - actions.add_parser('status',help='Inspect current source/control phase and controller binding') - for action in ('preview','report-page','stop','extract'): - command=actions.add_parser(action,help='Explicit extraction '+action) - command.add_argument('--binding',required=True) - if action=='preview':command.add_argument('--selection',type=Path,required=True,help='Closed native ownership selection JSON file; no credentials or cessation proof') - else:command.add_argument('--report-sha256',required=True) - if action=='report-page': - command.add_argument('--section',required=True,choices=('included','required','unassigned','excluded','blockers','warnings','artifacts','working_files')) - command.add_argument('--offset',type=int,default=0);command.add_argument('--limit',type=int,default=40) - if action=='stop': - command.add_argument('--backup-root',required=True);command.add_argument('--backup-directory',required=True) - if action in ('stop','extract'): - command.add_argument('--destination-root',required=True);command.add_argument('--directory',required=True) - if action=='extract': - command.add_argument('--name',required=True);command.add_argument('--request-key',required=True) opening = commands.add_parser('open', help='Register/inspect a project without executing its code') opening.add_argument('--root', required=True) opening.add_argument('--manifest', required=True) @@ -291,13 +251,8 @@ def main(argv=None, *, observer=None): try: status, http_status = 200, None client_type = PublicationOwnerClient if args.command in ('recover-publication', 'cancel-publication') else AgentClient - if args.command=='project-transfer':client_type=ProjectTransferOwnerClient if args.command == 'recover-coordinator': client_type = CoordinatorOwnerClient - if args.owner_project_files: - if not args.command.startswith('file-'): - raise AgentAPIError('--owner-project-files is restricted to file-* commands.', 'configuration') - client_type = ProjectFilesOwnerClient client = client_type.from_config(args.config) if args.command in WORKFLOW_COMMANDS: result, success = workflow_command(client, args) @@ -316,30 +271,7 @@ def main(argv=None, *, observer=None): observe_stage(observer, observation, phase) result = download(client, run_id, args.name, args.expected_sha256, args.out, result_ref=ref) else: - if args.command=='project-transfer': - action=args.transfer_action.replace('-','_');operation='project_transfer.'+action;arguments={} - if action!='status':arguments['binding']=args.binding - if action=='preview':arguments['selection']=read_arguments(args.selection) - if action in ('report_page','stop','extract'):arguments['report_sha256']=args.report_sha256 - if action=='report_page':arguments.update(section=args.section,offset=args.offset,limit=args.limit) - if action=='stop':arguments.update(backup_root=args.backup_root,backup_directory=args.backup_directory) - if action in ('stop','extract'):arguments.update(destination_root=args.destination_root,directory=args.directory) - if action=='extract':arguments.update(name=args.name,request_key=args.request_key) - elif args.command == 'file-status': - operation, arguments = 'project_files.status', {} - elif args.command.startswith('file-'): - action = args.command[5:].replace('-', '_') - operation = 'project_files.' + action - if action in ('new', 'open', 'save', 'close', 'save_as'): - arguments = dict(binding=args.binding, request_key=args.request_key, - expected_checkpoint=None if args.expected_checkpoint == 'null' else args.expected_checkpoint) - if action in ('new', 'open', 'save_as'): - arguments.update(root=args.root, directory=args.directory) - if action in ('new','save_as'): - arguments['name'] = args.name - else: - arguments = read_arguments(args.arguments) - elif args.command == 'call': + if args.command == 'call': operation, arguments = args.operation, read_arguments(args.arguments) elif args.command == 'phase-timings': operation, arguments = 'runs.phase_timings', {'run_id': args.run_id} diff --git a/project_composition.py b/project_composition.py deleted file mode 100644 index ee667167..00000000 --- a/project_composition.py +++ /dev/null @@ -1,328 +0,0 @@ -"""Pure retained-source assembly for native independent project composition. - -Sources are ordered: the primary supplies the base envelope for an overlapping -identity. Exact duplicates share custody. Only admission-proof grant/consumption -variants of otherwise byte-identical Runs can coexist as native Run observations. -No source root, catalog permission, runtime or editable overlay is activated here. -The lifecycle owner must validate each source and the complete final candidate, -clone primary edits and publish all required bytes through the native owners. -""" -import contextvars -from contextlib import contextmanager -import hashlib - -from project_fork import ARCHIVE_FAMILIES, _sha, _uuid, decode_fork -from project_history import capture_run_observation -from project_store import (MAX_RECORDS, MAX_TOTAL_BYTES, ProjectStoreError, - _encode, _records) - - -class ProjectCompositionError(ProjectStoreError): - pass - - -def _fail(message, code='invalid', status=400): - raise ProjectCompositionError(message, code, status) - - -_KNOWN_ENVELOPES = contextvars.ContextVar('known_envelope_digests', default=None) - - -@contextmanager -def known_envelope_digests(known): - """Supply verified envelope digests for exact record objects during validation. - - known maps id(record) -> (record, digest) for envelopes whose document is the - stored canonical revision. Only that same object uses its digest; any other - envelope (for example a constructed observation variant) is encoded. - """ - token = _KNOWN_ENVELOPES.set(known) - try: - yield - finally: - _KNOWN_ENVELOPES.reset(token) - - -def _record_sha(record): - known = _KNOWN_ENVELOPES.get() - if known is not None: - entry = known.get(id(record)) - if entry is not None and entry[0] is record: - return entry[1] - return hashlib.sha256(_encode(record)).hexdigest() - - -def _approval_variant(original, alternate): - if original['family'] != 'run_history': - return False - left, right = original['document'], alternate['document'] - if _encode({k:v for k,v in left.items() if k != 'admission_proof'}) != _encode( - {k:v for k,v in right.items() if k != 'admission_proof'}): - return False - left, right = left.get('admission_proof'), right.get('admission_proof') - fields = {'grant', 'consumption', 'admission'} - return (type(left) is dict and type(right) is dict and set(left) == fields and set(right) == fields - and _encode(left['admission']) == _encode(right['admission'])) - - -def merge_retained_sources(snapshots): - """Return owned originals, exact source memberships and inactive dispositions. - - This internal construction step is not a publication or a fresh domain - admission boundary. Each source must already have passed its domain owners. - Store shape/size checks are retained here. Sources may be supplied lazily - to keep one temporary source snapshot in memory at a time. Same-ID conflicts outside the one - explicit Run-proof case are rejected; nothing is renamed or overwritten. - Membership digests cover complete canonical envelopes, not Python equality. - """ - try: - snapshots = iter(snapshots) - except TypeError: - _fail('Composition requires explicit complete source snapshots.') - projects, indexed, hashes, sizes = set(), {}, {}, {} - memberships, archived = [], set() - for snapshot in snapshots: - if type(snapshot) is not dict or set(snapshot) != {'project_id','name','checkpoint','records'}: - _fail('Composition requires complete canonical source snapshots.') - _uuid(snapshot['project_id']); _sha(snapshot['checkpoint']) - if snapshot['project_id'] in projects: - _fail('Composition source project identity repeats.', 'conflict', 409) - projects.add(snapshot['project_id']) - if type(snapshot['name']) is not str or not snapshot['name'].strip() or len(snapshot['name']) > 200: - _fail('Composition source name differs from the project contract.') - previous = decode_fork(snapshot['records'], project_id=snapshot['project_id']) - if previous is not None: - archived.update((item['family'], item['id']) for item in previous['archived']) - membership = dict(project_id=snapshot['project_id'], name=snapshot['name'], - checkpoint=snapshot['checkpoint'], records={}, run_observations={}) - if len(snapshot['records']) > MAX_RECORDS: - _fail('Composition source exceeds the project record budget.', 'too_large', 413) - source_bytes = 0 - for original in snapshot['records']: - # Validate/copy one envelope at a time. Holding another complete - # source-sized serialized/deep-copy list is unnecessary for custody. - family, identity, document, raw = _records([original])[0] - source_bytes += len(raw) - if source_bytes > MAX_TOTAL_BYTES: - _fail('Composition source exceeds the project byte budget.', 'too_large', 413) - record = dict(family=family, id=identity, document=document) - key = family, identity - sha = _record_sha(record) - membership['records'].setdefault(family, {})[identity] = sha - if family in ARCHIVE_FAMILIES: - archived.add(key) - if key not in indexed: - indexed[key], hashes[key], sizes[key] = record, sha, len(raw) - elif hashes[key] != sha: - if not _approval_variant(indexed[key], record): - _fail('Composition has conflicting retained envelopes: '+family+'/'+identity+'.', 'conflict', 409) - observation = capture_run_observation(document) - observation_key = observation['family'], observation['id'] - observation_sha = _record_sha(observation) - if observation_key in indexed and hashes[observation_key] != observation_sha: - _fail('Composition Run observation identity conflicts.', 'conflict', 409) - indexed[observation_key], hashes[observation_key] = observation, observation_sha - sizes[observation_key] = len(_records([observation])[0][3]) - membership['run_observations'][identity] = observation['id'] - memberships.append(membership) - del snapshot - if len(projects) < 2: - _fail('Composition requires at least two explicit complete source snapshots.') - if len(indexed) > MAX_RECORDS or sum(sizes.values()) > MAX_TOTAL_BYTES: - _fail('Composed retained sources exceed the project record budget.', 'too_large', 413) - return dict(records=[indexed[key] for key in sorted(indexed)], sources=memberships, - archived=[dict(family=family,id=identity) for family,identity in sorted(archived)]) - - -FAMILY = 'project_composition' -FORMAT = 'project_composition_v1' - - -def composition_input(sources, directory, name, request_key): - return dict(format='project_composition_request_v1', - sources=[dict(project_id=s['project_id'],checkpoint=s['checkpoint']) for s in sources], - directory=directory,name=name,request_key=request_key) - - -def _request_digest(sources, directory, name, request_key): - return hashlib.sha256(_encode(composition_input(sources,directory,name,request_key))).hexdigest() - - -def _shape(value, fields, label): - if type(value) is not dict or set(value) != set(fields): - _fail('Invalid composition '+label+' fields.') - - -def _decode_memberships(document, indexed): - from project_store import REQUEST_KEY, _identifier, _relative_parts - _shape(document,('format','destination_project_id','directory','name','request_key','request_sha256','sources'),'provenance') - if document['format'] != FORMAT: - _fail('Unsupported composition provenance format.','unsupported',422) - _uuid(document['destination_project_id']);_relative_parts(document['directory']) - if type(document['name']) is not str or not document['name'].strip() or len(document['name'])>200: - _fail('Invalid composition name.') - if type(document['request_key']) is not str or not REQUEST_KEY.fullmatch(document['request_key']): - _fail('Invalid composition request key.') - _sha(document['request_sha256']) - sources=document['sources'] - if type(sources) is not list or len(sources)<2: - _fail('Composition provenance needs at least two sources.') - seen=set() - for source in sources: - _shape(source,('project_id','name','checkpoint','records','run_observations'),'source membership') - _uuid(source['project_id']);_sha(source['checkpoint']) - if source['project_id'] in seen or source['project_id']==document['destination_project_id']: - _fail('Composition source identities repeat or alias the destination.') - seen.add(source['project_id']) - if type(source['name']) is not str or not source['name'].strip() or len(source['name'])>200: - _fail('Invalid composition source name.') - if type(source['records']) is not dict or type(source['run_observations']) is not dict: - _fail('Composition membership must use explicit inventories.') - runs=source['records'].get('run_history',{}) - if type(runs) is not dict:_fail('Composition Run membership must be an object.') - for identity,sha in source['run_observations'].items(): - _identifier(identity,'Composition original Run ID');_sha(sha) - if not set(source['run_observations'])<=set(runs): - _fail('Composition has an observation mapping without an original Run.') - for family,members in source['records'].items(): - _identifier(family,'Composition member family') - if type(members) is not dict or not members: - _fail('Composition family membership must be a nonempty object.') - for identity,sha in members.items(): - _identifier(identity,'Composition member ID');_sha(sha) - alternate=source['run_observations'].get(identity) if family=='run_history' else None - if alternate is None: - retained=indexed.get((family,identity)) - if retained is None:_fail('Composition source member is missing.','unavailable',409) - else: - from project_history import _observation_document - _sha(alternate) - wrapped=indexed.get(('run_observation',alternate)) - if wrapped is None:_fail('Composition source observation is missing.','unavailable',409) - observed=_observation_document(wrapped) - retained=dict(family='run_history',id=identity,document=observed['observation']) - base=indexed.get(('run_history',identity)) - if (base is None or not _approval_variant(base,retained) - or _record_sha(base)==_record_sha(retained)): - _fail('Composition alternate observation is not an admission-proof-only variant.') - if _record_sha(retained)!=sha: - _fail('Composition source membership differs from its exact retained envelope.') - if document['request_sha256']!=_request_digest(sources,document['directory'],document['name'],document['request_key']): - _fail('Composition request receipt differs from its exact sources.') - - -def decode_records(records, *, project_id=None): - """Validate inert source provenance and its independently owned fork binding.""" - from project_fork import _decode_document - indexed={(r['family'],r['id']):r for r in records} - if len(indexed)!=len(records):_fail('Composition record identity repeats.') - result={} - for record in records: - if record['family']!=FAMILY:continue - document=record['document'];_decode_memberships(document,indexed) - if record['id']!=document['destination_project_id']: - _fail('Composition envelope belongs to a different destination.') - fork=indexed.get(('project_fork',record['id'])) - if fork is None:_fail('Composition has no native fork overlay.','unavailable',409) - _decode_document(fork['document'],record['id'],indexed) - overlay=fork['document'];primary=document['sources'][0] - if (overlay['source']!={key:primary[key] for key in ('project_id','checkpoint')} - or any(overlay[key]!=document[key] for key in ('directory','name','request_key'))): - _fail('Composition provenance differs from its primary fork binding.') - archived={(r['family'],r['id']) for r in overlay['archived']} - required={(family,identity) for source in document['sources'] for family,members in source['records'].items() - for identity in members if family in ARCHIVE_FAMILIES} - if not required<=archived: - _fail('Composition must archive all source editable values and operation receipts.') - result[record['id']]=document - if project_id is not None: - current=decode_fork(records,project_id=project_id) - archived=set() if current is None else {(r['family'],r['id']) for r in current['archived']} - if any(identity!=project_id and (FAMILY,identity) not in archived for identity in result): - _fail('Prior composition provenance must remain explicitly archived.') - return result - - -def validate_source_memberships(records, snapshots, destination_project_id): - """Bind a constructed candidate to the actual owner-selected complete inputs.""" - document=next((r['document'] for r in records if r['family']==FAMILY and r['id']==destination_project_id),None) - if document is None:_fail('Composition candidate has no destination provenance.') - if len(document['sources'])!=len(snapshots):_fail('Composition candidate source count differs.') - for membership,snapshot in zip(document['sources'],snapshots): - if any(membership[key]!=snapshot[key] for key in ('project_id','name','checkpoint')): - _fail('Composition candidate belongs to different source identities.') - expected={} - for record in snapshot['records']: - expected.setdefault(record['family'],{})[record['id']]=_record_sha(record) - if membership['records']!=expected: - _fail('Composition candidate does not preserve the complete source membership.') - _decode_memberships(document,{(r['family'],r['id']):r for r in records}) - - -def build_candidate(snapshots, destination_project_id, *, created_at, id_factory, - directory, name, request_key): - """Preserve all sources and independently fork only the primary editable state.""" - import copy - from project_fork_composition import build_candidate as fork_candidate, artifact_plan - from project_fork import ID - merged=merge_retained_sources(snapshots) - _uuid(destination_project_id) - used={s['project_id'] for s in snapshots} - if destination_project_id in used:_fail('Composition destination must have a fresh identity.') - used.update(r['id'] for r in merged['records'] if ID.fullmatch(r['id'])) - used.update(r['document']['current']['id'] for r in merged['records'] - if r['family'] in ('model_definition','pipeline_graph','recipe_definition')) - used.add(destination_project_id) - def allocate(family,identity,revision): - value=id_factory(family,identity,revision);_uuid(value) - if value in used:_fail('Composition allocator reused a source or allocated identity.') - used.add(value);return value - fork=fork_candidate(snapshots[0],destination_project_id,created_at=created_at,id_factory=allocate) - indexed={(r['family'],r['id']):r for r in merged['records']} - for record in fork['records']: - key=record['family'],record['id'] - if key in indexed: - if _record_sha(indexed[key])!=_record_sha(record):_fail('Primary fork changed an original envelope.') - else:indexed[key]=record - document=dict(format=FORMAT,destination_project_id=destination_project_id,directory=directory,name=name, - request_key=request_key,request_sha256=_request_digest(merged['sources'],directory,name,request_key), - sources=merged['sources']) - record=dict(family=FAMILY,id=destination_project_id,document=document) - if (FAMILY,destination_project_id) in indexed:_fail('Composition provenance identity collides.') - indexed[(FAMILY,destination_project_id)]=record - records=[indexed[key] for key in sorted(indexed)] - validate_source_memberships(records,snapshots,destination_project_id) - archived={(r['family'],r['id']) for r in merged['archived']+fork['archived']} - # Every input has its own valid current fork. The combined records receive - # their one new current overlay only in the publication owner. Ask each - # source owner for its preservation policy before combining those roots; - # independent editable forks introduce no new payload identities. - plans={} - inventories=[fork['artifacts']] - inventories.extend(artifact_plan(source['records']) for source in snapshots[1:]) - for inventory in inventories: - for edge in inventory: - sha=edge['declaration']['sha256'];previous=plans.get(sha) - if previous is not None and previous['declaration']!=edge['declaration']: - _fail('Composition source artifact declarations conflict.') - if previous is None or edge['preservation']=='unique': - plans[sha]=copy.deepcopy(edge) - return dict(records=records,mapping=copy.deepcopy(fork['mapping']), - archived=[dict(family=f,id=i) for f,i in sorted(archived)], - artifacts=[plans[sha] for sha in sorted(plans)]) - - -def publish_composition(primary_store, primary_artifacts, retained_sources, trusted_root_fd, directory, - *, root_path, name, expected_checkpoint, request_key, quiescent, - created_at, id_factory, validate_candidate): - """Publish through the native fork lifecycle with explicit additional owners.""" - from functools import partial - from project_fork import _publish_project - if type(retained_sources) not in (list,tuple) or not retained_sources: - _fail('Composition requires at least one explicit secondary source owner.') - return _publish_project(primary_store,primary_artifacts,trusted_root_fd,directory, - root_path=root_path,name=name,expected_checkpoint=expected_checkpoint,request_key=request_key, - quiescent=quiescent,retained_sources=retained_sources, - build_candidate=partial(build_candidate,created_at=created_at,id_factory=id_factory, - directory=directory,name=name,request_key=request_key), - validate_candidate=validate_candidate) diff --git a/project_custody.py b/project_custody.py deleted file mode 100644 index 1a188f1d..00000000 --- a/project_custody.py +++ /dev/null @@ -1,511 +0,0 @@ -"""Private original-host custody index; never part of a portable project. - -Only fixed Run and producer facts are recorded. These files can fence a local -physical projection and identify its exact receipts; they never grant execution, -select a PID to signal, or restore a Run/admission from project history. -""" -import copy -import hashlib -import json -import os -from pathlib import Path -import re -import stat -import threading -import uuid -from run_capacity import MAX_ATTEMPTS - -NATIVE_PHASE_LIMIT=3*MAX_ATTEMPTS+1 - -import agent_api -from workbench import WorkbenchError - -LIMIT = 1024 * 1024 -ID = re.compile(r'^[a-f0-9]{32}$') -SHA = re.compile(r'^[a-f0-9]{64}$') -PRODUCERS = frozenset(('generation_history','grounded_history','caption_proposal_history','text_proposal_history')) -PHASES = ('admitted','launch_intent','launched','ready','go_intent','drained') -DIR_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC -FILE_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC - - -def _fail(message, code='local_custody_unavailable'): - raise WorkbenchError(message, code, 409) - - -def _id(value): - if type(value) is not str or not ID.fullmatch(value):_fail('Invalid local custody identity.') - return value - - -def _sha(value): - if type(value) is not str or not SHA.fullmatch(value):_fail('Invalid local custody digest.') - return value - - -def _shape(value, fields): - if type(value) is not dict or set(value) != set(fields):_fail('Unclassified local custody fields.') - - -def records_sha256(records): - return hashlib.sha256(agent_api.encoded(records)).hexdigest() - - -def validate(value): - version=value.get('format') if type(value) is dict else None - fields=('format','host_catalog_id','project_id','root_identity','projection_id', - 'project_name','selected_checkpoint','attempted','runs','producers','runtime_effects','package_effects') - _shape(value,(*fields,*(['native_phases'] if version in ('project_local_custody_v2','project_local_custody_v3') else []), - *(['native_admissions'] if version=='project_local_custody_v3' else []))) - if version not in ('project_local_custody_v1','project_local_custody_v2','project_local_custody_v3'):_fail('Unsupported local custody format.') - for key in ('host_catalog_id','project_id','projection_id'):_id(value[key]) - if type(value['project_name']) is not str or not value['project_name'].strip() or len(value['project_name'])>200: - _fail('Invalid retained local project name.') - root=value['root_identity'] - if type(root) is not list or len(root)!=2 or any(type(n) is not int or n<0 for n in root): - _fail('Invalid local project physical identity.') - _sha(value['selected_checkpoint']) - if value['attempted'] is not None: - attempt=value['attempted'];_shape(attempt,('records_sha256','request_key','input_sha256')) - _sha(attempt['records_sha256']);_sha(attempt['input_sha256']) - if type(attempt['request_key']) is not str or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}',attempt['request_key']): - _fail('Invalid local canonical publication request.') - if type(value['runs']) is not list or len(value['runs'])>16:_fail('Too many unresolved local Runs.') - seen=set() - for run in value['runs']: - _shape(run,('run_id','admission_id','request_sha256','spec_sha256','phase','custody_identity','receipt_sha256')) - _id(run['run_id']);_id(run['admission_id']);_sha(run['request_sha256']);_sha(run['spec_sha256']) - if run['run_id'] in seen or run['phase'] not in PHASES:_fail('Invalid or duplicate local Run custody.') - seen.add(run['run_id']) - identity=run['custody_identity'] - if identity is not None: - _shape(identity,('attempt_id','request_sha256','spec_sha256','helper')) - from recipe_process import valid_identity - if (identity['attempt_id']!=run['run_id'] or identity['request_sha256']!=run['request_sha256'] - or identity['spec_sha256']!=run['spec_sha256'] or not valid_identity(identity['helper'])): - _fail('Local helper identity differs from its admitted specification.') - if run['phase'] in ('launched','ready','go_intent','drained') and identity is None: - _fail('Local launched Run requires its observed helper identity.') - if run['receipt_sha256'] is not None:_sha(run['receipt_sha256']) - if run['receipt_sha256'] is not None and run['phase']!='drained':_fail('Drain receipt precedes drainage.') - if type(value['producers']) is not list or len(value['producers'])>16:_fail('Too many unresolved local producers.') - seen=set() - for producer in value['producers']: - _shape(producer,('family','producer_id','admission_id','request_sha256','phase','provider_outcome','worker_drained')) - if producer['family'] not in PRODUCERS:_fail('Unsupported local producer custody family.') - for key in ('producer_id','admission_id'):_id(producer[key]) - _sha(producer['request_sha256']) - identity=producer['family'],producer['producer_id'] - if identity in seen or producer['phase'] not in ('admitted','effect_intent','unknown'): - _fail('Invalid or duplicate current producer custody.') - seen.add(identity) - if producer['provider_outcome'] not in ('not_admitted','unknown','result_received') or type(producer['worker_drained']) is not bool: - _fail('Invalid current producer outcome.') - for family,identity_field in (('runtime_effects','manifest_sha256'),('package_effects','material_sha256')): - entries=value[family] - if type(entries) is not list or len(entries)>8:_fail('Too many unresolved local file writers.') - seen=set() - for entry in entries: - _shape(entry,('request_key','input_sha256',identity_field)) - key=entry['request_key'] - if type(key) is not str or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}',key) or key in seen: - _fail('Invalid local file writer request.') - seen.add(key) - for field in ('input_sha256',identity_field): - if entry[field] is not None:_sha(entry[field]) - native=value.get('native_phases',[]) - if type(native) is not list or len(native)>(NATIVE_PHASE_LIMIT if version=='project_local_custody_v3' else 16):_fail('Too many unresolved native phases.') - seen=set() - for phase in native: - _shape(phase,('phase_id','phase','run_id','request_sha256','spec_sha256','state','helper','receipt_sha256')) - _id(phase['phase_id']);_sha(phase['request_sha256']);_sha(phase['spec_sha256']) - if phase['phase_id'] in seen or phase['phase'] not in ('probe','train','verify'): - _fail('Invalid or duplicate native phase custody.') - seen.add(phase['phase_id']) - if phase['phase']!='probe' and phase['run_id'] is None:_fail('Native phase Run association differs.') - if phase['run_id'] is not None:_id(phase['run_id']) - if phase['state'] not in ('prepared','launch_intent','launched','ready','go_intent','completed','failed','unknown'): - _fail('Unclassified native custody state.') - from recipe_process import valid_identity - if phase['helper'] is not None and not valid_identity(phase['helper']):_fail('Invalid native helper observation.') - if phase['state'] in ('launched','ready','go_intent','completed') and phase['helper'] is None: - _fail('Native launched custody lacks its actual helper identity.') - if phase['receipt_sha256'] is not None: - _sha(phase['receipt_sha256']) - if phase['state'] not in ('completed','failed') or phase['helper'] is None: - _fail('Native terminal receipt lacks exact helper drainage.') - if phase['state']=='completed' and phase['receipt_sha256'] is None:_fail('Native completion requires drainage proof.') - admissions=value.get('native_admissions',[]) - if type(admissions) is not list or len(admissions)>16:_fail('Too many unresolved native admissions.') - seen=set() - for admission in admissions: - _shape(admission,('run_id','admission_id','request_sha256','graph_sha256','release_sha256','runtime_sha256','phase_started')) - for key in ('run_id','admission_id'):_id(admission[key]) - for key in ('request_sha256','graph_sha256','release_sha256','runtime_sha256'):_sha(admission[key]) - if admission['run_id'] in seen or type(admission['phase_started']) is not bool: - _fail('Invalid or duplicate native admission custody.') - seen.add(admission['run_id']) - return copy.deepcopy(value) - - -def _read(directory, name): - try:descriptor=os.open(name,FILE_FLAGS,dir_fd=directory) - except FileNotFoundError:return None - except OSError:_fail('Local custody index is not an anchored ordinary file.') - try: - before=os.fstat(descriptor) - if (not stat.S_ISREG(before.st_mode) or before.st_uid!=os.getuid() or before.st_nlink!=1 - or before.st_mode & 0o077 or before.st_size>LIMIT):_fail('Unsafe local custody index file.') - chunks=[];size=0 - while size<=LIMIT: - chunk=os.read(descriptor,min(65536,LIMIT+1-size)) - if not chunk:break - chunks.append(chunk);size+=len(chunk) - raw=b''.join(chunks);after=os.fstat(descriptor) - if len(raw)>LIMIT or (before.st_dev,before.st_ino,before.st_size,before.st_mtime_ns,before.st_ctime_ns)!=( - after.st_dev,after.st_ino,after.st_size,after.st_mtime_ns,after.st_ctime_ns):_fail('Local custody changed while reading.') - def pairs(items): - result={} - for key,item in items: - if key in result:_fail('Duplicate local custody JSON field.') - result[key]=item - return result - return validate(json.loads(raw,object_pairs_hook=pairs)) - except (OSError,UnicodeError,ValueError,RecursionError): - _fail('Local custody index cannot be read safely.') - finally:os.close(descriptor) - - -class ProjectCustody: - """Original host's private index, using only configured control directory.""" - def __init__(self, control_path, host_catalog_id): - self.path=Path(control_path);self.host_catalog_id=_id(host_catalog_id) - self.lock=threading.RLock() - - def _directory(self, create=False): - parent=os.open(self.path,DIR_FLAGS) - try: - if create: - try:os.mkdir('.project-custody',0o700,dir_fd=parent);os.fsync(parent) - except FileExistsError:pass - try:directory=os.open('.project-custody',DIR_FLAGS,dir_fd=parent) - except FileNotFoundError:return None - info=os.fstat(directory) - if not stat.S_ISDIR(info.st_mode) or info.st_uid!=os.getuid() or info.st_mode&0o077: - os.close(directory);_fail('Unsafe original-host custody directory.') - return directory - finally:os.close(parent) - - def _binding(self, store, projection_id): - snapshot=store.summary() - with store.root_directory() as descriptor: - info=os.fstat(descriptor) - return dict(host_catalog_id=self.host_catalog_id,project_id=snapshot['project_id'], - root_identity=[info.st_dev,info.st_ino],projection_id=_id(projection_id)) - - @staticmethod - def _name(binding): - return binding['host_catalog_id']+'.'+binding['project_id']+'.'+binding['projection_id']+'.json' - - def for_projection(self, store, projection_id): - return CustodyJournal(self,store,self._binding(store,projection_id)) - - def pending(self, store): - """Read only; no SQL owner, directory creation or worker recovery.""" - binding=self._binding(store,'0'*32);directory=self._directory() - if directory is None:return [] - result=[] - try: - names=os.listdir(directory) - if len(names)>10000:_fail('Original-host custody index requires explicit maintenance.') - # Local custody belongs to the physical root. A different valid - # project UUID at that same inode cannot erase an unresolved effect. - prefix=self.host_catalog_id+'.' - for name in sorted(names): - if not name.startswith(prefix):continue - if not re.fullmatch(r'[a-f0-9]{32}\.[a-f0-9]{32}\.[a-f0-9]{32}\.json',name): - _fail('Unclassified original-host custody index filename.') - value=_read(directory,name) - if value is None:continue - if self._name(value)!=name or value['host_catalog_id']!=self.host_catalog_id: - _fail('Local custody filename differs from its host binding.') - if value['root_identity']!=binding['root_identity']:continue - if value['runs'] or value['producers'] or value['runtime_effects'] or value['package_effects'] or value.get('native_phases') or value.get('native_admissions'): - result.append({key:copy.deepcopy(value[key]) for key in ('host_catalog_id','project_id','root_identity','projection_id')}) - return result - finally:os.close(directory) - - -class CustodyJournal: - def __init__(self, owner, store, binding): - self.owner,self.store,self.binding=owner,store,binding - self.name=owner._name(binding);self.lock=threading.RLock() - directory=owner._directory() - try:value=_read(directory,self.name) if directory is not None else None - finally: - if directory is not None:os.close(directory) - persisted=value is not None - if value is None: - value=dict(format='project_local_custody_v1',**binding, - project_name=store.summary()['name'],selected_checkpoint=store.summary()['checkpoint'], - attempted=None,runs=[],producers=[],runtime_effects=[],package_effects=[]) - if any(value[key]!=item for key,item in binding.items()):_fail('Original physical custody binding differs.') - self.value=validate(value) - self._persisted=persisted - - @property - def has_pending(self): - with self.lock:return bool(self.value['runs'] or self.value['producers'] or self.value['runtime_effects'] or self.value['package_effects'] or self.value.get('native_phases') or self.value.get('native_admissions')) - - def snapshot(self): - with self.lock:return copy.deepcopy(self.value) - - def refresh(self): - """Read this exact private binding after an uncertain journal ACK. - - This neither reads a canonical checkout nor clears an effect. The - execution owner separately proves selection and current authority. - """ - with self.owner.lock,self.lock: - directory=self.owner._directory() - try:current=_read(directory,self.name) if directory is not None else None - finally: - if directory is not None:os.close(directory) - if current is None: - if self._persisted:_fail('Original physical custody index is missing.') - return self.snapshot() - if any(current[key]!=value for key,value in self.binding.items()): - _fail('Original physical custody binding changed.') - self.value=current;self._persisted=True - return self.snapshot() - - def _write(self, value): - value=validate(value);raw=agent_api.encoded(value) - if len(raw)>LIMIT:_fail('Local custody exceeds its bounded format.') - directory=self.owner._directory(create=True);temporary='.custody-'+uuid.uuid4().hex - try: - descriptor=os.open(temporary,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW|os.O_CLOEXEC,0o600,dir_fd=directory) - try: - os.fchmod(descriptor,0o600);view=memoryview(raw) - while view: - count=os.write(descriptor,view) - if not count:raise OSError('Short local custody write') - view=view[count:] - os.fsync(descriptor) - finally:os.close(descriptor) - os.replace(temporary,self.name,src_dir_fd=directory,dst_dir_fd=directory);os.fsync(directory) - self.value=value - self._persisted=True - except OSError as error: - raise WorkbenchError('Local custody publication outcome is uncertain. Execution remains fenced.','outcome_unknown',409) from error - finally: - try:os.unlink(temporary,dir_fd=directory) - except OSError:pass - os.close(directory) - - def _change(self, change): - with self.owner.lock,self.lock: - directory=self.owner._directory() - try:current=_read(directory,self.name) if directory is not None else None - finally: - if directory is not None:os.close(directory) - if current is not None and current!=self.value or current is None and self._persisted: - _fail('Original-host custody changed; refuse to replace another writer.','outcome_unknown') - value=copy.deepcopy(self.value);change(value);self._write(value) - - def expect_publication(self, before, records, key, sha): - if not self.has_pending:return - def change(value): - value['selected_checkpoint']=before['checkpoint'] - value['attempted']=dict(records_sha256=records_sha256(records),request_key=key,input_sha256=sha) - self._change(change) - - def acknowledge(self, snapshot): - def change(value):value.update(selected_checkpoint=snapshot['checkpoint'],attempted=None) - self._change(change) - - def validate_selection(self, snapshot): - value=self.snapshot();attempt=value['attempted'] - if snapshot['project_id']!=value['project_id'] or snapshot['name']!=value['project_name']: - _fail('Current canonical project differs from local custody.') - if snapshot['checkpoint']==value['selected_checkpoint']:return - if (attempt is not None and records_sha256(snapshot['records'])==attempt['records_sha256'] - and self.store.reconcile(attempt['request_key'],attempt['input_sha256']) is not None):return - _fail('Canonical checkout differs from this original-host custody outcome.','stale_project') - - def observe_run(self, run, *, receipt_sha256=None): - from recipe_process import encoded - spec_sha=hashlib.sha256(encoded(run['spec'])).hexdigest() - def change(value): - old=next((item for item in value['runs'] if item['run_id']==run['id']),None) - immutable=dict(run_id=run['id'],admission_id=run['admission_id'],request_sha256=run['request_sha256'],spec_sha256=spec_sha) - if old is None: - old=dict(**immutable,phase='admitted',custody_identity=None,receipt_sha256=None);value['runs'].append(old) - if any(old[key]!=item for key,item in immutable.items()):_fail('Current Run differs from its local admission journal.') - phase={'prepared':'admitted','barrier_ready':'ready','go_intent':'go_intent','drained':'drained'}[run['custody_phase']] - if PHASES.index(phase)>PHASES.index(old['phase']):old['phase']=phase - identity=run.get('custody_identity') - if identity is not None: - if old['custody_identity'] is not None and old['custody_identity']!=identity:_fail('Current helper identity changed.') - old['custody_identity']=copy.deepcopy(identity) - if receipt_sha256 is not None:old['receipt_sha256']=receipt_sha256 - value['runs'].sort(key=lambda item:item['run_id']) - self._change(change) - - def launch_intent(self, run_id): - def change(value): - run=next((item for item in value['runs'] if item['run_id']==run_id),None) - if run is None or run['phase']!='admitted':_fail('Launch requires its acknowledged current admission.') - run['phase']='launch_intent' - self._change(change) - - def launched(self, run_id, helper): - from recipe_process import valid_identity - if not valid_identity(helper):_fail('Current spawned custodian identity is unavailable.') - def change(value): - run=next((item for item in value['runs'] if item['run_id']==run_id),None) - if run is None or run['phase']!='launch_intent':_fail('Custodian has no current launch intent.') - run['phase']='launched' - run['custody_identity']=dict(attempt_id=run_id,request_sha256=run['request_sha256'],spec_sha256=run['spec_sha256'],helper=copy.deepcopy(helper)) - self._change(change) - - def settle_run(self, run_id): - def change(value):value['runs']=[item for item in value['runs'] if item['run_id']!=run_id] - self._change(change) - - def observe_native_admission(self,run): - """Original physical admission intent; never a replay/GO capability.""" - from pipeline import digest - entry=dict(run_id=run['id'],admission_id=run['admission_id'],request_sha256=run['request_sha256'], - graph_sha256=run['graph_sha256'],release_sha256=run['release_sha256'],runtime_sha256=digest(run['runtime'])) - def change(value): - value['format']='project_local_custody_v3';value.setdefault('native_phases',[]) - rows=value.setdefault('native_admissions',[]) - old=next((item for item in rows if item['run_id']==run['id']),None) - if old is None:rows.append(dict(entry,phase_started=False)) - elif any(old[key]!=item for key,item in entry.items()):_fail('Native admission differs from its original physical intent.') - rows.sort(key=lambda row:row['run_id']) - self._change(change) - - def settle_native_admission(self,run_id): - def change(value): - if any(row['run_id']==run_id for row in value.get('native_phases',[])): - _fail('Native admission still has unresolved phase custody.') - value['native_admissions']=[row for row in value.get('native_admissions',[]) if row['run_id']!=run_id] - self._change(change) - - def native_prepared(self, phase): - """Fence one acknowledged, current native phase on this physical host.""" - from recipe_process import encoded - entry=dict(phase_id=phase['phase_id'],phase=phase['phase'],run_id=phase['run_id'], - request_sha256=phase['request_sha256'],spec_sha256=hashlib.sha256(encoded(phase['spec'])).hexdigest(), - state='prepared',helper=None,receipt_sha256=None) - def change(value): - if value['format']=='project_local_custody_v1':value['format']='project_local_custody_v2' - value.setdefault('native_phases',[]) - if entry['run_id'] is not None: - admission=next((row for row in value.get('native_admissions',[]) if row['run_id']==entry['run_id']),None) - if admission is None:_fail('A current native Run phase requires its original physical admission.') - admission['phase_started']=True - old=next((row for row in value['native_phases'] if row['phase_id']==entry['phase_id']),None) - if old is not None:_fail('A native phase cannot register another current launch.') - value['native_phases'].append(entry);value['native_phases'].sort(key=lambda row:row['phase_id']) - self._change(change) - - def native_launch_intent(self, phase_id): - def change(value): - row=next((row for row in value.get('native_phases',[]) if row['phase_id']==phase_id),None) - if row is None or row['state']!='prepared':_fail('Native launch requires its current acknowledged specification.') - row['state']='launch_intent' - self._change(change) - - def native_launched(self, phase_id, helper): - def change(value): - row=next((row for row in value.get('native_phases',[]) if row['phase_id']==phase_id),None) - if row is None or row['state']!='launch_intent':_fail('Native helper has no current launch intent.') - row.update(state='launched',helper=copy.deepcopy(helper)) - self._change(change) - - def native_observed(self, phase): - from recipe_process import encoded - def change(value): - row=next((row for row in value.get('native_phases',[]) if row['phase_id']==phase['phase_id']),None) - if row is None:_fail('Native observation has no original local phase.') - event=phase['events'][-1] - if (row['request_sha256']!=phase['request_sha256'] or row['phase']!=phase['phase'] - or row['run_id']!=phase['run_id'] or event['helper']!=row['helper'] - or row['spec_sha256']!=hashlib.sha256(encoded(phase['spec'])).hexdigest()): - _fail('Native observation differs from original local custody.') - transitions={'prepared':('failed',),'launch_intent':('failed',), - 'launched':('ready','failed','unknown'),'ready':('go_intent','failed','unknown'), - 'go_intent':('completed','failed','unknown')} - if event['state'] not in transitions.get(row['state'],()): - _fail('Native custody observation cannot skip or replay its acknowledged phase.') - row.update(state=event['state'],receipt_sha256=event['receipt_sha256']) - self._change(change) - - def native_confirmed(self, phase_id): - def change(value): - row=next((row for row in value.get('native_phases',[]) if row['phase_id']==phase_id),None) - if row is None or row['state'] not in ('completed','failed') or row['receipt_sha256'] is None: - _fail('Native phase cannot settle without acknowledged terminal drainage.') - value['native_phases']=[item for item in value['native_phases'] if item['phase_id']!=phase_id] - self._change(change) - - def native_not_launched(self,phase_id): - """Settle only the caller's proven never-entered launch boundary.""" - def change(value): - row=next((item for item in value.get('native_phases',[]) if item['phase_id']==phase_id),None) - if row is None or row['state']!='failed' or row['helper'] is not None or row['receipt_sha256'] is not None: - _fail('Never-launched native settlement requires its acknowledged failed non-process fact.') - value['native_phases']=[item for item in value['native_phases'] if item['phase_id']!=phase_id] - if row['run_id'] is not None and not any(item['run_id']==row['run_id'] for item in value['native_phases']): - admission=next((item for item in value.get('native_admissions',[]) if item['run_id']==row['run_id']),None) - if admission is not None:admission['phase_started']=False - self._change(change) - - def register_producer(self, family, producer_id, admission_id, request_digest): - def change(value): - item=dict(family=family,producer_id=producer_id,admission_id=admission_id,request_sha256=request_digest, - phase='admitted',provider_outcome='not_admitted',worker_drained=False) - previous=next((row for row in value['producers'] if (row['family'],row['producer_id'])==(family,producer_id)),None) - if previous is not None: - if any(previous[key]!=item[key] for key in ('family','producer_id','admission_id','request_sha256')):_fail('Producer admission changed.') - else:value['producers'].append(item) - self._change(change) - - def producer_effect_intent(self, family, producer_id): - def change(value): - item=next((row for row in value['producers'] if (row['family'],row['producer_id'])==(family,producer_id)),None) - if item is None:_fail('Provider request has no current admission journal.') - item.update(phase='effect_intent',provider_outcome='unknown',worker_drained=False) - self._change(change) - - def producer_outcome(self, family, producer_id, provider_outcome, worker_drained): - def change(value): - item=next((row for row in value['producers'] if (row['family'],row['producer_id'])==(family,producer_id)),None) - if item is None:_fail('Provider result has no current admission journal.') - item.update(provider_outcome=provider_outcome,worker_drained=worker_drained) - if provider_outcome=='unknown':item['phase']='unknown' - elif worker_drained:value['producers'].remove(item) - self._change(change) - - def runtime_started(self, key, input_sha256=None, manifest_sha256=None): - self._file_started('runtime_effects',dict(request_key=key,input_sha256=input_sha256,manifest_sha256=manifest_sha256)) - - def package_started(self, key, input_sha256=None, material_sha256=None): - self._file_started('package_effects',dict(request_key=key,input_sha256=input_sha256,material_sha256=material_sha256)) - - def _file_started(self, family, entry): - def change(value): - previous=next((item for item in value[family] if item['request_key']==entry['request_key']),None) - if previous is not None and previous!=entry:_fail('File writer key has different retained input.','conflict') - if previous is None:value[family].append(entry) - self._change(change) - - def runtime_confirmed(self, key):self._file_confirmed('runtime_effects',key) - def package_confirmed(self, key):self._file_confirmed('package_effects',key) - - def _file_confirmed(self, family, key): - def change(value):value[family]=[item for item in value[family] if item['request_key']!=key] - self._change(change) diff --git a/project_data.py b/project_data.py deleted file mode 100644 index 1b58eae6..00000000 --- a/project_data.py +++ /dev/null @@ -1,1466 +0,0 @@ -"""Workbench-owned project data and history, composed by ProjectSession. - -Each family below has a closed semantic decoder. SQL is a disposable projection; -the adapter supplies changed owned documents to the shared session, never a -replacement inventory or an independent commit. Existing domain owners admit -mutations and immutable artifact storage owns exact source bytes. -""" -import copy -from contextlib import contextmanager -from datetime import datetime -import hashlib -import json -import re -from types import SimpleNamespace -import unicodedata - -from workbench import (Workbench, WorkbenchError, IDENTIFIER, TASKS, REVIEWS, MAX_TEXT, - encode, encode_retained, strings, text_value, timestamp, validate_annotation) -from project_text import decode_records as decode_text_records, SNAPSHOT_FIELDS -from saved_searches import SavedSearches, criteria -from saved_selections import SavedSelections -from dataset_releases import Releases -from project_session import session_active, session_records, session_records_for - -FAMILIES = { - 'workbench_record': 'workbench_record_v1', - 'workbench_response': 'workbench_response_v1', - 'workbench_preference': 'workbench_preference_v1', - 'saved_selection': 'saved_selection_v1', - 'saved_search': 'saved_search_v1', - 'dataset_release': 'dataset_release_v1', - 'dataset_exposure': 'dataset_exposure_v1', -} -SHA = re.compile(r'^[a-f0-9]{64}$') -ASSET_KINDS = ('sequence', 'mesh', 'pointcloud', 'volume', 'audio', 'field_sequence') -SOURCE_FIELDS = set('id sha256 filename width height book_id session_id split created_at updated_at revision annotation'.split()) -RESPONSE_FIELDS = set('id prompt_id revision completion review provenance created_at updated_at'.split()) -JUDGMENT_FIELDS = set('id prompt_id parent_revision source_revision left_id left_revision right_id right_revision revision outcome rationale review deleted created_at updated_at provenance'.split()) - - -def fail(message, code='invalid', status=400): - raise WorkbenchError(message, code, status) - - -def closed(value, fields, name): - if type(value) is not dict or set(value) != set(fields): - fail('Invalid ' + name + ' fields.') - - -def identity(value, name='identity'): - if type(value) is not str or not IDENTIFIER.fullmatch(value): - fail('Invalid project data ' + name + '.') - return value - - -def sha(value): - if type(value) is not str or not SHA.fullmatch(value): - fail('Invalid project data digest.') - return value - - -def natural(value, minimum=1): - if type(value) is not int or not minimum <= value <= 2**53 - 1: - fail('Invalid project data revision/count.') - return value - - -def date(value): - text_value(value, 'Timestamp', 80) - try: - parsed = datetime.fromisoformat(value) - if parsed.utcoffset() is None: - raise ValueError() - return parsed - except (ValueError, TypeError): - fail('Invalid project data timestamp.') - - -def digest(value): - return hashlib.sha256(encode_retained(value).encode()).hexdigest() - - -def _audit(facts): - if type(facts) is not list: - fail('Data audit must be a list.') - for fact in facts: - closed(fact, {'action', 'target', 'details', 'result', 'actor_id', 'occurred_at'}, 'data audit') - if type(fact['action']) is not str or fact['action'] not in DATA_ACTIONS or fact['result'] != 'success' or type(fact['details']) is not dict: - fail('Invalid data audit action/result.') - identity(fact['actor_id'], 'historical actor') - if fact['target'] is not None: - text_value(fact['target'], 'Audit target', 128) - date(fact['occurred_at']) - # The current domain owners publish hashes/revisions, never credentials. - if set(fact['details']) & {'credential_id', 'token', 'admission_id', 'grant_id'}: - fail('Current access authority cannot enter data audit facts.') - - -def _events(events): - if type(events) is not list: - fail('Data events must be a list.') - for event in events: - closed(event, {'operation', 'request_key', 'record_revision', 'source_revision', 'deleted', 'occurred_at', 'snapshot'}, 'data event') - if type(event['operation']) is not str or event['operation'] not in DATA_OPERATIONS|RUNTIME_DATA_OPERATIONS or type(event['deleted']) is not bool: - fail('Invalid data lifecycle event.') - text_value(event['request_key'], 'Operation key', 128) - natural(event['record_revision']) - if event['source_revision'] is not None: - natural(event['source_revision']) - date(event['occurred_at']) - state=event['snapshot'];validate_record_snapshot(state) - if state['revision']!=event['record_revision'] or state['source_revision']!=event['source_revision']: - fail('Lifecycle event snapshot revision differs.') - - -def _edge(value, declarations, kind, role): - closed(value, {'artifact', 'media_type', 'kind', 'role', 'preservation', 'required_for'}, 'artifact edge') - closed(value['artifact'], {'namespace', 'sha256'}, 'artifact reference') - identity(value['artifact']['namespace'], 'artifact namespace') - sha(value['artifact']['sha256']) - if value['kind'] != kind or value['role'] != role or value['preservation'] not in ('unique', 'regenerable'): - fail('Artifact edge meaning differs from its data owner.') - text_value(value['media_type'], 'Media type', 120) - if value['required_for'] != ['inspect', 'release']: - fail('Unsupported data artifact required operations.') - declaration = declarations.get(value['artifact']['sha256']) - if declaration is None: - fail('Data artifact declaration is missing.') - return declaration - - -def _source_row(row, record_id): - closed(row, SOURCE_FIELDS, 'image source snapshot') - if row['id'] != record_id: - fail('Image source identity differs.') - sha(row['sha256']); natural(row['revision']) - natural(row['width']); natural(row['height']) - if row['width'] * row['height'] > 40_000_000: - fail('Image geometry exceeds owner limit.') - text_value(row['filename'], 'Filename', 200) - from image_sources import metadata, validate_annotation as validate_corners - if metadata(row) != {key: row[key] for key in ('book_id', 'session_id', 'split')}: - fail('Image source metadata is not canonical.') - if row['annotation'] is not None and validate_corners(row['annotation']) != row['annotation']: - fail('Image corner annotation is not canonical.') - if date(row['updated_at']) < date(row['created_at']): - fail('Image source update precedes creation.') - - -def _source_metadata(kind,value): - if kind=='sequence': - from sequence_assets import SequenceAssets - SequenceAssets.validate_project_metadata(value) - elif kind=='mesh': - from meshes import MeshAssets - MeshAssets.validate_project_metadata(value) - elif kind in ('pointcloud','volume'): - from spatial_assets import SpatialAssets - SpatialAssets.validate_project_metadata(kind,value) - elif kind=='audio': - from audio_assets import validate_project_metadata - validate_project_metadata(value) - else: - from scientific_fields import validate_project_metadata - validate_project_metadata(value) - - -def _acquisition(value,record,source,records): - """Only the finite receipts produced by the existing import owners.""" - if record['kind']=='sequence': - from sequence_assets import SequenceAssets - SequenceAssets.validate_project_acquisition(value,source['metadata']) - return - if type(value) is not dict:fail('Acquisition facts must be an object.') - from bulk_import import leaf_name, REQUEST_ID, MAX_ROWS - if type(value.get('request_id')) is not str or not REQUEST_ID.fullmatch(value['request_id']):fail('Invalid retained import request marker.') - format_name=value.get('format');declared=value.get('declared') - if format_name=='tuldok_assets_jsonl_v1': - closed(value,{'format','request_id','row_sha256','declared'},'JSONL acquisition') - fields={'manifest_name','row_number'}|({'image_file'} if record['kind']=='image' else set()) - closed(declared,fields,'JSONL source declaration');leaf_name(declared['manifest_name'],'Manifest name') - if record['kind']=='image':leaf_name(declared['image_file'],'Image filename') - if type(declared['row_number']) is not int or not 1<=declared['row_number']<=MAX_ROWS:fail('Invalid retained JSONL row number.') - sha(value['row_sha256']);return - if format_name=='image_caption_v1': - from caption_import import snapshot, SNAPSHOT_KEYS, RECORD_KEYS - closed(value,{'format','request_id','row_sha256','observed','declared'},'caption acquisition') - closed(value['observed'],{'manifest_sha256','metadata_sha256','asset_sha256','pixel_sha256'},'caption observed hashes') - closed(declared,{'manifest_name','metadata_path','row_number','image_file','origin_record','protected_component'},'caption declaration') - if declared['manifest_name']!='manifest.json' or declared['metadata_path'] not in ('train/metadata.jsonl','val/metadata.jsonl','test/metadata.jsonl'):fail('Invalid retained caption metadata location.') - upstream=declared['origin_record'];closed(upstream,RECORD_KEYS,'caption origin record') - members=declared['protected_component'] - if type(members) is not list or not 1<=len(members)<=30:fail('Invalid retained protected caption component.') - for member in members:snapshot(member) - if not any(member=={k:upstream[k] for k in SNAPSHOT_KEYS} for member in members):fail('Caption source is absent from its declared component.') - for digest_value in value['observed'].values():sha(digest_value) - elif format_name=='canonical_v1': - text_source=record['kind']=='text' - fields={'format','request_id','archive_sha256','manifest_sha256','row_sha256','asset_sha256','input_sha256','input_basis','declared'} - fields|={'metadata_sha256'} if text_source else {'coco_sha256','row_hash_basis','pixel_sha256','upstream_graph'} - closed(value,fields,'native acquisition') - closed(declared,{'archive_name','metadata_path','row_number','asset','upstream'},'native source declaration') - leaf_name(declared['archive_name'],'Archive filename') - closed(declared['upstream'],{'record','original_status'},'native upstream declaration') - if declared['upstream']['original_status']!='unavailable':fail('Native imported original status differs.') - upstream=declared['upstream']['record'] - from native_text_import import RECORD_KEYS - closed(upstream,RECORD_KEYS,'native origin record') - if text_source: - sha(value['metadata_sha256']) - if value['input_basis']!='consumed_exported_text' or declared['metadata_path'] not in ('train/records.jsonl','validation/records.jsonl','test/records.jsonl'):fail('Invalid retained native text input.') - else: - closed(value['coco_sha256'],{'train','validation','test'},'native COCO hashes') - for item in value['coco_sha256'].values():sha(item) - sha(value['pixel_sha256']) - if value['input_basis']!='consumed_exported_png' or value['row_hash_basis']!='canonical_parsed_manifest_record' or value['upstream_graph']!='selected_declared_links_only' or declared['metadata_path']!='manifest.json':fail('Invalid retained native image input.') - for field in ('archive_sha256','manifest_sha256','asset_sha256','input_sha256'):sha(value[field]) - else:fail('Unsupported retained acquisition format.','unsupported',409) - sha(value['row_sha256']) - if type(declared['row_number']) is not int or not 1<=declared['row_number']<=1000:fail('Invalid retained source row number.') - path=declared['image_file'] if format_name=='image_caption_v1' else declared['asset'] - if type(path) is not str or path!=upstream['asset'] or any(part in ('','.','..') for part in path.split('/')) or '\\' in path or path.startswith('/'):fail('Invalid retained source asset reference.') - captured={k:upstream[k] for k in SNAPSHOT_FIELDS} - _record(captured,identity(captured['id'])) - foreign_source=dict(kind=captured['kind'],original={'artifact':{'sha256':captured['source_sha256']}}) - origin=dict(captured['provenance']);origin.pop('rights_note_correction',None) - _origin(origin,captured,foreign_source,records) - - -def _origin(value,record,source,records): - kind=record['kind'] - method=value.get('method') - if method=='project_fork': - from project_data_fork import validate_origin - validate_origin(value,record,source,records) - return - if method in ('model_generated','model_grounded_rewrite'): - from project_producers import validate_record_origin - validate_record_origin(value,record,records) - return - if method=='procedural': - closed(value,{'method','recipe','recipe_version','seed','ordinal','factors','rights','verification'},'procedural origin') - from dataset_recipes import RECIPES - if value['recipe'] not in RECIPES or RECIPES[value['recipe']]['kind']!=kind or type(value['recipe_version']) is not int or value['recipe_version']!=1 or type(value['seed']) is not int or not 0<=value['seed']<=2**32-1 or type(value['ordinal']) is not int or not 0<=value['ordinal']<100 or type(value['factors']) is not dict: - fail('Invalid retained procedural origin.') - factors=value['factors'] - if kind=='image': - closed(factors,{'canvas','objects','occlusion'},'procedural image factors') - if factors['canvas']!=[128,96] or any(type(n) is not int for n in factors['canvas']) or type(factors['objects']) is not int or factors['objects']!=value['ordinal']%4 or type(factors['occlusion']) is not int or factors['occlusion']!=0:fail('Procedural image factors differ from the owned recipe.') - else: - from dataset_recipes import TEMPLATES - closed(factors,{'topic','day','hour','template'},'procedural text factors') - if factors['topic'] not in ('design review','project planning','reading group','demo','studio session') or factors['day'] not in ('Monday','Tuesday','Wednesday','Thursday','Friday') or type(factors['hour']) is not str or not re.fullmatch(r'(?:[89]|1[0-8]):(?:00|15|30|45)',factors['hour']) or type(factors['template']) is not int or factors['template']!=value['ordinal']%len(TEMPLATES):fail('Procedural text factors differ from the owned recipe.') - if value['verification']!='Targets derived from authored scene/template state; no human review or real-world quality claim.':fail('Procedural verification declaration differs.') - return - if kind=='text': - fields={'method','rights','normalization'} - if 'acquisition' in value:fields.add('acquisition') - closed(value,fields,'text origin') - if method!='import' or value['normalization']!='NFC; LF newlines':fail('Unsupported text origin.','unsupported',409) - elif kind=='image': - fields={'method','rights','source_sha256'} - if 'acquisition' in value:fields.add('acquisition') - closed(value,fields,'image origin') - if method!='import' or value['source_sha256']!=source['original']['artifact']['sha256']:fail('Image origin differs from its original bytes.') - elif kind=='sequence': - fields={'method','rights','adapter','run_sha256','frames_sha256'} - if 'sequence_acquisition' in value:fields.add('sequence_acquisition') - closed(value,fields,'sequence origin');meta=source['metadata'] - if method!='simulation_import' or value['adapter']!=meta['adapter'] or value['run_sha256']!=meta['run_sha256'] or value['frames_sha256']!=meta['manifest']['frames_sha256']:fail('Sequence origin differs from its owner metadata.') - elif kind in ('mesh','pointcloud','volume'): - fields={'method','rights','format','manifest_sha256'}|({'geometry_sha256'} if kind=='mesh' else set()) - closed(value,fields,'native source origin');meta=source['metadata'] - if method!=kind+'_import' or value['format']!=meta['manifest']['format'] or value['manifest_sha256']!=meta['manifest_sha256'] or kind=='mesh' and value['geometry_sha256']!=meta['manifest']['geometry_sha256']:fail('Native source provenance differs from its metadata.') - elif kind=='audio': - closed(value,{'method','rights','audio','source_sha256','transform'},'audio origin') - if method!='import' or value['audio']!=source['metadata'] or value['source_sha256']!=source['asset']['artifact']['sha256'] or value['transform']!='none; original interleaved PCM WAV bytes preserved':fail('Audio origin differs from its original source.') - else: - key='project_source' if method=='project_data_import' else 'run_source' - closed(value,{'method','rights','field_sequence','source_sha256',key},'scientific origin') - if value['field_sequence']!=source['metadata'] or value['source_sha256']!=source['asset']['artifact']['sha256']:fail('Scientific source origin differs.') - from scientific_fields import validate_source_declaration - validate_source_declaration(method,value[key],value['source_sha256'],source['metadata']['source_bytes']) - acquisition=value.get('acquisition',value.get('sequence_acquisition')) - if 'acquisition' in value or 'sequence_acquisition' in value: - _acquisition(acquisition,record,source,records) - - -def _record(row, record_id, source=None): - if type(row) is not dict or row.get('kind') not in ('image', 'text', *ASSET_KINDS): - fail('Unsupported data record kind.', 'unsupported', 409) - extra = {row['kind']} if row['kind'] in ASSET_KINDS else set() - optional = {'target_proposal'} if 'target_proposal' in row else set() - closed(row, SNAPSHOT_FIELDS | extra | optional, 'Workbench record') - if row['id'] != record_id: - fail('Workbench record identity differs.') - natural(row['revision']); sha(row['content_hash']) - if row['source_revision'] is not None: - natural(row['source_revision']) - if row['source_sha256'] is not None: - sha(row['source_sha256']) - if row['pixel_hash'] is not None: - sha(row['pixel_hash']) - text_value(row['name'], 'Name') - if strings(row['groups'], 'Groups') != row['groups'] or not row['groups'] or strings(row['parents'], 'Parents') != row['parents']: - fail('Invalid Workbench groups or parents.') - for parent in row['parents']: - identity(parent, 'parent') - if row['task'] not in TASKS or not row['task'].startswith(row['kind'] + '_') or row['review'] not in REVIEWS: - fail('Invalid Workbench task or review.') - if type(row['source_available']) is not bool or type(row['source_lineage_known']) is not bool: - fail('Invalid historical source availability.') - if type(row['provenance']) is not dict: - fail('Invalid Workbench provenance.') - text_value(row['provenance'].get('rights'), 'Rights note', 1000) - correction = row['provenance'].get('rights_note_correction') - if correction is not None: - closed(correction, {'note', 'revision'}, 'rights correction') - text_value(correction['note'], 'Rights note', 1000) - natural(correction['revision']) - if correction['revision'] > row['revision']: - fail('Rights correction is newer than its record.') - annotation_record = row - deleted_geometry = row['task'] == 'image_detection' and (row['width'] is None or row['height'] is None) - if deleted_geometry and source is not None: - # Source deletion removes the live join, not the retained image geometry. - # The complete codec validates this source history before acknowledging it. - geometry = source['revisions'][-1] - annotation_record = dict(row, width=geometry['width'], height=geometry['height']) - # A standalone tombstone frame has no geometry to prove box fit. Its full - # record decoder supplies retained source geometry; receipt validation also - # requires an exact match to that fully validated retained state. - if row['annotation'] is not None and not (deleted_geometry and source is None): - if validate_annotation(row['task'], row['annotation'], annotation_record) != row['annotation']: - fail('Workbench annotation is not canonical.') - if row['review'] != 'draft' and row['annotation'] is None: - fail('Reviewed record has no annotation.') - if row['kind'] == 'text': - text_value(row['text'], 'Text', MAX_TEXT) - if hashlib.sha256(row['text'].encode()).hexdigest() != row['content_hash']: - fail('Normalized text digest differs.') - elif row['text'] is not None: - fail('Non-text record contains text source.') - if date(row['updated_at']) < date(row['created_at']): - fail('Workbench update precedes creation.') - return row - - -def validate_record_snapshot(record): - """Pure Workbench capture validation for the producer history owner.""" - if type(record) is not dict: - fail('Captured record must be an object.') - _record(record, identity(record.get('id'))) - return copy.deepcopy(record) - - -def _response(row, record_id): - closed(row, RESPONSE_FIELDS, 'response') - if row['id'] != record_id: - fail('Response identity differs.') - identity(row['prompt_id'], 'prompt'); natural(row['revision']) - text_value(row['completion'], 'Completion', MAX_TEXT) - if row['review'] not in ('draft', 'human_reviewed'): - fail('Invalid response review.') - closed(row['provenance'], {'method', 'prompt_id', 'prompt_sha256', 'source_sha256'}, 'response provenance') - if row['provenance']['method'] not in ('human_authored', 'agent_authored') or row['provenance']['prompt_id'] != row['prompt_id']: - fail('Invalid response origin.') - sha(row['provenance']['prompt_sha256']); sha(row['provenance']['source_sha256']) - if date(row['updated_at']) < date(row['created_at']): - fail('Response update precedes creation.') - - -def _judgment(row, record_id): - closed(row, JUDGMENT_FIELDS, 'preference') - if row['id'] != record_id: - fail('Preference identity differs.') - for key in ('id', 'prompt_id', 'left_id', 'right_id'): - identity(row[key], key) - for key in ('revision', 'parent_revision', 'source_revision', 'left_revision', 'right_revision'): - natural(row[key]) - if row['left_id'] == row['right_id'] or row['outcome'] not in ('left', 'right', 'tie', 'abstain') or row['review'] not in ('draft', 'human_reviewed') or type(row['deleted']) is not bool: - fail('Invalid preference meaning.') - text_value(row['rationale'], 'Rationale', 4000) - closed(row['provenance'], {'method'}, 'preference provenance') - if row['provenance']['method'] != ('human_comparative_judgment' if row['review'] == 'human_reviewed' else 'draft_comparative_judgment'): - fail('Preference origin differs from review.') - if date(row['updated_at']) < date(row['created_at']): - fail('Preference update precedes creation.') - - -def _audit_bindings(family,record_id,doc): - """Audit hashes describe exact owner snapshots, never free-standing claims.""" - for fact in doc.get('audit',[]): - action,details=fact['action'],fact['details'] - if action=='dataset.add_generated' and family=='workbench_record': - origin=doc['origin'] - if origin.get('method')!='model_generated' or fact['target']!=origin['generation']['job_id']: - fail('Generated source audit differs from its exact producer job.') - elif fact['target']!=record_id:fail('Data audit target differs from its owning record.') - expected=None - if family=='workbench_record': - states=[doc['current'],*doc['history'],*(e['snapshot'] for e in doc['events'])] - if action.startswith('workbench.'): - candidates=[s for s in states if s['revision']==details.get('revision') and s['source_revision']==details.get('source_revision') and digest(s)==details.get('content_sha256')] - if not candidates:fail('Workbench audit snapshot is not retained.') - state=candidates[0] - expected=dict(revision=state['revision'],source_revision=state['source_revision'],content_sha256=digest(state)) - if action in ('workbench.save','workbench.approve_review','workbench.verify','workbench.correct_rights_note'): - expected.update(before_revision=state['revision']-1,before_source_revision=state['source_revision']) - elif action in ('dataset.add','dataset.add_generated'): - source=doc['source'] - if source['kind']!='image':fail('Image audit is attached to another modality.') - expected=dict(sample_id=record_id,revision=1,source_sha256=source['revisions'][0]['sha256']) - elif action=='dataset.save': - source=doc['source'];sample=next((r for r in source.get('revisions',[]) if r['revision']==details.get('revision')),None) - if sample is None:fail('Image audit source revision is not retained.') - expected=dict(before_revision=sample['revision']-1,revision=sample['revision'],annotation_sha256=digest(sample['annotation'])) - elif action=='dataset.delete': - if not doc['source'].get('deleted'):fail('Image deletion audit lacks its tombstone.') - expected={} - elif action in ('immutable_assets.admit','audio_assets.admit','scientific_fields.admit'): - expected=dict(revision=1,source_sha256=doc['current']['content_hash']) - elif family=='workbench_response': - state=next((h for h in doc['history'] if h['response']['revision']==details.get('revision')),None) - if state and action in ('workbench.save_response','workbench.approve_response'): - r,p=state['response'],state['parent'] - expected=dict(before_revision=r['revision']-1,revision=r['revision'],prompt_id=p['id'],parent_revision=p['revision'],source_revision=p['source_revision'],content_sha256=digest(state)) - elif family=='workbench_preference': - state=next((h for h in doc['history'] if h['judgment']['revision']==details.get('revision')),None) - if state and action in ('preferences.save','preferences.delete'): - expected=dict(revision=state['judgment']['revision'],review=state['judgment']['review'],binding_sha256=digest(state)) - elif family in ('saved_selection','saved_search'): - owner='saved_selections' if family=='saved_selection' else 'saved_searches' - state=next((h for h in doc['history'] if h['revision']==details.get('revision')),None) - if state: - if action==owner+'.create':expected=dict(revision=1,content_sha256=digest(state['items'] if family=='saved_selection' else state['criteria'])) - elif action==owner+'.rename':expected=dict(revision=state['revision'],name_sha256=hashlib.sha256(state['name'].encode()).hexdigest()) - elif action==owner+'.delete' and doc['deleted']:expected=dict(revision=state['revision'],retired_sha256=digest(state)) - elif family=='dataset_release' and action=='dataset_releases.create': - expected=dict(release_sha256=record_id,format=doc['release_format']) - if expected is None or encode_retained(expected)!=encode_retained(details): - fail('Data audit details do not match the retained owner facts.') - - -def _decode_data_records(records): - """Validate complete owned closure, without opening missing payloads or SQL.""" - if type(records) is not list or any(type(r) is not dict for r in records): - fail('Project records must be a list.') - owned, assets, responses, declarations, seen = [], {}, {}, {}, set() - legacy = [r for r in records if r.get('family') == 'workbench_text'] - # Legacy parent closure may cross to a new record; validate each legacy - # record locally, then check parent closure across all data families below. - for item in legacy: - single = copy.deepcopy(item) - parents = single['document']['current']['parents'] - if parents: - # The unchanged M1 decoder requires its family-local parent set. - # Supply the legacy set, whose already-admitted parents stay v1. - decode_text_records(legacy) - break - decode_text_records([single]) - for item in legacy: - assets[item['id']] = item['document'] - for envelope in records: - if type(envelope) is not dict: - fail('Invalid project envelope.') - family = envelope.get('family') - if family not in FAMILIES: - continue - closed(envelope, {'family', 'id', 'document'}, 'data envelope') - record_id, doc = envelope['id'], envelope['document'] - if family in ('dataset_release', 'dataset_exposure'): - sha(record_id) if family == 'dataset_release' else identity(record_id) - else: - identity(record_id) - if (family, record_id) in seen: - fail('Duplicate data envelope.') - seen.add((family, record_id)) - if type(doc) is not dict or doc.get('format') != FAMILIES[family]: - fail('Unsupported data document format.', 'unsupported', 409) - owned.append(envelope) - if family == 'workbench_record': - if record_id in assets: - fail('An asset has more than one current domain family.') - assets[record_id] = doc - elif family == 'workbench_response': - responses[record_id] = doc - for envelope in records: - if envelope.get('family') == 'project_artifact': - declaration = envelope['document'] - closed(declaration, {'format', 'sha256', 'bytes'}, 'artifact declaration') - if declaration['format'] != 'project_artifact_v1' or declaration['sha256'] != envelope['id']: - fail('Artifact declaration identity differs.') - sha(declaration['sha256']) - if type(declaration['bytes']) is not int or not 0<=declaration['bytes']<=2**63-1:fail('Invalid artifact declared byte count.') - declarations[envelope['id']] = declaration - def binding(parent): - _record(parent, identity(parent.get('id'))) - retained = assets.get(parent['id']) - if retained is None: - fail('Captured prompt/source is missing from data closure.') - candidates = [retained['current'], *retained['history'], *(e['snapshot'] for e in retained.get('events',[]))] - if not any(state == parent for state in candidates): - fail('Captured prompt/source snapshot differs from its retained revision.') - for envelope in owned: - family, record_id, doc = envelope['family'], envelope['id'], envelope['document'] - if family == 'workbench_record': - closed(doc, {'format', 'origin', 'source', 'current', 'history', 'events', 'audit'}, 'data record document') - source, current, history = doc['source'], doc['current'], doc['history'] - if type(source) is not dict or source.get('kind') not in ('image', 'text', *ASSET_KINDS): - fail('Invalid data source kind.') - kind = source['kind'] - _record(current, record_id, source) - if current['kind'] != kind or type(doc['origin']) is not dict or 'rights_note_correction' in doc['origin']: - fail('Record source/origin differs.') - if kind == 'text': - closed(source, {'kind', 'original_text'}, 'text source') - text_value(source['original_text'], 'Original text', MAX_TEXT) - normalized = unicodedata.normalize('NFC', source['original_text'].replace('\r\n', '\n').replace('\r', '\n')) - if current['text'] != normalized or current['source_sha256'] != hashlib.sha256(source['original_text'].encode()).hexdigest(): - fail('Original text normalization/source hash differs.') - elif kind == 'image': - closed(source, {'kind', 'original', 'oriented', 'revisions', 'deleted'}, 'image source') - original = _edge(source['original'], declarations, kind, 'original') - oriented = _edge(source['oriented'], declarations, kind, 'oriented') - if type(source['deleted']) is not bool or type(source['revisions']) is not list or not source['revisions']: - fail('Invalid image source lifecycle.') - previous = 0 - for row in source['revisions']: - _source_row(row, record_id) - if row['revision'] != previous + 1 or row['sha256'] != original['sha256'] or any(row[k] != source['revisions'][0][k] for k in ('width','height','created_at')): - fail('Image source revisions or original hash differ.') - previous = row['revision'] - if current['content_hash'] != oriented['sha256']: - fail('Oriented image identity differs.') - else: - closed(source, {'kind', 'asset', 'metadata'}, 'immutable source') - declaration = _edge(source['asset'], declarations, kind, 'original') - if current['content_hash'] != declaration['sha256'] or current[kind] != source['metadata']: - fail('Immutable source metadata or content identity differs.') - if type(source['metadata']) is not dict: - fail('Immutable metadata must be an object.') - _source_metadata(kind,source['metadata']) - expected_bytes=source['metadata'].get('bundle_bytes',source['metadata'].get('source_bytes')) - if expected_bytes!=declaration['bytes'] or not set(source['metadata']['protected_groups'])<=set(current['groups']): - fail('Immutable byte count or protected groups differ.') - _origin(doc['origin'],current,source,records) - if type(history) is not list or not history: - fail('Workbench annotation history is missing.') - for revision, state in enumerate(history, 1): - _record(state, record_id, source) - if state['revision'] != revision or state['kind'] != kind or state['content_hash'] != current['content_hash']: - fail('Workbench immutable identity/history revisions differ.') - if state['created_at']!=current['created_at'] or state['parents']!=current['parents'] or state['pixel_hash']!=current['pixel_hash']: - fail('Workbench history rewrites immutable source lineage.') - if revision>1 and date(state['updated_at'])<date(history[revision-2]['updated_at']): - fail('Workbench annotation history reverses time.') - origin = dict(state['provenance']); origin.pop('rights_note_correction', None) - _origin(origin,state,source,records) - if 'target_proposal' in state: - from project_producers import validate_target_proposal - validate_target_proposal(state['target_proposal'], state, records) - if current['revision'] != len(history): - fail('Current annotation revision differs from retained history.') - annotation_fields={'id','kind','text','content_hash','pixel_hash','groups','parents','provenance','task','annotation','review','revision','created_at','updated_at'} - if any(current[k]!=history[-1][k] for k in annotation_fields): - fail('Current annotation differs from its final retained revision.') - _events(doc['events']); _audit(doc['audit']) - current_origin=dict(current['provenance']);current_origin.pop('rights_note_correction',None) - if current_origin!=doc['origin']:fail('Current raw origin differs from Workbench provenance.') - for event in doc['events']: - state=event['snapshot'] - _record(state, record_id, source) - if state['id']!=record_id or state['kind']!=kind or state['content_hash']!=current['content_hash']: - fail('Lifecycle event changes immutable asset identity.') - if kind=='image': - by_revision={r['revision']:r for r in source['revisions']} - if source['deleted']!=(current['source_revision'] is None):fail('Image tombstone/current source differs.') - for state in [current,*history,*(e['snapshot'] for e in doc['events'])]: - if state['source_revision'] is None:continue - sample=by_revision.get(state['source_revision']) - expected={'source_sha256':'sha256','book_id':'book_id','session_id':'session_id','source_split':'split','width':'width','height':'height','corner_annotation':'annotation'} - if sample is None or any(state[key]!=sample[value] for key,value in expected.items()):fail('Workbench image snapshot differs from its source revision.') - else: - for state in [current,*history,*(e['snapshot'] for e in doc['events'])]: - if type(state['source_revision']) is not int or state['source_revision']!=1 or state['source_split']!='unassigned' or state['source_sha256']!=current['source_sha256']: - fail('Immutable source revision/hash differs across history.') - elif family == 'workbench_response': - closed(doc, {'format', 'current', 'history', 'audit'}, 'response document') - _response(doc['current'], record_id) - if type(doc['history']) is not list or not doc['history']: - fail('Response history is missing.') - for revision, state in enumerate(doc['history'], 1): - closed(state, {'response', 'parent'}, 'response capture') - _response(state['response'], record_id); binding(state['parent']) - if state['response']['revision'] != revision or state['response']['prompt_id'] != state['parent']['id'] or state['parent']['kind'] != 'text': - fail('Response prompt/revision binding differs.') - if state['response']['provenance']['prompt_sha256']!=state['parent']['content_hash'] or state['response']['provenance']['source_sha256']!=state['parent']['source_sha256']: - fail('Response source hash differs from its captured prompt.') - if state['response']['created_at']!=doc['current']['created_at'] or revision>1 and date(state['response']['updated_at'])<date(doc['history'][revision-2]['response']['updated_at']): - fail('Response history changes creation or reverses time.') - if doc['current'] != doc['history'][-1]['response']: - fail('Current response differs from its last history.') - _audit(doc['audit']) - elif family == 'workbench_preference': - closed(doc, {'format', 'current', 'history', 'audit'}, 'preference document') - _judgment(doc['current'], record_id) - if type(doc['history']) is not list or not doc['history']: - fail('Preference history is missing.') - for revision, capture in enumerate(doc['history'], 1): - closed(capture, {'judgment', 'parent', 'left', 'right'}, 'preference capture') - judgment = capture['judgment']; _judgment(judgment, record_id); binding(capture['parent']) - if judgment['revision'] != revision or judgment['prompt_id'] != capture['parent']['id'] or judgment['parent_revision'] != capture['parent']['revision'] or judgment['source_revision'] != capture['parent']['source_revision']: - fail('Preference prompt binding differs.') - for side in ('left', 'right'): - answer = capture[side]; _response(answer, judgment[side + '_id']) - retained = responses.get(answer['id']) - if answer['revision'] != judgment[side + '_revision'] or answer['prompt_id'] != judgment['prompt_id'] or retained is None or not any(h['response'] == answer for h in retained['history']): - fail('Preference answer binding is missing or differs.') - if doc['current'] != doc['history'][-1]['judgment']: - fail('Current preference differs from its last history.') - _audit(doc['audit']) - elif family in ('saved_selection', 'saved_search'): - closed(doc, {'format', 'current', 'history', 'deleted', 'audit'}, 'saved intent document') - if type(doc['deleted']) is not bool or type(doc['history']) is not list or not doc['history']: - fail('Invalid saved intent lifecycle.') - for revision, state in enumerate(doc['history'], 1): - fields = {'id', 'name', 'revision', 'created_at', 'updated_at', 'mode', 'schema_version'} | ({'items', 'member_count'} if family == 'saved_selection' else {'criteria'}) - closed(state, fields, 'saved intent snapshot') - natural(state['revision']) - if state['id'] != record_id or state['revision'] != revision or type(state['schema_version']) is not int or state['schema_version'] != 1 or state['mode'] != ('fixed' if family == 'saved_selection' else 'dynamic'): - fail('Saved intent identity differs.') - text_value(state['name'], 'Saved intent name', 120); date(state['created_at']); date(state['updated_at']) - if family == 'saved_search': - if criteria(state['criteria']) != state['criteria']: - fail('Saved search criteria differ.') - else: - if type(state['items']) is not list or not 1 <= len(state['items']) <= 5000 or type(state['member_count']) is not int or state['member_count'] != len(state['items']): - fail('Saved membership count differs.') - member_ids = set() - for member in state['items']: - closed(member, {'id', 'name', 'kind', 'revision', 'source_revision', 'source_sha256', 'content_hash', 'pixel_hash'}, 'saved member') - if member['id'] in member_ids or member['id'] not in assets: - fail('Saved membership repeats or loses a source.') - member_ids.add(member['id']); natural(member['revision']); natural(member['source_revision']) - retained = assets[member['id']] - if not any(all(snapshot.get(key) == value for key, value in member.items()) for snapshot in [retained['current'], *retained['history'],*(e['snapshot'] for e in retained.get('events',[]))]): - fail('Saved member revision/source differs.') - if doc['current'] != doc['history'][-1]: - fail('Saved intent current value differs.') - _audit(doc['audit']) - elif family == 'dataset_release': - closed(doc, {'format', 'asset', 'manifest', 'release_format', 'audit'}, 'release document') - declaration = _edge(doc['asset'], declarations, 'release', 'frozen_archive') - if declaration['sha256'] != record_id or type(doc['manifest']) is not dict: - fail('Frozen release identity differs.') - manifest=doc['manifest'];Releases.validate_project_manifest(manifest) - if doc['release_format']!=manifest.get('format','canonical_v1'):fail('Release format differs from retained manifest.') - exported={'split','asset','asset_sha256','export_group','export_split','exported_pixel_sha256','class_index', - 'audio_target_schema_version','audio_coordinate_contract','target_label','pcm_sha256','audio_family_sha256', - 'sample_rate_hz','channels','channel','start_frame','end_frame'} - for state in manifest.get('records',[]): - fields=SNAPSHOT_FIELDS|({state['kind']} if state.get('kind') in ASSET_KINDS else set())|({'target_proposal'} if 'target_proposal' in state else set()) - if set(state)-fields-exported:fail('Unknown frozen release record fields.') - binding({key:state[key] for key in fields}) - if state.get('asset_sha256')!=state['content_hash']:fail('Frozen release source identity differs.') - for parent in manifest.get('prompts',[]):binding(parent) - for response in manifest.get('responses',[]): - _response(response,identity(response.get('id'))) - retained=responses.get(response['id']) - if retained is None or not any(h['response']==response for h in retained['history']):fail('Frozen release response binding is missing.') - for judgment in [*manifest.get('judgments',[]),*manifest.get('competing_reviewed_judgments',[])]: - _judgment(judgment,identity(judgment.get('id'))) - retained=next((r['document'] for r in owned if r['family']=='workbench_preference' and r['id']==judgment['id']),None) - if retained is None or not any(h['judgment']==judgment for h in retained['history']):fail('Frozen release judgment binding is missing.') - _audit(doc['audit']) - elif family == 'dataset_exposure': - closed(doc, {'format', 'id', 'record_id', 'source_hash', 'reason', 'run_id', 'release_id', 'created_at'}, 'exposure fact') - if doc['id'] != record_id or doc['record_id'] not in assets: - fail('Exposure source is missing.') - sha(doc['source_hash']); identity(doc['run_id']); sha(doc['release_id']) - if not any(r.get('family') == 'dataset_release' and r['id'] == doc['release_id'] for r in records): - fail('Exposure release is missing.') - text_value(doc['reason'], 'Exposure reason', 500); date(doc['created_at']) - # Parent references use the full retained data closure, including tombstones. - for record_id, doc in assets.items(): - for parent in doc['current']['parents']: - if parent not in assets: - fail('Workbench parent is missing from the project.') - pending, visited = list(doc['current']['parents']), {record_id} - while pending: - parent = pending.pop() - if parent == record_id: - fail('Workbench source lineage contains a cycle.') - if parent in visited: - continue - visited.add(parent); pending.extend(assets[parent]['current']['parents']) - for envelope in owned: - _audit_bindings(envelope['family'],envelope['id'],envelope['document']) - if any(r['family']=='project_fork' for r in records): - from project_data_fork import validate_records - validate_records(records) - return copy.deepcopy(owned) - - -def decode_data_records(records): - try: - return _decode_data_records(records) - except WorkbenchError: - raise - except (ValueError,KeyError,TypeError,AttributeError,OverflowError,RecursionError) as error: - raise WorkbenchError('Malformed retained project data.') from error - - -DATA_ACTIONS = { - 'workbench.import_asset', 'workbench.save', 'workbench.approve_review', 'workbench.verify', - 'workbench.correct_rights_note', 'workbench.preserve_deleted_source', 'workbench.save_response', 'workbench.approve_response', - 'preferences.save', 'preferences.delete', 'saved_selections.create', 'saved_selections.rename', 'saved_selections.delete', - 'saved_searches.create', 'saved_searches.rename', 'saved_searches.delete', 'immutable_assets.admit', - 'audio_assets.admit', 'scientific_fields.admit', 'dataset.add', 'dataset.add_generated', 'dataset.save', 'dataset.delete', 'dataset_releases.create', -} -RUNTIME_DATA_OPERATIONS = frozenset(('data.runtime_generated_image','data.runtime_grounded_text', - 'data.runtime_caption','data.runtime_text_classification')) -DATA_OPERATIONS = { - 'data.import_text', 'data.import_image', 'data.import_bundle', 'data.import_audio', 'data.import_scientific', - 'data.annotate', 'data.correct_rights_note', 'data.save_image_source', 'data.delete_image_source', - 'data.save_response', 'data.save_preference', 'data.delete_preference', - 'data.create_selection', 'data.rename_selection', 'data.delete_selection', - 'data.create_search', 'data.rename_search', 'data.delete_search', 'data.freeze_release', -} - - -def _validate_operation_result(operation, result, records=None): - """Closed result variants for the session's retained, inert retry receipts.""" - if operation not in DATA_OPERATIONS: - return False - if operation in ('data.import_text','data.import_image','data.import_bundle','data.import_audio','data.import_scientific','data.annotate'): - validate_record_snapshot(result) - elif operation == 'data.correct_rights_note': - closed(result, {'record','changed'}, 'rights result'); validate_record_snapshot(result['record']) - if type(result['changed']) is not bool:fail('Invalid rights change result.') - elif operation == 'data.save_image_source': - _source_row(result, identity(result.get('id'))) - elif operation == 'data.delete_image_source': - closed(result, {'deleted','record'}, 'source deletion result'); identity(result['deleted']) - validate_record_snapshot(result['record']) - if result['record']['id'] != result['deleted'] or result['record']['source_available']: - fail('Invalid source deletion result.') - elif operation == 'data.save_response': - closed(result, {'response','parent','changed'}, 'response result') - _response(result['response'], identity(result['response'].get('id'))); validate_record_snapshot(result['parent']) - if type(result['changed']) is not bool or result['response']['prompt_id'] != result['parent']['id']: - fail('Invalid response result binding.') - elif operation in ('data.save_preference','data.delete_preference'): - closed(result, {'judgment','changed'}, 'preference result') - _judgment(result['judgment'], identity(result['judgment'].get('id'))) - if type(result['changed']) is not bool:fail('Invalid preference change result.') - elif operation in ('data.delete_selection','data.delete_search'): - closed(result, {'deleted'}, 'intent deletion result'); identity(result['deleted']) - elif operation in ('data.create_selection','data.rename_selection','data.create_search','data.rename_search'): - family = 'saved_selection' if 'selection' in operation else 'saved_search' - doc = dict(format=FAMILIES[family], current=result, history=[result], deleted=False, audit=[]) - # Reuse the complete intent validator with retained revision history. - if records is not None: - actual = next((r for r in records if r['family']==family and r['id']==result.get('id')),None) - if actual is None or result not in actual['document']['history']:fail('Intent receipt result is not retained.') - else: - expected = {'id','name','revision','created_at','updated_at','mode','schema_version'} | ({'items','member_count'} if family=='saved_selection' else {'criteria'}) - closed(result, expected, 'intent result'); identity(result['id']); natural(result['revision']) - text_value(result['name'],'Saved intent name',120);date(result['created_at']);date(result['updated_at']) - if family=='saved_search':criteria(result['criteria']) - else: - required={'id','url','records','split_report'} - if type(result) is not dict: - fail('Invalid frozen release result.') - format_name=result.get('format','canonical_v1') - variants={'canonical_v1':required,'image_caption_v1':required|{'format','warnings'}, - 'image_classification_v1':required|{'format','class_vocabulary','warnings'}, - 'text_classification_v1':required|{'format','class_vocabulary','warnings'}, - 'audio_classification_v1':required|{'format','class_vocabulary','warnings'}, - 'text_corpus_v1':required|{'format','corpus_counts','artifact_bytes','warnings'}, - 'text_instruction_v1':required|{'format','example_count','unique_prompt_count'}, - 'text_preference_v1':required|{'format','example_count','unique_prompt_count'}} - if type(format_name) is not str or format_name not in variants or set(result)!=variants[format_name]: - fail('Invalid frozen release result variant.') - sha(result['id']);natural(result['records'],0) - if result['url']!='/api/workbench/releases/'+result['id']+'.zip' or type(result['split_report']) is not dict: - fail('Frozen release result identity differs.') - if records is not None: - by_key={(r['family'],r['id']):r['document'] for r in records} - def retained_record(row): - document=by_key.get(('workbench_record',row['id'])) or by_key.get(('workbench_text',row['id'])) - if document is None or not any(row==state for state in [document['current'],*document['history'],*(e['snapshot'] for e in document.get('events',[]))]): - fail('Operation result is not an exact retained record revision.') - return document - if operation in ('data.import_text','data.import_image','data.import_bundle','data.import_audio','data.import_scientific','data.annotate'): - retained_record(result) - if operation!='data.annotate' and result['revision']!=1: - fail('Import result must identify the initial retained revision.') - elif operation=='data.correct_rights_note': - document=retained_record(result['record']) - if result['changed'] and not any(e['operation']==operation and e['record_revision']==result['record']['revision'] for e in document.get('events',[])): - fail('Rights correction result lacks its retained lifecycle event.') - elif operation=='data.save_image_source': - document=by_key.get(('workbench_record',result['id'])) - if document is None or document['source']['kind']!='image' or result not in document['source']['revisions']: - fail('Source save result is not a retained source revision.') - elif operation=='data.delete_image_source': - document=retained_record(result['record']) - if not document['source'].get('deleted') or not any(e['operation']==operation and e['deleted'] for e in document['events']): - fail('Source deletion result has no retained tombstone.') - elif operation=='data.save_response': - document=by_key.get(('workbench_response',result['response']['id'])) - if document is None or not any(h=={'response':result['response'],'parent':result['parent']} for h in document['history']): - fail('Response operation result differs from its exact captured binding.') - elif operation in ('data.save_preference','data.delete_preference'): - document=by_key.get(('workbench_preference',result['judgment']['id'])) - if document is None or not any(h['judgment']==result['judgment'] for h in document['history']): - fail('Preference operation result is not retained.') - if operation=='data.delete_preference' and not result['judgment']['deleted']: - fail('Deleted preference result lacks its tombstone.') - elif operation in ('data.delete_selection','data.delete_search'): - family='saved_selection' if operation=='data.delete_selection' else 'saved_search' - document=by_key.get((family,result['deleted'])) - if document is None or not document['deleted']:fail('Intent deletion result lacks its tombstone.') - elif operation=='data.freeze_release': - document=by_key.get(('dataset_release',result['id'])) - if document is None or document['manifest']['split_report']!=result['split_report']: - fail('Release result is not bound to its retained frozen manifest.') - manifest=document['manifest'] - if manifest.get('format','canonical_v1')!=result.get('format','canonical_v1'):fail('Frozen result format differs.') - if 'class_vocabulary' in result and result['class_vocabulary']!=manifest['class_vocabulary']:fail('Frozen class vocabulary result differs.') - count=len(manifest['records']) if 'records' in manifest else sum(j['outcome'] in ('left','right') for j in manifest['judgments']) if 'judgments' in manifest else len(manifest['responses']) - if result['records']!=count:fail('Frozen release result count differs from manifest.') - return True - - -def validate_operation_result(operation,result,records=None): - try: - return _validate_operation_result(operation,result,records) - except WorkbenchError: - raise - except (ValueError,KeyError,TypeError,AttributeError,OverflowError,RecursionError) as error: - raise WorkbenchError('Malformed retained data operation result.') from error - - -class _DataAccess: - def __init__(self, session): - self.session, self.host, self.facts = session, session.access, None - def _selected(self): - from access import current_context, AccessError - context=current_context() - if context is not None and context.mode=='job': - if context.catalog_id!=self.session.db.dataset_id: - raise AccessError('Current job authority belongs to another project.') - runtime=getattr(self.session,'runtime_access',None) - if runtime is None or runtime.catalog_id!=self.session.db.dataset_id: - raise AccessError('Current local job authority is unavailable.') - return runtime - return self.host - def _context(self): - # Merely observing a context never authenticates it. A local job must - # still prove its exact local admission before returning any identity. - from access import current_context - authority=self._selected() - if authority is self.host: - authority.identity() - return current_context() - context=current_context() - return authority.require_job(context.kind,context.job_id,context.admission_id,context.request_digest) - def require(self, *args, **kwargs): - if args and args[0]!='inspect': - window=getattr(self.session,'_data_runtime_window',None) - if self.facts is None or window and (not self.session.db.in_transaction or window['records'] is not None): - fail('Project data writes require an uncaptured canonical transaction.', 'unavailable', 409) - return self._selected().require(*args, **kwargs) - def require_job(self, *args, **kwargs): - return self._selected().require_job(*args, **kwargs) - def identity(self): - self._context() - return self._selected().identity() - def denial(self, *args, **kwargs): - return self._selected().denial(*args, **kwargs) - def audit(self, action, target=None, details=None, result='success'): - window=getattr(self.session,'_data_runtime_window',None) - if self.facts is None or window and window['records'] is not None or action not in DATA_ACTIONS or result != 'success': - fail('Data effect is outside its canonical mutation.', 'unavailable', 409) - scope='freeze' if action=='dataset_releases.create' else 'approve-review' if action=='dataset.save' else 'edit' - actor = self._selected().require(scope, action, target) - self.facts.append(dict(action=action, target=target, details=copy.deepcopy(details or {}), result=result, - actor_id=actor.actor_id, occurred_at=timestamp())) - - -class ProjectData: - def __init__(self, session, image_sources=None, generation_entries=None): - self.session, self.db, self.access, self.artifacts = session, session.db, session.access, session.artifacts - self._authority = _DataAccess(session) - self.context = SimpleNamespace(db=self.db, lock=self.db.lock, access=self._authority, path=self.db.path, - project_sources={}, project_asset=self._asset, project_availability=self._available) - self.workbench = Workbench(self.context) - self.context.workbench = self.workbench - if generation_entries is not None: - self.context.generation_jobs = generation_entries - if image_sources is None: - from image_sources import ImageSources - image_sources = ImageSources(self.context) - self.images = image_sources - self.context.add = self.images.add - self.context.sample, self.context.rows = self.images.sample, self.images.rows - self.selections, self.searches = SavedSelections(self.workbench), SavedSearches(self.workbench) - self.releases = Releases(self.workbench) - self._declarations = {} - self._runtime_window = None - self._sync_sources() - - @contextmanager - def runtime_mutation(self, operation, key): - """Open one trusted owner window inside an already owned SQL TX. - - The execution hook captures and publishes before outer COMMIT. This - method neither owns that transaction nor calls Session.mutate. - """ - if operation not in RUNTIME_DATA_OPERATIONS or type(key) is not str or not 1<=len(key)<=128: - fail('Unsupported runtime data mutation.') - if not self.db.in_transaction or getattr(self.session,'runtime_access',None) is None: - fail('Runtime data capture requires current execution and its outer transaction.','unavailable',409) - if self._runtime_window is not None or self._authority.facts is not None: - fail('A data mutation window is already active.','conflict',409) - from access import current_context - context=current_context() - if context is not None and context.mode=='job': - self._authority._context() - if operation!='data.runtime_generated_image' or context.kind!='synthetic': - fail('This admitted job cannot own the requested data mutation.','unavailable',409) - self._authority._selected().require('edit','dataset.add_generated',context.job_id) - else: - self.access.require('edit',operation) - window=dict(operation=operation,key=key,before=None,records=None) - self._runtime_window=window - self.session._data_runtime_window=window - self._authority.facts=[];self._pending_artifacts={};self._request_body={} - def finish(): - if self._runtime_window is window:self.finish_runtime_mutation() - self.db.on_commit(finish);self.db.on_rollback(finish) - try: - yield self - except BaseException: - finish() - raise - - def finish_runtime_mutation(self): - """Execution outcome owner also calls this after unknown SQL outcome.""" - self._runtime_window=None - self.session._data_runtime_window=False - self._authority.facts=None - self._pending_artifacts={} - self._sync_sources() - - def capture_runtime_changes(self, before_records, *, operation, key): - window=self._runtime_window - if (not self.db.in_transaction or window is None or operation!=window['operation'] or key!=window['key']): - fail('Runtime data capture is outside its exact owned transaction.','unavailable',409) - if window['records'] is not None: - if before_records!=window['before']:fail('Runtime data capture checkpoint changed.','conflict',409) - return copy.deepcopy(window['records']) - if operation=='data.runtime_generated_image': - for fact in self._authority.facts: - if fact['action']=='dataset.add_generated': - self.workbench._sync_images(identity(fact['details'].get('sample_id'))) - captured=self._capture(None,before_records,operation,key)['records'] - window['before']=copy.deepcopy(before_records);window['records']=copy.deepcopy(captured) - return captured - - def _sync_sources(self): - # Only the selected families are copied; fork history is still checked. - active = session_active(self.session, 'workbench_record') - self.context.project_sources = {r['id']: r['document']['source'] for r in active if r['family'] == 'workbench_record'} - self._declarations = {r['id']: r['document'] for r in session_records(self.session, 'project_artifact')} - - def _available(self, record_id): - source = self.context.project_sources.get(record_id) - if source is None or source['kind'] == 'text': - return True - if source['kind'] == 'image': - if source['deleted']: - return False - edges = (source['original'], source['oriented']) - else: - edges = (source['asset'],) - # Browsing metadata checks presence without repeatedly hashing every - # large payload. Every actual consumer obtains a verified snapshot. - return all(self.artifacts.inspect(self._declarations[e['artifact']['sha256']])['state'] != 'missing' for e in edges) - - def _asset(self, record_id, kind): - source = self.context.project_sources[record_id] - edge = source['oriented'] if kind == 'image' else source['asset'] - declaration = self._declarations[edge['artifact']['sha256']] - return self.artifacts.read_blob({k:declaration[k] for k in ('sha256','bytes')}, maximum_bytes=40 * 1024 * 1024) - - def _publish(self, raw, kind, role, media_type): - declaration = dict(format='project_artifact_v1', sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) - self.artifacts.publish_bytes(raw, declaration) - self._pending_artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=self.session.summary['project_id'], sha256=declaration['sha256']), - media_type=media_type, kind=kind, role=role, preservation='unique', required_for=['inspect', 'release']) - - def _require_record(self, record_id, *, extended=False): - # Only these families are inspected; fork history is still checked. - for item in session_active(self.session, 'workbench_record', 'workbench_text'): - if item['id'] == record_id and item['family'] in ('workbench_record', 'workbench_text'): - if extended and item['family'] == 'workbench_text': - fail('This operation requires an explicitly converted data record.', 'unsupported', 409) - return item - fail('Project data record is unavailable.', 'unavailable', 404) - - def _permission(self, operation, target, body): - scope='freeze' if operation=='data.freeze_release' else 'approve-review' if operation=='data.save_image_source' else 'edit' - self.access.require(scope, operation, target) - if operation in ('data.annotate', 'data.save_response', 'data.save_preference') and body.get('review') == 'human_reviewed': - self.access.require('approve-review', operation, target, owner_only=True) - if operation == 'data.annotate' and body.get('review') == 'programmatically_verified': - fail('Programmatic verification requires its owned verifier.', 'unsupported', 409) - - def _mutate(self, operation, target, body, expected_checkpoint, key, invoke): - with self.access.lock: - if type(body) is not dict: - fail('Data mutation body must be an object.') - self._permission(operation, target, body) - with self.session.lock, self.db.lock: - self.session.check_current() - if operation in ('data.annotate','data.correct_rights_note','data.save_image_source','data.delete_image_source'): - self._require_record(target,extended=operation!='data.annotate') - legacy = self._legacy_replay(operation, target, body, key) - if legacy is not None: - return legacy - self._pending_artifacts, self._authority.facts = {}, [] - self._request_body = copy.deepcopy(body) - try: - scope='freeze' if operation=='data.freeze_release' else 'approve-review' if operation=='data.save_image_source' else 'edit' - result = self.session.mutate(operation, target, body, expected_checkpoint, key, invoke, - lambda result, before: self._capture(result, before, operation, key),scope=scope) - self._sync_sources() - return result - finally: - self._authority.facts = None - - def _legacy_replay(self, operation, target, body, key): - """Retain M1 request meaning after the shared M2 session cutover.""" - # Only receipts and legacy text are inspected below. - records = session_active(self.session, 'project_operation', 'workbench_text') - if any(r['family']=='project_operation' and r['document']['request_key']==key for r in records): - return None - if operation not in ('data.import_text','data.annotate'): - return None - action = 'import_text' if operation=='data.import_text' else 'annotate' - request_hash = hashlib.sha256(encode(dict(action=action,target=target,body=body)).encode()).hexdigest() - for envelope in records: - if envelope['family']!='workbench_text':continue - doc=envelope['document'] - for receipt in doc['operations']: - if receipt['request_key']!=key:continue - if receipt['request_sha256']!=request_hash: - fail('Request key already has different M1 input.','conflict',409) - if self.session.store.reconcile(key,request_hash) is None: - fail('M1 text operation has no canonical publication receipt.') - return copy.deepcopy(doc['history'][receipt['revision']-1]) - return None - - def import_text(self, body, expected_checkpoint, key): - if type(body) is not dict or set(body) - {'kind','text', 'name', 'groups', 'parents', 'rights'} or body.get('kind','text')!='text': - fail('Import exactly text and its source metadata.') - return self._mutate('data.import_text', None, body, expected_checkpoint, key, - lambda: self.workbench.import_asset(dict(body, kind='text'))) - - def import_image(self, body, expected_checkpoint, key): - if type(body) is not dict or body.get('kind', 'image') != 'image': - fail('Expected an image import.') - return self._mutate('data.import_image', None, body, expected_checkpoint, key, - lambda: self.workbench.import_asset(dict(body, kind='image'))) - - def import_bundle(self, kind, prepared, body, expected_checkpoint, key): - if kind not in ('sequence', 'mesh', 'pointcloud', 'volume'): - fail('Unsupported immutable bundle kind.', 'unsupported', 409) - # prepared belongs to the selected existing parser, not a public RPC. - request = dict(kind=kind, body=body, bundle_sha256=hashlib.sha256(prepared['bundle']).hexdigest()) - def invoke(): - owner=self.workbench.immutable_owners[kind] - owner.validate_project_source(prepared['bundle'],dict(prepared['metadata'],bundle_bytes=len(prepared['bundle']))) - return owner.admit(prepared,body) - return self._mutate('data.import_bundle', None, request, expected_checkpoint, key,invoke) - - def import_audio(self, body, expected_checkpoint, key): - return self._mutate('data.import_audio', None, body, expected_checkpoint, key, lambda: self.workbench.audio.admit(body)) - - def import_scientific(self, raw, body, source, expected_checkpoint, key): - request = dict(body=body, source=source, sha256=hashlib.sha256(raw).hexdigest()) - return self._mutate('data.import_scientific', None, request, expected_checkpoint, key, - lambda: self.workbench.scientific.admit(raw, body, source)) - - def annotate(self, record_id, body, expected_checkpoint, key): - return self._mutate('data.annotate', record_id, body, expected_checkpoint, key, lambda: self.workbench.save(record_id, body)) - - def correct_rights_note(self, record_id, body, expected_checkpoint, key): - return self._mutate('data.correct_rights_note', record_id, body, expected_checkpoint, key, - lambda: self.workbench.correct_rights_note(record_id, body)) - - def save_image_source(self, record_id, body, expected_checkpoint, key): - def invoke(): - if not self._available(record_id): - fail('Image source bytes are unavailable.', 'unavailable', 409) - return self.images.save(record_id, body) - return self._mutate('data.save_image_source', record_id, body, expected_checkpoint, key, invoke) - - def delete_image_source(self, record_id, body, expected_checkpoint, key): - def invoke(): - if not hasattr(self.context,'generation_jobs'): - fail('Image deletion requires its inert producer entry owner.','unavailable',409) - # Verify/preserve the exact source before the image owner can remove - # its disposable projection files. Canonical artifacts survive it. - source = self.context.project_sources[record_id] - for edge in (source['original'], source['oriented']): - declaration = self._declarations[edge['artifact']['sha256']] - self.artifacts.read_blob({k:declaration[k] for k in ('sha256','bytes')}, maximum_bytes=40 * 1024 * 1024) - result = self.images.delete(record_id, body) - return {'deleted': record_id, 'record': self.workbench._get(record_id)} - return self._mutate('data.delete_image_source', record_id, body, expected_checkpoint, key, invoke) - - def save_response(self, body, expected_checkpoint, key): - return self._mutate('data.save_response', body.get('id') if isinstance(body,dict) else None, body, expected_checkpoint, key, lambda: self.workbench.save_response(body)) - - def save_preference(self, body, expected_checkpoint, key): - return self._mutate('data.save_preference', body.get('id') if isinstance(body,dict) else None, body, expected_checkpoint, key, lambda: self.workbench.preferences.save(body)) - - def delete_preference(self, body, expected_checkpoint, key): - return self._mutate('data.delete_preference', body.get('id') if isinstance(body,dict) else None, body, expected_checkpoint, key, lambda: self.workbench.preferences.delete(body)) - - def create_selection(self, body, expected_checkpoint, key): - return self._mutate('data.create_selection', None, body, expected_checkpoint, key, lambda: self.selections.create(body)) - def rename_selection(self, record_id, body, expected_checkpoint, key): - return self._mutate('data.rename_selection', record_id, body, expected_checkpoint, key, lambda: self.selections.mutate(record_id, 'rename', body)) - def delete_selection(self, record_id, body, expected_checkpoint, key): - return self._mutate('data.delete_selection', record_id, body, expected_checkpoint, key, lambda: self.selections.mutate(record_id, 'delete', body)) - def create_search(self, body, expected_checkpoint, key): - return self._mutate('data.create_search', None, body, expected_checkpoint, key, lambda: self.searches.create(body)) - def rename_search(self, record_id, body, expected_checkpoint, key): - return self._mutate('data.rename_search', record_id, body, expected_checkpoint, key, lambda: self.searches.mutate(record_id, 'rename', body)) - def delete_search(self, record_id, body, expected_checkpoint, key): - return self._mutate('data.delete_search', record_id, body, expected_checkpoint, key, lambda: self.searches.mutate(record_id, 'delete', body)) - - def preview_release(self, body): - return self._read('dataset_releases.preview', lambda: self.releases.preview(body)) - - def freeze_release(self, body, expected_checkpoint, key): - return self._mutate('data.freeze_release', None, body, expected_checkpoint, key, lambda: self.releases._create(body)) - - def _read(self, action, invoke, target=None): - with self.access.lock, self.session.lock, self.db.lock: - self.access.require('inspect', action, target) - self.session.check_current() - self._sync_sources() - result = invoke() - self.session.check_current() - return result - - def query(self, options=None): - return self._read('workbench.query', lambda: self.workbench.query(options or {})) - def get(self, record_id): - return self._read('workbench.get', lambda: self.workbench.get(record_id), record_id) - def scientific(self, record_id, content_hash, **selection): - sha(content_hash) - def inspect(): - record = self.workbench.get(record_id) - if record['content_hash'] != content_hash: - fail('Scientific source differs from the selected content hash.', 'conflict', 409) - return self.workbench.scientific.inspect(record_id, **selection) - return self._read('scientific_fields.inspect', inspect, record_id) - def history(self, record_id): - return self._read('workbench.history', lambda: self.workbench.history(record_id), record_id) - def responses(self, record_id): - return self._read('workbench.responses', lambda: self.workbench.responses(record_id), record_id) - def response_history(self, record_id): - return self._read('workbench.response_history', lambda: self.workbench.response_history(record_id), record_id) - def preferences(self, prompt_id): - return self._read('preferences.list', lambda: self.workbench.preferences.list(prompt_id), prompt_id) - def preference_history(self, record_id): - return self._read('preferences.history', lambda: self.workbench.preferences.history(record_id), record_id) - def load_selection(self, record_id): - return self._read('saved_selections.load', lambda: self.selections.load(record_id), record_id) - def list_selections(self): - return self._read('saved_selections.list', self.selections.list) - def list_searches(self): - return self._read('saved_searches.list', self.searches.list) - - def availability(self, record_id): - def inspect(): - envelope=self._require_record(record_id) - if envelope['family']=='workbench_text': - return dict(record_id=record_id,kind='text',deleted=False,artifacts=[]) - source=envelope['document']['source'];kind=source['kind'] - edges=[] if kind=='text' else [source['original'],source['oriented']] if kind=='image' else [source['asset']] - return dict(record_id=record_id,kind=kind,deleted=source.get('deleted',False),artifacts=[ - dict(role=edge['role'],reference=edge['artifact'],observation=self.artifacts.status(self._declarations[edge['artifact']['sha256']])) for edge in edges]) - return self._read('workbench.get',inspect,record_id) - - def _capture_source(self, row, prior): - kind, record_id = row['kind'], row['id'] - if kind == 'text': - original = self.db.execute('SELECT original_text FROM workbench_records WHERE id=%s', (record_id,)).fetchone()[0] - return dict(kind=kind, original_text=original) - if kind == 'image': - source = copy.deepcopy(prior['source']) if prior else None - sample = self.db.execute('SELECT * FROM samples WHERE id=%s', (record_id,)).fetchone() - if sample is None: - if source is None: - fail('Deleted image lost its preserved source.', 'invalid', 409) - source['deleted'] = True - return source - sample = dict(sample) - sample['annotation'] = json.loads(sample['annotation']) if sample['annotation'] is not None else None - if source is None: - folder = self.db.path / 'images' / record_id - source = dict(kind='image', - original=self._publish((folder / 'source').read_bytes(), kind, 'original', 'application/octet-stream'), - oriented=self._publish((folder / 'image.png').read_bytes(), kind, 'oriented', 'image/png'), - revisions=[], deleted=False) - if not source['revisions'] or source['revisions'][-1] != sample: - source['revisions'].append(sample) - return source - if prior: - return copy.deepcopy(prior['source']) - if kind in ('sequence', 'mesh', 'pointcloud', 'volume'): - raw = bytes(self.db.execute('SELECT bundle FROM workbench_' + kind + '_assets WHERE id=%s', (record_id,)).fetchone()[0]) - media = 'application/zip' - elif kind == 'audio': - raw = (self.db.path / 'audio' / record_id / 'source.wav').read_bytes(); media = 'audio/wav' - else: - raw = self.workbench.scientific._capture(record_id); media = 'application/zip' - return dict(kind=kind, asset=self._publish(raw, kind, 'original', media), metadata=copy.deepcopy(row[kind])) - - def _capture(self, result, before_records, operation, key): - before = {(r['family'], r['id']): r for r in before_records} - changed = [] - for identity_row in self.db.execute('SELECT id FROM workbench_records ORDER BY id'): - record_id = identity_row[0] - legacy = before.get(('workbench_text', record_id)) - if legacy: - # M1 retains its unchanged decoder and annotation semantics. - row = self.workbench._get(record_id) - prior = copy.deepcopy(legacy) - if row != prior['document']['current']: - if operation != 'data.annotate': - fail('M1 text lifecycle requires explicit conversion.', 'unsupported', 409) - doc = prior['document']; doc['current'] = row; doc['history'].append(row) - facts = [f for f in self._authority.facts if f['target'] == record_id] - doc['audit'].extend(facts) - request = dict(action='annotate', target=record_id, body=self._request_body) - doc['operations'].append(dict(request_key=key, request_sha256=digest(request), action='annotate', target=record_id, revision=row['revision'])) - changed.append(prior) - continue - old = before.get(('workbench_record', record_id)) - old_doc = old['document'] if old else None - row = self.workbench._get(record_id) - if old_doc is not None and all(row.get(k)==v for k,v in old_doc['current'].items() if k!='source_available'): - # Physical payload availability is current local observation, - # never an authored change to a retained checkpoint snapshot. - row = copy.deepcopy(old_doc['current']) - raw = self.db.execute('SELECT provenance_json FROM workbench_records WHERE id=%s', (record_id,)).fetchone()[0] - history = [json.loads(r[0]) for r in self.db.execute('SELECT snapshot FROM workbench_history WHERE id=%s ORDER BY revision', (record_id,))] - source = self._capture_source(row, old_doc) - doc = dict(format='workbench_record_v1', origin=json.loads(raw), source=source, current=row, history=history, - events=copy.deepcopy(old_doc['events']) if old_doc else [], audit=copy.deepcopy(old_doc['audit']) if old_doc else []) - if old_doc is None or any(doc[field] != old_doc[field] for field in ('origin', 'source', 'current', 'history')): - doc['events'].append(dict(operation=operation, request_key=key, record_revision=row['revision'], source_revision=row['source_revision'], - deleted=source.get('deleted', False), occurred_at=timestamp(),snapshot=copy.deepcopy(row))) - doc['audit'].extend(f for f in self._authority.facts if f['target'] == record_id or f['details'].get('sample_id') == record_id) - changed.append(dict(family='workbench_record', id=record_id, document=doc)) - for family, table, history_table, key_name in ( - ('workbench_response', 'workbench_responses', 'workbench_response_history', 'response'), - ('workbench_preference', 'workbench_preferences', 'workbench_preference_history', 'judgment')): - for row in self.db.execute('SELECT id FROM ' + table + ' ORDER BY id'): - record_id = row[0] - current = self.workbench._response(record_id) if family == 'workbench_response' else self.workbench.preferences._get(record_id) - history = [json.loads(r[0]) for r in self.db.execute('SELECT snapshot FROM ' + history_table + ' WHERE id=%s ORDER BY revision', (record_id,))] - old = before.get((family, record_id)) - if old is None or old['document']['current'] != current or old['document']['history'] != history: - audit = copy.deepcopy(old['document']['audit']) if old else [] - audit.extend(f for f in self._authority.facts if f['target'] == record_id) - changed.append(dict(family=family, id=record_id, document=dict(format=FAMILIES[family], current=current, history=history, audit=audit))) - for family, table, owner in (('saved_selection', 'saved_selections', self.selections), ('saved_search', 'saved_searches', self.searches)): - live = {r[0] for r in self.db.execute('SELECT id FROM ' + table)} - from project_fork import active_records - all_ids = live | {r['id'] for r in active_records(before_records, project_id=self.session.project_id, - families=frozenset({family})) if r['family'] == family} - for record_id in all_ids: - old = before.get((family, record_id)) - current = owner._get(record_id) if record_id in live else copy.deepcopy(old['document']['current']) - deleted = record_id not in live - if old is None or old['document']['current'] != current or old['document']['deleted'] != deleted: - history = copy.deepcopy(old['document']['history']) if old else [] - if not history or history[-1] != current: - history.append(current) - audit = copy.deepcopy(old['document']['audit']) if old else [] - audit.extend(f for f in self._authority.facts if f['target'] == record_id) - changed.append(dict(family=family, id=record_id, document=dict(format=FAMILIES[family], current=current, history=history, deleted=deleted, audit=audit))) - if operation == 'data.freeze_release': - release_id = result['id'] - raw, _ = self.releases.capture_input(release_id, release_id, 40 * 1024 * 1024) - manifest = self.releases.project_manifest(raw) - edge = self._publish(raw, 'release', 'frozen_archive', 'application/zip') - old = before.get(('dataset_release', release_id)) - if old is None: - changed.append(dict(family='dataset_release', id=release_id, document=dict(format='dataset_release_v1', asset=edge, - manifest=manifest, release_format=self._request_body.get('format', 'canonical_v1'), audit=copy.deepcopy(self._authority.facts)))) - for row in self.db.execute('SELECT * FROM workspace_exposures'): - record = dict(row) - if ('dataset_exposure', record['id']) not in before: - changed.append(dict(family='dataset_exposure', id=record['id'], document=dict(format='dataset_exposure_v1', **record))) - for artifact_id, declaration in self._pending_artifacts.items(): - changed.append(dict(family='project_artifact', id=artifact_id, document=declaration)) - if operation=='data.delete_image_source': - generation=getattr(self.context,'generation_jobs',None) - if generation is not None: - changed.extend(generation.capture_records()) - unique={} - for envelope in changed: - identity=(envelope['family'],envelope['id']) - if identity in unique and unique[identity]!=envelope: - fail('Two data owners captured conflicting canonical facts.') - unique[identity]=envelope - # The session appends one operation receipt then validates the complete - # merged family closure; local upserts alone are not an inventory. - return {'records': list(unique.values()), 'result': copy.deepcopy(result)} - - def restore(self, snapshot=None): - if snapshot is not None and snapshot != self.session.snapshot: - fail('Data reconstruction snapshot is stale.', 'stale_project', 409) - # The decoder reads its own families and project_fork overlays, and its - # fork validation needs every record identity; unrelated documents are - # shared read-only instead of copying unrelated Run history. - families = (*FAMILIES, 'workbench_text', 'project_artifact', 'project_fork') - owned = decode_data_records(session_records_for(self.session, *families)) - self._sync_sources() - # Validate present source bytes through the existing semantic owners - # before touching any selected SQL row. Missing bytes retain metadata. - self._verify_sources(owned) - active = session_active(self.session, *families) - identities = {(r['family'], r['id']) for r in active} - owned = [r for r in owned if (r['family'], r['id']) in identities] - with self.db.lock, self.db.transaction(): - for table in ('workbench_history', 'workbench_rights_notes', 'workbench_target_proposals', 'workbench_responses', 'workbench_response_history', - 'workbench_preferences', 'workbench_preference_history', 'saved_selections', 'saved_searches', 'workbench_deleted_sources', - 'workbench_sequence_assets', 'workbench_mesh_assets', 'workbench_pointcloud_assets', 'workbench_volume_assets', 'samples', 'workbench_records', 'workspace_exposures'): - self.db.execute('DELETE FROM ' + table) - asset_docs = [r for r in owned if r['family'] == 'workbench_record'] - for legacy in (r for r in active if r['family'] == 'workbench_text'): - doc = legacy['document'] - asset_docs.append(dict(family='workbench_record', id=legacy['id'], document=dict(current=doc['current'], history=doc['history'], - origin=doc['current']['provenance'], source=dict(kind='text', original_text=doc['original_text'])))) - for envelope in asset_docs: - doc = envelope['document']; row, source = doc['current'], doc['source'] - if source['kind'] == 'image': - if not source['deleted']: - sample = source['revisions'][-1] - self.db.execute('INSERT INTO samples (id,sha256,filename,width,height,book_id,session_id,split,created_at,updated_at,revision,annotation) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', - tuple(encode_retained(sample[k]) if k == 'annotation' and sample[k] is not None else sample[k] for k in ('id','sha256','filename','width','height','book_id','session_id','split','created_at','updated_at','revision','annotation'))) - else: - sample = source['revisions'][-1] - self.db.execute('INSERT INTO workbench_deleted_sources VALUES (%s,%s,%s,%s)', (row['id'],sample['book_id'],sample['session_id'],sample['split'])) - self.db.execute('''INSERT INTO workbench_records - (id,kind,name,text,original_text,content_hash,pixel_hash,groups_json,parents_json,provenance_json,task,annotation_json,review,revision,created_at,updated_at) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)''', - (row['id'],row['kind'],row['name'],row['text'],source.get('original_text'),row['content_hash'],row['pixel_hash'], - encode_retained(row['groups']),encode_retained(row['parents']),encode_retained(doc['origin']),row['task'], - encode_retained(row['annotation']) if row['annotation'] is not None else None,row['review'],row['revision'],row['created_at'],row['updated_at'])) - correction = row['provenance'].get('rights_note_correction') - if correction: - self.db.execute('INSERT INTO workbench_rights_notes VALUES (%s,%s,%s)', (row['id'],correction['note'],correction['revision'])) - if 'target_proposal' in row: - self.db.execute('INSERT INTO workbench_target_proposals VALUES (%s,%s)', (row['id'],encode_retained(row['target_proposal']))) - for state in doc['history']: - self.workbench._history(state) - for envelope in owned: - family, doc, record_id = envelope['family'], envelope['document'], envelope['id'] - if family == 'workbench_response': - r=doc['current'] - self.db.execute('INSERT INTO workbench_responses VALUES (%s,%s,%s,%s,%s,%s,%s,%s)', - (record_id,r['prompt_id'],r['revision'],r['completion'],r['review'],encode_retained(r['provenance']),r['created_at'],r['updated_at'])) - for revision, state in enumerate(doc['history'],1): - self.db.execute('INSERT INTO workbench_response_history VALUES (%s,%s,%s)', (record_id,revision,encode_retained(state))) - elif family == 'workbench_preference': - r=doc['current'] - self.db.execute('INSERT INTO workbench_preferences VALUES (%s,%s,%s,%s)', (record_id,r['prompt_id'],r['revision'],encode_retained(r))) - for revision,state in enumerate(doc['history'],1): - self.db.execute('INSERT INTO workbench_preference_history VALUES (%s,%s,%s)', (record_id,revision,encode_retained(state))) - elif family == 'saved_selection' and not doc['deleted']: - r=doc['current'] - self.db.execute('INSERT INTO saved_selections VALUES (%s,%s,%s,%s,%s,%s,%s)', (record_id,r['name'],r['revision'],encode_retained(r['items']),r['member_count'],r['created_at'],r['updated_at'])) - elif family == 'saved_search' and not doc['deleted']: - r=doc['current'] - self.db.execute('INSERT INTO saved_searches VALUES (%s,%s,%s,%s,%s,%s)', (record_id,r['name'],r['revision'],encode_retained(r['criteria']),r['created_at'],r['updated_at'])) - elif family == 'dataset_exposure': - self.db.execute('INSERT INTO workspace_exposures VALUES (%s,%s,%s,%s,%s,%s,%s)', tuple(doc[k] for k in ('id','record_id','source_hash','reason','run_id','release_id','created_at'))) - self.session.check_current() - self._materialize_images_and_releases(owned) - return snapshot - - def _verify_sources(self, owned): - for envelope in owned: - if envelope['family']=='dataset_release': - doc=envelope['document'];declaration=self._declarations[doc['asset']['artifact']['sha256']] - status=self.artifacts.status(declaration)['state'] - if status=='corrupt':fail('Frozen release artifact is corrupt.') - if status=='verified': - raw=self.artifacts.read_blob({k:declaration[k] for k in ('sha256','bytes')},maximum_bytes=40*1024*1024) - if self.releases.project_manifest(raw)!=doc['manifest']:fail('Frozen manifest differs from the exact archive bytes.') - continue - if envelope['family'] != 'workbench_record': - continue - doc = envelope['document']; source, row = doc['source'], doc['current']; kind=source['kind'] - if kind == 'text': - continue - edges=[source['original'],source['oriented']] if kind=='image' else [source['asset']] - states=[self.artifacts.status(self._declarations[e['artifact']['sha256']])['state'] for e in edges] - if 'corrupt' in states:fail('Stored data artifact is corrupt.') - if 'missing' in states:continue - if kind == 'image': - from image_sources import validate_project_source - payloads=[] - for edge in (source['original'],source['oriented']): - declaration=self._declarations[edge['artifact']['sha256']] - payloads.append(self.artifacts.read_blob({k:declaration[k] for k in ('sha256','bytes')},maximum_bytes=40*1024*1024)) - validate_project_source(*payloads,source['revisions'][-1],row['pixel_hash']) - else: - # Historical source verification uses its exact envelope; - # only current editable records populate the runtime resolver. - declaration=self._declarations[source['asset']['artifact']['sha256']] - raw=self.artifacts.read_blob({k:declaration[k] for k in ('sha256','bytes')}, - maximum_bytes=40*1024*1024) - self.workbench.immutable_owners[kind].validate_project_source(raw,source['metadata']) if kind in ('sequence','mesh','pointcloud','volume') else None - if kind == 'audio': - from audio_assets import parse_wav - actual,_=parse_wav(raw); expected=dict(source['metadata']);expected.pop('protected_groups',None) - if actual!=expected:fail('Audio metadata differs from original bytes.') - elif kind == 'field_sequence': - from scientific_fields import parse_bundle,validate_source - if parse_bundle(raw)!=source['metadata']:fail('Scientific metadata differs from original bytes.') - origin=doc['origin'] - if origin['method']=='project_fork': - from project_data_fork import source_origin - origin=source_origin(origin,self.session.snapshot['records']) - method=origin['method']; key='project_source' if method=='project_data_import' else 'run_source' - validate_source(method,origin[key],raw) - - def _materialize_images_and_releases(self, owned): - for envelope in owned: - family,doc=envelope['family'],envelope['document'] - paths=[] - if family=='workbench_record' and doc['source']['kind']=='image' and not doc['source']['deleted']: - folder=self.db.path/'images'/envelope['id'];folder.mkdir(parents=True,exist_ok=True) - paths=[(folder/'source',doc['source']['original']),(folder/'image.png',doc['source']['oriented'])] - elif family=='dataset_release': - paths=[(self.releases.path/(envelope['id']+'.zip'),doc['asset'])] - for path,edge in paths: - declaration=self._declarations[edge['artifact']['sha256']] - if self.artifacts.status(declaration)['state']=='verified': - path.write_bytes(self.artifacts.read_blob({k:declaration[k] for k in ('sha256','bytes')},maximum_bytes=40*1024*1024)) diff --git a/project_data_fork.py b/project_data_fork.py deleted file mode 100644 index a0b34965..00000000 --- a/project_data_fork.py +++ /dev/null @@ -1,275 +0,0 @@ -"""Finite, pure constructors for independent editable Workbench copies. - -Original documents remain exact inert records. A validated project_fork overlay -binds every new initial snapshot to its original domain revision; no old audit, -operation receipt, approval or producer application is rewritten as a new fact. -""" -import copy - -from project_data import (FAMILIES, ASSET_KINDS, closed, date, decode_data_records, - fail, identity, natural, sha) -from workbench import rights_note - -OWNED = frozenset(('workbench_text', 'workbench_record', 'workbench_response', - 'workbench_preference', 'saved_selection', 'saved_search')) - - -def _destination_family(family): - return 'workbench_record' if family == 'workbench_text' else family - - -def _index(records): - return {(r['family'], r['id']): r for r in records} - - -def _revision(envelope): - return envelope['document']['current']['revision'] - - -def _source(envelope): - doc = envelope['document'] - if envelope['family'] == 'workbench_text': - return dict(kind='text', original_text=doc['original_text']) - return copy.deepcopy(doc['source']) - - -def _origin(source_project, checkpoint, envelope): - row = envelope['document']['current'] - return dict(method='project_fork', rights=rights_note(row), origin=dict( - project_id=source_project, checkpoint=checkpoint, family=envelope['family'], - id=envelope['id'], revision=row['revision'], source_revision=row['source_revision'])) - - -def _asset(envelope, destination_id, source_project, checkpoint, now, resolve_asset): - old = envelope['document']['current'] - source = _source(envelope) - row = copy.deepcopy(old) - row.update(id=destination_id, revision=1, source_revision=1, - created_at=now, updated_at=now, review='draft', - parents=[resolve_asset(parent) or parent for parent in old['parents']], - provenance=_origin(source_project, checkpoint, envelope)) - row.pop('target_proposal', None) - if source['kind'] == 'image': - if source['deleted']: - fail('Deleted image sources cannot become fresh editable records.') - sample = copy.deepcopy(source['revisions'][-1]) - sample.update(id=destination_id, revision=1, created_at=now, updated_at=now) - source['revisions'] = [sample] - for field, sample_field in {'width':'width','height':'height','source_sha256':'sha256', - 'book_id':'book_id','session_id':'session_id','source_split':'split', - 'corner_annotation':'annotation'}.items(): - row[field] = copy.deepcopy(sample[sample_field]) - return dict(family='workbench_record', id=destination_id, document=dict( - format='workbench_record_v1', origin=copy.deepcopy(row['provenance']), source=source, - current=row, history=[copy.deepcopy(row)], events=[], audit=[])) - - -def _response(envelope, revision, destination_id, parent, now): - original = next((h['response'] for h in envelope['document']['history'] - if h['response']['revision'] == revision), None) - if original is None: - fail('Fork response revision is missing.') - row = copy.deepcopy(original) - row.update(id=destination_id, prompt_id=parent['id'], revision=1, - review='draft', created_at=now, updated_at=now) - row['provenance'] = dict(method=original['provenance']['method'], prompt_id=parent['id'], - prompt_sha256=parent['content_hash'], source_sha256=parent['source_sha256']) - return dict(family='workbench_response', id=destination_id, document=dict( - format='workbench_response_v1', current=row, - history=[dict(response=copy.deepcopy(row), parent=copy.deepcopy(parent))], audit=[])) - - -def _preference(envelope, destination_id, parent, left, right, now): - row = copy.deepcopy(envelope['document']['current']) - if row['deleted']: - fail('Deleted judgments cannot become fresh editable judgments.') - row.update(id=destination_id, revision=1, prompt_id=parent['id'], parent_revision=1, - source_revision=1, left_id=left['id'], right_id=right['id'], left_revision=1, - right_revision=1, review='draft', provenance=dict(method='draft_comparative_judgment'), - created_at=now, updated_at=now) - return dict(family='workbench_preference', id=destination_id, document=dict( - format='workbench_preference_v1', current=row, history=[dict(judgment=copy.deepcopy(row), - parent=copy.deepcopy(parent), left=copy.deepcopy(left), right=copy.deepcopy(right))], audit=[])) - - -def _intent(envelope, destination_id, now, resolve_asset_row): - doc = envelope['document'] - if doc['deleted']: - fail('Deleted saved intent cannot become fresh editable intent.') - row = copy.deepcopy(doc['current']) - row.update(id=destination_id, revision=1, created_at=now, updated_at=now) - if envelope['family'] == 'saved_selection': - members=[] - for member in row['items']: - fresh=resolve_asset_row(member['id']) - # A missing/deleted source remains an explicit archived membership, - # rather than silently dropping the entered membership or reviving it. - members.append({k:fresh[k] for k in member} if fresh else member) - row['items']=members - return dict(family=envelope['family'], id=destination_id, document=dict( - format=FAMILIES[envelope['family']], current=row, history=[copy.deepcopy(row)], - deleted=False, audit=[])) - - -def fork_records(records, source_project_id, source_checkpoint, destination_project_id, - *, id_factory, created_at): - """Construct fresh drafts; the caller archives originals and publishes once. - - id_factory(family, source_id, domain_revision) owns fresh allocation. This - function performs no SQL, filesystem access, random allocation or publication. - """ - identity(source_project_id); identity(destination_project_id); sha(source_checkpoint); date(created_at) - if source_project_id == destination_project_id or not callable(id_factory): - fail('An independent fork needs a new project and an ID allocator.') - decode_data_records(records) - from project_fork import active_records - active = [r for r in active_records(records,project_id=source_project_id) if r['family'] in OWNED] - all_records=_index(records) - mapping=[]; allocated={}; fresh={}; used={r['id'] for r in records} - def allocate(envelope, revision=None): - revision=_revision(envelope) if revision is None else revision - key=envelope['family'],envelope['id'],revision - if key not in allocated: - new_id=identity(id_factory(*key)) - if new_id in used:fail('Fork allocator reused an existing identity.') - used.add(new_id);allocated[key]=new_id - mapping.append(dict(family=key[0],source_id=key[1],source_revision=key[2],destination_id=new_id)) - return allocated[key] - assets=[r for r in active if r['family'] in ('workbench_text','workbench_record') and not _source(r).get('deleted',False)] - source_assets={r['id']:r for r in assets} - for envelope in assets:allocate(envelope) - def resolve_asset(record_id): - envelope=source_assets.get(record_id) - return allocate(envelope) if envelope is not None else None - for envelope in assets: - made=_asset(envelope,allocate(envelope),source_project_id,source_checkpoint,created_at,resolve_asset) - fresh[(made['family'],made['id'])]=made - def asset_row(record_id): - new_id=resolve_asset(record_id) - return fresh[('workbench_record',new_id)]['document']['current'] if new_id else None - responses={(r['id'],_revision(r)):r for r in active if r['family']=='workbench_response'} - preferences=[r for r in active if r['family']=='workbench_preference' and not r['document']['current']['deleted']] - for envelope in preferences: - row=envelope['document']['current'] - for side in ('left','right'): - key=row[side+'_id'],row[side+'_revision'] - responses[key]=all_records[('workbench_response',key[0])] - for (record_id,revision),envelope in sorted(responses.items()): - original=next(h['response'] for h in envelope['document']['history'] if h['response']['revision']==revision) - parent=asset_row(original['prompt_id']) - if parent is None:fail('Fork response needs a current editable prompt.') - made=_response(envelope,revision,allocate(envelope,revision),parent,created_at) - fresh[(made['family'],made['id'])]=made - for envelope in preferences: - row=envelope['document']['current'];parent=asset_row(row['prompt_id']) - answers=[] - for side in ('left','right'): - original=all_records[('workbench_response',row[side+'_id'])] - new_id=allocate(original,row[side+'_revision']) - answers.append(fresh[('workbench_response',new_id)]['document']['current']) - made=_preference(envelope,allocate(envelope),parent,*answers,created_at) - fresh[(made['family'],made['id'])]=made - for envelope in active: - if envelope['family'] in ('saved_selection','saved_search') and not envelope['document']['deleted']: - made=_intent(envelope,allocate(envelope),created_at,asset_row) - fresh[(made['family'],made['id'])]=made - mapping.sort(key=lambda m:(m['family'],m['source_id'],m['source_revision'] or 0,m['destination_id'])) - return dict(records=[fresh[key] for key in sorted(fresh)],mapping=mapping) - - -def _overlay_maps(records): - if not any(r['family']=='project_fork' for r in records):return [] - from project_fork import decode_fork - decode_fork(records) - return [r['document'] for r in records if r['family']=='project_fork'] - - -def validate_origin(value, record, source, records): - """A fork origin is meaningful only with its exact archived source mapping.""" - closed(value,{'method','rights','origin'},'fork data provenance') - ref=value['origin'] - closed(ref,{'project_id','checkpoint','family','id','revision','source_revision'},'fork data source') - identity(ref['project_id']);sha(ref['checkpoint']);identity(ref['id']);natural(ref['revision']) - if ref['source_revision'] is not None:natural(ref['source_revision']) - indexed=_index(records) - original=indexed.get((ref['family'],ref['id'])) - if ref['family'] not in ('workbench_record','workbench_text') or original is None: - fail('Fork data origin has no retained source.') - old=original['document']['current'] - if old['revision']!=ref['revision'] or old['source_revision']!=ref['source_revision'] or value['rights']!=rights_note(old): - fail('Fork data origin differs from its archived current source.') - matches=[overlay for overlay in _overlay_maps(records) - if overlay['source']==dict(project_id=ref['project_id'],checkpoint=ref['checkpoint']) - and dict(family=ref['family'],source_id=ref['id'],source_revision=ref['revision'],destination_id=record['id']) in overlay['mapping']] - if len(matches)!=1:fail('Fork data origin lacks its exact independent-fork mapping.') - if (record['kind'],record['content_hash'],record['pixel_hash'])!=(old['kind'],old['content_hash'],old['pixel_hash']): - fail('Fork source identity differs from its original.') - original_source=_source(original) - if source['kind']=='image': - if any(source[k]!=original_source[k] for k in ('kind','original','oriented')): - fail('Fork image source bytes differ.') - elif source!=original_source:fail('Fork original source or immutable metadata differs.') - return True - - -def validate_records(records): - """Prove initial derived snapshots without rewriting original history.""" - indexed=_index(records) - for overlay in _overlay_maps(records): - own=[m for m in overlay['mapping'] if m['family'] in OWNED] - by_source={(m['family'],m['source_id'],m['source_revision']):m for m in own} - def asset_mapping(record_id): - values=[m for m in own if m['family'] in ('workbench_record','workbench_text') and m['source_id']==record_id] - if len(values)>1:fail('Fork asset has ambiguous destination mappings.') - return values[0]['destination_id'] if values else None - def asset_row(record_id): - new_id=asset_mapping(record_id) - return indexed[('workbench_record',new_id)]['document']['history'][0] if new_id else None - for mapping in own: - family=mapping['family'];original=indexed[(family,mapping['source_id'])] - destination=indexed[(_destination_family(family),mapping['destination_id'])] - doc=destination['document'] - initial=doc['history'][0] - row=initial['response'] if family=='workbench_response' else initial['judgment'] if family=='workbench_preference' else initial - now=row['created_at'];date(now) - if family in ('workbench_record','workbench_text'): - if mapping['source_revision']!=_revision(original):fail('Fork source domain revision differs.') - expected=_asset(original,destination['id'],overlay['source']['project_id'],overlay['source']['checkpoint'],now,asset_mapping)['document'] - if initial!=expected['history'][0] or doc['origin']!=expected['origin']: - fail('Fork initial Workbench snapshot differs from its exact derived draft.') - if doc['source']['kind']=='image': - if doc['source']['revisions'][0]!=expected['source']['revisions'][0]:fail('Fork initial image source differs.') - elif doc['source']!=expected['source']:fail('Fork immutable source differs.') - elif family=='workbench_response': - revision=mapping['source_revision'];natural(revision) - source_row=next((h['response'] for h in original['document']['history'] if h['response']['revision']==revision),None) - if source_row is None:fail('Fork response source revision is missing.') - parent=asset_row(source_row['prompt_id']) - if parent is None:fail('Fork response lost its current prompt.') - expected=_response(original,revision,destination['id'],parent,now)['document'] - if initial!=expected['history'][0]:fail('Fork initial response binding differs.') - elif family=='workbench_preference': - if mapping['source_revision']!=_revision(original):fail('Fork judgment source revision differs.') - source_row=original['document']['current'];parent=asset_row(source_row['prompt_id']);answers=[] - for side in ('left','right'): - answer=by_source.get(('workbench_response',source_row[side+'_id'],source_row[side+'_revision'])) - if answer is None:fail('Fork judgment exact answer revision is missing.') - answers.append(indexed[('workbench_response',answer['destination_id'])]['document']['history'][0]['response']) - expected=_preference(original,destination['id'],parent,*answers,now)['document'] - if initial!=expected['history'][0]:fail('Fork initial preference binding differs.') - else: - if mapping['source_revision']!=_revision(original):fail('Fork saved intent source revision differs.') - expected=_intent(original,destination['id'],now,asset_row)['document'] - if initial!=expected['history'][0]:fail('Fork initial saved intent differs.') - return True - - -def source_origin(origin, records): - """Follow only validated inert fork facts to a subtype source declaration.""" - indexed=_index(records);seen=set();value=origin - while value.get('method')=='project_fork': - ref=value['origin'];key=ref['family'],ref['id'] - if key in seen or key not in indexed:fail('Fork source provenance is cyclic or unavailable.') - seen.add(key);doc=indexed[key]['document'] - value=doc.get('origin',doc['current']['provenance']) - return copy.deepcopy(value) diff --git a/project_dispatch.py b/project_dispatch.py deleted file mode 100644 index e517c2e6..00000000 --- a/project_dispatch.py +++ /dev/null @@ -1,451 +0,0 @@ -"""Fixed M2 public-to-owner dispatch for one already bound project projection. - -The workspace validates selection, binding, checkpoint and the HTTP envelope. -This module projects bounded pages and calls existing domain methods. It neither -constructs live owners nor interprets scientific records or imported authority. -""" -import base64 -import binascii -import copy - -from workbench import WorkbenchError -from project_session import session_records - - -_BOUND_FIELDS = frozenset(('binding', 'checkpoint', 'expected_checkpoint', 'request_key')) -_DATA_ACTIONS = frozenset(('data_query', 'data_get', 'data_scientific', 'data_availability', 'data_history', - 'data_responses', 'data_response_history', 'data_preferences', 'data_preference_history', - 'data_import_text', 'data_import_image', 'data_import_audio', 'data_import_mesh', - 'data_import_pointcloud', 'data_import_volume', 'data_import_sequence', 'data_import_scientific', - 'data_annotate', 'data_rights', 'data_source_save', 'data_source_delete', 'data_response_save', - 'data_preference_save', 'data_preference_delete', 'selections', 'selection_get', 'selection_create', - 'selection_rename', 'selection_delete', 'searches', 'search_create', 'search_rename', - 'search_delete', 'release_preview', 'release_freeze')) -_MATERIAL_ACTIONS = frozenset(('definitions', 'models', 'model_get', 'model_create', 'model_save', - 'model_publish', 'graphs', 'graph_get', 'graph_create', 'graph_create_external', 'graph_save', 'graph_clone', 'recipes', - 'recipe_get', 'recipe_create', 'recipe_save', 'recipe_prepare', 'model_material', - 'recipe_material', 'runtime_material', 'snapshot', 'materials')) -_HISTORY_ACTIONS = frozenset(('studies', 'study', 'study_create', 'study_save', 'trial', 'trial_prepare', - 'lineage', 'lineage_record', 'progression_origins', 'progression', 'run', 'runs', 'candidate', 'candidate_select', 'evidence', - 'retained_analysis', 'retained_analysis_export', 'policies', 'policy', 'dependencies', - 'publication_result', 'package_state', 'expose_holdout', 'run_document_segment')) -_OTHER_ACTIONS = frozenset(('artifacts', 'artifact_status', 'artifact_resolve', 'producers', 'producer', 'source_register', 'source_references')) -_EXECUTION_ACTIONS = frozenset(('native_preflight','native_submit','native_retry','native_reconcile','execution_health', 'execution_preflight_external', 'execution_approve_external', - 'execution_submit_external', 'execution_cancel', 'execution_study_continue', 'execution_study_approve', - 'execution_study_admit', 'execution_study_stop', 'execution_reconcile_current', - 'execution_producer_start', 'execution_generation_resume', 'execution_grounded_review', - 'execution_producer_decide', 'execution_producer_cancel', 'execution_producer_health', - 'execution_study_prepare_package','execution_study_prepare_verifier','execution_study_abandon_preparation','execution_study_reconcile_preparation')) -_RUNTIME_ACTIONS=frozenset(('runtime_sources','runtime_inspect','runtime_capture','runtime_status','runtime_abandon')) -_MEASUREMENT_ACTIONS=frozenset(('measurement_import','measurement_sources','measurement_metadata', - 'measurement_subjects','measurement_query','measurement_inspect','measurement_compare')) -_M4_ACTIONS=frozenset(('files_inspect','file_get','file_history','file_adopt','files_adopt','file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')) -SUPPORTED_OPERATIONS = frozenset('project_files.' + action for action in - _DATA_ACTIONS | _MATERIAL_ACTIONS | _HISTORY_ACTIONS | _OTHER_ACTIONS | _EXECUTION_ACTIONS | _M4_ACTIONS | _RUNTIME_ACTIONS | _MEASUREMENT_ACTIONS) - - -def _body(arguments, *omit): - return {key: copy.deepcopy(value) for key, value in arguments.items() - if key not in _BOUND_FIELDS and key not in omit} - - -def _page(items, arguments, *, default=40): - offset, limit = arguments.get('offset', 0), arguments.get('limit', default) - if type(offset) is not int or offset < 0 or type(limit) is not int or not 1 <= limit <= (4 if default == 1 else 40): - raise WorkbenchError('Invalid project page bounds.') - return dict(items=copy.deepcopy(items[offset:offset + limit]), total=len(items), offset=offset, limit=limit) - - -def _origin_items(items, arguments): - origin = arguments.get('origin_catalog_id') - return [item for item in items if origin is None or item['origin_catalog_id'] == origin] - - -def _declarations(projection): - return {item['id']: item['document'] for item in session_records(projection.session, 'project_artifact')} - - -def _artifact(projection, sha): - declaration = _declarations(projection).get(sha) - if declaration is None: - raise WorkbenchError('Project artifact declaration is unavailable.', 'unavailable', 404) - return declaration - - -def _bundle(projection, kind, arguments): - """Existing native parsers alone assign bundle meaning, before publication.""" - body = _body(arguments) - files = body['files'] - try: - if kind == 'mesh': - import meshes - prepared = meshes.prepare(meshes.decode_file(files['mesh.ply'], meshes.PLY_LIMIT, 'mesh.ply'), - meshes.decode_file(files['mesh.json'], meshes.MANIFEST_LIMIT, 'mesh.json')) - elif kind in ('pointcloud', 'volume'): - import spatial_assets - parser, names, caps = spatial_assets.profile(kind) - raw = [] - for name, cap in zip(names, caps): - value = base64.b64decode(files[name], validate=True) - if not 0 < len(value) <= cap: - raise ValueError() - raw.append(value) - prepared = parser.prepare(*raw) - elif kind == 'sequence': - prepared = projection.data.workbench.sequences.prepare_project_files(files) - else: - raise WorkbenchError('Unsupported native project import.', 'unsupported', 409) - except WorkbenchError: - raise - except (ValueError, binascii.Error, KeyError, TypeError): - raise WorkbenchError('Invalid native project import bundle.') from None - return projection.data.import_bundle(kind, prepared, body, arguments['expected_checkpoint'], arguments['request_key']) - - -def _data(projection, action, a): - data = projection.data - if action == 'data_query': return data.query(_body(a)) - if action == 'data_get': return data.get(a['record_id']) - if action == 'data_scientific': return data.scientific(a['record_id'], a['content_hash'], - **{key: a[key] for key in ('frame', 'field', 'axis', 'index') if key in a}) - if action == 'data_availability': return data.availability(a['record_id']) - if action == 'data_history': return _page(data.history(a['record_id']), a, default=1) - if action == 'data_responses': return _page(data.responses(a['record_id'])['responses'], a) - if action == 'data_response_history': - return _page(sorted(data.response_history(a['record_id'])['history'], key=lambda item: item['revision'], reverse=True), a, default=1) - if action == 'data_preferences': return _page(data.preferences(a['record_id'])['judgments'], a) - if action == 'data_preference_history': - return _page(sorted(data.preference_history(a['record_id'])['history'], key=lambda item: item['revision'], reverse=True), a, default=1) - if action == 'selections': return _page(sorted(data.list_selections()['selections'], key=lambda item: item['id']), a) - if action == 'searches': return _page(sorted(data.list_searches()['searches'], key=lambda item: item['id']), a) - if action == 'selection_get': return data.load_selection(a['selection_id']) - if action == 'release_preview': return data.preview_release(_body(a)) - cp, key = a['expected_checkpoint'], a['request_key'] - if action == 'data_import_text': return data.import_text(_body(a), cp, key) - if action == 'data_import_image': return data.import_image(_body(a), cp, key) - if action == 'data_import_audio': return data.import_audio(_body(a), cp, key) - if action == 'data_import_mesh': return _bundle(projection, 'mesh', a) - if action == 'data_import_pointcloud': return _bundle(projection, 'pointcloud', a) - if action == 'data_import_volume': return _bundle(projection, 'volume', a) - if action == 'data_import_sequence': return _bundle(projection, 'sequence', a) - if action == 'data_import_scientific': return projection.source_adapter.import_scientific(_body(a), cp, key) - if action == 'data_annotate': return data.annotate(a['record_id'], _body(a, 'record_id'), cp, key) - if action == 'data_rights': return data.correct_rights_note(a['record_id'], _body(a, 'record_id'), cp, key) - if action == 'data_source_save': return data.save_image_source(a['record_id'], a['document'], cp, key) - if action == 'data_source_delete': return data.delete_image_source(a['record_id'], a['document'], cp, key) - if action == 'data_response_save': return data.save_response(a['document'], cp, key) - if action == 'data_preference_save': return data.save_preference(a['document'], cp, key) - if action == 'data_preference_delete': return data.delete_preference(a['document'], cp, key) - if action == 'selection_create': return data.create_selection(a['document'], cp, key) - if action == 'selection_rename': return data.rename_selection(a['selection_id'], _body(a, 'selection_id'), cp, key) - if action == 'selection_delete': return data.delete_selection(a['selection_id'], _body(a, 'selection_id'), cp, key) - if action == 'search_create': return data.create_search(a['document'], cp, key) - if action == 'search_rename': return data.rename_search(a['search_id'], _body(a, 'search_id'), cp, key) - if action == 'search_delete': return data.delete_search(a['search_id'], _body(a, 'search_id'), cp, key) - if action == 'release_freeze': return data.freeze_release(_body(a), cp, key) - raise AssertionError('Closed data dispatch is incomplete.') - - -def _definitions(projection, family, a): - docs = projection.materials._documents() - values = [doc['current'] for (kind, _), doc in docs.items() - if kind == family and doc['origin_catalog_id'] == projection.session.summary['project_id']] - return _page(sorted(values, key=lambda item: item['id']), a) - - -def _material(projection, action, a): - materials, models, pipelines = projection.materials, projection.context.models, projection.context.pipelines - if action == 'definitions': return pipelines.definitions() - if action == 'models': return _definitions(projection, 'model_definition', a) - if action == 'graphs': return _definitions(projection, 'pipeline_graph', a) - if action == 'recipes': return _definitions(projection, 'recipe_definition', a) - if action == 'model_get': return models.get(a['model_id']) - if action == 'graph_get': return pipelines.get(a['graph_id']) - if action == 'recipe_get': return pipelines.get_recipe(a['recipe_id']) - if action == 'model_material': return materials.model(a['ref'], verify=a.get('verify', False)) - if action == 'recipe_material': return materials.recipe(a['ref'], verify=a.get('verify', False)) - if action == 'runtime_material': return materials.runtime(a['ref'], verify=a.get('verify', False)) - if action == 'snapshot': return materials.snapshot_ref(a['project_id'], a['manifest_sha256'], - catalog_id=a['origin_catalog_id'], verify=a.get('verify', False)) - if action == 'materials': - items = [] - for (family, _), doc in materials._documents().items(): - if family not in ('model_material', 'recipe_material', 'runtime_material', 'project_snapshot'): continue - value = doc['value'] - identity = value.get('ref', {}).get('id', value.get('id')) - item = dict(family=family, origin_catalog_id=doc['origin_catalog_id'], id=identity, - format=value.get('material', value).get('format')) - if 'ref' in value: item['ref'] = copy.deepcopy(value['ref']) - else: item['manifest_sha256'] = value['manifest_sha256'] - items.append(item) - return _page(sorted(_origin_items(items, a), key=lambda item: (item['origin_catalog_id'], item['family'], item['id'])), a) - cp, key = a['expected_checkpoint'], a['request_key'] - if action == 'model_create': - body = _body(a) - return materials.mutate_definition('model_definition', 'models.create', None, body, cp, key, lambda: models.create(body)) - if action == 'model_save': - body = _body(a, 'model_id') - return materials.mutate_definition('model_definition', 'models.save', a['model_id'], body, cp, key, lambda: models.save(a['model_id'], body)) - if action == 'model_publish': return materials.publish_model(models, a['model_id'], _body(a, 'model_id'), cp, key) - if action == 'graph_create': - body = _body(a) - return materials.mutate_definition('pipeline_graph', 'graphs.create', None, body, cp, key, lambda: pipelines.create(body)) - if action == 'graph_create_external': - body = _body(a) - return materials.mutate_definition('pipeline_graph', 'graphs.create_external', None, body, cp, key, lambda: pipelines.create_external(body)) - if action == 'graph_save': - body = _body(a, 'graph_id') - return materials.mutate_definition('pipeline_graph', 'graphs.save', a['graph_id'], body, cp, key, lambda: pipelines.save(a['graph_id'], body)) - if action == 'graph_clone': - body = _body(a, 'graph_id') - return materials.mutate_definition('pipeline_graph', 'graphs.clone', a['graph_id'], body, cp, key, lambda: pipelines.clone(a['graph_id'], body)) - if action == 'recipe_create': - body = _body(a) - return materials.mutate_definition('recipe_definition', 'recipes.create', None, body, cp, key, lambda: pipelines.create_recipe(body)) - if action == 'recipe_save': - body = _body(a, 'recipe_id') - return materials.mutate_definition('recipe_definition', 'recipes.save', a['recipe_id'], body, cp, key, lambda: pipelines.save_recipe(a['recipe_id'], body)) - if action == 'recipe_prepare': - from pipeline import model_aware_graph - graph=pipelines.get(a['graph_id']);builtin=model_aware_graph(graph) - if graph.get('recipe')!='external_recipe' and not builtin: - raise WorkbenchError('This native graph does not support composed model material.','unsupported',409) - if 'builtin' in a and a['builtin'] is not builtin: - raise WorkbenchError('Recipe preparation kind differs from the saved graph.','conflict',409) - return materials.prepare_recipe(pipelines, a['graph_id'], a['revision'], a['model'], cp, key, - builtin=builtin, reconcile_capture=a.get('reconcile_capture', False), - runtime_manifest=projection.runtime_manifest) - raise AssertionError('Closed material dispatch is incomplete.') - - -def _study_summary(study): - summary = copy.deepcopy(study) - trial_ids = summary.pop('trial_ids') - summary['source_format'] = summary.pop('format') - summary['format'] = 'project_study_summary_v1' - summary['trial_count'] = len(trial_ids) - return summary - - -def _study_page(history, arguments): - value = history.study(arguments['study_id'], origin_catalog_id=arguments['origin_catalog_id'], - offset=arguments.get('offset', 0), limit=arguments.get('limit', 40), revision=arguments.get('revision')) - result = copy.deepcopy(value) - result['study'] = _study_summary(value['study']) - result['trials'] = _page([{'id': identity} for identity in value['study']['trial_ids']], arguments) - return result - - -def _run_page(history, arguments): - value = history.run(arguments['run_id'], origin_catalog_id=arguments['origin_catalog_id'], - offset=arguments.get('offset', 0), limit=arguments.get('limit', 40)) - # The Run owner applies the shared disclosure projection first. Large - # captured documents and collections remain exact through document segments. - run = value['run'] - fields = ('mode', 'id', 'dataset_id', 'request_key', 'state', 'created_at', 'started_at', - 'finished_at', 'graph_revision', 'graph_sha256', 'attempt', 'parent_run_id', - 'quality_claim', 'execution_kind', 'result_kind', 'material_sha256', - 'approval_id', 'actor_id', 'custody_phase', 'release_id', 'release_sha256', 'release_bytes') - summary = {key: copy.deepcopy(run[key]) for key in fields if key in run} - summary.update(format='project_run_summary_v1', source_schema_version=run['schema_version'], - available_fields=sorted(run), - collection_counts={key: len(run[key]) for key in - ('events', 'artifacts', 'node_attempts', 'exposure_events', 'prior_test_exposure', '_phase_timings') - if isinstance(run.get(key), list)}) - result = {key: copy.deepcopy(item) for key, item in value.items() if key != 'run'} - result['run'] = summary - return result - - -def _history(projection, action, a): - history = projection.history - origin = a.get('origin_catalog_id') - if action == 'studies': - return _page([dict(origin_catalog_id=item['origin_catalog_id'], study=_study_summary(item['study'])) - for item in _origin_items(history.studies()['studies'], a)], a) - if action == 'study': return _study_page(history, a) - if action == 'trial': return history.trial(a['trial_id'], origin_catalog_id=origin) - if action == 'run': return _run_page(history, a) - if action == 'run_document_segment': return history.run_document_segment(a['run_id'], - origin_catalog_id=origin, expected_sha256=a['expected_sha256'], offset=a['offset'], limit=a['limit']) - if action == 'candidate': return history.candidate(a['candidate_id'], origin_catalog_id=origin, verify=a.get('verify', False)) - if action == 'lineage': return _page(history.lineage(origin_catalog_id=origin)['records'], a) - if action == 'progression_origins': return history.progression_origins(offset=a.get('offset',0),limit=a.get('limit',40)) - if action == 'progression': return history.progression(origin,offset=a.get('offset',0),limit=a.get('limit',20),cohort=a.get('cohort')) - if action == 'policies': return _page(history.policies(origin_catalog_id=origin)['policies'], a) - if action == 'policy': return history.policy(a['ref']) - if action == 'runs': - items = [] - for (catalog, identity), doc in sorted(history._facts().runs.items()): - run = doc['run'] - summary = {key: copy.deepcopy(run[key]) for key in ('id', 'name', 'state', 'created_at', 'finished_at', 'execution_kind') if key in run} - items.append(dict(origin_catalog_id=catalog, run=summary)) - return _page(_origin_items(items, a), a) - if action == 'dependencies': return history.dependencies(a['run_id'], origin_catalog_id=origin, - offset=a.get('offset', 0), limit=a.get('limit', 40)) - if action == 'publication_result': return history.publication_result(a['run_id'], origin_catalog_id=origin) - if action == 'package_state': return history.package_state(a['candidate_id'], origin_catalog_id=origin) - if action == 'evidence': return history.evidence(a['run_id'], a.get('selectors'), origin_catalog_id=origin) - if action in ('retained_analysis', 'retained_analysis_export'): - return history.retained_analysis(origin, _body(a, 'origin_catalog_id'), export=action == 'retained_analysis_export') - cp, key = a['expected_checkpoint'], a['request_key'] - if action == 'study_create': return history.create_study(_body(a), cp, key) - if action == 'study_save': return history.save_study(a['study_id'], _body(a, 'study_id'), cp, key) - if action == 'trial_prepare': return history.prepare_trial_from_material(a['study_id'], _body(a, 'study_id'), cp, key) - if action == 'candidate_select': return history.select_candidate(a['study_id'], _body(a, 'study_id'), cp, key) - if action == 'lineage_record': return history.record_lineage(_body(a), cp, key) - if action == 'expose_holdout': return projection.disclose(a['run_id'], _body(a, 'run_id'), cp, key) - raise AssertionError('Closed historical dispatch is incomplete.') - - -def _execution(projection, action, a): - execution = projection.execution - if execution is None: - raise WorkbenchError('Current execution is unavailable for this project.', 'unavailable', 409) - if action == 'execution_health': return execution.health(offset=a.get('offset', 0), limit=a.get('limit', 40)) - if action == 'execution_preflight_external': return execution.preflight_external(a['graph_id'], a['revision']) - if action == 'execution_reconcile_current': return execution.reconcile_current() - if action == 'execution_producer_health': - if execution.producer_execution is None: - raise WorkbenchError('Current producer execution is unavailable.', 'unavailable', 409) - return execution.producer_execution.status(offset=a.get('offset', 0), limit=a.get('limit', 40)) - if action=='execution_study_reconcile_preparation': - if execution.study_execution is None:raise WorkbenchError('Current Study execution is unavailable.','unavailable',409) - return execution.study_execution.reconcile_preparation(a['study_id'],_body(a,'study_id'),a['expected_checkpoint']) - if action=='native_reconcile':return execution.native_reconcile(a['phase_id'],a['expected_checkpoint']) - cp, key = a['expected_checkpoint'], a['request_key'] - if action=='native_preflight':return execution.native_preflight(a['graph_id'],a['revision'],cp,key) - if action=='native_submit':return execution.native_submit(_body(a),cp,key) - if action=='native_retry':return execution.native_retry(a['run_id'],cp,key) - if action == 'execution_producer_start' or action in ('execution_generation_resume', 'execution_grounded_review', 'execution_producer_decide', 'execution_producer_cancel'): - producers = execution.producer_execution - if producers is None: - raise WorkbenchError('Current producer execution is unavailable.', 'unavailable', 409) - if action == 'execution_producer_start': return producers.start(a['family'], a['document'], cp, key) - if action == 'execution_generation_resume': return producers.resume_generation(a['producer_id'], cp, key) - if action == 'execution_grounded_review': return producers.review_grounded(a['producer_id'], a['document'], cp, key) - if action == 'execution_producer_decide': return producers.decide(a['family'], a['producer_id'], a['document'], cp, key) - return producers.cancel(a['family'], a['producer_id'], cp, key) - if action == 'execution_approve_external': return execution.approve_external(_body(a), cp, key) - if action == 'execution_submit_external': return execution.submit_external(_body(a), cp, key) - if action == 'execution_cancel': return execution.cancel_run(a['run_id'], cp, key) - studies = execution.study_execution - if studies is None: - raise WorkbenchError('Current Study execution is unavailable.', 'unavailable', 409) - if action=='execution_study_prepare_package':return studies.prepare_package(a['study_id'],_body(a,'study_id'),cp,key) - if action=='execution_study_prepare_verifier':return studies.prepare_verifier(a['study_id'],_body(a,'study_id'),cp,key) - if action=='execution_study_abandon_preparation':return studies.abandon_preparation(a['study_id'],_body(a,'study_id'),cp,key) - if action == 'execution_study_continue': return studies.continue_study(a['study_id'], _body(a, 'study_id'), cp, key) - if action == 'execution_study_approve': return studies.approve(a['study_id'], _body(a, 'study_id'), cp, key) - if action == 'execution_study_admit': return studies.admit(a['study_id'], _body(a, 'study_id'), cp, key) - if action == 'execution_study_stop': return studies.stop(a['study_id'], _body(a, 'study_id'), cp, key) - raise AssertionError('Closed execution dispatch is incomplete.') - - -def _runtime(projection, action, a): - if action=='runtime_sources': - return _page([dict(source=alias) for alias in projection.runtime_sources()['sources']],a) - if action=='runtime_inspect': - value=projection.runtime.inspect({'source':a['source']});manifest=value['manifest'] - return dict(format='project_runtime_inspection_v1',source=a['source'], - **{field:copy.deepcopy(value[field]) for field in ('inventory_sha256','file_count','total_bytes','manifest_bytes','free_bytes')}, - profile=manifest['profile'],python_version=manifest['python_version'],launch=copy.deepcopy(manifest['launch']), - packages=[dict(name=name,version=version) for name,version in sorted(manifest['packages'].items())], - files=_page(manifest['files'],a)) - if action=='runtime_status':return projection.runtime.status(a['capture_key']) - if action=='runtime_abandon':return projection.runtime.abandon(a['capture_key'],a['reason'],a['expected_checkpoint'],a['request_key']) - if action=='runtime_capture': - options={'reserve_bytes':a['reserve_bytes']} if 'reserve_bytes' in a else {} - return projection.runtime.capture({'source':a['source']},a['expected_inventory_sha256'], - a['expected_checkpoint'],a['request_key'],**options) - raise AssertionError('Closed runtime dispatch is incomplete.') - - -def _m4(projection,action,a): - session=projection.session - if action=='file_adopt': - return projection.files.adopt(_body(a),a['expected_checkpoint'],a['request_key']) - if action=='files_adopt': - return projection.files.adopt_many(_body(a),a['expected_checkpoint'],a['request_key']) - if action=='file_adopt_deletion': - return projection.files.adopt_deletion(_body(a),a['expected_checkpoint'],a['request_key']) - with session.access.lock,session.lock,session.db.lock: - session.access.require('inspect','project_files.'+action) - if action=='fork_info': - snapshot=session.check() - from project_fork import decode_fork - overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) - return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], - mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, - copied_count=len(overlay['artifacts']['copied']) if overlay else 0, - missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) - session.check_current() - if action=='files_inspect':return _page(projection.files.inspect()['items'],a) - if action=='file_get':return projection.files.read(a['file_id'],a.get('revision')) - if action=='file_history': - values=projection.files.history(a['file_id']) - return dict(id=a['file_id'],**_page([{k:v[k] for k in ('revision','sha256','bytes','adopted_at','actor_id',*(['deleted'] if v.get('deleted') is True else []))} for v in values],a)) - if action in ('material_intents','material_intent'): - intents=projection.materials.intents() - if action=='material_intent': - result=next((item for item in intents if item['family']==a['family'] and item['current']['id']==a['definition_id']),None) - if result is None:raise WorkbenchError('Unresolved material intent is unavailable.','unavailable',404) - return result - return _page([dict(family=item['family'],id=item['id'],catalog_id=item['current']['catalog_id'], - definition_id=item['current']['id'],name=item['current']['name'],revision=item['current']['revision'], - required_reselection=item['current']['required_reselection'],fork_origin=item['fork_origin']) for item in intents],a) - if action=='material_intent_capabilities': - value=projection.materials.intent_capabilities() - items=[dict(item,requires_reselection=[]) for item in value['editable']] - items.extend(dict(family=item['family'],id=item['id'],requires_reselection=item['fields']) for item in value['requires_reselection']) - return _page(sorted(items,key=lambda item:(item['family'],item['id'])),a) - raise AssertionError('Closed M4 dispatch is incomplete.') - - -def dispatch(projection, name, arguments): - if name not in SUPPORTED_OPERATIONS: - raise WorkbenchError('Unsupported external-project operation.', 'unsupported', 409) - action = name.split('.')[1] - projection.session.access.require('inspect', name) - if action in _MEASUREMENT_ACTIONS:return _measurements(projection,action,arguments) - if action in _RUNTIME_ACTIONS:return _runtime(projection,action,arguments) - if action in _M4_ACTIONS:return _m4(projection,action,arguments) - if action in _EXECUTION_ACTIONS: return _execution(projection, action, arguments) - if action in _DATA_ACTIONS: return _data(projection, action, arguments) - if action in _MATERIAL_ACTIONS: return _material(projection, action, arguments) - if action in _HISTORY_ACTIONS: return _history(projection, action, arguments) - if action == 'artifacts': - items = [dict(copy.deepcopy(declaration), availability=projection.artifacts.inspect(declaration)) - for _, declaration in sorted(_declarations(projection).items())] - return _page(items, arguments) - if action == 'artifact_status': return projection.artifacts.status(_artifact(projection, arguments['sha256'])) - if action == 'artifact_resolve': - projection.session.access.require('edit', 'project_files.artifact_resolve', arguments['sha256'], owner_only=True) - return projection.resolve_artifact(_body(arguments), arguments['expected_checkpoint'], arguments['request_key']) - if action == 'producers': return projection.producers.list(arguments['family'], - origin_catalog_id=arguments.get('origin_catalog_id'), offset=arguments.get('offset', 0), limit=arguments.get('limit', 40)) - if action == 'producer': return projection.producers.get(arguments['family'], arguments['producer_id'], - origin_catalog_id=arguments['origin_catalog_id'], entries_offset=arguments.get('entries_offset', 0), - entries_limit=arguments.get('entries_limit', 40)) - if action == 'source_references': - return projection.source_adapter.references( - {key: arguments[key] for key in ('origin_catalog_id', 'project_id', 'manifest_sha256')}, - offset=arguments.get('offset', 0), limit=arguments.get('limit', 40)) - if action == 'source_register': - projection.session.access.require('edit', 'project_files.source_register', owner_only=True) - return projection.source_adapter.register(_body(arguments), arguments['expected_checkpoint'], arguments['request_key']) - raise AssertionError('Closed project dispatch is incomplete.') - - -def _measurements(projection,action,a): - owner=projection.measurements - if action=='measurement_import': - return projection.source_adapter.import_measurements(_body(a),a['expected_checkpoint'],a['request_key']) - if action=='measurement_sources':return owner.sources(offset=a.get('offset',0),limit=a.get('limit',40)) - if action=='measurement_compare':return owner.compare(a['sources'],**_body(a,'sources')) - origin,packet=a['origin_catalog_id'],a['packet_sha256'] - if action=='measurement_metadata':return owner.metadata(origin,packet) - if action=='measurement_subjects':return owner.subjects(origin,packet,offset=a.get('offset',0),limit=a.get('limit',40)) - if action=='measurement_query':return owner.query(origin,packet,**_body(a,'origin_catalog_id','packet_sha256')) - if action=='measurement_inspect':return owner.inspect(origin,packet,a['observation_id']) - raise AssertionError('Closed measurement dispatch is incomplete.') diff --git a/project_execution.py b/project_execution.py deleted file mode 100644 index 9f7a5878..00000000 --- a/project_execution.py +++ /dev/null @@ -1,1057 +0,0 @@ -"""Current-host project execution, separate from inert imported Run history. - -Construction installs transaction capture hooks but never constructs Runs, -claims runtime ownership, recovers a process or wakes a queue. Only an explicit -currently authorized action may create the local execution owner. -""" -import copy -from contextlib import contextmanager -import hashlib -import json -import os -import re -import threading -import time -import uuid - -import agent_api -from access import Access, AccessError -from model_definitions import natural -from pipeline import closed, digest, identifier -from project_history import capture_run, declarations -from project_session import operation_records -from workbench import WorkbenchError -from project_session import session_operations - - -OPERATIONS = frozenset(('runs.submit_external', 'runs.submit', 'runs.retry', 'runs.cancel', 'runs.approve_external')) - - -class _FileEffect: - """One of the two named nonexecuting filesystem publication guards.""" - def __init__(self, execution, kind, key, input_sha256, content_sha256): - self.execution,self.kind,self.key=execution,kind,key - self.input_sha256,self.content_sha256=input_sha256,content_sha256 - self.started=False;self.done=False - - def publication_started(self): - execution=self.execution - pending=execution._runtime_pending if self.kind=='runtime' else execution._package_pending - identity_field='manifest_sha256' if self.kind=='runtime' else 'material_sha256' - entry=dict(request_key=self.key,input_sha256=self.input_sha256,**{identity_field:self.content_sha256}) - previous=pending.get(self.key) - if previous is not None and previous!=entry: - raise WorkbenchError('File publication key has different current input.','conflict',409) - # Retain a conservative local fence even if journal fsync acknowledgement - # fails. No owner filesystem write follows an unacknowledged start. - pending[self.key]=entry;self.started=True - if execution.custody is not None: - with execution._journal_guard(): - if not execution.custody.has_pending:execution.custody.acknowledge(execution.session.snapshot) - if self.kind=='runtime':execution.custody.runtime_started(self.key,self.input_sha256,self.content_sha256) - else:execution.custody.package_started(self.key,self.input_sha256,self.content_sha256) - - def confirmed(self): - if not self.started:raise RuntimeError('File publication must have a durable start before confirmation.') - execution=self.execution;execution.session.check_current() - if execution.db.in_transaction: - raise RuntimeError('File publication confirms only after its canonical and outer SQL acknowledgement.') - if execution.custody is not None: - with execution._journal_guard(): - if self.kind=='runtime':execution.custody.runtime_confirmed(self.key) - else:execution.custody.package_confirmed(self.key) - pending=execution._runtime_pending if self.kind=='runtime' else execution._package_pending - pending.pop(self.key,None);self.done=True - - -def _admission_result(run): - return dict(format='project_run_admission_v1', origin_catalog_id=run['dataset_id'], - run_id=run['id'], request_sha256=run['request_sha256']) - - -def validate_operation_result(operation, result, records=None): - """Validate inert operation facts, never a live grant or resume instruction.""" - if operation not in OPERATIONS: - return False - if operation == 'runs.approve_external': - closed(result, ('format','origin_catalog_id','approval_id','material_sha256', - 'consumer_actor_id','approver_actor_id','created_at','expires_at')) - if result['format'] != 'project_execution_approval_v1': - raise WorkbenchError('Invalid project approval observation.') - for key in ('origin_catalog_id','approval_id','consumer_actor_id','approver_actor_id'): - identifier(result[key], key) - if type(result['material_sha256']) is not str or not re.fullmatch('[a-f0-9]{64}', result['material_sha256']): - raise WorkbenchError('Invalid approved material identity.') - natural(result['created_at'], 'Approval time');natural(result['expires_at'], 'Approval expiry') - if not result['created_at'] < result['expires_at'] <= result['created_at'] + 3600: - raise WorkbenchError('Invalid approval observation lifetime.') - return True - closed(result, ('format','origin_catalog_id','run_id','request_sha256')) - if result['format'] != 'project_run_admission_v1': - raise WorkbenchError('Invalid project Run observation.') - identifier(result['origin_catalog_id'], 'Run origin');identifier(result['run_id'], 'Run ID') - if type(result['request_sha256']) is not str or not re.fullmatch('[a-f0-9]{64}', result['request_sha256']): - raise WorkbenchError('Invalid Run request digest.') - if records is not None: - matches = [item['document']['run'] for item in records if item['family'] == 'run_history' - and item['id'] == result['origin_catalog_id']+'.'+result['run_id']] - if len(matches) != 1 or _admission_result(matches[0]) != result: - raise WorkbenchError('Run operation observation differs from its retained Run.') - return True - - -class ProjectExecution: - def __init__(self, projection): - self.projection = projection - self.session, self.db = projection.session, projection.db - self.access = projection.access - self.custody = getattr(projection, 'custody', None) - local=self.custody.snapshot() if self.custody is not None else dict(runtime_effects=[],package_effects=[]) - self._runtime_pending={item['request_key']:item for item in local['runtime_effects']} - self._package_pending={item['request_key']:item for item in local['package_effects']} - self._runtime_active=self._package_active=None - self._journal_publication_pending=False - self.runtime_access = Access(self.db, self.db.path, authority=self.access) - self.session.runtime_access = self.runtime_access - self._runs = None - self._constructing = False - self._reconstructing = False - self._dirty_runs = set() - self._pending_studies = set() - self._pending_producers = {kind:set() for kind in ('generation','grounded','caption','text_proposal')} - self.study_execution = None - self.producer_execution = None - self._custody_lock = threading.RLock() - self._handles = {} - self._active_attempts = set() - from project_native_execution import ProjectNativeExecution - self.native=ProjectNativeExecution(self) - self.session.extra_capture = self.capture_dirty - if self.custody is not None:self.session.publication_intent=self._publication_intent - self.db.set_project_hooks(self.before_commit, self.outcome) - - def attach_runs(self, owner): - if not self._constructing or owner.project_execution is not self or self._runs is not None: - raise RuntimeError('Only explicit local construction can attach a Run owner.') - self._runs = owner - - @contextmanager - def _journal_guard(self): - """A failed private custody acknowledgement fences every SQL outcome.""" - try: - yield - except BaseException: - self.session.available = False - if self._runs is not None:self._runs._fence_coordinator() - raise - - def attempt_started(self, identity): - with self._custody_lock:self._active_attempts.add(identity) - - def attempt_finished(self, identity): - with self._custody_lock:self._active_attempts.discard(identity) - - def register_custody(self, identity, handle): - if handle.spec['attempt_id'] != identity: - raise RuntimeError('Current custodian differs from its Run.') - with self._custody_lock: - if identity in self._handles: - raise RuntimeError('A current Run cannot replace its custodian.') - self._handles[identity] = handle - if self.custody is not None: - from recipe_process import identity as process_identity - with self._journal_guard(): - self.custody.launched(identity,process_identity(handle.process.pid)) - - def before_launch(self, identity): - with self.db.lock: - self.session.check_current();run=self.runs._load(identity);self._proof(run) - self.runtime_access.require_job('runs',identity,run['admission_id'],run['request_sha256']) - if self.custody is not None: - with self._journal_guard():self.custody.launch_intent(identity) - - def _publication_intent(self, before, records, key, sha): - if self.custody is not None and self.custody.has_pending: - with self._journal_guard():self.custody.expect_publication(before,records,key,sha) - self._journal_publication_pending=True - - def _local_owner(self, *, recover=True): - """Construct execution only after the caller checked exact authority.""" - self.session.check_current() - if self._runs is None: - from runs import Runs - from pipeline import Pipelines - context = copy.copy(self.projection.context) - context.access, context.lock = self.runtime_access, self.db.lock - context.project_execution = self - pipelines = Pipelines(context) - context.pipelines = pipelines - self._constructing = True - try: - with self.runtime_access._reconcile(): - owner = Runs(context, pipelines, recover=recover) - finally: - self._constructing = False - self._runs = owner - context.runs = owner - return self._runs - - @property - def runs(self): - """An already activated owner, never an implicit constructor.""" - if self._runs is None: - raise WorkbenchError('No current local execution owner is active.', 'unavailable', 409) - return self._runs - - @property - def has_local_custody(self): - """Observe only current physical owners and their unresolved local index.""" - with self._custody_lock: - return (self._runs is not None or bool(self._handles) - or self.native.has_local_custody - or self.producer_execution is not None and self.producer_execution.has_local_custody - or bool(self._runtime_pending or self._package_pending or self._runtime_active or self._package_active) - or self.custody is not None and self.custody.has_pending) - - @contextmanager - def runtime_inspection(self): - with self.access.lock,self.session.lock,self.db.lock: - self.assert_quiescent() - self._runtime_active='inspection' - try:yield - finally:self._runtime_active=None - - def runtime_effect(self, request_key, input_sha256=None, manifest_sha256=None): - return self._file_effect('runtime',request_key,input_sha256,manifest_sha256) - - def package_effect(self, request_key, input_sha256=None, material_sha256=None): - return self._file_effect('package',request_key,input_sha256,material_sha256) - - @contextmanager - def package_reconciliation(self, request_key, input_sha256, material_sha256, expected_checkpoint): - """Rebind one stopped original physical package writer; never rerun it. - - The caller publishes its fixed final/abandonment fact before confirming - this guard. Imported intent alone cannot select or clear local custody. - """ - with self._file_reconciliation('package',request_key,input_sha256,material_sha256,expected_checkpoint) as effect: - yield effect - - @contextmanager - def runtime_reconciliation(self, request_key, input_sha256, manifest_sha256, expected_checkpoint): - """Hold one original stopped runtime capture for a canonical owner decision.""" - with self._file_reconciliation('runtime',request_key,input_sha256,manifest_sha256,expected_checkpoint) as effect: - yield effect - - @contextmanager - def _file_reconciliation(self, kind, request_key, input_sha256, content_sha256, expected_checkpoint): - with self.access.lock,self.session.lock,self.db.lock: - self._restore_file_writer(kind,request_key,input_sha256,content_sha256,expected_checkpoint) - if kind=='runtime':self._runtime_active=request_key - else:self._package_active=request_key - effect=_FileEffect(self,kind,request_key,input_sha256,content_sha256) - effect.started=True - try: - yield effect - finally: - if kind=='runtime':self._runtime_active=None - else:self._package_active=None - - def runtime_pending(self, request_key): - """Observe one current-host private runtime marker, never an imported intent.""" - self.access.require('admin','project_execution.runtime_pending',owner_only=True) - if type(request_key) is not str or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}',request_key): - raise WorkbenchError('Invalid current runtime publication key.') - with self.access.lock,self.session.lock,self.db.lock: - self.access.require('admin','project_execution.runtime_pending',owner_only=True) - if self.custody is None:return None - with self._journal_guard():journal=self.custody.refresh() - return next((copy.deepcopy(entry) for entry in journal['runtime_effects'] - if entry['request_key']==request_key),None) - - def prepare_runtime_reconciliation(self, request_key, input_sha256, manifest_sha256, expected_checkpoint): - """Restore one stopped capture before an explicit normal owner retry. - - No active file guard or marker is changed. The caller must still verify - its actual target and acknowledge normal canonical completion. - """ - return self._restore_file_writer('runtime',request_key,input_sha256,manifest_sha256,expected_checkpoint) - - def _restore_file_writer(self, kind, request_key, input_sha256, content_sha256, expected_checkpoint): - # Exactly the two admitted synchronous filesystem owners share this - # physical-custody proof. There is no extensible effect registry. - if kind=='runtime': - authority,scope,action=self.access,'admin','project_execution.runtime_reconciliation' - field='manifest_sha256' - elif kind=='package': - authority,scope,action=self.runtime_access,'cancel','project_execution.package_reconciliation' - field='material_sha256' - else:raise RuntimeError('Unsupported file reconciliation owner.') - authority.require(scope,action,owner_only=True) - if type(request_key) is not str or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}',request_key): - raise WorkbenchError('Invalid current file publication key.') - for value in (input_sha256,content_sha256,expected_checkpoint): - if type(value) is not str or not re.fullmatch('[a-f0-9]{64}',value): - raise WorkbenchError('File reconciliation requires exact input and checkpoint digests.') - with self.access.lock,self.session.lock,self.db.lock: - authority.require(scope,action,owner_only=True) - if self.db.in_transaction: - raise RuntimeError('File reconciliation requires no outer SQL transaction.') - if self.session.summary['checkpoint']!=expected_checkpoint: - raise WorkbenchError('Expected current file checkpoint changed.','stale_project',409) - if self.custody is None: - raise WorkbenchError('No original physical file custody exists.','unavailable',409) - if self._runtime_active or self._package_active: - raise WorkbenchError('A current runtime or package writer is active.','busy',409) - expected=dict(request_key=request_key,input_sha256=input_sha256,**{field:content_sha256}) - with self._journal_guard():journal=self.custody.refresh() - own,other=(('runtime_effects','package_effects') if kind=='runtime' else ('package_effects','runtime_effects')) - pending=self._runtime_pending if kind=='runtime' else self._package_pending - other_pending=self._package_pending if kind=='runtime' else self._runtime_pending - if expected not in journal[own]: - raise WorkbenchError('No exact original file writer matches this request.','unavailable',409) - if (journal['runs'] or journal['producers'] or journal.get('native_phases') or journal.get('native_admissions') or journal[other] - or journal[own]!=[expected] or other_pending - or any(key!=request_key or value!=expected for key,value in pending.items())): - raise WorkbenchError('Other original local custody remains unresolved.','outcome_unknown',409) - self._assert_run_drained() - self.native.assert_quiescent() - if self.producer_execution is not None:self.producer_execution.assert_quiescent() - binding=self.custody.binding - if (binding['projection_id']!=self.db.projection_id or binding['project_id']!=self.db.dataset_id - or binding['host_catalog_id']!=self.access.catalog_id): - raise WorkbenchError('File custody belongs to another physical owner.','local_custody_unavailable',409) - # Recovery remains fenced until every fixed inert owner has been - # reconstructed from this exact acknowledged/attempted selection. - self.session.available=False - snapshot=self.session.store.recover_checkpoint() - with self.session.store.root_directory() as descriptor: - info=os.fstat(descriptor) - if [info.st_dev,info.st_ino]!=binding['root_identity']: - raise WorkbenchError('File custody belongs to another physical root.','local_custody_unavailable',409) - self.custody.validate_selection(snapshot) - self.projection.reconcile_inert(snapshot) - pending[request_key]=expected - self.assert_quiescent(_allow_file=(kind,request_key)) - return self.session.snapshot - - @contextmanager - def _file_effect(self, kind, key, input_sha256, content_sha256): - if type(key) is not str or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}',key): - raise WorkbenchError('Invalid current file publication key.') - for value in (input_sha256,content_sha256): - if value is not None and (type(value) is not str or not re.fullmatch('[a-f0-9]{64}',value)): - raise WorkbenchError('Invalid current file publication digest.') - with self.access.lock,self.session.lock,self.db.lock: - self.assert_quiescent(_allow_file=(kind,key)) - if kind=='runtime':self._runtime_active=key - else:self._package_active=key - effect=_FileEffect(self,kind,key,input_sha256,content_sha256) - try:yield effect - finally: - if kind=='runtime':self._runtime_active=None - else:self._package_active=None - - def mark_run(self, run_id): - identifier(run_id, 'Dirty Run ID') - if not self.db.in_transaction: - raise RuntimeError('Run changes require their owned SQL transaction.') - self._dirty_runs.add(run_id) - - def mark_study(self, study_id): - identifier(study_id, 'Dirty Study ID') - if not self.db.in_transaction: - raise RuntimeError('Study changes require their owned SQL transaction.') - if self.study_execution is None: - self._pending_studies.add(study_id) - else: - self.study_execution.mark_study(study_id) - - def _mark_producer(self, kind, identity): - identifier(identity, 'Dirty producer ID') - if not self.db.in_transaction: - raise RuntimeError('Producer changes require their owned SQL transaction.') - if self.producer_execution is None: - self._pending_producers[kind].add(identity) - else: - getattr(self.producer_execution, 'mark_'+kind)(identity) - - def mark_generation(self, identity):self._mark_producer('generation', identity) - def mark_grounded(self, identity):self._mark_producer('grounded', identity) - def mark_caption(self, identity):self._mark_producer('caption', identity) - def mark_text_proposal(self, identity):self._mark_producer('text_proposal', identity) - - def _producer_authority(self, family, identity, admission_id, request_digest): - from project_producer_execution import JOB_KINDS, FINISH - from access import current_context - if family not in JOB_KINDS:raise WorkbenchError('Unsupported producer custody family.') - row=self.db.execute('SELECT * FROM access_admissions WHERE id=%s',(admission_id,)).fetchone() - if (row is None or row['catalog_id']!=self.db.dataset_id or row['kind']!=JOB_KINDS[family] - or row['job_id']!=identity or row['request_digest']!=request_digest): - raise AccessError('Producer custody requires its exact current local admission.') - context=current_context() - if context is not None and context.mode=='job': - self.runtime_access.require_job(JOB_KINDS[family],identity,admission_id,request_digest) - self.runtime_access.require('edit',FINISH[family],identity) - elif context is not None and context.mode=='system': - self.runtime_access.require('cancel',JOB_KINDS[family]+'.reconcile',identity) - else:self.runtime_access.require('inspect','producers.custody',identity) - - def register_producer(self, family, identity, admission_id, request_digest): - with self.db.lock: - self.session.check_current();self._producer_authority(family,identity,admission_id,request_digest) - if self.custody is not None: - with self._journal_guard(): - if not self.custody.has_pending:self.custody.acknowledge(self.session.snapshot) - self.custody.register_producer(family,identity,admission_id,request_digest) - - def producer_effect_intent(self, family, identity): - with self.db.lock: - self.session.check_current() - if self.custody is not None: - value=next((item for item in self.custody.snapshot()['producers'] - if (item['family'],item['producer_id'])==(family,identity)),None) - if value is None:raise AccessError('Provider dispatch has no original-host custody admission.') - self._producer_authority(family,identity,value['admission_id'],value['request_sha256']) - with self._journal_guard():self.custody.producer_effect_intent(family,identity) - - def update_producer_outcome(self, family, identity, provider_outcome, worker_drained): - with self.db.lock: - self.session.check_current() - if self.custody is not None: - value=next((item for item in self.custody.snapshot()['producers'] - if (item['family'],item['producer_id'])==(family,identity)),None) - if value is None:return - self._producer_authority(family,identity,value['admission_id'],value['request_sha256']) - with self._journal_guard():self.custody.producer_outcome(family,identity,provider_outcome,worker_drained) - - def _proof(self, run): - admission = self.db.execute('SELECT * FROM access_admissions WHERE id=%s', (run['admission_id'],)).fetchone() - if admission is None: - raise AccessError('Current local Run admission is unavailable.') - admission = dict(admission) - admission['effects'] = json.loads(admission.pop('effects_json')) - if (admission['catalog_id'] != self.db.dataset_id or admission['kind'] != 'runs' - or admission['job_id'] != run['id'] or admission['request_digest'] != run['request_sha256']): - raise AccessError('Current local admission differs from its Run.') - grant = consumption = None - if run.get('execution_kind') == 'external_recipe': - grant = self.db.execute('SELECT * FROM access_execution_grants WHERE id=%s', (run['approval_id'],)).fetchone() - consumption = self.db.execute('SELECT * FROM access_execution_consumptions WHERE grant_id=%s', (run['approval_id'],)).fetchone() - if grant is None or consumption is None: - raise AccessError('Current local execution consumption is unavailable.') - return dict(grant=dict(grant) if grant is not None else None, - consumption=dict(consumption) if consumption is not None else None, admission=admission) - - def _authorize_observation(self): - self.native.authorize_dirty() - context = self.runtime_access._context() if self._dirty_runs else None - for identity in self._dirty_runs: - row = self.db.execute('SELECT document FROM workspace_runs WHERE id=%s', (identity,)).fetchone() - if row is None: - raise AccessError('Current local Run is unavailable.') - run = json.loads(row['document']) - proof = self._proof(run) - if context.mode == 'job': - self.runtime_access.require_job('runs', identity, run['admission_id'], run['request_sha256']) - self.runtime_access.require('edit', 'runs.update', identity) - elif context.mode == 'system': - if self._runs is None and not self._constructing: - raise AccessError('Reconciliation requires the current local execution owner.') - if not self._runs.runtime_owner or not self.db.runtime_owned('workspace_runs'): - raise AccessError('Current local runtime custody ownership is unavailable.') - self.runtime_access.require('cancel', 'runs.reconcile', identity) - else: - # The domain owner has already checked submit/cancel authority; - # recheck live host credentials without inventing a worker owner. - self.runtime_access.require('inspect', 'runs.observe', identity) - if proof['admission']['catalog_id'] != self.session.summary['project_id']: - raise AccessError('Local custody belongs to another project.') - - def _capture_run(self, identity): - owner = self.runs - run = owner._load(identity) - proof = self._proof(run) - payloads = owner._capture_project_payloads(run, self.session.artifacts) - record = capture_run(run, admission_proof=proof, payloads=payloads) - records = [record, *declarations(record)] - if run.get('execution_kind') == 'external_recipe': - environment = run['material'].get('torch_runtime', run['material'].get('cpu_snapshot', run['material']['runtime'])) - descriptor = environment if environment.get('profile') in ('captured_cpu_python_v1','managed_torch_python_v1') else None - sha = digest(environment) - ref = dict(kind='runtime', catalog_id=run['dataset_id'], id=sha, sha256=sha) - from project_material import decode_records - if ('runtime_material', ref['catalog_id']+'.'+sha) not in decode_records(self.session.snapshot['records']): - raw = self.projection.runtime_manifest(descriptor) if descriptor is not None else None - manifest = None - if raw is not None: - from recipe_process import decode - manifest = decode(raw) - runtime = self.projection.materials.capture_runtime(dict(ref=ref, descriptor=descriptor, - environment=None if descriptor is not None else environment, manifest=manifest), raw) - records.extend([runtime, *self.projection.materials.declarations(runtime)]) - if self.custody is not None and run.get('execution_kind')=='external_recipe': - with self._journal_guard(): - if not self.custody.has_pending:self.custody.acknowledge(self.session.snapshot) - receipt=next((edge['blob']['sha256'] for edge in payloads if edge['name']=='custody-receipt.json'),None) - self.custody.observe_run(run,receipt_sha256=receipt) - elif self.custody is not None: - with self._journal_guard(): - if not self.custody.has_pending:self.custody.acknowledge(self.session.snapshot) - if run['state'] in ('queued','running','cancelling','unknown'): - self.custody.observe_native_admission(run) - return records - - def capture_dirty(self, before_records): - """The finite owner capture seam used by foreground and background TXs.""" - if self._reconstructing: - return [] - if self._pending_studies or any(self._pending_producers.values()): - raise WorkbenchError('The changed execution family has no installed capture owner.', 'unavailable', 409) - if self.study_execution is not None: - self.study_execution.settle_runs(self._dirty_runs) - self._authorize_observation() - records = [] - for identity in sorted(self._dirty_runs): - records.extend(self._capture_run(identity)) - if self.study_execution is not None: - records.extend(self.study_execution.capture_dirty(before_records)) - if self.producer_execution is not None: - records.extend(self.producer_execution.capture_dirty(before_records)) - records.extend(self.native.capture_dirty(before_records)) - # Shared immutable blobs can appear on several domain edges. - merged = {} - for record in records: - identity = record['family'], record['id'] - if identity in merged and merged[identity] != record: - raise WorkbenchError('Execution captures disagree on shared project facts.') - merged[identity] = record - return list(merged.values()) - - def _capture_pending(self): - """Whether capture_dirty could return any record (see its owners).""" - if self._dirty_runs or self._pending_studies or any(self._pending_producers.values()) or self.native.has_dirty: - return True - study = self.study_execution - if study is not None and (study._dirty or study._package_records): - return True - producers = self.producer_execution - if producers is not None and (producers._runtime_data is not None or any(producers._dirty.values())): - return True - return False - - def before_commit(self): - if self._reconstructing or self.session.foreground_publication: - return - if not self._dirty_runs and not self._pending_studies and not any(self._pending_producers.values()) and not self.native.has_dirty and self.study_execution is None and self.producer_execution is None: - return - if not self._capture_pending(): - # Nothing can be captured; never materialize the record set. - return - snapshot = self.session.snapshot - records = self.capture_dirty(snapshot['records']) - if records: - key = 'execution-observation-' + uuid.uuid4().hex - sha = hashlib.sha256(agent_api.encoded(records)).hexdigest() - self.session._publish_observation(records, key, sha, authorize=self._authorize_observation) - - def outcome(self, outcome): - if outcome not in ('committed','rolled_back','unknown'): - raise ValueError('Unsupported project SQL outcome.') - if self._reconstructing: - return - self.session._observation_outcome(outcome) - self.native.outcome(outcome) - if self.custody is not None and outcome=='committed': - try: - if self._journal_publication_pending:self.custody.acknowledge(self.session.snapshot) - for identity in sorted(self._dirty_runs): - run=self.runs._load(identity) - self.native.settle_run(run) - if run.get('execution_kind')!='external_recipe' and run['state'] in ('completed','failed','cancelled','interrupted'): - if any(row['run_id']==identity for row in self.custody.snapshot().get('native_admissions',[])): - self.custody.settle_native_admission(identity) - entry=next((item for item in self.custody.snapshot()['runs'] if item['run_id']==identity),None) - if entry is not None and run['state'] in ('completed','failed','cancelled','interrupted') and ( - run['custody_phase']=='drained' and entry['receipt_sha256'] is not None - or entry['phase']=='admitted' and run.get('custody_identity') is None): - self.custody.settle_run(identity) - except BaseException: - self.session.available=False - if self._runs is not None:self._runs._fence_coordinator() - raise - self._journal_publication_pending=False - self._dirty_runs.clear();self._pending_studies.clear() - for values in self._pending_producers.values():values.clear() - for adapter in (self.study_execution, self.producer_execution): - if adapter is not None:adapter.outcome(outcome) - if outcome == 'unknown':self.session.available = False - if not self.session.available: - if self._runs is not None:self._runs._fence_coordinator() - - def abandon_pending(self): - """Discard failed capture bookkeeping, never an admission or grant.""" - if not self._reconstructing: - raise RuntimeError('Pending execution capture may clear only during guarded inert reconstruction.') - self._dirty_runs.clear();self._pending_studies.clear() - self.native.abandon_pending() - for values in self._pending_producers.values():values.clear() - - def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capture, *, scope='edit'): - try: - return self.session.mutate(operation, target, body, expected_checkpoint, key, invoke, capture, scope=scope) - except BaseException: - if not self.session.available and self._runs is not None:self._runs._fence_coordinator() - raise - - def preflight_external(self, graph_id, revision): - self.runtime_access.require('inspect', 'pipeline.preflight_external', graph_id) - with self.access.lock, self.session.lock, self.db.lock: - self.session.check_current() - return self.projection.pipelines.preflight_external(graph_id, revision) - - def native_preflight(self,graph_id,revision,expected_checkpoint,key): - self.runtime_access.require('execute','runs.native_preflight',graph_id,owner_only=True) - return self.projection.native_preparation.validate_graph(self.projection.pipelines, - graph_id,revision,expected_checkpoint,key) - - def _native_replay(self,operation,target,body,expected_checkpoint,key): - prior=session_operations(self.session).get(key) - if prior is None:return None - if (prior['operation']!=operation or self.db.execute('SELECT id FROM workspace_runs WHERE id=%s', - (prior['result'].get('run_id'),)).fetchone() is None): - raise WorkbenchError('A retained native receipt cannot create current local execution.','unavailable',409) - def forbidden():raise RuntimeError('Native admission replay attempted another qualification.') - return self.mutate(operation,target,body,expected_checkpoint,key,forbidden, - lambda value,before:dict(records=[],result=value),scope='execute') - - def native_submit(self,body,expected_checkpoint,key): - from pipeline import closed - closed(body,('graph_id','graph_revision','validation_token')) - actor=self.runtime_access.require('execute','runs.submit',body['graph_id'],owner_only=True) - effective=dict(body,actor_id=actor.actor_id) - with self.db.lock: - self.session.check_current() - replay=self._native_replay('runs.submit',body['graph_id'],effective,expected_checkpoint,key) - if replay is not None:return replay - with self.projection.native_preparation.current_graph(self.projection.pipelines,body['graph_id'], - body['graph_revision'],expected_checkpoint,key) as context: - if context.validation['validation_token']!=body['validation_token']: - raise WorkbenchError('Native validation is stale. Explicitly validate this saved graph.','stale',409) - def invoke(): - context.consume_admission() - owner=self._local_owner();public=owner.submit(dict(body,request_key=key)) - return _admission_result(owner._load(public['id'])) - return self.mutate('runs.submit',body['graph_id'],effective,self.session.summary['checkpoint'],key, - invoke,lambda value,before:dict(records=[],result=value),scope='execute') - - def native_retry(self,run_id,expected_checkpoint,key): - actor=self.runtime_access.require('execute','runs.retry',run_id,owner_only=True) - body=dict(actor_id=actor.actor_id) - with self.db.lock: - self.session.check_current() - replay=self._native_replay('runs.retry',run_id,body,expected_checkpoint,key) - if replay is not None:return replay - if self._runs is None:raise WorkbenchError('Retry requires the actual current local Run owner.','unavailable',409) - parent=self._runs._load(run_id) - if self._runs._external(parent) or parent['state'] not in ('failed','cancelled','interrupted'): - raise WorkbenchError('Choose a drained failed/cancelled/interrupted current native Run.','conflict',409) - with self.projection.native_preparation.current_graph(self.projection.pipelines,parent['graph']['id'], - parent['graph_revision'],expected_checkpoint,key) as context: - def invoke(): - context.consume_admission() - owner=self.runs;public=owner.retry(run_id,dict(request_key=key)) - return _admission_result(owner._load(public['id'])) - return self.mutate('runs.retry',run_id,body,self.session.summary['checkpoint'],key, - invoke,lambda value,before:dict(records=[],result=value),scope='execute') - - def native_reconcile(self,phase_id,expected_checkpoint): - return self.native.reconcile(phase_id,expected_checkpoint) - - def submit_external(self, body, expected_checkpoint, key): - authority = self.runtime_access.require_external_submission() - with self.access.lock, self.session.lock, self.db.lock: - self.session.check_current() - prior = session_operations(self.session).get(key) - if prior is None and expected_checkpoint != self.session.summary['checkpoint']: - raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) - if prior is not None: - # A retained request never recreates a local Run/admission or wakes - # imported queue state. A same-host local receipt can reconcile. - result = prior['result'] - if (prior['operation'] != 'runs.submit_external' or - self.db.execute('SELECT id FROM workspace_runs WHERE id=%s', (result.get('run_id'),)).fetchone() is None): - raise WorkbenchError('This saved request has no current local Run admission.', 'unavailable', 409) - request = dict(body, request_key=key) - def invoke(): - owner = self._local_owner() - public = owner.submit_external(request) - return _admission_result(owner._load(public['id'])) - effective = dict(body, actor_id=authority.actor_id) - return self.mutate('runs.submit_external', body.get('graph_id'), effective, expected_checkpoint, key, - invoke, lambda value,before:dict(records=[], result=value), scope='inspect') - - def approve_external(self, body, expected_checkpoint, key): - authority = self.runtime_access.require('approve-execution', 'runs.approve_external', owner_only=True) - with self.access.lock, self.session.lock, self.db.lock: - self.session.check_current() - prior = session_operations(self.session).get(key) - if prior is None and expected_checkpoint != self.session.summary['checkpoint']: - raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) - if prior is not None: - value = prior['result'] - if prior['operation'] != 'runs.approve_external' or self.db.execute( - 'SELECT id FROM access_execution_grants WHERE id=%s', (value.get('approval_id'),)).fetchone() is None: - raise WorkbenchError('This approval is historical; approve explicitly under current authority.', 'unavailable', 409) - def invoke(): - owner = self._local_owner() - grant = owner.approve_external(body) - return dict(format='project_execution_approval_v1', origin_catalog_id=grant['catalog_id'], - approval_id=grant['id'], **{key:grant[key] for key in ('material_sha256','consumer_actor_id', - 'approver_actor_id','created_at','expires_at')}) - effective = dict(body, approver_actor_id=authority.actor_id) - value = self.mutate('runs.approve_external', body.get('graph_id'), effective, expected_checkpoint, key, - invoke, lambda result,before:dict(records=[], result=result), scope='approve-execution') - grant = self.db.execute('SELECT * FROM access_execution_grants WHERE id=%s', (value['approval_id'],)).fetchone() - if grant is None: - raise WorkbenchError('Current local execution approval is unavailable.', 'unavailable', 409) - fields = ('material_sha256','consumer_actor_id','approver_actor_id','created_at','expires_at') - if (grant['catalog_id'] != value['origin_catalog_id'] or grant['id'] != value['approval_id'] - or any(grant[field] != value[field] for field in fields) - or grant['approver_actor_id'] != authority.actor_id or grant['revoked_at'] is not None - or grant['expires_at'] <= int(time.time())): - raise WorkbenchError('This historical approval has no exact current live grant. Use a fresh key.', 'unavailable', 409) - return dict(format='project_current_execution_approval_v1', current_local=True, grant=dict(grant)) - - def cancel_run(self, run_id, expected_checkpoint, key): - self.runtime_access.require('cancel', 'runs.cancel', run_id) - with self.access.lock, self.session.lock, self.db.lock: - if self._runs is None or self.db.execute('SELECT id FROM workspace_runs WHERE id=%s', (run_id,)).fetchone() is None: - raise WorkbenchError('A historical Run has no current local process to cancel.', 'unavailable', 409) - def invoke(): - self.runs.cancel(run_id) - return _admission_result(self.runs._load(run_id)) - return self.mutate('runs.cancel', run_id, {}, expected_checkpoint, key, invoke, - lambda result,before:dict(records=[], result=result), scope='cancel') - - def health(self, offset=0, limit=40): - self.access.require('inspect', 'project_execution.health') - natural(offset,'Execution offset',0,100000);natural(limit,'Execution limit',1,40) - with self.db.lock: - rows = [json.loads(row['document']) for row in self.db.execute('SELECT document FROM workspace_runs')] - active = {row['id'] for row in rows if row['state'] in ('queued','running','cancelling','unknown')} - with self._custody_lock:active.update(self._active_attempts) - native=self.custody.snapshot().get('native_phases',[]) if self.custody is not None else [] - native=sorted(native,key=lambda item:item['phase_id']) - phases=[dict(format='project_native_phase_v1',origin_catalog_id=self.db.dataset_id, - phase_id=item['phase_id'],phase=item['phase'],run_id=item['run_id'], - state='prepared' if item['state'] in ('launch_intent','launched') else item['state'], - spec_sha256=item['spec_sha256'],receipt_sha256=item['receipt_sha256']) for item in native[offset:offset+limit]] - return dict(format='project_execution_health_v1', active_run_ids=sorted(active)[offset:offset+limit], - active_count=len(active), offset=offset, limit=limit, - native_phases=dict(items=phases,total=len(native),offset=offset,limit=limit), - local_owner_active=self._runs is not None, - available=self.session.available and self.db.dataset_id != self.access.catalog_id, - drain_required=bool(self.custody is not None and self.custody.has_pending - or self.native.has_local_custody - or self._runtime_active or self._package_active - or self._runs is not None and self._runs._thread is not None and self._runs._thread.is_alive())) - - def assert_quiescent(self, *, _allow_file=None): - with self.db.lock: - if not self.session.available: - raise WorkbenchError('Current execution outcome requires explicit reconciliation.', 'projection_unavailable', 503) - if self._runtime_active or self._package_active: - raise WorkbenchError('A current runtime or package file writer is active.','busy',409) - for kind,pending in (('runtime',self._runtime_pending),('package',self._package_pending)): - if any(_allow_file!=(kind,key) for key in pending): - raise WorkbenchError('A current runtime or package publication requires exact reconciliation.','outcome_unknown',409) - self._assert_run_drained() - self.native.assert_quiescent() - for adapter in (self.study_execution,self.producer_execution): - if adapter is not None:adapter.assert_quiescent() - if self.custody is not None: - journal=self.custody.snapshot() - if journal['runs'] or journal['producers'] or journal.get('native_phases') or journal.get('native_admissions'): - raise WorkbenchError('Original local custody remains unresolved.','outcome_unknown',409) - - def _assert_run_drained(self): - """Check only actual current Run effects, including an idle supervisor.""" - rows = self.db.execute("SELECT id FROM workspace_runs WHERE document::jsonb->>'state' IN ('queued','running','cancelling','unknown')").fetchall() - if rows: - raise WorkbenchError('Drain or cancel current local Runs before closing or copying this project.', 'busy', 409) - with self._custody_lock: - if self._active_attempts or any(handle.process.poll() is None for handle in self._handles.values()): - raise WorkbenchError('Current local custodians are still draining.', 'busy', 409) - if self._runs is not None and self._runs._thread is not None and self._runs._thread.is_alive(): - raise WorkbenchError('Reconcile current local custody to stop its idle supervisor before closing.', 'busy', 409) - if self._runs is not None and (self._runs._publication_active is not None or self._runs._publication_lock.locked()): - raise WorkbenchError('Current Run publication has not drained.', 'busy', 409) - - def close(self): - if not self.has_local_custody: - return - self.assert_quiescent() - if self._runs is not None: - with self.runtime_access._reconcile():self._runs.close() - if self.producer_execution is not None:self.producer_execution.close() - - def shutdown(self): - """Explicit application shutdown, outside project command locks. - - Existing local continuations own cancellation and final observations. - This neither constructs an owner nor creates human execution authority. - """ - if self.db.lock.owned_by_current_thread or self.db.in_transaction: - raise WorkbenchError('Shutdown must drain outside the project command lock.', 'busy', 409) - try: - if self._runs is not None: - with self.runtime_access._reconcile():self._runs.close() - finally: - if self.producer_execution is not None:self.producer_execution.shutdown() - - def reconcile_current(self): - """Reconcile only retained current-local custody, never imported handles.""" - self.runtime_access.require('cancel', 'project_execution.reconcile', owner_only=True) - self._require_no_native_phase_reconciliation() - if self._runs is None: - raise WorkbenchError('No current local custody owner exists to reconcile.', 'unavailable', 409) - if self.db.lock.owned_by_current_thread or self.db.in_transaction: - raise WorkbenchError('Drain current custody outside the project command lock.', 'busy', 409) - owner = self._runs - owner._fence_coordinator() - with self._custody_lock:handles = dict(self._handles) - receipts = {} - # Only actual Handle objects retained by this application's launch path - # close control channels. Saved PID/spec/receipt text never selects one. - for identity, handle in handles.items(): - receipt = handle.close_control() - if not receipt or not receipt.get('drained'): - raise WorkbenchError('Current custodian drainage is still unconfirmed.', 'unknown', 409) - receipts[identity] = receipt - deadline = time.monotonic() + 60 - thread = owner._thread - if thread is not None: - thread.join(max(0, deadline-time.monotonic())) - if thread.is_alive(): - raise WorkbenchError('Current execution supervisor has not drained.', 'busy', 409) - with self._custody_lock: - if self._active_attempts: - raise WorkbenchError('Current execution attempts have not drained.', 'busy', 409) - if self.producer_execution is not None:self.producer_execution.assert_quiescent() - with self.access.lock, self.session.lock, self.db.lock: - self.runtime_access.require('cancel','project_execution.reconcile',owner_only=True) - self._require_no_native_phase_reconciliation() - snapshot = self.session.store.recover_checkpoint() - self.projection.reconcile_inert(snapshot) - canonical = {item['document']['run']['id']:item['document']['run'] for item in snapshot['records'] - if item['family']=='run_history' and item['document']['origin_catalog_id']==self.db.dataset_id} - reconciled = [] - try: - with self.runtime_access._reconcile(), self.db.transaction(): - for row in self.db.execute('SELECT id FROM workspace_runs'): - local = owner._load(row['id']) - if local['state'] not in ('queued','running','cancelling','unknown'): - continue - self._proof(local) - retained = canonical.get(local['id']) - if retained is None or any(retained.get(key)!=local.get(key) for key in ( - 'dataset_id','id','request_sha256','admission_id','approval_id','material_sha256','spec')): - raise WorkbenchError('Current local admission differs from canonical custody.', 'unknown', 409) - run = copy.deepcopy(retained) - handle = handles.get(run['id']) - if run.get('execution_kind')!='external_recipe': - # No native phase can launch without first entering - # the private phase journal. A drained original - # supervisor plus a never-started native admission - # may therefore terminate without inventing receipt - # or interpreting an external custody specification. - self._native_never_started(local,retained) - owner._event(run,'interrupted','Original native admission ended before phase dispatch; execution was not replayed.') - from runs import LOCAL_RUN_FIELDS - run.update({key:copy.deepcopy(value) for key,value in local.items() if key in LOCAL_RUN_FIELDS}) - owner._save(run);reconciled.append(run['id']);continue - if handle is None: - if run['custody_phase']!='prepared' or run.get('custody_identity') is not None: - raise WorkbenchError('Canonical execution has no current local custody proof.', 'unknown', 409) - state = 'cancelled' - else: - receipt = receipts[run['id']] - if (handle.expected_identity['attempt_id']!=run['id'] or - handle.expected_identity['request_sha256']!=run['request_sha256'] or - run.get('custody_identity') not in (None,handle.expected_identity)): - raise WorkbenchError('Current custodian differs from its canonical admission.', 'unknown', 409) - run['custody_identity'] = copy.deepcopy(handle.expected_identity) - run['custody_phase'] = 'drained' - if run['state']=='completed': - owner._verify_external_artifacts(run) - state = 'completed' - else: - state = 'cancelled' if receipt['outcome']=='cancelled' else 'interrupted' - owner._event(run,state,'Explicit current-host reconciliation confirmed custody drainage; execution was not replayed.') - owner._save(run) - reconciled.append(run['id']) - # A foreground canonical admission may have survived a rolled - # back SQL admission. Clearing that original non-effect intent - # neither restores SQL authority nor rewrites historical state. - for entry in self.custody.snapshot().get('native_admissions',[]) if self.custody is not None else []: - if self.db.execute('SELECT id FROM workspace_runs WHERE id=%s',(entry['run_id'],)).fetchone() is None: - self._native_unacknowledged(entry,canonical.get(entry['run_id'])) - with self._journal_guard():self.custody.settle_native_admission(entry['run_id']) - self.assert_quiescent() - except BaseException: - self.session.available = False - owner._fence_coordinator() - raise - return dict(format='project_execution_reconciliation_v1', origin_catalog_id=self.db.dataset_id, - run_ids=sorted(reconciled), available=self.session.available) - - def _require_no_native_phase_reconciliation(self): - if self.native.has_local_custody or self.custody is not None and self.custody.snapshot().get('native_phases'): - raise WorkbenchError('Reconcile the exact current native phase with native_reconcile before generic Run reconciliation.','unavailable',409) - - def _native_never_started(self,local,retained): - """Prove absence of phase dispatch from the original private journal.""" - entry=next((row for row in self.custody.snapshot().get('native_admissions',[]) - if row['run_id']==local['id']),None) if self.custody is not None else None - if (entry is None or entry['phase_started'] or local.get('process_phase')!='not_started' - or local.get('worker') is not None or local.get('execution_kind')=='external_recipe' - or any(retained.get(key)!=local.get(key) for key in - ('id','dataset_id','admission_id','request_sha256','graph_sha256','release_sha256','runtime')) - or entry['admission_id']!=local['admission_id'] or entry['request_sha256']!=local['request_sha256'] - or entry['graph_sha256']!=local['graph_sha256'] or entry['release_sha256']!=local['release_sha256'] - or entry['runtime_sha256']!=digest(local['runtime'])): - raise WorkbenchError('Native admission lacks original never-dispatched phase proof.','unknown',409) - - def _native_unacknowledged(self,entry,retained): - if entry['phase_started']: - raise WorkbenchError('A native phase may have dispatched; its current receipt is required.','unknown',409) - if self.db.execute('SELECT id FROM access_admissions WHERE id=%s',(entry['admission_id'],)).fetchone() is not None: - raise WorkbenchError('An incomplete original native SQL admission needs investigation.','unknown',409) - if retained is not None and (retained.get('execution_kind')=='external_recipe' or retained['state']!='queued' - or any(entry[key]!=retained[key] for key in ('admission_id','request_sha256','graph_sha256','release_sha256')) - or entry['runtime_sha256']!=digest(retained['runtime'])): - raise WorkbenchError('Original unacknowledged native admission differs from canonical history.','unknown',409) - - def reconcile_original(self, *, expected_checkpoint, timeout=15): - """Explicit original-host receipt reconciliation, without PID control. - - The Workspace supplies only an indexed existing physical projection. - Imported history never selects that projection or creates admissions. - No Run owner exists until all journal/local-SQL/receipt proofs pass. - """ - self.runtime_access.require('cancel','project_execution.reconcile_original',owner_only=True) - self._require_no_native_phase_reconciliation() - if self.custody is None or self._runs is not None: - raise WorkbenchError('Original custody requires its indexed inactive physical projection.','local_custody_unavailable',409) - if self.db.lock.owned_by_current_thread or self.db.in_transaction: - raise WorkbenchError('Original custody must drain outside command locks.','busy',409) - if type(timeout) not in (int,float) or not 0<=timeout<=30: - raise WorkbenchError('Original receipt wait must be zero through thirty seconds.') - journal=self.custody.snapshot();snapshot=self.session.store.recover_checkpoint() - if snapshot['checkpoint']!=expected_checkpoint: - raise WorkbenchError('Expected original project checkpoint changed.','stale_project',409) - self.custody.validate_selection(snapshot) - canonical={item['document']['run']['id']:item['document']['run'] for item in snapshot['records'] - if item['family']=='run_history' and item['document']['origin_catalog_id']==self.db.dataset_id} - plans=[];unacknowledged=[];receipts={} - with self.db.lock: - if (journal['runs'] or journal.get('native_admissions')) and not self.db.claim_runtime('workspace_runs'): - raise WorkbenchError('The original Run application still owns this physical projection.','busy',409) - from project_producer_execution import TABLES as PRODUCER_TABLES - for producer in journal['producers']: - if not self.db.claim_runtime(PRODUCER_TABLES[producer['family']]): - raise WorkbenchError('An original producer application still owns this physical projection.','busy',409) - if producer['phase']!='admitted' or producer['provider_outcome']!='not_admitted': - raise WorkbenchError('Original provider outcome remains unknown; a closed local worker is not cessation proof.','outcome_unknown',409) - for entry in journal['runs']: - row=self.db.execute('SELECT document FROM workspace_runs WHERE id=%s',(entry['run_id'],)).fetchone() - if row is None: - if entry['phase']!='admitted': - raise WorkbenchError('Launched custody has no original local admission.','local_custody_unavailable',409) - unacknowledged.append(entry['run_id']);continue - local=json.loads(row['document']);retained=canonical.get(entry['run_id']) - self._proof(local) - from recipe_process import encoded - if (retained is None or local['id']!=entry['run_id'] or local['admission_id']!=entry['admission_id'] - or local['request_sha256']!=entry['request_sha256'] or local['dataset_id']!=self.db.dataset_id - or hashlib.sha256(encoded(local['spec'])).hexdigest()!=entry['spec_sha256'] - or any(retained.get(key)!=local.get(key) for key in ( - 'id','dataset_id','admission_id','request_sha256','approval_id','material_sha256','spec')) - or local['spec']['cwd']!=str(self.db.path/'workspace-runs'/('.external-'+local['id']))): - raise WorkbenchError('Original local admission/specification differs from its indexed canonical observation.','local_custody_unavailable',409) - if entry['phase']!='admitted' and entry['custody_identity'] is None: - raise WorkbenchError('Original spawn window lacks an observed helper identity; cessation remains unknown.','unknown',409) - plans.append((entry,copy.deepcopy(retained))) - native_plans=[] - for entry in journal.get('native_admissions',[]): - row=self.db.execute('SELECT document FROM workspace_runs WHERE id=%s',(entry['run_id'],)).fetchone() - if row is None: - self._native_unacknowledged(entry,canonical.get(entry['run_id'])) - unacknowledged.append(entry['run_id']);continue - local=json.loads(row['document']);retained=canonical.get(entry['run_id']) - self._proof(local) - if retained is None:raise WorkbenchError('Original native Run is missing its canonical admission.','unknown',409) - self._native_never_started(local,retained) - native_plans.append((local,copy.deepcopy(retained))) - # Poll only exact original local receipt files. Never signal a stored PID, - # import a controller handle, replay GO or execute a project script. - if any(entry['custody_identity'] is not None for entry,run in plans): - from projects import Projects - from recipe_process import validate_receipt - reader=Projects(self.projection.context,{'original_runs':self.db.path/'workspace-runs'}) - deadline=time.monotonic()+timeout - try: - for entry,run in plans: - if entry['custody_identity'] is None:continue - while True: - try: - raw=reader._read('original_runs','.external-'+run['id']+'/custody/receipt.json',16384) - receipt=validate_receipt(raw,entry['custody_identity']) - if not receipt['drained']: - raise WorkbenchError('Original custodian has not established drainage.','unknown',409) - receipts[run['id']]=receipt;break - except FileNotFoundError: - if time.monotonic()>=deadline: - raise WorkbenchError('Original exact drain receipt is not available yet.','unknown',409) from None - time.sleep(min(.1,max(0,deadline-time.monotonic()))) - finally:reader.close() - with self.access.lock,self.session.lock,self.db.lock: - self.runtime_access.require('cancel','project_execution.reconcile_original',owner_only=True) - self.custody.validate_selection(self.session.store.snapshot()) - self.projection.reconcile_inert(snapshot) - owner=self._local_owner(recover=False) if plans or native_plans else None - if owner is not None:owner._fence_coordinator() - try: - if owner is not None: - # Verification uses the current human's existing disclosure - # policy. The following system scope owns only terminal SQL. - for entry,run in plans: - if run['state']=='completed':owner._verify_external_artifacts(run) - with self.runtime_access._reconcile(),self.db.transaction(): - for entry,run in plans: - if entry['custody_identity'] is not None: - run['custody_identity']=copy.deepcopy(entry['custody_identity']);run['custody_phase']='drained' - state=run['state'] if run['state']=='completed' else ( - 'cancelled' if receipts[run['id']]['outcome']=='cancelled' else 'interrupted') - else:state='cancelled' - owner._event(run,state,'Original-host exact local custody reconciled; execution was not resumed.') - owner._save(run) - for local,run in native_plans: - from runs import LOCAL_RUN_FIELDS - run.update({key:copy.deepcopy(value) for key,value in local.items() if key in LOCAL_RUN_FIELDS}) - owner._event(run,'interrupted','Original native admission ended before phase dispatch; execution was not resumed.') - owner._save(run) - for identity in unacknowledged: - if any(row['run_id']==identity for row in journal.get('native_admissions',[])): - self.custody.settle_native_admission(identity) - else:self.custody.settle_run(identity) - for producer in journal['producers']: - self.custody.producer_outcome(producer['family'],producer['producer_id'],'not_admitted',True) - self.custody.acknowledge(self.session.snapshot) - except BaseException: - self.session.available=False - if owner is not None:owner._fence_coordinator() - raise - return dict(format='project_execution_reconciliation_v1',origin_catalog_id=self.db.dataset_id, - run_ids=sorted([run['id'] for entry,run in plans]+[run['id'] for local,run in native_plans]),available=self.session.available) diff --git a/project_extraction.py b/project_extraction.py deleted file mode 100644 index 461b0d0f..00000000 --- a/project_extraction.py +++ /dev/null @@ -1,836 +0,0 @@ -"""Finite read-only legacy capture for an explicitly selected project. - -The caller owns the repeatable-read PostgreSQL transaction, pinned directory -capabilities and stopped-source proof. This module never opens a catalog, -constructs runtime owners, enrolls records or writes source metadata. Scientific -shapes are admitted by their existing project/domain codecs. Missing observations -are reported; they are never replaced with invented history or regenerated bytes. -""" -import copy -from collections import deque -from datetime import datetime, timezone -import hashlib -import json -from types import SimpleNamespace - -from workbench import Workbench, WorkbenchError -import project_data as data_codec -import project_material as material_codec -import project_history as history_codec -import project_producers as producer_codec -from projects import decode_retained_snapshot, decode_retained_manifest, decode as decode_manifest, relative_path -from project_files import file_id, MAX_FILE_BYTES - -MAX_PAYLOAD = 40 * 1024 * 1024 -MAX_RECORDS = 100000 -ROOT_FAMILIES = { - 'records': 'workbench_record', 'responses': 'workbench_response', - 'preferences': 'workbench_preference', 'selections': 'saved_selection', - 'searches': 'saved_search', 'graphs': 'pipeline_graph', - 'recipe_drafts': 'recipe_definition', 'model_definitions': 'model_definition', - 'studies': 'study_history', 'runs': 'run_history', 'releases': 'dataset_release', - 'exposures': 'dataset_exposure', 'model_materials': 'model_material', - 'recipe_materials': 'recipe_material', 'runtime_materials': 'runtime_material', -} -PRODUCERS = {'generation': 'generation_history', 'grounded': 'grounded_history', - 'caption': 'caption_proposal_history', 'text_proposal': 'text_proposal_history'} -NAMESPACED = frozenset(material_codec.FAMILIES) | frozenset(history_codec.FAMILY_FORMATS) | frozenset(producer_codec.FAMILIES) - - -def _json(value): - return json.loads(value) if isinstance(value, str) else copy.deepcopy(value) - - -def _missing(detail): - raise WorkbenchError(detail, 'missing_source', 409) - - -class _ReadAccess: - """An internal read-view guard, never an imported actor or grant.""" - def require(self, scope, *args, **kwargs): - if scope != 'inspect': - raise WorkbenchError('Extraction source views are read-only.', 'forbidden', 403) - def audit(self, *args, **kwargs): - raise WorkbenchError('Extraction cannot append source audit.', 'forbidden', 403) - - -class _Capture: - def __init__(self, source): - self.source, self.db = source, source.db - self.origin = data_codec.identity(self.db.dataset_id) - self.records, self.inventory, self.dependencies = {}, {}, {} - self.blockers, self.warnings, self.working_files = [], [], [] - # Preserve emitted order without scanning prior runtime diagnostics. - self._issue_keys = {False: set(), True: set()} - self.anchor = None - self.queue, self.visited, self.queued = deque(), set(), set() - self.snapshot_by_id = {} - self.raw_cache = {} - self.payload_gaps = [] - self.deleted_intents = set() - self.session = SimpleNamespace(snapshot=source.snapshot, db=self.db, - artifacts=source.artifacts, check_current=lambda: None) - # Workbench and each metadata-view constructor only assign references; - # none runs SQL, creates directories or constructs Dataset/Jobs/Runs. - # Never call get/history: those public owners may enroll images. - dataset = SimpleNamespace(db=self.db, lock=self.db.lock, access=_ReadAccess(), - path=self.db.path, project_sources={}, project_availability=self._available) - self.reader = Workbench(dataset) - # Scientific metadata must use the caller's pinned catalog FD reader. - self.reader.scientific._capture = lambda identity: self._catalog_source('field_sequence', identity) - - def locator(self, family, identity): - if type(identity) is not str: - raise WorkbenchError('Source identities must be exact strings.') - bare = identity - if family in NAMESPACED and identity.startswith(self.origin + '.'): - bare = identity[len(self.origin) + 1:] - if family in ('project_artifact', 'dataset_release', 'project_file', 'model_material', - 'recipe_material', 'runtime_material', 'project_snapshot'): - data_codec.sha(bare) - else: - data_codec.identity(bare) - return self.origin + '.' + bare if family in NAMESPACED else bare - - def bare(self, family, identity): - if family in NAMESPACED: - prefix = self.origin + '.' - if not identity.startswith(prefix): - raise WorkbenchError('Source dependency belongs to another catalog.', 'unsupported', 409) - return identity[len(prefix):] - return identity - - def observe(self, family, identity, revision=None): - key = family, self.locator(family, identity) - previous = self.inventory.get(key) - value = dict(family=key[0], id=key[1], revision=revision) - if previous is not None and previous != value and revision is not None: - raise WorkbenchError('Source inventory identity has conflicting revisions.', 'invalid', 409) - if previous is None or revision is not None: - if previous is None and len(self.inventory) >= MAX_RECORDS: - raise WorkbenchError('Legacy source exceeds the supported finite inventory.', 'too_large', 413) - self.inventory[key] = value - return key - - def need(self, family, identity, parent=None, reason='required scientific dependency'): - target = self.observe(family, identity) - if parent is not None: - source = dict(family=parent[0], id=parent[1]) - edge = dict(source=source, target=dict(family=target[0], id=target[1]), reason=reason) - self.dependencies[(parent, target, reason)] = edge - if target not in self.visited and target not in self.queued: - self.queue.append(target);self.queued.add(target) - return target - - def issue(self, key, code, detail, *, warning=False): - detail = str(detail)[:1000] - identity = key[0], key[1], code, detail - seen = self._issue_keys[bool(warning)] - if identity not in seen: - seen.add(identity) - destination = self.warnings if warning else self.blockers - destination.append(dict(family=key[0], id=key[1], code=code, detail=detail)) - - def add(self, record): - key = record['family'], record['id'] - if key in self.records and self.records[key] != record: - raise WorkbenchError('Two source owners disagree about the same retained fact.', 'invalid', 409) - self.records[key] = copy.deepcopy(record) - self.observe(*key) - - def intrinsic(self, parent, blob, *, preservation='unique', role='source', raw=None): - sha, size = blob['sha256'], blob['bytes'] - data_codec.sha(sha) - if type(size) is not int or not 0 <= size <= 2**63 - 1: - raise WorkbenchError('Invalid source payload size.') - declaration = dict(format='project_artifact_v1', sha256=sha, bytes=size) - self.add(dict(family='project_artifact', id=sha, document=declaration)) - self.need('project_artifact', sha, parent, preservation + ':' + role) - try: - if raw is None: - if sha in self.raw_cache: - raw = self.raw_cache[sha] - elif size > MAX_PAYLOAD and hasattr(self.source, 'payload_fd'): - with self.source.payload_fd(sha, size) as handle: - descriptor = handle[0] if isinstance(handle, tuple) else handle - self.source.artifacts.publish_fd(descriptor, declaration, maximum_bytes=size) - return - else: - raw = self.source.resolve_payload(sha, size) - if type(raw) is not bytes or len(raw) != size or hashlib.sha256(raw).hexdigest() != sha: - raise WorkbenchError('Original payload differs from its declared byte identity.', 'corrupt_payload', 409) - self.source.artifacts.publish_bytes(raw, declaration, maximum_bytes=max(size, 1)) - except (OSError, WorkbenchError) as error: - self.payload_gaps.append((parent, sha, 'missing_regenerable' if preservation == 'regenerable' else getattr(error, 'code', 'missing_payload'), role + ': ' + str(error), preservation)) - - def raw(self, parent, raw, role, *, media='application/octet-stream', kind=None): - blob = dict(sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) - self.raw_cache[blob['sha256']] = raw - self.intrinsic(parent, blob, raw=raw, role=role) - if kind is None: - return blob - return dict(artifact=dict(namespace=self.origin, sha256=blob['sha256']), media_type=media, - kind=kind, role=role, preservation='unique', required_for=['inspect', 'release']) - - def _catalog_source(self, kind, identity): - folder, name = {'audio': ('audio', 'source.wav'), 'field_sequence': ('field_sequence', 'source.zip')}[kind] - return self.source.read_catalog(folder + '/' + identity + '/' + name, MAX_PAYLOAD) - - def _available(self, identity): - row = self.db.execute('SELECT kind FROM workbench_records WHERE id=%s', (identity,)).fetchone() - if row is None: - return False - try: - self._catalog_source(row[0], identity) - return True - except (OSError, WorkbenchError): - return False - - def _audit(self, target, actions, *, include_denied=False): - values = [] - for row in self.db.execute('SELECT actor_id,action,target_json,details_json,result,created_at FROM access_audit WHERE target_json::jsonb=%s::jsonb ORDER BY created_at,id', (json.dumps(target),)): - if row['action'] not in actions or row['result'] != 'success' and not include_denied: - continue - values.append(dict(actor_id=row['actor_id'], action=row['action'], target=_json(row['target_json']), - details=_json(row['details_json']), result=row['result'], - occurred_at=datetime.fromtimestamp(row['created_at'], timezone.utc).isoformat())) - return values - - def discover(self): - # Only identity/revision columns are read for unrelated global rows. - for row in self.db.execute('SELECT id,revision FROM workbench_records ORDER BY id'): - self.observe('workbench_record', row[0], row[1]) - for family, statement in ( - ('workbench_response', 'SELECT id,revision FROM workbench_responses ORDER BY id'), - ('workbench_preference', 'SELECT id,revision FROM workbench_preferences ORDER BY id'), - ('saved_selection', 'SELECT id,revision FROM saved_selections ORDER BY id'), - ('saved_search', 'SELECT id,revision FROM saved_searches ORDER BY id'), - ('pipeline_graph', 'SELECT id,revision FROM workspace_graphs ORDER BY id'), - ('recipe_definition', 'SELECT id,revision FROM workspace_recipe_drafts ORDER BY id'), - ('model_definition', 'SELECT id,revision FROM model_definition_drafts ORDER BY id'), - ('study_history', 'SELECT id,revision FROM model_studies ORDER BY id'), - ('run_history', 'SELECT id,NULL FROM workspace_runs ORDER BY id'), - ('trial_history', 'SELECT id,NULL FROM model_trials ORDER BY id'), - ('candidate_history', 'SELECT id,NULL FROM model_candidates ORDER BY id'), - ('model_material', 'SELECT sha256,revision FROM model_materials ORDER BY sha256'), - ('recipe_material', 'SELECT sha256,recipe_revision FROM workspace_recipe_materials ORDER BY sha256'), - ('dataset_exposure', 'SELECT id,NULL FROM workspace_exposures ORDER BY id'), - ('generation_history', 'SELECT id,NULL FROM generation_jobs ORDER BY id'), - ('grounded_history', 'SELECT id,NULL FROM grounded_jobs ORDER BY id'), - ('caption_proposal_history', 'SELECT id,NULL FROM caption_proposals ORDER BY id'), - ('text_proposal_history', 'SELECT id,NULL FROM text_classification_proposals ORDER BY id')): - for row in self.db.execute(statement): - self.observe(family, row[0], row[1]) - # These two owners remove the current row on deletion. Their closed - # successful audit facts retain a known identity, not its lost contents. - # Keep those identities visible in the global selection report. - for row in self.db.execute("SELECT action,target_json,details_json FROM access_audit WHERE result='success' AND action IN ('saved_selections.delete','saved_searches.delete') ORDER BY created_at,id"): - family = 'saved_selection' if row[0] == 'saved_selections.delete' else 'saved_search' - try: - identity, details = _json(row[1]), _json(row[2]) - data_codec.identity(identity) - if type(details) is not dict or set(details) != {'revision', 'retired_sha256'} or type(details['revision']) is not int or details['revision'] < 1: - continue - data_codec.sha(details['retired_sha256']) - except (WorkbenchError, ValueError, TypeError): - continue - key = family, identity - if key not in self.inventory: - self.observe(family, identity, details['revision']) - self.deleted_intents.add(key) - if len(self.inventory) > MAX_RECORDS: - raise WorkbenchError('Legacy source exceeds the supported finite inventory.', 'too_large', 413) - - def capture_record(self, identity, key): - current = Workbench._get(self.reader, identity) - history = [_json(row[0]) for row in self.db.execute('SELECT snapshot FROM workbench_history WHERE id=%s ORDER BY revision', (identity,))] - origin = _json(self.db.execute('SELECT provenance_json FROM workbench_records WHERE id=%s', (identity,)).fetchone()[0]) - kind = current['kind'] - if kind == 'text': - original = self.db.execute('SELECT original_text FROM workbench_records WHERE id=%s', (identity,)).fetchone()[0] - source = dict(kind=kind, original_text=original) - elif kind == 'image': - row = self.db.execute('SELECT * FROM samples WHERE id=%s', (identity,)).fetchone() - if row is None: - _missing('The deleted image has no preserved original source row or source geometry.') - sample = dict(row); sample['annotation'] = _json(sample['annotation']) - original = self.source.read_catalog('images/' + identity + '/source', MAX_PAYLOAD) - oriented = self.source.read_catalog('images/' + identity + '/image.png', MAX_PAYLOAD) - if hashlib.sha256(original).hexdigest() != sample['sha256'] or hashlib.sha256(oriented).hexdigest() != current['content_hash']: - raise WorkbenchError('Original image bytes changed outside their recorded identity.', 'corrupt_payload', 409) - source = dict(kind=kind, original=self.raw(key, original, 'original', kind=kind), - oriented=self.raw(key, oriented, 'oriented', media='image/png', kind=kind), revisions=[sample], deleted=False) - if sample['revision'] != 1: - self.issue(key, 'source_history_unavailable', 'Earlier image source revisions were not retained by the legacy catalog.') - return - elif kind in ('sequence', 'mesh', 'pointcloud', 'volume'): - statements = {'sequence': 'SELECT bundle FROM workbench_sequence_assets WHERE id=%s', - 'mesh': 'SELECT bundle FROM workbench_mesh_assets WHERE id=%s', - 'pointcloud': 'SELECT bundle FROM workbench_pointcloud_assets WHERE id=%s', - 'volume': 'SELECT bundle FROM workbench_volume_assets WHERE id=%s'} - row = self.db.execute(statements[kind], (identity,)).fetchone() - if row is None: - _missing('Original immutable BYTEA bundle is missing.') - raw = bytes(row[0]) - if hashlib.sha256(raw).hexdigest() != current['content_hash']: - raise WorkbenchError('Original immutable BYTEA identity differs.', 'corrupt_payload', 409) - source = dict(kind=kind, asset=self.raw(key, raw, 'original', media='application/zip', kind=kind), metadata=copy.deepcopy(current[kind])) - elif kind in ('audio', 'field_sequence'): - raw = self._catalog_source(kind, identity) - if hashlib.sha256(raw).hexdigest() != current['content_hash']: - raise WorkbenchError('Original scientific/audio source identity differs.', 'corrupt_payload', 409) - source = dict(kind=kind, asset=self.raw(key, raw, 'original', kind=kind, - media='audio/wav' if kind == 'audio' else 'application/zip'), metadata=copy.deepcopy(current[kind])) - else: - raise WorkbenchError('Unsupported source data kind.', 'unsupported', 409) - doc = dict(format='workbench_record_v1', origin=origin, source=source, current=current, - history=history, events=[], audit=self._audit(identity, data_codec.DATA_ACTIONS)) - self.add(dict(family='workbench_record', id=identity, document=doc)) - for parent in current['parents']: - self.need('workbench_record', parent, key, 'immutable data parent') - for response in self.db.execute('SELECT id FROM workbench_responses WHERE prompt_id=%s ORDER BY id', (identity,)): - self.need('workbench_response', response[0], key, 'captured response to selected prompt') - for preference in self.db.execute('SELECT id FROM workbench_preferences WHERE prompt_id=%s ORDER BY id', (identity,)): - self.need('workbench_preference', preference[0], key, 'captured preference for selected prompt') - provenance = origin - generation = provenance.get('generation') - if generation is not None: - self.need('generation_history', generation['job_id'], key, 'exact generating producer') - for field, family in (('caption_proposal', 'caption_proposal_history'), ('text_classification_proposal', 'text_proposal_history'), ('grounded', 'grounded_history')): - if field in provenance and isinstance(provenance[field], dict) and 'job_id' in provenance[field]: - self.need(family, provenance[field]['job_id'], key, 'exact source producer evidence') - - def capture_response(self, identity, key): - current = Workbench._response(self.reader, identity) - history = [_json(row[0]) for row in self.db.execute('SELECT snapshot FROM workbench_response_history WHERE id=%s ORDER BY revision', (identity,))] - self.add(dict(family='workbench_response', id=identity, document=dict(format='workbench_response_v1', - current=current, history=history, audit=self._audit(identity, data_codec.DATA_ACTIONS)))) - self.need('workbench_record', current['prompt_id'], key, 'exact captured response prompt') - - def capture_preference(self, identity, key): - from preferences import Preferences - current = Preferences._get(SimpleNamespace(db=self.db), identity) - history = [_json(row[0]) for row in self.db.execute('SELECT snapshot FROM workbench_preference_history WHERE id=%s ORDER BY revision', (identity,))] - self.add(dict(family='workbench_preference', id=identity, document=dict(format='workbench_preference_v1', - current=current, history=history, audit=self._audit(identity, data_codec.DATA_ACTIONS)))) - self.need('workbench_record', current['prompt_id'], key, 'exact captured preference prompt') - for response in (current['left_id'], current['right_id']): - self.need('workbench_response', response, key, 'exact captured preference answer') - - def capture_saved(self, family, identity, key): - from saved_selections import SavedSelections - from saved_searches import SavedSearches - if key in self.deleted_intents: - self.issue(key, 'intent_history_unavailable', 'A successful legacy deletion retains this identity and retired hash, but its original name and criteria/membership history are unavailable.') - return - current = (SavedSelections if family == 'saved_selection' else SavedSearches)._get(SimpleNamespace(db=self.db), identity) - if family == 'saved_selection': - for member in current['items']: - self.need('workbench_record', member['id'], key, 'saved exact membership') - if current['revision'] != 1: - self.issue(key, 'intent_history_unavailable', 'Legacy storage retains only the current saved name and criteria/membership, not its earlier revisions.') - return - self.add(dict(family=family, id=identity, document=dict(format=data_codec.FAMILIES[family], - current=current, history=[copy.deepcopy(current)], deleted=False, audit=self._audit(identity, data_codec.DATA_ACTIONS)))) - - def capture_definition(self, family, identity, key): - statement = {'pipeline_graph': 'SELECT document FROM workspace_graphs WHERE id=%s', - 'model_definition': 'SELECT document FROM model_definition_drafts WHERE id=%s', - 'recipe_definition': 'SELECT document FROM workspace_recipe_drafts WHERE id=%s'}[family] - row = self.db.execute(statement, (identity,)).fetchone() - if row is None: - _missing('Selected editable definition is missing.') - value = material_codec._definition(family, _json(row[0]), self.origin) - audits = [] - for fact in self._audit(identity, material_codec.AUDIT_ACTIONS): - try: - material_codec._definition_audit(family, value, fact, self.origin) - except WorkbenchError: - continue - audits.append(fact) - if len(audits) != 1: - _missing('The current editable definition has no unique exact original audit fact.') - record = dict(family=family, id=key[1], document=dict(format=material_codec.FAMILIES[family], - origin_catalog_id=self.origin, current=value, revisions=[copy.deepcopy(value)], audit=audits)) - material_codec.decode_records([record]);self.add(record) - if value['revision'] > 1: - self.issue(key, 'observed_history_incomplete', 'Only the actual retained current definition and its exact audit are available; earlier definition documents were not stored.', warning=True) - selector = value.get('source_selector') - if selector is not None: - self.snapshot_dependency(selector['project_id'], key) - if family == 'pipeline_graph': - # Graph-specific dependency extraction is delegated to the existing - # graph grammar below rather than inferred from arbitrary JSON. - for node in value['nodes']: - if node['type'] == 'release': - self.need('dataset_release', node['parameters']['release_id'], key, 'frozen graph input release') - elif node['type'] == 'input' and value['recipe'] == 'external_recipe': - self.snapshot_dependency(node['parameters']['project_id'], key) - - def snapshot_dependency(self, identity, parent): - row = self.db.execute('SELECT document FROM workspace_projects WHERE id=%s', (identity,)).fetchone() - if row is None: - _missing('Required registered project snapshot is missing.') - saved = decode_retained_snapshot(_json(row[0]));self.snapshot_by_id[identity] = saved - self.need('project_snapshot', saved['manifest_sha256'], parent, 'exact retained source registration') - - def capture_snapshot(self, sha, key): - matches = [value for value in self.snapshot_by_id.values() if value['manifest_sha256'] == sha] - if len(matches) != 1: - _missing('An exact selected project snapshot is not available in this pinned source.') - saved = matches[0] - # Explicitly recovered bytes retain the original registration as inert - # history. Its former directory/inode is not current read authority. - raw = self.source.read_bound_payload(sha, maximum_bytes=65536) - if raw is None: - root = self.source.roots.get(saved['root']) - if root is None or list(root[1]) != saved['root_identity']: - _missing('The retained registration has no matching current explicit source-root capability.') - raw = self.source.read_root(saved['root'], saved['manifest_path'], 65536) - if hashlib.sha256(raw).hexdigest() != sha or decode_retained_manifest(decode_manifest(raw)) != saved['manifest']: - raise WorkbenchError('Original registered manifest bytes differ.', 'corrupt_payload', 409) - blob = self.raw(key, raw, 'project_manifest') - record = dict(family='project_snapshot', id=key[1], document=dict(format=material_codec.FAMILIES['project_snapshot'], - origin_catalog_id=self.origin, value=saved, artifacts=material_codec._edges([dict(name=saved['manifest_path'], **blob)], 'project_snapshot'), audit=[])) - material_codec.decode_records([record]);self.add(record) - - def capture_material(self, family, sha, key): - statement = ('SELECT document FROM model_materials WHERE sha256=%s' if family == 'model_material' - else 'SELECT document FROM workspace_recipe_materials WHERE sha256=%s') - row = self.db.execute(statement, (sha,)).fetchone() - if row is None: - _missing('Required immutable material is missing.') - value = _json(row[0]);inventory = material_codec._inventory(family, value) - target = value['ref']['id'] if family == 'model_material' else sha - action = 'models.publish' if family == 'model_material' else 'recipes.material' - expected = (dict(revision=value['ref']['revision'], material_sha256=sha) if family == 'model_material' - else dict(recipe_id=value['material']['recipe_id'], revision=value['material']['recipe_revision'])) - audits = [fact for fact in self._audit(target, {action}) if fact['details'] == expected] - record = dict(family=family, id=key[1], document=dict(format=material_codec.FAMILIES[family], - origin_catalog_id=self.origin, value=value, artifacts=material_codec._edges(inventory, family), audit=audits)) - material_codec.decode_records([record]);self.add(record) - folder = 'model-materials' if family == 'model_material' else 'workspace-recipe-materials' - for item in inventory: - try: - raw = self.source.read_catalog(folder + '/' + sha + '/' + item['name'], min(MAX_PAYLOAD, max(item['bytes'], 1))) - except (OSError, WorkbenchError): - raw = None - self.intrinsic(key, item, role=item['name'], raw=raw) - material = value['material'] - if family == 'model_material': - self.need('model_definition', material['model_definition_id'], key, 'original editable model definition') - if material['source_selector'] is not None: - self.snapshot_dependency(material['source_selector']['project_id'], key) - else: - self.need('model_material', material['model']['sha256'], key, 'exact composed model material') - self.need('pipeline_graph', material['recipe_id'], key, 'original editable composition graph') - for release in material['data']: - self.need('dataset_release', release['id'], key, 'exact frozen composition data') - recipe = material['recipe_document'] - raw = self.source.resolve_payload(recipe['sha256'], recipe['bytes']) - snapshot = _json(raw.decode('utf-8')) - if snapshot['format'] == 'builtin_recipe_snapshot_v1': - environment = snapshot['validation']['runtime'] - else: - execution = snapshot['preflight']['material'] - environment = execution.get('torch_runtime', execution.get('cpu_snapshot', execution['runtime'])) - self.capture_runtime(environment, key) - - def capture_runtime(self, environment, parent): - from pipeline import digest - from recipe_process import decode - sha = digest(environment);key = self.need('runtime_material', sha, parent, 'original measured runtime material') - if key in self.records: - return - descriptor = environment if environment.get('profile') in ('captured_cpu_python_v1', 'managed_torch_python_v1') else None - raw = self.source.resolve_payload(descriptor['manifest_sha256'], descriptor['manifest_bytes']) if descriptor else None - decoded = decode(raw) if raw is not None else None - value = dict(ref=dict(kind='runtime', catalog_id=self.origin, id=sha, sha256=sha), descriptor=descriptor, - environment=environment if descriptor is None else None, manifest=decoded) - value = material_codec.decode_runtime_material(value, catalog_id=self.origin) - inventory = ([dict(name='manifest.json', sha256=descriptor['manifest_sha256'], bytes=descriptor['manifest_bytes'])] + - [dict(name=item['path'], sha256=item['sha256'], bytes=item['bytes']) for item in decoded['files']]) if descriptor else [] - record = dict(family='runtime_material', id=key[1], document=dict(format=material_codec.FAMILIES['runtime_material'], - origin_catalog_id=self.origin, value=value, artifacts=material_codec._edges(inventory, 'runtime_material'), audit=[])) - material_codec.decode_records([record]);self.add(record) - for item in record['document']['artifacts']: - self.intrinsic(key, item['blob'], preservation=item['preservation'], role=item['name'], raw=raw if item['name'] == 'manifest.json' else None) - - def capture_study(self, identity, key): - row = self.db.execute('SELECT document FROM model_studies WHERE id=%s', (identity,)).fetchone() - if row is None:_missing('Selected Study is missing.') - study = _json(row[0]);ref = study['policy'] - policy_row = self.db.execute('SELECT document FROM model_selection_policies WHERE sha256=%s', (ref['sha256'],)).fetchone() - if policy_row is None:_missing('Exact Study policy is missing.') - memberships = [dict(revision=row[0], trial_ids=_json(row[1])['trial_ids']) for row in self.db.execute( - 'SELECT revision,document FROM model_study_memberships WHERE study_id=%s ORDER BY revision', (identity,))] - reservations = [_json(row[0]) for row in self.db.execute('SELECT document FROM model_attempt_reservations WHERE study_id=%s ORDER BY id', (identity,))] - doc = dict(format=history_codec.FAMILY_FORMATS['study_history'], origin_catalog_id=self.origin, study=study, - memberships=memberships, policies=[dict(ref=ref, policy=_json(policy_row[0]))], reservations=reservations, - operations=[], revisions=[copy.deepcopy(study)], audit=[], revision_history_complete=study['revision'] == 1) - # Native operation receipts contain scientific results, not replay - # instructions in the destination. Classify them through the existing - # Study owner's finite result-to-Study projection. - from project_study_execution import ProjectStudyExecution - def original_trial(trial_id): - selected = self.db.execute('SELECT document FROM model_trials WHERE id=%s', (trial_id,)).fetchone() - if selected is None:_missing('Native lineage operation lost its original Trial.') - return _json(selected[0]) - classifier = SimpleNamespace(studies=SimpleNamespace(trial=original_trial)) - observed_revisions = {study['revision']: copy.deepcopy(study)} - policies = {ref['sha256']: doc['policies'][0]} - for row in self.db.execute('SELECT actor_id,operation_id,request_key,request_sha256,document FROM model_operation_receipts ORDER BY actor_id,operation_id,request_key'): - if row['operation_id'] not in history_codec.OPERATIONS: - continue - try: - result = _json(row['document']) - belongs = ProjectStudyExecution._operation_study(classifier, row['operation_id'], result) - except (WorkbenchError, KeyError, TypeError, ValueError): - self.issue(key, 'unassigned_operation_unreadable', 'A native operation result cannot be assigned to a Study: ' + row['operation_id'] + '/' + row['request_key'], warning=True) - continue - if belongs != identity: - continue - fact = dict(actor_id=row['actor_id'], operation_id=row['operation_id'], request_key=row['request_key'], - request_sha256=row['request_sha256'], result=result) - history_codec._operations([fact], self.origin) - doc['operations'].append(fact) - if row['operation_id'] in ('studies.create', 'studies.save', 'studies.continue'): - previous = observed_revisions.get(result['revision']) - if previous is not None and previous != result: - raise WorkbenchError('Native Study receipts disagree about an observed revision.') - observed_revisions[result['revision']] = copy.deepcopy(result) - policy_ref = result['policy'] - policy_row = self.db.execute('SELECT document FROM model_selection_policies WHERE sha256=%s', (policy_ref['sha256'],)).fetchone() - if policy_row is None:_missing('Observed Study revision lost its exact policy.') - policies[policy_ref['sha256']] = dict(ref=policy_ref, policy=_json(policy_row[0])) - doc['revisions'] = [observed_revisions[revision] for revision in sorted(observed_revisions)] - doc['policies'] = [policies[sha] for sha in sorted(policies)] - doc['revision_history_complete'] = sorted(observed_revisions) == list(range(1, study['revision'] + 1)) - for trial in study['trial_ids']: - self.need('trial_history', trial, key, 'immutable Study Trial membership') - for reservation in reservations: - self.need('run_history', reservation['run_id'], key, 'spent or reserved scientific attempt') - candidate_ids = [] - for row in self.db.execute('SELECT id,document FROM model_candidates WHERE study_id=%s ORDER BY id', (identity,)): - self.need('candidate_history', row[0], key, 'retained scientific candidate') - candidate_ids.append(row[0]);candidate = _json(row[1]);policy_ref = candidate['policy'] - if policy_ref['sha256'] not in policies: - policy_row = self.db.execute('SELECT document FROM model_selection_policies WHERE sha256=%s', (policy_ref['sha256'],)).fetchone() - if policy_row is None:_missing('Candidate lost its exact historical selection policy.') - policies[policy_ref['sha256']] = dict(ref=policy_ref, policy=_json(policy_row[0])) - doc['policies'] = [policies[sha] for sha in sorted(policies)] - targets = {identity, *study['trial_ids'], *candidate_ids, *(item['run_id'] for item in reservations)} - for target in sorted(targets): - doc['audit'].extend(self._audit(target, history_codec.OPERATIONS | {'studies.admit', 'studies.approve', 'studies.reconcile', 'studies.candidate_selected', 'studies.lineage.record', 'runs.admit', 'runs.admit_external', 'runs.update', 'runs.reconcile', 'runs.expose_test'})) - history_codec._audit(doc['audit']) - self.add(history_codec.envelope('study_history', self.origin, identity, doc)) - if not doc['revision_history_complete']: - self.issue(key, 'observed_history_incomplete', 'The legacy catalog retained only this Study revision; immutable memberships, policies, Trials and reservations are preserved.', warning=True) - - def capture_trial(self, identity, key): - row = self.db.execute('SELECT document FROM model_trials WHERE id=%s', (identity,)).fetchone() - if row is None:_missing('Required historical Trial is missing.') - trial = _json(row[0]);lineage_row = self.db.execute('SELECT document FROM model_trial_lineage WHERE trial_id=%s', (identity,)).fetchone() - lineage = _json(lineage_row[0]) if lineage_row is not None else None - self.add(history_codec.envelope('trial_history', self.origin, identity, dict(format=history_codec.FAMILY_FORMATS['trial_history'], - origin_catalog_id=self.origin, trial=trial, lineage=lineage))) - self.need('study_history', trial['study_id'], key, 'original Trial Study') - self.need('recipe_material', trial['material']['sha256'], key, 'exact immutable Trial composition') - if lineage is not None: - for relation in ('parent', 'baseline'): - ref = lineage['intent'][relation] - if ref is not None: - if ref['catalog_id'] != self.origin: - self.issue(key, 'unsupported_origin', 'Lineage crosses a source catalog not supplied by this read capability.') - else:self.need('trial_history', ref['trial_id'], key, 'explicit ' + relation + ' lineage') - - def capture_candidate(self, identity, key): - row = self.db.execute('SELECT document FROM model_candidates WHERE id=%s', (identity,)).fetchone() - if row is None:_missing('Required candidate observation is missing.') - candidate = _json(row[0]) - self.add(history_codec.envelope('candidate_history', self.origin, identity, dict(format=history_codec.FAMILY_FORMATS['candidate_history'], - origin_catalog_id=self.origin, candidate=candidate))) - self.need('study_history', candidate['study_id'], key, 'original candidate Study') - self.need('trial_history', candidate['trial_id'], key, 'original candidate Trial') - self.need('run_history', candidate['run_id'], key, 'original candidate Run evidence') - - def capture_run(self, identity, key): - row = self.db.execute('SELECT document FROM workspace_runs WHERE id=%s', (identity,)).fetchone() - if row is None:_missing('Selected historical Run is missing.') - run = _json(row[0]);history_codec.project_run_projection(run) - admission = self.db.execute('SELECT * FROM access_admissions WHERE id=%s', (run['admission_id'],)).fetchone() - grant = consumption = None - if run.get('approval_id') is not None: - grant = self.db.execute('SELECT * FROM access_execution_grants WHERE id=%s', (run['approval_id'],)).fetchone() - consumption = self.db.execute('SELECT * FROM access_execution_consumptions WHERE grant_id=%s', (run['approval_id'],)).fetchone() - proof = None - if admission is not None: - admission = dict(admission);admission['effects'] = _json(admission.pop('effects_json')) - proof = dict(grant=dict(grant) if grant is not None else None, consumption=dict(consumption) if consumption is not None else None, admission=admission) - history_codec._approval(proof, run) - payloads = [] - def payload(name, blob, role, population='unknown', raw=None, preservation='unique'): - self.intrinsic(key, blob, role=name, raw=raw, preservation=preservation) - payloads.append(dict(name=name, blob={field:blob[field] for field in ('sha256', 'bytes')}, population=population, - role=role, preservation=preservation, required_for=['scientific_inspection', 'execution_proof'])) - external = run.get('execution_kind') == 'external_recipe' - if external: - for name, item in run['material']['captures'].items(): - try:raw = self.source.read_catalog('workspace-runs/.external-' + identity + '/' + name, min(MAX_PAYLOAD, max(item['bytes'], 1))) - except (OSError, WorkbenchError):raw = None - payload(name, item, 'execution_source', raw=raw) - environment = run['material'].get('torch_runtime', run['material'].get('cpu_snapshot', run['material']['runtime'])) - self.capture_runtime(environment, key) - for ordinal, (name, item) in enumerate(sorted(run['material']['runtime'].get('files', {}).items())): - payload('runtime-files/' + str(ordinal), item, 'historical_runtime', preservation='regenerable') - for name, sha in sorted(run['material']['host_files'].items()): - declared = self.records.get(('project_artifact', sha)) - candidates = {declared['document']['bytes']} if declared is not None else { - item['bytes'] for item in self.source.external_inventory if item['sha256'] == sha} - if len(candidates) != 1: - self.issue(key, 'source_byte_count_unavailable', 'Original host source ' + name + ' retains a hash without original byte-count/custody evidence.') - else: - payload('host-sources/' + name, dict(sha256=sha, bytes=candidates.pop()), 'historical_host_source') - if run['state'] == 'completed': - receipt = self.source.read_catalog('workspace-runs/' + identity + '/custody-receipt.json', 16384) - inventory = self.source.read_catalog('workspace-runs/' + identity + '/artifact-manifest.json', 65536) - from runs import validate_external_artifact_inventory - validate_external_artifact_inventory(run, receipt, inventory) - payload('custody-receipt.json', dict(sha256=hashlib.sha256(receipt).hexdigest(), bytes=len(receipt)), 'custody_proof', raw=receipt) - payload('artifact-manifest.json', dict(sha256=hashlib.sha256(inventory).hexdigest(), bytes=len(inventory)), 'custody_proof', raw=inventory) - else: - payload('release.zip', dict(sha256=run['release_sha256'], bytes=run['release_bytes']), 'frozen_input') - self.need('dataset_release', run['release_id'], key, 'exact native frozen input') - for source in run['source_records']: - self.need('workbench_record', source['id'], key, 'retained native scientific input') - from pipeline import adapter_for_graph, decode_probe_observation - import release_validation - # Whole-asset integrity runs execute in-process and do not create - # the training source staging directory. Trainable native runs do; - # only those original, hash-bound copies prove their implementation. - if adapter_for_graph(run['graph']) is not release_validation: - if 'definition_files' not in run['runtime']: - self.issue(key, 'source_identity_unavailable', 'Native training Run has no retained original source inventory.') - else: - observation = decode_probe_observation(run['graph'].get('recipe', 'image_classification'), - run['runtime'], with_definition=True) - for name, expected in sorted(observation['definition_files'].items()): - try: - raw = self.source.read_catalog('workspace-runs/.staging-' + identity + '/' + name, MAX_PAYLOAD) - except (OSError, WorkbenchError): - self.issue(key, 'missing_source', 'Original native training source ' + name + ' is unavailable.') - continue - if hashlib.sha256(raw).hexdigest() != expected: - self.issue(key, 'corrupt_source', 'Original native training source ' + name + ' differs from its retained hash.') - continue - payload('source/' + name, dict(sha256=expected, bytes=len(raw)), 'execution_source', raw=raw) - for item in run['artifacts']: - try:raw = self.source.read_catalog('workspace-runs/' + identity + '/' + item['name'], min(MAX_PAYLOAD, max(item['bytes'], 1))) - except (OSError, WorkbenchError):raw = None - payload(item['name'], item, item.get('role', 'output'), item.get('split', 'unknown'), raw=raw) - if run.get('approval_material_sha256') is not None: - self.need('recipe_material', run['approval_material_sha256'], key, 'original approved composition') - if run.get('parent_run_id') is not None:self.need('run_history', run['parent_run_id'], key, 'original retry parent') - for association in self.db.execute('SELECT trial_id,study_id FROM model_attempt_reservations WHERE run_id=%s', (identity,)): - self.need('study_history', association[1], key, 'original budget association') - self.need('trial_history', association[0], key, 'original attempt Trial') - if external: - for item in run['material']['inputs'].values() if isinstance(run['material'].get('inputs'), dict) else []: - source = item.get('source', {}) - if item.get('kind') == 'run_artifact': - if source.get('catalog_id') != self.origin: - self.issue(key, 'unsupported_origin', 'Run artifact input crosses a source catalog not supplied by this read capability.') - else: - self.need('run_history', source['run_id'], key, 'exact producer Run input') - publication = None - if run.get('publication_continuation') is not None: - publication = dict(continuation=copy.deepcopy(run['publication_continuation']), source=None, result=None) - if run.get('publication_source') is not None: - from run_publication import validate_publication_source, validate_publication_result, MAX_DOCUMENT_BYTES - raw = self.source.read_catalog('workspace-runs/.external-' + identity + '/publication-source-v1.json', MAX_DOCUMENT_BYTES) - source_proof = validate_publication_source(raw, run=run) - blob = dict(sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) - payload('publication-source-v1.json', blob, 'publication_proof', raw=raw) - if publication is not None: - publication['source'] = blob - continuation = publication['continuation'];result_ref = continuation.get('result_ref') - if result_ref is not None: - result_raw = self.source.read_catalog('workspace-runs/.external-' + identity + '/publication-result-v1.json', MAX_DOCUMENT_BYTES) - validate_publication_result(result_raw, run=run, source=source_proof, ref=result_ref, continuation=continuation) - result_blob = dict(sha256=hashlib.sha256(result_raw).hexdigest(), bytes=len(result_raw)) - payload('publication-result-v1.json', result_blob, 'publication_result', raw=result_raw) - publication['result'] = result_blob - self.add(history_codec.capture_run(run, admission_proof=proof, payloads=payloads, publication=publication)) - - def capture_producer(self, family, identity, key): - statements = {'generation_history': 'SELECT data,ordinal FROM generation_jobs WHERE id=%s', - 'grounded_history': 'SELECT data,ordinal FROM grounded_jobs WHERE id=%s', - 'caption_proposal_history': 'SELECT data,ordinal FROM caption_proposals WHERE id=%s', - 'text_proposal_history': 'SELECT data,ordinal FROM text_classification_proposals WHERE id=%s'} - row = self.db.execute(statements[family], (identity,)).fetchone() - if row is None:_missing('Selected producer history is missing.') - original = _json(row[0]);entries = [] - if original.get('id') != identity:raise WorkbenchError('Producer row identity differs.') - if family == 'generation_history': - for entry in self.db.execute('SELECT data,storage_order FROM generation_entries WHERE job_id=%s ORDER BY storage_order', (identity,)): - entries.append((_json(entry[0]), entry[1])) - owner = SimpleNamespace(_origin=self.origin, _pending_payloads={}, _artifacts={}, session=self.session) - record = producer_codec._capture_job(owner, family, original, row[1], entries) - self.add(record) - for sha, declaration in owner._artifacts.items():self.intrinsic(key, declaration, role='producer_evidence', raw=owner._pending_payloads[sha]) - source = original.get('source') - if source is not None:self.need('workbench_record', source['id'], key, 'exact captured producer source') - if family == 'generation_history': - for entry, _ in entries: - if entry.get('sample_id') is not None:self.need('workbench_record', entry['sample_id'], key, 'retained generated source') - - def capture_release(self, identity, key): - # A release is immutable original ZIP evidence, never re-exported from - # current SQL. Its hash is its identity; inventory supplies original size. - from dataset_releases import Releases - raw = self.source.read_catalog('releases/' + identity + '.zip', MAX_PAYLOAD) - if hashlib.sha256(raw).hexdigest() != identity: - raise WorkbenchError('Original frozen release hash differs.', 'corrupt_payload', 409) - manifest_value = Releases.project_manifest(raw) - edge = self.raw(key, raw, 'frozen_archive', kind='release', media='application/zip') - self.add(dict(family='dataset_release', id=identity, document=dict(format='dataset_release_v1', asset=edge, - manifest=manifest_value, release_format=manifest_value.get('format', 'canonical_v1'), audit=self._audit(identity, data_codec.DATA_ACTIONS)))) - for item in manifest_value.get('records', manifest_value.get('prompts', [])): - self.need('workbench_record', item['id'], key, 'original frozen release record') - for item in manifest_value.get('responses', []): - self.need('workbench_response', item['id'], key, 'original frozen response') - for item in manifest_value.get('judgments', []): - self.need('workbench_preference', item['id'], key, 'original frozen preference') - - def capture_exposure(self, identity, key): - row = self.db.execute('SELECT * FROM workspace_exposures WHERE id=%s', (identity,)).fetchone() - if row is None:_missing('Selected disclosure observation is missing.') - value = dict(row);self.add(dict(family='dataset_exposure', id=identity, document=dict(format='dataset_exposure_v1', **value))) - self.need('workbench_record', value['record_id'], key, 'original exposed data') - self.need('run_history', value['run_id'], key, 'original disclosure Run') - self.need('dataset_release', value['release_id'], key, 'original exposed input Release') - - def working(self, item, saved): - kind, path = item['kind'], item['path'] - identity = file_id(self.origin, kind, path);key = self.observe('project_file', identity) - relative = saved['manifest_path'].rsplit('/', 1)[0] + '/' + path if '/' in saved['manifest_path'] else path - root = self.source.roots.get(saved['root']) - if root is None or list(root[1]) != saved['root_identity']: - _missing('Working files require the exact current registered root capability.') - raw = self.source.read_root(saved['root'], relative, MAX_FILE_BYTES) - text = raw.decode('utf-8', 'strict');blob = self.raw(key, raw, 'selected_working_file') - self.working_files.append(dict(kind=kind, path=path, sha256=blob['sha256'], bytes=blob['bytes'], text=text)) - - def execute(self, key): - family, locator = key;identity = self.bare(family, locator) - if key in self.records:return - if family == 'project_artifact': - match = [item for item in self.source.external_inventory if item['sha256'] == identity] - if not match:_missing('Selected artifact has no original byte-count inventory.') - sizes = {item['bytes'] for item in match} - if len(sizes) != 1:raise WorkbenchError('Selected artifact byte-count observations disagree.') - self.intrinsic(key, dict(sha256=identity, bytes=sizes.pop()), role='explicit_selected_artifact') - return - if family == 'workbench_record':return self.capture_record(identity, key) - if family == 'workbench_response':return self.capture_response(identity, key) - if family == 'workbench_preference':return self.capture_preference(identity, key) - if family in ('saved_selection', 'saved_search'):return self.capture_saved(family, identity, key) - if family in material_codec.EDITABLE:return self.capture_definition(family, identity, key) - if family == 'project_snapshot':return self.capture_snapshot(identity, key) - if family in ('model_material', 'recipe_material'):return self.capture_material(family, identity, key) - if family == 'project_file':_missing('Working-file adoption requires its explicit selected kind and path, not an inferred identifier.') - if family == 'runtime_material':_missing('Runtime material has no exact owning recipe/Run descriptor in the selected source closure.') - if family == 'study_history':return self.capture_study(identity, key) - if family == 'trial_history':return self.capture_trial(identity, key) - if family == 'candidate_history':return self.capture_candidate(identity, key) - if family == 'run_history':return self.capture_run(identity, key) - if family in producer_codec.FAMILIES:return self.capture_producer(family, identity, key) - if family == 'dataset_release':return self.capture_release(identity, key) - if family == 'dataset_exposure':return self.capture_exposure(identity, key) - raise WorkbenchError('Unsupported extraction family.', 'unsupported', 409) - - def validate(self): - for parent, sha, code, detail, preservation in self.payload_gaps: - declaration = self.records[('project_artifact', sha)]['document'] - if self.source.artifacts.status(declaration)['state'] != 'verified': - self.issue(parent, code, detail, warning=preservation == 'regenerable') - records = list(self.records.values()) - # Failed dependencies remain explicit blockers. Do not turn their - # absence into a fabricated valid empty closure. - if self.blockers:return - try: - data_codec.decode_data_records(records) - materials = material_codec.MaterialSnapshot(records) - history_codec.decode_records(records, materials=materials) - producer_codec.decode_records(records, artifacts=self.source.artifacts) - except WorkbenchError as error: - self.issue(self.anchor, error.code, str(error)) - - -def capture_legacy(source, selection): - """Capture selected facts from a caller-owned immutable read capability. - - All returned locators use exact canonical envelope identities. Global - inventory is distinct from selected closure. Source credentials, current - admissions and runtime owners are never reconstructed in the destination. - """ - if source.db.schema_version != 10: - raise WorkbenchError('Only the qualified legacy schema 10 can be extracted.', 'unsupported', 409) - if type(selection) is not dict or set(selection) - {'project_snapshot', 'roots', 'decisions'}: - raise WorkbenchError('Invalid finite extraction selection.') - snapshot = selection.get('project_snapshot') - if type(snapshot) is not dict or set(snapshot) != {'id', 'manifest_sha256'}: - raise WorkbenchError('Select one exact registered project snapshot.') - roots = selection.get('roots', {}) - if type(roots) is not dict or set(roots) - set(ROOT_FAMILIES) - {'producer_jobs', 'working_files'}: - raise WorkbenchError('Unknown extraction root family.') - source.require() - proof = source.db.execute("SELECT current_setting('transaction_read_only'),current_setting('transaction_isolation')").fetchone() - if tuple(proof) != ('on', 'repeatable read'): - raise WorkbenchError('Extraction requires a pinned repeatable-read READ ONLY source.', 'unavailable', 409) - capture = _Capture(source) - with source.db.lock: - capture.discover() - capture.snapshot_dependency(snapshot['id'], None) - saved = capture.snapshot_by_id[snapshot['id']] - if saved['manifest_sha256'] != snapshot['manifest_sha256']: - raise WorkbenchError('Selected registered snapshot digest changed.', 'stale', 409) - capture.anchor = capture.need('project_snapshot', saved['manifest_sha256']) - for root, family in ROOT_FAMILIES.items(): - values = roots.get(root, []) - if type(values) is not list or len(values) > MAX_RECORDS: - raise WorkbenchError('Extraction roots must be bounded lists.') - for identity in values: - if not isinstance(identity, str):raise WorkbenchError('Invalid selected identity.') - capture.need(family, identity) - decisions = selection.get('decisions', []) - if type(decisions) is not list or len(decisions) > MAX_RECORDS: - raise WorkbenchError('Ownership decisions must be bounded lists.') - supported = set(ROOT_FAMILIES.values()) | NAMESPACED | {'project_artifact', 'project_file', 'trial_history', 'candidate_history'} - for decision in decisions: - if (type(decision) is not dict or set(decision) != {'family', 'id', 'include', 'reason'} - or decision['family'] not in supported or type(decision['include']) is not bool): - raise WorkbenchError('Invalid explicit ownership decision.') - if decision['include']: - capture.need(decision['family'], decision['id']) - for item in roots.get('producer_jobs', []): - if type(item) is not dict or set(item) != {'kind', 'id'} or item['kind'] not in PRODUCERS: - raise WorkbenchError('Invalid selected producer family.') - capture.need(PRODUCERS[item['kind']], item['id']) - for item in roots.get('working_files', []): - try:capture.working(item, saved) - except (OSError, WorkbenchError, UnicodeError) as error: - capture.issue(('project_file', file_id(capture.origin, item['kind'], item['path'])), getattr(error, 'code', 'missing_source'), str(error)) - while capture.queue: - key = capture.queue.popleft();capture.queued.discard(key) - if key in capture.visited:continue - capture.visited.add(key) - if len(capture.visited) > MAX_RECORDS:raise WorkbenchError('Extraction closure exceeds its bound.', 'too_large', 413) - try:capture.execute(key) - except (OSError, WorkbenchError, KeyError, TypeError, ValueError) as error: - capture.issue(key, getattr(error, 'code', 'invalid_source'), str(error)) - capture.validate() - return dict(records=[capture.records[key] for key in sorted(capture.records)], - inventory=[capture.inventory[key] for key in sorted(capture.inventory)], - dependencies=[capture.dependencies[key] for key in sorted(capture.dependencies)], - blockers=capture.blockers, working_files=capture.working_files, warnings=capture.warnings) diff --git a/project_files.py b/project_files.py deleted file mode 100644 index ad0f6b3f..00000000 --- a/project_files.py +++ /dev/null @@ -1,522 +0,0 @@ -"""Project-owned scripts and notes: inert working files, explicit adoption. - -Captured UTF-8 text is immutable checkpoint history. Inspecting a working tree -never rewrites it, restores a missing file, executes a script, or installs tools. -""" -import copy -from datetime import datetime -from functools import cache, partial -import errno -import hashlib -import os -import stat - -from project_store import DIR_FLAGS -from workbench import WorkbenchError, timestamp - -FAMILY='project_file' -FORMAT='project_file_v1' -DELETION_FORMAT='project_file_v2' -MAX_FILE_BYTES=1024*1024 -MAX_FILES=1000 -MAX_ADOPT_ITEMS=40 -MAX_ADOPT_BYTES=8*1024*1024 -KINDS={'script':'scripts','note':'notes'} -VERSION_FIELDS={'revision','sha256','bytes','text','adopted_at','actor_id'} -RESULT_FIELDS={'id','project_id','kind','path','revision','sha256','bytes','adopted_at','changed'} - - -def fail(message,code='invalid',status=400): - raise WorkbenchError(message,code,status) - - -def closed(value,fields,label): - if type(value) is not dict or set(value)!=fields:fail('Invalid '+label+' fields.') - - -def digest(raw):return hashlib.sha256(raw).hexdigest() - - -def sha(value): - if type(value) is not str or len(value)!=64 or any(c not in '0123456789abcdef' for c in value):fail('Invalid working-file digest.') - - -def _path(kind,path): - if type(kind) is not str or kind not in KINDS or type(path) is not str or len(path)>512 or '\\' in path or any(ord(c)<32 for c in path): - fail('Choose a bounded relative script or note path.') - try:path.encode('utf-8') - except UnicodeEncodeError:fail('Working-file path must be strict UTF-8.') - parts=path.split('/') - if not 2<=len(parts)<=16 or parts[0]!=KINDS[kind] or any(part in ('','.','..') or len(part.encode('utf-8'))>255 for part in parts): - fail('Working-file path must remain inside its scripts or notes root.') - return parts - - -def _project_id(value): - if type(value) is not str or len(value)!=32 or any(c not in '0123456789abcdef' for c in value):fail('Invalid working-file project identity.') - - -def file_id(project_id,kind,path): - _project_id(project_id) - _path(kind,path) - return digest((project_id+'\0'+kind+'\0'+path).encode('utf-8')) - - -def _date(value): - try: - if type(value) is not str or len(value)>80:raise ValueError() - parsed=datetime.fromisoformat(value) - if parsed.utcoffset() is None:raise ValueError() - return parsed - except (ValueError,TypeError):fail('Invalid adoption timestamp.') - - -def _version(value,*,allow_deleted=False): - deleted=allow_deleted and type(value) is dict and value.get('deleted') is True - closed(value,VERSION_FIELDS|{'deleted'} if deleted else VERSION_FIELDS,'adopted working-file revision') - if type(value['revision']) is not int or not 1<=value['revision']<=2**53-1:fail('Invalid adoption revision.') - if deleted: - if value['text'] is not None or value['sha256'] is not None or type(value['bytes']) is not int or value['bytes']!=0: - fail('A deleted note revision must retain an exact absence marker.') - else: - if type(value['text']) is not str:fail('Adopted source must be UTF-8 text.') - try:raw=value['text'].encode('utf-8') - except UnicodeEncodeError:fail('Adopted source must be strict UTF-8.') - sha(value['sha256']) - if type(value['bytes']) is not int or value['bytes']!=len(raw) or len(raw)>MAX_FILE_BYTES or digest(raw)!=value['sha256']: - fail('Adopted text bytes differ from their exact identity.') - if type(value['actor_id']) is not str or len(value['actor_id'])!=32 or any(c not in '0123456789abcdef' for c in value['actor_id']):fail('Invalid historical adoption actor.') - _date(value['adopted_at']) - - -def decode_records(records): - if type(records) is not list:fail('Working-file inventory must be a list.') - owned=[];seen=set() - for envelope in records: - if type(envelope) is not dict:fail('Invalid working-file envelope.') - if envelope.get('family')!=FAMILY:continue - closed(envelope,{'family','id','document'},'working-file envelope') - doc=envelope['document'] - if type(doc) is not dict or doc.get('format') not in (FORMAT,DELETION_FORMAT):fail('Unsupported working-file format.','unsupported',409) - closed(doc,{'format','project_id','kind','path','current','history','origin'},'working-file document') - expected=file_id(doc['project_id'],doc['kind'],doc['path']) - if envelope['id']!=expected or expected in seen:fail('Duplicate or mismatched working-file identity.') - seen.add(expected) - if type(doc['history']) is not list or not doc['history']:fail('Working-file history is missing.') - previous=None;previous_deleted=False;has_deletion=False - for revision,value in enumerate(doc['history'],1): - _version(value,allow_deleted=doc['format']==DELETION_FORMAT) - deleted=value.get('deleted') is True - if deleted and (doc['kind']!='note' or revision==1 or previous_deleted):fail('Only an adopted, present note can acquire a deleted revision.') - if value['revision']!=revision or previous is not None and _date(value['adopted_at'])<previous:fail('Working-file history revisions or time differ.') - previous=_date(value['adopted_at']);previous_deleted=deleted;has_deletion=has_deletion or deleted - if doc['format']==DELETION_FORMAT and not has_deletion:fail('Working-file v2 requires an explicit retained deletion.') - if doc['current']!=doc['history'][-1]:fail('Working-file current revision differs from its history.') - owned.append(copy.deepcopy(envelope)) - counts={} - for envelope in owned: - project=envelope['document']['project_id'];counts[project]=counts.get(project,0)+1 - if any(count>MAX_FILES for count in counts.values()):fail('Project tracks more than 1,000 working files.','too_large',413) - _validate_forks(owned,records) - return owned - - -def _result(envelope,changed): - doc=envelope['document'];value=doc['current'] - result=dict(id=envelope['id'],project_id=doc['project_id'],kind=doc['kind'],path=doc['path'],changed=changed, - **{key:value[key] for key in ('revision','sha256','bytes','adopted_at')}) - if value.get('deleted') is True:result['deleted']=True - return result - - -def _validate_result(operation,result,lookup=None): - if operation=='files.adopt_many': - closed(result,{'items','changed_count'},'batch adoption result') - if type(result['items']) is not list or not 1<=len(result['items'])<=MAX_ADOPT_ITEMS: - fail('Invalid batch adoption result size.') - seen=set();projects=set() - for item in result['items']: - _validate_result('files.adopt',item,lookup) - if item['id'] in seen:fail('Batch adoption result repeats a file.') - seen.add(item['id']);projects.add(item['project_id']) - if (len(projects)!=1 or type(result['changed_count']) is not int - or result['changed_count']!=sum(item['changed'] for item in result['items']) - or sum(item['bytes'] for item in result['items'])>MAX_ADOPT_BYTES): - fail('Batch adoption result count or project differs.') - return True - if operation not in ('files.adopt','files.adopt_deletion'):return False - deleted=operation=='files.adopt_deletion' - closed(result,RESULT_FIELDS|{'deleted'} if deleted else RESULT_FIELDS,'adoption result') - if result['id']!=file_id(result['project_id'],result['kind'],result['path']) or type(result['changed']) is not bool:fail('Invalid adoption result identity.') - if deleted: - if result['deleted'] is not True or result['kind']!='note' or result['sha256'] is not None or result['bytes']!=0 or result['changed'] is not True or type(result['revision']) is not int or result['revision']<2: - fail('Invalid deleted note result.') - else:sha(result['sha256']) - _date(result['adopted_at']) - if type(result['revision']) is not int or not 1<=result['revision']<=2**53-1 or type(result['bytes']) is not int or not 0<=result['bytes']<=MAX_FILE_BYTES:fail('Invalid adoption result bounds.') - if lookup is not None: - envelope=lookup(result['id']) - if envelope is None or not any((value.get('deleted') is True)==deleted and all(value[key]==result[key] for key in ('revision','sha256','bytes','adopted_at')) for value in envelope['document']['history']): - fail('Adoption result is not an exact retained working-file revision.') - return True - - -def receipt_validator(records): - """Own one decoded candidate's file histories for its receipt checks. - - The returned validator has no live Session or publication authority. It - owns copied records, so caller mutation cannot change this fixed view. - A different candidate requires a new validator. - """ - owned = {record['id']: record for record in decode_records(records)} - return partial(_validate_result, lookup=owned.get) - - -def validate_operation_result(operation, result, records=None): - # Keep standalone shape/error ordering, and share only this call's lookup - # across batch items. There is no retained cache between invocations. - @cache - def owned(): - return {record['id']: record for record in decode_records(records)} - lookup = None if records is None else lambda identity: owned().get(identity) - return _validate_result(operation, result, lookup) - - -class _WorkingError(Exception): - def __init__(self,state):self.state=state - - -def _missing(root,parts): - """Confirm absence beneath the actual root, never through an unsafe alias.""" - parents=[os.dup(root)];missing=None - try: - for part in parts[:-1]: - try:child=os.open(part,DIR_FLAGS,dir_fd=parents[-1]) - except FileNotFoundError: - missing=part;break - parents.append(child) - if missing is None: - try: - entry=os.stat(parts[-1],dir_fd=parents[-1],follow_symlinks=False) - if not stat.S_ISREG(entry.st_mode) or entry.st_nlink!=1:raise _WorkingError('unsafe_path') - except FileNotFoundError:missing=parts[-1] - for index in range(1,len(parents)): - entry=os.stat(parts[index-1],dir_fd=parents[index-1],follow_symlinks=False) - if not os.path.samestat(entry,os.fstat(parents[index])):raise _WorkingError('unstable') - if missing is not None: - try:os.stat(missing,dir_fd=parents[-1],follow_symlinks=False) - except FileNotFoundError:return True - raise _WorkingError('unstable') - return False - except OSError as error: - state='unsafe_path' if error.errno in (errno.ELOOP,errno.ENOTDIR) else 'unstable' if error.errno==errno.ENOENT else 'unreadable' - raise _WorkingError(state) from None - finally: - for descriptor in reversed(parents):os.close(descriptor) - - -def _read(root,parts): - parent=os.dup(root);descriptor=None - try: - for part in parts[:-1]: - child=os.open(part,DIR_FLAGS,dir_fd=parent);os.close(parent);parent=child - descriptor=os.open(parts[-1],os.O_RDONLY|os.O_NOFOLLOW|os.O_CLOEXEC|os.O_NONBLOCK,dir_fd=parent) - before=os.fstat(descriptor) - if not stat.S_ISREG(before.st_mode) or before.st_nlink!=1:raise _WorkingError('unsafe_path') - if before.st_size>MAX_FILE_BYTES:raise _WorkingError('too_large') - chunks=[];total=0 - while True: - block=os.read(descriptor,min(65536,MAX_FILE_BYTES+1-total)) - if not block:break - chunks.append(block);total+=len(block) - if total>MAX_FILE_BYTES:raise _WorkingError('too_large') - after=os.fstat(descriptor);entry=os.stat(parts[-1],dir_fd=parent,follow_symlinks=False) - fields=lambda value:(value.st_dev,value.st_ino,value.st_mode,value.st_nlink,value.st_size,value.st_mtime_ns,value.st_ctime_ns) - if fields(before)!=fields(after) or fields(after)!=fields(entry):raise _WorkingError('unstable') - current=os.dup(root) - try: - for part in parts[:-1]: - child=os.open(part,DIR_FLAGS,dir_fd=current);os.close(current);current=child - if (os.fstat(current).st_dev,os.fstat(current).st_ino)!=(os.fstat(parent).st_dev,os.fstat(parent).st_ino): - raise _WorkingError('unstable') - finally:os.close(current) - raw=b''.join(chunks) - try:text=raw.decode('utf-8') - except UnicodeDecodeError:raise _WorkingError('invalid_utf8') from None - return text,digest(raw),len(raw) - except OSError as error: - state='missing' if error.errno==errno.ENOENT else 'unsafe_path' if error.errno in (errno.ELOOP,errno.ENOTDIR) else 'unreadable' - raise _WorkingError(state) from None - finally: - if descriptor is not None:os.close(descriptor) - os.close(parent) - - -def inspect_captured(records, project_id, root): - """Observe captured working files using only a locked project capability.""" - return _inspect_validated(decode_records(records), project_id, root) - - -def _inspect_validated(records, project_id, root): - """Consume full-decoder output or fresh values from a validated Session.""" - owned=[r for r in records if r['document']['project_id']==project_id] - items=[] - for envelope in sorted(owned,key=lambda r:r['document']['path']): - doc=envelope['document'];value=doc['current'] - observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], - expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) - try: - parts=_path(doc['kind'],doc['path']) - if value.get('deleted') is True and _missing(root,parts): - observation['state']='deleted' - else: - if value.get('deleted') is True:observation['state']='changed' - _,observed,size=_read(root,parts) - observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') - except _WorkingError as error: - if value.get('deleted') is True: - observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) - else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) - items.append(observation) - return items - - -class ProjectFiles: - def __init__(self,session): - self.session,self.access,self.db,self.store=session,session.access,session.db,session.store - - def inspect(self): - with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.inspect');snapshot=self.session.check_records(FAMILY) - with self.store.root_directory() as root: - items=_inspect_validated(snapshot['records'],snapshot['project_id'],root) - self.session.check_current() - return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) - - def read(self,record_id,revision=None): - with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.read',record_id);snapshot=self.session.check_records(FAMILY) - envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) - if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) - values=envelope['document']['history'] - if revision is None:value=values[-1] - elif type(revision) is int and 1<=revision<=len(values):value=values[revision-1] - else:fail('Adopted working-file revision is unavailable.','unavailable',404) - return dict(id=record_id,project_id=envelope['document']['project_id'],kind=envelope['document']['kind'],path=envelope['document']['path'],version=copy.deepcopy(value)) - - def history(self,record_id): - with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.history',record_id);snapshot=self.session.check_records(FAMILY) - envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) - if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) - return copy.deepcopy(envelope['document']['history']) - - def require_clean(self): - """Save As must not silently discard unadopted or missing source files.""" - report=self.inspect() - if any(item['state'] not in ('unchanged','deleted') for item in report['items']): - error=WorkbenchError('Adopt or restore changed, missing or invalid working files before Save As.','working_files_changed',409) - error.report=report - raise error - return report - - @staticmethod - def materialize_captured(records,project_id,root_fd): - return materialize_captured(records,project_id,root_fd) - - def _adoption_intent(self,body): - closed(body,{'kind','path','expected_revision','expected_sha256'},'working-file adoption') - parts=_path(body['kind'],body['path']);project_id=self.session.summary['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) - if type(body['expected_revision']) is not int or not 0<=body['expected_revision']<=2**53-1:fail('Invalid expected adoption revision.') - return dict(body),parts,record_id - - def _capture_adoption(self,intent,prior,root): - body,parts,record_id=intent - revision=prior['document']['current']['revision'] if prior else 0 - if revision!=body['expected_revision']:fail('Working-file adoption revision changed.','conflict',409) - try:text,observed,size=_read(root,parts) - except _WorkingError as error:fail('Working file is '+error.state+'.','unavailable' if error.state=='missing' else 'invalid',404 if error.state=='missing' else 400) - if observed!=body['expected_sha256']:fail('Working file changed after review.','conflict',409) - if prior is not None and observed==prior['document']['current']['sha256']:return _result(prior,False),None,size - actor=self.access.require('edit','files.adopt',record_id) - value=dict(revision=revision+1,sha256=observed,bytes=size,text=text,adopted_at=timestamp(),actor_id=actor.actor_id) - history=copy.deepcopy(prior['document']['history']) if prior else [];history.append(value) - envelope=dict(family=FAMILY,id=record_id,document=dict(format=prior['document']['format'] if prior else FORMAT, - project_id=self.session.summary['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), - history=history,origin=copy.deepcopy(prior['document']['origin']) if prior else None)) - return _result(envelope,True),envelope,size - - def adopt(self,body,expected_checkpoint,key): - intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.summary['project_id'] - pending=[] - with self.access.lock: - self.access.require('edit','files.adopt',record_id) - def invoke(): - snapshot=self.session.snapshot;owned=[r for r in decode_records(snapshot['records']) if r['document']['project_id']==project_id] - prior=next((r for r in owned if r['id']==record_id),None) - if prior is None and len(owned)>=MAX_FILES:fail('Project already tracks 1,000 working files.','too_large',413) - with self.store.root_directory() as root: - result,envelope,_=self._capture_adoption(intent,prior,root) - if envelope is not None: - decode_records([r for r in snapshot['records'] if not (r['family']==FAMILY and r['id']==record_id)]+[envelope]);pending.append(envelope) - return result - return self.session.mutate('files.adopt',record_id,body,expected_checkpoint,key,invoke, - lambda result,before:dict(records=copy.deepcopy(pending),result=copy.deepcopy(result))) - - def adopt_many(self,body,expected_checkpoint,key): - """Capture one bounded, all-or-nothing group through ordinary adoption. - - Every file retains its permission/revision/hash and anchored read checks. - One verified root scope and one canonical publication cover the group. - No working file is rewritten, even when another input rejects the group. - """ - closed(body,{'items'},'batch working-file adoption') - if type(body['items']) is not list or not 1<=len(body['items'])<=MAX_ADOPT_ITEMS: - fail('Choose 1 through 40 working files for adoption.','too_large',413) - intents=[self._adoption_intent(item) for item in body['items']] - ids=[intent[2] for intent in intents] - if len(set(ids))!=len(ids):fail('Batch adoption repeats a working-file path.') - body=dict(items=[intent[0] for intent in intents]);project_id=self.session.summary['project_id'] - target=digest('\0'.join(ids).encode());pending=[] - with self.access.lock: - for identity in ids:self.access.require('edit','files.adopt',identity) - def invoke(): - snapshot=self.session.snapshot - owned={r['id']:r for r in decode_records(snapshot['records']) if r['document']['project_id']==project_id} - if len(owned)+sum(identity not in owned for identity in ids)>MAX_FILES: - fail('Project already tracks 1,000 working files.','too_large',413) - results=[];total=0 - with self.store.root_directory(verify_after=True) as root: - for intent in intents: - result,envelope,size=self._capture_adoption(intent,owned.get(intent[2]),root) - total+=size - if total>MAX_ADOPT_BYTES:fail('Batch working-file capture exceeds 8 MiB.','too_large',413) - if envelope is not None:pending.append(envelope) - results.append(result) - changed={r['id'] for r in pending} - decode_records([r for r in snapshot['records'] if not (r['family']==FAMILY and r['id'] in changed)]+pending) - return dict(items=results,changed_count=sum(item['changed'] for item in results)) - return self.session.mutate('files.adopt_many',target,body,expected_checkpoint,key,invoke, - lambda result,before:dict(records=copy.deepcopy(pending),result=copy.deepcopy(result))) - - def adopt_deletion(self,body,expected_checkpoint,key): - closed(body,{'path','expected_revision','expected_sha256'},'note deletion adoption') - parts=_path('note',body['path']);sha(body['expected_sha256']) - if type(body['expected_revision']) is not int or not 1<=body['expected_revision']<=2**53-1:fail('Invalid expected note revision.') - project_id=self.session.summary['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] - with self.access.lock: - self.access.require('edit','files.adopt_deletion',record_id) - def invoke(): - snapshot=self.session.snapshot - prior=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) - if prior is None:fail('Only an already adopted note can acquire a deletion.','unavailable',404) - current=prior['document']['current'] - if current['revision']!=body['expected_revision'] or current['sha256']!=body['expected_sha256'] or current.get('deleted') is True: - fail('Canonical note revision or digest changed.','conflict',409) - with self.store.root_directory() as root: - try:missing=_missing(root,parts) - except _WorkingError as error:fail('Note path is '+error.state+'.','invalid',400) - if not missing:fail('The reviewed note still exists; deletion cannot be adopted.','conflict',409) - actor=self.access.require('edit','files.adopt_deletion',record_id) - value=dict(revision=current['revision']+1,deleted=True,sha256=None,bytes=0,text=None, - adopted_at=timestamp(),actor_id=actor.actor_id) - envelope=copy.deepcopy(prior);doc=envelope['document'] - doc['format']=DELETION_FORMAT;doc['history'].append(value);doc['current']=copy.deepcopy(value) - decode_records([r for r in snapshot['records'] if not (r['family']==FAMILY and r['id']==record_id)]+[envelope]);pending.append(envelope) - return _result(envelope,True) - return self.session.mutate('files.adopt_deletion',record_id,body,expected_checkpoint,key,invoke, - lambda result,before:dict(records=copy.deepcopy(pending),result=copy.deepcopy(result))) - - -def _validate_forks(owned,records): - indexed={r['id']:r for r in owned} - overlays=[r['document'] for r in records if r.get('family')=='project_fork'] - if overlays: - from project_fork import decode_fork - decode_fork(records) - for envelope in owned: - doc=envelope['document'];origin=doc['origin'] - if origin is None:continue - closed(origin,{'project_id','checkpoint','id','revision'},'working-file fork origin') - _project_id(origin['project_id']);sha(origin['checkpoint']);sha(origin['id']) - if type(origin['revision']) is not int or not 1<=origin['revision']<=2**53-1:fail('Invalid original working-file revision.') - original=indexed.get(origin['id']) - if original is None:fail('Working-file fork original is unavailable.') - source=original['document'] - if source['project_id']!=origin['project_id'] or source['kind']!=doc['kind'] or source['path']!=doc['path'] or source['current']['revision']!=origin['revision']: - fail('Working-file fork origin differs from its exact archived source.') - mapping=dict(family=FAMILY,source_id=origin['id'],source_revision=origin['revision'],destination_id=envelope['id']) - matches=[o for o in overlays if o['source']==dict(project_id=origin['project_id'],checkpoint=origin['checkpoint']) and o['destination_project_id']==doc['project_id'] and mapping in o['mapping']] - if len(matches)!=1 or doc['history'][:origin['revision']]!=source['history']: - fail('Working-file fork requires its exact archived history and origin mapping.') - - -def fork_records(records,source_project_id,source_checkpoint,destination_project_id): - """Independent file identity; preserve actual historical adoptions exactly.""" - _project_id(source_project_id);_project_id(destination_project_id);sha(source_checkpoint) - if source_project_id==destination_project_id:fail('An independent working-file fork needs a new project.') - decoded=decode_records(records) - from project_fork import active_records - active={(r['family'],r['id']) for r in active_records(records,project_id=source_project_id)} - owned=[r for r in decoded if (r['family'],r['id']) in active] - made=[];mapping=[] - for envelope in owned: - doc=envelope['document'] - if doc['project_id']!=source_project_id:fail('Current working-file scope differs from the selected project.') - new_id=file_id(destination_project_id,doc['kind'],doc['path']) - if any(r['family']==FAMILY and r['id']==new_id for r in records):fail('Working-file destination identity already exists.') - fresh=copy.deepcopy(envelope);fresh['id']=new_id - fresh['document'].update(project_id=destination_project_id,origin=dict(project_id=source_project_id,checkpoint=source_checkpoint,id=envelope['id'],revision=doc['current']['revision'])) - made.append(fresh);mapping.append(dict(family=FAMILY,source_id=envelope['id'],source_revision=doc['current']['revision'],destination_id=new_id)) - made.sort(key=lambda r:r['id']);mapping.sort(key=lambda m:(m['family'],m['source_id'],m['source_revision'],m['destination_id'])) - return dict(records=made,mapping=mapping) - - -def materialize_captured(records,project_id,root_fd): - """Write exact declared text only into the caller's new staging capability. - - The fork owner verifies destination non-overlap and owns all staging cleanup - and publication. O_EXCL never replaces an existing working file. A partial - failed staging tree is not a published fork and must not be acknowledged. - """ - _project_id(project_id) - if type(root_fd) is not int or not stat.S_ISDIR(os.fstat(root_fd).st_mode):fail('A staging directory capability is required.') - owned=[r for r in decode_records(records) if r['document']['project_id']==project_id] - copied=[] - try: - for envelope in sorted(owned,key=lambda r:r['document']['path']): - doc=envelope['document'];value=doc['current'];parts=_path(doc['kind'],doc['path']) - if value.get('deleted') is True:continue - parent=os.dup(root_fd);descriptor=None - try: - for part in parts[:-1]: - try:os.mkdir(part,0o700,dir_fd=parent);os.fsync(parent) - except FileExistsError:pass - child=os.open(part,DIR_FLAGS,dir_fd=parent);os.close(parent);parent=child - descriptor=os.open(parts[-1],os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW|os.O_CLOEXEC,0o600,dir_fd=parent) - captured=os.fstat(descriptor) - if not stat.S_ISREG(captured.st_mode) or captured.st_nlink!=1:fail('Unsafe staging working file.') - raw=value['text'].encode('utf-8');remaining=memoryview(raw) - while remaining: - count=os.write(descriptor,remaining) - if count<=0:raise OSError(errno.EIO,'Working-file staging write failed') - remaining=remaining[count:] - os.fsync(descriptor) - entry=os.stat(parts[-1],dir_fd=parent,follow_symlinks=False) - after=os.fstat(descriptor) - if (entry.st_dev,entry.st_ino,entry.st_nlink,entry.st_size)!=(captured.st_dev,captured.st_ino,1,len(raw)) or after.st_size!=len(raw):fail('Working-file staging entry changed.','stale_project',409) - check=os.dup(root_fd) - try: - for part in parts[:-1]: - child=os.open(part,DIR_FLAGS,dir_fd=check);os.close(check);check=child - if (os.fstat(check).st_dev,os.fstat(check).st_ino)!=(os.fstat(parent).st_dev,os.fstat(parent).st_ino):fail('Working-file staging directory changed.','stale_project',409) - finally:os.close(check) - os.fsync(parent) - copied.append(dict(id=envelope['id'],path=doc['path'],sha256=value['sha256'],bytes=value['bytes'])) - finally: - if descriptor is not None:os.close(descriptor) - os.close(parent) - os.fsync(root_fd) - except OSError: - fail('Working-file staging failed; no complete fork was published.','storage',500) - return copied diff --git a/project_fork.py b/project_fork.py deleted file mode 100644 index f35b9812..00000000 --- a/project_fork.py +++ /dev/null @@ -1,529 +0,0 @@ -"""Independent fork publication and its explicit inert-origin overlay. - -Domain owners construct new editable values and their revision-specific origin -mappings. This owner never rewrites domain documents. It preserves source -records, verifies declared payload custody, and atomically publishes a complete -new directory. Runtime quiescence and current authorization belong to the caller. -""" -import copy -import errno -import hashlib -import os -import re -import stat -import uuid - -from project_artifacts import ProjectArtifacts, validate_declaration -from project_store import (ProjectStore, ProjectStoreError, REQUEST_KEY, anchored_directory, - DIR_FLAGS, _encode, _identifier, _physical, _relative_parts, - _rename_noreplace, _walk) - -FAMILY = 'project_fork' -FORMAT = 'project_fork_v1' -# Immutable scientific evidence stays origin-scoped and inspectable. Only these -# formerly active values/receipts become inactive through the fork overlay. -ARCHIVE_FAMILIES = frozenset(('workbench_text', 'workbench_record', 'workbench_response', - 'workbench_preference', 'saved_selection', 'saved_search', 'model_definition', - 'pipeline_graph', 'recipe_definition', 'project_file', 'project_operation', 'project_composition', FAMILY)) -ID = re.compile(r'^[a-f0-9]{32}$') -SHA = re.compile(r'^[a-f0-9]{64}$') - - -class ProjectForkError(ProjectStoreError): - pass - - -def _fail(message, code='invalid', status=400): - raise ProjectForkError(message, code, status) - - -def _shape(value, fields, label): - if type(value) is not dict or set(value) != set(fields): - _fail('Invalid fork ' + label + ' fields.') - - -def _uuid(value): - if type(value) is not str or not ID.fullmatch(value): - _fail('Fork project identity must be a UUID.') - - -def _sha(value): - if type(value) is not str or not SHA.fullmatch(value): - _fail('Fork checkpoint must be a SHA256 digest.') - - -def _input(source, directory, name, key): - return dict(source=source, directory=directory, name=name, request_key=key) - - -def _digest(value): - return hashlib.sha256(_encode(value)).hexdigest() - - -def _locator(value): - _shape(value, ('family', 'id'), 'archive locator') - _identifier(value['family'], 'Fork record family') - _identifier(value['id'], 'Fork record ID') - return value['family'], value['id'] - - -def _mapping_key(value): - return value['family'], value['source_id'], value['source_revision'] or 0, value['destination_id'] - - -def _decode_document(value, identity, indexed): - _shape(value, ('format', 'source', 'destination_project_id', 'request_key', 'request_sha256', - 'directory', 'name', 'mapping', 'archived', 'artifacts'), 'origin overlay') - if value['format'] != FORMAT: - _fail('Unsupported fork overlay format.', 'unsupported', 422) - _shape(value['source'], ('project_id', 'checkpoint'), 'source identity') - _uuid(value['source']['project_id']) - _sha(value['source']['checkpoint']) - _uuid(value['destination_project_id']) - if identity != value['destination_project_id'] or identity == value['source']['project_id']: - _fail('Fork source and destination identities must be independent.') - if type(value['request_key']) is not str or not REQUEST_KEY.fullmatch(value['request_key']): - _fail('Invalid fork request key.') - _sha(value['request_sha256']) - _relative_parts(value['directory']) - if type(value['name']) is not str or not value['name'].strip() or len(value['name']) > 200: - _fail('Invalid fork display name.') - if value['request_sha256'] != _digest(_input(value['source'], value['directory'], value['name'], value['request_key'])): - _fail('Fork request receipt differs from its exact input.') - if type(value['archived']) is not list: - _fail('Fork archive inventory must be a list.') - archived = [_locator(item) for item in value['archived']] - if archived != sorted(set(archived)) or any(key not in indexed for key in archived): - _fail('Fork archive must be sorted, unique and refer to retained records.') - if (FAMILY, identity) in archived: - _fail('Current fork overlay cannot archive itself.') - archived = set(archived) - if type(value['mapping']) is not list: - _fail('Fork mappings must be a list.') - sources, destinations, keys = set(), set(), [] - for item in value['mapping']: - _shape(item, ('family', 'source_id', 'source_revision', 'destination_id'), 'editable mapping') - _identifier(item['family'], 'Fork mapping family') - _identifier(item['source_id'], 'Fork source record ID') - _identifier(item['destination_id'], 'Fork destination record ID') - revision = item['source_revision'] - if revision is not None and (type(revision) is not int or revision < 1): - _fail('Fork source revision must be positive or null.') - source = (item['family'], item['source_id']) - destination_family = 'workbench_record' if item['family'] == 'workbench_text' else item['family'] - destination = (destination_family, item['destination_id']) - if (source not in archived or destination not in indexed or destination in archived or - item['source_id'] == item['destination_id'] or (source, revision) in sources or destination in destinations): - _fail('Fork mapping must bind an archived source revision to a unique new current record.') - sources.add((source, revision)) - destinations.add(destination) - keys.append(_mapping_key(item)) - if keys != sorted(keys): - _fail('Fork mappings must be sorted.') - _shape(value['artifacts'], ('copied', 'missing_regenerable'), 'artifact completeness') - listed = set() - for field in ('copied', 'missing_regenerable'): - items = value['artifacts'][field] - if type(items) is not list: - _fail('Fork artifact completeness must use lists.') - for digest in items: - _sha(digest) - if items != sorted(set(items)) or listed.intersection(items): - _fail('Fork artifact completeness must be disjoint, sorted and unique.') - if any(('project_artifact', digest) not in indexed for digest in items): - _fail('Fork payload completeness has no intrinsic declaration.') - listed.update(items) - return archived - - -def decode_fork(records, *, project_id=None): - """Decode the one current overlay; originals remain independently validated. - - Pure owner decoders consume all records for historical closure. Projection - and receipt owners use active_records only when constructing current state. - """ - if type(records) is not list: - _fail('Fork records must be a list.') - indexed, overlays = {}, {} - for item in records: - _shape(item, ('family', 'id', 'document'), 'record envelope') - _identifier(item['family'], 'Fork record family') - _identifier(item['id'], 'Fork record ID') - key = item['family'], item['id'] - if key in indexed: - _fail('Duplicate fork record identity.') - indexed[key] = item - if item['family'] == FAMILY: - overlays[key] = item['document'] - if project_id is not None: - _uuid(project_id) - if not overlays: - return None - archives = {key: _decode_document(doc, key[1], indexed) for key, doc in overlays.items()} - archived_overlays = {key for values in archives.values() for key in values if key[0] == FAMILY} - candidates = set(overlays) - archived_overlays - if len(candidates) != 1: - _fail('Fork history must have exactly one current origin overlay.') - current = next(iter(candidates)) - if project_id is not None and current[1] != project_id: - _fail('Fork overlay belongs to a different current project.') - archive = archives[current] - for key, previous in archives.items(): - if key != current and (key not in archive or not previous <= archive): - _fail('Current fork must preserve all prior archived dispositions.') - return copy.deepcopy(overlays[current]) - - -def active_records(records, *, project_id=None, families=None): - """Own selected active copies while validating the complete fork history.""" - if families is not None: - if type(families) is not frozenset: - _fail('Active record families must be an explicit frozenset.') - for family in families:_identifier(family,'Active record family') - overlay = decode_fork(records, project_id=project_id) - archived = set() if overlay is None else {(item['family'], item['id']) for item in overlay['archived']} - return [copy.deepcopy(item) for item in records - if (families is None or item['family'] in families) and (item['family'], item['id']) not in archived] - - -def _inside(candidate_fd, ancestor_fd): - """Compare physical ancestry, including directory aliases; never string-prefix.""" - current = os.dup(candidate_fd) - try: - expected = _physical(ancestor_fd) - while True: - if _physical(current) == expected: - return True - parent = os.open('..', DIR_FLAGS, dir_fd=current) - if _physical(parent) == _physical(current): - os.close(parent) - return False - os.close(current) - current = parent - finally: - os.close(current) - - -def _declarations(records): - result = {} - for item in records: - if item['family'] == 'project_artifact': - value = validate_declaration(item['document']) - if item['id'] != value['sha256'] or item['id'] in result: - _fail('Fork artifact declaration identity differs or repeats.') - result[item['id']] = value - return result - - -def _artifact_plan(items, declarations): - if type(items) is not list: - _fail('Fork requires an explicit payload preservation plan.') - plan = {} - for item in items: - _shape(item, ('declaration', 'preservation'), 'payload preservation edge') - value = validate_declaration(item['declaration']) - if item['preservation'] not in ('unique', 'regenerable') or declarations.get(value['sha256']) != value: - _fail('Fork payload policy refers to an undeclared or changed artifact.') - previous = plan.get(value['sha256']) - if previous is None or item['preservation'] == 'unique': - plan[value['sha256']] = item['preservation'] - if set(plan) != set(declarations): - _fail('Every retained artifact declaration needs an explicit fork preservation disposition.') - return plan - - -def _source_current(store, source): - current = store.snapshot() - if current['project_id'] != source['project_id'] or current['checkpoint'] != source['checkpoint']: - _fail('Source project changed during fork preparation.', 'stale', 409) - - -def _result(snapshot, overlay, reconciled): - return dict(snapshot=copy.deepcopy(snapshot), fork=copy.deepcopy(overlay), reconciled=reconciled) - - -def _reconcile_existing(sources, trusted_root_fd, relative, root_path, - expected_input, request_key, digest, validate_candidate, *, composed): - with ProjectStore.open_at(trusted_root_fd, relative, root_path=root_path) as destination: - snapshot = destination.snapshot() - overlay = decode_fork(snapshot['records'], project_id=snapshot['project_id']) - if composed: - from project_composition import decode_records, composition_input - document = decode_records(snapshot['records'],project_id=snapshot['project_id']).get(snapshot['project_id']) - observed_input = None if document is None else composition_input(document['sources'],document['directory'],document['name'],document['request_key']) - observed_digest = None if document is None else document['request_sha256'] - else: - observed_input = None if overlay is None else _input(overlay['source'],overlay['directory'],overlay['name'],overlay['request_key']) - observed_digest = None if overlay is None else overlay['request_sha256'] - if observed_input != expected_input or observed_digest != digest: - _fail('Existing destination does not belong to this exact fork request.', 'conflict', 409) - if destination.reconcile(request_key, digest) is None: - _fail('Existing destination lacks the canonical fork publication receipt.', 'conflict', 409) - declarations = _declarations(snapshot['records']) - with ProjectArtifacts(destination) as artifacts: - validate_candidate(copy.deepcopy(snapshot['records']),snapshot['project_id'],artifacts) - for identity in overlay['artifacts']['copied']: - if artifacts.status(declarations[identity])['state'] != 'verified': - _fail('Previously copied fork payloads are missing or corrupt.', 'unavailable', 409) - for item in sources: - _source_current(item['store'],item['identity']) - destination.save(snapshot['checkpoint']) - result = _result(destination.snapshot(),overlay,True) - if composed:result['composition']=copy.deepcopy(document) - return result - - -def publish_fork(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, - name, expected_checkpoint, request_key, quiescent, build_candidate, - validate_candidate): - """Publish an independent single-source fork through the shared lifecycle.""" - if not callable(validate_candidate): - _fail('Fork requires complete validation.','unavailable',503) - return _publish_project(source_store,source_artifacts,trusted_root_fd,relative, - root_path=root_path,name=name,expected_checkpoint=expected_checkpoint,request_key=request_key, - quiescent=quiescent,build_candidate=build_candidate, - validate_candidate=lambda records,identity,_artifacts:validate_candidate(records,identity)) - - -def recover_committed_publication(trusted_root_fd, staging, relative, *, root_path, - expected_project_id, expected_checkpoint, - expected_request, quiescent): - """Publish one explicitly selected committed stage without rebuilding it. - - This operator recovery accepts a frozen historical snapshot, not a new read - of its original sources. The caller owns current filesystem authority and - cessation of the former process. The store's exclusive root lock, complete - domain validation, exact original request/commit receipt, payload custody, - and clean captured files are required before a no-overwrite rename. No SQL - projection, source permission, source runtime or worker is restored here. - After an uncertain rename, repeat this exact recovery to inspect the final - destination; never repeat composition or create another stage. - """ - if quiescent is not True: - _fail('Recovery requires proven cessation of the former publication owner.', 'busy', 409) - _uuid(expected_project_id);_sha(expected_checkpoint) - stage_parts=_relative_parts(staging);parts=_relative_parts(relative) - if (stage_parts[:-1]!=parts[:-1] - or not re.fullmatch(r'\.tuldok-fork-[a-f0-9]{32}',stage_parts[-1]) - or stage_parts==parts): - _fail('Recovery requires an explicit fork stage beside its original destination.') - if type(expected_request) is not dict or expected_request.get('directory')!=relative: - _fail('Recovery requires the exact original publication request.') - key=expected_request.get('request_key') - if type(key) is not str or not REQUEST_KEY.fullmatch(key): - _fail('Recovery requires the original publication request key.') - digest=_digest(expected_request) - published=False - try: - with anchored_directory(trusted_root_fd,'/'.join(parts[:-1]),root_path=root_path) as parent: - def exists(name): - try:os.stat(name,dir_fd=parent,follow_symlinks=False);return True - except FileNotFoundError:return False - stage_exists,final_exists=exists(stage_parts[-1]),exists(parts[-1]) - if stage_exists and final_exists: - _fail('Both stage and destination exist; neither was replaced.', 'conflict', 409) - if not stage_exists and not final_exists: - _fail('Neither the selected stage nor its destination is available.', 'unavailable', 409) - selected=relative if final_exists else staging - with ProjectStore.open_at(trusted_root_fd,selected,root_path=root_path) as store: - snapshot=store.snapshot() - if (snapshot['project_id']!=expected_project_id - or snapshot['checkpoint']!=expected_checkpoint): - _fail('Recovery project identity or checkpoint differs.', 'stale', 409) - overlay=decode_fork(snapshot['records'],project_id=expected_project_id) - if overlay is None:_fail('Recovery requires a committed independent publication.') - from project_composition import decode_records, composition_input - composition=decode_records(snapshot['records'],project_id=expected_project_id).get(expected_project_id) - observed=(composition_input(composition['sources'],composition['directory'],composition['name'],composition['request_key']) - if composition is not None else _input(overlay['source'],overlay['directory'],overlay['name'],overlay['request_key'])) - if observed!=expected_request or store.reconcile(key,digest) is None: - _fail('Committed stage does not match the original request and receipt.', 'conflict', 409) - from project_projection import validate_project_records - from project_files import inspect_captured - declarations=_declarations(snapshot['records']) - with ProjectArtifacts(store) as artifacts: - validate_project_records(snapshot['records'],project_id=expected_project_id,artifacts=artifacts) - checks=artifacts.status_many([declarations[identity] for identity in overlay['artifacts']['copied']]) - if any(item['state']!='verified' for item in checks): - _fail('Committed publication payloads are missing or corrupt.', 'unavailable', 409) - with store.root_directory() as project_root: - if any(item['state'] not in ('unchanged','deleted') - for item in inspect_captured(snapshot['records'],expected_project_id,project_root)): - _fail('Committed stage has changed or missing working files.', 'working_files_changed', 409) - store.save(expected_checkpoint) - if not final_exists: - # Recheck the selected directory entry while the store root - # remains locked; an external rename must not substitute it. - with store.root_directory() as project_root: - entry=os.stat(stage_parts[-1],dir_fd=parent,follow_symlinks=False) - actual=os.fstat(project_root) - if (entry.st_dev,entry.st_ino)!=(actual.st_dev,actual.st_ino): - _fail('Selected stage directory changed.', 'stale', 409) - _rename_noreplace(parent,stage_parts[-1],parts[-1]);published=True - os.fsync(parent) - return dict(project_id=expected_project_id,checkpoint=expected_checkpoint, - directory=relative,request_key=key,reconciled=final_exists, - payloads_copied=0,selected=False) - except BaseException as error: - # Anchored directory owners also translate IO failures. Preserve the - # uncertain post-rename outcome through either exception representation. - if published: - raise ProjectForkError('Committed publication was renamed but acknowledgement is uncertain; reconcile this exact recovery.', - 'unknown',503) from error - if isinstance(error,OSError): - code='conflict' if error.errno==errno.EEXIST else 'unavailable' - raise ProjectForkError('Committed publication recovery is unavailable; no stage was rebuilt.', - code,503 if code!='conflict' else 409) from error - raise - - -def _publish_project(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, - name, expected_checkpoint, request_key, quiescent, build_candidate, - validate_candidate, retained_sources=()): - """One native stage/commit/no-clobber lifecycle for forks and composition. - - The workspace owns authorization, quiescence and all borrowed source owners. - Every source is pinned and rechecked. Composed candidates additionally prove - complete source provenance. Validators receive the actual source/destination - artifact owner; no imported runtime, permission or source path is activated. - No failed or unknown stage is deleted or selected by this owner. - """ - if quiescent is not True: - _fail('Fork requires proven quiescence under the workspace lifecycle guard.', 'busy', 409) - if not callable(build_candidate) or not callable(validate_candidate): - _fail('Fork requires pure domain construction and complete validation.', 'unavailable', 503) - if type(retained_sources) not in (tuple,list):_fail('Explicit retained source owners required.') - if type(request_key) is not str or not REQUEST_KEY.fullmatch(request_key):_fail('Invalid fork request key.') - _sha(expected_checkpoint) - if type(name) is not str or not name.strip() or len(name)>200:_fail('Fork name must contain 1–200 characters.') - _encode(name);parts=_relative_parts(relative) - owners=[dict(store=source_store,artifacts=source_artifacts,checkpoint=expected_checkpoint)] - for item in retained_sources: - _shape(item,('store','artifacts','checkpoint'),'retained source owner') - _sha(item['checkpoint']);owners.append(dict(item)) - sources=[];snapshots=[];seen=set() - for item in owners: - snapshot=item['store'].snapshot() - if snapshot['checkpoint']!=item['checkpoint']:_fail('Expected source checkpoint changed.','stale',409) - if snapshot['project_id'] in seen:_fail('Retained source project identity repeats.','conflict',409) - seen.add(snapshot['project_id']) - validate_candidate(copy.deepcopy(snapshot['records']),snapshot['project_id'],item['artifacts']) - identity={key:snapshot[key] for key in ('project_id','checkpoint')} - sources.append(dict(store=item['store'],artifacts=item['artifacts'],identity=identity)) - snapshots.append(snapshot) - composed=bool(retained_sources);source=sources[0]['identity'] - if composed: - from project_composition import composition_input - expected_input=composition_input(snapshots,relative,name,request_key) - else:expected_input=_input(source,relative,name,request_key) - digest=_digest(expected_input) - source_roots=[];staged=artifacts=None;staging_relative=None;committed=None;published=False - def sources_current(): - for item in sources:_source_current(item['store'],item['identity']) - try: - for item in sources: - with item['store'].root_directory() as borrowed:source_roots.append(os.dup(borrowed)) - with anchored_directory(trusted_root_fd,'/'.join(parts[:-1]),root_path=root_path) as parent: - if any(_inside(parent,root) for root in source_roots):_fail('Fork destination cannot overlap any source project.') - try:existing=os.stat(parts[-1],dir_fd=parent,follow_symlinks=False) - except FileNotFoundError:existing=None - if existing is not None: - if not stat.S_ISDIR(existing.st_mode):_fail('Fork destination is an existing or unsafe filesystem object.','conflict',409) - selected=os.open(parts[-1],DIR_FLAGS,dir_fd=parent) - try: - if any(_inside(root,selected) or _inside(selected,root) for root in source_roots): - _fail('Fork destination aliases or overlaps a source project.') - finally:os.close(selected) - return _reconcile_existing(sources,trusted_root_fd,relative,root_path, - expected_input,request_key,digest,validate_candidate,composed=composed) - staging_name='.tuldok-fork-'+uuid.uuid4().hex - staging_relative='/'.join([*parts[:-1],staging_name]) - staged=ProjectStore.create_at(trusted_root_fd,staging_relative,name, - 'fork-stage.'+hashlib.sha256(request_key.encode()).hexdigest(),digest,root_path=root_path) - destination_id=staged.snapshot()['project_id'] - if destination_id in seen:_fail('Fork destination identity must be new.') - candidate=build_candidate(copy.deepcopy(snapshots if composed else snapshots[0]),destination_id) - _shape(candidate,('records','mapping','archived','artifacts'),'owned candidate') - if type(candidate['records']) is not list:_fail('Fork candidate records must be a list.') - records=copy.deepcopy(candidate['records']);indexed={} - for item in records: - _shape(item,('family','id','document'),'candidate envelope') - _identifier(item['family'],'Fork candidate family');_identifier(item['id'],'Fork candidate record ID') - key=item['family'],item['id'] - if key in indexed:_fail('Fork candidate repeats a record identity.') - indexed[key]=item - source_index={(r['family'],r['id']):r for snapshot in snapshots for r in snapshot['records']} - if composed: - from project_composition import validate_source_memberships - validate_source_memberships(records,snapshots,destination_id) - else: - for identity,original in source_index.items(): - if identity not in indexed or _encode(indexed[identity])!=_encode(original): - _fail('Fork must preserve every original canonical envelope exactly.') - archived=[_locator(item) for item in candidate['archived']] - if not set(archived)<=set(source_index):_fail('Fork may archive only retained source records.') - required={key for key in source_index if key[0] in ARCHIVE_FAMILIES} - for snapshot in snapshots: - previous=decode_fork(snapshot['records'],project_id=snapshot['project_id']) - if previous is not None:required.update(_locator(item) for item in previous['archived']) - if not required<=set(archived):_fail('Fork must archive every source editable value and operation receipt.') - declarations=_declarations(records);expected_declarations={} - for snapshot in snapshots: - for identity,value in _declarations(snapshot['records']).items(): - if identity in expected_declarations and expected_declarations[identity]!=value: - _fail('Source artifact declarations conflict.') - expected_declarations[identity]=value - if declarations!=expected_declarations:_fail('Fork must retain the source payload inventory without regenerating bytes.') - plan=_artifact_plan(candidate['artifacts'],declarations);selected={} - for number,item in enumerate(sources): - own=_declarations(snapshots[number]['records']);identities=sorted(own) - observations=item['artifacts'].status_many([own[key] for key in identities]) - for identity,observation in zip(identities,observations): - status=observation['state'] - if status=='corrupt':_fail('Source fork payload is corrupt.','invalid',400) - if status=='verified':selected.setdefault(identity,number) - elif status!='missing':_fail('Fork requires verified payload custody.','unavailable',409) - copied=sorted(selected);missing=sorted(set(plan)-set(selected)) - if any(plan[key]=='unique' for key in missing): - _fail('Irreplaceable source bytes are missing; complete fork is unavailable.','unavailable',409) - overlay=dict(format=FORMAT,source=source,destination_project_id=destination_id, - request_key=request_key,request_sha256=_digest(_input(source,relative,name,request_key)),directory=relative,name=name, - mapping=copy.deepcopy(candidate['mapping']),archived=copy.deepcopy(candidate['archived']), - artifacts=dict(copied=copied,missing_regenerable=missing)) - records.append(dict(family=FAMILY,id=destination_id,document=overlay)) - decode_fork(records,project_id=destination_id) - validate_candidate(copy.deepcopy(records),destination_id,source_artifacts) - sources_current();artifacts=ProjectArtifacts(staged) - for number,item in enumerate(sources): - group=[declarations[key] for key in copied if selected[key]==number] - if group:item['artifacts'].copy_many_to(artifacts,group) - from project_files import materialize_captured - with staged.root_directory() as destination_root:materialize_captured(records,destination_id,destination_root) - sources_current();validate_candidate(copy.deepcopy(records),destination_id,artifacts) - committed=staged.commit(records,staged.snapshot()['checkpoint'],request_key,digest) - staged.save(committed['checkpoint']);sources_current() - try:_rename_noreplace(parent,staging_name,parts[-1]) - except FileExistsError as error: - raise ProjectForkError('Fork destination appeared during publication; it was not overwritten.','conflict',409) from error - published=True;os.fsync(parent);sources_current() - result=_result(committed,overlay,False) - if composed: - from project_composition import decode_records - result['composition']=copy.deepcopy(decode_records(records,project_id=destination_id)[destination_id]) - return result - except BaseException as error: - if published: - failure=ProjectForkError('Fork destination was published but acknowledgement is uncertain; retry this exact request.','unknown',503) - if committed is not None: - failure.receipt=dict(committed=True,selected=False,project_id=committed['project_id'],checkpoint=committed['checkpoint'],request_key=request_key,directory=relative) - raise failure from error - if isinstance(error,OSError): - code='conflict' if error.errno==errno.EEXIST else 'unavailable' - failure=ProjectForkError('Fork publication is unavailable; retained staging has not been selected.',code,409 if code=='conflict' else 503) - failure.staging_directory=staging_relative;raise failure from error - if staging_relative is not None:error.staging_directory=staging_relative - raise - finally: - if artifacts is not None:artifacts.close() - if staged is not None:staged.close() - for descriptor in source_roots:os.close(descriptor) diff --git a/project_fork_composition.py b/project_fork_composition.py deleted file mode 100644 index ee13410a..00000000 --- a/project_fork_composition.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Pure fixed-owner composition for an independent project fork. - -The lifecycle owner supplies a validated source checkpoint, creation fact and -fresh ID allocator. This module never opens files, SQL, runtimes or permissions. -The publication owner adds the fork overlay and verifies actual payload custody. -""" -import copy - -from project_artifacts import validate_declaration -from project_data_fork import fork_records as fork_data -from project_material_fork import fork_records as fork_material, OWNED as MATERIAL_EDITABLE -from project_files import fork_records as fork_files -from project_fork import ARCHIVE_FAMILIES, ProjectForkError, decode_fork, ID, _uuid, _sha -from project_store import _records - - -def _fail(message, code='invalid', status=400): - raise ProjectForkError(message, code, status) - - -def artifact_plan(records): - """Give every intrinsic declaration a conservative explicit disposition. - - Runtime files alone can be regenerable. Any unique owner edge to the same - bytes wins, including archived sources and past producer wire observations. - Unclassified declarations remain unique; no filename/path guesses are used. - """ - try: - declarations = {} - for item in records: - if item['family'] != 'project_artifact': continue - declaration = validate_declaration(item['document']) - sha = declaration['sha256'] - if item['id'] != sha or sha in declarations: - _fail('Fork artifact declaration identity differs or repeats.') - declarations[sha] = declaration - unique, regenerable, missing = set(), set(), [] - - def mark(sha, record, role, name, size=None, *, policy='unique'): - _sha(sha) - declaration = declarations.get(sha) - if declaration is None: - if policy == 'unique': - missing.append(dict(family=record['family'],id=record['id'],role=role,name=name, - sha256=sha,bytes=size)) - return - if size is not None and declaration['bytes'] != size: - _fail('Fork owner byte count differs from its intrinsic declaration.') - (unique if policy == 'unique' else regenerable).add(sha) - - # Finite Workbench source fields. Both original and oriented image bytes - # are historical identity; re-encoding does not reproduce their hashes. - for record in records: - family, doc = record['family'], record['document'] - edges = [] - if family == 'workbench_record': - source = doc['source'] - if source['kind'] == 'image': edges = [source['original'],source['oriented']] - elif source['kind'] != 'text': edges = [source['asset']] - elif family == 'dataset_release': edges = [doc['asset']] - for edge in edges: - mark(edge['artifact']['sha256'],record,edge['role'],edge['role']) - - # The material owner has already closed and correlated these inventories - # with its hashed model, recipe, snapshot and runtime definitions. - from project_material import decode_records as material_records - for (family, locator), doc in material_records(records).items(): - record = dict(family=family,id=locator) - for edge in doc.get('artifacts',[]): - runtime_file = family == 'runtime_material' and edge['role'] == 'runtime_file' and edge['preservation'] == 'regenerable' - mark(edge['blob']['sha256'],record,edge['role'],edge['name'],edge['blob']['bytes'], - policy='regenerable' if runtime_file else 'unique') - - # The history owner distinguishes recorded runtime dependencies from - # outputs/proofs/host sources; a weaker payload label alone grants none. - from project_history import declarations as run_declarations, fork_preservation - for record in records: - if record['family'] not in ('run_history', 'run_observation'): continue - doc = (record['document']['observation'] if record['family'] == 'run_observation' - else record['document']) - by_sha = {edge['blob']['sha256']:edge['name'] for edge in doc['payloads']} - publication = doc['publication'] - preservation = fork_preservation(doc) - if publication is not None: - for name in ('source','result'): - if publication[name] is not None: by_sha[publication[name]['sha256']] = 'publication.'+name - for declaration in run_declarations(dict(family='run_history', id=record['id'], document=doc)): - blob = declaration['document'] - mark(blob['sha256'],record,'run_payload',by_sha[blob['sha256']],blob['bytes'], - policy=preservation[blob['sha256']]) - run = doc['run'] - if run.get('execution_kind') == 'external_recipe': - for name, sha in run['material']['host_files'].items(): - mark(sha,record,'host_source',name) - - # Native measured proof/source/output bytes retain unique custody even - # when identical bytes also occur in a regenerable runtime dependency. - for record in records: - if record['family'] != 'native_execution_history': continue - for edge in record['document']['payloads']: - mark(edge['blob']['sha256'],record,edge['role'],edge['name'],edge['blob']['bytes']) - - # Scalar sources are immutable original evidence, including when the - # same bytes are also declared by a regenerable runtime dependency. - from project_measurements import decode_records as measurement_records, FAMILY as MEASUREMENT_FAMILY - for locator, doc in measurement_records(records).items(): - blob = doc['blob'] - mark(blob['sha256'], dict(family=MEASUREMENT_FAMILY, id=locator), - 'measurement_packet', 'measurements.json', blob['bytes']) - - # Existing pure getter covers the four producer families, entry history - # payload maps, and variation prompt-batch request/response observations. - from project_producers import FAMILIES as PRODUCER_FAMILIES, _referenced_artifacts - for record in records: - if record['family'] in PRODUCER_FAMILIES: - for sha in _referenced_artifacts([record]): - mark(sha,record,'producer_wire','producer_wire') - if missing: - failure = ProjectForkError('Complete fork requires declarations for unique historical payloads.', - 'missing_unique',409) - failure.missing_unique = sorted(missing,key=lambda item:(item['family'],item['id'],item['role'],item['name'],item['sha256'])) - raise failure - return [dict(declaration=copy.deepcopy(declarations[sha]), - preservation='regenerable' if sha in regenerable and sha not in unique else 'unique') - for sha in sorted(declarations)] - except ProjectForkError: raise - except (KeyError, TypeError, ValueError, AttributeError, OverflowError, RecursionError): - _fail('Malformed fixed-owner fork artifact closure.') - - -def build_candidate(snapshot, destination_project_id, *, created_at, id_factory): - """Compose originals, independent editable drafts and complete byte policy. - - id_factory(family, source_envelope_id, source_revision) must return fresh UUID - strings. Domain owners retain their own version, provenance and binding rules. - No historical ID, origin catalog or hash-bearing document is rewritten. - """ - if type(snapshot) is not dict or set(snapshot) != {'project_id','name','checkpoint','records'}: - _fail('Fork requires one complete canonical source snapshot.') - _uuid(snapshot['project_id']); _uuid(destination_project_id); _sha(snapshot['checkpoint']) - if destination_project_id == snapshot['project_id'] or not callable(id_factory): - _fail('Fork requires an independent destination identity and explicit ID allocator.') - _records(snapshot['records']) - source = copy.deepcopy(snapshot['records']) - previous = decode_fork(source,project_id=snapshot['project_id']) - used = {snapshot['project_id'],destination_project_id} - used.update(r['id'] for r in source if ID.fullmatch(r['id'])) - used.update(r['document']['current']['id'] for r in source if r['family'] in MATERIAL_EDITABLE) - allocated = {} - - def allocate(family, source_id, revision): - key = family,source_id,revision - if key not in allocated: - identity = id_factory(*key) - _uuid(identity) - if identity in used: _fail('Fork allocator reused an existing or allocated editable identity.') - used.add(identity); allocated[key] = identity - return allocated[key] - - parts = [fork_data(source,snapshot['project_id'],snapshot['checkpoint'],destination_project_id, - id_factory=allocate,created_at=created_at), - fork_material(source,snapshot['project_id'],snapshot['checkpoint'],destination_project_id, - id_factory=allocate,created_at=created_at), - fork_files(source,snapshot['project_id'],snapshot['checkpoint'],destination_project_id)] - indexed = {(r['family'],r['id']):r for r in source} - mapping = [] - for part in parts: - mapping.extend(copy.deepcopy(part['mapping'])) - for record in part['records']: - key = record['family'],record['id'] - if key in indexed: _fail('Fork owner generated a colliding canonical record.') - indexed[key] = copy.deepcopy(record) - records = [indexed[key] for key in sorted(indexed)] - _records(records) - archived = {(r['family'],r['id']) for r in source if r['family'] in ARCHIVE_FAMILIES} - if previous is not None: archived.update((item['family'],item['id']) for item in previous['archived']) - mapping.sort(key=lambda item:(item['family'],item['source_id'],item['source_revision'] or 0,item['destination_id'])) - return dict(records=records,mapping=mapping, - archived=[dict(family=family,id=identity) for family,identity in sorted(archived)], - artifacts=artifact_plan(source)) diff --git a/project_history.py b/project_history.py deleted file mode 100644 index e3b1c7c7..00000000 --- a/project_history.py +++ /dev/null @@ -1,1392 +0,0 @@ -"""Inert project Study, lineage and Run facts, with owner-shared validation. - -Historical identity is always (origin catalog, original ID). No imported fact is -inserted into a worker, admission, grant or request replay table. The optional -SQL connection is only a query engine for bounded supplied immutable JSON. -""" -import base64 -from collections.abc import Mapping -import copy -import hashlib -import re -import uuid - -import agent_api - -from experiments import (DIMENSIONS, Experiments, budget, request_key, - select_retained_checkpoint, reservation_totals, validate_candidate_evidence, new_study_document, edit_study_document, new_trial_document, new_candidate_document, validate_retained_candidate, - validate_retained_policy, validate_retained_reservation, - validate_retained_study, validate_retained_trial) -from experiment_lineage import (INTENT_LIMIT, validate_retained_ancestry, - validate_retained_intent, validate_retained_lineage) -from model_definitions import checked_json, natural, reference, text -from pipeline import closed, digest, identifier, model_aware_graph, source_content_hash -from project_session import operation_documents, operation_records -from projects import relative_path -from runs import (ACTIVE, TERMINAL, TEST_FILES, MAX_ARTIFACT_BYTES, validate_retained_run, - project_run_projection, public_run_projection, artifact_capture_source, validate_external_artifact_inventory) -from workbench import WorkbenchError, timestamp -from project_session import session_derived, session_operations, session_record, session_records, session_records_for - -FAMILY_FORMATS = { - 'study_history': 'project_study_history_v1', - 'trial_history': 'project_trial_history_v1', - 'run_history': 'project_run_history_v1', - 'run_observation': 'project_run_observation_v1', - 'candidate_history': 'project_candidate_history_v1', -} -MAX_HISTORY_BYTES = 40 * 1024**2 -MAX_HISTORY_RECORDS = 100_000 -NAME = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.-]{0,119}$') -SHA = re.compile(r'^[a-f0-9]{64}$') -RUN_OBSERVATION_IDENTITY = ('dataset_id','id','request_key','request_sha256','graph_sha256','graph_document_sha256', - 'material_sha256','release_id','release_sha256','release_bytes','admission_id','approval_id', - 'approval_material_sha256','actor_id','parent_run_id','attempt','created_at','execution_kind','result_kind') - -OPERATIONS = frozenset(('studies.create','studies.save','studies.prepare_trial','studies.record_lineage', - 'studies.branch_trial','studies.prepare_branch','studies.select_candidate','studies.prepare_package', - 'studies.prepare_verifier','studies.continue','studies.stop')) - - -def _invalid(message, code='invalid'): - raise WorkbenchError(message, code, 409) - - -def _shape(value, fields, name): - if type(value) is not dict or set(value) != set(fields): - _invalid('Invalid '+name+' fields.') - - -def _items(value, name, maximum=10000): - if type(value) is not list or len(value)>maximum: - _invalid(name+' must be a bounded list.') - return value - - -def _sha(value): - if type(value) is not str or not SHA.fullmatch(value): - _invalid('Historical digest must be an exact SHA256.') - return value - - -def envelope(family, origin, identity, document): - identifier(origin,'Origin catalog');identifier(identity,'Historical identity') - return {'family':family,'id':origin+'.'+identity,'document':document} - - -def _audit(facts): - """Retain classified scientific audit facts, never arbitrary access payloads.""" - for fact in _items(facts,'Scientific audit',100000): - _shape(fact,('actor_id','action','target','details','result','occurred_at'),'scientific audit') - if fact['actor_id'] is not None:identifier(fact['actor_id'],'Historical actor') - action=fact['action'];details=fact['details'] - text(fact['occurred_at'],'Audit time',80);identifier(fact['target'],'Audit target') - allowed=OPERATIONS|{'studies.admit','studies.approve','studies.reconcile','studies.candidate_selected','studies.lineage.record', - 'runs.admit','runs.admit_external','runs.update','runs.reconcile','runs.expose_test'} - if action not in allowed:_invalid('Unclassified historical scientific audit action.','unsupported') - if fact['result'] in ('denied','failed'): - closed(details,('code',)) - if details['code'] not in ('invalid','unavailable','unsupported','stale','conflict','not_found','denied', - 'forbidden','unknown','failed','cancelled','busy','interrupted'):_invalid('Unclassified historical failure code.') - continue - if fact['result'] not in ('success','interrupted'):_invalid('Unsupported historical audit result.') - keys=set(details) if type(details) is dict else set() - if action=='studies.create': - closed(details,('policy_sha256','limits'));_sha(details['policy_sha256']);budget(details['limits']) - elif action in ('studies.save','studies.continue','studies.stop'): - if keys not in ({'revision'},{'revision','epoch'}):_invalid('Unclassified Study transition audit.') - elif action=='studies.reconcile': - if keys not in ({'revision','epoch'},{'state'}):_invalid('Unclassified Study recovery audit.') - elif action=='studies.prepare_trial': - if keys not in ({'trial_id'},{'trial_id','revision','epoch'}):_invalid('Unclassified Trial preparation audit.') - elif action=='studies.admit':closed(details,('run_id','reservation_id','request_sha256','revision','epoch')) - elif action=='studies.approve': - if keys not in ({'trial_id','material_sha256','approval_id'}, - {'trial_id','material_sha256','approval_id','consumer_actor_id','created_at','expires_at'}): - _invalid('Unclassified Study approval audit.') - if 'created_at' in details: - from access import MAX_APPROVAL_TTL - if (type(details['created_at']) is not int or type(details['expires_at']) is not int or - not details['created_at']<details['expires_at']<=details['created_at']+MAX_APPROVAL_TTL): - _invalid('Invalid historical Study approval lifetime.') - elif action=='studies.select_candidate': - if keys not in ({'study_id','run_id','checkpoint_sha256'},{'study_id','run_id','checkpoint_sha256','decision'}): - _invalid('Unclassified candidate selection audit.') - if 'decision' in details and details['decision'] not in ('diagnostic','qualified'):_invalid('Invalid decision audit.') - elif action=='studies.candidate_selected':closed(details,('candidate_id','revision','epoch')) - elif action=='studies.lineage.record': - if keys not in ({'intent_sha256'},{'material_sha256','intent_sha256'}):_invalid('Unclassified lineage audit.') - elif action in ('studies.prepare_package','studies.prepare_verifier'):closed(details,('study_id','trial_id','material_sha256')) - elif action=='runs.admit':closed(details,('request_sha256','graph_sha256','release_sha256')) - elif action=='runs.admit_external':closed(details,('material_sha256','approved_material_sha256','approval_id','request_sha256')) - elif action in ('runs.update','runs.reconcile'): - if keys not in ({'state','request_sha256','graph_sha256','release_sha256'}, - {'state','request_sha256','graph_sha256','material_sha256','approval_id','custody_phase','receipt_sha256'}): - _invalid('Unclassified Run observation audit.') - elif action=='runs.expose_test':closed(details,('event_id','release_sha256','reason')) - else:_invalid('Unclassified historical scientific audit details.','unsupported') - for key,value in details.items(): - if key.endswith('_sha256') and value is not None:_sha(value) - elif key.endswith('_id') and value is not None:identifier(value,key) - elif key in ('revision','epoch','created_at','expires_at'):natural(value,key,1 if key=='revision' else 0) - elif key=='state' and value not in set(ACTIVE)|set(TERMINAL)|{'open','stopping','stopped'}:_invalid('Invalid historical state audit.') - elif key=='custody_phase' and value not in ('prepared','barrier_ready','go_intent','drained'):_invalid('Invalid custody audit.') - elif key=='reason':text(value,'Disclosure reason',1000) - - -def _operations(facts, catalog_id): - seen=set() - for fact in _items(facts,'Historical operations',100000): - _shape(fact,('actor_id','operation_id','request_key','request_sha256','result'),'historical operation') - identifier(fact['actor_id'],'Historical actor');request_key(fact['request_key']);_sha(fact['request_sha256']) - if fact['operation_id'] not in OPERATIONS: - _invalid('Unsupported historical operation.','unsupported') - key=(fact['actor_id'],fact['operation_id'],fact['request_key']) - if key in seen:_invalid('Duplicate historical operation fact.') - seen.add(key);checked_json(fact['result']) - if not validate_operation_result(fact['operation_id'],fact['result']): - _invalid('Unclassified historical operation result.','unsupported') - - -def _payloads(edges): - result={} - for edge in _items(edges,'Historical payloads',100000): - _shape(edge,('name','blob','population','role','preservation','required_for'),'historical payload') - name=edge['name'] - relative_path(name) - if name in result: - _invalid('Unsafe or duplicate historical payload name.') - _shape(edge['blob'],('sha256','bytes'),'payload byte identity') - _sha(edge['blob']['sha256']);natural(edge['blob']['bytes'],'Payload bytes') - if edge['population'] not in ('train','validation','test','unknown'): - _invalid('Unsupported historical payload population.') - if edge['preservation'] not in ('unique','regenerable'): - _invalid('Unsupported payload preservation meaning.') - text(edge['role'],'Payload role',120) - for use in _items(edge['required_for'],'Payload required uses',64):text(use,'Required use',120) - result[name]=edge - return result - - -def _approval(proof, run): - """Validate exact old associations; return only the common proof projection.""" - if proof is None:return None - _shape(proof,('grant','consumption','admission'),'historical admission proof') - grant,consumption,admission=proof['grant'],proof['consumption'],proof['admission'] - if grant is not None: - _shape(grant,('id','catalog_id','approver_actor_id','approver_credential_id','consumer_actor_id', - 'material_sha256','created_at','expires_at','revoked_at'),'historical approval') - for key in ('id','catalog_id','approver_actor_id','approver_credential_id','consumer_actor_id'): - identifier(grant[key],key) - _sha(grant['material_sha256']) - for key in ('created_at','expires_at'):natural(grant[key],key) - if grant['expires_at']<=grant['created_at']:_invalid('Historical approval lifetime differs.') - if grant['revoked_at'] is not None:natural(grant['revoked_at'],'Revocation time') - if consumption is not None: - _shape(consumption,('grant_id','run_id','admission_id','consumer_credential_id','request_sha256','consumed_at'), - 'historical approval consumption') - for key in ('grant_id','run_id','admission_id','consumer_credential_id'):identifier(consumption[key],key) - _sha(consumption['request_sha256']);natural(consumption['consumed_at'],'Consumption time') - if admission is not None: - _shape(admission,('id','catalog_id','actor_id','credential_id','kind','job_id','request_digest','effects','created_at'), - 'historical admission') - for key in ('id','catalog_id','actor_id','credential_id','job_id'):identifier(admission[key],key) - _sha(admission['request_digest']);natural(admission['created_at'],'Admission time') - if (admission['kind']!='runs' or admission['id']!=run['admission_id'] or admission['catalog_id']!=run['dataset_id'] or - admission['job_id']!=run['id'] or admission['request_digest']!=run['request_sha256']): - _invalid('Historical admission differs from its Run.') - # These are original closed owner-emitted effect observations, never - # capabilities. Unknown action/target text cannot enter this proof. - if run.get('execution_kind')=='external_recipe': - expected={'runs.update':[run['id']]} - else: - recipe=run['graph'].get('recipe','image_classification') - expected={'pipeline.inspect_release':[run['release_id']], - 'dataset_releases.locate':[run['release_id']],'pipeline.probe':[recipe], - 'pipeline.probe_native':[recipe],'pipeline.definition_identity':[recipe], - 'pipeline.exposures':[None],'runs.update':[run['id']]} - if admission['effects']!=expected:_invalid('Unclassified historical admission effects.','unsupported') - if grant is None and consumption is None:return None - if grant is None or consumption is None or admission is None: - _invalid('Historical approval proof is incomplete.') - if (grant['id']!=run.get('approval_id') or consumption['grant_id']!=grant['id'] or - consumption['run_id']!=run['id'] or consumption['admission_id']!=admission['id'] or - consumption['request_sha256']!=run['request_sha256'] or - consumption['consumer_credential_id']!=admission['credential_id'] or - admission['actor_id']!=grant['consumer_actor_id']): - _invalid('Historical approval consumption association differs.') - return dict(material_sha256=grant['material_sha256'],consumer_actor_id=grant['consumer_actor_id'], - catalog_id=grant['catalog_id'],run_id=consumption['run_id'],admission_id=consumption['admission_id'], - request_sha256=consumption['request_sha256']) - - -def capture_run(run, *, admission_proof=None, payloads, publication=None): - """Create an inert evidence projection; reject unknown source fields.""" - # Continuation state has its own closed evidence shape, never a replayable - # field in the saved Run consumed by recovery. - if publication is None and run.get('publication_continuation') is not None: - publication={'continuation':copy.deepcopy(run['publication_continuation']),'source':None,'result':None} - value=project_run_projection(run) - payloads=copy.deepcopy(payloads) - if run.get('execution_kind')!='external_recipe' and not any(item['name']=='release.zip' for item in payloads): - payloads.append({'name':'release.zip','blob':{'sha256':run['release_sha256'],'bytes':run['release_bytes']}, - 'population':'unknown','role':'frozen_input','preservation':'unique','required_for':['input_proof','scientific_inspection']}) - doc=dict(format=FAMILY_FORMATS['run_history'],origin_catalog_id=run['dataset_id'],run=value, - admission_proof=copy.deepcopy(admission_proof),payloads=copy.deepcopy(payloads),publication=copy.deepcopy(publication)) - return envelope('run_history',run['dataset_id'],run['id'],doc) - - -def payload_set_sha256(origin_catalog_id,run_id,payloads): - """New disclosure identity, separate from every historical proof namespace.""" - identifier(origin_catalog_id,'Origin catalog');identifier(run_id,'Run ID') - edges=_payloads(payloads) - value={'format':'project_disclosure_payload_set_v1','origin_catalog_id':origin_catalog_id,'run_id':run_id, - 'payloads':[{'name':name,**edge['blob']} for name,edge in sorted(edges.items())]} - return hashlib.sha256(agent_api.encoded(value)).hexdigest() - - -def validate_disclosure_fact(value): - fields=('format','origin_catalog_id','run_id','payload_set_sha256','population','reason','actor_id','occurred_at') - if type(value) is dict and value.get('format')=='project_disclosure_v2': - fields+=('observation_sha256',) - _shape(value,fields, - 'historical disclosure observation') - if value['format'] not in ('project_disclosure_v1','project_disclosure_v2') or value['population']!='all': - _invalid('Unsupported historical disclosure observation.','unsupported') - for key in ('origin_catalog_id','run_id','actor_id'):identifier(value[key],key) - _sha(value['payload_set_sha256']);text(value['reason'],'Disclosure reason',1000) - if value['format']=='project_disclosure_v2':_sha(value['observation_sha256']) - text(value['occurred_at'],'Disclosure observation time',80) - return copy.deepcopy(value) - - -def _observation_document(record): - _shape(record,('family','id','document'),'Run observation envelope') - doc=checked_json(record['document'],MAX_HISTORY_BYTES) - _shape(doc,('format','origin_catalog_id','run_id','sha256','observation'),'Run observation') - if record['family']!='run_observation' or doc['format']!=FAMILY_FORMATS['run_observation']: - _invalid('Unsupported Run observation.','unsupported') - identifier(doc['origin_catalog_id']);identifier(doc['run_id']);_sha(doc['sha256']) - from project_store import _encode as canonical - if (record['id']!=doc['sha256'] or hashlib.sha256(canonical(doc['observation'])).hexdigest()!=doc['sha256'] - or doc['observation'].get('origin_catalog_id')!=doc['origin_catalog_id'] - or doc['observation'].get('run',{}).get('id')!=doc['run_id']): - _invalid('Retained Run observation identity differs.') - return doc - - -def capture_run_observation(document): - """Retain one already validated Run-history document as inert exact evidence. - - This creates no disclosure fact or execution permission. The complete - history decoder still owns Run, admission and payload closure validation. - """ - from project_store import _encode as canonical - sha=hashlib.sha256(canonical(document)).hexdigest() - record={'family':'run_observation','id':sha,'document':{ - 'format':FAMILY_FORMATS['run_observation'],'origin_catalog_id':document['origin_catalog_id'], - 'run_id':document['run']['id'],'sha256':sha,'observation':copy.deepcopy(document)}} - _observation_document(record) - return record - - -def declarations(record): - if record['family']=='run_observation': - return declarations({'family':'run_history','document':record['document']['observation']}) - if record['family']!='run_history':return [] - values={} - edges=list(record['document']['payloads']) - publication=record['document']['publication'] - blobs=[edge['blob'] for edge in edges] - if publication is not None:blobs.extend(publication[key] for key in ('source','result') if publication[key] is not None) - for blob in blobs: - value={'format':'project_artifact_v1',**blob} - if blob['sha256'] in values and values[blob['sha256']]['document']!=value:_invalid('Historical artifact declarations differ.') - values[blob['sha256']]={'family':'project_artifact','id':blob['sha256'],'document':value} - return list(values.values()) - - -def fork_preservation(document): - """Copy obligations for an already decoded historical Run document. - - A payload label alone cannot weaken custody. Only an explicitly regenerable - historical runtime payload with the exact hash/count in this Run's recorded - runtime inventory may remain absent. Outputs, publication proofs and measured - host sources override that classification when they share the same bytes. - This projects copy policy; it does not rewrite the original retained facts. - """ - run=document['run'];runtime=set();protected={item['sha256'] for item in run.get('artifacts',[])} - if run.get('execution_kind')=='external_recipe': - material=run['material'] - for item in material.get('runtime',{}).get('files',{}).values(): - _sha(item['sha256']);natural(item['bytes'],'Historical runtime file bytes') - runtime.add((item['sha256'],item['bytes'])) - protected.update(material['host_files'].values()) - protected.update(item['sha256'] for item in material.get('captures',{}).values()) - result={} - for edge in document['payloads']: - blob=edge['blob'];digest=blob['sha256'] - regenerative=(edge.get('role')=='historical_runtime' and edge.get('preservation')=='regenerable' - and (blob['sha256'],blob['bytes']) in runtime) - policy='regenerable' if regenerative else 'unique' - if result.get(digest)!='unique':result[digest]=policy - publication=document['publication'] - if publication is not None: - protected.update(publication[name]['sha256'] for name in ('source','result') if publication[name] is not None) - for digest in protected:result[digest]='unique' - return result - - -class HistoryFacts: - def __init__(self): - self.studies={};self.trials={};self.runs={};self.candidates={};self.policies={};self.reservations={};self.observations={} - - def trial_ref(self, value): - _shape(value,('catalog_id','trial_id','material_sha256'),'historical Trial reference') - key=(value['catalog_id'],value['trial_id']) - if key not in self.trials:_invalid('Historical Trial dependency is missing.','unavailable') - trial=self.trials[key]['trial'] - if value['material_sha256']!=trial['material']['sha256']:_invalid('Historical Trial material differs.') - return trial - - -def _validate_run_document(doc, *, facts, declarations, materials): - """Shared exact scientific Run decoder for current and retained observations.""" - _shape(doc,('format','origin_catalog_id','run','admission_proof','payloads','publication'),'Run history') - if doc['format']!=FAMILY_FORMATS['run_history']:_invalid('Unsupported observed Run format.','unsupported') - origin=doc['origin_catalog_id'];identity=doc['run']['id'] - identifier(origin);identifier(identity) - run=project_run_projection(doc['run'],imported=True) - if run.get('dataset_id')!=origin:_invalid('Historical Run origin differs.') - edges=_payloads(doc['payloads']) - for edge in edges.values(): - declaration=declarations.get(edge['blob']['sha256']) - if declaration!={'format':'project_artifact_v1',**edge['blob']}: - _invalid('Historical artifact declaration is missing or contradictory.','unavailable') - for item in run.get('artifacts',[]): - edge=edges.get(item.get('name')) - if edge is None or edge['blob']!={key:item[key] for key in ('sha256','bytes')}: - _invalid('Run artifact closure differs.','unavailable') - if run.get('execution_kind')=='external_recipe': - for name,item in run['material']['captures'].items(): - edge=edges.get(name) - if edge is None or edge['blob']!={key:item[key] for key in ('sha256','bytes')}: - _invalid('Historical execution capture closure is missing or differs.','unavailable') - descriptor=run['material'].get('torch_runtime',run['material'].get('cpu_snapshot')) - if descriptor is not None: - sha=digest(descriptor) - if materials is None:_invalid('Historical runtime decoder is unavailable.','unavailable') - materials.runtime({'kind':'runtime','catalog_id':origin,'id':sha,'sha256':sha}) - if run['state']=='completed': - receipt=edges.get('custody-receipt.json') - if receipt is None or receipt['blob']['sha256']!=run['custody_receipt_sha256']: - _invalid('Historical custody receipt declaration is missing or differs.','unavailable') - if 'artifact-manifest.json' not in edges: - _invalid('Historical artifact manifest declaration is missing.','unavailable') - else: - release=edges.get('release.zip') - if release is None or release['blob']!={'sha256':run['release_sha256'],'bytes':run['release_bytes']}: - _invalid('Historical frozen input Release declaration is missing or differs.','unavailable') - reservation=facts.reservations.get((origin,identity));association=None;composition=None - if reservation is not None: - trial=facts.trials.get((origin,reservation['trial_id'])) - if trial is None:_invalid('Reservation Trial is missing.','unavailable') - association={key:reservation[key] for key in ('study_id','trial_id','actor_id','request_sha256','material_sha256')} - composition=materials.recipe(trial['trial']['material']) - elif run.get('approval_material_sha256') is not None: - sha=run['approval_material_sha256'] - composition=materials.recipe({'kind':'recipe_material','catalog_id':origin,'id':sha,'sha256':sha}) - approval=_approval(doc['admission_proof'],run) - # This validates the stored context/spec binding. Exact static context - # is rederived from retained source bytes on explicit evidence use. - validate_retained_run(run,catalog_id=origin,run_id=identity,request_key=run['request_key'], - request_sha256=run['request_sha256'],association=association,approval=approval,composition=composition, - extension_context=run.get('extension_context')) - parent=run.get('parent_run_id') - if parent is not None and (origin,parent) not in facts.runs:_invalid('Retry parent Run is missing.','unavailable') - if run.get('execution_kind')=='external_recipe': - for item in run['material']['inputs'].values(): - if item.get('kind')=='run_artifact': - source=item['source'];producer=facts.runs.get((source['catalog_id'],source['run_id'])) - if producer is None:_invalid('Producer Run closure is missing.','unavailable') - match=[edge for edge in producer['payloads'] if edge['name']==source['artifact_name']] - if len(match)!=1 or match[0]['blob']!={'sha256':source['measured_sha256'],'bytes':source['artifact_bytes']}: - _invalid('Producer Run input byte identity differs.') - publication=doc['publication'] - if publication is not None: - _shape(publication,('continuation','source','result'),'publication history') - from run_publication import validate_continuation - historical=dict(run,publication_continuation=publication['continuation']) - validate_continuation(historical) - for key in ('source','result'): - if publication[key] is not None: - _shape(publication[key],('sha256','bytes'),'publication byte reference') - if declarations.get(publication[key]['sha256'])!={'format':'project_artifact_v1',**publication[key]}: - _invalid('Historical publication byte declaration is missing.','unavailable') - - -def _decode_records(records, *, materials=None, artifacts=None): - """Validate finite owned families and complete metadata closure before effects. - - Declared but absent payload bytes are legitimate metadata. Semantic byte - verification is explicit on use and cannot confer current permission. - """ - if type(records) is not list:_invalid('Project records must be a list.') - _items([record for record in records if type(record) is dict and record.get('family') in FAMILY_FORMATS], - 'Project history records',MAX_HISTORY_RECORDS) - facts=HistoryFacts();declarations={} - for record in records: - if record.get('family')=='project_artifact': - declarations[record['document']['sha256']]=record['document'] - family=record.get('family') - if family not in FAMILY_FORMATS:continue - _shape(record,('family','id','document'),'history envelope') - doc=checked_json(record['document'],MAX_HISTORY_BYTES) - if doc.get('format')!=FAMILY_FORMATS[family]:_invalid('Unsupported historical format.','unsupported') - origin=doc.get('origin_catalog_id');identifier(origin,'Origin catalog') - if family=='run_observation': - doc=_observation_document(record) - key=(origin,doc['run_id'],doc['sha256']) - if key in facts.observations:_invalid('Duplicate retained Run observation.') - facts.observations[key]=doc - continue - if family=='study_history': - _shape(doc,('format','origin_catalog_id','study','memberships','policies','reservations','operations', - 'revisions','audit','revision_history_complete'),'Study history') - identity=doc['study'].get('id');target=facts.studies - elif family=='trial_history': - _shape(doc,('format','origin_catalog_id','trial','lineage'),'Trial history') - identity=doc['trial'].get('id');target=facts.trials - elif family=='run_history': - _shape(doc,('format','origin_catalog_id','run','admission_proof','payloads','publication'),'Run history') - identity=doc['run'].get('id');target=facts.runs - else: - _shape(doc,('format','origin_catalog_id','candidate'),'Candidate history') - identity=doc['candidate'].get('id');target=facts.candidates - identifier(identity,'Historical identity') - if record['id']!=origin+'.'+identity or (origin,identity) in target: - _invalid('Historical envelope identity differs or is duplicated.') - target[(origin,identity)]=doc - for (origin,identity),doc in facts.studies.items(): - policies={} - for item in _items(doc['policies'],'Retained policies'): - _shape(item,('ref','policy'),'historical policy') - ref=reference(item['ref'],'selection_policy',origin);validate_retained_policy(item['policy']) - if digest(item['policy'])!=ref['sha256']:_invalid('Historical selection policy digest differs.') - key=(origin,ref['id'],ref['revision'],ref['sha256']) - if key in policies or (key in facts.policies and facts.policies[key]!=item['policy']): - _invalid('Historical selection policy is duplicated or contradictory.') - policies[key]=item['policy'];facts.policies[key]=item['policy'] - memberships={} - previous=[] - for item in _items(doc['memberships'],'Study memberships'): - _shape(item,('revision','trial_ids'),'historical membership') - revision=natural(item['revision'],'Membership revision',1) - if revision!=len(memberships)+1:_invalid('Historical membership revisions must be contiguous.') - trials=_items(item['trial_ids'],'Historical trial members') - if len(set(trials))!=len(trials) or trials[:len(previous)]!=previous: - _invalid('Historical immutable membership changed or repeats a Trial.') - for trial_id in trials:identifier(trial_id) - memberships[revision]={'trial_ids':trials};previous=trials - revisions=_items(doc['revisions'],'Study revisions') - if not revisions or type(doc['revision_history_complete']) is not bool:_invalid('Study revision history is missing.') - last_revision=0 - for study in revisions: - if study.get('id')!=identity or study.get('revision',0)<=last_revision:_invalid('Study revision sequence differs.') - ref=study['policy'];policy=policies.get((origin,ref['id'],ref['revision'],ref['sha256'])) - if policy is None:_invalid('Study policy closure is missing.','unavailable') - validate_retained_study(study,catalog_id=origin,membership=memberships.get(study['membership_revision']),policy=policy) - last_revision=study['revision'] - if revisions[-1]!=doc['study']:_invalid('Current Study differs from its revision stream.') - if doc['revision_history_complete'] and [item['revision'] for item in revisions]!=list(range(1,last_revision+1)): - _invalid('Complete Study history has a revision gap.') - _audit(doc['audit']);_operations(doc['operations'],origin) - for reservation in _items(doc['reservations'],'Study reservations'): - if reservation.get('study_id')!=identity:_invalid('Historical reservation belongs to another Study.') - key=(origin,reservation.get('run_id')) - if key in facts.reservations:_invalid('Run has duplicate historical reservation associations.') - facts.reservations[key]=reservation - for (origin,identity),doc in facts.trials.items(): - trial=validate_retained_trial(doc['trial'],catalog_id=origin) - study=facts.studies.get((origin,trial['study_id'])) - if study is None or identity not in study['study']['trial_ids']:_invalid('Historical Trial membership is missing.','unavailable') - if materials is None:_invalid('Historical material decoder is unavailable.','unavailable') - materials.recipe(trial['material']) - if doc['lineage'] is not None:validate_retained_lineage(doc['lineage'],trial=trial,resolve=facts.trial_ref) - for (origin,_identity),doc in facts.studies.items(): - for trial_id in doc['study']['trial_ids']: - if (origin,trial_id) not in facts.trials:_invalid('Study contains a missing Trial.','unavailable') - for (origin,identity),doc in facts.trials.items(): - parent=(doc['lineage']['intent']['parent'] or {}).get('trial_id') if doc['lineage'] else None - def parent_of(trial_id): - record=facts.trials.get((origin,trial_id)) - if record is None:_invalid('Historical ancestor is missing.','unavailable') - return (record['lineage']['intent']['parent'] or {}).get('trial_id') if record['lineage'] else None - validate_retained_ancestry(identity,parent,parent_of=parent_of) - for doc in facts.runs.values(): - _validate_run_document(doc,facts=facts,declarations=declarations,materials=materials) - for (origin,identity,sha),saved in facts.observations.items(): - doc=saved['observation'];current=facts.runs.get((origin,identity)) - if current is None:_invalid('Disclosure Run observation has no historical Run.','unavailable') - for field in RUN_OBSERVATION_IDENTITY: - if doc['run'].get(field)!=current['run'].get(field): - _invalid('Disclosure Run observation immutable identity differs.') - _validate_run_document(doc,facts=facts,declarations=declarations,materials=materials) - def visit_run(key,path): - if key in path:_invalid('Cyclic historical Run ancestry or producer inputs.') - if len(path)>=64:_invalid('Historical Run ancestry exceeds its bound.','unsupported') - run=facts.runs[key]['run'];next_path=path|{key} - if run.get('parent_run_id') is not None:visit_run((key[0],run['parent_run_id']),next_path) - if run.get('execution_kind')=='external_recipe': - for item in run['material']['inputs'].values(): - if item.get('kind')=='run_artifact': - source=item['source'];visit_run((source['catalog_id'],source['run_id']),next_path) - for key in facts.runs:visit_run(key,set()) - for (origin,_identity),reservation in facts.reservations.items(): - trial=facts.trials[(origin,reservation['trial_id'])]['trial'] - run=facts.runs.get((origin,reservation['run_id'])) - if run is None:_invalid('Historical reservation Run is missing.','unavailable') - validate_retained_reservation(reservation,trial=trial,composition=materials.recipe(trial['material']),run=run['run']) - for (origin,_identity),doc in facts.candidates.items(): - value=doc['candidate'];trial=facts.trials.get((origin,value.get('trial_id'))) - reservation=facts.reservations.get((origin,value.get('run_id'))) - if trial is None or reservation is None:_invalid('Candidate dependency closure is missing.','unavailable') - ref=value['policy'];policy=facts.policies.get((origin,ref['id'],ref['revision'],ref['sha256'])) - if policy is None:_invalid('Candidate policy closure is missing.','unavailable') - validate_retained_candidate(value,catalog_id=origin,trial=trial['trial'],policy=policy,reservation=reservation) - for (origin,_),doc in facts.studies.items(): - for operation in doc['operations']: - validate_operation_result(operation['operation_id'],operation['result'],records) - validate_data_exposures(records,facts) - for record in records: - if record.get('family')=='project_operation': - value=record['document'];validate_operation_result(value['operation'],value['result'],records) - return facts - - -def assemble_validated_facts(records, *, owned=False): - """Collect HistoryFacts from records whose complete validation is on record. - - decode_records validates and collects exactly these values: each history - document (as a checked JSON copy), Run observations, and the policy and - reservation indexes taken from Study documents. Only for a checkpoint that - already passed complete domain validation (which runs decode_records over - these same records) may the collection be repeated without re-running the - validators. tests/test_project_history.py and the portfolio equivalence - check compare this against decode_records field by field. owned=True means - the caller supplies independent copies the facts may keep without recopying. - """ - facts=HistoryFacts() - for record in records: - family=record.get('family') - if family not in FAMILY_FORMATS:continue - if family=='run_observation': - doc=_observation_document(record) - facts.observations[(doc['origin_catalog_id'],doc['run_id'],doc['sha256'])]=doc - continue - doc=record['document'] if owned else checked_json(record['document'],MAX_HISTORY_BYTES) - origin=doc['origin_catalog_id'] - if family=='study_history':target,identity=facts.studies,doc['study']['id'] - elif family=='trial_history':target,identity=facts.trials,doc['trial']['id'] - elif family=='run_history':target,identity=facts.runs,doc['run']['id'] - else:target,identity=facts.candidates,doc['candidate']['id'] - target[(origin,identity)]=doc - for (origin,_identity),doc in facts.studies.items(): - for item in doc['policies']: - ref=item['ref'];facts.policies[(origin,ref['id'],ref['revision'],ref['sha256'])]=item['policy'] - for reservation in doc['reservations']: - facts.reservations[(origin,reservation.get('run_id'))]=reservation - return facts - - -class _LazyRuns(Mapping): - """Validated Run history documents by (origin, run_id), loaded on first use. - - Keys come from verified record identities (origin + '.' + run_id); each - document is read from the working copy once per checkpoint and shared like - every other fact (callers copy what they return). - """ - def __init__(self, identities, load): - self._identities = identities - self._load = load - self._documents = {} - - def __getitem__(self, key): - if key not in self._identities: - raise KeyError(key) - if key not in self._documents: - envelope = self._load(self._identities[key]) - if envelope is None: - _invalid('Historical Run is unavailable.', 'unavailable') - self._documents[key] = envelope['document'] - return self._documents[key] - - def __iter__(self): - return iter(self._identities) - - def __len__(self): - return len(self._identities) - - -def checkpoint_facts(session, materials): - """Decoded history of the session's bound checkpoint, shared by its owners. - - Decoded once per checkpoint; callers copy any part they return. When the - session records that this exact checkpoint passed complete validation, - the facts are collected without repeating the validators. - """ - def compute(): - validated=getattr(session,'validated_checkpoint',None) - if validated is not None and validated==session.summary['checkpoint'] and getattr(session,'_lazy',False): - # A working copy serves Run documents on demand: decode everything - # except Run history now, and keep only Run identities. - selected=('study_history','trial_history','run_observation','candidate_history') - view=session_records_for(session,*selected) - facts=assemble_validated_facts([r for r in view if r['family'] in selected],owned=True) - identities={} - for envelope in view: - if envelope['family']=='run_history': - origin,run_id=envelope['id'].split('.',1) - identities[(origin,run_id)]=envelope['id'] - facts.runs=_LazyRuns(identities,lambda record_id:session_record(session,'run_history',record_id)) - return facts - if validated is not None and validated==session.summary['checkpoint']: - # records_for returns independent copies of exactly these families. - return assemble_validated_facts(session_records_for(session,*FAMILY_FORMATS),owned=True) - records=session.snapshot['records'] - return decode_records(records,materials=materials.read_view(records)) - return session_derived(session, 'history_facts',compute) - - -def decode_records(records, *, materials=None, artifacts=None): - try: - return _decode_records(records,materials=materials,artifacts=artifacts) - except WorkbenchError: - raise - except (KeyError,TypeError,ValueError,AttributeError,RecursionError): - _invalid('Invalid or incomplete project history document.') - - -def _validate_operation_shape(operation,result): - if operation=='history.disclose': - validate_disclosure_fact(result);return True - if operation in ('studies.create','studies.save'): - closed(result,('format','id','revision','name','admission_epoch','open','state','limits','policy', - 'membership_revision','trial_ids','stop_receipt_id')) - if result['format']!='study_v1':_invalid('Historical Study operation result differs.') - identifier(result['id']);natural(result['revision'],'Study revision',1);budget(result['limits']) - reference(result['policy'],'selection_policy',result['policy']['catalog_id']) - return True - if operation in ('studies.prepare_trial','studies.prepare_branch'): - validate_retained_trial(result,catalog_id=result['material']['catalog_id']);return True - if operation=='studies.record_lineage': - closed(result,('format','trial','intent','actor_id','recorded_at','timing')) - if result['format']!='trial_lineage_v1':_invalid('Historical lineage operation result differs.') - return True - if operation=='studies.select_candidate': - closed(result,('ref','candidate','run_id','checkpoint')) - ref=reference(result['ref'],'candidate',result['ref']['catalog_id']) - if (ref['sha256']!=digest(result['candidate']) or ref['id']!=result['candidate']['id'] or - result['run_id']!=result['candidate']['run_id'] or result['checkpoint']['sha256']!=result['candidate']['checkpoint_sha256']): - _invalid('Historical candidate operation identity differs.') - return True - if operation=='studies.continue': - return _validate_operation_shape('studies.save',result) - if operation=='studies.stop': - closed(result,('study_id','stop_receipt_id','admission_epoch','run_ids')) - identifier(result['study_id']);identifier(result['stop_receipt_id']);natural(result['admission_epoch'],'Admission epoch') - for run_id in _items(result['run_ids'],'Stopped Run ids'):identifier(run_id) - return True - if operation in ('studies.prepare_package','studies.prepare_verifier'): - from candidate_packages import decode_preparation_receipt - origin=result['trial']['material']['catalog_id'] - decode_preparation_receipt(operation,result,catalog_id=origin, - validate_trial=lambda value:validate_retained_trial(value,catalog_id=origin)) - return True - if operation=='studies.branch_trial': - from pipeline import decode_graph - closed(result,('request_key','graph','model','parent','intent','study_id','source')) - request_key(result['request_key']);decode_graph(result['graph']);identifier(result['study_id']) - reference(result['model'],'model',result['model']['catalog_id']) - closed(result['source'],('id','revision','material'));identifier(result['source']['id']) - natural(result['source']['revision'],'Source graph revision',1) - reference(result['source']['material'],'recipe_material',result['model']['catalog_id']) - closed(result['parent'],('catalog_id','trial_id','material_sha256')) - identifier(result['parent']['catalog_id']);identifier(result['parent']['trial_id']);_sha(result['parent']['material_sha256']) - if result['intent']['parent']!=result['parent']:_invalid('Branch parent differs from retained intent.') - # Exact lineage intent is checked with the complete Trial closure below. - def ref(value): - closed(value,('catalog_id','trial_id','material_sha256')) - identifier(value['catalog_id']);identifier(value['trial_id']);_sha(value['material_sha256']) - validate_retained_intent(result['intent'],resolve=ref) - return True - if operation=='studies.admit': - # The legacy owner replays admission from reservations and never calls - # _save_receipt for this operation. M3 must own any future publication - # receipt; an invented public Run snapshot is not a portable v1 fact. - _invalid('Portable admission operation receipts are unsupported; retain exact reservation and Run proof.','unsupported') - return False - - -def _bind_operation_result(operation,result,records): - def document(family,origin,identity): - matches=[item['document'] for item in records if item.get('family')==family and item.get('id')==origin+'.'+identity] - if len(matches)!=1:_invalid('Operation result has no unique canonical history fact.') - return matches[0] - if operation=='history.disclose': - doc=document('run_history',result['origin_catalog_id'],result['run_id']) - if result['format']=='project_disclosure_v2': - matches=[item for item in records if item.get('family')=='run_observation' - and item.get('id')==result['observation_sha256']] - if len(matches)!=1:_invalid('Disclosure has no unique retained Run observation.') - saved=_observation_document(matches[0]) - if (saved['origin_catalog_id']!=result['origin_catalog_id'] or saved['run_id']!=result['run_id'] - or saved['sha256']!=result['observation_sha256']): - _invalid('Disclosure Run observation identity differs.') - observation=saved['observation'] - for field in RUN_OBSERVATION_IDENTITY: - if observation['run'].get(field)!=doc['run'].get(field): - _invalid('Disclosure Run observation immutable identity differs.') - declarations={item['id']:item['document'] for item in records if item.get('family')=='project_artifact'} - for edge in _payloads(observation['payloads']).values(): - if declarations.get(edge['blob']['sha256'])!={'format':'project_artifact_v1',**edge['blob']}: - _invalid('Disclosure observation artifact declaration is missing or contradictory.','unavailable') - doc=observation - if result['payload_set_sha256']!=payload_set_sha256(doc['origin_catalog_id'],doc['run']['id'],doc['payloads']): - _invalid('Disclosure observation does not identify the retained Run payload set.') - elif operation in ('studies.create','studies.save'): - doc=document('study_history',result['policy']['catalog_id'],result['id']) - if result not in doc['revisions']:_invalid('Study operation result has no retained revision.') - elif operation in ('studies.prepare_trial','studies.prepare_branch'): - doc=document('trial_history',result['material']['catalog_id'],result['id']) - if doc['trial']!=result:_invalid('Trial operation result has no retained Trial.') - elif operation=='studies.record_lineage': - ref=result['trial'];doc=document('trial_history',ref['catalog_id'],ref['trial_id']) - if doc['lineage']!=result:_invalid('Lineage operation result has no retained sidecar.') - elif operation=='studies.select_candidate': - ref=result['ref'];doc=document('candidate_history',ref['catalog_id'],ref['id']) - if doc['candidate']!=result['candidate']:_invalid('Candidate operation result has no retained decision.') - run=document('run_history',ref['catalog_id'],result['run_id'])['run'] - checkpoint=result['checkpoint'] - if not any(all(item.get(key)==checkpoint.get(key) for key in ('name','sha256','bytes','media_type')) - for item in run['artifacts']): - _invalid('Candidate operation checkpoint differs from its canonical Run artifact.') - - elif operation=='studies.continue': - _bind_operation_result('studies.save',result,records) - elif operation=='studies.stop': - docs=[item['document'] for item in records if item.get('family')=='study_history' - and item['document']['study']['id']==result['study_id']] - matches=[doc for doc in docs if any(value['stop_receipt_id']==result['stop_receipt_id'] - and value['admission_epoch']==result['admission_epoch'] for value in doc['revisions'])] - if len(matches)!=1:_invalid('Stop receipt has no unique retained epoch fact.') - members={value['run_id'] for value in matches[0]['reservations']} - if not set(result['run_ids'])<=members:_invalid('Stop receipt references an unrelated Run.') - elif operation in ('studies.prepare_package','studies.prepare_verifier'): - origin=result['trial']['material']['catalog_id'] - _bind_operation_result('studies.prepare_trial',result['trial'],records) - document('candidate_history',origin,result['candidate_id']) - graph_docs=[item['document'] for item in records if item.get('family')=='pipeline_graph' - and item['document']['origin_catalog_id']==origin] - if not any(result['graph'] in doc['revisions'] for doc in graph_docs): - _invalid('Preparation graph has no retained revision.') - if operation=='studies.prepare_verifier':document('run_history',origin,result['package_run_id']) - elif operation=='studies.branch_trial': - ref=result['parent'];origin=ref['catalog_id'];parent=document('trial_history',origin,ref['trial_id'])['trial'] - if parent['material']['sha256']!=ref['material_sha256']:_invalid('Branch parent material differs.') - resolve=lambda value:document('trial_history',value['catalog_id'],value['trial_id'])['trial'] - validate_retained_intent(result['intent'],resolve=resolve) - document('study_history',origin,result['study_id']) - graph_docs=[item['document'] for item in records if item.get('family')=='pipeline_graph' - and item['document']['origin_catalog_id']==origin] - if not any(result['graph'] in doc['revisions'] for doc in graph_docs):_invalid('Branch graph has no retained revision.') - -def validate_operation_result(operation,result,records=None): - """Validate exact returned meaning and, when supplied, its canonical fact.""" - try: - claimed=_validate_operation_shape(operation,result) - if claimed and records is not None:_bind_operation_result(operation,result,records) - return claimed - except WorkbenchError: - raise - except (KeyError,TypeError,ValueError,AttributeError): - _invalid('Invalid historical operation result or fact binding.') - - -def validate_data_exposures(records,history): - """Exposure is a historical Run/source fact, never present permission.""" - try: - for item in records: - if item.get('family')!='dataset_exposure':continue - doc=item['document'] - _shape(doc,('format','id','record_id','source_hash','reason','run_id','release_id','created_at'),'dataset exposure') - if doc['format']!='dataset_exposure_v1':_invalid('Unsupported dataset exposure.','unsupported') - matches=[] - for (origin,run_id),saved in history.runs.items(): - run=saved['run'] - if run_id!=doc['run_id'] or run.get('release_id')!=doc['release_id']:continue - sources=[source for source in run.get('source_records',[]) if source.get('id')==doc['record_id'] - and source.get('split')=='test' and source_content_hash(source)==doc['source_hash']] - events=[event for event in run.get('exposure_events',[]) if event.get('created_at')==doc['created_at'] - and event.get('reason')==doc['reason']] - if len(sources)==1 and events:matches.append(origin) - if len(matches)!=1:_invalid('Dataset exposure has no unique exact historical Run/source/release binding.') - return True - except WorkbenchError: - raise - except (KeyError,TypeError,ValueError,AttributeError): - _invalid('Invalid historical dataset exposure binding.') - - -class ProjectHistory: - """Read-only imported facts plus canonical, non-executing Study metadata edits.""" - def __init__(self,session,*,materials=None,authorize_disclosure=None): - self.session=session;self.access=session.access;self.artifacts=session.artifacts - if materials is None: - from project_material import ProjectMaterial - materials=ProjectMaterial(session) - self.materials=materials;self.authorize_disclosure=authorize_disclosure - self._decoded_checkpoint=None - self._decoded_facts=None - - def _facts(self): - self.session.check_current() - # Current canonical bytes are guarded on every call. A ProjectSession - # shares decoded metadata per checkpoint with the Study owner; other - # snapshot views keep this owner's private per-checkpoint cache. - # Returned values and permissions remain each read's responsibility. - if callable(getattr(self.session,'derived',None)): - return checkpoint_facts(self.session,self.materials) - summary=self.session.summary - checkpoint=(summary['project_id'],summary['checkpoint']) - if self._decoded_checkpoint==checkpoint: - return self._decoded_facts - records=self.session.snapshot['records'] - facts=decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) - self._decoded_facts=facts - self._decoded_checkpoint=checkpoint - return facts - - def _authorize(self,action,target=None): - self.access.require('inspect',action,target) - - def _origin(self,origin): - return self.session.summary['project_id'] if origin is None else origin - - def _record(self,collection,identity,origin=None): - origin=self._origin(origin);identifier(origin,'Origin catalog');identifier(identity) - result=collection.get((origin,identity)) - if result is None:_invalid('Historical record is unavailable.','not_found') - return result - - def study(self,study_id,*,origin_catalog_id=None,offset=0,limit=100,revision=None): - self._authorize('studies.get',study_id) - natural(offset,'Reservation offset');natural(limit,'Reservation limit',1,100) - facts=self._facts();doc=self._record(facts.studies,study_id,origin_catalog_id) - if offset and revision is None:_invalid('Later reservation pages require a Study revision.') - if revision is not None and revision!=doc['study']['revision']:_invalid('Study changed between pages.','stale') - totals=reservation_totals(doc['reservations']) - self._authorize('studies.get',study_id) - end=min(offset+limit,len(doc['reservations'])) - return {'study':copy.deepcopy(doc['study']),'reservations':copy.deepcopy(doc['reservations'][offset:end]), - 'charged':totals,'revision_history_complete':doc['revision_history_complete'], - 'page':{'offset':offset,'limit':limit,'total':len(doc['reservations']), - 'study_revision':doc['study']['revision'],'next_offset':end if end<len(doc['reservations']) else None}} - - def studies(self): - self._authorize('studies.list') - result=[{'origin_catalog_id':origin,'study':copy.deepcopy(doc['study'])} - for (origin,_),doc in sorted(self._facts().studies.items())] - return {'studies':result} - - def policies(self,*,origin_catalog_id=None): - self._authorize('studies.policy') - origin=self._origin(origin_catalog_id) - return {'policies':[{'ref':{'kind':'selection_policy','catalog_id':namespace,'id':identity,'revision':revision,'sha256':sha}, - 'policy':copy.deepcopy(value)} - for (namespace,identity,revision,sha),value in sorted(self._facts().policies.items()) if namespace==origin]} - - def policy(self,ref): - self._authorize('studies.policy',ref.get('id')) - reference(ref,'selection_policy',ref['catalog_id']) - value=self._facts().policies.get((ref['catalog_id'],ref['id'],ref['revision'],ref['sha256'])) - if value is None:_invalid('Historical selection policy is unavailable.','not_found') - return copy.deepcopy(value) - - def trial(self,trial_id,*,origin_catalog_id=None): - self._authorize('studies.trial',trial_id) - return copy.deepcopy(self._record(self._facts().trials,trial_id,origin_catalog_id)['trial']) - - def lineage(self,*,origin_catalog_id=None): - self._authorize('studies.lineage') - origin=self._origin(origin_catalog_id) - return {'records':[copy.deepcopy(doc) for (namespace,_),doc in sorted(self._facts().trials.items()) if namespace==origin]} - - def progression_origins(self,*,offset=0,limit=40): - """Discover original namespaces without inventing current project ancestry.""" - self._authorize('studies.lineage') - natural(offset,'Origin offset');natural(limit,'Origin page limit',1,40) - facts=self._facts();counts={} - # One pass over Studies and Trials instead of origins x (studies+trials). - for namespace,_ in facts.studies:counts.setdefault(namespace,[0,0])[0]+=1 - for namespace,_ in facts.trials:counts.setdefault(namespace,[0,0])[1]+=1 - items=[{'origin_catalog_id':origin,'study_count':counts[origin][0],'trial_count':counts[origin][1]} - for origin in sorted(counts)] - self._authorize('studies.lineage') - return {'items':items[offset:offset+limit],'offset':offset,'limit':limit,'total':len(items)} - - def progression(self,origin_catalog_id,*,offset=0,limit=20,cohort=None): - """Bounded original Trial graph, with saved Run status only. - - Numeric evidence stays behind its existing disclosure/query owners. - All references come from the already validated immutable history; - this projection reads no payload bytes or execution/recovery owner. - """ - self._authorize('studies.lineage');identifier(origin_catalog_id,'Origin catalog') - natural(offset,'Progression offset');natural(limit,'Progression page limit',1,20) - if cohort is not None:text(cohort,'Recorded cohort',120) - facts=self._facts();origin=origin_catalog_id - if not any(namespace==origin for namespace,_ in facts.studies): - _invalid('Historical origin is unavailable.','not_found') - documents=[doc for (namespace,_),doc in sorted(facts.trials.items()) if namespace==origin - and (cohort is None or doc['lineage'] is not None and doc['lineage']['intent']['cohort']==cohort)] - nodes=[] - for doc in documents[offset:offset+limit]: - trial=doc['trial'];study=facts.studies[(origin,trial['study_id'])]['study'] - attempts=[facts.runs[(origin,value['run_id'])]['run'] - for (namespace,_),value in facts.reservations.items() - if namespace==origin and value['trial_id']==trial['id']] - attempts.sort(key=lambda value:(value['created_at'],value['id']),reverse=True) - runs=[{key:copy.deepcopy(run.get(key)) for key in - ('id','state','parent_run_id','created_at','finished_at')} for run in attempts[:5]] - nodes.append({'trial':copy.deepcopy(trial),'lineage':copy.deepcopy(doc['lineage']), - 'study':{'id':study['id'],'name':study['name']}, - 'attempt_count':len(attempts),'runs':runs,'runs_truncated':len(attempts)>5}) - visible={node['trial']['id'] for node in nodes};related={} - for node in nodes: - for kind in ('parent','baseline'): - ref=(node['lineage'] or {}).get('intent',{}).get(kind) - if ref is not None and (ref['catalog_id']!=origin or ref['trial_id'] not in visible): - related[(ref['catalog_id'],ref['trial_id'])]=copy.deepcopy(ref) - self._authorize('studies.lineage') - return {'origin_catalog_id':origin,'evidence':'saved_metadata_only','historical':True, - 'execution_available':False,'nodes':nodes,'related':[related[key] for key in sorted(related)], - 'offset':offset,'limit':limit,'total':len(documents),'cohort':cohort} - - def run(self,run_id,*,origin_catalog_id=None,offset=None,limit=None): - self._authorize('runs.get',run_id) - facts=self._facts();doc=self._record(facts.runs,run_id,origin_catalog_id) - self._transitive(facts,doc) - public=public_run_projection(doc['run']) - public.pop('exposure_events',None) - disclosures=self.disclosure_facts(run_id,origin_catalog_id=doc['origin_catalog_id']) - payloads=doc['payloads'];page=None - if offset is not None or limit is not None: - offset=0 if offset is None else offset;limit=40 if limit is None else limit - natural(offset,'Run payload offset');natural(limit,'Run payload limit',1,100) - end=min(offset+limit,len(payloads));disclosure_end=min(offset+limit,len(disclosures)) - page={'offset':offset,'limit':limit,'total':len(payloads), - 'next_offset':end if end<len(payloads) else None, - 'disclosures_total':len(disclosures), - 'disclosures_next_offset':disclosure_end if disclosure_end<len(disclosures) else None} - payloads=payloads[offset:end];disclosures=disclosures[offset:disclosure_end] - availability=self.artifacts.inspect_many([ - {'format':'project_artifact_v1',**edge['blob']} for edge in payloads]) - result={'origin_catalog_id':doc['origin_catalog_id'],'run':public, - 'historical':True,'execution_available':False,'evidence':'unverified', - 'disclosures':disclosures,'document':self.run_document_descriptor(run_id,origin_catalog_id=doc['origin_catalog_id']), - 'payloads':[dict(copy.deepcopy(edge),availability=observed) - for edge,observed in zip(payloads,availability)]} - if page is not None:result['page']=page - self._authorize('runs.get',run_id) - return result - - def _run_document(self,run_id,origin_catalog_id): - from project_store import _encode as canonical - facts=self._facts();doc=self._record(facts.runs,run_id,origin_catalog_id) - raw=canonical(doc) - return facts,doc,raw,{'sha256':hashlib.sha256(raw).hexdigest(),'bytes':len(raw)} - - def run_document_descriptor(self,run_id,*,origin_catalog_id=None): - self._authorize('runs.get',run_id) - _facts,_doc,_raw,descriptor=self._run_document(run_id,origin_catalog_id) - self._authorize('runs.get',run_id) - return descriptor - - def run_document_segment(self,run_id,expected_sha256,offset=0,limit=262144,*,origin_catalog_id=None): - self._authorize('runs.get',run_id);_sha(expected_sha256) - natural(offset,'Run document offset');natural(limit,'Run document segment limit',1,262144) - facts,doc,raw,descriptor=self._run_document(run_id,origin_catalog_id) - self._transitive(facts,doc);self._disclose(doc['run'],'unknown') - if descriptor['sha256']!=expected_sha256:_invalid('Run document changed.','stale') - if offset>len(raw):_invalid('Run document offset exceeds its size.') - end=min(offset+limit,len(raw)) - result={'format':'project_run_document_segment_v1','origin_catalog_id':doc['origin_catalog_id'], - 'run_id':run_id,**descriptor,'offset':offset,'limit':limit, - 'base64':base64.b64encode(raw[offset:end]).decode('ascii'), - 'next_offset':end if end<len(raw) else None} - self._authorize('runs.get',run_id);self._disclose(doc['run'],'unknown') - return result - - def payload_set_sha256(self,run_id,*,origin_catalog_id=None): - self._authorize('runs.get',run_id) - doc=self._record(self._facts().runs,run_id,origin_catalog_id) - return payload_set_sha256(doc['origin_catalog_id'],run_id,doc['payloads']) - - def disclosure_facts(self,run_id,*,origin_catalog_id=None): - self._authorize('runs.get',run_id);origin=self._origin(origin_catalog_id) - result=[] - for operation in operation_documents(session_records(self.session, 'project_operation')): - if operation['operation']!='history.disclose':continue - value=validate_disclosure_fact(operation['result']) - if value['origin_catalog_id']==origin and value['run_id']==run_id:result.append(value) - return sorted(result,key=lambda value:(value['occurred_at'],value['actor_id'],value['reason'])) - - def record_disclosure(self,run_id,body,expected_checkpoint,key): - closed(body,('origin_catalog_id','reason'));body=checked_json(body) - identifier(run_id);identifier(body['origin_catalog_id'],'Origin catalog');text(body['reason'],'Disclosure reason',1000) - authority=self.access.require('expose-holdout','history.disclose',run_id) - actor_id=authority.actor_id - effective=dict(body,actor_id=actor_id) - captured=[] - def invoke(): - current=self.access.require('expose-holdout','history.disclose',run_id) - if current.actor_id!=actor_id:_invalid('Disclosure caller changed.','conflict') - doc=self._record(self._facts().runs,run_id,body['origin_catalog_id']) - observation=capture_run_observation(doc) - sha=observation['id'] - captured.append(observation) - return validate_disclosure_fact({'format':'project_disclosure_v2','origin_catalog_id':body['origin_catalog_id'], - 'run_id':run_id,'payload_set_sha256':payload_set_sha256(doc['origin_catalog_id'],run_id,doc['payloads']), - 'population':'all','reason':body['reason'],'actor_id':actor_id,'occurred_at':timestamp(), - 'observation_sha256':sha}) - def capture(result,before_records):return {'records':captured,'result':result} - return self.session.mutate('history.disclose',run_id,effective,expected_checkpoint,key,invoke,capture,scope='expose-holdout') - - def dependencies(self,run_id,*,origin_catalog_id=None,offset=0,limit=100): - self._authorize('runs.get',run_id);natural(offset,'Dependency offset');natural(limit,'Dependency limit',1,100) - doc=self._record(self._facts().runs,run_id,origin_catalog_id);run=doc['run'] - declarations_by_sha={item['document']['sha256']:item['document'] for item in session_records(self.session, 'project_artifact')} - required=[] - for edge in doc['payloads']: - required.append({'name':edge['name'],'sha256':edge['blob']['sha256'],'bytes':edge['blob']['bytes'], - 'preservation':edge['preservation'],'required_for':edge['required_for']}) - if run.get('execution_kind')=='external_recipe': - for path,item in sorted(run['material']['runtime']['files'].items()): - required.append({'name':path,'sha256':item['sha256'],'bytes':item['bytes'], - 'preservation':'regenerable','required_for':['historical_runtime']}) - for name,sha in sorted(run['material']['host_files'].items()): - declaration=declarations_by_sha.get(sha) - required.append({'name':name,'sha256':sha,'bytes':declaration['bytes'] if declaration else None, - 'preservation':'unique','required_for':['historical_host_source']}) - end=min(offset+limit,len(required));items=[] - for item in required[offset:end]: - declaration=declarations_by_sha.get(item['sha256']) - availability=self.artifacts.inspect(declaration) if declaration else { - 'state':'missing','expected_sha256':item['sha256'],'expected_bytes':item['bytes'], - 'observed_sha256':None,'observed_bytes':None} - items.append(dict(copy.deepcopy(item),availability=availability)) - return {'items':items,'page':{'offset':offset,'limit':limit,'total':len(required), - 'next_offset':end if end<len(required) else None}} - - def publication_result(self,run_id,*,origin_catalog_id=None): - """Inspect hash-bound publication proof without any continuation authority.""" - self._authorize('runs.publication_result',run_id) - facts=self._facts();doc=self._record(facts.runs,run_id,origin_catalog_id);self._transitive(facts,doc) - publication=doc['publication'] - if publication is None or publication['source'] is None: - _invalid('Historical publication source is unavailable.','unavailable') - from run_publication import validate_publication_source,validate_publication_result,MAX_DOCUMENT_BYTES - run=doc['run'];self._disclose(run,'unknown');raw=self.artifacts.read_blob(publication['source'],maximum_bytes=MAX_DOCUMENT_BYTES) - source=validate_publication_source(raw,run=run) - result=None;continuation=publication['continuation'] - if publication['result'] is not None: - if continuation is None or continuation['state']!='published':_invalid('Historical publication result is not attached.') - raw=self.artifacts.read_blob(publication['result'],maximum_bytes=MAX_DOCUMENT_BYTES) - result=validate_publication_result(raw,run=run,source=source,ref=continuation['result_ref'],continuation=continuation) - self._authorize('runs.publication_result',run_id) - return {'origin_catalog_id':doc['origin_catalog_id'],'source':source,'result':result, - 'evidence':'receipt_verified','execution_available':False} - - def _disclose(self,run,population,name=None): - # Portable labels describe evidence; they cannot grant present access. - if name=='release.zip':population='unknown' - elif name in TEST_FILES or any(item['name']==name and item.get('split')=='test' for item in run['artifacts']): - population='test' - elif run.get('execution_kind')=='external_recipe' and (name is None or not name.startswith(('source/','runtime-bootstrap/','native-policy/'))): - population='unknown' - if population in ('test','unknown'): - if self.authorize_disclosure is None:_invalid('Current explicit disclosure is required.','unavailable') - self.authorize_disclosure(run,population) - - def _transitive(self,facts,doc,seen=None): - run=doc['run'];key=(doc['origin_catalog_id'],run['id']);seen=set() if seen is None else seen - if key in seen:_invalid('Cyclic historical Run input lineage.') - if len(seen)>=64:_invalid('Historical Run input lineage exceeds its bound.','unsupported') - seen=seen|{key} - if run.get('execution_kind')!='external_recipe':return - for item in run['material']['inputs'].values(): - if item.get('kind')!='run_artifact':continue - source=item['source'];parent=facts.runs[(source['catalog_id'],source['run_id'])] - self._authorize('runs.artifact',source['run_id']) - edge=next(edge for edge in parent['payloads'] if edge['name']==source['artifact_name']) - self._disclose(parent['run'],edge['population'],edge['name']);self._transitive(facts,parent,seen) - - def _read(self,doc,name,maximum): - edges=_payloads(doc['payloads']);edge=edges.get(name) - if edge is None:_invalid('Historical artifact is unavailable.','not_found') - self._authorize('runs.artifact',doc['run']['id']);self._disclose(doc['run'],edge['population'],name) - raw=self.artifacts.read_blob(edge['blob'],maximum_bytes=maximum) - self._authorize('runs.artifact',doc['run']['id']);self._disclose(doc['run'],edge['population'],name) - return raw - - def artifact(self,run_id,name,*,origin_catalog_id=None,maximum_bytes=40*1024**2): - facts=self._facts();doc=self._record(facts.runs,run_id,origin_catalog_id) - self._transitive(facts,doc) - return self._read(doc,name,maximum_bytes) - - def authorize_material_inputs(self,material): - """Bind retained producer receipts under present disclosure authority.""" - from recipe_contract import validate_material - material=validate_material(copy.deepcopy(material)) - for name,item in material['inputs'].items(): - if item.get('kind')!='run_artifact':continue - source=item['source'] - _raw,actual=self.capture_artifact(source['run_id'],source['artifact_name'],source['measured_sha256'], - max_bytes=material['recipe']['inputs'][name]['max_bytes'],origin_catalog_id=source['catalog_id']) - if actual!=source:_invalid('Retained producer proof differs from the captured input receipt.','unavailable') - - def capture_artifact(self,run_id,name,expected_sha256,*,max_bytes=MAX_ARTIFACT_BYTES,origin_catalog_id=None): - """Capture an exact completed input without live recovery or admissions.""" - self._authorize('runs.capture_artifact',run_id);_sha(expected_sha256) - natural(max_bytes,'Artifact input byte bound',1,MAX_ARTIFACT_BYTES) - facts=self._facts();doc=self._record(facts.runs,run_id,origin_catalog_id);run=doc['run'] - self._transitive(facts,doc) - if run['state']!='completed':_invalid('Only completed Runs provide artifact inputs.','unavailable') - external=run.get('execution_kind')=='external_recipe' - if not external and (run.get('graph',{}).get('recipe')!='image_classification_model_v2' or name=='release.zip'): - _invalid('Capture requires qualified classifier evidence or an external Run.','unsupported') - item=next((item for item in run['artifacts'] if item['name']==name),None) - if item is None or item['bytes']>max_bytes:_invalid('Artifact input is unavailable or exceeds its bound.','unavailable') - if item['sha256']!=expected_sha256:_invalid('Artifact differs from expected SHA256.','conflict') - if external: - self._disclose(run,'unknown') - receipt=self._read(doc,'custody-receipt.json',16384) - manifest=self._read(doc,'artifact-manifest.json',65536) - try:validate_external_artifact_inventory(run,receipt,manifest) - except (ValueError,TypeError,KeyError) as error: - raise WorkbenchError('Retained external inventory differs.','unavailable',409) from error - if run.get('extension_context') is not None:self.evidence(run_id,origin_catalog_id=doc['origin_catalog_id']) - if not run['material']['recipe']['resources'].get('profile')=='captured_cpu_large_outputs_v1': - for artifact in run['artifacts']:self._read(doc,artifact['name'],MAX_ARTIFACT_BYTES) - proof=receipt - else: - # Shared model evidence verifies the full frozen prediction/checkpoint closure. - self.evidence(run_id,origin_catalog_id=doc['origin_catalog_id']) - proof=self._read(doc,'artifact-manifest.json',65536) - raw=self._read(doc,name,max_bytes) - source=artifact_capture_source(run,item,raw,proof) - self._authorize('runs.capture_artifact',run_id) - return raw,source - - def evidence(self,run_id,selectors=None,*,origin_catalog_id=None): - self._authorize('runs.model_evidence',run_id) - facts=self._facts();doc=self._record(facts.runs,run_id,origin_catalog_id) - self._transitive(facts,doc) - if doc['run'].get('execution_kind')=='external_recipe':self._disclose(doc['run'],'unknown') - result=self.materials.evidence(doc['run'],selectors or {}, - read_blob=lambda name,maximum:self._read(doc,name,maximum)) - self._authorize('runs.model_evidence',run_id) - return result - - def candidate(self,candidate_id,*,origin_catalog_id=None,verify=False): - self._authorize('studies.candidate',candidate_id) - facts=self._facts();doc=self._record(facts.candidates,candidate_id,origin_catalog_id);value=doc['candidate'] - result={'ref':{'kind':'candidate','catalog_id':doc['origin_catalog_id'],'id':candidate_id,'sha256':digest(value)}, - 'candidate':copy.deepcopy(value),'run_id':value['run_id'],'evidence':'unverified'} - if verify: - evidence=self.evidence(value['run_id'],{'checkpoint_sha256':value['checkpoint_sha256']}, - origin_catalog_id=doc['origin_catalog_id']) - trial=facts.trials[(doc['origin_catalog_id'],value['trial_id'])]['trial'] - validate_candidate_evidence(evidence,trial) - ref=value['policy'];policy=facts.policies[(doc['origin_catalog_id'],ref['id'],ref['revision'],ref['sha256'])] - checkpoint=select_retained_checkpoint(evidence,policy,value['checkpoint_sha256']) - run=facts.runs[(doc['origin_catalog_id'],value['run_id'])]['run'] - evaluation='model-evidence.json' if run.get('extension_context') is not None else 'metrics.json' - if not any(item['name']==evaluation and item['sha256']==value['evaluation_sha256'] for item in run['artifacts']): - _invalid('Candidate evaluation identity differs.') - result.update(checkpoint=checkpoint,evidence='verified') - return result - - def package_state(self,candidate_id,*,origin_catalog_id=None): - from candidate_packages import retained_package_state - self._authorize('studies.package_state',candidate_id) - facts=self._facts();origin=self._origin(origin_catalog_id) - candidate=self.candidate(candidate_id,origin_catalog_id=origin,verify=True) - preparations=[] - for (namespace,_),doc in facts.studies.items(): - if namespace!=origin:continue - for operation in doc['operations']: - if operation['operation_id'] in ('studies.prepare_package','studies.prepare_verifier') and operation['result'].get('candidate_id')==candidate_id: - preparations.append({'operation_id':operation['operation_id'],'result':operation['result']}) - def runs_for_trial(trial_id): - result=[] - for (namespace,run_id),reservation in sorted(facts.reservations.items()): - if namespace==origin and reservation['trial_id']==trial_id: - self._authorize('runs.get',run_id) - doc=facts.runs[(origin,run_id)];self._transitive(facts,doc);result.append(copy.deepcopy(doc['run'])) - return result - result=retained_package_state(candidate,preparations,runs_for_trial=runs_for_trial, - read_artifact=lambda run_id,name:self.artifact(run_id,name,origin_catalog_id=origin)) - self._authorize('studies.package_state',candidate_id) - return result - - def retained_analysis(self,origin_catalog_id,body,*,export=False): - from retained_analysis import query_history - run_id=body['run_id'];doc=self._record(self._facts().runs,run_id,origin_catalog_id) - evidence=self.evidence(run_id,{'checkpoint_sha256':body['checkpoint_sha256']},origin_catalog_id=origin_catalog_id) - return query_history(self.session.db,doc['run'],evidence,body,authorize=self.access.require,export=export) - - def _mutate(self,operation,target,body,expected_checkpoint,key,build): - prepared={} - def invoke(): - self.access.require('edit',operation,target) - facts=self._facts();records,result=build(facts) - prepared['records']=records - return result - def capture(result,before_records): - adapter=getattr(self,'study_execution',None) - if adapter is not None: - # Authoring rows are a disposable projection of THIS pending - # canonical change. Update them inside the same SQL boundary so - # a background local Run cannot later publish stale Study intent. - merged={(item['family'],item['id']):item for item in before_records} - merged.update(((item['family'],item['id']),item) for item in prepared['records']) - adapter.refresh_authoring(list(merged.values())) - return {'records':prepared['records'],'result':result} - self.access.require('edit',operation,target,owner_only=operation in ('studies.create','studies.save')) - return self.session.mutate(operation,target,body,expected_checkpoint,key,invoke,capture) - - def _audit_fact(self,action,target,details): - return {'actor_id':self.access._context().actor_id,'action':action,'target':target,'details':copy.deepcopy(details), - 'result':'success','occurred_at':timestamp()} - - def create_study(self,body,expected_checkpoint,request_key): - closed(body,('name','limits','policy'));body=checked_json(body) - text(body['name'],'Study name');limits=budget(body['limits']);policy=validate_retained_policy(body['policy']) - def build(facts): - origin=self.session.summary['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) - prior=next((key for key,value in facts.policies.items() if key[0]==origin and key[3]==sha),None) - policy_id=prior[1] if prior else uuid.uuid4().hex - ref={'kind':'selection_policy','catalog_id':origin,'id':policy_id,'revision':1,'sha256':sha} - study=new_study_document(body,catalog_id=origin,study_id=identity,policy_id=policy_id) - doc={'format':FAMILY_FORMATS['study_history'],'origin_catalog_id':origin,'study':study, - 'memberships':[{'revision':1,'trial_ids':[]}],'policies':[{'ref':ref,'policy':policy}], - 'reservations':[],'operations':[],'revisions':[copy.deepcopy(study)], - 'audit':[self._audit_fact('studies.create',identity,{'policy_sha256':sha,'limits':limits})], - 'revision_history_complete':True} - return [envelope('study_history',origin,identity,doc)],study - return self._mutate('studies.create',None,body,expected_checkpoint,request_key,build) - - def save_study(self,study_id,body,expected_checkpoint,request_key): - closed(body,('revision','name','limits','policy'));body=checked_json(body) - text(body['name'],'Study name');limits=budget(body['limits']) - def build(facts): - origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] - if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') - ref=reference(body['policy'],'selection_policy',origin) - policy=facts.policies.get((origin,ref['id'],ref['revision'],ref['sha256'])) - if policy is None:_invalid('Selection policy is unavailable.','unavailable') - totals=reservation_totals(doc['reservations']) - study=edit_study_document(study,body,policy=policy,charged=totals,catalog_id=origin) - study['revision']=natural(study['revision']+1,'Study revision',1);doc['study']=study - if not any(item['ref']==ref for item in doc['policies']):doc['policies'].append({'ref':ref,'policy':policy}) - doc['revisions'].append(copy.deepcopy(study));doc['audit'].append(self._audit_fact('studies.save',study_id,{'revision':study['revision']})) - return [envelope('study_history',origin,study_id,doc)],study - return self._mutate('studies.save',study_id,body,expected_checkpoint,request_key,build) - - def prepare_trial_from_material(self,study_id,body,expected_checkpoint,request_key): - """Attach a fully retained composition without capture, approval or execution.""" - closed(body,('revision','material','replicate'));body=checked_json(body) - natural(body['replicate'],'Replicate') - self.access.require('edit','studies.prepare_trial',study_id) - if request_key not in session_operations(self.session): - self.materials.recipe(body['material'],verify=True) - def build(facts): - origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) - study=doc['study'] - if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') - ref=reference(body['material'],'recipe_material',origin);self.materials.recipe(ref) - for (namespace,_),other in facts.trials.items(): - trial=other['trial'] - if namespace==origin and trial['study_id']==study_id and trial['material']==ref and trial['replicate']==body['replicate']: - _invalid('Material/replicate already belongs to this study.','conflict') - trial=new_trial_document(study,ref,body['replicate'],trial_id=uuid.uuid4().hex) - study['trial_ids'].append(trial['id']);study['membership_revision']+=1 - study['revision']=natural(study['revision']+1,'Study revision',1) - doc['memberships'].append({'revision':study['membership_revision'],'trial_ids':copy.deepcopy(study['trial_ids'])}) - doc['revisions'].append(copy.deepcopy(study)) - doc['audit'].append(self._audit_fact('studies.prepare_trial',study_id,{'trial_id':trial['id']})) - trial_doc={'format':FAMILY_FORMATS['trial_history'],'origin_catalog_id':origin,'trial':trial,'lineage':None} - return [envelope('study_history',origin,study_id,doc),envelope('trial_history',origin,trial['id'],trial_doc)],trial - return self._mutate('studies.prepare_trial',study_id,body,expected_checkpoint,request_key,build) - - def select_candidate(self,study_id,body,expected_checkpoint,request_key): - closed(body,('revision','trial_id','run_id'),('checkpoint_sha256',));body=checked_json(body) - identifier(body['trial_id']);identifier(body['run_id']) - # Expensive semantic proof precedes the write transaction. All exact - # associations and selected policy are checked again under publication. - origin=self.session.summary['project_id'] - self.access.require('edit','studies.select_candidate',study_id) - evidence=(self.evidence(body['run_id'],{},origin_catalog_id=origin) - if request_key not in session_operations(self.session) else None) - def build(facts): - doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] - if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') - trial=self._record(facts.trials,body['trial_id'],origin)['trial'] - reservation=facts.reservations.get((origin,body['run_id'])) - if reservation is None or reservation['trial_id']!=trial['id']:_invalid('Run does not belong to the selected trial.') - validate_candidate_evidence(evidence,trial) - ref=study['policy'];policy=facts.policies[(origin,ref['id'],ref['revision'],ref['sha256'])] - checkpoint=select_retained_checkpoint(evidence,policy,body.get('checkpoint_sha256')) - run=facts.runs[(origin,body['run_id'])]['run'] - name='model-evidence.json' if run.get('extension_context') is not None else 'metrics.json' - evaluation=next((item for item in run['artifacts'] if item['name']==name),None) - candidate=new_candidate_document(study,trial,run,checkpoint,evaluation,candidate_id=uuid.uuid4().hex) - study['revision']=natural(study['revision']+1,'Study revision',1) - doc['revisions'].append(copy.deepcopy(study));doc['audit'].append(self._audit_fact( - 'studies.select_candidate',candidate['id'],{'study_id':study_id,'run_id':run['id'],'checkpoint_sha256':checkpoint['sha256']})) - candidate_doc={'format':FAMILY_FORMATS['candidate_history'],'origin_catalog_id':origin,'candidate':candidate} - result={'ref':{'kind':'candidate','catalog_id':origin,'id':candidate['id'],'sha256':digest(candidate)}, - 'candidate':candidate,'run_id':run['id'],'checkpoint':checkpoint} - return [envelope('study_history',origin,study_id,doc), - envelope('candidate_history',origin,candidate['id'],candidate_doc)],result - return self._mutate('studies.select_candidate',study_id,body,expected_checkpoint,request_key,build) - - def record_lineage(self,body,expected_checkpoint,request_key): - closed(body,('trial','intent'));body=checked_json(body,INTENT_LIMIT+4096) - def build(facts): - origin=self.session.summary['project_id'];trial=facts.trial_ref(body['trial']) - if body['trial']['catalog_id']!=origin:_invalid('Inherited Trials remain immutable historical evidence.','conflict') - doc=copy.deepcopy(facts.trials[(origin,trial['id'])]) - if doc['lineage'] is not None:_invalid('Trial lineage is already recorded and immutable.','conflict') - intent=validate_retained_intent(body['intent'],resolve=facts.trial_ref) - value={'format':'trial_lineage_v1','trial':copy.deepcopy(body['trial']),'intent':intent, - 'actor_id':self.access._context().actor_id,'recorded_at':timestamp(), - 'timing':'retrospective' if any(item['trial_id']==trial['id'] for item in facts.reservations.values()) else 'before_admission'} - validate_retained_lineage(value,trial=trial,resolve=facts.trial_ref) - parent=(intent['parent'] or {}).get('trial_id') - validate_retained_ancestry(trial['id'],parent,parent_of=lambda identity: - (facts.trials[(origin,identity)]['lineage']['intent']['parent'] or {}).get('trial_id') - if facts.trials[(origin,identity)]['lineage'] else None) - doc['lineage']=value - study_doc=copy.deepcopy(facts.studies[(origin,trial['study_id'])]) - study_doc['audit'].append(self._audit_fact('studies.lineage.record',trial['id'],{'intent_sha256':digest(intent)})) - return [envelope('trial_history',origin,trial['id'],doc), - envelope('study_history',origin,trial['study_id'],study_doc)],value - return self._mutate('studies.record_lineage',body.get('trial',{}).get('trial_id'),body,expected_checkpoint,request_key,build) - - def unavailable_execution(self,*args,**kwargs): - _invalid('Project execution and candidate preparation require the M3 custody boundary.','unavailable') - - approve=admit=continue_study=stop=prepare_package=prepare_verifier=unavailable_execution diff --git a/project_locks.py b/project_locks.py deleted file mode 100644 index ed906d8a..00000000 --- a/project_locks.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Preserve host -> project session -> projection command lock ordering.""" -from contextlib import contextmanager, ExitStack -import threading -import time - -from metadata import CATALOG_LOCAL_TIMEOUT, CatalogBusy - - -class ProjectLock: - """CatalogLock interface for owners that begin at the projection lock. - - Owners retain the actual PostgreSQL advisory lock and its deadlines. Every - entry also owns the host authorization connection before the project - session, including background workers which have no foreground request. - """ - def __init__(self, host, session, projection): - self.host, self.session, self.projection = host, session, projection - self._local = threading.local() - - @property - def depth(self): - return self.projection.depth - - @property - def mutex(self): - return self.projection.mutex - - @property - def owned_by_current_thread(self): - return bool(getattr(self._local, 'stacks', ()) or getattr(self._local, 'bounded_depth', 0)) - - def __enter__(self): - stack = ExitStack() - try: - stack.enter_context(self.host) - if not self.session.acquire(timeout=CATALOG_LOCAL_TIMEOUT): - raise CatalogBusy('Project command ownership did not become available within its local wait bound.') - stack.callback(self.session.release) - stack.enter_context(self.projection) - if not hasattr(self._local, 'stacks'): - self._local.stacks = [] - self._local.stacks.append(stack) - return self - except BaseException: - stack.close() - raise - - def __exit__(self, kind, error, tb): - return self._local.stacks.pop().__exit__(kind, error, tb) - - @contextmanager - def bounded(self, deadline, cancelled=None): - with ExitStack() as stack: - stack.enter_context(self.host.bounded(deadline, cancelled)) - while not self.session.acquire(timeout=min(.1, max(0, deadline-time.monotonic()))): - if (cancelled and cancelled()) or time.monotonic() >= deadline: - raise CatalogBusy('Project command ownership reached its recovery deadline.') - stack.callback(self.session.release) - if (cancelled and cancelled()) or time.monotonic() >= deadline: - raise CatalogBusy('Project command ownership was cancelled or reached its recovery deadline.') - stack.enter_context(self.projection.bounded(deadline, cancelled)) - self._local.bounded_depth = getattr(self._local, 'bounded_depth', 0) + 1 - try: - yield self - finally: - self._local.bounded_depth -= 1 diff --git a/project_material.py b/project_material.py deleted file mode 100644 index 75af5b96..00000000 --- a/project_material.py +++ /dev/null @@ -1,827 +0,0 @@ -"""Portable model/source/recipe facts, using their existing semantic owners. - -Open decodes inert metadata. Only explicit publication reads projection-local -captures and snapshots their exact bytes before Session publishes a checkpoint. -No runtime is provisioned, captured source imported, or model state loaded here. -""" -import copy -import hashlib -import os -import re -import shutil -import tempfile -from functools import wraps -from datetime import datetime -from pathlib import Path - -from workbench import WorkbenchError, timestamp, encode -from model_definitions import (decode_model_draft, decode_model_material, verify_model_closure, - reference, natural, retained_json, MAX_SOURCE_BYTES) -from pipeline import decode_graph, decode_recipe_draft, decode_recipe_material, verify_recipe_closure, digest, execution_digest -from project_store import ProjectStoreError -from project_session import session_active, session_records_for - -FAMILIES = { - 'model_definition': 'model_definition_history_v1', - 'pipeline_graph': 'pipeline_graph_history_v1', - 'recipe_definition': 'recipe_definition_history_v1', - 'model_material': 'retained_model_material_v1', - 'recipe_material': 'retained_recipe_material_v1', - 'project_snapshot': 'retained_project_snapshot_v1', - 'runtime_material': 'retained_runtime_material_v1', -} -EDITABLE = ('model_definition', 'pipeline_graph', 'recipe_definition') -ID = re.compile(r'^[a-f0-9]{32}$') -SHA = re.compile(r'^[a-f0-9]{64}$') -AUDIT_FIELDS = {'action','target','details','result','actor_id','occurred_at'} -AUDIT_DETAILS = { - 'models.create':('revision','document_sha256'), - 'models.save':('revision','document_sha256'), - 'models.publish':('revision','material_sha256'), - 'pipeline.create':('revision','document_sha256','execution_sha256'), - 'pipeline.create_external':('revision','document_sha256','execution_sha256'), - 'pipeline.save':('revision','document_sha256','execution_sha256'), - 'pipeline.clone':('source_graph_id','revision','document_sha256','execution_sha256'), - 'pipeline.clone_retained':('id','revision','material'), - 'recipes.create':('revision','document_sha256'), - 'recipes.save':('revision','document_sha256'), - 'recipes.material':('recipe_id','revision'), -} -AUDIT_ACTIONS = frozenset(AUDIT_DETAILS) - - -def _boundary(function): - @wraps(function) - def checked(*args, **kwargs): - try:return function(*args, **kwargs) - except (WorkbenchError,ProjectStoreError):raise - except ImportError: - raise WorkbenchError('Retained material validator is unavailable.','unavailable',409) from None - except (KeyError, TypeError, ValueError, AttributeError, OverflowError, RecursionError, UnicodeError): - raise WorkbenchError('Malformed retained material document.','invalid',400) from None - return checked - - -def _closed(value, fields, label): - if type(value) is not dict or set(value) != set(fields): - raise WorkbenchError('Invalid retained ' + label + ' fields.') - - -def _catalog(value): - if type(value) is not str or not ID.fullmatch(value): - raise WorkbenchError('Invalid retained origin catalog.') - return value - - -def _intrinsic(item): - return {'sha256': item['sha256'], 'bytes': item['bytes']} - - -def _declaration(item): - return dict(format='project_artifact_v1', **_intrinsic(item)) - - -def _edges(items, family): - required = 'execution' if family == 'runtime_material' else 'snapshot_verification' if family == 'project_snapshot' else 'material_verification' if family == 'model_material' else 'composition_verification' - result=[] - for item in items: - runtime_file=family=='runtime_material' and item['name']!='manifest.json' - role='runtime_file' if runtime_file else 'runtime_manifest' if family=='runtime_material' else 'project_manifest' if family=='project_snapshot' else 'model_source' if family=='model_material' and item['name'].startswith('source-') else 'model_policy' if family=='model_material' else 'recipe_capture' if item['name'].startswith('capture-') else 'recipe_document' - result.append(dict(name=item['name'],blob=_intrinsic(item),role=role, - preservation='regenerable' if runtime_file else 'unique',required_for=[required])) - return result - - -def _inventory(family, value): - if family == 'model_material': - material = value['material'] - return [*(material[key] for key in ('specification', 'preprocessing_policy', 'output_policy')), - *(item['blob'] for item in material['source_closure'])] - if family == 'recipe_material': - material = value['material'] - return [material['recipe_document'], material['resolved_profile'], *material['captures']] - return [] - - -def _definition(family, value, catalog_id): - if family == 'model_definition': return decode_model_draft(value, catalog_id=catalog_id) - if family == 'recipe_definition': return decode_recipe_draft(value, catalog_id=catalog_id) - if family == 'pipeline_graph': return copy.deepcopy(decode_graph(value)) - raise WorkbenchError('Unsupported material definition family.', 'unsupported', 409) - - -def _audit(value): - _closed(value,AUDIT_FIELDS,'material audit') - if (value['action'] not in AUDIT_ACTIONS or value['result']!='success' or - type(value['target']) is not str or not 1<=len(value['target'])<=128 or - value['actor_id'] is not None and (type(value['actor_id']) is not str or not ID.fullmatch(value['actor_id'])) or - type(value['occurred_at']) is not str or len(value['occurred_at'])>80): - raise WorkbenchError('Invalid material audit fact.') - target_pattern=SHA if value['action']=='recipes.material' else ID - if not target_pattern.fullmatch(value['target']):raise WorkbenchError('Invalid material audit target.') - details=value['details'];_closed(details,AUDIT_DETAILS[value['action']],'material action details') - natural(details['revision'],'Audit revision',1) - for key,item in details.items(): - if key.endswith('_sha256') and (type(item) is not str or not SHA.fullmatch(item)): - raise WorkbenchError('Invalid material audit digest.') - if key in ('id','recipe_id','source_graph_id') and (type(item) is not str or not ID.fullmatch(item)): - raise WorkbenchError('Invalid material audit source identity.') - if 'material' in details: - _catalog(details['material']['catalog_id']) - reference(details['material'],'recipe_material',details['material']['catalog_id']) - instant=datetime.fromisoformat(value['occurred_at']) - if instant.utcoffset() is None:raise WorkbenchError('Material audit requires an offset timestamp.') - - -def _definition_audit(family,value,audit,catalog_id): - _audit(audit) - actions={'model_definition':('models.create','models.save'), - 'recipe_definition':('recipes.create','recipes.save'), - 'pipeline_graph':('pipeline.create','pipeline.create_external','pipeline.save','pipeline.clone','pipeline.clone_retained')} - if audit['action'] not in actions[family] or audit['target']!=value['id']: - raise WorkbenchError('Definition audit action or identity differs.') - details=audit['details'] - if audit['action']=='pipeline.clone_retained': - _closed(details,('id','revision','material'),'retained clone audit') - if not ID.fullmatch(details['id']) or value['revision']!=1: - raise WorkbenchError('Retained clone audit identity differs.') - natural(details['revision'],'Source recipe revision',1) - reference(details['material'],'recipe_material',catalog_id) - elif details.get('revision')!=value['revision']: - raise WorkbenchError('Definition audit revision differs.') - if audit['action'].endswith(('.create','.create_external','.clone','.clone_retained')) and value['revision']!=1: - raise WorkbenchError('Created definition audit must identify its initial revision.') - if audit['action'].endswith('.save') and value['revision']<2: - raise WorkbenchError('Saved definition audit requires a successor revision.') - if 'document_sha256' in details and details['document_sha256']!=digest(value): - raise WorkbenchError('Definition audit document hash differs.') - if 'execution_sha256' in details and details['execution_sha256']!=execution_digest(value): - raise WorkbenchError('Definition audit execution hash differs.') - - -class _MaterialAccess: - """Host authorization plus transaction-scoped inert scientific audit facts.""" - def __init__(self,session):self.session=session;self.host=session.access;self.facts=[];self.registered=False - def require(self,*args,**kwargs):return self.host.require(*args,**kwargs) - def denial(self,*args,**kwargs):return self.host.denial(*args,**kwargs) - def _context(self):return self.host._context() - @property - def lock(self):return self.host.lock - def audit(self,action,target=None,details=None,result='success'): - if action not in AUDIT_ACTIONS or result!='success' or not self.session.db.in_transaction: - raise WorkbenchError('Material audit must share its canonical mutation.','unavailable',409) - actor=self.host.require('edit',action,target) - fact=dict(action=action,target=target,details=copy.deepcopy(details or {}),result=result, - actor_id=actor.actor_id,occurred_at=timestamp()) - _audit(fact) - if not self.registered: - self.registered=True - def clear():self.facts.clear();self.registered=False - self.session.db.on_commit(clear);self.session.db.on_rollback(clear) - self.facts.append(fact) - - -@_boundary -def decode_records(records): - """Closed family-local metadata checks; missing payloads do not erase facts.""" - result = {} - for envelope in records: - if envelope.get('family') not in FAMILIES: - continue - _closed(envelope, ('family', 'id', 'document'), 'material envelope') - family, doc = envelope['family'], envelope['document'] - if family in EDITABLE: - _closed(doc, ('format', 'origin_catalog_id', 'current', 'revisions', 'audit') + - (('fork_origin',) if 'fork_origin' in doc else ()), 'definition history') - else: - _closed(doc, ('format', 'origin_catalog_id', 'value', 'artifacts','audit'), 'immutable material') - if doc['format'] != FAMILIES[family]: - raise WorkbenchError('Unsupported material document format.', 'unsupported', 409) - catalog_id = _catalog(doc['origin_catalog_id']) - if family in EDITABLE: - revisions = doc['revisions'] - forked = 'fork_origin' in doc - if (type(revisions) is not list or not revisions or type(doc['audit']) is not list or - len(doc['audit']) != len(revisions) - int(forked)): - raise WorkbenchError('Definition history must retain its observed revisions and audit.') - previous = None - for index, value in enumerate(revisions): - from project_material_fork import is_intent, decode_intent - if is_intent(value): - if not forked or index != 0 or len(revisions) != 1: - raise WorkbenchError('Unresolved fork intents cannot acquire owner revisions.') - value = decode_intent(value, family, catalog_id) - else: - value = _definition(family, value, catalog_id) - if previous and (value['id'] != previous['id'] or value['revision'] != previous['revision'] + 1): - raise WorkbenchError('Definition history identity or revision changed.') - if not forked or index: - _definition_audit(family, value, doc['audit'][index - int(forked)], catalog_id) - previous = value - if doc['current'] != revisions[-1]: - raise WorkbenchError('Definition current value differs from its history.') - expected = catalog_id + '.' + doc['current']['id'] - else: - if type(doc['audit']) is not list:raise WorkbenchError('Material audit must retain its fact list.') - for audit in doc['audit']:_audit(audit) - value = doc['value'] - if family == 'model_material': value = decode_model_material(value, catalog_id=catalog_id) - elif family == 'recipe_material': value = decode_recipe_material(value, catalog_id=catalog_id) - elif family == 'project_snapshot': - from projects import decode_retained_snapshot - value = decode_retained_snapshot(value) - else: - value = decode_runtime_material(value, catalog_id=catalog_id) - for audit in doc['audit']: - if family=='model_material': - if (audit['action']!='models.publish' or audit['target']!=value['ref']['id'] or - audit['details']!={'revision':value['ref']['revision'],'material_sha256':value['ref']['sha256']}): - raise WorkbenchError('Published Model audit differs.') - elif family=='recipe_material': - if (audit['action']!='recipes.material' or audit['target']!=value['ref']['sha256'] or - audit['details']!={'recipe_id':value['material']['recipe_id'],'revision':value['material']['recipe_revision']}): - raise WorkbenchError('Retained Recipe audit differs.') - else:raise WorkbenchError('Unsupported immutable material audit fact.') - inventory = _inventory(family, value) - if family == 'project_snapshot': - if type(doc['artifacts']) is not list or len(doc['artifacts'])!=1:raise WorkbenchError('Snapshot requires its exact manifest edge.') - size=doc['artifacts'][0]['blob']['bytes'];natural(size,'Manifest bytes',1,64*1024) - inventory=[dict(name=value['manifest_path'],sha256=value['manifest_sha256'],bytes=size)] - if family == 'runtime_material': - inventory=([dict(name='manifest.json',sha256=value['descriptor']['manifest_sha256'],bytes=value['descriptor']['manifest_bytes'])]+ - [dict(name=item['path'],sha256=item['sha256'],bytes=item['bytes']) for item in value['manifest']['files']]) if value['descriptor'] is not None else [] - if type(doc['artifacts']) is not list or doc['artifacts'] != _edges(inventory,family): - raise WorkbenchError('Retained material artifact edges differ from the owner inventory.') - expected = catalog_id + '.' + (value['manifest_sha256'] if family == 'project_snapshot' else value['ref']['sha256']) - if envelope['id'] != expected or (family, expected) in result: - raise WorkbenchError('Retained material locator differs or repeats.') - result[(family, expected)] = copy.deepcopy(doc) - from project_material_fork import validate_records - validate_records(records) - return result - - -@_boundary -def decode_runtime_material(value, *, catalog_id): - """Keep original physical descriptors as historical facts, never live bindings.""" - _closed(value, ('ref', 'descriptor', 'environment','manifest'), 'runtime material') - reference(value['ref'], 'runtime', catalog_id) - descriptor = value['descriptor'] - if descriptor is None: - if value['manifest'] is not None:raise WorkbenchError('Built-in environment cannot carry an external runtime manifest.') - if type(value['environment']) is not dict or digest(value['environment']) != value['ref']['sha256']: - raise WorkbenchError('Retained built-in runtime identity differs.') - else: - if value['environment'] is not None: - raise WorkbenchError('Retained runtime cannot mix descriptor and environment.') - if descriptor.get('profile') == 'captured_cpu_python_v1': - from recipe_process import validate_cpu_snapshot_manifest,encoded - manifest=validate_cpu_snapshot_manifest(descriptor,value['manifest']) - elif descriptor.get('profile') == 'managed_torch_python_v1': - from torch_runtime import verify_retained_descriptor,encoded - manifest=verify_retained_descriptor(descriptor,value['manifest']) - else: raise WorkbenchError('Unsupported retained runtime profile.', 'unsupported', 409) - raw=encoded(manifest) - if len(raw)!=descriptor['manifest_bytes'] or hashlib.sha256(raw).hexdigest()!=descriptor['manifest_sha256']: - raise WorkbenchError('Retained runtime manifest identity differs.') - if descriptor['catalog_id'] != catalog_id or digest(descriptor) != value['ref']['sha256']: - raise WorkbenchError('Retained runtime descriptor identity differs.') - return copy.deepcopy(value) - - -@_boundary -def validate_operation_result(operation, result, records=None): - """True for a validated owned result; False for another fixed owner.""" - if operation in ('models.create','models.save'): - decode_model_draft(result,catalog_id=result['task']['catalog_id']);family='model_definition' - elif operation=='models.publish': - decode_model_material(result,catalog_id=result['ref']['catalog_id']);family='model_material' - elif operation in ('graphs.create','graphs.create_external','graphs.save','graphs.clone'): - decode_graph(result);family='pipeline_graph' - elif operation in ('recipes.create','recipes.save'): - decode_recipe_draft(result,catalog_id=result['task']['catalog_id']);family='recipe_definition' - elif operation in ('recipes.material','recipes.prepare_external_material','recipes.prepare_builtin_material'): - decode_recipe_material(result,catalog_id=result['ref']['catalog_id']);family='recipe_material' - else:return False - if family in EDITABLE: - create=operation.endswith(('.create','.create_external','.clone')) - if create and result['revision']!=1 or not create and result['revision']<2: - raise WorkbenchError('Retained definition result revision differs from its operation.') - if records is not None: - documents=decode_records(records) - values=[value for (owned,_),doc in documents.items() if owned==family - for value in (doc['revisions'] if family in EDITABLE else [doc['value']])] - if not any(encode(value)==encode(result) for value in values): - raise WorkbenchError('Material operation result differs from its retained family closure.') - return True - - -def _retained_value(get_documents, family, ref): - catalog_id = _catalog(ref['catalog_id']) - reference(ref, {'model_material':'model', 'recipe_material':'recipe_material', - 'runtime_material':'runtime'}[family], catalog_id) - key = catalog_id + '.' + ref['sha256'] - document = get_documents().get((family, key)) - if document is None: - raise WorkbenchError('Retained material is unavailable.', 'unavailable', 409) - if document['value']['ref'] != ref: - raise WorkbenchError('Retained material reference differs.', 'invalid', 400) - # Decoded retained values are plain JSON: an exact independent copy. - from project_store import copy_validated_json - return copy_validated_json(document['value']) - - -class MaterialSnapshot: - """Owned, validated material facts for one fixed checkpoint read. - - The decoder copies the selected documents. Neither input mutation nor a - returned value can alter this view. It has no live Session, artifact reader, - publication operation or current execution authority. - """ - @_boundary - def __init__(self, records): - self.__documents = decode_records(records) - - @_boundary - def recipe(self, ref): - return _retained_value(lambda: self.__documents, 'recipe_material', ref) - - @_boundary - def runtime(self, ref): - return _retained_value(lambda: self.__documents, 'runtime_material', ref) - - def artifact_blobs(self): - for document in self.__documents.values(): - for edge in document.get('artifacts', []): - yield copy.deepcopy(edge['blob']) - - -class ProjectMaterial: - def __init__(self, session): - self.session = session - self.owner_access = _MaterialAccess(session) if hasattr(session,'access') else None - - def _documents(self): - self.session.check_current() - return decode_records(session_records_for(self.session, *FAMILIES, 'project_fork')) - - def read_view(self, records): - """Own validated material facts from one guarded checkpoint read. - - The history reader supplies its selected records. This view carries no - live publication, verification or execution capability; later reads - must select their own checkpoint and construct a new view. - """ - self.session.check_current() - return MaterialSnapshot(records) - - @_boundary - def intents(self): - """Inspect unresolved active definitions; they confer no execution binding.""" - if self.owner_access: - self.session.access.require('inspect', 'project_material.intents') - from project_fork import active_records - from project_material_fork import is_intent - documents = self._documents() - active = {(r['family'], r['id']) for r in session_active(self.session, - *FAMILIES, project_id=self.session.summary['project_id'])} - return [dict(family=family, id=locator, current=copy.deepcopy(doc['current']), - fork_origin=copy.deepcopy(doc['fork_origin'])) - for (family, locator), doc in sorted(documents.items()) - if (family, locator) in active and family in EDITABLE and is_intent(doc['current'])] - - @_boundary - def intent_capabilities(self): - intents = self.intents() - unresolved = {(r['family'],r['id']) for r in intents} - from project_fork import active_records - active = session_active(self.session, *EDITABLE, project_id=self.session.summary['project_id']) - return dict(requires_reselection=[dict(family=r['family'],id=r['id'],fields=r['current']['required_reselection']) for r in intents], - editable=[dict(family=r['family'],id=r['id']) for r in active - if r['family'] in EDITABLE and (r['family'],r['id']) not in unresolved]) - - def _require_resolved(self, family, identity): - for intent in self.intents(): - if intent['family'] == family and intent['current']['id'] == identity: - raise WorkbenchError('Fork intent requires explicit binding reselection.', 'requires_reselection', 409) - - @_boundary - def _value(self, family, ref): - return _retained_value(self._documents, family, ref) - - @_boundary - def model(self, ref, *, verify=False): - value = self._value('model_material', ref) - if verify: verify_model_closure(value, read_blob=lambda item, maximum: self.blob(ref, item, maximum)) - return value - - @_boundary - def recipe(self, ref, *, verify=False): - value = self._value('recipe_material', ref) - if verify: self.composition_closure(ref) - return value - - @_boundary - def blob(self, ref, item, maximum=40*1024**2): - family = 'model_material' if ref['kind'] == 'model' else 'recipe_material' - value = self._value(family, ref) - if item not in _inventory(family, value): raise WorkbenchError('Undeclared material blob.', 'invalid', 400) - return self.session.artifacts.read_blob(_intrinsic(item), maximum_bytes=maximum) - - @_boundary - def composition_closure(self, ref): - value = self.recipe(ref) - return verify_recipe_closure(value, self.model(value['material']['model']), read_blob=self.blob) - - @_boundary - def runtime(self, ref, *, verify=False): - value = self._value('runtime_material', ref) - if verify and value['descriptor'] is not None: - descriptor = value['descriptor'] - raw = self.session.artifacts.read_blob({'sha256': descriptor['manifest_sha256'], 'bytes': descriptor['manifest_bytes']}, maximum_bytes=32*1024**2) - if descriptor['profile'] == 'captured_cpu_python_v1': - from recipe_process import decode, encoded, validate_cpu_manifest - manifest = validate_cpu_manifest(decode(raw)) - else: - from torch_runtime import decode, encoded, validate_retained_manifest - manifest = validate_retained_manifest(decode(raw)) - if encoded(manifest) != raw: raise WorkbenchError('Retained runtime manifest encoding differs.', 'invalid', 400) - return value - - @_boundary - def snapshot_ref(self, project_id, manifest_sha256, *, catalog_id=None, verify=False): - candidates = [doc for (family, _), doc in self._documents().items() if family == 'project_snapshot' and - doc['value']['id'] == project_id and doc['value']['manifest_sha256'] == manifest_sha256 and - (catalog_id is None or doc['origin_catalog_id'] == catalog_id)] - if len(candidates) != 1: raise WorkbenchError('Retained project snapshot is unavailable or ambiguous.', 'unavailable', 409) - doc = candidates[0] - if verify: - from projects import decode, decode_retained_manifest - edge = doc['artifacts'][0]['blob'] - raw = self.session.artifacts.read_blob(edge, maximum_bytes=64*1024) - if decode_retained_manifest(decode(raw)) != doc['value']['manifest']: - raise WorkbenchError('Retained project manifest differs.', 'invalid', 400) - return copy.deepcopy(doc['value']) - - def capture_definition(self, family, value, before_records, *, action, actor_id=None, audit_facts=None): - catalog_id = self.session.summary['project_id'] - value = _definition(family, value, catalog_id) - key = catalog_id + '.' + value['id'] - prior = decode_records(before_records).get((family, key)) - revisions = copy.deepcopy(prior['revisions']) if prior else [] - audits = copy.deepcopy(prior['audit']) if prior else [] - if revisions and value == revisions[-1]: - return {'family': family, 'id': key, 'document': prior} - if revisions and value['revision'] != revisions[-1]['revision'] + 1: - raise WorkbenchError('Definition changed outside its canonical history.', 'stale', 409) - details={'revision':value['revision'],'document_sha256':digest(value)} - if family=='pipeline_graph':details['execution_sha256']=execution_digest(value) - facts=copy.deepcopy(audit_facts) if audit_facts is not None else [dict(action=action,target=value['id'],details=details,result='success',actor_id=actor_id,occurred_at=timestamp())] - if len(facts)!=1:raise WorkbenchError('A definition revision requires its exact owner audit fact.') - revisions.append(copy.deepcopy(value));audits.extend(facts) - record = dict(family=family, id=key, document=dict(format=FAMILIES[family], origin_catalog_id=catalog_id, - current=copy.deepcopy(value), revisions=revisions, audit=audits)) - if prior and 'fork_origin' in prior: - from project_material_fork import is_intent - if is_intent(prior['current']): - raise WorkbenchError('Fork intent requires explicit binding reselection.', 'requires_reselection', 409) - record['document']['fork_origin'] = copy.deepcopy(prior['fork_origin']) - decode_records([r for r in before_records if (r['family'],r['id']) != (family,key)] + [record]) - else: - decode_records([record]) - return record - - def _capture_material(self, family, value, root): - catalog_id = value['ref']['catalog_id'] - value = decode_model_material(value, catalog_id=catalog_id) if family == 'model_material' else decode_recipe_material(value, catalog_id=catalog_id) - inventory = _inventory(family, value) - root_fd = material_fd = None - try: - root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - material_fd = os.open(value['ref']['sha256'], os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root_fd) - for item in inventory: - fd = os.open(item['name'], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=material_fd) - try: self.session.artifacts.publish_fd(fd, _declaration(item), maximum_bytes=item['bytes']) - finally: os.close(fd) - except OSError: - raise WorkbenchError('Projection-local material capture is unavailable.','unavailable',409) from None - finally: - if material_fd is not None: os.close(material_fd) - if root_fd is not None: os.close(root_fd) - record = dict(family=family, id=catalog_id+'.'+value['ref']['sha256'], document=dict( - format=FAMILIES[family], origin_catalog_id=catalog_id, value=copy.deepcopy(value), artifacts=_edges(inventory,family), - audit=copy.deepcopy([fact for fact in self.owner_access.facts - if fact['action']==('models.publish' if family=='model_material' else 'recipes.material') - and fact['target'] in (value['ref'].get('id'),value['ref']['sha256'])]) if self.owner_access else [])) - prior=decode_records(session_records_for(self.session, *FAMILIES, 'project_fork')).get((family,record['id'])) - if prior is not None: - if prior['value']!=record['document']['value'] or prior['artifacts']!=record['document']['artifacts']: - raise WorkbenchError('Retained immutable material changed.','invalid',400) - facts=copy.deepcopy(prior['audit']) - facts.extend(fact for fact in record['document']['audit'] if fact not in facts) - record['document']['audit']=facts - decode_records([record]); return record - - @staticmethod - def declarations(record): - """Explicit intrinsic declaration upserts, coalesced without hidden state.""" - result={} - for edge in record['document'].get('artifacts',[]): - item=edge['blob']; declaration=_declaration(item) - if item['sha256'] in result and result[item['sha256']]['document']!=declaration: - raise WorkbenchError('Artifact declarations disagree.','invalid',400) - result[item['sha256']]=dict(family='project_artifact',id=item['sha256'],document=declaration) - return list(result.values()) - - def capture_model_material(self, value, root): return self._capture_material('model_material', value, root) - def capture_recipe_material(self, value, root): return self._capture_material('recipe_material', value, root) - - @staticmethod - def _coalesce(records): - result={} - for record in records: - key=record['family'],record['id'] - if key in result and result[key]!=record: - raise WorkbenchError('Owned material upserts disagree.','invalid',400) - result[key]=record - return list(result.values()) - - @_boundary - def capture_recipe_dependencies(self,record,before_records,*,runtime_manifest=None): - """Retain exact measured runtime facts without using origin paths. - - Captured runtime manifests must be supplied by an explicit trusted - current-runtime reader, or already be part of the canonical checkpoint. - This method never provisions or captures runtime binaries. - """ - value=record['document']['value'];ref=value['material']['runtime'] - snapshot=retained_json(self.session.artifacts.read_blob( - _intrinsic(value['material']['recipe_document']),maximum_bytes=2*1024**2).decode(),2*1024**2) - if snapshot['format']=='builtin_recipe_snapshot_v1':environment=snapshot['validation']['runtime'] - else: - execution=snapshot['preflight']['material'] - environment=execution.get('torch_runtime',execution.get('cpu_snapshot',execution['runtime'])) - descriptor=environment if environment.get('profile') in ('captured_cpu_python_v1','managed_torch_python_v1') else None - prior=decode_records(before_records).get(('runtime_material',ref['catalog_id']+'.'+ref['sha256'])) - if prior is not None: - expected=prior['value']['descriptor'] if descriptor is not None else prior['value']['environment'] - if expected!=environment or prior['value']['ref']!=ref: - raise WorkbenchError('Recipe and retained runtime identity differ.','invalid',400) - if descriptor is not None: - if runtime_manifest is not None: - raw=runtime_manifest(copy.deepcopy(descriptor)) if callable(runtime_manifest) else runtime_manifest - retained=self.capture_runtime(prior['value'],raw) - return [*self.declarations(retained),retained] - self.session.artifacts.read_blob({'sha256':descriptor['manifest_sha256'],'bytes':descriptor['manifest_bytes']},maximum_bytes=32*1024**2) - return [] - manifest=None - manifest_raw=None - if descriptor is not None: - manifest_raw=runtime_manifest(copy.deepcopy(descriptor)) if callable(runtime_manifest) else runtime_manifest - if type(manifest_raw) is not bytes: - raise WorkbenchError('Explicit captured runtime manifest is unavailable.','unavailable',409) - if descriptor['profile']=='captured_cpu_python_v1': - from recipe_process import decode,validate_cpu_manifest - manifest=validate_cpu_manifest(decode(manifest_raw)) - else: - from torch_runtime import decode,validate_retained_manifest - manifest=validate_retained_manifest(decode(manifest_raw)) - runtime=self.capture_runtime(dict(ref=ref,descriptor=descriptor, - environment=None if descriptor is not None else environment,manifest=manifest),manifest_raw) - return [*self.declarations(runtime),runtime] - - @_boundary - def capture_snapshot(self, value, raw, *, catalog_id): - from projects import decode_retained_snapshot,decode,decode_retained_manifest - value=decode_retained_snapshot(value);_catalog(catalog_id) - if type(raw) is not bytes or len(raw)>64*1024 or hashlib.sha256(raw).hexdigest()!=value['manifest_sha256'] or decode_retained_manifest(decode(raw))!=value['manifest']: - raise WorkbenchError('Exact original project manifest bytes differ.') - key=catalog_id+'.'+value['manifest_sha256'] - prior=decode_records(session_records_for(self.session, *FAMILIES, 'project_fork')).get(('project_snapshot',key)) - if prior is not None: - original=prior['value'] - if any(original[field]!=value[field] for field in ('id','name','manifest_sha256','manifest')): - raise WorkbenchError('Original snapshot identity differs.','invalid',400) - # Explicit current root registration may move. It cannot replace - # original historical root/path identity facts at the same hash. - return dict(family='project_snapshot',id=key,document=prior) - item=dict(name=value['manifest_path'],sha256=value['manifest_sha256'],bytes=len(raw)) - self.session.artifacts.publish_bytes(raw,_declaration(item),maximum_bytes=64*1024) - record=dict(family='project_snapshot',id=catalog_id+'.'+value['manifest_sha256'],document=dict( - format=FAMILIES['project_snapshot'],origin_catalog_id=catalog_id,value=value,artifacts=_edges([item],'project_snapshot'),audit=[])) - decode_records([record]);return record - - @_boundary - def capture_runtime(self,value,manifest_raw=None): - value=decode_runtime_material(value,catalog_id=value['ref']['catalog_id']) - items=[] - if value['descriptor'] is not None: - descriptor=value['descriptor'] - if type(manifest_raw) is not bytes or len(manifest_raw)!=descriptor['manifest_bytes'] or hashlib.sha256(manifest_raw).hexdigest()!=descriptor['manifest_sha256']: - raise WorkbenchError('Exact original runtime manifest bytes differ.') - item=dict(name='manifest.json',sha256=descriptor['manifest_sha256'],bytes=len(manifest_raw)) - self.session.artifacts.publish_bytes(manifest_raw,_declaration(item),maximum_bytes=32*1024**2) - items=[item,*[dict(name=row['path'],sha256=row['sha256'],bytes=row['bytes']) for row in value['manifest']['files']]] - record=dict(family='runtime_material',id=value['ref']['catalog_id']+'.'+value['ref']['sha256'],document=dict( - format=FAMILIES['runtime_material'],origin_catalog_id=value['ref']['catalog_id'],value=value,artifacts=_edges(items,'runtime_material'),audit=[])) - decode_records([record]);return record - - def restore(self): - """Rebuild only current-namespace SQL values, without mutable owners.""" - documents=self._documents();catalog_id=self.session.db.dataset_id - from project_fork import active_records - from project_material_fork import is_intent - active={(r['family'],r['id']) for r in session_active(self.session, *FAMILIES,project_id=catalog_id)} - with self.session.db.transaction(): - for table in ('model_definition_drafts','workspace_graphs','workspace_recipe_drafts','model_materials','workspace_recipe_materials'): - self.session.db.execute('DELETE FROM '+table) - for (family,locator),doc in documents.items(): - if (family,locator) not in active or doc['origin_catalog_id']!=catalog_id:continue - if family in EDITABLE and is_intent(doc['current']):continue - if family in EDITABLE: - value=doc['current'] - table={'model_definition':'model_definition_drafts','pipeline_graph':'workspace_graphs','recipe_definition':'workspace_recipe_drafts'}[family] - self.session.db.execute('INSERT INTO '+table+' VALUES(%s,%s,%s)',(value['id'],value['revision'],encode(value))) - elif family=='model_material': - value=doc['value'];ref=value['ref'] - self.session.db.execute('INSERT INTO model_materials VALUES(%s,%s,%s,%s)',(ref['id'],ref['revision'],ref['sha256'],encode(value))) - elif family=='recipe_material': - value=doc['value'];material=value['material'] - self.session.db.execute('INSERT INTO workspace_recipe_materials VALUES(%s,%s,%s,%s)',(value['ref']['sha256'],material['recipe_id'],material['recipe_revision'],encode(value))) - - def reconcile_projection(self): - """Restore selected canonical material facts without deleting local custody. - - The caller holds projection/session/access guards and has drained custody. - Immutable SQL conflicts fence recovery instead of replacing hash identity. - """ - documents = self._documents() - catalog_id = self.session.db.dataset_id - from project_fork import active_records - from project_material_fork import is_intent - active = {(r['family'], r['id']) for r in session_active(self.session, *FAMILIES, project_id=catalog_id)} - with self.session.db.transaction(): - for (family, locator), doc in documents.items(): - if (family, locator) not in active or doc['origin_catalog_id'] != catalog_id: - continue - if family in EDITABLE: - value = doc['current'] - if is_intent(value): continue - table = {'model_definition':'model_definition_drafts', 'pipeline_graph':'workspace_graphs', - 'recipe_definition':'workspace_recipe_drafts'}[family] - self.session.db.execute('INSERT INTO '+table+'(id,revision,document) VALUES(%s,%s,%s) ' - 'ON CONFLICT(id) DO UPDATE SET revision=EXCLUDED.revision,document=EXCLUDED.document', - (value['id'], value['revision'], encode(value))) - elif family in ('model_material', 'recipe_material'): - value = doc['value']; ref = value['ref'] - table = 'model_materials' if family == 'model_material' else 'workspace_recipe_materials' - if family == 'model_material': - rows = self.session.db.execute('SELECT id,revision,sha256,document FROM model_materials ' - 'WHERE sha256=%s OR (id=%s AND revision=%s)', - (ref['sha256'],ref['id'],ref['revision'])).fetchall() - consistent = len(rows) == 1 and tuple(rows[0][:3]) == (ref['id'],ref['revision'],ref['sha256']) and retained_json(rows[0][3]) == value - else: - rows = self.session.db.execute('SELECT recipe_id,recipe_revision,document FROM workspace_recipe_materials ' - 'WHERE sha256=%s', (ref['sha256'],)).fetchall() - consistent = len(rows) == 1 and tuple(rows[0][:2]) == (value['material']['recipe_id'],value['material']['recipe_revision']) and retained_json(rows[0][2]) == value - if rows: - if not consistent: - raise WorkbenchError('Immutable projection material differs from its canonical hash.', - 'projection_unavailable', 503) - continue - if family == 'model_material': - self.session.db.execute('INSERT INTO model_materials(id,revision,sha256,document) VALUES(%s,%s,%s,%s)', - (ref['id'], ref['revision'], ref['sha256'], encode(value))) - else: - material = value['material'] - self.session.db.execute('INSERT INTO workspace_recipe_materials(sha256,recipe_id,recipe_revision,document) ' - 'VALUES(%s,%s,%s,%s)', (ref['sha256'],material['recipe_id'],material['recipe_revision'],encode(value))) - - def materialize_projection(self,ref,root): - """Explicit preparation may recreate a disposable owner byte cache. - - The destination must live inside the current physical SQL projection. - Canonical files and original runtime paths are never cleanup targets. - """ - root=Path(root) - if not root.resolve().is_relative_to(Path(self.session.db.path).resolve()): - raise WorkbenchError('Material cache must belong to the disposable projection.','invalid',400) - value=self.model(ref) if ref['kind']=='model' else self.recipe(ref) - family='model_material' if ref['kind']=='model' else 'recipe_material' - inventory=_inventory(family,value) - root.mkdir(mode=0o700,parents=True,exist_ok=True) - target=root/ref['sha256'] - if target.exists(): - from model_definitions import read_owned_blob - if target.is_symlink() or set(os.listdir(target))!={item['name'] for item in inventory}: - raise WorkbenchError('Disposable material cache differs.','unavailable',409) - for item in inventory:read_owned_blob(root,ref['sha256'],item,item['bytes']) - return target - staging=Path(tempfile.mkdtemp(prefix='.material-rebuild-',dir=root)) - try: - for item in inventory: - with self.session.artifacts.open_verified(_declaration(item),maximum_bytes=item['bytes']) as source: - with (staging/item['name']).open('xb') as destination: - while True: - raw=os.read(source,1024**2) - if not raw:break - destination.write(raw) - destination.flush();os.fchmod(destination.fileno(),0o400);os.fsync(destination.fileno()) - from metadata import _sync_directory - _sync_directory(staging) - # The Session lock serializes owner cache publication. An externally - # created target is refused, never adopted or overwritten. - if os.path.lexists(target):raise WorkbenchError('Material cache changed during reconstruction.','stale',409) - os.rename(staging,target);_sync_directory(root) - return target - finally: - if staging.exists():shutil.rmtree(staging) - - def prepare_recipe(self,owner,graph_id,revision,model_ref,expected_checkpoint,key,*,builtin=False,reconcile_capture=False,runtime_manifest=None): - self._require_resolved('pipeline_graph', graph_id) - operation='recipes.prepare_builtin_material' if builtin else 'recipes.prepare_external_material' - self.session.access.require('edit',operation,graph_id,owner_only=reconcile_capture and not builtin) - if builtin: - if reconcile_capture: - raise WorkbenchError('Built-in orphan capture reconciliation is unsupported.','unsupported',409) - from project_native_preparation import preparation_owner - current=preparation_owner(owner) - if current is None:raise WorkbenchError('Current project preparation for the runtime probe is unavailable.','unavailable',409) - return current.prepare_recipe(owner,graph_id,revision,model_ref,expected_checkpoint,key) - body=dict(graph_id=graph_id,revision=revision,model=model_ref,reconcile_capture=reconcile_capture) - def invoke(): - self.materialize_projection(model_ref,owner.dataset.models.path) - prepare=owner.prepare_builtin_material if builtin else owner.prepare_external_material - prepared=prepare(graph_id,revision,model_ref,reconcile_capture=reconcile_capture) - return owner._retain_material(prepared) - def capture(result,before): - record=self.capture_recipe_material(result,owner.material_path) - return {'records':self._coalesce([*self.declarations(record),record, - *self.capture_recipe_dependencies(record,before,runtime_manifest=runtime_manifest)]),'result':result} - return self.session.mutate(operation,graph_id,body,expected_checkpoint,key,invoke,capture) - - def mutate_definition(self, family, operation, target, body, expected_checkpoint, key, invoke): - authority = self.session.access.require('edit', operation, target) - def capture(result, before): - validate_operation_result(operation, result) - return {'records': [self.capture_definition(family,result,before,action=operation,actor_id=authority.actor_id, - audit_facts=[fact for fact in self.owner_access.facts if fact['target']==result['id']])], 'result': result} - return self.session.mutate(operation, target, body, expected_checkpoint, key, invoke, capture) - - def publish_model(self, owner, model_id, body, expected_checkpoint, key): - self._require_resolved('model_definition', model_id) - self.session.access.require('edit', 'models.publish', model_id, owner_only=body.get('reconcile_capture', False)) - def capture(result, before): - record=self.capture_model_material(result,owner.path) - return {'records': [*self.declarations(record),record], 'result': result} - return self.session.mutate('models.publish', model_id, body, expected_checkpoint, key, - lambda: owner.publish(model_id, body), capture) - - def retain_recipe(self, owner, prepared, expected_checkpoint, key, *, runtime_manifest=None): - value = prepared['document']; target = value['material']['recipe_id'] - self.session.access.require('edit', 'recipes.prepare_external_material', target, - owner_only=prepared.get('reconcile_capture', False)) - body = {'ref': value['ref']} - def capture(result, before): - record=self.capture_recipe_material(result,owner.material_path) - return {'records': self._coalesce([*self.declarations(record),record, - *self.capture_recipe_dependencies(record,before,runtime_manifest=runtime_manifest)]), 'result': result} - return self.session.mutate('recipes.material', target, body, expected_checkpoint, key, - lambda: owner._retain_material(prepared), capture) - - @_boundary - def evidence(self, run, selectors=None, *, read_blob=None): - import model_evidence - import recipe_contract - selectors = {} if selectors is None else selectors - inventory = {item['name']: item for item in run['artifacts']} - def read(name, maximum): - if name not in inventory: raise WorkbenchError('Evidence artifact is unavailable.', 'unavailable', 409) - item = inventory[name] - raw = read_blob(name, maximum) if read_blob else self.session.artifacts.read_blob(_intrinsic(item), maximum_bytes=maximum) - if type(raw) is not bytes or len(raw)>maximum or len(raw)!=item['bytes'] or hashlib.sha256(raw).hexdigest()!=item['sha256']: - raise WorkbenchError('Evidence artifact differs.', 'invalid', 400) - return raw - if run.get('extension_context') is not None: - import extension_evidence - ref = {'kind': 'recipe_material', 'catalog_id': run['dataset_id'], 'id': run['approval_material_sha256'], 'sha256': run['approval_material_sha256']} - composition = self.recipe(ref); closure = self.composition_closure(ref) - context = extension_evidence.validate_context(run['extension_context'], run, composition, closure['static_context']) - verified = extension_evidence.verify_blobs(run, context, read_blob=read, captured=closure['captured_inputs']) - return extension_evidence.query(verified, selectors) - from pipeline import adapter_for_graph, model_aware_graph - import image_training - if not model_aware_graph(run['graph']): return {'state': 'unsupported', 'reason': 'This retained adapter did not capture model_evidence_v1.'} - if set(selectors) - {'checkpoint_sha256', 'split'} or selectors.get('split', 'validation') != 'validation': - raise WorkbenchError('Unsupported retained evidence selector.', 'unsupported', 409) - if adapter_for_graph(run['graph']) is image_training: evaluator = model_evidence - else: - import one_fit_evidence as evaluator - result, descriptors, predictions = evaluator.verify_blobs(run, run['metrics'], read_blob=read) - result.update(checkpoint=None, predictions=None, metric_definition_hashes={item['id']: model_evidence.digest(item) for item in result['metric_definitions']}) - selected = selectors.get('checkpoint_sha256') - if selected is not None: - matches = [index for index, entry in enumerate(result['checkpoint_index']['entries']) if entry['checkpoint']['sha256'] == selected] - if not matches: raise WorkbenchError('Checkpoint does not belong to this Run.', 'not_found', 404) - result.update(checkpoint=descriptors[matches[0]], predictions=predictions[matches[0]]) - return result diff --git a/project_material_fork.py b/project_material_fork.py deleted file mode 100644 index deb35c8c..00000000 --- a/project_material_fork.py +++ /dev/null @@ -1,202 +0,0 @@ -"""Pure, finite reconstruction of editable materials for an independent fork. - -Original definitions and hash-bearing materials remain unchanged in the archive. -Unresolved bindings are retained as inert, inspectable intents, never SQL drafts. -""" -import copy -from datetime import datetime - -from workbench import WorkbenchError - -INTENT_FORMAT = 'project_material_fork_intent_v1' -OWNED = ('model_definition', 'pipeline_graph', 'recipe_definition') - - -def _fail(message): - raise WorkbenchError(message, 'invalid', 400) - - -def _date(value): - if type(value) is not str or len(value) > 80: - _fail('Fork creation requires an offset timestamp.') - try: - instant = datetime.fromisoformat(value) - if instant.utcoffset() is None: raise ValueError() - except ValueError: - _fail('Fork creation requires an offset timestamp.') - - -def is_intent(value): - return type(value) is dict and value.get('format') == INTENT_FORMAT - - -def _model_bindings(value, destination): - """Qualify only exact pure templates accepted by the existing task owners.""" - import image_training - import model_evidence - import one_fit_evidence - import text_training - import audio_training - task = value['task'] - owners = [(model_evidence, image_training), - (one_fit_evidence.contract(text_training), text_training), - (one_fit_evidence.contract(audio_training), audio_training)] - for owner, adapter in owners: - if task != owner.task_ref(task['catalog_id']): continue - required = [] - definition = None - specification = value['specification'] - try: - if adapter is image_training: - if specification.get('template') != 'tinycnn_v2': raise ValueError() - architecture = image_training.validate_architecture( - {k:v for k,v in specification.items() if k != 'template'}) - definition = dict(specification=dict(template='tinycnn_v2', **architecture), - preprocessing_policy=image_training.transform_for_parameters(architecture), - output_policy=model_evidence.OUTPUT_POLICY) - else: - if specification.get('template') != owner.template: raise ValueError() - definition = owner.model_definition({k:v for k,v in specification.items() if k != 'template'}) - except ValueError: - required.append('specification') - if definition is not None: - required.extend(field for field in ('specification','preprocessing_policy','output_policy') - if value[field] != definition[field]) - elif value['output_policy'] != owner.OUTPUT_POLICY: - required.append('output_policy') - return owner.task_ref(destination), sorted(set(required)) - return None, ['task'] - - -def _construct(family, source, identity, catalog): - """Rebuild only explicitly understood bindings; retain every other field.""" - from project_material import _definition - original = copy.deepcopy(source['definition'] if is_intent(source) else source) - required = [] - if family == 'model_definition': - task, required = _model_bindings(original, catalog) - if original['kind'] != 'template': required.append('source_selector') - if not required: - value = dict(original, id=identity, revision=1, task=task) - return _definition(family, value, catalog) - elif family == 'pipeline_graph': - if original.get('recipe') == 'external_recipe': - required = ['recipe_source', 'inputs', 'runtime'] - elif any(node['type'] == 'release' for node in original['nodes']): - required = ['release'] - elif any(node['type'] not in ('train', 'evaluate', 'validate', 'publish') or - node['type'] != 'train' and node['parameters'] for node in original['nodes']): - required = ['nodes'] - else: - # The graph owner owns its closed structure and adapter selection. - # No release, recipe-source or external input binding is adopted. - from pipeline import adapter_for_graph, types_for_graph - import image_training - adapter = adapter_for_graph(original) - try: - for node in original['nodes']: - if node['type'] not in types_for_graph(original): raise ValueError() - if node['type'] == 'train': - validate = (image_training.validate_model_parameters if original.get('recipe') == image_training.MODEL_RECIPE - else adapter.validate_parameters) - validate(node['parameters']) - except ValueError: - required = ['nodes'] - else: - return _definition(family, dict(original, id=identity, revision=1), catalog) - elif family == 'recipe_definition': - required = ['data', 'evaluation_profile', 'model', 'runtime', 'task', 'trainer'] - else: - _fail('Unsupported editable material fork family.') - return dict(format=INTENT_FORMAT, catalog_id=catalog, id=identity, revision=1, - name=original['name'], definition_family=family, definition=original, - required_reselection=sorted(required)) - - -def decode_intent(value, family, catalog): - from project_material import _closed, _catalog, _definition, ID - _closed(value, ('format', 'catalog_id', 'id', 'revision', 'name', - 'definition_family', 'definition', 'required_reselection'), 'fork intent') - if (value['format'] != INTENT_FORMAT or value['catalog_id'] != catalog or - value['definition_family'] != family or type(value['id']) is not str or - not ID.fullmatch(value['id']) or type(value['revision']) is not int or value['revision'] != 1): - _fail('Invalid material fork intent identity.') - old = value['definition'] - origin = old['task']['catalog_id'] if family != 'pipeline_graph' else catalog - _catalog(origin) - _definition(family, old, origin) - if _construct(family, old, value['id'], catalog) != value: - _fail('Material fork intent differs from its exact unresolved definition.') - return copy.deepcopy(value) - - -def fork_records(records, source_project_id, source_checkpoint, destination_project_id, *, id_factory, created_at): - from project_material import decode_records, FAMILIES, _catalog, SHA, ID - from project_fork import active_records - _catalog(source_project_id); _catalog(destination_project_id); _date(created_at) - if source_project_id == destination_project_id or type(source_checkpoint) is not str or not SHA.fullmatch(source_checkpoint): - _fail('Independent material fork requires a distinct identity and source checkpoint.') - decode_records(records) - active = active_records(records, project_id=source_project_id) - existing = {r['document']['current']['id'] for r in records if r['family'] in OWNED} - fresh, mapping = [], [] - for envelope in sorted(active, key=lambda r: (r['family'], r['id'])): - family = envelope['family'] - if family not in OWNED: continue - old = envelope['document']['current'] - identity = id_factory(family, envelope['id'], old['revision']) - if type(identity) is not str or not ID.fullmatch(identity) or identity in existing: - _fail('Material fork allocator must supply a fresh editable UUID.') - existing.add(identity) - value = _construct(family, old, identity, destination_project_id) - locator = destination_project_id + '.' + identity - fresh.append(dict(family=family, id=locator, document=dict(format=FAMILIES[family], - origin_catalog_id=destination_project_id, current=value, revisions=[copy.deepcopy(value)], audit=[], - fork_origin=dict(project_id=source_project_id, checkpoint=source_checkpoint, family=family, - id=envelope['id'], revision=old['revision'], created_at=created_at)))) - mapping.append(dict(family=family, source_id=envelope['id'], source_revision=old['revision'], destination_id=locator)) - mapping.sort(key=lambda m: (m['family'],m['source_id'],m['source_revision'],m['destination_id'])) - return dict(records=fresh, mapping=mapping) - - -def validate_records(records): - """Prove each fork's unaudited initial revision from exact retained owners.""" - from project_material import _closed, _catalog, SHA - from project_fork import decode_fork - if not any(r.get('family') == 'project_fork' or - r.get('family') in OWNED and 'fork_origin' in r.get('document', {}) for r in records): - return - decode_fork(records) - indexed = {(r['family'],r['id']):r for r in records} - overlays = [r['document'] for r in records if r['family'] == 'project_fork'] - mapped = {} - for overlay in overlays: - for mapping in overlay['mapping']: - if mapping['family'] in OWNED: - key = mapping['family'],mapping['destination_id'] - if key in mapped: _fail('Material fork target repeats across origin overlays.') - mapped[key] = (overlay, mapping) - for record in records: - if record['family'] not in OWNED: continue - doc = record['document']; key = record['family'],record['id'] - origin = doc.get('fork_origin') - if origin is None: - if key in mapped: _fail('Mapped material draft lacks its explicit fork origin.') - continue - _closed(origin, ('project_id','checkpoint','family','id','revision','created_at'), 'material fork origin') - _catalog(origin['project_id']); _date(origin['created_at']) - if type(origin['checkpoint']) is not str or not SHA.fullmatch(origin['checkpoint']): - _fail('Invalid material fork source checkpoint.') - if key not in mapped: _fail('Material fork origin lacks a retained mapping.') - overlay, mapping = mapped[key] - expected = dict(project_id=overlay['source']['project_id'], checkpoint=overlay['source']['checkpoint'], - family=mapping['family'], id=mapping['source_id'], revision=mapping['source_revision'], - created_at=origin['created_at']) - if origin != expected or doc['origin_catalog_id'] != overlay['destination_project_id']: - _fail('Material fork origin differs from its archived source mapping.') - original = indexed[(mapping['family'],mapping['source_id'])]['document']['current'] - if original['revision'] != origin['revision']: - _fail('Material fork source revision differs from its archived current value.') - initial = _construct(record['family'], original, doc['current']['id'], doc['origin_catalog_id']) - if doc['revisions'][0] != initial: - _fail('Material fork initial revision differs from its owner reconstruction.') diff --git a/project_measurements.py b/project_measurements.py deleted file mode 100644 index e64a307e..00000000 --- a/project_measurements.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Native custody and current access for immutable retained scalar packets. - -The scalar owner interprets the packet. ProjectArtifacts owns bytes and -ProjectSession owns publication/receipts. Retained source paths are facts only; -reads after Open need no current source registration or producer execution. -""" -import copy -from datetime import datetime -from functools import cache, partial - -from project_artifacts import validate_declaration -from project_material import decode_records as material_records -from projects import ID, NAME, SHA, relative_path -from retained_measurements import (MeasurementTable, MeasurementComparison, MAX_BYTES, MAX_RESPONSE_BYTES, - MAX_COMPARISON_SOURCES, MAX_COMPARISON_BYTES, encode, text) -from workbench import WorkbenchError, timestamp - - -FAMILY = 'measurement_source' -FORMAT = 'project_measurement_source_v1' -FIELDS = {'format', 'origin_catalog_id', 'id', 'title', 'blob', 'source', 'actor_id', 'imported_at'} - - -def _closed(value, fields): - if type(value) is not dict or set(value) != set(fields): - raise WorkbenchError('Invalid native measurement source fields.') - - -def _identity(value, pattern): - if type(value) is not str or not pattern.fullmatch(value): - raise WorkbenchError('Invalid native measurement source identity.') - return value - - -def declaration(blob): - return validate_declaration(dict(format='project_artifact_v1', **blob)) - - -def _document(value): - _closed(value, FIELDS) - if value['format'] != FORMAT: - raise WorkbenchError('Unsupported native measurement source format.', 'unsupported', 409) - for key in ('origin_catalog_id', 'actor_id'): - _identity(value[key], ID) - _identity(value['id'], SHA) - text(value['title']) - _closed(value['blob'], ('sha256', 'bytes')) - blob = declaration(value['blob']) - if not 0 < blob['bytes'] <= MAX_BYTES or blob['sha256'] != value['id']: - raise WorkbenchError('Measurement packet identity or byte bound differs.') - source = value['source'] - _closed(source, ('origin_catalog_id', 'project_id', 'manifest_sha256', 'reference_id', 'path', 'measured_sha256')) - for key in ('origin_catalog_id', 'project_id'): - _identity(source[key], ID) - for key in ('manifest_sha256', 'measured_sha256'): - _identity(source[key], SHA) - _identity(source['reference_id'], NAME) - relative_path(source['path']) - if source['measured_sha256'] != blob['sha256']: - raise WorkbenchError('Measurement source capture hash differs from its packet.') - try: - if type(value['imported_at']) is not str or len(value['imported_at']) > 80: - raise ValueError() - if datetime.fromisoformat(value['imported_at']).utcoffset() is None: - raise ValueError() - except ValueError: - raise WorkbenchError('Measurement import requires an attributed timestamp.') from None - return value - - -def decode_records(records, *, artifacts=None): - """Validate candidate closure, including exact available packet bytes. - - Missing payloads remain discoverable as missing. They never yield numerical - results. Present payloads must pass custody and scalar-source validation. - """ - declarations = {item['id']: item['document'] for item in records if item['family'] == 'project_artifact'} - snapshots = material_records(records) - result = {} - for item in records: - if item['family'] != FAMILY: - continue - doc = _document(item['document']) - identity = doc['origin_catalog_id'] + '.' + doc['id'] - if item['id'] != identity or identity in result: - raise WorkbenchError('Duplicate or inconsistent native measurement identity.') - if declarations.get(doc['blob']['sha256']) != declaration(doc['blob']): - raise WorkbenchError('Measurement artifact declaration is missing or inconsistent.') - source = doc['source'] - snapshot = snapshots.get(('project_snapshot', source['origin_catalog_id'] + '.' + source['manifest_sha256'])) - if snapshot is None or snapshot['value']['id'] != source['project_id']: - raise WorkbenchError('Measurement source snapshot is missing or inconsistent.') - reference = next((r for r in snapshot['value']['manifest']['data'] if r['id'] == source['reference_id']), None) - if (reference is None or reference['path'] != source['path'] - or reference.get('sha256', source['measured_sha256']) != source['measured_sha256']): - raise WorkbenchError('Measurement capture differs from its retained source reference.') - if artifacts is not None and artifacts.inspect(declaration(doc['blob']))['state'] != 'missing': - table = MeasurementTable(artifacts.read_blob(doc['blob'], maximum_bytes=MAX_BYTES)) - if table.metadata()['title'] != doc['title']: - raise WorkbenchError('Measurement title differs from the exact retained packet.') - result[identity] = copy.deepcopy(doc) - return result - - -def _validate_result(operation, result, lookup=None): - if operation != 'measurements.import': - return False - _document(result) - if lookup is not None: - identity = result['origin_catalog_id'] + '.' + result['id'] - if lookup(identity) != result: - raise WorkbenchError('Measurement receipt has no exact retained source.') - return True - - -def receipt_validator(records, *, artifacts=None): - """Own one candidate's validated source bindings and packet custody. - - Decode and copy every retained measurement once, including current available - packet bytes when supplied. The returned receipt checker is fixed to those - facts; it carries no live source or publication capability. - """ - owned = decode_records(records, artifacts=artifacts) - return partial(_validate_result, lookup=owned.get) - - -def validate_operation_result(operation, result, records=None): - @cache - def owned(): - return decode_records(records) - lookup = None if records is None else lambda identity: owned().get(identity) - return _validate_result(operation, result, lookup) - - -class ProjectMeasurements: - def __init__(self, session): - self.session, self.artifacts = session, session.artifacts - self.access, self.db = session.access, session.db - - def _documents(self): - # Session adoption/publication already validates the complete candidate. - # Verify current canonical bytes, then own only the measurement family. - records = self.session.check_records(FAMILY)['records'] - return {item['id']: item['document'] for item in records} - - def import_packet(self, raw, source, expected_checkpoint, key): - """Called only after SourceAdapter's explicit current rooted capture.""" - with self.access.lock, self.session.lock, self.db.lock: - actor = self.access.require('edit', 'measurements.import') - self.session.check_current() - table = MeasurementTable(raw) - blob = dict(sha256=table.sha256, bytes=len(raw)) - origin = self.session.summary['project_id'] - request = dict(blob=blob, source=copy.deepcopy(source)) - - def invoke(): - prior = self._documents().get(origin + '.' + table.sha256) - if prior is not None: - if prior['source'] != source: - raise WorkbenchError('This packet already has different retained capture facts.', 'conflict', 409) - return prior - self.artifacts.publish_bytes(raw, declaration(blob), maximum_bytes=MAX_BYTES) - return dict(format=FORMAT, origin_catalog_id=origin, id=table.sha256, - title=table.metadata()['title'], blob=blob, source=copy.deepcopy(source), - actor_id=actor.actor_id, imported_at=timestamp()) - - def capture(result, before): - return dict(records=[dict(family='project_artifact', id=table.sha256, document=declaration(blob)), - dict(family=FAMILY, id=origin + '.' + table.sha256, document=result)], result=result) - - return self.session.mutate('measurements.import', table.sha256, request, expected_checkpoint, key, invoke, capture) - - def sources(self, *, offset=0, limit=40): - if type(offset) is not int or not 0 <= offset <= 2**53-1 or type(limit) is not int or not 1 <= limit <= 40: - raise WorkbenchError('Invalid measurement source page.') - with self.access.lock, self.session.lock, self.db.lock: - self.access.require('inspect', 'measurements.sources') - self.session.check_current() - docs = sorted(self._documents().values(), key=lambda d: (d['origin_catalog_id'], d['id'])) - items = [dict(doc, availability=self.artifacts.inspect(declaration(doc['blob']))) - for doc in docs[offset:offset+limit]] - self.session.check_current() - self.access.require('inspect', 'measurements.sources') - return self._bounded(dict(items=items, total=len(docs), offset=offset, limit=limit)) - - @staticmethod - def _bounded(value): - if len(encode(value)) > MAX_RESPONSE_BYTES: - raise WorkbenchError('Measurement response exceeds its bound; select a smaller page.', 'too_large', 413) - return value - - def _read(self, origin_catalog_id, packet_sha256, action, select): - _identity(origin_catalog_id, ID) - _identity(packet_sha256, SHA) - target = origin_catalog_id + '.' + packet_sha256 - with self.access.lock, self.session.lock, self.db.lock: - self.access.require('inspect', action, target) - self.session.check_current() - doc = self._documents().get(target) - if doc is None: - raise WorkbenchError('Measurement source is unavailable.', 'not_found', 404) - raw = self.artifacts.read_blob(doc['blob'], maximum_bytes=MAX_BYTES) - table = MeasurementTable(raw) - if table.metadata()['title'] != doc['title']: - raise WorkbenchError('Measurement title differs from the exact retained packet.') - value = self._bounded(dict(source=doc, value=select(table))) - self.session.check_current() - self.access.require('inspect', action, target) - return value - - def metadata(self, origin_catalog_id, packet_sha256): - return self._read(origin_catalog_id, packet_sha256, 'measurements.metadata', lambda table: table.metadata()) - - def subjects(self, origin_catalog_id, packet_sha256, *, offset=0, limit=40): - return self._read(origin_catalog_id, packet_sha256, 'measurements.subjects', - lambda table: table.subjects(offset=offset, limit=limit)) - - def query(self, origin_catalog_id, packet_sha256, metric_id, **selection): - return self._read(origin_catalog_id, packet_sha256, 'measurements.query', - lambda table: table.query(metric_id, **selection)) - - def inspect(self, origin_catalog_id, packet_sha256, observation_id): - return self._read(origin_catalog_id, packet_sha256, 'measurements.inspect', - lambda table: table.inspect(observation_id)) - - def compare(self, sources, metric_id=None, **selection): - """Read one checkpoint's bounded group with every original source intact.""" - if type(sources) is not list or not 1 <= len(sources) <= MAX_COMPARISON_SOURCES: - raise WorkbenchError('Choose 1 through 16 native measurement sources.', 'too_large', 413) - targets = [] - for source in sources: - _closed(source, ('origin_catalog_id','packet_sha256')) - _identity(source['origin_catalog_id'], ID) - _identity(source['packet_sha256'], SHA) - targets.append(source['origin_catalog_id'] + '.' + source['packet_sha256']) - if len(set(targets)) != len(targets): - raise WorkbenchError('Native comparison repeats a measurement source.') - with self.access.lock, self.session.lock, self.db.lock: - for target in targets: - self.access.require('inspect', 'measurements.compare', target) - self.session.check_current() - documents = self._documents() - if any(target not in documents for target in targets): - raise WorkbenchError('A comparison source is unavailable.', 'not_found', 404) - chosen = [documents[target] for target in targets] - if sum(doc['blob']['bytes'] for doc in chosen) > MAX_COMPARISON_BYTES: - raise WorkbenchError('Native comparison exceeds 128 MiB of original packets.', 'too_large', 413) - tables = [] - for locator, doc in zip(sources, chosen): - raw = self.artifacts.read_blob(doc['blob'], maximum_bytes=MAX_BYTES) - table = MeasurementTable(raw) - if table.metadata()['title'] != doc['title']: - raise WorkbenchError('Comparison title differs from its original packet.') - tables.append((locator, table)) - value = MeasurementComparison(tables).read(metric_id, **selection) - result = self._bounded(dict(sources=chosen, value=value)) - self.session.check_current() - for target in targets: - self.access.require('inspect', 'measurements.compare', target) - return result diff --git a/project_native_execution.py b/project_native_execution.py deleted file mode 100644 index b2ff98ea..00000000 --- a/project_native_execution.py +++ /dev/null @@ -1,902 +0,0 @@ -"""Current native probe/train/verifier phases; retained phases are inert facts. - -Only an explicitly authorized preparation or an actual local Run continuation -may call the phase owner. Reading a checkpoint validates these documents and -their immutable evidence without preparing files, importing workers, or GO. -""" -import copy -from datetime import datetime -import hashlib -import os -from pathlib import Path -import stat -import sys -import time -import uuid - -import agent_api -import recipe_process -from pipeline import adapter_for_graph, adapter_identity_for_graph, decode_probe_observation, digest -from project_native_process import ENVIRONMENT, PHASES, WORKERS, prepare, validate_prepared_spec, validate_job_evidence -from project_session import REQUEST_KEY -from workbench import WorkbenchError, timestamp -from project_session import session_record, session_records - -FAMILY = 'native_execution_history' -FORMAT = 'native_execution_history_v1' -STATES = ('prepared', 'ready', 'go_intent', 'completed', 'failed', 'unknown') -FIELDS = {'format','origin_catalog_id','phase_id','phase','run_id','recipe', - 'request_key','request_sha256','actor_id','definition_identity','spec','events','payloads','observations','probe_result'} -OBSERVATIONS = {'job_template':('native-job-template.json',65536), - 'effective_job':('effective-job.json',65536),'parent_observation':('native-parent-observation.json',65536), - 'drain_receipt':('receipt.json',16384),'probe_stdout':('stdout.log',32768)} -PHASE_WALL_SECONDS = {'probe':45,'train':600,'verify':90} -PARENT_DRAIN_ALLOWANCE = 5 - - -def native_admission_wall_seconds(): - """New material's finite probe/train/verifier and custody time allowance. - - This budgets child/custodian phases, not a storage availability guarantee. - Cleanup already includes the custodian's TERM grace interval. - """ - overhead=recipe_process.READY_SECONDS+recipe_process.GO_WAIT_SECONDS+recipe_process.CLEANUP_SECONDS+PARENT_DRAIN_ALLOWANCE - return sum(PHASE_WALL_SECONDS.values())+len(PHASE_WALL_SECONDS)*overhead - - -def _fail(message, code='invalid'): - raise WorkbenchError(message, code, 409 if code != 'invalid' else 400) - - -def _id(value): - if type(value) is not str or recipe_process.ID.fullmatch(value) is None: - _fail('Invalid native execution identity.') - - -def _sha(value): - if type(value) is not str or recipe_process.HASH.fullmatch(value) is None: - _fail('Invalid native execution digest.') - - -def phase_reference(value): - event=value['events'][-1] - return dict(format='project_native_phase_v1',origin_catalog_id=value['origin_catalog_id'], - phase_id=value['phase_id'],phase=value['phase'],run_id=value['run_id'],state=event['state'], - spec_sha256=hashlib.sha256(recipe_process.encoded(value['spec'])).hexdigest(), - receipt_sha256=event['receipt_sha256']) - - -def classify_probe(recipe,raw): - """Process drainage and a valid native runtime observation are distinct.""" - result=dict(stdout=dict(sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw))) - if len(raw)>32768:return dict(result,outcome='too_large') - try:text=raw.decode('utf8') - except UnicodeError:return dict(result,outcome='invalid_utf8') - try:decode_probe_observation(recipe,recipe_process.decode(text)) - except (WorkbenchError,recipe_process.ProcessError):return dict(result,outcome='invalid_observation') - return dict(result,outcome='valid') - - -def decode_records(records, *, artifacts=None): - """Check exact finite phase/evidence closure, never local process state.""" - result={} - availability={} - declarations={r['id']:r['document'] for r in records if r['family']=='project_artifact'} - try: - for record in records: - if record['family']!=FAMILY:continue - value=record['document'] - if type(value) is not dict or set(value)!=FIELDS or value['format']!=FORMAT: - _fail('Invalid native phase document.') - for key in ('origin_catalog_id','phase_id','actor_id'):_id(value[key]) - if record['id']!=value['origin_catalog_id']+'.'+value['phase_id'] or record['id'] in result: - _fail('Native phase envelope identity differs.') - if value['phase'] not in PHASES or value['phase']!='probe' and value['run_id'] is None: - _fail('Native phase does not match its Run association.') - if value['run_id'] is not None:_id(value['run_id']) - if type(value['request_key']) is not str or REQUEST_KEY.fullmatch(value['request_key']) is None: - _fail('Invalid native phase request key.') - _sha(value['request_sha256']) - adapter_identity_for_graph(dict(recipe=value['recipe'])) - definition=value['definition_identity'] - if (type(definition) is not dict or set(definition)!={'definition_files','adapter_definition_sha256'} - or type(definition['definition_files']) is not dict or not definition['definition_files'] - or len(definition['definition_files'])>120 - or digest(definition['definition_files'])!=definition['adapter_definition_sha256']): - _fail('Native definition identity differs.') - for name,sha in definition['definition_files'].items(): - if (type(name) is not str or '/' in name or '\\' in name or not name.endswith('.py')): - _fail('Invalid native definition source name.') - _sha(sha) - worker=Path(adapter_for_graph(dict(recipe=value['recipe'])).__file__).name - spec=validate_prepared_spec(value['spec'],phase=value['phase'],worker=worker) - if (spec['attempt_id']!=value['phase_id'] or spec['request_sha256']!=value['request_sha256'] - or spec['env']!=ENVIRONMENT or spec.get('native_loader_policy')!='glibc_x86_64_no_preload_v1' - or spec['wall_seconds']!=PHASE_WALL_SECONDS[value['phase']] - or spec['max_output_bytes']!=64*1024*1024 or not 1<=len(spec['runtime_files'])<=3): - _fail('Native phase specification differs from its exact owner identity or environment.') - events=value['events'] - if type(events) is not list or not 1<=len(events)<=4: - _fail('Native phase has invalid bounded transitions.') - previous=None;helper=None - for index,event in enumerate(events): - if type(event) is not dict or set(event)!={'state','at','actor_id','helper','receipt_sha256'}: - _fail('Unclassified native phase event.') - _id(event['actor_id']) - state=event['state'] - allowed={'prepared':{'ready','failed','unknown'},'ready':{'go_intent','failed','unknown'}, - 'go_intent':{'completed','failed','unknown'}} - if (index==0 and state!='prepared' or index>0 and state not in allowed.get(previous,set())): - _fail('Invalid native phase transition.') - if type(event['at']) is not str or len(event['at'])>80 or datetime.fromisoformat(event['at']).utcoffset() is None: - _fail('Native phase requires an offset timestamp.') - if event['helper'] is not None: - if not recipe_process.valid_identity(event['helper']) or helper is not None and helper!=event['helper']: - _fail('Native phase helper identity changed.') - helper=event['helper'] - if state in ('ready','go_intent','completed') and event['helper'] is None: - _fail('Native phase lacks its actual READY helper observation.') - if event['receipt_sha256'] is not None: - _sha(event['receipt_sha256']) - if state not in ('completed','failed'):_fail('Native drainage proof precedes terminal observation.') - if state=='completed' and event['receipt_sha256'] is None: - _fail('Native completed phase requires its exact drainage proof.') - previous=state - payloads=value['payloads'] - if type(payloads) is not list or len(payloads)>180:_fail('Native payload inventory exceeds its bound.') - runs=[];output_names=set() - if value['run_id'] is not None: - runs=[r['document']['run'] for r in records if r['family']=='run_history' - and r['id']==value['origin_catalog_id']+'.'+value['run_id']] - if len(runs)!=1:_fail('Native phase lacks its exact retained Run.') - from runs import native_payload_names - output_names={'output/'+name for name in native_payload_names(runs[0])|{'artifact-manifest.json'}} - names=set();byname={} - for edge in payloads: - if (type(edge) is not dict or set(edge)!={'name','blob','role','preservation','required_for'} - or edge['role'] not in ('execution_source','execution_spec','custody_proof','measured_job','measured_stream','measured_output') - or edge['preservation']!='unique' or edge['required_for']!=['execution_proof'] - or type(edge['name']) is not str or edge['name'] in names): - _fail('Invalid native phase evidence edge.') - name=edge['name'];names.add(name);blob=edge['blob'] - if (name not in ('spec.json','native-job-template.json','effective-job.json','native-parent-observation.json', - 'stdout.log','stderr.log','receipt.json') and - not (name.startswith('source/') and name[7:] in {*definition['definition_files'],'native_process_bootstrap.py'}) - and name not in output_names): - _fail('Unsupported native phase evidence path.') - role=('execution_source' if name.startswith('source/') else 'measured_output' if name.startswith('output/') - else 'execution_spec' if name in ('spec.json','native-job-template.json') - else 'measured_stream' if name in ('stdout.log','stderr.log') - else 'custody_proof' if name=='receipt.json' else 'measured_job') - if edge['role']!=role:_fail('Native evidence edge has a different finite role.') - if type(blob) is not dict or set(blob)!={'sha256','bytes'}:_fail('Invalid native evidence blob.') - _sha(blob['sha256']) - if type(blob['bytes']) is not int or not 0<=blob['bytes']<=64*1024*1024: - _fail('Native evidence exceeds its owner bound.') - if declarations.get(blob['sha256'])!=dict(format='project_artifact_v1',**blob): - _fail('Native evidence declaration is missing or differs.') - byname[name]=blob - if artifacts is not None and blob['sha256'] not in availability: - observed=artifacts.status(dict(format='project_artifact_v1',**blob)) - if observed['state']=='corrupt':_fail('Present native evidence differs from its declared bytes.') - availability[blob['sha256']]=observed['state']=='verified' - observations=value['observations'] - if type(observations) is not dict or set(observations)!=set(OBSERVATIONS): - _fail('Native phase has unclassified canonical observations.') - proofs={} - for key,(name,maximum) in OBSERVATIONS.items(): - text=observations[key] - if text is not None: - if type(text) is not str:_fail('Native proof must retain its exact UTF8 observation.') - raw=text.encode('utf8') - if len(raw)>maximum or byname.get(name)!=dict(sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw)): - _fail('Canonical native observation differs from its exact evidence edge.') - proofs[key]=raw - spec_raw=recipe_process.encoded(spec) - if byname.get('spec.json')!=dict(sha256=hashlib.sha256(spec_raw).hexdigest(),bytes=len(spec_raw)): - _fail('Native phase lacks its exact prepared specification.') - for name,sha in definition['definition_files'].items(): - if byname.get('source/'+name,{}).get('sha256')!=sha: - _fail('Native phase lacks exact qualified source bytes.') - captures={str(Path(item['path']).relative_to(spec['cwd'])):dict(sha256=item['sha256'],bytes=item['bytes']) - for item in spec['captured_files']} - if set(captures)!={'source/'+name for name in definition['definition_files']}|( - {'source/native_process_bootstrap.py','native-job-template.json'} if value['phase']!='probe' else set()): - _fail('Native phase capture inventory differs from its finite definition closure.') - for name,blob in captures.items(): - if byname.get(name)!=blob:_fail('Native phase lacks exact immutable capture evidence.') - if value['run_id'] is not None: - if (len(runs)!=1 or runs[0]['graph']['recipe']!=value['recipe'] - or runs[0]['runtime']['definition_files']!=definition['definition_files']): - _fail('Native phase differs from its retained Run and exact adapter.') - if (value['phase']=='probe')!=(observations['job_template'] is None): - _fail('Native job template observation differs from its phase.') - if value['phase']!='probe' and observations['probe_stdout'] is not None: - _fail('Training/verifier logs are not probe runtime observations.') - if (observations['effective_job'] is None)!=(observations['parent_observation'] is None): - _fail('Native measured parent/job observations must be retained together.') - if value['phase']!='probe': - template=recipe_process.decode(proofs['job_template']) - if (type(template) is not dict or set(template)!={'format','phase','worker','attempt_id','request_sha256','job'} - or template['format']!='native_parent_binding_v1' or template['phase']!=value['phase'] - or template['worker']!=worker or template['attempt_id']!=value['phase_id'] - or template['request_sha256']!=value['request_sha256']): - _fail('Native job template differs from its exact phase.') - job=adapter_for_graph(dict(recipe=value['recipe']))._checked_job(template['job']) - run=runs[0] - from pipeline import model_aware_graph - import image_training - expected_job=dict(schema_version=1,dataset_id=run['dataset_id'],run_id=run['id'], - graph_sha256=run['graph_sha256'],release_id=run['release_id'],release_sha256=run['release_sha256'], - release_path=job['release_path'],output_dir=spec['output_dir'],parameters=run['parameters'],parent_pid=1) - if value['recipe']==image_training.MODEL_RECIPE: - expected_job.update(schema_version=2,evidence_context=run['evidence_context']) - elif model_aware_graph(run['graph']): - from one_fit_evidence import job_for_run - expected_job=job_for_run(run,spec['output_dir']) - # Historical Run projection deliberately carries no live - # release path. Its absolute template observation is inert; - # scientific release identity and all evidence selections - # remain exact projections of the retained Run owner. - expected_job['release_path']=job['release_path'] - if (agent_api.encoded(job)!=agent_api.encoded(template['job']) - or agent_api.encoded(job)!=agent_api.encoded(expected_job)): - _fail('Native job template differs from its exact scientific Run inputs.') - receipt_sha=events[-1]['receipt_sha256'] - if (receipt_sha is None)!=(observations['drain_receipt'] is None): - _fail('Native terminal receipt observation differs from its acknowledged phase.') - if receipt_sha is not None: - if byname.get('receipt.json',{}).get('sha256')!=receipt_sha: - _fail('Native terminal observation differs from its retained receipt.') - if observations['drain_receipt'] is not None: - raw=proofs['drain_receipt'] - expected=dict(attempt_id=value['phase_id'],request_sha256=value['request_sha256'], - spec_sha256=hashlib.sha256(spec_raw).hexdigest(),helper=helper) - receipt=recipe_process.validate_receipt(raw,expected) - validate_job_evidence(spec,phase=value['phase'],helper=helper, - template=proofs.get('job_template'),effective_job=proofs.get('effective_job'), - parent_observation=proofs.get('parent_observation'),receipt=receipt) - if not receipt['drained'] or (previous=='completed')!=(receipt['outcome']=='completed'): - _fail('Native terminal state differs from measured drainage.') - for stream in ('stdout','stderr'): - if byname.get(stream+'.log')!=receipt['streams'][stream]: - _fail('Native measured stream differs from its exact receipt.') - if value['phase']=='probe': - classified=value['probe_result'] - if (type(classified) is not dict or set(classified)!={'outcome','stdout'} - or classified['outcome'] not in ('valid','invalid_utf8','invalid_observation','too_large') - or classified['stdout']!=byname.get('stdout.log')): - _fail('Native probe protocol observation differs from measured stdout.') - if observations['probe_stdout'] is not None: - if classify_probe(value['recipe'],proofs['probe_stdout'])!=classified: - _fail('Canonical native probe classification differs from exact stdout.') - elif classified['outcome'] not in ('invalid_utf8','too_large'): - _fail('A bounded UTF8 native probe requires its exact canonical stdout.') - if (classified['outcome']=='too_large')!=(classified['stdout']['bytes']>32768): - _fail('Native oversized probe observation has a different bound.') - if artifacts is not None and availability.get(classified['stdout']['sha256']): - actual=artifacts.read_blob(classified['stdout'],maximum_bytes=recipe_process.STREAM_LIMIT) - if classify_probe(value['recipe'],actual)!=classified: - _fail('Present native probe bytes differ from the classified observation.') - elif value['probe_result'] is not None: - _fail('Native probe protocol result precedes its measured terminal receipt.') - if value['phase']!='probe' and value['probe_result'] is not None: - _fail('Only a probe phase may classify a runtime response.') - result[record['id']]=copy.deepcopy(value) - return result - except WorkbenchError:raise - except (ValueError,TypeError,KeyError,AttributeError,OverflowError,RecursionError): - _fail('Malformed retained native execution phase.') - - -class _RunAuthority: - """One actual current SQL admission, never constructed from a checkpoint.""" - def __init__(self,native,run_id): - self.native=native;self.run_id=run_id - self.recipe=native.execution.runs._load(run_id)['graph']['recipe'] - - def authorize_probe(self,recipe): - execution=self.native.execution;owner=execution.runs - with execution.db.lock: - run=owner._load(self.run_id);execution._proof(run) - context=execution.runtime_access.require_job('runs',run['id'],run['admission_id'],run['request_sha256']) - execution.runtime_access.require('edit','runs.update',run['id']) - execution.runtime_access.require('execute','pipeline.probe_native',recipe) - if recipe!=self.recipe or run['state'] not in ('queued','running','cancelling'): - _fail('Current native Run cannot observe this phase.','unavailable') - inputs=self.native._run_inputs.get(run['id']) - if inputs is None:_fail('A retained native Run has no current one-use qualification.','unavailable') - if (os.environ.get('TULDOK_TRAINING_PYTHON') or sys.executable)!=inputs['runtime']['executable']: - _fail('Configured native interpreter changed after admission.','stale') - from project_native_process import current_runtime - if current_runtime(inputs['runtime']['executable'])!=inputs['runtime']: - _fail('Current native interpreter changed after admission.','stale') - if owner.pipelines.definition_identity(recipe)!=inputs['definition_identity']: - _fail('Native adapter changed after admission.','stale') - return context - - def dispatch_allowed(self): - owner=self.native.execution.runs;run=owner._load(self.run_id) - if run['state'] not in ('queued','running') or owner._stop.is_set() or owner._blocked: - _fail('Current native Run no longer permits dispatch.','unavailable') - - -class _ReconciliationAuthority: - def __init__(self,native,recipe):self.native=native;self.recipe=recipe - def authorize_probe(self,recipe): - if recipe!=self.recipe:_fail('Native reconciliation phase changed.','conflict') - return self.native.access.require('cancel','native_execution.reconcile',owner_only=True) - - -class ProjectNativeExecution: - """The fixed current phase owner; construction is entirely inert.""" - def __init__(self, execution): - self.execution=execution;self.projection=execution.projection - self.session=execution.session;self.db=execution.db;self.access=execution.runtime_access - self._dirty={};self._handles={};self._prepared={};self._contexts={};self._active=set();self._run_inputs={} - self._never_launched=set() - self._foreground_phase=None - - @property - def has_dirty(self):return bool(self._dirty) - - @property - def has_local_custody(self): - return bool(self._active or self._handles) - - def capture_dirty(self, before_records): - records=[] - for phase_id,document in self._dirty.items(): - self._authorize(phase_id) - records.append(dict(family=FAMILY,id=document['origin_catalog_id']+'.'+phase_id,document=copy.deepcopy(document))) - records.extend(dict(family='project_artifact',id=edge['blob']['sha256'], - document=dict(format='project_artifact_v1',**edge['blob'])) for edge in document['payloads']) - return records - - def outcome(self, outcome): - if outcome not in ('committed','rolled_back','unknown'):raise ValueError('Invalid native SQL outcome.') - self._dirty.clear() - if outcome=='unknown' or not self.session.available: - for handle in self._handles.values():handle.close_control(wait=False) - - def abandon_pending(self): - if not self.execution._reconstructing:raise RuntimeError('Native pending capture clears only during guarded reconstruction.') - self._dirty.clear() - - def _authorize(self, phase_id): - context=self._contexts.get(phase_id) - if context is None:_fail('Native phase has no actual current preparation context.','unavailable') - return context.authorize_probe(context.recipe) - - def authorize_dirty(self): - for phase_id in self._dirty:self._authorize(phase_id) - - def _dispatch_authorized(self,phase_id): - actor=self._authorize(phase_id) - context=self._contexts[phase_id] - if isinstance(context,_RunAuthority):context.dispatch_allowed() - return actor - - def _retain(self, document, name, raw, role): - declaration=dict(format='project_artifact_v1',sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw)) - self.session.artifacts.publish_bytes(raw,declaration,maximum_bytes=64*1024*1024) - document['payloads'].append(dict(name=name,blob={k:declaration[k] for k in ('sha256','bytes')}, - role=role,preservation='unique',required_for=['execution_proof'])) - for key,(filename,maximum) in OBSERVATIONS.items(): - if name==filename and (key!='probe_stdout' or document['phase']=='probe'): - if key=='probe_stdout': - document['probe_result']=classify_probe(document['recipe'],raw) - try:text=raw.decode('utf8') if len(raw)<=maximum else None - except UnicodeError:text=None - document['observations'][key]=text - else: - if len(raw)>maximum:_fail('Native canonical observation exceeds its fixed bound.') - document['observations'][key]=raw.decode('utf8') - - def _retain_outputs(self,document,*,run=None,output=None): - """Preserve measured native outputs without claiming their validity.""" - if document['phase']=='probe':return - from runs import native_payload_names - run=run if run is not None else self.execution.runs._load(document['run_id']) - allowed=native_payload_names(run)|{'artifact-manifest.json'} - # A saved pathname is an observation, never a directory capability. - # Select only this physical projection's exact phase stage, including - # the earlier training stage explicitly associated with a verifier. - from project_store import DIR_FLAGS, FILE_FLAGS, ProjectStoreError, _open_directory_path, anchored_directory - stage_id=document['phase_id'] - if document['phase']=='verify': - matches=[row['document'] for row in session_records(self.session, FAMILY) - if row['family']==FAMILY and row['document']['phase']=='train' - and row['document']['origin_catalog_id']==self.db.dataset_id - and row['document']['run_id']==document['run_id']] - if len(matches)!=1:_fail('Native verifier lacks its exact training stage.','unknown') - stage_id=matches[0]['phase_id'] - _id(stage_id) - relative='project-native-phases/'+stage_id+'/output' - expected=self.db.path/relative - if document['spec']['output_dir']!=str(expected) or output is not None and Path(output)!=expected: - _fail('Native measured output is not its original local stage.','unknown') - root=None;captured=[];total=0 - def identity(info): - return info.st_dev,info.st_ino,info.st_size,info.st_mtime_ns,info.st_ctime_ns - try: - root_path,root=_open_directory_path(self.db.path) - with anchored_directory(root,relative,root_path=root_path) as directory: - before=os.fstat(directory);names=sorted(os.listdir(directory)) - for name in names: - if name not in allowed:_fail('Native worker produced an unclassified output; retain local custody.','unknown') - descriptor=os.open(name,FILE_FLAGS,dir_fd=directory) - try: - info=os.fstat(descriptor) - if not stat.S_ISREG(info.st_mode) or info.st_size>document['spec']['max_output_bytes']: - _fail('Native output is not an admitted bounded regular file.','unknown') - chunks=[];remaining=document['spec']['max_output_bytes']-total+1 - while remaining: - chunk=os.read(descriptor,min(remaining,65536)) - if not chunk:break - chunks.append(chunk);remaining-=len(chunk) - raw=b''.join(chunks);total+=len(raw) - if (total>document['spec']['max_output_bytes'] or identity(info)!=identity(os.fstat(descriptor)) - or identity(info)!=identity(os.stat(name,dir_fd=directory,follow_symlinks=False))): - _fail('Native output changed during its measured observation.','unknown') - captured.append((name,raw)) - finally:os.close(descriptor) - if identity(before)!=identity(os.fstat(directory)) or names!=sorted(os.listdir(directory)): - _fail('Native output directory changed during observation.','unknown') - # The anchored context rechecks the complete directory binding - # before any bytes receive canonical measured-output attribution. - for name,raw in captured:self._retain(document,'output/'+name,raw,'measured_output') - except (OSError,ProjectStoreError) as error: - raise WorkbenchError('Native output stage is unavailable or changed; retain local custody.','unknown',409) from error - finally: - if root is not None:os.close(root) - - def _publish(self, document, state, *, helper=None, receipt_sha256=None): - with self.db.lock,self.db.transaction(): - self.session.check_current();actor=self._authorize(document['phase_id']) - document['events'].append(dict(state=state,at=timestamp(),actor_id=actor.actor_id, - helper=copy.deepcopy(helper),receipt_sha256=receipt_sha256)) - self._dirty[document['phase_id']]=copy.deepcopy(document) - if state!='prepared' and self.execution.custody is not None: - with self.execution._journal_guard():self.execution.custody.native_observed(document) - - def assert_quiescent(self): - if self.has_local_custody:_fail('Current native phase custody requires explicit drainage.','busy') - - def reconcile(self,phase_id,expected_checkpoint): - """Observe exact original-host drainage; never prepare, import or GO.""" - import json - from project_native_process import read_phase_evidence - _id(phase_id);_sha(expected_checkpoint) - self.access.require('cancel','native_execution.reconcile',phase_id,owner_only=True) - if self.db.lock.owned_by_current_thread or self.db.in_transaction: - _fail('Native custody reconciliation must drain outside command locks.','busy') - execution=self.execution;journal=execution.custody - if journal is None:_fail('There is no original local native custody.','unavailable') - with self.db.lock: - if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') - local=journal.refresh() - selected=next((row for row in local.get('native_phases',[]) if row['phase_id']==phase_id),None) - if selected is None:_fail('A retained phase alone cannot select current native custody.','unavailable') - if (local['runs'] or local['producers'] or local['runtime_effects'] or local['package_effects'] - or any(row['run_id']!=selected['run_id'] for row in local.get('native_admissions',[])) - or execution._runtime_active or execution._package_active): - _fail('Other current effect owners must reconcile first.','busy') - entries=[row for row in local['native_phases'] - if row['phase_id']==phase_id or selected['run_id'] is not None and row['run_id']==selected['run_id']] - if len(entries)!=len(local['native_phases']):_fail('Other native custody remains unresolved.','busy') - if not self.db.claim_runtime('project_native_phases'):_fail('The original native application is still active.','busy') - other=self.db.execute("SELECT id FROM workspace_runs WHERE document::jsonb->>'state' IN ('queued','running','cancelling','unknown') AND id<>%s", - (selected['run_id'] or '',)).fetchall() - if other:_fail('Another current Run must drain before native recovery.','busy') - owner=execution._runs - if owner is not None:owner._fence_coordinator() - for entry in entries: - handle=self._handles.get(entry['phase_id']) - if handle is not None: - receipt=handle.close_control() - if not receipt or not receipt.get('drained'):_fail('Actual native helper has not drained.','unknown') - if owner is not None and owner._thread is not None: - owner._thread.join(60) - if owner._thread.is_alive():_fail('Current native supervisor has not drained.','busy') - if self._active:_fail('Current native owner has not finished its bounded observation.','busy') - with execution.access.lock,self.session.lock,self.db.lock: - self.access.require('cancel','native_execution.reconcile',phase_id,owner_only=True) - if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') - if journal.refresh()!=local:_fail('Original native custody changed while draining.','stale_project') - binding=journal.binding - if (binding['project_id']!=self.db.dataset_id or binding['projection_id']!=self.db.projection_id - or binding['host_catalog_id']!=execution.access.catalog_id): - _fail('Native custody belongs to another physical owner.','local_custody_unavailable') - self.session.available=False - snapshot=self.session.store.recover_checkpoint();journal.validate_selection(snapshot) - with self.session.store.root_directory() as descriptor: - info=os.fstat(descriptor) - if [info.st_dev,info.st_ino]!=binding['root_identity']: - _fail('Native custody belongs to another physical project root.','local_custody_unavailable') - documents=decode_records(snapshot['records'],artifacts=self.session.artifacts) - phases=[];local_run=None;retained_run=None - if selected['run_id'] is not None: - if owner is None and not self.db.claim_runtime('workspace_runs'): - _fail('The original Run application still owns this projection.','busy') - row=self.db.execute('SELECT document FROM workspace_runs WHERE id=%s',(selected['run_id'],)).fetchone() - if row is None:_fail('Native custody has no original local Run admission.','local_custody_unavailable') - local_run=json.loads(row['document']);execution._proof(local_run) - retained_run=next((row['document']['run'] for row in snapshot['records'] if row['family']=='run_history' - and row['id']==self.db.dataset_id+'.'+selected['run_id']),None) - immutable=('id','dataset_id','request_key','request_sha256','admission_id','graph_sha256','release_sha256','runtime') - if retained_run is None or any(local_run.get(key)!=retained_run.get(key) for key in immutable): - _fail('Original native Run admission differs from retained science.','local_custody_unavailable') - for entry in entries: - document=copy.deepcopy(documents.get(self.db.dataset_id+'.'+entry['phase_id'])) - stage=self.db.path/'project-native-phases'/entry['phase_id'] - if (document is None or document['run_id']!=entry['run_id'] or document['phase']!=entry['phase'] - or document['request_sha256']!=entry['request_sha256'] or document['spec']['cwd']!=str(stage) - or hashlib.sha256(recipe_process.encoded(document['spec'])).hexdigest()!=entry['spec_sha256']): - _fail('Original native phase differs from its exact physical journal.','local_custody_unavailable') - if entry['helper'] is None: - # A prepared journal has not acknowledged launch intent. - # After intent, only this process's retained call boundary - # proves Popen was never entered; a restarted owner cannot - # convert that ambiguous spawn window into cessation. - if entry['state']!='prepared' and not (entry['state']=='launch_intent' and entry['phase_id'] in self._never_launched): - _fail('Original native launch lacks an exact helper drainage identity.','unknown') - evidence=None - else:evidence=read_phase_evidence(stage,document['spec'],phase=entry['phase'],helper=entry['helper']) - phases.append((entry,document,evidence)) - # Fixed cache reconstruction happens only after every actual/local - # effect has a complete drain receipt. It restores no Run or grant. - self.projection.reconcile_inert(snapshot) - try: - for entry,document,evidence in phases: - identity=entry['phase_id'];self._contexts[identity]=_ReconciliationAuthority(self,document['recipe']) - if evidence is None: - if document['events'][-1]['state']!='failed':self._publish(document,'failed') - elif entry['state']!='failed': - with execution._journal_guard():journal.native_observed(document) - with execution._journal_guard():journal.native_not_launched(identity) - self._never_launched.discard(identity) - continue - names={edge['name'] for edge in document['payloads']} - for name,raw,role in (('stdout.log',evidence['stdout'],'measured_stream'), - ('stderr.log',evidence['stderr'],'measured_stream'), - ('receipt.json',recipe_process.encoded(evidence['receipt']),'custody_proof'), - ('effective-job.json',evidence['effective_job'],'measured_job'), - ('native-parent-observation.json',evidence['parent_observation'],'measured_job')): - if raw is not None and name not in names:self._retain(document,name,raw,role) - if document['phase']!='probe' and not any(name.startswith('output/') for name in names): - output=stage/'output' - if document['phase']=='verify': - train=next((value for value in documents.values() if value['run_id']==document['run_id'] - and value['phase']=='train'),None) - if train is None:_fail('Original verifier has no exact retained training phase.','unknown') - output=self.db.path/'project-native-phases'/train['phase_id']/'output' - if document['spec']['output_dir']!=str(output):_fail('Native output path differs from original local custody.','unknown') - if output.exists():self._retain_outputs(document,run=retained_run,output=output) - receipt_sha=hashlib.sha256(recipe_process.encoded(evidence['receipt'])).hexdigest() - if document['events'][-1]['state'] not in ('completed','failed'): - state='completed' if evidence['receipt']['outcome']=='completed' else 'failed' - self._publish(document,state,helper=entry['helper'],receipt_sha256=receipt_sha) - elif document['events'][-1]['receipt_sha256']!=receipt_sha: - _fail('Original native terminal receipt changed.','unknown') - elif entry['state'] not in ('completed','failed'): - with execution._journal_guard():journal.native_observed(document) - self._handles.pop(identity,None) - if retained_run is not None: - current=copy.deepcopy(retained_run) - from runs import LOCAL_RUN_FIELDS - current.update({key:copy.deepcopy(value) for key,value in local_run.items() if key in LOCAL_RUN_FIELDS}) - owner=execution._local_owner(recover=False) if owner is None else owner - owner._fence_coordinator() - with self.db.transaction(): - execution._proof(current) - state='completed' if current['state']=='completed' else 'interrupted' - owner._event(current,state,'Original native custody drained; no phase was resumed or regenerated.') - owner._save(current) - elif evidence is not None: - journal.native_confirmed(phase_id) - journal.acknowledge(self.session.snapshot) - return phase_reference(next(document for entry,document,evidence in phases if entry['phase_id']==phase_id)) - except BaseException: - self.session.available=False - if owner is not None:owner._fence_coordinator() - raise - - def settle_run(self,run): - if self.execution.custody is None or run['state'] not in ('completed','failed','cancelled','interrupted'):return - for entry in self.execution.custody.snapshot().get('native_phases',[]): - if entry['run_id']==run['id'] and entry['state'] in ('completed','failed') and entry['receipt_sha256'] is not None: - self.execution.custody.native_confirmed(entry['phase_id']) - - def bind_run(self,run): - """Retain a private measured qualification only during actual admission.""" - if not self.db.in_transaction:_fail('Native qualification binds only inside its admission transaction.') - if self.execution.runs._validation_run(run):return - context=self.projection.native_preparation.current(self.execution.runs.pipelines) - _actor,inputs=self._inputs(run['graph']['recipe'],context) - if (run['runtime']['definition_files']!=inputs['definition_identity']['definition_files'] - or run['runtime']['adapter_definition_sha256']!=inputs['definition_identity']['adapter_definition_sha256']): - _fail('Native Run differs from its qualified adapter.','stale') - if run['id'] in self._run_inputs:_fail('A native Run qualification cannot be replaced.','conflict') - self._run_inputs[run['id']]=copy.deepcopy(inputs) - self.db.on_rollback(lambda:self._run_inputs.pop(run['id'],None)) - - def probe_current_run(self,owner,recipe): - from access import current_context - context=current_context() - if (context is None or context.mode!='job' or context.kind!='runs' - or context.catalog_id!=self.db.dataset_id or owner.db is not self.db): - _fail('Native runtime probing requires its exact current local Run.','unavailable') - authority=_RunAuthority(self,context.job_id);actor=authority.authorize_probe(recipe) - inputs=self._run_inputs[context.job_id] - result=self._run_phase('probe',recipe,'native-run-probe-'+context.job_id,actor,inputs,authority,run_id=context.job_id) - return dict(runtime=self._probe_result(recipe,inputs,result['evidence']),phase=phase_reference(result['document'])) - - def capture_run_payloads(self,run,artifacts): - """Copy a newly admitted frozen native input, then reuse exact custody.""" - from projects import Projects - prior=session_record(self.session, 'run_history',run['dataset_id']+'.'+run['id']) - prior=None if prior is None else prior['document'] - if prior is not None: - retained=[copy.deepcopy(edge) for edge in prior['payloads'] - if edge['name']=='release.zip' or edge['name'].startswith('source/')] - expected={'release.zip':run['release_sha256'], - **{'source/'+name:sha for name,sha in run['runtime']['definition_files'].items()}} - if {edge['name']:edge['blob']['sha256'] for edge in retained}!=expected: - _fail('Native Run input/source custody is incomplete.','unavailable') - for edge in retained: - with artifacts.open_verified(dict(format='project_artifact_v1',**edge['blob'])):pass - return retained - if run['state']!='queued':_fail('Only an actual newly admitted native Run may capture inputs.','unavailable') - edges=[] - release=Path(run['release_path']) - reader=Projects(self.execution.runs.dataset,{'native_release':release.parent}) - try: - declaration=dict(format='project_artifact_v1',sha256=run['release_sha256'],bytes=run['release_bytes']) - with reader._file('native_release',release.name) as (descriptor,_info):artifacts.publish_fd(descriptor,declaration) - edges.append(dict(name='release.zip',blob={key:declaration[key] for key in ('sha256','bytes')}, - population='unknown',role='captured_input',preservation='unique',required_for=['scientific_inspection','execution_proof'])) - finally:reader.close() - inputs=self._run_inputs.get(run['id']) - if self.execution.runs._validation_run(run): - sources={name:(self.execution.runs.pipelines.source_path/name).read_bytes() - for name in run['runtime']['definition_files']} - elif inputs is None:_fail('Native input capture has no current qualification.','unavailable') - else:sources=inputs['sources'] - for name,raw in sorted(sources.items()): - if hashlib.sha256(raw).hexdigest()!=run['runtime']['definition_files'][name]: - _fail('Native source changed before canonical admission.','stale') - declaration=dict(format='project_artifact_v1',sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw)) - artifacts.publish_bytes(raw,declaration) - edges.append(dict(name='source/'+name,blob={key:declaration[key] for key in ('sha256','bytes')}, - population='unknown',role='execution_source',preservation='unique',required_for=['scientific_inspection','execution_proof'])) - return edges - - def execute_run(self,run_id): - """Run the existing CPU owners through three fixed custody phases.""" - from pipeline import model_aware_graph - from types import SimpleNamespace - from runs import _sync - owner=self.execution.runs - with self.db.lock,self.db.transaction(): - run=owner._load(run_id) - self.access.require_job('runs',run_id,run['admission_id'],run['request_sha256']) - if run['state']!='queued' or owner._stop.is_set() or owner._blocked:return - owner._event(run,'running','Verifying exact frozen native inputs before current CPU phases.') - run['started_at']=timestamp();run['progress']['stage']='validate';owner._save(run) - try: - if owner._validation_run(run):return owner._execute_validation(run) - runtime=owner.pipelines.probe(run['graph']['recipe']) - if not runtime['available'] or runtime!=run['runtime']:_fail('Native runtime changed after admission.','stale') - release=owner.pipelines.inspect_release(run['release_id'],private=True,recipe=run['graph']['recipe']) - if release['sha256']!=run['release_sha256']:_fail('Native frozen release changed after admission.','stale') - authority=_RunAuthority(self,run_id);actor=authority.authorize_probe(run['graph']['recipe']) - phase_id=uuid.uuid4().hex;output=self.db.path/'project-native-phases'/phase_id/'output' - job=dict(schema_version=1,dataset_id=run['dataset_id'],run_id=run_id,graph_sha256=run['graph_sha256'], - release_id=run['release_id'],release_sha256=run['release_sha256'],release_path=release['path'], - output_dir=str(output),parameters=run['parameters'],parent_pid=1) - if run['graph']['recipe']=='image_classification_model_v2': - job.update(schema_version=2,evidence_context=run['evidence_context']) - elif model_aware_graph(run['graph']): - from one_fit_evidence import job_for_run - job.update(job_for_run(run,output),parent_pid=1,release_path=release['path']) - with self.db.lock,self.db.transaction(): - current=owner._load(run_id);current['progress']['stage']='train' - current['execution_test_exposure']=owner.pipelines.exposures(current['source_records']) - for node in current['node_attempts']: - node['state']='completed' if node['type'] in ('release','validate') else 'running' if node['type']=='train' else 'pending' - owner._save(current) - trained=self._run_phase('train',run['graph']['recipe'],'native-train-'+run_id,actor, - self._run_inputs[run_id],authority,run_id=run_id,job=job,phase_id=phase_id) - if trained['evidence']['receipt']['outcome']!='completed': - state='cancelled' if trained['evidence']['receipt']['outcome']=='cancelled' else 'failed' - return owner._finish(run_id,state,'Native trainer drained without publishing results.') - with (trained['prepared'].stage/'custody/stdout.log').open('rb') as stream: - owner._read_progress(SimpleNamespace(stdout=stream),run,dict(buffer=b'',bytes=0,epoch=0)) - artifacts,metrics=owner._verify_directory(run,output) - if model_aware_graph(run['graph']): - verified=self._run_phase('verify',run['graph']['recipe'],'native-verify-'+run_id,actor, - self._run_inputs[run_id],authority,run_id=run_id,job=job) - if verified['evidence']['receipt']['outcome']!='completed': - state='cancelled' if verified['evidence']['receipt']['outcome']=='cancelled' else 'failed' - return owner._finish(run_id,state,'Independent checkpoint verification drained without accepting outputs.') - for path in output.iterdir(): - with path.open('rb') as source:os.fsync(source.fileno()) - _sync(output) - with self.db.lock,self.db.transaction(): - current=owner._load(run_id) - if current['state']=='cancelling' or owner._stop.is_set(): - owner._event(current,'cancelled','Native workers drained before final publication.');owner._save(current);return - os.rename(output,owner.path/run_id);_sync(owner.path) - current['artifacts']=artifacts;current['metrics']=metrics - for node in current['node_attempts']:node['state']='completed' - owner._event(current,'completed','Measured native CPU outputs and independent evidence published after custody drainage.') - owner._save(current) - except (WorkbenchError,recipe_process.ProcessError) as error: - state='unknown' if error.code in ('unknown','outcome_unknown') else 'failed' - owner._finish(run_id,state,str(error),error.code) - - def _inputs(self, recipe, preparation): - from project_native_preparation import NativePreparationContext - if type(preparation) is not NativePreparationContext or preparation.recipe!=recipe: - _fail('Native probing requires its actual current preparation context.','unavailable') - actor=preparation.authorize_probe(recipe) - inputs=preparation.native_inputs(recipe) - if type(inputs) is not dict or set(inputs)!={'runtime','worker','sources','model_aware','definition_identity'}: - _fail('Native preparation supplied unclassified phase inputs.') - if inputs['worker'] not in WORKERS or type(inputs['model_aware']) is not bool: - _fail('Native preparation supplied an unsupported adapter.') - definition=inputs['definition_identity'] - if (type(definition) is not dict or set(definition)!={'definition_files','adapter_definition_sha256'} - or type(inputs['sources']) is not dict or set(inputs['sources'])!=set(definition['definition_files']) - or digest(definition['definition_files'])!=definition['adapter_definition_sha256'] - or any(type(raw) is not bytes or hashlib.sha256(raw).hexdigest()!=definition['definition_files'][name] - for name,raw in inputs['sources'].items())): - _fail('Native measured sources differ from their exact definition.','stale') - return actor,inputs - - def probe(self, recipe, *, expected_checkpoint, request_key, preparation): - """Explicitly measure a native runtime behind canonical launch/GO ACKs.""" - if self.db.in_transaction: - _fail('Native probing must precede the foreground admission transaction.','busy') - if type(request_key) is not str or REQUEST_KEY.fullmatch(request_key) is None: - _fail('Invalid native probe request key.') - with self.db.lock: - current=self.session.check() - if current['checkpoint']!=expected_checkpoint:_fail('Native preparation checkpoint changed.','stale_project') - actor,inputs=self._inputs(recipe,preparation) - prior=[value for value in decode_records(current['records']).values() - if value['origin_catalog_id']==self.session.project_id and value['request_key']==request_key] - if prior: - _fail('A previous native probe is historical; start a fresh explicit preparation.','unavailable') - result=self._run_phase('probe',recipe,request_key,actor,inputs,preparation) - return dict(runtime=self._probe_result(recipe,inputs,result['evidence']),phase=phase_reference(result['document'])) - - @staticmethod - def _probe_result(recipe, inputs, evidence): - receipt=evidence['receipt'] - if not receipt['go_received'] or receipt['outcome'] not in ('completed','failed'): - _fail('The native runtime probe did not produce an acknowledged observation.','unavailable') - if classify_probe(recipe,evidence['stdout'])['outcome']!='valid': - _fail('The native runtime probe returned an invalid bounded observation.','unavailable') - try:probe=decode_probe_observation(recipe,recipe_process.decode(evidence['stdout'])) - except (WorkbenchError,recipe_process.ProcessError): - _fail('The native runtime probe did not return its bounded observation.','unavailable') - if receipt['leader_returncode'] and probe['available']: - _fail('Failed native runtime probe claimed availability.','invalid') - return dict(probe,**inputs['definition_identity']) - - def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, run_id=None, job=None, phase_id=None): - if self.db.in_transaction:_fail('Native phase cannot launch inside a metadata transaction.','busy') - with self.db.lock: - self.session.check_current() - if run_id is None and self._foreground_phase is not None: - _fail('Another explicit native preparation phase is active.','busy') - if not self.db.claim_runtime('project_native_phases'):_fail('Another current native phase owner is active.','busy') - phase_id=phase_id or uuid.uuid4().hex - if run_id is None:self._foreground_phase=phase_id - self._active.add(phase_id);self._contexts[phase_id]=authority - document=None;handle=None - self._never_launched.add(phase_id) - try: - parent=self.db.path/'project-native-phases';parent.mkdir(mode=0o700,exist_ok=True) - request_sha=hashlib.sha256(agent_api.encoded(dict(recipe=recipe,phase=phase,run_id=run_id,job=job,runtime=inputs['runtime'], - definition_identity=inputs['definition_identity'],request_key=request_key))).hexdigest() - prepared=prepare(parent/phase_id,phase=phase,attempt_id=phase_id,request_sha256=request_sha, - runtime=inputs['runtime'],worker=inputs['worker'],sources=inputs['sources'],model_aware=inputs['model_aware'],job=job) - self._prepared[phase_id]=prepared - document=dict(format=FORMAT,origin_catalog_id=self.session.project_id,phase_id=phase_id,phase=phase, - run_id=run_id,recipe=recipe,request_key=request_key,request_sha256=request_sha,actor_id=actor.actor_id, - definition_identity=copy.deepcopy(inputs['definition_identity']),spec=prepared.spec,events=[],payloads=[], - observations=dict.fromkeys(OBSERVATIONS),probe_result=None) - self._retain(document,'spec.json',recipe_process.encoded(prepared.spec),'execution_spec') - for item in prepared.spec['captured_files']: - name=str(Path(item['path']).relative_to(prepared.stage)) - raw=recipe_process.read_regular(item['path'],64*1024*1024) - if len(raw)!=item['bytes'] or hashlib.sha256(raw).hexdigest()!=item['sha256']: - _fail('Prepared native source or job changed before acknowledgement.','stale') - self._retain(document,name,raw,'execution_spec' if name=='native-job-template.json' else 'execution_source') - self._publish(document,'prepared') - if self.execution.custody is None:_fail('Native launch requires its original physical custody journal.','unavailable') - with self.db.lock,self.execution._journal_guard(): - self._dispatch_authorized(phase_id);self.session.check_current() - if not self.execution.custody.has_pending:self.execution.custody.acknowledge(self.session.snapshot) - self.execution.custody.native_prepared(document) - self.execution.custody.native_launch_intent(phase_id) - self._dispatch_authorized(phase_id) - # Once the foundation launch call is entered, failure may mean a - # spawned helper whose identity ACK was lost. Never infer no launch - # from an exception, absent Handle, or a saved PID scan. - handle=prepared.launch(prepared.stage/'custody', - _before_spawn=lambda:self._never_launched.discard(phase_id)) - self._handles[phase_id]=handle - with self.execution._journal_guard(): - helper=recipe_process.identity(handle.process.pid) - self.execution.custody.native_launched(phase_id,helper) - until=time.monotonic()+recipe_process.preparation_seconds(prepared.spec)+2 - terminal=False - while not terminal: - for event in handle.poll(): - if event['type']=='ready': - self._publish(document,'ready',helper=handle.ready) - try:self._dispatch_authorized(phase_id) - except WorkbenchError: - handle.cancel('cancelled') - until=time.monotonic()+recipe_process.CLEANUP_SECONDS+PARENT_DRAIN_ALLOWANCE - continue - self._publish(document,'go_intent',helper=handle.ready) - # Recheck actual authority after both durable ACKs. A - # saved GO intent is never permission to retry this send. - with self.db.lock: - self.session.check_current();self._dispatch_authorized(phase_id) - if not self.execution.custody.has_pending:_fail('Native local custody disappeared.','unknown') - prepared.check_current_runtime() - handle.go(handle.ready) - until=(time.monotonic()+prepared.spec['wall_seconds'] - +recipe_process.verification_seconds(prepared.spec)+recipe_process.CLEANUP_SECONDS+5) - elif event['type']=='terminal':terminal=True - elif event['type']=='lost':_fail('Native custodian control closed without its measured receipt.','unknown') - if run_id is not None: - with self.db.lock: - current=self.execution.runs._load(run_id) - if current['state']=='cancelling' or self.execution.runs._stop.is_set():handle.cancel('cancelled') - if time.monotonic()>until:_fail('Native phase exceeded its bounded custody deadline.','unknown') - if not terminal:time.sleep(.02) - evidence=prepared.evidence(handle) - self._retain_outputs(document) - for name,raw in (('stdout.log',evidence['stdout']),('stderr.log',evidence['stderr']), - ('receipt.json',recipe_process.encoded(evidence['receipt']))): - self._retain(document,name,raw,'custody_proof' if name=='receipt.json' else 'measured_stream') - for name,key in (('effective-job.json','effective_job'),('native-parent-observation.json','parent_observation')): - if evidence[key] is not None:self._retain(document,name,evidence[key],'measured_job') - receipt_sha=hashlib.sha256(recipe_process.encoded(evidence['receipt'])).hexdigest() - state='completed' if evidence['receipt']['outcome']=='completed' else 'failed' - self._publish(document,state,helper=handle.ready,receipt_sha256=receipt_sha) - if run_id is None: - with self.execution._journal_guard():self.execution.custody.native_confirmed(phase_id) - return dict(document=document,evidence=evidence,prepared=prepared) - except BaseException as error: - # An unacknowledged phase cannot send GO or publish a successful - # terminal fact. Close only this process's actual control channel. - if handle is not None: - try:handle.close_control() - except (OSError,recipe_process.ProcessError):pass - if phase_id not in self._never_launched: - retained=session_record(self.session, FAMILY,self.session.project_id+'.'+phase_id) - retained=None if retained is None else retained['document'] - acknowledged=(self.session.available and retained is not None - and retained['events'][-1]['state'] in ('completed','failed') - and retained['events'][-1]['receipt_sha256'] is not None) - if not acknowledged: - already_fenced=not self.session.available - self.session.available=False - if self.execution._runs is not None:self.execution._runs._fence_coordinator() - if not already_fenced: - raise WorkbenchError('Native spawn outcome is uncertain; reconcile exact original custody before further dispatch.', - 'outcome_unknown',409) from error - raise - finally: - self._active.discard(phase_id) - if self._foreground_phase==phase_id:self._foreground_phase=None - if handle is not None and handle.process.poll() is not None and handle.terminal: - self._handles.pop(phase_id,None) diff --git a/project_native_preparation.py b/project_native_preparation.py deleted file mode 100644 index ee4195a8..00000000 --- a/project_native_preparation.py +++ /dev/null @@ -1,430 +0,0 @@ -"""Explicit current native qualification, before the short admission transaction. - -The configured interpreter and installed packages are operator-managed inputs. -Measured interpreter/source bytes and observed package versions are evidence; -they do not promise a complete package closure or scientific reproducibility. -No serialized validation result can become this process-local preparation. -""" -import copy -from contextlib import contextmanager -from contextvars import ContextVar -import hashlib -import os -from pathlib import Path -import shutil -import sys - -from model_definitions import checked_json, natural, reference -from pipeline import (adapter_for_graph, decode_graph, digest, execution_digest, - identifier, model_aware_graph, closed, decode_probe_observation) -from project_session import operation_records -from workbench import WorkbenchError, timestamp -from project_session import session_operations - - -OPERATIONS=frozenset(('native_preparation.intent','native_preparation.validate')) -_CURRENT=ContextVar('project_native_preparation',default=None) - - -def _fail(message,code='unavailable'): - raise WorkbenchError(message,code,409) - - -def _sha(value): - if type(value) is not str or len(value)!=64 or any(c not in '0123456789abcdef' for c in value): - _fail('Native preparation requires an exact SHA256.','invalid') - - -def _key(operation,key): - from experiments import request_key - request_key(key) - return 'native-'+digest(dict(operation=operation,key=key)) - - -def _validation_observation(result,records): - closed(result,('status','valid','errors','validation_token','graph_revision','execution_sha256','release','runtime'),('selection_policy',)) - if type(result['valid']) is not bool or type(result['errors']) is not list or len(result['errors'])>100: - _fail('Invalid native validation result.','invalid') - natural(result['graph_revision'],'Graph revision',1);_sha(result['execution_sha256']) - if result['validation_token'] is not None:_sha(result['validation_token']) - for error in result['errors']: - closed(error,('path','message','code')) - if any(type(error[k]) is not str or len(error[k])>2000 for k in error):_fail('Invalid native validation error.','invalid') - if (result['valid'] and (result['errors'] or result['status']!='valid' or result['validation_token'] is None) - or not result['valid'] and (not result['errors'] or result['status']!=result['errors'][0]['code'] or result['validation_token'] is not None)): - _fail('Contradictory native validation result.','invalid') - release=result['release'] - if release is not None: - import release_validation - integrity=release.get('recipe') in release_validation.RECIPES - fields=('id','sha256','format','recipe','allowed_recipes','records','split_counts','prior_test_exposure','status') - if integrity: - fields+=('result_kind','validation_summary') - if release['recipe'] in release_validation.CLASSIFICATION_RECIPES:fields+=('class_vocabulary',) - closed(release,fields) - if release['result_kind']!='release_validation':_fail('Invalid integrity release result kind.','invalid') - else:closed(release,(*fields,'class_vocabulary'),('audio_summary',)) - _sha(release['id']);_sha(release['sha256']) - if release['id']!=release['sha256'] or release['status']!='available':_fail('Invalid frozen input observation.','invalid') - for name in ('format','recipe'): - if type(release[name]) is not str or not 1<=len(release[name])<=120:_fail('Invalid release task.','invalid') - for name in ('allowed_recipes',): - if type(release[name]) is not list or len(release[name])>10000 or any(type(v) is not str or len(v)>1000 for v in release[name]): - _fail('Invalid release vocabulary.','invalid') - if 'class_vocabulary' in release: - manifests=[] if records is None else [record['document']['manifest'] for record in records - if record['family']=='dataset_release' and record['id']==release['id']] - release_validation.decode_class_vocabulary(release['class_vocabulary'],release['recipe'], - manifest=manifests[0] if len(manifests)==1 else None) - natural(release['records'],'Frozen record count');closed(release['split_counts'],('train','validation','test')) - for count in release['split_counts'].values():natural(count,'Split count') - exposure=release['prior_test_exposure'];closed(exposure,('exposed','events')) - if type(exposure['exposed']) is not bool or type(exposure['events']) is not list or len(exposure['events'])>100000: - _fail('Invalid prior disclosure observations.','invalid') - for event in exposure['events']: - closed(event,('id','record_id','source_hash','reason','run_id','release_id','created_at')) - for field in ('id','run_id'):identifier(event[field]) - for field in ('source_hash','release_id'):_sha(event[field]) - for field in ('record_id','reason','created_at'): - if type(event[field]) is not str or len(event[field])>1000:_fail('Invalid disclosure observation text.','invalid') - if 'audio_summary' in release: - audio=release['audio_summary'];closed(audio,('sample_rates_hz','analyzed_frames','coordinate_contract')) - if type(audio['sample_rates_hz']) is not list or len(audio['sample_rates_hz'])>1000:_fail('Invalid audio rate inventory.','invalid') - for rate in audio['sample_rates_hz']:natural(rate,'Sample rate',1) - natural(audio['analyzed_frames'],'Analyzed frames') - if type(audio['coordinate_contract']) is not str or len(audio['coordinate_contract'])>1000:_fail('Invalid audio coordinate contract.','invalid') - if integrity: - matches=[] if records is None else [record['document']['manifest'] for record in records - if record['family']=='dataset_release' and record['id']==release['id']] - if len(matches)!=1:_fail('Integrity validation needs its exact retained frozen manifest.') - manifest=matches[0] - summary=release_validation.decode_summary(release['validation_summary'],release['recipe'], - manifest=manifest,release_id=release['id']) - if (release['records']!=len(manifest['records']) or release['split_counts']!=summary['split_counts'] - or release['format']!=manifest.get('format','canonical_v1')): - _fail('Integrity release header differs from its frozen summary.','invalid') - if release['recipe'] in release_validation.CLASSIFICATION_RECIPES: - if release['class_vocabulary']!=summary['class_vocabulary']:_fail('Integrity class vocabulary differs.','invalid') - runtime=result['runtime'] - recipes=[] - if runtime is not None: - import release_validation - from pipeline import MODEL_RECIPES - for recipe in ('image_classification','text_classification','audio_classification',*MODEL_RECIPES,*release_validation.RECIPES): - try:decode_probe_observation(recipe,runtime,with_definition=True) - except (WorkbenchError,TypeError,ValueError):continue - recipes.append(recipe) - if len(recipes)!=1:_fail('Unclassified native validation runtime.','invalid') - if 'selection_policy' in result: - from experiments import validate_retained_policy - validate_retained_policy(result['selection_policy']) - if result['valid']: - if release is None or runtime is None or not runtime['available'] or release['recipe']!=recipes[0]: - _fail('Native validation lacks its exact runtime and frozen input.','invalid') - if records is not None: - from project_material_fork import is_intent - matches=[] - for record in records: - if record['family']!='pipeline_graph':continue - for graph in record['document']['revisions']: - if is_intent(graph):continue - if graph['revision']!=result['graph_revision'] or execution_digest(graph)!=result['execution_sha256']:continue - token=digest(dict(graph_id=graph['id'],revision=graph['revision'],execution_sha256=execution_digest(graph), - release_sha256=release['sha256'],runtime=runtime)) - release_id=next((node['parameters']['release_id'] for node in graph['nodes'] if node['type']=='release'),None) - if graph['recipe']==recipes[0] and token==result['validation_token'] and release_id==release['id']:matches.append(graph) - if not matches:_fail('Native validation has no exact retained graph/runtime binding.','invalid') - - -def preparation_owner(owner): - """Select only the fixed current project owner; legacy paths stay legacy.""" - from project_material import _MaterialAccess - from project_execution import ProjectExecution - adapter=getattr(owner.dataset,'project_native_preparation',None) - project=isinstance(owner.access,_MaterialAccess) or isinstance(getattr(owner.dataset,'project_execution',None),ProjectExecution) - if not project and adapter is None:return None - if not isinstance(adapter,ProjectNativePreparation) or adapter.db is not owner.db: - _fail('Current native preparation for the runtime probe is unavailable.') - return adapter - - -def validate_operation_result(operation,result,records=None): - if operation not in OPERATIONS:return False - try: - if operation=='native_preparation.intent': - closed(result,('format','origin_catalog_id','actor_id','purpose','graph','model','composition', - 'release_id','definition_identity','runtime','occurred_at')) - if result['format']!='project_native_preparation_intent_v1' or result['purpose'] not in ('material','validation','admission'): - _fail('Unsupported native preparation intent.','unsupported') - for field in ('origin_catalog_id','actor_id'):identifier(result[field]) - graph=result['graph'];closed(graph,('id','revision','document_sha256','execution_sha256')) - identifier(graph['id']);natural(graph['revision'],'Graph revision',1) - for field in ('document_sha256','execution_sha256'):_sha(graph[field]) - _sha(result['release_id']) - if result['model'] is not None:reference(result['model'],'model',result['origin_catalog_id']) - if result['composition'] is not None:reference(result['composition'],'recipe_material',result['origin_catalog_id']) - identity=result['definition_identity'];closed(identity,('definition_files','adapter_definition_sha256')) - if type(identity['definition_files']) is not dict or not 1<=len(identity['definition_files'])<=120: - _fail('Invalid native definition inventory.','invalid') - from project_native_process import WORKERS, validate_current_runtime - for name,sha in identity['definition_files'].items(): - if type(name) is not str or Path(name).name!=name or not name.endswith('.py'): - _fail('Invalid native definition filename.','invalid') - _sha(sha) - if not set(identity['definition_files'])&WORKERS or digest(identity['definition_files'])!=identity['adapter_definition_sha256']: - _fail('Native definition identity differs.','invalid') - validate_current_runtime(result['runtime']) - from datetime import datetime - if datetime.fromisoformat(result['occurred_at']).utcoffset() is None:_fail('Native intent time requires an offset.','invalid') - if records is not None: - matches=[r['document'] for r in records if r['family']=='pipeline_graph' - and r['document']['origin_catalog_id']==result['origin_catalog_id'] - and r['document']['current']['id']==graph['id']] - if len(matches)!=1 or not any(v['revision']==graph['revision'] and digest(v)==graph['document_sha256'] - and execution_digest(v)==graph['execution_sha256'] for v in matches[0]['revisions']): - _fail('Native intent has no exact retained graph.','invalid') - actual=next(v for v in matches[0]['revisions'] if v['revision']==graph['revision']) - if (next((node['parameters']['release_id'] for node in actual['nodes'] if node['type']=='release'),None)!=result['release_id'] - or Path(adapter_for_graph(actual).__file__).name not in identity['definition_files']): - _fail('Native intent source or frozen input differs from its graph.','invalid') - from types import SimpleNamespace - from project_material import ProjectMaterial - retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None)) - if result['model'] is not None:retained.model(result['model']) - if result['composition'] is not None: - composition=retained.recipe(result['composition'])['material'] - if (composition['model']!=result['model'] or composition['recipe_id']!=graph['id'] - or composition['recipe_revision']!=graph['revision']): - _fail('Native admission intent differs from its retained composition.','invalid') - else: - _validation_observation(result,records) - checked_json(result,1024**2) - return True - except WorkbenchError:raise - except (ValueError,TypeError,KeyError,AttributeError,ImportError): - _fail('Invalid native preparation observation.','invalid') - - -class NativePreparationContext: - """A lexical current qualification, never a portable admission capability.""" - def __init__(self,adapter,owner,graph,model,composition,purpose,key): - self.adapter,self.owner=adapter,owner - self.graph=copy.deepcopy(decode_graph(graph));self.model=copy.deepcopy(model) - self.recipe=self.graph['recipe'] - self.composition=copy.deepcopy(composition);self.purpose=purpose;self.key=key - self.actor=adapter.access.require('execute','pipeline.probe_native',self.graph['recipe'],owner_only=True) - self.configured_executable=os.environ.get('TULDOK_TRAINING_PYTHON') or sys.executable - self.definition=owner.definition_identity(self.graph['recipe']) - self.sources={name:(owner.source_path/name).read_bytes() for name in self.definition['definition_files']} - self.release_id=next(node['parameters']['release_id'] for node in self.graph['nodes'] if node['type']=='release') - self.release=owner.inspect_release(self.release_id,private=True,recipe=self.graph['recipe']) - self.model_value=adapter.materials.model(model,verify=True) if model is not None else None - if model is not None: - current_model,_definition=owner._builtin_model_for_graph(self.graph,model) - if current_model!=self.model_value or {item['import_path']:item['blob']['sha256'] - for item in current_model['material']['source_closure']}!=self.definition['definition_files']: - _fail('Published native model differs from its graph or current source.','stale') - from project_native_process import current_runtime - self.runtime=current_runtime(self.configured_executable) - self._active=False;self._probe=None;self._prepared=None;self._consumed=False;self.phase=None - - def _check(self,recipe): - if not self._active or _CURRENT.get() is not self or recipe!=self.graph['recipe']: - _fail('An explicit current native preparation is required.') - self.adapter.session.check_current() - actor=self.adapter.access.require('execute','pipeline.probe_native',recipe,owner_only=True) - if actor.actor_id!=self.actor.actor_id:_fail('Native preparation caller changed.','conflict') - if (os.environ.get('TULDOK_TRAINING_PYTHON') or sys.executable)!=self.configured_executable: - _fail('Configured native interpreter changed.','stale') - if self.owner.get(self.graph['id'])!=self.graph or self.owner.definition_identity(recipe)!=self.definition: - _fail('Native graph or source changed during qualification.','stale') - if any(hashlib.sha256((self.owner.source_path/name).read_bytes()).hexdigest()!=sha - for name,sha in self.definition['definition_files'].items()): - _fail('Native source bytes changed.','stale') - current=self.owner.inspect_release(self.release_id,private=True,recipe=recipe) - if current!=self.release:_fail('Native frozen input observation changed.','stale') - if self.model is not None and self.adapter.materials.model(self.model,verify=True)!=self.model_value: - _fail('Native model observation changed.','stale') - from project_native_process import current_runtime - if current_runtime(self.configured_executable)!=self.runtime:_fail('Configured native interpreter identity changed.','stale') - return actor - - def authorize_probe(self,recipe): - return self._check(recipe) - - def native_inputs(self,recipe): - self.authorize_probe(recipe) - return dict(runtime=copy.deepcopy(self.runtime),worker=Path(adapter_for_graph(self.graph).__file__).name, - sources=copy.deepcopy(self.sources),model_aware=model_aware_graph(self.graph),definition_identity=copy.deepcopy(self.definition)) - - def probe(self,owner,recipe): - if owner.db is not self.adapter.db:_fail('Native preparation belongs to another projection.') - self._check(recipe) - if self._probe is None: - if self.adapter.db.in_transaction:_fail('Native probe cannot run inside a metadata transaction.','unsupported') - self.authorize_probe(recipe) - result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.summary['checkpoint'], - request_key=self.key+'.probe',preparation=self) - self._probe=copy.deepcopy(result['runtime']);self.phase=copy.deepcopy(result['phase']) - return copy.deepcopy(self._probe) - - def capture_builtin(self,owner,value): - self._check(self.graph['recipe']) - if self.composition is None or value!=self.composition:_fail('Prepared native material differs.','stale') - if self.adapter.db.in_transaction: - if self._prepared is None or self._consumed:_fail('Native qualification is missing or already consumed.') - self._consumed=True - return copy.deepcopy(self._prepared) - if self._prepared is None:self._prepared=owner._prepare_builtin_recipe_material_current(value) - return copy.deepcopy(self._prepared) - - def consume_admission(self): - self._check(self.recipe) - if self._consumed:_fail('Native qualification has already been consumed.') - self._consumed=True - - -class PurePreparationContext: - """Current integrity qualification; it cannot authorize a child phase.""" - def __init__(self,adapter,owner,graph): - self.adapter,self.owner=adapter,owner;self.graph=copy.deepcopy(graph) - self.recipe=graph['recipe'];self.definition=owner.definition_identity(self.recipe) - self.actor=adapter.access.require('execute','pipeline.snapshot',graph['id'],owner_only=True) - self.validation=owner.validate_graph(graph) - if not self.validation['valid']: - first=self.validation['errors'][0];_fail(first['message'],first['code']) - self.release=owner.inspect_release(self.validation['release']['id'],private=True,recipe=self.recipe) - self._consumed=False;self._active=True - - def consume_admission(self): - if not self._active or self._consumed:_fail('Current integrity qualification has already been consumed.') - self.adapter.session.check_current() - actor=self.adapter.access.require('execute','pipeline.snapshot',self.graph['id'],owner_only=True) - if actor.actor_id!=self.actor.actor_id:_fail('Current integrity caller changed.','conflict') - if self.owner.get(self.graph['id'])!=self.graph or self.owner.definition_identity(self.recipe)!=self.definition: - _fail('Current integrity graph or source changed.','stale') - if self.owner.inspect_release(self.release['id'],private=True,recipe=self.recipe)!=self.release: - _fail('Current integrity frozen input changed.','stale') - self._consumed=True - - -class ProjectNativePreparation: - def __init__(self,projection): - self.projection=projection;self.session=projection.session;self.db=projection.db - self.access=projection.execution.runtime_access;self.execution=projection.execution;self.materials=projection.materials - - def current(self,owner): - context=_CURRENT.get() - if not isinstance(context,NativePreparationContext) or context.adapter is not self or owner.db is not self.db: - _fail('Native qualification requires an explicit current preparation.') - return context - - def probe(self,owner,recipe): - actor=self.access._context() - if actor.mode=='job': - if actor.kind!='runs' or actor.catalog_id!=self.session.project_id: - _fail('Native probe requires its exact current Run admission.') - self.access.require('execute','pipeline.probe_native',recipe) - result=self.execution.native.probe_current_run(owner,recipe) - return copy.deepcopy(result['runtime']) - return self.current(owner).probe(owner,recipe) - - @contextmanager - def _scope(self,owner,graph,model,composition,purpose,expected_checkpoint,key): - if self.db.in_transaction:_fail('Native preparation must precede the metadata transaction.','unsupported') - if _CURRENT.get() is not None:_fail('A native preparation is already active.','busy') - self.access.require('edit','native_preparation.'+purpose,graph['id'],owner_only=True) - self.session.check_current() - if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') - context=NativePreparationContext(self,owner,graph,model,composition,purpose,_key(purpose,key)) - result=dict(format='project_native_preparation_intent_v1',origin_catalog_id=self.session.project_id, - actor_id=context.actor.actor_id,purpose=purpose,graph=dict(id=graph['id'],revision=graph['revision'], - document_sha256=digest(graph),execution_sha256=execution_digest(graph)),model=model, - composition=None if composition is None else composition['ref'],release_id=context.release_id, - definition_identity=context.definition,runtime=context.runtime,occurred_at=timestamp()) - body={name:copy.deepcopy(value) for name,value in result.items() if name not in ('format','occurred_at')} - self.session.mutate('native_preparation.intent',graph['id'],body,expected_checkpoint,context.key, - lambda:copy.deepcopy(result),lambda value,before:dict(records=[],result=value),scope='execute') - context._active=True;token=_CURRENT.set(context) - try:yield context - finally:_CURRENT.reset(token);context._active=False;context._prepared=None - - def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): - self.access.require('edit','native_preparation.validation',graph_id,owner_only=True) - self.access.require('execute','pipeline.probe_native',graph_id,owner_only=True) - body=dict(graph_id=graph_id,revision=revision) - prior=session_operations(self.session).get(key) - if prior is not None: - return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, - lambda:(_ for _ in ()).throw(RuntimeError('Native validation replay launched a probe.')), - lambda value,before:dict(records=[],result=value),scope='execute') - graph=owner.get(graph_id) - if graph['revision']!=revision:_fail('Graph changed before native validation.','stale') - import release_validation - if adapter_for_graph(graph) is release_validation: - self.session.check_current() - if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') - value=owner.validate_graph(graph) - return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, - lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') - with self._scope(owner,graph,None,None,'validation',expected_checkpoint,key) as context: - value=owner.validate_graph(graph);context._check(graph['recipe']) - return self.session.mutate('native_preparation.validate',graph_id,body,self.session.summary['checkpoint'],key, - lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') - - @contextmanager - def current_graph(self,owner,graph_id,revision,expected_checkpoint,key): - if self.db.in_transaction:_fail('Current native validation must precede admission.','unsupported') - self.access.require('edit','native_preparation.admission',graph_id,owner_only=True) - self.session.check_current() - if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') - graph=owner.get(graph_id) - if graph['revision']!=revision:_fail('Graph changed before native admission.','stale') - import release_validation - if adapter_for_graph(graph) is release_validation: - context=PurePreparationContext(self,owner,graph) - try:yield context - finally:context._active=False - return - with self._scope(owner,graph,None,None,'admission',expected_checkpoint,key) as context: - context.validation=owner.validate_graph(graph) - if not context.validation['valid']: - first=context.validation['errors'][0];_fail(first['message'],first['code']) - context._check(graph['recipe']) - yield context - - def prepare_recipe(self,owner,graph_id,revision,model_ref,expected_checkpoint,key): - self.materials._require_resolved('pipeline_graph',graph_id) - self.access.require('edit','recipes.prepare_builtin_material',graph_id,owner_only=True) - self.access.require('execute','pipeline.probe_native',graph_id,owner_only=True) - body=dict(graph_id=graph_id,revision=revision,model=model_ref,reconcile_capture=False) - operation='recipes.prepare_builtin_material' - if key in session_operations(self.session): - return self.session.mutate(operation,graph_id,body,expected_checkpoint,key, - lambda:(_ for _ in ()).throw(RuntimeError('Material replay launched a native probe.')), - lambda value,before:dict(records=[],result=value)) - graph=owner.get(graph_id) - if graph['revision']!=revision or not model_aware_graph(graph):_fail('Exact model-aware graph is unavailable.','stale') - self.materials.materialize_projection(model_ref,owner.dataset.models.path) - with self._scope(owner,graph,model_ref,None,'material',expected_checkpoint,key) as context: - prepared=owner.prepare_builtin_material(graph_id,revision,model_ref) - try: - context._check(graph['recipe']) - def capture(value,before): - record=self.materials.capture_recipe_material(value,owner.material_path) - return dict(records=self.materials._coalesce([record,*self.materials.declarations(record), - *self.materials.capture_recipe_dependencies(record,before)]),result=value) - return self.session.mutate(operation,graph_id,body,self.session.summary['checkpoint'],key, - lambda:owner._retain_material(prepared),capture) - finally: - if prepared['staging'].exists():shutil.rmtree(prepared['staging']) - - @contextmanager - def admission(self,owner,material_ref,expected_checkpoint,key): - value=self.materials.recipe(material_ref,verify=True) - import json - snapshot=json.loads(self.materials.blob(material_ref,value['material']['recipe_document'],2*1024**2)) - graph=decode_graph(snapshot['graph']) - with self._scope(owner,graph,value['material']['model'],value,'admission',expected_checkpoint,key) as context: - context.capture_builtin(owner,value) - yield context diff --git a/project_native_process.py b/project_native_process.py deleted file mode 100644 index 41b28f38..00000000 --- a/project_native_process.py +++ /dev/null @@ -1,445 +0,0 @@ -"""Finite native probe/train/verifier preparation over the existing custodian. - -Only the current native domain owner supplies measured runtime and source bytes. -Preparation owns new disposable files, not SQL, portable publication, authority, -or GO. The caller acknowledges READY through its canonical boundary, then uses -the returned ordinary recipe_process.Handle explicitly. -""" -import copy -import hashlib -import os -from pathlib import Path -import re -import stat - -import recipe_process as custody - -WORKERS = frozenset(('image_training.py', 'text_training.py', 'audio_training.py')) -PHASES = frozenset(('probe', 'train', 'verify')) -TEMPLATE_FORMAT = 'native_parent_binding_v1' -TEMPLATE_LIMIT = 64 * 1024 -BOOTSTRAP = Path(__file__).with_name('native_process_bootstrap.py') -ENVIRONMENT = dict(PATH='/usr/bin:/bin', LANG='C.UTF-8', CUDA_VISIBLE_DEVICES='', - OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', BLIS_NUM_THREADS='1', - VECLIB_MAXIMUM_THREADS='1', NUMEXPR_NUM_THREADS='1', PYTHONDONTWRITEBYTECODE='1') - - -def _fail(message, code='invalid'): - raise custody.ProcessError(message, code) - - -def validate_current_runtime(value): - """Pure observed runtime metadata, not a complete package/library closure.""" - if (type(value) is not dict or set(value) != {'executable', 'executable_source', 'executable_sha256', - 'trusted_install_owner_uid', 'files'} - or type(value['files']) is not dict or not 1 <= len(value['files']) <= 3 - or type(value['trusted_install_owner_uid']) is not int or value['trusted_install_owner_uid'] < 0): - _fail('Invalid observed current native runtime.') - for key in ('executable', 'executable_source'): - path = value[key] - if (type(path) is not str or not 1 <= len(path) <= 4096 or '\0' in path - or not Path(path).is_absolute() or str(Path(path)) != path or '..' in Path(path).parts): - _fail('Invalid observed native interpreter path.') - if type(value['executable_sha256']) is not str or custody.HASH.fullmatch(value['executable_sha256']) is None: - _fail('Invalid observed native interpreter hash.') - allowed = {value['executable_source'], str(Path(value['executable']).parent/'pyvenv.cfg'), - str(Path(value['executable']).parent.parent/'pyvenv.cfg')} - for path, row in value['files'].items(): - if (path not in allowed or type(row) is not dict or set(row) != {'sha256', 'bytes'} - or type(row['sha256']) is not str or custody.HASH.fullmatch(row['sha256']) is None - or type(row['bytes']) is not int or not 0 <= row['bytes'] <= 64*1024*1024): - _fail('Invalid observed native runtime file.') - if value['files'].get(value['executable_source'], {}).get('sha256') != value['executable_sha256']: - _fail('Observed native interpreter source differs.') - return copy.deepcopy(value) - - -def current_runtime(executable): - """Measure only the explicit current interpreter and venv configuration. - - Installed packages and native libraries remain operator-managed dependencies; - the actual native probe observes their reported versions. No executable runs - here and no alternative interpreter is selected. - """ - if type(executable) is not str or not executable or '\0' in executable: - _fail('Choose the explicitly configured native interpreter.') - configured = Path(os.path.abspath(executable)) - try: - physical = configured.resolve(strict=True) - files = {} - for path in dict.fromkeys((physical, configured.parent/'pyvenv.cfg', configured.parent.parent/'pyvenv.cfg')): - if path != physical and not os.path.lexists(path): continue - raw = custody.read_regular(path, 64*1024*1024) - files[str(path)] = dict(sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) - if configured.resolve(strict=True) != physical or not os.path.samestat(configured.stat(),physical.stat()): - _fail('Configured native interpreter alias changed.', 'stale') - return validate_current_runtime(dict(executable=str(configured), executable_source=str(physical), - executable_sha256=files[str(physical)]['sha256'], trusted_install_owner_uid=os.stat('/usr').st_uid, - files=files)) - except OSError: - _fail('Configured current native interpreter is unavailable.', 'unavailable') - - -def validate_prepared_spec(value, *, phase, worker): - """Pure finite native invocation grammar; never a current runtime resolver.""" - value = custody.decode_retained_spec(value) - if (type(phase) is not str or phase not in PHASES or type(worker) is not str or worker not in WORKERS - or value['schema_version'] != 1 or value['env'] != ENVIRONMENT - or value.get('native_loader_policy') != 'glibc_x86_64_no_preload_v1' - or any(key in value for key in ('cpu_snapshot', 'torch_runtime', 'resource_profile'))): - _fail('Unsupported retained native phase specification.') - root = Path(value['cwd']) - if str(root) != value['cwd'] or '..' in root.parts: - _fail('Retained native working directory differs.') - source = root / 'source' - captures = {row['path'] for row in value['captured_files']} - template = str(root / 'native-job-template.json') - bootstrap = str(source / BOOTSTRAP.name) - for path in captures: - item = Path(path) - if path == template and phase != 'probe': continue - if (item.parent != source or str(item) != path - or re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]{0,119}\.py', item.name) is None - or item.name == BOOTSTRAP.name and phase == 'probe'): - _fail('Unclassified retained native capture path.') - if str(source / worker) not in captures: - _fail('Retained native worker capture is missing.') - executable = value['argv'][0] - if phase == 'probe': - expected = [executable, '-B', str(source / worker), '--probe'] - if value['argv'] not in (expected, expected + ['--model-aware']): - _fail('Retained native probe argument vector differs.') - elif (value['argv'] != [executable, '-I', '-B', bootstrap, phase, template] - or not {bootstrap, template} <= captures): - _fail('Retained native job bootstrap differs.') - if phase in ('probe', 'train') and value['output_dir'] != str(root / 'output'): - _fail('Retained native output directory differs.') - return value - - -def _job_template(spec, phase, template): - if (type(template) is not bytes or not 1 <= len(template) <= TEMPLATE_LIMIT): - _fail('Native retained job template is unavailable or exceeds its bound.', 'stale') - frame = custody.decode(template) - if (type(frame) is not dict or set(frame) != {'format','phase','worker','attempt_id','request_sha256','job'} - or frame['format'] != TEMPLATE_FORMAT or frame['phase'] != phase - or frame['attempt_id'] != spec['attempt_id'] or frame['request_sha256'] != spec['request_sha256'] - or type(frame['job']) is not dict or type(frame['job'].get('parent_pid')) is not int - or frame['job']['parent_pid'] != 1 or custody.encoded(frame) != template): - _fail('Native retained job template differs from its phase.', 'stale') - return frame - - -def validate_job_evidence(spec, *, phase, helper, template, effective_job, parent_observation, receipt): - """Pure exact native job proof for inline history or current measured bytes. - - The existing native worker still owns scientific job semantics. This seam - binds only the immutable template, qualified helper, distinct effective job - and closed custodian receipt; it confers no current execution authority. - """ - spec = custody.decode_retained_spec(spec) - if type(phase) is not str or phase not in PHASES or not custody.valid_identity(helper): - _fail('Invalid native job evidence phase or helper.') - if phase == 'probe': - if any(value is not None for value in (template,effective_job,parent_observation)): - _fail('Native probe cannot carry job execution evidence.', 'stale') - worker = Path(spec['argv'][2]).name if len(spec['argv']) > 2 else '' - frame = None - else: - frame = _job_template(spec,phase,template) - worker = frame['worker'] - spec = validate_prepared_spec(spec,phase=phase,worker=worker) - expected = dict(attempt_id=spec['attempt_id'],request_sha256=spec['request_sha256'], - spec_sha256=hashlib.sha256(custody.encoded(spec)).hexdigest(),helper=helper) - if type(receipt) is not dict:_fail('Native job evidence requires its closed custody receipt.') - receipt = custody.validate_receipt(custody.encoded(receipt),expected) - if not receipt['drained']:_fail('Native descendant drainage is unconfirmed.', 'unknown') - if frame is not None: - row = next(item for item in spec['captured_files'] if item['path'] == str(Path(spec['cwd'])/'native-job-template.json')) - if len(template) != row['bytes'] or hashlib.sha256(template).hexdigest() != row['sha256']: - _fail('Native retained job template differs from its captured identity.', 'stale') - if effective_job is not None or parent_observation is not None: - if (type(effective_job) is not bytes or type(parent_observation) is not bytes - or not 1 <= len(effective_job) <= TEMPLATE_LIMIT - or not 1 <= len(parent_observation) <= TEMPLATE_LIMIT): - _fail('Native measured job evidence is incomplete or exceeds its bound.', 'unknown') - expected_job = dict(frame['job'],parent_pid=helper['pid']) - expected_observation = dict(format='native_parent_observation_v1', - attempt_id=spec['attempt_id'],request_sha256=spec['request_sha256'], - template_sha256=hashlib.sha256(template).hexdigest(), - effective_job_sha256=hashlib.sha256(effective_job).hexdigest(),helper=helper) - if (effective_job != custody.encoded(expected_job) or parent_observation != custody.encoded(expected_observation) - or not receipt['go_received']): - _fail('Native effective job differs from its exact qualified parent.', 'stale') - elif receipt['outcome'] == 'completed': - _fail('Completed native phase lacks measured job evidence.', 'unknown') - return dict(receipt=receipt,template=template,effective_job=effective_job,parent_observation=parent_observation) - - -def _record(path, raw): - return dict(path=str(path), sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) - - -def _write(directory, name, raw): - descriptor = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, - 0o400, dir_fd=directory) - with os.fdopen(descriptor, 'wb') as target: - target.write(raw); target.flush(); os.fsync(target.fileno()) - - -def _parent(path): - """Walk the caller's explicit current path without directory symlinks.""" - if not path.is_absolute() or '..' in path.parts or path.name in ('', '.', '..'): - _fail('Native staging requires an absolute new path.') - descriptor = os.open('/', os.O_RDONLY | os.O_DIRECTORY) - try: - for component in path.parent.parts[1:]: - following = os.open(component, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor) - os.close(descriptor); descriptor = following - return descriptor - except BaseException: - os.close(descriptor); raise - - -def _directory(path): - parent = _parent(path) - try: - descriptor = os.open(path.name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) - finally:os.close(parent) - info = os.fstat(descriptor) - if info.st_uid != os.getuid() or info.st_mode & 0o077: - os.close(descriptor) - _fail('Native phase evidence requires its current private physical directory.', 'stale') - return descriptor - - -def _read_at(directory, name, maximum, *, immutable=False): - descriptor = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory) - with os.fdopen(descriptor, 'rb') as source: - before = os.fstat(source.fileno()) - if (not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 or before.st_uid != os.getuid() - or before.st_size > maximum or immutable and before.st_mode & 0o222): - _fail('Native phase evidence file is unsafe or exceeds its bound.', 'stale') - raw = source.read(maximum + 1) - after = os.fstat(source.fileno()) - identity = lambda row: (row.st_dev,row.st_ino,row.st_mode,row.st_nlink,row.st_uid, - row.st_size,row.st_mtime_ns,row.st_ctime_ns) - if (len(raw) > maximum or identity(before) != identity(after) - or identity(after) != identity(os.stat(name, dir_fd=directory, follow_symlinks=False))): - _fail('Native phase evidence changed during observation.', 'stale') - return raw - - -def read_phase_evidence(stage, spec, *, phase, helper): - """Read one caller-qualified original physical phase, never restore custody. - - The caller must first prove its current original-host/root/projection journal - binding. This function uses only that explicit stage and its custody child; - retained absolute paths cannot select another filesystem root. No runtime - lookup, process observation, signal, launch or GO occurs here. - """ - return _read_phase_evidence(stage, spec, phase=phase, helper=helper, - custody_directory=Path(stage) / 'custody') - - -def _read_phase_evidence(stage, spec, *, phase, helper, custody_directory): - stage = Path(stage) - spec = custody.decode_retained_spec(spec) - if spec['cwd'] != str(stage) or not custody.valid_identity(helper): - _fail('Native phase evidence differs from its qualified physical stage or helper.', 'stale') - if type(phase) is not str or phase not in PHASES: - _fail('Unsupported native phase evidence.') - root = source = owner = None - try: - root = _directory(stage) - template = None - if phase == 'probe': - worker = Path(spec['argv'][2]).name if len(spec['argv']) > 2 else '' - else: - template = _read_at(root, 'native-job-template.json', TEMPLATE_LIMIT, immutable=True) - frame = _job_template(spec,phase,template) - worker = frame['worker'] - spec = validate_prepared_spec(spec, phase=phase, worker=worker) - source = os.open('source', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=root) - for item in spec['captured_files']: - name = Path(item['path']).name - raw = template if item['path'] == str(stage/'native-job-template.json') else _read_at( - source, name, item['bytes'], immutable=True) - if raw is None or len(raw) != item['bytes'] or hashlib.sha256(raw).hexdigest() != item['sha256']: - _fail('Native retained capture differs from its exact specification.', 'stale') - expected = dict(attempt_id=spec['attempt_id'],request_sha256=spec['request_sha256'], - spec_sha256=hashlib.sha256(custody.encoded(spec)).hexdigest(),helper=helper) - try: - owner = _directory(Path(custody_directory)) - receipt_raw = _read_at(owner,'receipt.json',16384) - receipt = custody.validate_receipt(receipt_raw,expected) - if custody.encoded(receipt) != receipt_raw: - _fail('Native custody receipt lost its exact canonical bytes.', 'stale') - except (OSError,custody.ProcessError): - _fail('Native phase has no exact drainage receipt.', 'unknown') - if not receipt['drained']: _fail('Native descendant drainage is unconfirmed.', 'unknown') - streams = {} - for name in ('stdout','stderr'): - row = receipt['streams'][name] - raw = _read_at(owner,name+'.log',custody.STREAM_LIMIT) - if len(raw) != row['bytes'] or hashlib.sha256(raw).hexdigest() != row['sha256']: - _fail('Native measured stream differs from its custody receipt.', 'stale') - streams[name] = raw - effective = observation = None - if template is not None: - present = [] - for name in ('effective-job.json','native-parent-observation.json'): - try:os.stat(name,dir_fd=root,follow_symlinks=False);present.append(True) - except FileNotFoundError:present.append(False) - if any(present): - if not all(present): _fail('Native measured job evidence is incomplete.', 'unknown') - effective = _read_at(root,'effective-job.json',TEMPLATE_LIMIT,immutable=True) - observation = _read_at(root,'native-parent-observation.json',TEMPLATE_LIMIT,immutable=True) - proof = validate_job_evidence(spec,phase=phase,helper=helper,template=template, - effective_job=effective,parent_observation=observation,receipt=receipt) - if not os.path.samestat(os.fstat(root),os.stat(stage,follow_symlinks=False)): - _fail('Native physical phase changed during observation.', 'stale') - if (not os.path.samestat(os.fstat(source),os.stat('source',dir_fd=root,follow_symlinks=False)) - or not os.path.samestat(os.fstat(owner),os.stat(custody_directory,follow_symlinks=False))): - _fail('Native phase evidence directory changed during observation.', 'stale') - return dict(proof,stdout=streams['stdout'],stderr=streams['stderr']) - finally: - for descriptor in (owner,source,root): - if descriptor is not None:os.close(descriptor) - - -class PreparedNative: - """A fixed phase specification and its new files; never an execution grant.""" - def __init__(self, stage, phase, spec, template, runtime=None): - self.stage, self.phase = stage, phase - self._spec, self.template = copy.deepcopy(spec), template - self._runtime = copy.deepcopy(runtime) - - @property - def spec(self): - return copy.deepcopy(self._spec) - - def launch(self, custody_directory, *, _before_spawn=None): - """Return a READY/GO Handle. This method never sends GO.""" - if self._runtime is None: - return custody.launch(self.spec, custody_directory, _before_spawn=_before_spawn) - self.check_current_runtime() - descriptor = os.open(self._runtime['executable_source'], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) - try: - if not os.path.samestat(os.fstat(descriptor), os.stat(self._runtime['executable'])): - _fail('Configured interpreter alias changed before launch.', 'stale') - return custody.launch(self.spec, custody_directory, executable_fd=descriptor, - _before_spawn=_before_spawn) - finally:os.close(descriptor) - - def check_current_runtime(self): - """Explicit owner check before GO; never dispatches or grants permission.""" - if self._runtime is not None and current_runtime(self._runtime['executable']) != self._runtime: - _fail('Current native interpreter or venv configuration changed.', 'stale') - return copy.deepcopy(self._runtime) - - def evidence(self, handle): - """Read the same exact phase proof used by original-host recovery.""" - if handle.spec != self._spec or handle.ready is None: - _fail('Native phase handle differs from its prepared specification.', 'unknown') - result = _read_phase_evidence(self.stage, self._spec, phase=self.phase, - helper=handle.ready, custody_directory=handle.directory) - if result['template'] != self.template: - _fail('Native immutable job template differs from its prepared value.', 'stale') - return result - - -def prepare(stage, *, phase, attempt_id, request_sha256, runtime, worker, sources, - job=None, model_aware=False, wall_seconds=None, max_output_bytes=64 * 1024 * 1024): - """Create one new phase, using only caller-supplied current measurements. - - runtime is current_runtime's explicit interpreter/venv observation, or the - four-field regular-file input executable/executable_sha256/ - trusted_install_owner_uid/files. These are observed files, not complete - installed package/library closure. sources maps bounded Python - basenames to exact captured bytes. The existing worker owns job semantics; - this boundary requires parent_pid=1 and retains the whole template exactly. - A verifier may inspect the preceding training phase's private output folder. - """ - if (type(phase) is not str or phase not in PHASES or type(worker) is not str - or worker not in WORKERS or type(model_aware) is not bool - or type(attempt_id) is not str or custody.ID.fullmatch(attempt_id) is None - or type(request_sha256) is not str or custody.HASH.fullmatch(request_sha256) is None): - _fail('Invalid fixed native phase identity.') - if (type(runtime) is not dict or set(runtime) not in ( - {'executable', 'executable_sha256', 'trusted_install_owner_uid', 'files'}, - {'executable', 'executable_source', 'executable_sha256', 'trusted_install_owner_uid', 'files'}) - or type(runtime['files']) is not dict or not runtime['files']): - _fail('Supply an explicitly measured current native runtime.') - measured = validate_current_runtime(runtime) if 'executable_source' in runtime else None - if (type(sources) is not dict or worker not in sources or not 1 <= len(sources) <= 120 - or any(type(name) is not str or re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]{0,119}\.py', name) is None - or name == BOOTSTRAP.name or type(raw) is not bytes for name, raw in sources.items())): - _fail('Supply the exact finite native worker source closure.') - if sum(map(len, sources.values())) > 64 * 1024 * 1024 - TEMPLATE_LIMIT: - _fail('Native source closure exceeds its custody bound.') - stage = Path(stage) - if phase == 'probe': - if job is not None: _fail('Native probe cannot accept a job.') - template = None - output = stage / 'output' - else: - if type(job) is not dict or type(job.get('parent_pid')) is not int or job['parent_pid'] != 1: - _fail('Native job must retain its parent_pid=1 template.') - if type(job.get('output_dir')) is not str or not Path(job['output_dir']).is_absolute(): - _fail('Native job requires its current private output directory.') - output = Path(job['output_dir']) - if phase == 'train' and output != stage / 'output': - _fail('Native training output must belong to its new phase.') - template = custody.encoded(dict(format=TEMPLATE_FORMAT, phase=phase, worker=worker, - attempt_id=attempt_id, request_sha256=request_sha256, job=job)) - if len(template) > TEMPLATE_LIMIT: _fail('Native job template exceeds 64 KiB.') - files = [] - for path, item in runtime['files'].items(): - if (type(path) is not str or not Path(path).is_absolute() or type(item) is not dict - or set(item) != {'sha256', 'bytes'}): _fail('Invalid measured native runtime file.') - files.append(dict(path=path, **item)) - executable = runtime['executable'] - executable_source = runtime.get('executable_source', executable) - if (executable_source not in runtime['files'] or runtime['files'][executable_source]['sha256'] != runtime['executable_sha256']): - _fail('Native interpreter is absent from its measured runtime closure.') - captures = [_record(stage / 'source' / name, raw) for name, raw in sorted(sources.items())] - if template is not None: - bootstrap = BOOTSTRAP.read_bytes() - captures.extend((_record(stage / 'source' / BOOTSTRAP.name, bootstrap), - _record(stage / 'native-job-template.json', template))) - argv = [executable, '-I', '-B', str(stage / 'source' / BOOTSTRAP.name), phase, - str(stage / 'native-job-template.json')] - else: - argv = [executable, '-B', str(stage / 'source' / worker), '--probe', - *(['--model-aware'] if model_aware else [])] - spec = custody.validate_spec(dict(schema_version=1, attempt_id=attempt_id, request_sha256=request_sha256, - argv=argv, env=ENVIRONMENT.copy(), cwd=str(stage), stdin_base64='', - wall_seconds=wall_seconds if wall_seconds is not None else {'probe':45, 'train':600, 'verify':90}[phase], - executable_sha256=runtime['executable_sha256'], max_output_bytes=max_output_bytes, - output_dir=str(output), captured_files=captures, runtime_files=sorted(files, key=lambda row:row['path']), - trusted_install_owner_uid=runtime['trusted_install_owner_uid'], native_loader_policy='glibc_x86_64_no_preload_v1')) - parent = _parent(stage) - directory = source = None - try: - os.mkdir(stage.name, 0o700, dir_fd=parent) - directory = os.open(stage.name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) - os.mkdir('source', 0o700, dir_fd=directory) - source = os.open('source', os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory) - if output == stage / 'output': os.mkdir('output', 0o700, dir_fd=directory) - elif phase == 'verify': - check = os.open(output, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - try: - info = os.fstat(check) - if info.st_uid != os.getuid() or info.st_mode & 0o077: - _fail('Native verifier output must be a current private directory.') - finally: os.close(check) - for name, raw in sorted(sources.items()): _write(source, name, raw) - if template is not None: - _write(source, BOOTSTRAP.name, bootstrap) - _write(directory, 'native-job-template.json', template) - os.fsync(source); os.fsync(directory); os.fsync(parent) - finally: - for descriptor in (source, directory, parent): - if descriptor is not None: os.close(descriptor) - return PreparedNative(stage, phase, spec, template, measured) diff --git a/project_producer_execution.py b/project_producer_execution.py deleted file mode 100644 index c8833909..00000000 --- a/project_producer_execution.py +++ /dev/null @@ -1,447 +0,0 @@ -"""Explicit current-host producers over inert canonical producer history. - -No owner is constructed by this adapter's constructor. Preparation is part of -Session's canonical foreground transaction; worker launch requires its confirmed -acknowledgement and is never inferred from a saved receipt or imported history. -""" -import copy -import json -import threading -import uuid - -from access import AccessError, current_context -from image_sources import ImageSources -from project_producers import (FAMILIES, TABLES, ProjectProducers, _capture_job, - _CapturedEvidence, _composed, _declarations, _referenced_artifacts, decode_records, - identity, shape) -from workbench import WorkbenchError -from project_session import session_record, session_records - -KINDS = {'generation_history':'generation', 'grounded_history':'grounded', - 'caption_proposal_history':'caption', 'text_proposal_history':'text_proposal'} -JOB_KINDS = {'generation_history':'synthetic', 'grounded_history':'grounded', - 'caption_proposal_history':'caption', 'text_proposal_history':'text_classification'} -FINISH = {'generation_history':'synthetic.finish', 'grounded_history':'grounded.finish', - 'caption_proposal_history':'caption.finish', 'text_proposal_history':'text_classification.finish'} -ACTIONS = frozenset(('start','resume','review','decide','cancel')) -OPERATIONS = frozenset('producers.'+value for value in ACTIONS) - - -def validate_operation_result(operation, target, result, records): - if operation not in OPERATIONS: - return False - shape(result, {'format','family','origin_catalog_id','producer_id','action'}) - if (result['format'] != 'project_producer_action_v1' or result['family'] not in FAMILIES - or result['action'] not in ACTIONS or operation != 'producers.'+result['action']): - raise WorkbenchError('Invalid retained producer action.') - if ((result['action']=='resume' and result['family']!='generation_history') - or (result['action']=='review' and result['family']!='grounded_history') - or (result['action']=='decide' and result['family'] not in ('caption_proposal_history','text_proposal_history'))): - raise WorkbenchError('Producer action does not belong to this finite family.') - identity(result['origin_catalog_id']);identity(result['producer_id']) - if target != result['producer_id'] and not (result['action']=='start' and target is None): - raise WorkbenchError('Producer action target differs.') - if not any(r['family']==result['family'] and r['id']==result['origin_catalog_id']+'.'+result['producer_id'] for r in records): - raise WorkbenchError('Producer action has no retained scientific history.') - return True - - -class _LiveContext: - def __init__(self, adapter): - self.adapter = adapter - self.access = adapter.runtime_access - self.project_execution = adapter.execution - def __getattr__(self, name): - return getattr(self.adapter.data.context, name) - - -class ProjectProducerExecution: - def __init__(self, session, data, archive, runtime_access=None, *, execution=None): - self.session, self.db, self.data, self.archive = session, session.db, data, archive - self.runtime_access = runtime_access or session.runtime_access - self.execution = execution or self - self._owners = {} - self._custody_lock = threading.RLock() - self._journaled = set() - self._journal_observations = {} - self._dirty = {family:set() for family in FAMILIES} - self._runtime_data = None - self._fenced = False - - def _family(self, family): - if family not in FAMILIES: - raise WorkbenchError('Unsupported project producer family.') - return family - - def _owner(self, family): - self._family(family);self.session.check_current() - if self._fenced: - raise WorkbenchError('Current producer custody is uncertain. Reopen for inert inspection.', 'outcome_unknown', 409) - self._refuse_unknown_provider() - if family not in self._owners: - # Imported science retains its physical ordering. A fresh local - # authority table must allocate beyond those observed ordinals. - table=TABLES[family] - observed=max((r['document']['ordinal'] for r in session_records(self.session, family) - if r['family']==family and r['document']['origin_catalog_id']==self.db.dataset_id),default=0) - current=self.db.execute('SELECT COALESCE(max(ordinal),0) FROM '+table).fetchone()[0] - maximum=max(observed,current) - self.db.execute('SELECT setval(pg_get_serial_sequence(%s,%s),%s,%s)',(table,'ordinal',max(1,maximum),maximum>0)) - # The calling action has checked current host authority. Never restore - # archived jobs into these local authority tables. - context = _LiveContext(self) - workbench = copy.copy(self.data.workbench) - workbench.dataset = context - if family == 'generation_history': - from synthetic import Jobs - from image_generation import ImageRequests - context.image_requests = ImageRequests(self.runtime_access) - context.workbench = workbench - image_context = _LiveContext(self) - image_context.access = self.data._authority - image_context.workbench = workbench - images = ImageSources(image_context) - def add(body, generation=None): - if generation is None: - raise WorkbenchError('Current producer context only accepts admitted output.') - with self.db.lock, self.db.transaction(): - key = 'generated-image-'+uuid.uuid4().hex - with self.data.runtime_mutation('data.runtime_generated_image', key): - self._runtime_data = ('data.runtime_generated_image', key) - return images.add(body, generation=generation) - context.add = add - owner = Jobs(context) - context.generation_jobs = owner - image_context.generation_jobs = owner - else: - if family == 'grounded_history': - from grounded_candidates import Proposals as Owner - elif family == 'caption_proposal_history': - from caption_proposals import CaptionProposals as Owner - else: - from text_classification_proposals import TextClassificationProposals as Owner - owner = Owner(workbench) - self._owners[family] = owner - return self._owners[family] - - def _mark(self, family, producer_id): - identity(producer_id) - if not self.db.in_transaction: - raise RuntimeError('Producer writes require their owned SQL transaction.') - self._dirty[family].add(producer_id) - - def mark_generation(self, producer_id):self._mark('generation_history', producer_id) - def mark_grounded(self, producer_id):self._mark('grounded_history', producer_id) - def mark_caption(self, producer_id):self._mark('caption_proposal_history', producer_id) - def mark_text_proposal(self, producer_id):self._mark('text_proposal_history', producer_id) - - def _proof(self, family, job): - row = self.db.execute('SELECT * FROM access_admissions WHERE id=%s', (job.get('admission_id'),)).fetchone() - if row is None: - raise AccessError('Current local producer admission is unavailable.') - row = dict(row) - if (row['catalog_id'] != self.db.dataset_id or row['kind'] != JOB_KINDS[family] - or row['job_id'] != job['id'] or row['request_digest'] != job.get('request_digest')): - raise AccessError('Current producer admission differs from its retained request.') - context = current_context() - if context is None: - raise AccessError('Producer observation requires current authority.') - if context.mode == 'job': - self.runtime_access.require_job(JOB_KINDS[family], job['id'], job['admission_id'], job['request_digest']) - self.runtime_access.require('edit', FINISH[family], job['id']) - elif context.mode == 'system': - # Only a constructed local owner can reconcile its own table. - self.runtime_access.require('cancel', JOB_KINDS[family]+'.reconcile', job['id']) - else: - self.runtime_access.require('inspect', 'producers.observe', job['id']) - - def capture_dirty(self, before_records): - result = [] - if self._runtime_data is not None: - operation, key = self._runtime_data - result.extend(self.data.capture_runtime_changes(before_records, operation=operation, key=key)) - if not any(self._dirty.values()): - return result - capture = copy.copy(self.archive) - capture._origin = self.db.dataset_id - capture._artifacts = {} - capture._pending_payloads = {} - for family, ids in self._dirty.items(): - for producer_id in sorted(ids): - row = self.db.execute('SELECT data,ordinal FROM '+TABLES[family]+' WHERE id=%s', (producer_id,)).fetchone() - if row is None: - raise WorkbenchError('Current producer row disappeared.') - job = json.loads(row[0]);self._proof(family, job) - entries = [] - if family == 'generation_history': - entries = [(json.loads(raw),order) for raw,order in self.db.execute( - 'SELECT data,storage_order FROM generation_entries WHERE job_id=%s ORDER BY ordinal',(producer_id,))] - record = _capture_job(capture, family, job, row[1], entries) - # Keep all observed entry facts across explicit continuation and - # deletion; the current view remains the final observation. - previous = next((r for r in before_records if r['family']==family and r['id']==record['id']),None) - if previous is not None and family == 'generation_history': - old = {v['entry']['id']:v for v in previous['document']['entries']} - for value in record['document']['entries']: - if value['entry']['id'] in old: - history = copy.deepcopy(old[value['entry']['id']]['history']) - payload_history=copy.deepcopy(old[value['entry']['id']].get('history_payloads', - [old[value['entry']['id']]['payloads'] for _ in history])) - if history[-1] != value['entry']: - history.append(copy.deepcopy(value['entry'])) - payload_history.append(copy.deepcopy(value['payloads'])) - value['history'] = history - value['history_payloads'] = payload_history - result.append(record) - result.extend(dict(family='project_artifact',id=sha,document=decl) for sha,decl in capture._artifacts.items()) - decode_records(_composed(before_records,result), artifacts=_CapturedEvidence(capture)) - for sha,raw in capture._pending_payloads.items(): - self.session.artifacts.publish_bytes(raw,capture._artifacts[sha],maximum_bytes=32*1024*1024) - return result - - def outcome(self, outcome): - if outcome not in ('committed','rolled_back','unknown'): - raise ValueError('Invalid producer transaction outcome.') - for values in self._dirty.values():values.clear() - if self._runtime_data is not None: - self.data.finish_runtime_mutation() - self._runtime_data = None - if outcome == 'unknown' or not self.session.available: - self._fenced = True - # Unknown acknowledgement fences authority. It never authorizes an - # unacknowledged cancellation signal to a current worker. - if outcome == 'committed': - from project_session import session_records - records = session_records(self.session, *FAMILIES, 'project_artifact') - self.archive._records = copy.deepcopy([r for r in records if r['family'] in FAMILIES]) - declarations = _declarations(records) - self.archive._artifacts = {sha:declarations[sha] for sha in _referenced_artifacts(self.archive._records)} - - def _ack(self, family, producer_id, action): - return dict(format='project_producer_action_v1',family=family, - origin_catalog_id=self.db.dataset_id,producer_id=producer_id,action=action) - - def _mutate(self, action, family, producer_id, body, checkpoint, key, invoke, *, scope='edit'): - return self.session.mutate('producers.'+action,producer_id,dict(family=family,document=body),checkpoint,key, - invoke,lambda result,before:dict(records=[],result=result),scope=scope) - - def _dispatch_prepared(self, family, owner, job): - try: - # Host-local custody must be durable before any worker can perform - # capability discovery or inference. Saved project receipts cannot - # recreate this registration or launch a worker. - self.execution.register_producer(family,job['id'],job['admission_id'],job['request_digest']) - self._journaled.add((family,job['id'])) - self.execution.producer_effect_intent(family,job['id']) - with self._custody_lock: - owner._project_worker_id=job['id'] - owner._launch(job) - except BaseException: - self._fenced=True - self.session.available=False - raise - - def _settle_drained(self): - if not self.session.available: - return - records=self.session.snapshot['records'] - for family,owner in self._owners.items(): - producer_id=getattr(owner,'_project_worker_id',None) - identity_key=(family,producer_id) - if identity_key not in self._journaled or owner.worker is not None and owner.worker.is_alive(): - continue - doc=next((r['document'] for r in records if r['family']==family and r['id']==self.db.dataset_id+'.'+producer_id),None) - if doc is None or doc['job']['status'] in ('preparing','generating','stopping'): - continue - job=doc['job'];outcome=job.get('provider_outcome') - entries=[v['entry'] for v in doc.get('entries',[])] - batches=[v['batch'] for v in doc.get('prompt_batches',[])] - if (outcome=='unknown' or any(e.get('provider_outcome')=='unknown' for e in entries) - or any(b['status']=='requesting' for b in batches)): - outcome='unknown' - elif (outcome=='result_received' or any(e.get('provider_outcome')=='result_received' for e in entries) - or any(b['response_sha256'] is not None for b in batches)): - outcome='result_received' - else: - # Every effectful request is preceded by canonical uncertainty; - # a terminal job with no such request is known not admitted. - outcome='not_admitted' - observation=(outcome,True,self.session.summary['checkpoint']) - if self._journal_observations.get(identity_key)!=observation: - self.execution.update_producer_outcome(family,producer_id,outcome,True) - self._journal_observations[identity_key]=observation - if outcome!='unknown':self._journaled.discard(identity_key) - - def start(self, family, body, expected_checkpoint, key): - self._family(family) - if family == 'generation_history': - shape(body, {'server_url','model','prompt','count','strategy','session_id'}, - {'width','height','seed','protocol','profile','prompt_url','prompt_model','book_id','split'}) - self.runtime_access.require('execute','producers.start',owner_only=True) - prepared = [] - def invoke(): - requested = body.get('request_id') if type(body) is dict else None - if requested is not None and session_record(self.session, family,self.db.dataset_id+'.'+requested) is not None: - if self.db.execute('SELECT id FROM '+TABLES[family]+' WHERE id=%s',(requested,)).fetchone() is None: - raise WorkbenchError('A retained producer identity cannot acquire current admission. Start with a new identity.','unavailable',409) - owner = self._owner(family) - job, launch = owner._prepare_start(body) - if launch:prepared.append((owner,copy.deepcopy(job))) - return self._ack(family,job['id'],'start') - with self.session.access.lock, self.session.lock, self.db.lock: - self._settle_drained() - result = self._mutate('start',family,None,body,expected_checkpoint,key,invoke,scope='execute') - # Replays never invoke preparation. Unknown/failed commits never reach - # this line. Hold custody locks until the local worker is installed. - for owner,job in prepared: - self._dispatch_prepared(family,owner,job) - return result - - def resume_generation(self, producer_id, expected_checkpoint, key): - identity(producer_id) - self.runtime_access.require('execute','producers.resume',owner_only=True) - prepared=[] - def invoke(): - self._refuse_unknown_provider() - owner=self._owners.get('generation_history') - if owner is None: - raise WorkbenchError('Only a current local generation queue can continue. Imported history is inert.','unavailable',409) - job,launch=owner._prepare_resume(producer_id) - if launch:prepared.append((owner,copy.deepcopy(job))) - return self._ack('generation_history',producer_id,'resume') - with self.session.access.lock,self.session.lock,self.db.lock: - self._settle_drained() - result=self._mutate('resume','generation_history',producer_id,{},expected_checkpoint,key,invoke,scope='execute') - for owner,job in prepared: - self._dispatch_prepared('generation_history',owner,job) - return result - - def _decision(self, action, family, producer_id, body, checkpoint, key): - self._family(family);identity(producer_id) - self.runtime_access.require('edit','producers.'+action,producer_id) - def invoke(): - owner=self._owners.get(family) - if owner is None: - raise WorkbenchError('Imported producer history cannot be applied as current authority.','unavailable',409) - operation={'grounded_history':'data.runtime_grounded_text','caption_proposal_history':'data.runtime_caption', - 'text_proposal_history':'data.runtime_text_classification'}[family] - with self.data.runtime_mutation(operation,key): - self._runtime_data=(operation,key) - if action=='review':owner.review(producer_id,body) - else:owner.decide(producer_id,body) - return self._ack(family,producer_id,action) - return self._mutate(action,family,producer_id,body,checkpoint,key,invoke) - - def review_grounded(self, producer_id, body, expected_checkpoint, key): - return self._decision('review','grounded_history',producer_id,body,expected_checkpoint,key) - - def decide(self, family, producer_id, body, expected_checkpoint, key): - if family not in ('caption_proposal_history','text_proposal_history'): - raise WorkbenchError('Only caption/classification proposals accept a decision.') - return self._decision('decide',family,producer_id,body,expected_checkpoint,key) - - def cancel(self, family, producer_id, expected_checkpoint, key, *, drain_timeout=10): - self._family(family);identity(producer_id) - self.runtime_access.require('cancel','producers.cancel',producer_id) - if getattr(self.db.lock,'owned_by_current_thread',False): - raise WorkbenchError('Cancellation must signal and drain outside project command locks.','producer_active',409) - prepared=[] - def invoke(): - owner=self._owners.get(family) - if owner is None: - raise WorkbenchError('No current local producer custody exists for this history.','unavailable',409) - _result,pending=owner._prepare_cancel(producer_id) - if pending is not None:prepared.append((owner,pending)) - return self._ack(family,producer_id,'cancel') - result=self._mutate('cancel',family,producer_id,{},expected_checkpoint,key,invoke,scope='cancel') - # A replay never invokes preparation. Failure or uncertain COMMIT never - # reaches this point, and therefore never stops transport or a worker. - for owner,pending in prepared: - with self._custody_lock: - signalled=owner._signal_cancel(pending) - # The image transport owner audits through host/SQL locks. Never - # hold the custody mutex while entering those ordered guards. - if signalled and family=='generation_history':owner._cancel_transport(pending) - worker=pending[0] - if worker is not None: - worker.join(drain_timeout) - if worker.is_alive(): - raise WorkbenchError('Cancellation intent is saved; current worker has not drained.','producer_draining',409) - return result - - def _refuse_unknown_provider(self): - for family in self._owners: - for raw, in self.db.execute('SELECT data FROM '+TABLES[family]): - job=json.loads(raw) - if (job.get('provider_outcome')=='unknown' - or any(v['status']=='requesting' for v in job.get('prompt_batches',[]))): - raise WorkbenchError('Downstream producer outcome is unknown; local drainage is not provider cessation.','outcome_unknown',409) - if family=='generation_history': - if any(json.loads(row[0]).get('provider_outcome')=='unknown' for row in self.db.execute('SELECT data FROM generation_entries WHERE job_id=%s',(job['id'],))): - raise WorkbenchError('Image provider outcome is unknown.','outcome_unknown',409) - - def abandon_pending(self): - if not getattr(self.execution,'_reconstructing',False): - raise RuntimeError('Pending producer facts can be abandoned only by explicit current reconstruction.') - for owner in self._owners.values(): - if owner.worker is not None and owner.worker.is_alive(): - raise WorkbenchError('Actual local producer custody has not drained.','producer_active',409) - self._refuse_unknown_provider() - for values in self._dirty.values():values.clear() - self.data.finish_runtime_mutation() - self._runtime_data=None - self._fenced=False - - def assert_quiescent(self): - if not self.has_local_custody: - return - self._settle_drained() - for family,owner in self._owners.items(): - if owner.worker is not None and owner.worker.is_alive(): - raise WorkbenchError('Stop and drain the current producer before changing project.','producer_active',409) - self._refuse_unknown_provider() - for raw, in self.db.execute('SELECT data FROM '+TABLES[family]): - job=json.loads(raw) - if job['status'] in ('preparing','generating','stopping') or job.get('provider_outcome')=='unknown': - raise WorkbenchError('Current producer is active or its provider outcome is unknown.','producer_active',409) - - def close(self): - # Project lifecycle refuses active/unknown custody; it never turns Close - # into an implicit cancellation or recovery instruction. - self.assert_quiescent() - - @property - def has_local_custody(self): - return bool(self._owners) - - def shutdown(self): - """Explicit application shutdown drains only actual constructed owners. - - Historical rows, process IDs and endpoints never select a custodian. - Shutdown is separate from ordinary project Close, which refuses work. - """ - if getattr(self.db.lock,'owned_by_current_thread',self.db.lock.depth) or self.db.in_transaction: - raise WorkbenchError('Producer shutdown must drain outside project command locks.','producer_active',409) - for owner in self._owners.values(): - with self.runtime_access._reconcile():owner.close() - with self.runtime_access._reconcile():self._settle_drained() - - def status(self, *, offset=0, limit=40): - if type(offset) is not int or not 0<=offset<=1_000_000 or type(limit) is not int or not 1<=limit<=40: - raise WorkbenchError('Invalid current producer health page.') - with self.session.access.lock,self.session.lock,self.db.lock: - self.session.access.require('inspect','producers.health') - self.session.check_current() - self._settle_drained() - items=[] - for family,owner in self._owners.items(): - for raw, in self.db.execute('SELECT data FROM '+TABLES[family]+' ORDER BY ordinal'): - job=json.loads(raw) - observed=job.get('provider_outcome') - if any(v['status']=='requesting' for v in job.get('prompt_batches',[])):observed='unknown' - if family=='generation_history' and any(json.loads(row[0]).get('provider_outcome')=='unknown' for row in self.db.execute('SELECT data FROM generation_entries WHERE job_id=%s',(job['id'],))):observed='unknown' - items.append(dict(family=family,producer_id=job['id'],status=job['status'], - provider_outcome=observed,worker_alive=bool(owner.worker - and owner.worker.is_alive() and getattr(owner,'_project_worker_id',None)==job['id']))) - return dict(format='project_producer_health_v1',items=items[offset:offset+limit],total=len(items), - offset=offset,limit=limit,fenced=bool(self._fenced or self.runtime_access.catalog_id==self.session.access.catalog_id or any(v['provider_outcome']=='unknown' for v in items))) diff --git a/project_producers.py b/project_producers.py deleted file mode 100644 index 8e052b34..00000000 --- a/project_producers.py +++ /dev/null @@ -1,1247 +0,0 @@ -"""Inert, origin-scoped producer observations and exact evidence payloads. - -No producer constructor, admission, worker, cancellation or resume API enters this -owner. Four closed codecs preserve the scientific observations their existing -owners validate. Local transport configuration and admission envelopes have an -explicit exclusion; unknown fields fail instead of disappearing during export. -""" -import base64 -import copy -from datetime import datetime -import hashlib -import json -import re - -from access import authorized -import caption_proposals -import grounded_candidates -import image_generation -import pumas_operations as pumas -import synthetic -import text_classification_proposals -from workbench import WorkbenchError, encode, text_value, validate_annotation - -FAMILIES = { - 'generation_history': 'project_generation_history_v1', - 'grounded_history': 'project_grounded_history_v1', - 'caption_proposal_history': 'project_caption_proposal_history_v1', - 'text_proposal_history': 'project_text_proposal_history_v1', -} -TABLES = {'generation_history': 'generation_jobs', 'grounded_history': 'grounded_jobs', - 'caption_proposal_history': 'caption_proposals', 'text_proposal_history': 'text_classification_proposals'} -ID = re.compile(r'^[a-f0-9]{32}$') -SHA = re.compile(r'^[a-f0-9]{64}$') -MAX_RECORDS = 100_000 -MAX_RESPONSE = 256 * 1024 -LOCAL_JOB_FIELDS = {'admission_id', 'request_digest'} -LOCAL_CONFIG_FIELDS = {'server_url', 'requested_server_url', 'prompt_url'} -AUTHORITY_FIELDS = {'token', 'token_sha256', 'access_token', 'refresh_token', 'api_key', 'password', - 'authorization', 'cookie', 'set-cookie', 'credentials', 'admission_id', 'grant_id'} -EVIDENCE_FIELDS = {'response_sha256', 'capability_observation', 'producer_contract_source', 'canonical_request', - 'provider_outcome', 'provider_error_code', 'provider_http_status', 'build_observation', 'canonical_request_sha256'} -JOB_COMMON = {'schema_version', 'id', 'revision', 'status', 'error', 'created_at', 'updated_at', - 'source', 'config', 'system_prompt', 'prompt_version', 'prompt_sha256', - 'canonical_request_sha256', 'response_sha256', 'verification'} -JOB_OPTIONAL = {'reported_model', 'catalog_qualification', 'listed_models'} | EVIDENCE_FIELDS -PAYLOAD_FIELDS = {'raw_response_base64': 'raw_response', 'canonical_request': 'canonical_request', - 'input_image_base64': 'input_image'} -ENTRY_REQUIRED = {'id', 'job_id', 'ordinal', 'prompt', 'status', 'sample_id', 'error', 'model', - 'width', 'height', 'requested_seed'} -ENTRY_OPTIONAL = {'size', 'metadata', 'deleted_at', 'request_id', 'provider_outcome', 'attempt_evidence'} - - -def fail(message, code='invalid', status=400): - raise WorkbenchError(message, code, status) - - -def shape(value, required, optional=(), name='producer'): - if type(value) is not dict or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): - fail('Unsupported ' + name + ' fields.') - - -def identity(value): - if type(value) is not str or not ID.fullmatch(value): - fail('Invalid producer historical identity.') - return value - - -def digest(value): - if type(value) is not str or not SHA.fullmatch(value): - fail('Invalid producer digest.') - return value - - -def integer(value, minimum=0, maximum=2**53-1): - if type(value) is not int or not minimum <= value <= maximum: - fail('Invalid producer count/revision.') - return value - - -def date(value): - try: - if type(value) is not str or len(value) > 80: - raise ValueError() - parsed = datetime.fromisoformat(value) - if parsed.utcoffset() is None: - raise ValueError() - return parsed - except ValueError: - fail('Invalid producer observation timestamp.') - - -def _safe(value, depth=0): - if depth > 64: - fail('Producer evidence exceeds its nesting bound.') - if type(value) is dict: - if any(type(key) is not str or key.lower() in AUTHORITY_FIELDS for key in value): - fail('Producer evidence contains live credential or admission material.') - for child in value.values(): - _safe(child, depth+1) - elif type(value) is list: - for child in value: - _safe(child, depth+1) - elif type(value) not in (str, int, float, bool, type(None)): - fail('Producer evidence is not finite JSON.') - try: - if depth == 0: - json.dumps(value, allow_nan=False, ensure_ascii=False).encode('utf-8') - except (ValueError, UnicodeError, OverflowError): - fail('Producer evidence contains invalid Unicode or nonfinite values.') - - -def _sha(raw): - return hashlib.sha256(raw).hexdigest() - - -def _bytes(value, maximum): - try: - if type(value) is not str or len(value) > 4*((maximum+2)//3): - raise ValueError() - raw = base64.b64decode(value, validate=True) - if len(raw) > maximum or base64.b64encode(raw).decode('ascii') != value: - raise ValueError() - return raw - except ValueError: - fail('Producer payload requires bounded exact base64 bytes.') - - -def _observations(job): - for field in ('canonical_request_sha256', 'response_sha256'): - if job.get(field) is not None: - digest(job[field]) - if 'capability_observation' in job: - observed = job['capability_observation'] - shape(observed, {'capabilities', 'observed_sha256'}, name='capability observation') - # The original wire digest is retained independently: the manifest is an - # already-decoded observation and cannot recreate its original spacing. - pumas.validate_manifest(encode(observed['capabilities']).encode()) - digest(observed['observed_sha256']) - if observed['capabilities']['model'] != job['config']['model']: - fail('Producer capability observation belongs to another model.') - if 'producer_contract_source' in job: - source = job['producer_contract_source'] - if type(source) is not str or not re.fullmatch('[a-f0-9]{40}', source): - fail('Invalid producer contract source identity.') - if 'provider_outcome' in job and job['provider_outcome'] not in ('not_admitted', 'unknown', 'result_received'): - fail('Unsupported historical provider outcome.') - if job.get('provider_http_status') is not None: - integer(job['provider_http_status'], 100, 599) - if job.get('provider_error_code') is not None: - text_value(job['provider_error_code'], 'Provider error code', 200) - if 'build_observation' in job: - import pumas_gateway_descriptor as descriptor - build = job['build_observation'] - # Build observation interpretation stays with its existing owner. - shape(build, {'advertisement', 'observed_sha256'}, name='retained build observation') - descriptor.decode(encode(build['advertisement']).encode()) - digest(build['observed_sha256']) - - -def _config(family, config): - if family == 'generation_history': - required = {'model', 'prompt', 'width', 'height', 'count', 'strategy'} - optional = {'seed', 'protocol', 'profile', 'prompt_model', 'size'} - else: - required = {'model', 'instruction', 'seed'} - optional = {'protocol', 'profile'} - if family != 'grounded_history': - required.add('provider') - if family == 'grounded_history': - required.add('count') - if family == 'text_proposal_history': - required |= {'labels', 'requested_model'} - shape(config, required, optional, 'producer configuration') - pumas.alias(config['model']) - mode = pumas.configuration(config, image=family == 'generation_history') - if (family == 'generation_history' and config.get('seed') is not None - or family != 'generation_history' and not mode): - integer(config.get('seed'), 0, 2**32-1) - if family == 'generation_history': - dimensions = synthetic.saved_dimensions(config) - if (dimensions['width'], dimensions['height']) != (config['width'], config['height']): - fail('Saved producer geometry differs.') - integer(config['count'], 1, 10000) - text_value(config['prompt'], 'Generation prompt', 4000) - if config['strategy'] not in ('varied', 'repeat') or mode and config['strategy'] != 'repeat': - fail('Unsupported saved prompt strategy.') - if config['strategy'] == 'varied': - pumas.alias(config.get('prompt_model')) - else: - text_value(config['instruction'], 'Producer guidance', 2000) - if family == 'grounded_history': - integer(config['count'], 1, 10) - if family == 'text_proposal_history': - text_classification_proposals.validate_labels(config['labels']) - text_value(config['requested_model'], 'Requested model identity', 200) - if family != 'grounded_history' and config['provider'] not in ( - 'pumas_chat_compatible', 'pumas_image_to_text', pumas.PROTOCOL): - fail('Unsupported producer identity.') - - -def _entry(entry, job): - shape(entry, ENTRY_REQUIRED, ENTRY_OPTIONAL, 'generation entry') - identity(entry['id']); identity(entry['job_id']) - if entry['job_id'] != job['id'] or entry['model'] != job['config']['model']: - fail('Generation entry belongs to another job/model.') - integer(entry['ordinal'], 0, job['config']['count']-1) - dimensions = synthetic.saved_dimensions(entry) - if (dimensions['width'], dimensions['height']) != (job['config']['width'], job['config']['height']): - fail('Generation entry geometry differs from its job.') - text_value(entry['prompt'], 'Saved generation prompt', 4000) - text_value(entry['error'], 'Generation error', 800, empty=True) - if entry['status'] not in ('pending', 'generating', 'completed', 'failed', 'deleted'): - fail('Unsupported saved generation entry status.') - if entry['sample_id'] is not None: - identity(entry['sample_id']) - if ((entry['status'] == 'completed') != (entry['sample_id'] is not None)): - fail('Generation entry source/deletion state differs.') - expected_seed = ((job['config']['seed'] + entry['ordinal']) % 2**32 - if job['config'].get('seed') is not None else None) - if entry['requested_seed'] != expected_seed: - fail('Generation entry seed differs from its saved ordinal.') - if 'deleted_at' in entry: - date(entry['deleted_at']) - if 'request_id' in entry: - identity(entry['request_id']) - if 'provider_outcome' in entry and entry['provider_outcome'] not in ('not_admitted', 'unknown', 'result_received'): - fail('Unsupported generation provider outcome.') - for field in ('metadata', 'attempt_evidence'): - if field in entry: - value = entry[field] - optional = {'seed', 'steps', 'guidance', 'duration_seconds', 'memory_policy', 'protocol'} | EVIDENCE_FIELDS - shape(value, (), optional if field == 'metadata' else EVIDENCE_FIELDS, 'generation result evidence') - for number in ('seed', 'steps', 'guidance', 'duration_seconds'): - if number in value and type(value[number]) not in (int, float): - fail('Generation numeric observation differs from its owner.') - if 'memory_policy' in value and value['memory_policy'] not in ('sequential_cpu_offload', 'model_cpu_offload', 'gpu'): - fail('Unsupported generation memory policy observation.') - if 'protocol' in value and value['protocol'] != pumas.PROTOCOL: - fail('Unsupported generation protocol observation.') - _observations(dict(value, config=job['config'])) - _safe(value) - _safe(entry) - - -def _validate_job_value(family, job): - if family == 'generation_history': - shape(job, {'id', 'config', 'meta', 'status', 'error', 'completed'}, EVIDENCE_FIELDS, - 'generation job') - from image_sources import metadata - if metadata(job['meta']) != job['meta']: - fail('Generation source metadata differs from its owner.') - _config(family, job['config']) - integer(job['completed'], 0, job['config']['count']) - states = ('preparing', 'generating', 'stopping', 'completed', 'failed', 'cancelled', 'interrupted') - else: - extra = {'candidates', 'raw_response_text_available'} if family == 'grounded_history' else {'intent_sha256', 'annotation', 'application'} - if family == 'caption_proposal_history': - extra.add('input_image') - if family == 'text_proposal_history': - extra |= {'labels_sha256', 'source_text_sha256'} - shape(job, JOB_COMMON | extra, JOB_OPTIONAL - {'canonical_request'}, 'proposal history') - if job['schema_version'] != 1: - fail('Unsupported producer schema version.', 'unsupported', 422) - _config(family, job['config']) - integer(job['revision'], 1) - if date(job['updated_at']) < date(job['created_at']): - fail('Producer update precedes creation.') - from project_data import validate_record_snapshot - validate_record_snapshot(job['source']) - expected_kind = 'image' if family == 'caption_proposal_history' else 'text' - if job['source']['kind'] != expected_kind: - fail('Captured source has the wrong producer modality.') - text_value(job['system_prompt'], 'Captured system prompt', 20000) - module = {'grounded_history': grounded_candidates, 'caption_proposal_history': caption_proposals, 'text_proposal_history': text_classification_proposals}[family] - if job['prompt_version'] != module.PROMPT_VERSION or job['system_prompt'] != module.SYSTEM_PROMPT: - fail('Unsupported historical producer prompt contract.', 'unsupported', 422) - if _sha(job['system_prompt'].encode()) != job['prompt_sha256']: - fail('Captured system prompt digest differs.') - digest(job['canonical_request_sha256']) - if job['response_sha256'] is not None: - digest(job['response_sha256']) - if family == 'text_proposal_history': - if _sha(encode(job['config']['labels']).encode()) != job['labels_sha256'] or _sha(job['source']['text'].encode()) != job['source_text_sha256']: - fail('Frozen labels or source text digest differs.') - if family != 'grounded_history': - digest(job['intent_sha256']) - text_value(job['verification'], 'Producer verification qualification', 2000) - if job.get('reported_model') is not None: - text_value(job['reported_model'], 'Reported model', 200) - if 'catalog_qualification' in job: - text_value(job['catalog_qualification'], 'Model catalog qualification', 2000) - if 'listed_models' in job: - if type(job['listed_models']) is not list or len(job['listed_models']) > 10000: - fail('Invalid observed serving model catalog.') - for model in job['listed_models']: - text_value(model, 'Observed serving model', 256) - if family == 'grounded_history': - if type(job['raw_response_text_available']) is not bool: - fail('Invalid exact-response text-view observation.') - if job['source']['task'] != 'text_classification' or job['source']['review'] != 'human_reviewed' or job['source']['annotation'] is None: - fail('Grounded source lacks its captured human-reviewed class.') - if len(job['source']['text']) > grounded_candidates.MAX_SOURCE: - fail('Captured grounded source exceeds its owner bound.') - _candidates(job) - elif job['application'] is not None: - shape(job['application'], {'record_id', 'revision', 'source_revision'}, name='proposal application') - application = job['application'] - if (application['record_id'] != job['source']['id'] or application['revision'] != job['source']['revision']+1 - or application['source_revision'] != job['source']['source_revision'] or job['status'] != 'applied'): - fail('Proposal application differs from its captured revisions.') - elif job['status'] == 'applied': - fail('Applied producer observation lacks its application facts.') - if family == 'caption_proposal_history': - shape(job['input_image'], {'mime', 'size', 'bytes', 'sha256', 'preparation'}, name='prepared image') - image = job['input_image'] - if image['mime'] != 'image/jpeg' or type(image['size']) is not list or len(image['size']) != 2: - fail('Invalid prepared image format.') - for size in image['size']: - integer(size, 1, 1600) - integer(image['bytes'], 1, caption_proposals.MAX_IMAGE); digest(image['sha256']) - if image['preparation'] != 'oriented canonical RGB; thumbnail max 1600; JPEG quality 95': - fail('Unsupported prepared image procedure.', 'unsupported', 422) - if job['annotation'] is not None: - validate_annotation('image_caption', job['annotation'], job['source']) - if family == 'text_proposal_history' and job['annotation'] is not None: - if job['annotation'] not in [{'label': label} for label in job['config']['labels']]: - fail('Saved target is outside its frozen label set.') - states = ('preparing', 'generating', 'stopping', 'completed', 'failed', 'cancelled', 'interrupted', 'applied', 'rejected') - if family == 'text_proposal_history': - states += ('abstained',) - identity(job['id']) - if job['status'] not in states: - fail('Unsupported producer lifecycle observation.') - text_value(job['error'], 'Producer error observation', 800, empty=True) - _observations(job); _safe(job) - - -def _job(family, job): - try: - _validate_job_value(family, job) - except WorkbenchError: - raise - except (ValueError, TypeError, KeyError, AttributeError, OverflowError): - fail('Invalid retained producer value.') - - -def _candidates(job): - values = job['candidates'] - if type(values) is not list or len(values) > job['config']['count']: - fail('Invalid retained grounded candidate list.') - if job['status'] == 'completed' and len(values) != job['config']['count']: - fail('Completed grounded output is incomplete.') - seen, texts = set(), {job['source']['text']} - for candidate in values: - shape(candidate, {'id', 'text', 'label', 'evidence', 'status', 'record_id', 'review_note'}, name='grounded candidate') - identity(candidate['id']) - if candidate['id'] in seen: - fail('Duplicate grounded candidate identity.') - seen.add(candidate['id']) - grounded_candidates.validate_candidate_values([candidate], job['source'], existing_texts=texts) - texts.add(candidate['text']) - if candidate['status'] not in ('pending_review', 'admitted', 'rejected'): - fail('Invalid grounded review disposition.') - if candidate['record_id'] is not None: - identity(candidate['record_id']) - if (candidate['status'] == 'admitted') != (candidate['record_id'] is not None): - fail('Grounded admission identity differs from its disposition.') - text_value(candidate['review_note'], 'Grounded review note', 1000, empty=True) - - -def _declarations(records): - result = {} - for envelope in records: - if envelope.get('family') == 'project_artifact': - from project_artifacts import validate_declaration - declaration = validate_declaration(envelope['document']) - if envelope['id'] != declaration['sha256'] or declaration['sha256'] in result: - fail('Invalid duplicate producer artifact declaration.') - result[declaration['sha256']] = declaration - return result - - -def _edge(edge, declarations, role): - shape(edge, {'artifact', 'media_type', 'kind', 'role', 'preservation', 'required_for'}, name='producer artifact edge') - shape(edge['artifact'], {'namespace', 'sha256'}, name='producer artifact identity') - identity(edge['artifact']['namespace']); digest(edge['artifact']['sha256']) - if edge['kind'] != 'producer_evidence' or edge['role'] != role or edge['preservation'] != 'unique' or edge['required_for'] != ['verify_producer']: - fail('Producer payload role or retention differs.') - expected_media = 'image/jpeg' if role == 'input_image' else 'application/octet-stream' if role == 'raw_response' else 'application/json' - if edge['media_type'] != expected_media: - fail('Producer payload media type differs from its role.') - declaration = declarations.get(edge['artifact']['sha256']) - if declaration is None: - fail('Producer payload declaration is missing.') - return declaration - - -def _payloads(value, declarations, roles): - shape(value, (), roles, 'producer payloads') - for role, edge in value.items(): - _edge(edge, declarations, role) - - -def decode_records(records, *, artifacts=None): - """Validate four fixed history families; absent payloads remain declared. - - With an artifact capability, every available payload is decoded by its - original owner. Missing bytes permit metadata inspection, never inference - or a new verified result. Corrupt available bytes fail closed. - """ - if type(records) is not list or len(records) > MAX_RECORDS or any(type(item) is not dict for item in records): - fail('Producer inventory exceeds its record bound.') - declarations = _declarations(records) - result = [] - seen, observed_order = set(), set() - for envelope in records: - family = envelope.get('family') - if family not in FAMILIES: - continue - shape(envelope, {'family', 'id', 'document'}, name='producer envelope') - doc = envelope['document'] - shape(doc, {'format', 'origin_catalog_id', 'ordinal', 'job', 'payloads', 'observation_integrity'} | ({'entries'} if family == 'generation_history' else set()), {'prompt_batches'} if family == 'generation_history' else (), name='producer document') - if doc['format'] != FAMILIES[family]: - fail('Unsupported producer document format.', 'unsupported', 422) - identity(doc['origin_catalog_id']); _job(family, doc['job']) - integer(doc['ordinal'], 1, 2**63-1) - order_key = (family, doc['origin_catalog_id'], doc['ordinal']) - if order_key in observed_order: - fail('Duplicate historical producer ordering.') - observed_order.add(order_key) - if doc['observation_integrity'] != _observation_integrity(doc['job']): - fail('Producer observation wire-integrity classification differs.') - if envelope['id'] != doc['origin_catalog_id'] + '.' + doc['job']['id'] or (family, envelope['id']) in seen: - fail('Producer origin/identity differs or repeats.') - seen.add((family, envelope['id'])) - roles = {'raw_response', 'canonical_request'} | ({'input_image'} if family == 'caption_proposal_history' else set()) - _payloads(doc['payloads'], declarations, roles) - if family != 'generation_history' and 'canonical_request' not in doc['payloads']: - fail('Producer canonical request declaration is missing.') - if doc['job'].get('response_sha256') is not None and 'raw_response' not in doc['payloads']: - fail('Producer response declaration is missing.') - if family == 'caption_proposal_history' and 'input_image' not in doc['payloads']: - fail('Prepared producer input declaration is missing.') - if family == 'generation_history': - _prompt_batches(doc,declarations) - if type(doc['entries']) is not list or len(doc['entries']) > doc['job']['config']['count']: - fail('Invalid generation queue closure.') - ordinals, identities, storage_orders = set(), set(), set() - for value in doc['entries']: - shape(value, {'entry', 'history', 'payloads', 'observation_integrity', 'storage_order'}, {'history_payloads'}, name='generation entry history') - integer(value['storage_order'], 1, 2**63-1) - if value['storage_order'] in storage_orders: - fail('Duplicate historical generation storage ordering.') - storage_orders.add(value['storage_order']) - if value['observation_integrity'] != _entry_integrity(value['entry']): - fail('Generation observation wire-integrity classification differs.') - _entry_history(value, doc['job']) - _entry(value['entry'], doc['job']) - ordinal, entry_id = value['entry']['ordinal'], value['entry']['id'] - if ordinal in ordinals or entry_id in identities: - fail('Duplicate generation queue entry.') - ordinals.add(ordinal); identities.add(entry_id) - _payloads(value['payloads'], declarations, {'attempt_request', 'result_request'}) - histories=value.get('history_payloads',[value['payloads'] for _ in value['history']]) - if type(histories) is not list or len(histories)!=len(value['history']) or histories[-1]!=value['payloads']: - fail('Generation evidence history differs from its observations.') - for entry,mapping in zip(value['history'],histories): - _payloads(mapping,declarations,{'attempt_request','result_request'}) - for field,role in (('attempt_evidence','attempt_request'),('metadata','result_request')): - sha=entry.get(field,{}).get('canonical_request_sha256') - if sha is not None and (role not in mapping or mapping[role]['artifact']['sha256']!=sha): - fail('Generation request declaration is missing or differs.') - if sorted(ordinals) != list(range(len(ordinals))): - fail('Generation queue ordinals have a gap.') - if doc['job']['completed'] != sum(value['entry']['status'] in ('completed', 'deleted') for value in doc['entries']): - fail('Generation completed count differs from its retained entries.') - if doc['job']['status'] == 'completed' and doc['job']['completed'] != doc['job']['config']['count']: - fail('Completed generation job lacks its complete queue.') - try: - synthetic.validate_saved_queue(doc['job'], [value['entry'] for value in doc['entries']]) - except ValueError: - fail('Retained generation geometry differs from its queue owner.') - if artifacts is not None: - _verify_payloads(family, doc, declarations, artifacts) - result.append(copy.deepcopy(envelope)) - _source_closure(result, records) - return result - - -def _read(edge, declarations, artifacts, maximum): - declaration = declarations[edge['artifact']['sha256']] - state = artifacts.status(declaration)['state'] - if state == 'missing': - return None - if state != 'verified': - fail('Producer evidence payload is corrupt.', 'integrity', 409) - return artifacts.read_blob({'sha256': declaration['sha256'], 'bytes': declaration['bytes']}, maximum_bytes=maximum) - - -def _verify_payloads(family, doc, declarations, artifacts): - job = copy.deepcopy(doc['job']) - payloads = doc['payloads'] - for role in ('raw_response', 'canonical_request', 'input_image'): - if role not in payloads: - continue - maximum = caption_proposals.MAX_IMAGE if role == 'input_image' else 32*1024*1024 if role == 'canonical_request' else image_generation.MAX_JSON if family == 'generation_history' else MAX_RESPONSE - raw = _read(payloads[role], declarations, artifacts, maximum) - if raw is None: - continue - if role == 'canonical_request': - parsed = pumas.strict_json(raw); _safe(parsed) - if _sha(raw) != job.get('canonical_request_sha256'): - fail('Canonical producer request digest differs.') - job['canonical_request'] = parsed - elif role == 'input_image': - if _sha(raw) != job['input_image']['sha256'] or len(raw) != job['input_image']['bytes']: - fail('Prepared producer image differs from its observation.') - job['input_image_base64'] = base64.b64encode(raw).decode('ascii') - else: - if _sha(raw) != job.get('response_sha256'): - fail('Producer response digest differs.') - _response_values(raw) - job['raw_response_base64'] = base64.b64encode(raw).decode('ascii') - if family == 'generation_history': - _verify_prompt_batches(doc,declarations,artifacts) - for wrapper in doc['entries']: - histories=wrapper.get('history_payloads',[wrapper['payloads'] for _ in wrapper['history']]) - for entry,mapping in zip(wrapper['history'],histories): - for field,role in (('attempt_evidence','attempt_request'),('metadata','result_request')): - if role not in mapping:continue - raw=_read(mapping[role],declarations,artifacts,32*1024*1024) - if raw is None:continue - evidence=copy.deepcopy(entry.get(field,{})) - evidence['canonical_request']=pumas.strict_json(raw) - _safe(evidence) - try:synthetic.validate_retained_attempt(job,entry,evidence) - except (ValueError,TypeError,KeyError):fail('Generation request differs from its retained entry intent.') - - else: - module = {'grounded_history': grounded_candidates, 'caption_proposal_history': caption_proposals, - 'text_proposal_history': text_classification_proposals}[family] - module.validate_retained_result(job) - if family != 'caption_proposal_history' or 'input_image_base64' in job: - expected = module.request_payload(job) if family == 'grounded_history' else module.payload(job) - if pumas.typed(job['config']) and 'capability_observation' in job: - expected['profile'] = job['capability_observation']['capabilities']['profile'] - if _sha(encode(expected).encode()) != job['canonical_request_sha256']: - fail('Producer request differs from its captured source/configuration.') - if 'canonical_request' in job and expected != job['canonical_request']: - fail('Retained producer request body differs from its owner.') - - -class InertGenerationEntries: - """Source-deletion projection; deliberately has no Jobs lifecycle methods.""" - def __init__(self, owner): - self.owner, self.db, self.access = owner, owner.session.db, owner.session.access - self.lock = self.db.lock - - @authorized('inspect', 'synthetic.snapshot') - def snapshot(self): - with self.lock: - jobs = [copy.deepcopy(item['document']['job']) for item in sorted(self.owner._records, - key=lambda item: (item['document']['origin_catalog_id'], item['document']['ordinal'])) - if item['family'] == 'generation_history'] - entries = [json.loads(row[0]) for row in self.db.execute('SELECT data FROM generation_entries ORDER BY storage_order')] - return {'jobs': jobs, 'entries': entries} - - def capture_records(self): - return self.owner.capture() - - def _entry(self, entry): - self.access.require('edit', 'dataset.delete', entry.get('sample_id')) - previous = self.db.execute('SELECT data FROM generation_entries WHERE id=%s', (entry.get('id'),)).fetchone() - if previous is None: - fail('Only an existing inert generation entry can be changed.') - old = json.loads(previous[0]) - expected = dict(old, status='deleted', sample_id=None, deleted_at=entry.get('deleted_at')) - if entry != expected or old['status'] not in ('completed', 'deleted'): - fail('Only source-deletion facts may change an inert generation entry.') - date(entry['deleted_at']) - self.db.execute('UPDATE generation_entries SET data=%s WHERE id=%s', (encode(entry), entry['id'])) - - -class ProjectProducers: - def __init__(self, session): - self.session = session - self._records, self._artifacts = [], {} - self._pending_payloads = {} - self.generation_jobs = InertGenerationEntries(self) - self.generation_entries = self.generation_jobs - - def _read(self, action, invoke): - with self.session.access.lock, self.session.lock, self.session.db.lock: - self.session.access.require('inspect', action) - self.session.check_current() - result = invoke(self.session.snapshot['records']) - self.session.check_current() - return result - - def list(self, family, *, origin_catalog_id=None, offset=0, limit=40): - if family not in FAMILIES: - fail('Unsupported producer family.') - if origin_catalog_id is not None: - identity(origin_catalog_id) - integer(offset); integer(limit, 1, 100) - def read(records): - values = sorted((item['document'] for item in records if item['family'] == family - and (origin_catalog_id is None or item['document']['origin_catalog_id'] == origin_catalog_id)), - key=lambda doc: (doc['origin_catalog_id'], doc['ordinal'])) - return dict(items=[_summary(family, doc) for doc in values[offset:offset+limit]], - total=len(values), offset=offset, limit=limit) - return self._read('project_files.producers', read) - - def get(self, family, id, *, origin_catalog_id, entries_offset=0, entries_limit=40): - if family not in FAMILIES: - fail('Unsupported producer family.') - identity(id); identity(origin_catalog_id) - integer(entries_offset); integer(entries_limit, 1, 100) - def read(records): - doc = next((item['document'] for item in records if item['family'] == family - and item['id'] == origin_catalog_id + '.' + id), None) - if doc is None: - fail('Historical producer was not found.', 'not_found', 404) - declarations = _declarations(records) - result = _summary(family, doc) - result['job'] = {key:copy.deepcopy(value) for key,value in doc['job'].items() if key != 'source'} - source = doc['job'].get('source') - result['source_reference'] = ({key:source[key] for key in - ('id','kind','revision','source_revision','content_hash','source_sha256')} if source else None) - result['payloads'] = self._payload_status(doc['payloads'], declarations) - result['observation_integrity'] = copy.deepcopy(doc['observation_integrity']) - entries = doc.get('entries', []) - page, used = [], 0 - for value in entries[entries_offset:entries_offset+entries_limit]: - item = copy.deepcopy(value) - item['payloads'] = self._payload_status(item['payloads'], declarations) - if 'history_payloads' in item:item['history_payloads']=[self._payload_status(value,declarations) for value in item['history_payloads']] - size = len(encode(item).encode()) - if used + size > 8*1024*1024: - break - page.append(item); used += size - next_offset = entries_offset + len(page) - batches=[] - for value in doc.get('prompt_batches',[])[entries_offset:entries_offset+entries_limit]: - item=copy.deepcopy(value) - item['payloads']=self._payload_status(item['payloads'],declarations) - if len(encode(batches+[item]).encode())>8*1024*1024:break - batches.append(item) - next_batch=entries_offset+len(batches) - result['prompt_batches']=dict(items=batches,total=len(doc.get('prompt_batches',[])),offset=entries_offset,limit=entries_limit, - next_offset=next_batch if next_batch<len(doc.get('prompt_batches',[])) else None) - result['entries'] = dict(items=page, total=len(entries), offset=entries_offset, limit=entries_limit, - next_offset=next_offset if next_offset < len(entries) else None) - return result - return self._read('project_files.producer', read) - - def _payload_status(self, payloads, declarations): - result = copy.deepcopy(payloads) - for edge in result.values(): - # Presence is not byte verification. Reads by scientific consumers - # still obtain verified snapshots from the artifact owner. - edge['availability'] = self.session.artifacts.inspect(declarations[edge['artifact']['sha256']]) - return result - - def capture_source(self, *, origin_catalog_id=None): - previous = self._artifacts - self._artifacts = {} - try: - return _capture_source(self, origin_catalog_id=origin_catalog_id) - except BaseException: - self._artifacts = previous - self._pending_payloads = {} - raise - - def restore(self, snapshot): - records = decode_records(snapshot['records'], artifacts=self.session.artifacts) - with self.session.db.transaction(): - for table in TABLES.values(): - if self.session.db.execute('SELECT 1 FROM ' + table + ' LIMIT 1').fetchone(): - fail('Inert project reconstruction cannot adopt live producer queues.') - self.session.db.execute('DELETE FROM generation_entries') - seen, storage_orders = set(), set() - for envelope in records: - if envelope['family'] != 'generation_history': - continue - for value in envelope['document']['entries']: - entry = value['entry'] - if entry['id'] in seen or value['storage_order'] in storage_orders: - fail('Ambiguous generation entry identity across origins.', 'unsupported', 422) - seen.add(entry['id']) - storage_orders.add(value['storage_order']) - self.session.db.execute('INSERT INTO generation_entries(id,job_id,ordinal,data,storage_order) VALUES(%s,%s,%s,%s,%s)', - (entry['id'], entry['job_id'], entry['ordinal'], encode(entry), value['storage_order'])) - maximum=self.session.db.execute('SELECT COALESCE(max(storage_order),0) FROM generation_entries').fetchone()[0] - self.session.db.execute('SELECT setval(pg_get_serial_sequence(%s,%s),%s,%s)',('generation_entries','storage_order',max(1,maximum),maximum>0)) - self._records = records - all_declarations = _declarations(snapshot['records']) - referenced = _referenced_artifacts(records) - self._artifacts = {sha: all_declarations[sha] for sha in referenced} - return self - - def refresh(self, snapshot): - """Refresh canonical science after proven current custody drainage. - - This never creates producer jobs/admissions, deletes a local job table, - or turns a retained endpoint/identity into current execution authority. - """ - records=decode_records(snapshot['records'],artifacts=self.session.artifacts) - seen=set();orders=set() - for envelope in records: - if envelope['family']=='generation_history': - for value in envelope['document']['entries']: - if value['entry']['id'] in seen or value['storage_order'] in orders: - fail('Ambiguous generation entry identity across origins.','unsupported',422) - seen.add(value['entry']['id']);orders.add(value['storage_order']) - with self.session.db.transaction(): - for envelope in records: - if envelope['family']!='generation_history':continue - for value in envelope['document']['entries']: - entry=value['entry'] - self.session.db.execute('INSERT INTO generation_entries(id,job_id,ordinal,data,storage_order) VALUES(%s,%s,%s,%s,%s) ON CONFLICT(id) DO UPDATE SET job_id=excluded.job_id,ordinal=excluded.ordinal,data=excluded.data,storage_order=excluded.storage_order', - (entry['id'],entry['job_id'],entry['ordinal'],encode(entry),value['storage_order'])) - maximum=self.session.db.execute('SELECT COALESCE(max(storage_order),0) FROM generation_entries').fetchone()[0] - self.session.db.execute('SELECT setval(pg_get_serial_sequence(%s,%s),%s,%s)',('generation_entries','storage_order',max(1,maximum),maximum>0)) - self._records=records - declarations=_declarations(snapshot['records']) - self._artifacts={sha:declarations[sha] for sha in _referenced_artifacts(records)} - return self - - def capture(self): - result = copy.deepcopy(self._records) - by_id = {entry['id']: entry for entry in [json.loads(row[0]) for row in self.session.db.execute('SELECT data FROM generation_entries ORDER BY storage_order')]} - for envelope in result: - if envelope['family'] == 'generation_history': - for value in envelope['document']['entries']: - if value['entry']['id'] not in by_id: - fail('An inert generation entry disappeared from its projection.') - current = copy.deepcopy(by_id.pop(value['entry']['id'])) - if current != value['entry']: - value['history'].append(current) - if 'history_payloads' in value:value['history_payloads'].append(copy.deepcopy(value['payloads'])) - value['entry'] = current - if by_id: - fail('Unclassified generation entries appeared in the project projection.') - result.extend(dict(family='project_artifact', id=sha, document=copy.deepcopy(value)) for sha, value in sorted(self._artifacts.items())) - decode_records(_composed(self.session.snapshot['records'], result), artifacts=self.session.artifacts) - return result - - -def _summary(family, document): - job = document['job'] - return dict(family=family, origin_catalog_id=document['origin_catalog_id'], id=job['id'], - format=document['format'], ordinal=document['ordinal'], status=job['status'], - revision=job.get('revision'), created_at=job.get('created_at'), updated_at=job.get('updated_at'), - entry_count=len(document.get('entries', [])), candidate_count=len(job.get('candidates', [])), - prompt_batch_count=len(document.get('prompt_batches',[]))) - - -def _observation_integrity(value): - return {key: 'unverified_missing_original_wire' for key in ('capability_observation', 'build_observation') if key in value} - - -def _entry_integrity(entry): - result = {field + '.' + key: state for field in ('metadata', 'attempt_evidence') - for key, state in _observation_integrity(entry.get(field, {})).items()} - # Live image generation retains the successful response digest and decoded - # PNG, but not the original response envelope. Do not invent that envelope. - for field in ('metadata', 'attempt_evidence'): - if entry.get(field, {}).get('response_sha256') is not None: - result[field + '.response_sha256'] = 'unverified_missing_original_wire' - return result - - -def _entry_history(value, job): - history = value['history'] - if type(history) is not list or not 1 <= len(history) <= 100_000 or history[-1] != value['entry']: - fail('Invalid generation observation history.') - for entry in history: - _entry(entry, job) - if any(entry[k] != history[0][k] for k in ('id', 'job_id', 'ordinal', 'prompt', 'model', 'width', 'height', 'requested_seed')): - fail('Generation history rewrites its immutable producer intent.') - for previous, current in zip(history, history[1:]): - if previous['status'] == 'deleted': - fail('Deleted generation history cannot resume.') - if current['status'] == 'deleted': - expected = dict(previous, status='deleted', sample_id=None, deleted_at=current.get('deleted_at')) - if previous['status'] != 'completed' or current != expected: - fail('Unsupported generation deletion transition.') - elif previous['status'] == 'completed': - fail('Completed generation history cannot be rewritten.') - - -def _prompt_batches(doc, declarations): - values=doc.get('prompt_batches',[]) - if type(values) is not list or len(values)>10000: - fail('Generation prompt batch observations exceed their bound.') - if values and doc['job']['config']['strategy']!='varied': - fail('Repeat generation cannot import variation observations.') - seen=set();last_offset=0 - for wrapper in values: - shape(wrapper,{'batch','payloads'},name='prompt batch observation') - batch=wrapper['batch'] - shape(batch,{'id','offset','count','status','error','request_sha256','response_sha256','prompts'},name='prompt batch') - identity(batch['id']);digest(batch['request_sha256']) - if batch['id'] in seen:fail('Prompt batch identity repeats.') - seen.add(batch['id']) - integer(batch['offset'],0,doc['job']['config']['count']-1) - integer(batch['count'],1,min(synthetic.BATCH_SIZE,doc['job']['config']['count']-batch['offset'])) - if batch['offset']<last_offset:fail('Prompt batch ordering moves backwards.') - last_offset=batch['offset'] - if batch['status'] not in ('requesting','completed','failed'):fail('Unsupported prompt batch lifecycle.') - text_value(batch['error'],'Prompt batch error',800,empty=True) - if batch['status']=='requesting': - if batch['prompts'] is not None or batch['response_sha256'] is not None or batch['error']: - fail('Unreceived prompt batch contains a claimed result.') - else: - digest(batch['response_sha256']) - if batch['status']=='completed': - if (batch['error'] or type(batch['prompts']) is not list or len(batch['prompts'])!=batch['count'] - or any(type(p) is not str or not p.strip() or len(p)>4000 or p!=p.strip() for p in batch['prompts'])): - fail('Prompt batch result shape differs.') - elif batch['prompts'] is not None or not batch['error']: - fail('Failed prompt batch claims valid prompts.') - _safe(batch) - roles={'canonical_request'} | ({'raw_response'} if batch['response_sha256'] is not None else set()) - if set(wrapper['payloads'])!=roles:fail('Prompt batch exact payload declarations differ.') - _payloads(wrapper['payloads'],declarations,roles) - for role,field in (('canonical_request','request_sha256'),('raw_response','response_sha256')): - if role in wrapper['payloads'] and wrapper['payloads'][role]['artifact']['sha256']!=batch[field]: - fail('Prompt batch payload digest differs.') - - -def _verify_prompt_batches(doc,declarations,artifacts): - entries=[v['entry'] for v in doc['entries']] - expected=None;available=True - for wrapper in doc.get('prompt_batches',[]): - batch=wrapper['batch'] - request=_read(wrapper['payloads']['canonical_request'],declarations,artifacts,32*1024*1024) - response=(_read(wrapper['payloads']['raw_response'],declarations,artifacts,256*1024) - if 'raw_response' in wrapper['payloads'] else None) - if response is not None:_response_values(response) - if request is None: - available=False;continue - parsed=caption_proposals.strict_json(request);_safe(parsed) - try:synthetic.validate_retained_prompt_batch(doc['job']['config'],batch,parsed,response) - except (ValueError,KeyError,TypeError,IndexError):fail('Retained prompt batch wire differs from its queue intent.') - if expected is None:expected=[v['prompt'] for v in entries[:batch['offset']]] - if available and batch['offset']!=len(expected):fail('Prompt batch offset differs from retained unique results.') - if available and json.loads(parsed['messages'][1]['content'])['recent_prompts_to_avoid']!=expected[-synthetic.BATCH_SIZE:]: - fail('Prompt batch context differs from preceding retained queue prompts.') - if batch['status']=='completed' and available: - seen={' '.join(value.casefold().split()) for value in expected} - for prompt in batch['prompts']: - key=' '.join(prompt.casefold().split()) - if key not in seen:expected.append(prompt);seen.add(key) - if available and expected is not None and [v['prompt'] for v in entries]!=expected[:len(entries)]: - fail('Saved generation prompts differ from exact variation results.') - - -def _capture_prompt_batches(owner,job,batches): - result=[] - for original in batches: - batch=copy.deepcopy(original) - request=batch.pop('canonical_request') - raw=batch.pop('raw_response_base64') - synthetic.validate_retained_prompt_batch(job['config'],batch,request,_bytes(raw,256*1024) if raw is not None else None) - _safe(request) - payloads={'canonical_request':_publish(owner,encode(request).encode(),'canonical_request')} - if raw is not None: - data=_bytes(raw,256*1024);_response_values(data) - payloads['raw_response']=_publish(owner,data,'raw_response','application/octet-stream') - result.append(dict(batch=batch,payloads=payloads)) - return result - - -def _referenced_artifacts(records): - result = set() - for envelope in records: - doc = envelope['document'] - mappings = [doc['payloads']] - mappings.extend(value['payloads'] for value in doc.get('prompt_batches',[])) - if envelope['family'] == 'generation_history': - mappings.extend(value['payloads'] for value in doc['entries']) - mappings.extend(mapping for value in doc['entries'] for mapping in value.get('history_payloads',[])) - result.update(edge['artifact']['sha256'] for mapping in mappings for edge in mapping.values()) - return result - - -def _portable_observations(value): - value = copy.deepcopy(value) - if 'build_observation' in value: - build = value['build_observation'] - shape(build, {'advertisement', 'observed_sha256'}, {'server_url'}, 'stored build observation') - build.pop('server_url', None) - return value - - -def _historical_urls(value): - """Known attribution URLs are historical facts, never current resolvers.""" - if type(value) is dict: - for key, child in value.items(): - if key in ('server_url', 'requested_server_url', 'prompt_url'): - import ai_http - ai_http.validate_url('llamacpp', child) - _historical_urls(child) - elif type(value) is list: - for child in value: - _historical_urls(child) - - -def _publish(owner, raw, role, media='application/json'): - declaration = dict(format='project_artifact_v1', sha256=_sha(raw), bytes=len(raw)) - previous = owner._pending_payloads.get(declaration['sha256']) - if previous is not None and previous != raw: - fail('Conflicting exact producer payload bytes.') - owner._pending_payloads[declaration['sha256']] = raw - owner._artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=owner.session.summary['project_id'], sha256=declaration['sha256']), - media_type=media, kind='producer_evidence', role=role, - preservation='unique', required_for=['verify_producer']) - - -def _capture_payloads(owner, family, job): - payloads = {} - for field, role in PAYLOAD_FIELDS.items(): - value = job.pop(field, None) - if value is None: - continue - if field == 'canonical_request': - _safe(value) - raw = encode(value).encode() - else: - maximum = caption_proposals.MAX_IMAGE if role == 'input_image' else image_generation.MAX_JSON if family == 'generation_history' else MAX_RESPONSE - raw = _bytes(value, maximum) - if role == 'raw_response': - _response_values(raw) - payloads[role] = _publish(owner, raw, role, 'image/jpeg' if role == 'input_image' else 'application/octet-stream' if role == 'raw_response' else 'application/json') - return payloads - - -def _response_values(raw): - try: - # The existing chat owner accepts JSON's historical UTF-16 byte - # encodings too. Inspect values without rewriting original bytes. - parsed = caption_proposals.strict_json(raw) - except (UnicodeError, ValueError, RecursionError): - fail('Opaque provider response bytes lack a supported credential-free classification.', 'unsupported', 422) - _safe(parsed) - return parsed - - -def _capture_entry(owner, entry, storage_order): - result, payloads = copy.deepcopy(entry), {} - for field, role in (('attempt_evidence', 'attempt_request'), ('metadata', 'result_request')): - if field in result: - result[field] = _portable_observations(result[field]) - request = result[field].pop('canonical_request', None) - if request is not None: - _safe(request) - raw = encode(request).encode() - if _sha(raw) != result[field]['canonical_request_sha256']: - fail('Generation request body digest differs.') - payloads[role] = _publish(owner, raw, role) - return dict(entry=result, history=[copy.deepcopy(result)], payloads=payloads, storage_order=storage_order, - observation_integrity=_entry_integrity(result)) - - -def _capture_job(owner, family, original, ordinal, entries): - job = copy.deepcopy(original) - prompt_batches = job.pop('prompt_batches',None) if family == 'generation_history' else None - # These are the complete classified local fields, not a denylist fallback. - # All remaining source fields must satisfy the owned closed codec below. - for key in LOCAL_JOB_FIELDS: - if key in job: - (identity if key == 'admission_id' else digest)(job[key]) - job.pop(key, None) - if type(job.get('config')) is not dict: - fail('Missing producer configuration.') - for field in LOCAL_CONFIG_FIELDS: - if field in job['config']: - import ai_http - ai_http.validate_url('llamacpp', job['config'][field]) - job['config'].pop(field, None) - if family != 'caption_proposal_history' and 'input_image_base64' in job: - fail('Unexpected image payload in this producer family.') - job = _portable_observations(job) - if 'source' in job: - _historical_urls(job['source']) - if family == 'grounded_history': - view = job.pop('raw_response', None) - job['raw_response_text_available'] = view is not None - if view is not None: - if type(view) is not str: - fail('Invalid retained grounded response view.') - raw = _bytes(job['raw_response_base64'], MAX_RESPONSE) - if raw.decode('utf-8') != view: - fail('Grounded response text view differs from its exact bytes.') - if family != 'generation_history' and 'canonical_request' not in job: - module = {'grounded_history': grounded_candidates, 'caption_proposal_history': caption_proposals, - 'text_proposal_history': text_classification_proposals}[family] - job['canonical_request'] = module.request_payload(job) if family == 'grounded_history' else module.payload(job) - if _sha(encode(job['canonical_request']).encode()) != job['canonical_request_sha256']: - fail('Reconstructed canonical producer request differs from its stored identity.') - payloads = _capture_payloads(owner, family, job) - document = dict(format=FAMILIES[family], origin_catalog_id=owner._origin, ordinal=ordinal, - job=job, payloads=payloads, observation_integrity=_observation_integrity(job)) - if prompt_batches is not None: - document['prompt_batches'] = _capture_prompt_batches(owner,job,prompt_batches) - if family == 'generation_history': - document['entries'] = [_capture_entry(owner, entry, storage_order) for entry, storage_order in entries if entry['job_id'] == job['id']] - return dict(family=family, id=owner._origin+'.'+job['id'], document=document) - - -def _capture_source(self, *, origin_catalog_id=None): - """Explicit historical extraction primitive; never called by inert Open. - - Reads the five named producer tables. It may publish exact evidence bytes, - but neither alters the source rows nor constructs any producer supervisor. - """ - self._origin = identity(origin_catalog_id or self.session.db.dataset_id) - self._pending_payloads = {} - entries = [] - for row_id, job_id, ordinal, raw, storage_order in self.session.db.execute( - 'SELECT id,job_id,ordinal,data,storage_order FROM generation_entries ORDER BY storage_order'): - entry = json.loads(raw) - if (entry.get('id'), entry.get('job_id'), entry.get('ordinal')) != (row_id, job_id, ordinal): - fail('Generation row identity differs from its observed document.') - entries.append((entry, storage_order)) - records = [] - for family, table in TABLES.items(): - for row_id, raw, ordinal in self.session.db.execute('SELECT id,data,ordinal FROM ' + table + ' ORDER BY ordinal'): - job = json.loads(raw) - if job.get('id') != row_id: - fail('Producer row identity differs from its observed document.') - records.append(_capture_job(self, family, job, ordinal, entries)) - known_jobs = {item['document']['job']['id'] for item in records if item['family'] == 'generation_history'} - if any(entry['job_id'] not in known_jobs for entry, _ in entries): - fail('Generation entry has no retained producer job.') - records.extend(dict(family='project_artifact', id=sha, document=copy.deepcopy(declaration)) - for sha, declaration in sorted(self._artifacts.items())) - composed = _composed(self.session.snapshot['records'], records) - decode_records(composed, artifacts=_CapturedEvidence(self)) - for sha, raw in self._pending_payloads.items(): - self.session.artifacts.publish_bytes(raw, self._artifacts[sha], maximum_bytes=32*1024*1024) - self._pending_payloads = {} - return records - - - -def _producer_document(records, family, job_id, source_id=None): - candidates = [item['document'] for item in decode_records(records) - if item['family'] == family and item['document']['job']['id'] == job_id - and (source_id is None or item['document']['job'].get('source', {}).get('id') == source_id)] - if len(candidates) != 1: - fail('Producer evidence has a missing or ambiguous historical producer closure.') - return candidates[0] - - -def _same_payload(value, document, role): - edge = document['payloads'].get(role) - if edge is None or _sha(encode(value).encode()) != edge['artifact']['sha256']: - fail('Retained request evidence differs from its declared producer payload.') - - -def validate_target_proposal(evidence, record, records): - """Validate owner-generated draft evidence, never mint current approval.""" - if type(evidence) is not dict: - fail('Applied target proposal evidence must be an object.') - text_target = 'source_text_sha256' in evidence - required = {'job_id', 'applied_revision', 'captured_revision', 'source_revision', 'source_sha256', - 'config', 'prompt_version', 'prompt_sha256', 'canonical_request_sha256', - 'response_sha256', 'reported_model', 'verification'} - required |= {'source_text_sha256', 'labels_sha256'} if text_target else {'canonical_image_sha256', 'input_image'} - shape(evidence, required, EVIDENCE_FIELDS - required, 'applied proposal evidence') - _safe(evidence); _historical_urls(evidence) - family = 'text_proposal_history' if text_target else 'caption_proposal_history' - doc = _producer_document(records, family, identity(evidence['job_id']), record['id']) - expected_verification = ('Exact label shape only; applied as draft, never approved.' if text_target - else 'Caption shape only; applied as draft, never approved.') - if evidence['verification'] != expected_verification: - fail('Applied target verification qualification differs from its owner.') - job, source = doc['job'], doc['job']['source'] - for key in ('prompt_version', 'prompt_sha256', 'canonical_request_sha256', 'response_sha256'): - if evidence[key] != job[key]: - fail('Applied target evidence differs from its producer ' + key + '.') - if type(evidence['config']) is not dict: - fail('Applied target configuration must be an object.') - config = {key: value for key, value in evidence['config'].items() if key not in LOCAL_CONFIG_FIELDS} - if config != job['config'] or evidence['reported_model'] != job.get('reported_model'): - fail('Applied target evidence rewrites its producer configuration/result.') - if (type(evidence['applied_revision']) is not int or evidence['applied_revision'] != source['revision'] + 1 - or evidence['captured_revision'] != source['revision'] - or evidence['source_revision'] != source['source_revision'] - or evidence['source_sha256'] != source['source_sha256'] - or evidence['applied_revision'] > record['revision']): - fail('Applied target evidence belongs to different captured revisions.') - if job['status'] != 'applied' or job['application'] != { - 'record_id': record['id'], 'revision': evidence['applied_revision'], 'source_revision': evidence['source_revision']}: - fail('Applied target lacks its historical application disposition.') - if record['revision'] == evidence['applied_revision'] and ( - record['annotation'] != job['annotation'] or record['review'] != 'draft' - or record['task'] != ('text_classification' if text_target else 'image_caption')): - fail('A proposal application cannot establish human review or a different target.') - if text_target: - if evidence['source_text_sha256'] != job['source_text_sha256'] or evidence['labels_sha256'] != job['labels_sha256']: - fail('Applied target source/label identity differs.') - elif evidence['canonical_image_sha256'] != source['content_hash'] or evidence['input_image'] != job['input_image']: - fail('Applied target prepared/oriented image identity differs.') - for key in EVIDENCE_FIELDS - {'canonical_request', 'canonical_request_sha256', 'response_sha256'}: - if key in evidence and evidence[key] != job.get(key): - fail('Applied target provider observation differs.') - if 'canonical_request' in evidence: - _same_payload(evidence['canonical_request'], doc, 'canonical_request') - return True - - -def validate_record_origin(origin, record, records): - """Recognize only the two producer provenance variants owned here.""" - if type(origin) is not dict or origin.get('method') not in ('model_generated', 'model_grounded_rewrite'): - return False - _safe(origin); _historical_urls(origin) - if origin['method'] == 'model_generated': - shape(origin, {'method', 'source_sha256', 'rights', 'generation'}, {'rights_note_correction'}, 'generated source provenance') - generation = origin['generation'] - shape(generation, {'job_id', 'ordinal', 'prompt', 'model', 'width', 'height', 'requested_seed'}, {'metadata', 'size'}, 'generated source binding') - docs = [item['document'] for item in decode_records(records) if item['family'] == 'generation_history' - and item['document']['job']['id'] == generation['job_id']] - matched = [] - for doc in docs: - for wrapper in doc['entries']: - for entry in wrapper['history']: - if entry['sample_id'] == record['id'] and entry['ordinal'] == generation['ordinal']: - matched.append((wrapper, entry)) - if len(matched) != 1: - fail('Generated source lacks an exact producer entry/source closure.') - wrapper, entry = matched[0] - for key in ('job_id', 'ordinal', 'prompt', 'model', 'width', 'height', 'requested_seed', 'size'): - if key in generation and generation[key] != entry.get(key): - fail('Generated source intent differs from its retained entry.') - if 'metadata' in generation: - metadata = copy.deepcopy(generation['metadata']) - request = metadata.pop('canonical_request', None) - if request is not None: - edge = wrapper['payloads'].get('result_request') - if edge is None or _sha(encode(request).encode()) != edge['artifact']['sha256']: - fail('Generated source result request differs.') - if _portable_observations(metadata) != entry.get('metadata'): - fail('Generated source result metadata differs.') - if record['kind'] != 'image' or origin['source_sha256'] not in {record['source_sha256'], *( - row['sha256'] for item in records if item['family'] == 'workbench_record' and item['id'] == record['id'] - for row in item['document']['source'].get('revisions', []))}: - fail('Generated source hash differs from its retained image lineage.') - else: - required = {'method', 'rights', 'normalization', 'provider', 'server_url', 'requested_model', 'requested_seed', - 'instruction', 'system_prompt', 'prompt_version', 'prompt_sha256', 'reported_model', - 'canonical_request_sha256', 'requested_count', 'max_tokens', 'job_id', 'candidate_id', - 'response_sha256', 'source', 'evidence', 'verification', 'admission_note'} - shape(origin, required, EVIDENCE_FIELDS | {'profile', 'rights_note_correction'}, 'grounded source provenance') - doc = _producer_document(records, 'grounded_history', identity(origin['job_id'])) - job = doc['job'] - values = [candidate for candidate in job['candidates'] if candidate['id'] == origin['candidate_id']] - if len(values) != 1: - fail('Grounded source candidate closure is missing.') - candidate = values[0] - if candidate['status'] != 'admitted' or candidate['record_id'] != record['id'] or candidate['text'] != record['text']: - fail('Grounded source differs from its reviewed candidate.') - for key, expected in {'requested_model': job['config']['model'], 'requested_seed': job['config']['seed'], - 'instruction': job['config']['instruction'], 'requested_count': job['config']['count'], 'max_tokens': 6000, - 'system_prompt': job['system_prompt'], 'prompt_version': job['prompt_version'], - 'prompt_sha256': job['prompt_sha256'], 'canonical_request_sha256': job['canonical_request_sha256'], - 'response_sha256': job['response_sha256'], 'reported_model': job.get('reported_model'), - 'evidence': candidate['evidence'], 'admission_note': candidate['review_note'], 'verification': job['verification']}.items(): - if origin[key] != expected: - fail('Grounded source provenance differs from its producer ' + key + '.') - if (origin['provider'] != (pumas.PROTOCOL if pumas.typed(job['config']) else 'pumas_chat_compatible') - or origin['normalization'] != 'NFC; LF newlines' - or pumas.typed(job['config']) and origin.get('profile') != job['config'].get('profile')): - fail('Grounded source producer or normalization attribution differs.') - if record['revision'] == 1 and (record['review'] != 'draft' or record['task'] != 'text_classification' - or record['annotation'] != {'label':candidate['label']}): - fail('Grounded admission cannot establish human review or a different target.') - for key in EVIDENCE_FIELDS - {'canonical_request', 'canonical_request_sha256', 'response_sha256'}: - if key in origin and origin[key] != job.get(key): - fail('Grounded source provider observation differs.') - expected_source = {key: job['source'][key] for key in ('id', 'revision', 'source_revision', 'content_hash', 'text', 'annotation')} - if origin['source'] != expected_source: - fail('Grounded source provenance rewrites its captured input.') - if 'canonical_request' in origin: - _same_payload(origin['canonical_request'], doc, 'canonical_request') - return True - - -def validate_operation_result(operation, target, result, records): - from project_producer_execution import validate_operation_result as validate_action - return validate_action(operation, target, result, records) - - -def _composed(current, upserts): - result = {(item['family'], item['id']): item for item in current} - result.update({(item['family'], item['id']): item for item in upserts}) - return [result[key] for key in sorted(result)] - - -def _source_closure(producers, records): - assets = {item['id']: item['document'] for item in records - if item['family'] in ('workbench_record', 'workbench_text')} - for envelope in producers: - doc, family = envelope['document'], envelope['family'] - job = doc['job'] - if family == 'generation_history': - for wrapper in doc['entries']: - for entry in wrapper['history']: - if entry['sample_id'] is not None and entry['sample_id'] not in assets: - fail('Generated source canonical record is missing.') - continue - source = job['source'] - asset = assets.get(source['id']) - if asset is None or source not in asset.get('history', []): - fail('Producer captured source differs from its retained Workbench history.') - if family == 'grounded_history': - for candidate in job['candidates']: - if candidate['record_id'] is not None: - target = assets.get(candidate['record_id']) - if target is None or target['current']['text'] != candidate['text']: - fail('Admitted grounded candidate canonical record is missing or differs.') - - -class _CapturedEvidence: - """Immutable source buffers prove owner semantics before any publication.""" - def __init__(self, owner): - self.owner = owner - - def status(self, declaration): - raw = self.owner._pending_payloads.get(declaration['sha256']) - if raw is None: - return self.owner.session.artifacts.status(declaration) - if len(raw) != declaration['bytes'] or _sha(raw) != declaration['sha256']: - fail('Captured producer payload identity differs.') - return {'state': 'verified'} - - def read_blob(self, descriptor, *, maximum_bytes): - raw = self.owner._pending_payloads.get(descriptor['sha256']) - if raw is None: - return self.owner.session.artifacts.read_blob(descriptor, maximum_bytes=maximum_bytes) - if len(raw) != descriptor['bytes'] or len(raw) > maximum_bytes: - fail('Captured producer payload exceeds its owner bound.') - return raw diff --git a/project_projection.py b/project_projection.py deleted file mode 100644 index e7aa8366..00000000 --- a/project_projection.py +++ /dev/null @@ -1,434 +0,0 @@ -"""Concrete inert composition of external-project projection owners. - -No registry, worker constructor, process recovery, dependency installation or -project-script import belongs here. The fixed codecs validate the same candidate -checkpoint before reconstruction and before each canonical publication. -""" -import copy -import hashlib -from datetime import datetime -import re -from types import SimpleNamespace - -from project_artifacts import ProjectArtifacts, validate_declaration -from project_data import ProjectData, FAMILIES as DATA_FAMILIES, decode_data_records -from project_data import validate_operation_result as validate_data_result -from project_history import ProjectHistory, FAMILY_FORMATS as HISTORY_FAMILIES -from project_history import decode_records as decode_history_records -from project_history import validate_operation_result as validate_history_result -from project_material import ProjectMaterial, MaterialSnapshot, FAMILIES as MATERIAL_FAMILIES -from project_material import validate_operation_result as validate_material_result -from project_producers import ProjectProducers, FAMILIES as PRODUCER_FAMILIES -from project_producers import decode_records as decode_producer_records -from project_session import session_records_for -from project_session import ProjectSession, operation_documents, operation_records -from project_locks import ProjectLock -from project_execution import ProjectExecution, validate_operation_result as validate_execution_result -from project_producer_execution import ProjectProducerExecution, validate_operation_result as validate_producer_result -from project_files import ProjectFiles, decode_records as decode_file_records -from project_files import validate_operation_result as validate_file_result -from project_files import receipt_validator as file_receipt_validator -from project_study_execution import ProjectStudyExecution, validate_operation_result as validate_study_execution_result -from project_runtime import ProjectRuntime, validate_operation_result as validate_runtime_result -from project_native_execution import FAMILY as NATIVE_FAMILY, decode_records as decode_native_records -from project_native_preparation import ProjectNativePreparation, validate_operation_result as validate_preparation_result -from project_sources import SourceAdapter, validate_operation_result as validate_source_result -from project_sources import receipt_validator as source_receipt_validator -from project_measurements import ProjectMeasurements, FAMILY as MEASUREMENT_FAMILY -from project_measurements import decode_records as decode_measurement_records -from project_measurements import validate_operation_result as validate_measurement_result -from project_measurements import receipt_validator as measurement_receipt_validator -from workbench import WorkbenchError -from workbench import timestamp -from project_session import session_record - - -def validate_artifact_resolution(operation, result, records): - if operation != 'artifacts.resolve': - return False - if (type(result) is not dict or set(result) != { - 'format', 'sha256', 'bytes', 'actor_id', 'observed_at'} - or result['format'] != 'project_artifact_resolution_v1' - or type(result['actor_id']) is not str or not re.fullmatch('[a-f0-9]{32}', result['actor_id']) - or type(result['observed_at']) is not str): - raise WorkbenchError('Invalid artifact resolution observation.') - try: - if datetime.fromisoformat(result['observed_at']).utcoffset() is None: - raise ValueError() - except ValueError: - raise WorkbenchError('Artifact observation timestamp requires an offset.') from None - declaration = validate_declaration(dict(format='project_artifact_v1', sha256=result['sha256'], bytes=result['bytes'])) - if not any(item['family'] == 'project_artifact' and item['id'] == declaration['sha256'] - and item['document'] == declaration for item in records): - raise WorkbenchError('Artifact resolution observation has no exact declaration.') - return True - - -def validate_project_records(records, *, project_id, artifacts): - """Validate one inert candidate through every owning domain, without SQL reconstruction. - - Under an edit delta (project_validation_delta.edit_delta) the candidate is - a validated base checkpoint plus changed records; components and receipts - that cannot be affected are skipped. TULDOK_VALIDATION_CROSSCHECK=1 also - runs complete validation and requires the same outcome. - """ - from project_validation_delta import current_delta - delta = current_delta() - if delta is None: - return _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=None) - import os - if os.environ.get('TULDOK_VALIDATION_CROSSCHECK') == '1': - full_error = scoped_error = None - try: - _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=None) - except Exception as error: - full_error = error - try: - _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=delta) - except Exception as error: - scoped_error = error - if (full_error is None) != (scoped_error is None): - raise AssertionError(f'Edit-scoped validation disagrees with complete validation: ' - f'complete={full_error!r} scoped={scoped_error!r}') - if full_error is not None: - raise full_error - return None - return _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=delta) - - -def _composition_members(records): - members = set() - for item in records: - if item['family'] != 'project_composition': - continue - for source in item['document'].get('sources', ()): - for family, entries in source.get('records', {}).items(): - members.update((family, identity) for identity in entries) - members.update(('run_observation', sha) for sha in source.get('run_observations', {}).values()) - return members - - -def _validate_project_records(records, *, project_id, artifacts, delta): - def needed(component): - return delta is None or delta.touches(component) - claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( - MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) - identities = set() - declarations = {} - for item in records: - identity = (item['family'], item['id']) - if item['family'] not in claimed or identity in identities: - raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) - identities.add(identity) - if item['family'] == 'project_artifact': - declaration = validate_declaration(item['document']) - if declaration['sha256'] != item['id']: - raise WorkbenchError('Artifact inventory identity differs.') - declarations[item['id']] = declaration - if needed('data'): - decode_data_records(records) - from project_fork import decode_fork - decode_fork(records, project_id=project_id) - if needed('composition') or (delta is not None and delta.changed & _composition_members(records)): - from project_composition import decode_records as decode_compositions - decode_compositions(records, project_id=project_id) - if needed('extraction'): - from project_transfer import decode_records as decode_extraction_records - decode_extraction_records(records) - operation_records(records, project_id=project_id) - if needed('native'): - decode_native_records(records,artifacts=artifacts) - material_cache = [] - def materials(): - if not material_cache: - material_cache.append(MaterialSnapshot(records)) - return material_cache[0] - if needed('material'): - for blob in materials().artifact_blobs(): - if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): - raise WorkbenchError('Material artifact declaration is missing or inconsistent.') - # Resolve every cross-owner reference against THIS candidate, rather - # than a resolver that still sees the preceding committed checkpoint. - if needed('producers'): - decode_producer_records(records, artifacts=artifacts) - if needed('history') or (delta is not None and delta.history_receipt_added()): - decode_history_records(records, materials=materials(), artifacts=artifacts) - validators = {} - def lazy(name, component, claims, build): - # claims(operation) is exactly what the built validator can accept; - # for any other operation it would return False, so it is not built. - def validate(operation, result): - if not claims(operation): - return False - if name not in validators: - validators[name] = build() - return validators[name](operation, result) - if delta is None or needed(component): - validators[name] = build() - return validate - validate_file_receipt = lazy('file', 'files', lambda op: op.startswith('files.'), - lambda: file_receipt_validator(records)) - validate_measurement_receipt = lazy('measurement', 'measurements', lambda op: op == 'measurements.import', - lambda: measurement_receipt_validator(records, artifacts=artifacts)) - validate_source_receipt = lazy('source', 'sources', lambda op: op == 'source.register', - lambda: source_receipt_validator(records)) - from project_session import OPERATION_FAMILY - for receipt in operation_documents(records): - operation, result = receipt['operation'], receipt['result'] - if delta is not None: - identity = (OPERATION_FAMILY, receipt['project_id'] + '.' + hashlib.sha256(receipt['request_key'].encode('utf-8')).hexdigest() - if receipt.get('format') == 'project_operation_v2' else - hashlib.sha256(receipt['request_key'].encode('utf-8')).hexdigest()) - if not delta.receipt_needed(operation, identity): - continue - if (validate_data_result(operation, result, records) - or validate_material_result(operation, result, records) - or validate_history_result(operation, result, records) - or validate_source_receipt(operation, result) - or validate_measurement_receipt(operation, result) - or validate_execution_result(operation, result, records) - or validate_producer_result(operation, receipt['target'], result, records) - or validate_file_receipt(operation, result) - or validate_study_execution_result(operation, result, records) - or validate_runtime_result(operation, result, records) - or validate_preparation_result(operation, result, records) - or validate_artifact_resolution(operation, result, records)): - continue - raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) - - -class ProjectProjection: - def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None, - validation_cache=None): - self.store, self.db, self.access = store, db, access - self.custody = custody - self.pending_original = [] - self.project_id = store.summary()['project_id'] - self.artifacts = ProjectArtifacts(store) - self.sources = None - self._disclosures = set() - # Only installation configuration supplies these aliases. Neither a - # project descriptor nor an HTTP request supplies filesystem roots. - self._runtime_sources = copy.deepcopy(runtime_sources or {}) - try: - self.session = ProjectSession(store, db, access, self.artifacts, - validate_records=self.validate_records, - validation_cache=validation_cache) - # Install before domain constructors retain their lock references. - db.lock = ProjectLock(access.lock, self.session.lock, db.lock) - self.producers = ProjectProducers(self.session) - self.data = ProjectData(self.session, generation_entries=self.producers.generation_jobs) - self.materials = ProjectMaterial(self.session) - self.measurements = ProjectMeasurements(self.session) - self.history = ProjectHistory(self.session, materials=self.materials, - authorize_disclosure=self.authorize_disclosure) - if restore: - # Producer restore reads its families, declarations and forks. - self.producers.restore(dict(records=session_records_for( - self.session, *PRODUCER_FAMILIES, 'project_artifact', 'project_fork'))) - self.data.restore() - self.materials.restore() - # These owners create only disposable local directories. Their - # semantic capture/preparation methods run solely on explicit calls. - from projects import Projects - from model_definitions import ModelDefinitions - from pipeline import Pipelines - self.context = SimpleNamespace(path=db.path, db=db, lock=db.lock, - access=self.materials.owner_access, - workbench=self.data.workbench, - releases=self.data.releases, runs=self.history) - self.sources = Projects(self.context, roots) - self.context.projects = self.sources - self.context.pipelines = self.pipelines = Pipelines(self.context) - self.context.models = self.models = ModelDefinitions(self.context) - self.source_adapter = SourceAdapter(self) - self.text = _TextView(self) - self.execution = ProjectExecution(self) - self.native_preparation=ProjectNativePreparation(self) - self.context.project_native_preparation=self.native_preparation - self.runtime = ProjectRuntime(self, resolve_configuration=self.runtime_configuration) - self.execution.producer_execution = ProjectProducerExecution( - self.session, self.data, self.producers, - runtime_access=self.execution.runtime_access, execution=self.execution) - self.files = ProjectFiles(self.session) - self.study_execution = ProjectStudyExecution(self, self.execution) - if restore: - self.study_execution.restore_authoring() - else: - # Explicit original-host recovery validates custody before any - # science cache reconstruction or local execution constructor. - self.session.available = False - except BaseException: - self.close() - raise - - @property - def schema(self): - return self.db.schema - - def execute(self, *args, **kwargs): - return self.db.execute(*args, **kwargs) - - def validate_records(self, records, *, project_id=None, artifacts=None): - return validate_project_records(records, - project_id=self.project_id if project_id is None else project_id, - artifacts=self.artifacts if artifacts is None else artifacts) - - def resolve_artifact(self, body, expected_checkpoint, key): - """Explicitly supply exact bytes from a configured current local root.""" - if type(body) is not dict or set(body) != {'sha256', 'root', 'path'}: - raise WorkbenchError('Choose one declared artifact and a current rooted file.') - with self.access.lock, self.session.lock, self.db.lock: - actor = self.access.require('edit', 'artifacts.resolve', body['sha256'], owner_only=True) - self.session.check_current() - declaration = session_record(self.session, 'project_artifact', body['sha256']) - declaration = None if declaration is None else declaration['document'] - if declaration is None: - raise WorkbenchError('Artifact is not declared by this project.', 'not_found', 404) - def invoke(): - with self.sources._file(body['root'], body['path']) as (descriptor, info): - if info.st_size != declaration['bytes']: - raise WorkbenchError('Supplied file length differs from the declared artifact.', 'conflict', 409) - self.artifacts.publish_fd(descriptor, declaration) - return dict(format='project_artifact_resolution_v1', sha256=declaration['sha256'], - bytes=declaration['bytes'], actor_id=actor.actor_id, observed_at=timestamp()) - return self.session.mutate('artifacts.resolve', body['sha256'], body, expected_checkpoint, key, - invoke, lambda result, before: dict(records=[], result=result)) - - def runtime_manifest(self, descriptor): - """Read only a current local registration during explicit preparation. - - An imported descriptor's root is never a filesystem capability. Captured - manifests already in canonical artifacts need no current registration. - """ - from pathlib import Path - import hashlib - if descriptor.get('catalog_id') != self.project_id: - raise WorkbenchError('Register this runtime under the current project before preparation.', - 'unavailable', 409) - if descriptor.get('profile') == 'captured_cpu_python_v1': - from runtime_snapshots import RuntimeSnapshots - from recipe_process import read_regular, CPU_MANIFEST_LIMIT - with self.execution.runtime_inspection(): - current = RuntimeSnapshots(self.db, self.execution.runtime_access, self.db.path).get( - descriptor['manifest_sha256'], verify=False) - if current != descriptor: - raise WorkbenchError('Current runtime registration differs from the selected descriptor.', 'stale', 409) - raw = read_regular(Path(current['root']) / 'manifest.json', CPU_MANIFEST_LIMIT) - elif descriptor.get('profile') == 'managed_torch_python_v1': - from torch_runtimes import TorchRuntimes - current, raw = TorchRuntimes(self.db, self.access, self.db.path).get( - descriptor['manifest_sha256'], verify=False) - if current != descriptor: - raise WorkbenchError('Current runtime registration differs from the selected descriptor.', 'stale', 409) - else: - raise WorkbenchError('Unsupported current runtime registration.', 'unsupported', 409) - if len(raw) != descriptor['manifest_bytes'] or hashlib.sha256(raw).hexdigest() != descriptor['manifest_sha256']: - raise WorkbenchError('Current runtime manifest changed during capture.', 'stale', 409) - return raw - - def runtime_configuration(self, request): - if (type(request) is not dict or set(request) != {'source'} - or type(request['source']) is not str or request['source'] not in self._runtime_sources): - raise WorkbenchError('Choose an explicitly configured current runtime source.', 'unavailable', 409) - return copy.deepcopy(self._runtime_sources[request['source']]) - - def runtime_sources(self): - with self.access.lock, self.session.lock, self.db.lock: - self.access.require('admin', 'runtime.sources', owner_only=True) - self.session.check_current() - return dict(sources=sorted(self._runtime_sources)) - - def authorize_disclosure(self, run, population): - self.access.require('inspect', 'project_history.artifact', run['id']) - self.access.require('expose-holdout', 'project_history.disclosure', run['id']) - origin = run['dataset_id'] - digest = self.history.payload_set_sha256(run['id'], origin_catalog_id=origin) - actor = self.access._context().actor_id - if (actor, origin, run['id'], digest) not in self._disclosures: - raise WorkbenchError('Explicitly disclose this retained Run under current local authority first.', - 'forbidden', 403) - - def disclose(self, run_id, body, expected_checkpoint, key): - with self.access.lock, self.session.lock, self.db.lock: - result = self.history.record_disclosure(run_id, body, expected_checkpoint, key) - actor = self.access._context().actor_id - self._disclosures.add((actor, result['origin_catalog_id'], run_id, result['payload_set_sha256'])) - return result - - def reconcile_inert(self, snapshot): - """Rebuild selected science after actual local custody has drained. - - This preserves existing local Run/admission rows solely for the current - reconciler's terminal proof. No imported authority table is populated. - """ - execution = self.execution - with self.access.lock, self.session.lock, self.db.lock: - if execution._reconstructing: - raise RuntimeError('Project reconstruction already has an owner.') - execution._reconstructing = True - try: - self.session.reconcile_selection(snapshot) - execution.abandon_pending() - self.study_execution.abandon_pending() - execution.producer_execution.abandon_pending() - self.producers.refresh(snapshot) - self.data.restore(snapshot) - self.materials.reconcile_projection() - self.study_execution.restore_authoring(snapshot['records']) - except BaseException: - self.session.available = False - raise - finally: - execution._reconstructing = False - - def close(self): - execution = getattr(self, 'execution', None) - if execution is not None: - execution.close() - if self.sources is not None: - self.sources.close() - self.artifacts.close() - self._disclosures.clear() - self.db.close() - - -class _TextView: - """Preserve the M1 text wire projection over the complete data owner.""" - def __init__(self, projection): - self.projection = projection - - @property - def available(self): - return self.projection.session.available - - def _tag(self, value): - summary = self.projection.session.summary - return dict(value, project_id=summary['project_id'], checkpoint=summary['checkpoint']) - - def records(self): - self.projection.session.check_current() - from project_fork import active_records - snapshot = self.projection.session.snapshot - records = active_records(snapshot['records'], project_id=snapshot['project_id'], - families=frozenset(('workbench_record', 'workbench_text'))) - return self._tag(dict(items=[copy.deepcopy(item['document']['current']) for item in records - if item['family'] in ('workbench_record', 'workbench_text') - and item['document']['current']['kind'] == 'text'])) - - def get(self, identity): - result = self.projection.data.get(identity) - if result['kind'] != 'text': - raise WorkbenchError('Use the modality-aware data operation for this record.', 'unsupported', 409) - return self._tag(result) - - def history(self, identity): - self.get(identity) - return self.projection.data.history(identity) - - def import_text(self, body, expected_checkpoint, key): - return self._tag(self.projection.data.import_text(body, expected_checkpoint, key)) - - def annotate(self, identity, body, expected_checkpoint, key): - self.get(identity) - return self._tag(self.projection.data.annotate(identity, body, expected_checkpoint, key)) diff --git a/project_record_dedupe.py b/project_record_dedupe.py deleted file mode 100644 index aff3948d..00000000 --- a/project_record_dedupe.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Physical deduplication of retained runtime closures in Run history records. - -Retained external-recipe Runs each embed the same captured runtime closure three -times: run.spec.runtime_files, run.material.runtime and a contiguous block of -'historical_runtime' payload edges (about 2.5 MB per Run). The store keeps one -content-addressed `run_runtime_closure` record and a compact Run document -(`project_run_history_v2`) that references it. Stores expand compact Runs back -to the exact logical v1 document on read, so domain owners and validators see -unchanged documents. A Run is compacted only when expansion reproduces its -canonical bytes exactly. -""" -import copy -import hashlib - -RUN_FAMILY = 'run_history' -RUN_V1 = 'project_run_history_v1' -RUN_V2 = 'project_run_history_v2' -CLOSURE_FAMILY = 'run_runtime_closure' -CLOSURE_FORMAT = 'project_run_runtime_closure_v1' -RUNTIME_ROLE = 'historical_runtime' - - -def _encode(value): - from project_store import _encode as canonical - return canonical(value) - - -def closure_id(closure): - """Content address of a closure document.""" - return hashlib.sha256(_encode(closure)).hexdigest() - - -def compact(document): - """(compact v2 document, closure document) or None when not exactly reversible.""" - if type(document) is not dict or document.get('format') != RUN_V1: - return None - run = document.get('run') - payloads = document.get('payloads') - if type(run) is not dict or type(payloads) is not list: - return None - spec, material = run.get('spec'), run.get('material') - if (type(spec) is not dict or type(material) is not dict or 'runtime_files' not in spec - or 'runtime' not in material or 'runtime_closure' in document): - return None - positions = [index for index, edge in enumerate(payloads) - if type(edge) is dict and edge.get('role') == RUNTIME_ROLE] - if not positions or positions != list(range(positions[0], positions[-1] + 1)): - return None - start, end = positions[0], positions[-1] + 1 - closure = dict(format=CLOSURE_FORMAT, runtime_files=copy.deepcopy(spec['runtime_files']), - runtime=copy.deepcopy(material['runtime']), payload_block=copy.deepcopy(payloads[start:end])) - reduced = copy.deepcopy(document) - del reduced['run']['spec']['runtime_files'] - del reduced['run']['material']['runtime'] - reduced['payloads'] = reduced['payloads'][:start] + reduced['payloads'][end:] - reduced['format'] = RUN_V2 - reduced['runtime_closure'] = dict(id=closure_id(closure), payload_index=start) - if _encode(expand(reduced, closure)) != _encode(document): - return None - return reduced, closure - - -def expand(document, closure): - """The exact logical v1 Run document of a compact v2 document.""" - reference = document['runtime_closure'] - if closure.get('format') != CLOSURE_FORMAT or closure_id(closure) != reference['id']: - from project_store import _fail - _fail('Run runtime closure differs from its reference.') - result = copy.deepcopy(document) - del result['runtime_closure'] - result['format'] = RUN_V1 - result['run']['spec']['runtime_files'] = copy.deepcopy(closure['runtime_files']) - result['run']['material']['runtime'] = copy.deepcopy(closure['runtime']) - index = reference['payload_index'] - if type(index) is not int or not 0 <= index <= len(result['payloads']): - from project_store import _fail - _fail('Invalid Run runtime closure position.') - result['payloads'] = result['payloads'][:index] + copy.deepcopy(closure['payload_block']) + result['payloads'][index:] - return result - - -def is_compact(family, document): - return family == RUN_FAMILY and type(document) is dict and document.get('format') == RUN_V2 - - -def logical_records(physical): - """Logical envelopes of physical records: closures hidden, compact Runs expanded.""" - closures = {item['id']: item['document'] for item in physical if item['family'] == CLOSURE_FAMILY} - result = [] - for item in physical: - if item['family'] == CLOSURE_FAMILY: - continue - document = item['document'] - if is_compact(item['family'], document): - closure = closures.get(document['runtime_closure']['id']) - if closure is None: - from project_store import _fail - _fail('Compact Run references a missing runtime closure.') - document = expand(document, closure) - result.append(dict(family=item['family'], id=item['id'], document=document)) - return result - - -def physical_records(logical, *, existing_closures=None): - """Physical envelopes for logical records, compacting Runs where exact. - - existing_closures maps closure id -> document already stored; compacted - Runs reference them and every referenced closure is included once. - """ - closures = dict(existing_closures or {}) - used = set() - result = [] - for item in logical: - document = item['document'] - if item['family'] == RUN_FAMILY: - compacted = compact(document) - if compacted is not None: - document, closure = compacted - identity = document['runtime_closure']['id'] - closures.setdefault(identity, closure) - used.add(identity) - result.append(dict(family=item['family'], id=item['id'], document=document)) - result.extend(dict(family=CLOSURE_FAMILY, id=identity, document=closures[identity]) for identity in sorted(used)) - return sorted(result, key=lambda item: (item['family'], item['id'])) diff --git a/project_runtime.py b/project_runtime.py deleted file mode 100644 index d14c59d4..00000000 --- a/project_runtime.py +++ /dev/null @@ -1,320 +0,0 @@ -"""Explicit current-host CPU runtime capture through a canonical effect intent. - -Open never constructs this adapter's filesystem capabilities from retained roots. -The injected resolver supplies current trusted local configuration only. Captured -binaries remain an ignored local cache; portable metadata declares exact bytes. -""" -import copy -from datetime import datetime -import hashlib -import os - -import agent_api -from model_definitions import reference -from project_material import decode_records as material_records -from project_session import operation_records, REQUEST_KEY, _digest as operation_digest -from runtime_snapshots import RuntimeSnapshots, DEFAULT_RESERVE, MAX_FILES, MAX_BYTES, MAX_MANIFEST -from workbench import WorkbenchError, timestamp -from project_session import session_operations - -OPERATIONS = frozenset(('runtime.capture_intent', 'runtime.capture', 'runtime.abandon')) -RESULT_FIELDS = frozenset(('format','project_id','manifest_sha256','file_count','total_bytes', - 'actor_id','observed_at','request_key')) - - -def _fail(message, code='invalid', status=400): - raise WorkbenchError(message, code, status) - - -def _sha(value): - if type(value) is not str or len(value) != 64 or any(c not in '0123456789abcdef' for c in value): - _fail('Runtime capture requires an exact SHA256.') - - -def _id(value): - if type(value) is not str or len(value) != 32 or any(c not in '0123456789abcdef' for c in value): - _fail('Runtime capture requires a project or actor UUID.') - - -def _key(value): - if type(value) is not str or not REQUEST_KEY.fullmatch(value): - _fail('Invalid runtime capture request key.') - - -def _intent_key(key): - return 'runtime-intent.' + hashlib.sha256(key.encode('utf-8')).hexdigest() - - -def _body(config_request, expected_inventory_sha256, reserve_bytes): - _sha(expected_inventory_sha256) - if type(reserve_bytes) is not int or not 0 <= reserve_bytes <= 2**63-1: - _fail('Runtime capture reserve must be a nonnegative integer.') - try: - if type(config_request) is not dict or len(agent_api.encoded(config_request)) > 65536: - raise ValueError() - except (agent_api.AgentAPIError, TypeError, ValueError, RecursionError, UnicodeError): - _fail('Runtime capture requires a bounded current-local configuration request.') - return dict(configuration=copy.deepcopy(config_request), expected_inventory_sha256=expected_inventory_sha256, - reserve_bytes=reserve_bytes) - - -def _intent_input(body): - """Use the canonical intent owner's exact request identity for local custody.""" - return operation_digest(dict(operation='runtime.capture_intent', - target=body['expected_inventory_sha256'], body=body)) - - -def _summary(project_id, observed, actor_id, key, *, runtime_ref=None): - value = dict(format='project_runtime_capture_intent_v1' if runtime_ref is None else 'project_runtime_capture_v1', - project_id=project_id,manifest_sha256=observed['inventory_sha256'],file_count=observed['file_count'], - total_bytes=observed['total_bytes'],actor_id=actor_id,observed_at=timestamp(),request_key=key) - if runtime_ref is not None: value['runtime_ref'] = copy.deepcopy(runtime_ref) - return value - - -def validate_operation_result(operation, result, records=None): - """Validate inert capture facts; no retained path becomes a local binding.""" - if operation not in OPERATIONS: return False - try: - if operation == 'runtime.abandon': - fields = {'format','project_id','capture_key','input_sha256','manifest_sha256', - 'actor_id','reason','observed_checkpoint','observed_at'} - if type(result) is not dict or set(result) != fields or result['format'] != 'project_runtime_capture_abandon_v1': - _fail('Invalid runtime abandonment observation.') - _id(result['project_id']); _id(result['actor_id']); _key(result['capture_key']) - for field in ('input_sha256','manifest_sha256','observed_checkpoint'): _sha(result[field]) - if type(result['reason']) is not str or not result['reason'].strip() or len(result['reason']) > 1000: - _fail('Runtime abandonment requires the retained owner reason.') - if type(result['observed_at']) is not str or datetime.fromisoformat(result['observed_at']).utcoffset() is None: - _fail('Runtime abandonment requires an offset timestamp.') - if records is not None: - intents = [r['document'] for r in records if r['family']=='project_operation' - and r['document']['operation']=='runtime.capture_intent' - and r['document']['request_key']==_intent_key(result['capture_key']) - and r['document']['input_sha256']==result['input_sha256'] - and r['document']['result'].get('project_id')==result['project_id'] - and r['document']['result'].get('manifest_sha256')==result['manifest_sha256']] - if len(intents) != 1: _fail('Runtime abandonment has no exact original capture intent.') - if any(r['family']=='project_operation' - and r['document']['operation']=='runtime.capture' - and r['document']['request_key']==result['capture_key'] - and r['document']['result'].get('project_id')==result['project_id'] - for r in records): - _fail('Completed runtime capture cannot be abandoned.') - return True - complete = operation == 'runtime.capture' - fields = RESULT_FIELDS | ({'runtime_ref'} if complete else set()) - if type(result) is not dict or set(result) != fields: - _fail('Invalid runtime capture observation fields.') - expected_format = 'project_runtime_capture_v1' if complete else 'project_runtime_capture_intent_v1' - if result['format'] != expected_format: _fail('Unsupported runtime capture observation.') - _id(result['project_id']); _id(result['actor_id']); _sha(result['manifest_sha256']); _key(result['request_key']) - if (type(result['file_count']) is not int or not 1 <= result['file_count'] <= MAX_FILES or - type(result['total_bytes']) is not int or not 0 <= result['total_bytes'] <= MAX_BYTES): - _fail('Runtime capture observation exceeds the owner inventory bounds.') - instant = result['observed_at'] - if type(instant) is not str or len(instant) > 80 or datetime.fromisoformat(instant).utcoffset() is None: - _fail('Runtime capture observation requires an offset timestamp.') - if not complete: return True - ref = reference(result['runtime_ref'], 'runtime', result['project_id']) - if records is None: return True - doc = material_records(records).get(('runtime_material', result['project_id']+'.'+ref['sha256'])) - descriptor = None if doc is None else doc['value']['descriptor'] - if (doc is None or doc['value']['ref'] != ref or descriptor is None or - descriptor['profile'] != 'captured_cpu_python_v1' or - any(descriptor[field] != result[field] for field in ('manifest_sha256','file_count','total_bytes'))): - _fail('Runtime capture observation differs from its exact retained material.') - declarations = {r['id']:r['document'] for r in records if r['family']=='project_artifact'} - for edge in doc['artifacts']: - if declarations.get(edge['blob']['sha256']) != dict(format='project_artifact_v1', **edge['blob']): - _fail('Runtime capture lacks its complete exact artifact declarations.') - intents = [r['document']['result'] for r in records if r['family']=='project_operation' - and r['document']['operation']=='runtime.capture_intent' - and r['document']['request_key']==_intent_key(result['request_key'])] - matching = [intent for intent in intents if all(intent.get(field)==result[field] - for field in ('project_id','manifest_sha256','file_count','total_bytes','request_key'))] - if len(matching) != 1: _fail('Runtime capture lacks its unique durable publication intent.') - validate_operation_result('runtime.capture_intent', matching[0]) - return True - except WorkbenchError: raise - except (ValueError, TypeError, KeyError, AttributeError, OverflowError, RecursionError): - _fail('Malformed runtime capture observation.') - - -class ProjectRuntime: - def __init__(self, projection, *, resolve_configuration): - if not callable(resolve_configuration): - raise TypeError('An explicit current-local runtime configuration resolver is required.') - self.projection = projection - self.session, self.db, self.access = projection.session, projection.db, projection.access - self.execution, self.materials = projection.execution, projection.materials - self.resolve_configuration = resolve_configuration - self.owner = RuntimeSnapshots(self.db, self.execution.runtime_access, self.db.path) - - def _authorize(self, action): - return self.access.require('admin', action, owner_only=True) - - def inspect(self, config_request): - """Read an explicitly selected current source; never provision or execute.""" - with self.access.lock, self.session.lock, self.db.lock: - self._authorize('runtime_snapshots.inspect'); self.session.check_current() - _body(config_request, '0'*64, 0) - with self.execution.runtime_inspection(): - return self.owner.inspect(self.resolve_configuration(copy.deepcopy(config_request))) - - def status(self, key): - """Read historical outcome only; captured does not claim local readiness.""" - _key(key) - with self.access.lock, self.session.lock, self.db.lock: - self.access.require('inspect', 'runtime.capture_status'); self.session.check_current() - receipts = session_operations(self.session) - intent = receipts.get(_intent_key(key)); complete = receipts.get(key) - if intent is not None and intent['operation'] != 'runtime.capture_intent': intent = None - if complete is not None and complete['operation'] != 'runtime.capture': complete = None - abandoned = next((r['result'] for r in receipts.values() if r['operation']=='runtime.abandon' - and r['result']['capture_key']==key), None) - if abandoned is not None: - return dict(state='abandoned',intent=copy.deepcopy(intent['result']) if intent else None, - result=None,abandonment=copy.deepcopy(abandoned)) - return dict(state='captured' if complete else 'intent_recorded' if intent else 'absent', - intent=copy.deepcopy(intent['result']) if intent else None, - result=copy.deepcopy(complete['result']) if complete else None) - - def recover_capture(self, config_request, expected_inventory_sha256, expected_checkpoint, key, *, reserve_bytes=DEFAULT_RESERVE): - """Explicitly restore one original local copy before its exact retry.""" - body = _body(config_request, expected_inventory_sha256, reserve_bytes) - body_sha = _intent_input(body) - self._authorize('runtime_snapshots.create') - self.execution.prepare_runtime_reconciliation(key, body_sha, expected_inventory_sha256, expected_checkpoint) - return self.capture(config_request, expected_inventory_sha256, self.session.summary['checkpoint'], key, - reserve_bytes=reserve_bytes) - - def abandon(self, capture_key, reason, expected_checkpoint, key): - """Preserve partial bytes and acknowledge one stopped local copy.""" - _key(capture_key); _key(key); _sha(expected_checkpoint) - if key == capture_key or type(reason) is not str or not reason.strip() or len(reason) > 1000: - _fail('Use a separate abandonment key and a retained reason.') - actor = self._authorize('runtime.capture_abandon') - body = dict(capture_key=capture_key, reason=reason, actor_id=actor.actor_id) - marker = self.execution.runtime_pending(capture_key) - if marker is None: - # Pure receipt replay cannot clear a local writer or run anything. - def forbidden(): _fail('No original local runtime copy exists to abandon.', 'unavailable', 409) - return self.session.mutate('runtime.abandon', capture_key, body, expected_checkpoint, key, - forbidden, lambda *args: forbidden(), scope='admin') - with self.execution.runtime_reconciliation(capture_key, marker['input_sha256'], - marker['manifest_sha256'], expected_checkpoint) as guard: - receipts = session_operations(self.session) - intent = receipts.get(_intent_key(capture_key)) - if (intent is None or intent['operation']!='runtime.capture_intent' - or intent['input_sha256']!=marker['input_sha256'] - or intent['result']['project_id']!=self.session.project_id - or intent['result']['manifest_sha256']!=marker['manifest_sha256']): - _fail('Runtime intent differs from its exact original local writer.', 'conflict', 409) - if capture_key in receipts: - _fail('Capture completed; reconcile its acknowledgement with the exact capture request.', 'conflict', 409) - observed = self.session.summary['checkpoint'] - def invoke(): - current = self._authorize('runtime.capture_abandon') - if current.actor_id != actor.actor_id: _fail('Runtime abandonment caller changed.', 'conflict', 409) - return dict(format='project_runtime_capture_abandon_v1',project_id=self.session.project_id, - capture_key=capture_key,input_sha256=marker['input_sha256'],manifest_sha256=marker['manifest_sha256'], - actor_id=actor.actor_id,reason=reason,observed_checkpoint=observed,observed_at=timestamp()) - result = self.session.mutate('runtime.abandon', capture_key, body, observed, key, invoke, - lambda value,before:dict(records=[],result=value),scope='admin') - guard.confirmed() - return result - - def _barrier(self, descriptor): - # Native creation fsyncs all staged files before its atomic rename. This - # explicit barrier also closes a previous parent-fsync unknown outcome. - from runtime_snapshots import sync_directory - root = self.owner.path / descriptor['manifest_sha256'] - if descriptor['root'] != str(root): _fail('Current runtime binding differs.', 'stale', 409) - for directory in (root, self.owner.path, self.owner.path.parent): sync_directory(directory) - - def _replay(self, key, body, expected_checkpoint): - def forbidden(): _fail('Runtime replay has no canonical receipt.', 'unavailable', 409) - return self.session.mutate('runtime.capture', body['expected_inventory_sha256'], body, - expected_checkpoint, key, forbidden, lambda *args: forbidden(), scope='admin') - - def capture(self, config_request, expected_inventory_sha256, expected_checkpoint, key, *, reserve_bytes=DEFAULT_RESERVE): - """Explicitly capture bytes after durable intent and current quiescence. - - The resolver is trusted current-host configuration authority. It must not - derive source directories from canonical descriptors or historical roots. - """ - _key(key); _sha(expected_checkpoint) - body = _body(config_request, expected_inventory_sha256, reserve_bytes) - body_sha = _intent_input(body) - intent = None - with self.access.lock, self.session.lock, self.db.lock: - actor = self._authorize('runtime_snapshots.create'); self.session.check_current() - receipts = session_operations(self.session) - if any(r['operation']=='runtime.abandon' and r['result']['capture_key']==key for r in receipts.values()): - _fail('This capture was abandoned; use a fresh key for a new measurement.', 'conflict', 409) - if (_intent_key(key) in receipts and key not in receipts - and self.execution.runtime_pending(key) is None): - _fail('This incomplete capture intent is inert history; use a fresh key for a new capture.', 'unavailable', 409) - try: - with self.execution.runtime_effect(key, body_sha, expected_inventory_sha256) as guard: - receipts = session_operations(self.session) - if key in receipts: - result = self._replay(key, body, expected_checkpoint) - descriptor = self.owner.get(expected_inventory_sha256) - from pipeline import digest - if digest(descriptor) != result['runtime_ref']['sha256']: - _fail('Acknowledged runtime belongs to a different local binding; capture with a new key.', 'unavailable', 409) - guard.publication_started() - self._barrier(descriptor); guard.confirmed() - return result - config = None - if _intent_key(key) in receipts: - def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409) - intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, - expected_checkpoint, _intent_key(key), invalid, lambda *args: invalid(), scope='admin') - else: - if expected_checkpoint != self.session.summary['checkpoint']: - _fail('Expected project checkpoint changed.', 'stale_project', 409) - config = self.resolve_configuration(copy.deepcopy(config_request)) - observed = self.owner.inspect(config) - if observed['inventory_sha256'] != expected_inventory_sha256: - _fail('Runtime inventory changed after inspection.', 'stale', 409) - intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, - expected_checkpoint, _intent_key(key), - lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key), - lambda result,before:dict(records=[],result=result), scope='admin') - guard.publication_started() - if os.path.lexists(self.owner.path / expected_inventory_sha256): - descriptor = self.owner.get(expected_inventory_sha256) - else: - if config is None: config = self.resolve_configuration(copy.deepcopy(config_request)) - descriptor = self.owner.create(config, expected_inventory_sha256, quiesced=True, reserve_bytes=reserve_bytes) - self._barrier(descriptor) - from recipe_process import read_regular, decode - from pipeline import digest - from pathlib import Path - raw = read_regular(Path(descriptor['root'])/'manifest.json', MAX_MANIFEST) - runtime_ref = dict(kind='runtime',catalog_id=self.session.summary['project_id'], - id=digest(descriptor),sha256=digest(descriptor)) - value = dict(ref=runtime_ref,descriptor=descriptor,environment=None,manifest=decode(raw)) - observed = dict(inventory_sha256=descriptor['manifest_sha256'],file_count=descriptor['file_count'],total_bytes=descriptor['total_bytes']) - def capture(result, before): - record = self.materials.capture_runtime(value, raw) - return dict(records=[*self.materials.declarations(record),record],result=result) - result = self.session.mutate('runtime.capture', expected_inventory_sha256, body, - self.session.summary['checkpoint'], key, - lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') - guard.confirmed() - return result - except BaseException as error: - if isinstance(error, (OSError, RuntimeError)): - failure = WorkbenchError('Runtime capture outcome requires explicit same-key reconciliation.', - 'outcome_unknown' if intent else 'unavailable', 503) - else: failure = error - if intent is not None and not hasattr(failure, 'receipt'): - snapshot = self.session.snapshot - failure.receipt = dict(committed=True,selected=True,project_id=snapshot['project_id'], - checkpoint=snapshot['checkpoint'],request_key=key) - if failure is error: raise - raise failure from error diff --git a/project_session.py b/project_session.py deleted file mode 100644 index cfd60555..00000000 --- a/project_session.py +++ /dev/null @@ -1,656 +0,0 @@ -"""Canonical publication shared by the fixed external-project domain owners. - -The supplied validator composes those owners' closed codecs and reference -closure. This boundary owns transaction ordering and receipts, not domain policy. -""" -import copy -import hashlib -import re -import threading - -import agent_api -from project_store import copy_validated_json -from workbench import WorkbenchError - - -OPERATION_FAMILY = 'project_operation' -OPERATION_FORMAT = 'project_operation_v2' -REQUEST_KEY = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') -SHA = re.compile(r'^[a-f0-9]{64}$') -OPERATION = re.compile(r'^[a-z][a-z0-9_.]{0,119}$') - - -class ProjectSessionError(WorkbenchError): - pass - - -def _digest(value): - return hashlib.sha256(agent_api.encoded(value)).hexdigest() - - -def operation_documents(records): - """Validate all historical receipts without collapsing independent keys.""" - result, seen = [], set() - for envelope in records: - if envelope['family'] != OPERATION_FAMILY: - continue - value = envelope['document'] - base = {'format', 'operation', 'target', 'request_key', 'input_sha256', 'result'} - namespaced = type(value) is dict and value.get('format') == 'project_operation_v2' - if (type(value) is not dict or set(value) != base | ({'project_id'} if namespaced else set()) - or value['format'] not in ('project_operation_v1', 'project_operation_v2') - or type(value['operation']) is not str or not OPERATION.fullmatch(value['operation']) - or value['target'] is not None and (type(value['target']) is not str or len(value['target']) > 512) - or type(value['request_key']) is not str or not REQUEST_KEY.fullmatch(value['request_key']) - or type(value['input_sha256']) is not str or not SHA.fullmatch(value['input_sha256'])): - raise ProjectSessionError('Invalid canonical project operation receipt.') - identity = hashlib.sha256(value['request_key'].encode('utf-8')).hexdigest() - if namespaced: - if type(value['project_id']) is not str or not re.fullmatch('[a-f0-9]{32}', value['project_id']): - raise ProjectSessionError('Invalid project operation namespace.') - identity = value['project_id'] + '.' + identity - if envelope['id'] != identity or identity in seen: - raise ProjectSessionError('Invalid or duplicate canonical project operation identity.') - seen.add(identity) - if len(agent_api.encoded(value['result'])) > agent_api.RESPONSE_LIMIT: - raise ProjectSessionError('Canonical operation result exceeds its transport bound.', 'too_large', 413) - result.append(copy.deepcopy(value)) - return result - - -def operation_records(records, *, project_id=None): - """Current request keys are independent of exactly retained fork history.""" - operation_documents(records) - from project_fork import active_records, decode_fork - overlay = decode_fork(records, project_id=project_id) - if project_id is None and overlay is not None: - project_id = overlay['destination_project_id'] - result = {} - for value in operation_documents(active_records(records, project_id=project_id, - families=frozenset({OPERATION_FAMILY}))): - if overlay is not None and value['format'] == 'project_operation_v1': - raise ProjectSessionError('A fork must retain legacy operation receipts as archived historical facts.') - if project_id is not None and value.get('project_id', project_id) != project_id: - raise ProjectSessionError('Active project operation belongs to another namespace.') - if value['request_key'] in result: - raise ProjectSessionError('Duplicate active project operation request key.') - result[value['request_key']] = value - return result - - -class ProjectSession: - # Sessions over a database working copy hold no record set (see __init__). - _lazy = False - - def __init__(self, store, db, access, artifacts, *, validate_records, validation_cache=None): - if not callable(validate_records): - raise TypeError('A complete owner-codec validator is required.') - self.store, self.db, self.access, self.artifacts = store, db, access, artifacts - self.lock = threading.RLock() - self._validate = validate_records - self.validation_cache = validation_cache - # Checkpoint whose exact records passed complete domain validation. - self.validated_checkpoint = None - self.available = False - self._snapshot = None - self.project_id = store.summary()['project_id'] - self.extra_capture = None - self.publication_intent = None - self.runtime_access = None - self._foreground_publication = False - self._pending_observation = None - self._recovery_records = None - self._recovery_request = None - # A database working copy serves record reads directly; the session - # then binds checkpoint metadata only and holds no record set. - self._lazy = callable(getattr(store, 'records_view', None)) - if self._lazy: - self._adopt_working_copy() - else: - self.adopt(store.snapshot(), _owned=True) - - def _adopt_working_copy(self): - """Bind the working copy's checkpoint; validate its records only if unknown.""" - with self.access.lock, self.lock, self.db.lock: - summary = self.store.summary() - cache = getattr(self, 'validation_cache', None) - if cache is not None and cache.known(self.project_id, summary['checkpoint']): - self.validated_checkpoint = summary['checkpoint'] - else: - snapshot = self.store.snapshot() - if snapshot['checkpoint'] != summary['checkpoint']: - raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - with self._known_envelopes(snapshot['records']): - self._validated(snapshot['records'], snapshot['checkpoint']) - del snapshot - if self.store.summary()['checkpoint'] != summary['checkpoint']: - raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._bind(summary) - self.available = True - - def _known_envelopes(self, records, changed=frozenset()): - """Validation context with stored envelope digests for unchanged records.""" - from contextlib import nullcontext - if not self._lazy or not callable(getattr(self.store, 'envelope_digests', None)): - return nullcontext() - from project_composition import known_envelope_digests - digests = self.store.envelope_digests() - known = {} - for record in records: - identity = (record['family'], record['id']) - if identity not in changed and identity in digests and digests[identity] is not None: - known[id(record)] = (record, digests[identity]) - return known_envelope_digests(known) - - def _edit_scope(self, base_checkpoint, changed, operations=(), base_records=None): - """Validate only what this edit can affect when its base is validated. - - The candidate is the base checkpoint's records plus changed identities - (never deletions); see project_validation_delta. - """ - from contextlib import nullcontext - if getattr(self, 'validated_checkpoint', None) != base_checkpoint: - return nullcontext() - from project_validation_delta import ValidationDelta, edit_delta - base = set() if base_records is None else {(item['family'], item['id']) for item in base_records} - added = frozenset(identity for identity in changed if base_records is not None and identity not in base) - return edit_delta(ValidationDelta(frozenset(changed), frozenset(operations), added)) - - def _bind(self, snapshot): - """Bind a committed checkpoint; a lazy session keeps only its metadata.""" - if self._lazy: - self._snapshot = dict(project_id=snapshot['project_id'], name=snapshot['name'], - checkpoint=snapshot['checkpoint'], records=None) - else: - self._snapshot = snapshot - - def _fresh_store(self): - """The working copy must still be at the bound checkpoint.""" - if self.store.summary()['checkpoint'] != self._snapshot['checkpoint']: - raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) - return self.store - - @property - def foreground_publication(self): - return self._foreground_publication - - @staticmethod - def _merge(records, upserts, *, owned=False): - """Candidate record map: base records plus copied upserts. - - owned=True means the caller owns these base records (a fresh working - copy read), so they are used without another whole-project copy. - """ - merged = ({(item['family'], item['id']): item for item in records} if owned else - {(item['family'], item['id']): copy.deepcopy(item) for item in records}) - seen = {} - for item in upserts: - if type(item) is not dict or set(item) != {'family', 'id', 'document'}: - raise ProjectSessionError('Invalid owned project envelope.') - identity = (item['family'], item['id']) - if item['family'] == OPERATION_FAMILY: - raise ProjectSessionError('Owned capture cannot replace a project operation receipt.') - if identity in seen and seen[identity] != item: - raise ProjectSessionError('Owned captures disagree about the same project record.') - seen[identity] = item - merged[identity] = copy.deepcopy(item) - return merged - - def _publish_observation(self, records, key, digest, *, authorize): - """Publish fixed background science while its owned SQL TX is open. - - This is a private trusted-owner seam, not a transport operation or an - imported receipt capability. The callback verifies current local job - custody/admission. SQL outcome selects or fences the resulting cache. - """ - if (self.foreground_publication or not self.db.in_transaction - or not self.db.lock.depth or self._pending_observation is not None): - raise ProjectSessionError('Background publication requires its own outer project transaction.') - if (type(records) is not list or type(key) is not str or not REQUEST_KEY.fullmatch(key) - or type(digest) is not str or not SHA.fullmatch(digest) or not callable(authorize)): - raise ProjectSessionError('Invalid background publication identity.') - authorize() - current = self.check() - merged = self._merge(current['records'], records, owned=self._lazy) - candidate = [merged[identity] for identity in sorted(merged)] - changed = {(item['family'], item['id']) for item in records} - with self._known_envelopes(candidate, changed), \ - self._edit_scope(current['checkpoint'], changed, base_records=current['records']): - self._validated(candidate) - if self.publication_intent is not None: - self.publication_intent(copy.deepcopy(current), copy.deepcopy(candidate), key, digest) - try: - if self._lazy: - changed = {(item['family'], item['id']) for item in records} - self._pending_observation = self.store.commit(candidate, current['checkpoint'], key, digest, - changed=changed) - else: - self._pending_observation = self.store.commit(candidate, current['checkpoint'], key, digest) - except BaseException as error: - if getattr(error, 'code', None) in ('unknown', 'outcome_unknown'): - self.available = False - self._recovery_records = copy.deepcopy(candidate) - self._recovery_request = (key, digest) - raise - self._remember_validated(self._pending_observation['checkpoint']) - return copy.deepcopy(self._pending_observation) - - def _observation_outcome(self, outcome): - if outcome not in ('committed', 'rolled_back', 'unknown'): - raise ValueError('Invalid project SQL outcome.') - pending, self._pending_observation = self._pending_observation, None - if pending is not None and self._atomic_store() and outcome == 'rolled_back': - # The record commit rolled back with the SQL transaction, so the - # bound checkpoint is still exactly current. Background local - # effects (for example a live phase) may have advanced, so their - # custody is reconciled explicitly before further use. A foreground - # mutate reports its own definite rollback and stays available. - if not self._foreground_publication: - self.available = False - return - if pending is not None: - self._bind(pending) - if outcome != 'committed': - self.available = False - if outcome == 'unknown': - self.available = False - - def _atomic_store(self): - """Whether record commits share this session's SQL transactions.""" - bound = getattr(self.store, '_db', None) - return bound is not None and bound is self.db - - def reconcile_selection(self, snapshot): - """Bind only this owner's selected or uncertain publication, never a checkout. - - The execution controller first proves local cessation. Callers retain - all ordered guards until fixed inert reconstruction succeeds or fences. - """ - with self.access.lock, self.lock, self.db.lock: - if self._lazy: - known = all(snapshot[key] == self._snapshot[key] for key in ('project_id', 'name', 'checkpoint')) - else: - known = snapshot == self._snapshot - attempted = (self._recovery_records is not None - and snapshot['project_id'] == self._snapshot['project_id'] - and snapshot['name'] == self._snapshot['name'] - and snapshot['records'] == self._recovery_records - and self._recovery_request is not None - and self.store.reconcile(*self._recovery_request) is not None) - if not known and not attempted: - raise ProjectSessionError('Canonical selection differs from this local execution outcome.', 'stale_project', 409) - self.adopt(snapshot) - self._recovery_records = None - self._recovery_request = None - - @property - def summary(self): - """Scalar metadata for the same bound checkpoint as snapshot.""" - with self.lock: - return {key: self._snapshot[key] for key in ('project_id', 'name', 'checkpoint')} - - @property - def snapshot(self): - with self.lock: - if self._lazy: - # Complete independent record set from the working copy (costly). - current = self._fresh_store().snapshot() - if current['checkpoint'] != self._snapshot['checkpoint']: - raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) - return current - return copy_validated_json(self._snapshot) - - def records(self, *families): - """Independent copies of only the selected families at the bound checkpoint. - - Equivalent to filtering a complete snapshot copy, without copying the - unselected families (most bytes are usually retained Run history). - """ - if not families or any(type(family) is not str for family in families): - raise TypeError('Select one or more record families by name.') - selected = frozenset(families) - with self.lock: - if self._lazy: - return self._fresh_store().records(*families) - return copy_validated_json([item for item in self._snapshot['records'] - if item['family'] in selected]) - - def records_for(self, *families): - """Every bound record, in order, for one owner's pure decoder. - - Envelopes of the selected families are independent copies the owner - may keep or mutate. Every other envelope is a fresh envelope sharing its - verified document read-only, so fork/membership validators still see - the complete record set without copying unrelated Run history. Owners - must not mutate documents outside their selected families. - """ - if not families or any(type(family) is not str for family in families): - raise TypeError('Select one or more record families by name.') - selected = frozenset(families) - with self.lock: - if self._lazy: - # Unselected envelopes carry document None (identity only). - return self._fresh_store().records_view(*families) - return [copy_validated_json(item) if item['family'] in selected else - dict(family=item['family'], id=item['id'], document=item['document']) - for item in self._snapshot['records']] - - def active(self, *families, project_id=None): - """active_records for selected families over the bound checkpoint. - - Validates the complete fork history against every bound identity, but - copies only the selected active envelopes. - """ - if not families or any(type(family) is not str for family in families): - raise TypeError('Select one or more record families by name.') - from project_fork import active_records - with self.lock: - if self._lazy: - return active_records(self._fresh_store().records_view(*families, 'project_fork'), - project_id=self.project_id if project_id is None else project_id, - families=frozenset(families)) - return active_records(self._snapshot['records'], - project_id=self.project_id if project_id is None else project_id, - families=frozenset(families)) - - def derived(self, name, compute): - """Memoize one owner-derived read-only fact set per bound checkpoint. - - compute() must be a pure function of the bound records. Owners share - the result; they must copy any part they return or mutate. - """ - if type(name) is not str or not callable(compute): - raise TypeError('Derived facts need a name and a compute function.') - with self.lock: - checkpoint = self._snapshot['checkpoint'] - cache = getattr(self, '_derived_cache', None) - if cache is None or cache[0] != checkpoint: - cache = (checkpoint, {}) - self._derived_cache = cache - if name not in cache[1]: - value = compute() - if self._snapshot['checkpoint'] != checkpoint: - raise ProjectSessionError('Project changed while deriving facts.', 'stale_project', 409) - cache[1][name] = value - return cache[1][name] - - def record(self, family, record_id): - """An independent copy of one bound record envelope, or None.""" - if type(family) is not str or type(record_id) is not str: - raise TypeError('Select a record by family and ID.') - with self.lock: - if self._lazy: - return self._fresh_store().record(family, record_id) - index = getattr(self, '_record_index', None) - if index is None or index[0] != self._snapshot['checkpoint']: - index = (self._snapshot['checkpoint'], - {(item['family'], item['id']): item for item in self._snapshot['records']}) - self._record_index = index - found = index[1].get((family, record_id)) - return None if found is None else copy_validated_json(found) - - def operations(self, *, project_id=None): - """operation_records over the bound checkpoint, computed once per checkpoint. - - The bound records are immutable for a checkpoint, so the result is a - pure function of (checkpoint, project_id). Callers receive independent - copies, exactly as from operation_records(self.snapshot['records']). - """ - with self.lock: - key = (self._snapshot['checkpoint'], project_id) - cached = getattr(self, '_operations_cache', None) - if cached is None or cached[0] != key: - # operation_records copies what it returns and never mutates input. - records = (self._fresh_store().records_view(OPERATION_FAMILY, 'project_fork') if self._lazy - else self._snapshot['records']) - cached = (key, operation_records(records, project_id=project_id)) - self._operations_cache = cached - return copy.deepcopy(cached[1]) - - def _validated(self, records, checkpoint=None): - """Completely validate records; reuse an exact earlier success. - - checkpoint, when given, must be the Store checkpoint naming exactly - these records. Only then can a remembered validation be reused. - """ - cache = getattr(self, 'validation_cache', None) - if checkpoint is not None and cache is not None and cache.known(self.project_id, checkpoint): - self.validated_checkpoint = checkpoint - return - operation_records(records, project_id=self.project_id) - self._validate(records) - if checkpoint is not None: - self.validated_checkpoint = checkpoint - if cache is not None: - cache.remember(self.project_id, checkpoint) - - def _remember_validated(self, checkpoint): - # The caller completely validated exactly the records this names. - self.validated_checkpoint = checkpoint - cache = getattr(self, 'validation_cache', None) - if cache is not None: - cache.remember(self.project_id, checkpoint) - - def adopt(self, snapshot, *, _owned=False): - """Bind a fully validated checkpoint after inert reconstruction/cutover. - - The caller owns SQL reconstruction. This method never repairs or writes - canonical bytes, and is not authorization to trust an imported receipt. - _owned marks a snapshot this session just read from its own Store: its - checkpoint then names its records, so freshness is a checkpoint check - and the independent snapshot needs no second copy or full comparison. - """ - with self.access.lock, self.lock, self.db.lock: - if _owned: - self._validated(snapshot['records'], snapshot['checkpoint']) - if self.store.summary()['checkpoint'] != snapshot['checkpoint']: - raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._bind(snapshot) - self.available = True - return - self._validated(snapshot['records']) - current = self.store.snapshot() - if current != snapshot: - raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._remember_validated(snapshot['checkpoint']) - self._bind(copy_validated_json(snapshot) if not self._lazy else snapshot) - self.available = True - - def _checked(self, read): - if not self.available: - raise ProjectSessionError('Reopen this project to reconstruct its projection.', 'projection_unavailable', 503) - current = read() - if current['checkpoint'] != self._snapshot['checkpoint']: - raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) - return current - - def check_current(self): - """Verify actual canonical bytes without returning an unused record graph. - - The caller holds the same host/session/projection guards as check(). - Store.summary reads and hashes all current records before returning. - """ - self._checked(self.store.summary) - - def check(self): - """Return independent current records under caller-owned projection guards.""" - return self._checked(self.store.snapshot) - - def check_records(self, family): - """Verify every canonical record, then copy one bound record family. - - The complete snapshot, including cross-family closure, was validated - and isolated by adopt/publication. Returned copies retain that validity - inside the trusted owner boundary; they are not new imported raw input. - Exact current bytes and checkpoint must still match before using it. - Callers hold the same ordered guards as check(); no cached alias escapes. - """ - if type(family) is not str: - raise TypeError('Record-family selection requires a string.') - current = self._checked(self.store.summary) - if self._lazy: - current['records'] = self.store.records(family) - return current - current['records'] = copy.deepcopy([ - item for item in self._snapshot['records'] if item['family'] == family]) - return current - - def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capture, *, scope='edit'): - """Invoke existing policy, capture fixed-owner upserts, publish, commit SQL. - - invoke() returns the owner's result. capture(result, before_records) - returns {'records': upserts, 'result': original bounded domain result}. - Wrappers check any stronger scope/owner/disclosure requirement before - entering, including before replay. The supplied current scope belongs to - the wrapper's existing policy (for example freeze rather than edit). - """ - if (type(operation) is not str or not OPERATION.fullmatch(operation) - or target is not None and (type(target) is not str or len(target) > 512) - or type(key) is not str or not REQUEST_KEY.fullmatch(key) - or type(expected_checkpoint) is not str or not SHA.fullmatch(expected_checkpoint)): - raise ProjectSessionError('Invalid project mutation identity.') - digest = _digest(dict(operation=operation, target=target, body=body)) - committed = None - attempted_records = None - with self.access.lock, self.lock, self.db.lock: - self.access.require(scope, operation, target) - current = self.check() - if self.db.in_transaction: - raise ProjectSessionError('Project mutation must own its outer SQL transaction.', 'unavailable', 409) - receipts = operation_records(current['records']) - prior = receipts.get(key) - if prior is not None: - if prior['operation'] != operation or prior['input_sha256'] != digest or prior['target'] != target: - raise ProjectSessionError('Request key already has different input.', 'conflict', 409) - # The current composed codec validates the exact historical - # result too; receipt facts can never authorize execution. - self._validated(current['records'], current['checkpoint']) - return copy.deepcopy(prior['result']) - if self.store.reconcile(key, digest) is not None: - raise ProjectSessionError('Request key belongs to another canonical operation.', 'conflict', 409) - if expected_checkpoint != current['checkpoint']: - raise ProjectSessionError('Expected project checkpoint changed.', 'stale_project', 409) - try: - self._foreground_publication = True - with self.db.transaction(): - result = invoke() - # A working-copy read is already this mutation's own fresh - # copy; owner captures read it and copy what they change. - before = current['records'] if self._lazy else copy.deepcopy(current['records']) - captured = capture(result, before) - if type(captured) is not dict or set(captured) != {'records', 'result'} or type(captured['records']) is not list: - raise ProjectSessionError('Invalid owned project capture.') - if len(agent_api.encoded(captured['result'])) > agent_api.RESPONSE_LIMIT: - raise ProjectSessionError('Project result exceeds its transport bound.', 'too_large', 413) - upserts = captured['records'] - if self.extra_capture is not None: - extra = self.extra_capture(current['records'] if self._lazy else copy.deepcopy(current['records'])) - if type(extra) is not list: - raise ProjectSessionError('Invalid fixed runtime capture.') - upserts = upserts + extra - merged = self._merge(current['records'], upserts, owned=self._lazy) - document = dict(format=OPERATION_FORMAT, project_id=self.project_id, operation=operation, target=target, - request_key=key, input_sha256=digest, result=copy.deepcopy(captured['result'])) - identity = (OPERATION_FAMILY, self.project_id + '.' + hashlib.sha256(key.encode('utf-8')).hexdigest()) - merged[identity] = dict(family=identity[0], id=identity[1], document=document) - records = [merged[identity] for identity in sorted(merged)] - changed = {(item['family'], item['id']) for item in upserts} | {identity} - with self._known_envelopes(records, changed), \ - self._edit_scope(current['checkpoint'], changed, (operation,), - base_records=current['records']): - self._validated(records) - # Unique payloads have already reached artifact custody; - # SQL callbacks can delete only disposable projection files. - attempted_records = records - if self.publication_intent is not None: - self.publication_intent(copy.deepcopy(current), copy.deepcopy(records), key, digest) - if self._lazy: - # Only the captured upserts and this receipt can differ. - changed = {(item['family'], item['id']) for item in upserts} | {identity} - committed = self.store.commit(records, current['checkpoint'], key, digest, changed=changed) - else: - committed = self.store.commit(records, current['checkpoint'], key, digest) - # Outcome hooks see canonical selection before SQL outcome - # and before any deferred dispatch/wake can run. - self._pending_observation = committed - self._bind(committed) - # These exact records were completely validated as the candidate. - self._remember_validated(committed['checkpoint']) - return copy.deepcopy(captured['result']) - except BaseException as error: - if committed is not None and self._atomic_store(): - from metadata import CommitOutcomeUnknown - if isinstance(error, CommitOutcomeUnknown): - # PostgreSQL may or may not have committed both the - # records and the projection; reopening reads the outcome. - self.available = False - raise ProjectSessionError('PostgreSQL did not confirm this edit. Reopen the project to see whether it was saved.', - 'unknown', 503) from error - # Records and projection rolled back together; nothing changed. - if isinstance(error, ProjectSessionError): - raise - raise ProjectSessionError('PostgreSQL rejected this edit; nothing was saved.', - 'rejected', 409) from error - if committed is not None: - self._bind(committed) - self.available = False - failure = ProjectSessionError('Canonical operation committed; reopen to reconstruct the projection.', - 'projection_unavailable', 503) - failure.receipt = dict(committed=True, selected=True, project_id=committed['project_id'], - checkpoint=committed['checkpoint'], request_key=key) - raise failure from error - if getattr(error, 'code', None) in ('unknown', 'outcome_unknown'): - self.available = False - if attempted_records is not None: - self._recovery_records = copy.deepcopy(attempted_records) - self._recovery_request = (key, digest) - raise - finally: - self._foreground_publication = False - self._pending_observation = None - - -# Owners also accept lightweight snapshot views (for example inert candidate or -# retained-material views that expose only snapshot/check_current). These -# helpers use a ProjectSession's copy-avoiding bound reads when available and -# otherwise compute exactly the same values from the view's snapshot. - -def session_records(session, *families): - if hasattr(session, 'records') and callable(session.records): - return session.records(*families) - return [item for item in session.snapshot['records'] if item['family'] in families] - - -def session_record(session, family, record_id): - if hasattr(session, 'record') and callable(session.record): - return session.record(family, record_id) - return next((item for item in session.snapshot['records'] - if item['family'] == family and item['id'] == record_id), None) - - -def session_records_for(session, *families): - if hasattr(session, 'records_for') and callable(session.records_for): - return session.records_for(*families) - return session.snapshot['records'] - - -def session_active(session, *families, project_id=None): - if hasattr(session, 'active') and callable(session.active): - return session.active(*families, project_id=project_id) - from project_fork import active_records - snapshot = session.snapshot - return active_records(snapshot['records'], - project_id=session.project_id if project_id is None else project_id, - families=frozenset(families)) - - -def session_operations(session, *, project_id=None): - if hasattr(session, 'operations') and callable(session.operations): - return session.operations(project_id=project_id) - return operation_records(session.snapshot['records'], project_id=project_id) - - -def session_derived(session, name, compute): - if hasattr(session, 'derived') and callable(session.derived): - return session.derived(name, compute) - return compute() diff --git a/project_sources.py b/project_sources.py deleted file mode 100644 index a17b02a5..00000000 --- a/project_sources.py +++ /dev/null @@ -1,238 +0,0 @@ -"""Explicit current source bindings, separate from portable snapshot facts. - -Opening a project never restores workspace_projects. Only an authenticated owner -registering an anchored current manifest can create that disposable binding. -Canonical receipts describe retained facts; they are not filesystem capabilities. -""" -import copy -from datetime import datetime -from functools import cache, partial -import hashlib -import os - -from project_material import decode_records as material_records -from project_session import ProjectSessionError, REQUEST_KEY -from projects import (DATA_LIMIT, ID, INSPECTION_LIMIT, MANIFEST_LIMIT, NAME, SHA, decode, - decode_retained_snapshot, manifest, relative_path, text) -from workbench import WorkbenchError, encode, timestamp - -RESULT_FIELDS = frozenset(('origin_catalog_id','project_id','manifest_sha256','name', - 'schema_version','actor_id','registered_at')) - - -def _closed(value, fields): - if type(value) is not dict or set(value)!=set(fields): - raise WorkbenchError('Invalid explicit source operation fields.') - - -def _identity(value, pattern, label): - if type(value) is not str or not pattern.fullmatch(value): - raise WorkbenchError('Invalid source '+label+'.') - return value - - -def _summary(saved, origin): - return dict(origin_catalog_id=origin,project_id=saved['id'], - manifest_sha256=saved['manifest_sha256'],name=saved['name'], - schema_version=saved['manifest']['schema_version']) - - -def _validate_result(operation,result,lookup=None): - """Validate the fixed source receipt against its exact retained snapshot.""" - if operation!='source.register':return False - try: - _closed(result,RESULT_FIELDS) - for key in ('origin_catalog_id','project_id','actor_id'):_identity(result[key],ID,key) - _identity(result['manifest_sha256'],SHA,'manifest SHA256') - text(result['name'],'Source project name',120) - if type(result['schema_version']) is not int or result['schema_version'] not in (1,2): - raise WorkbenchError('Unsupported source registration schema.','unsupported',400) - if type(result['registered_at']) is not str or len(result['registered_at'])>80: - raise WorkbenchError('Source registration requires an attributed timestamp.') - if datetime.fromisoformat(result['registered_at']).utcoffset() is None: - raise WorkbenchError('Source registration timestamp requires an offset.') - if lookup is not None: - document=lookup(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) - expected={key:result[key] for key in ('origin_catalog_id','project_id','manifest_sha256','name','schema_version')} - if document is None or _summary(document['value'],document['origin_catalog_id'])!=expected: - raise WorkbenchError('Source receipt differs from its exact retained snapshot.') - return True - except WorkbenchError:raise - except (ValueError,TypeError,KeyError,AttributeError,OverflowError,RecursionError): - raise WorkbenchError('Malformed retained source operation result.') from None - - -def receipt_validator(records): - """Own one candidate's retained material facts for source receipt checks.""" - owned = material_records(records) - return partial(_validate_result, lookup=owned.get) - - -def validate_operation_result(operation, result, records=None): - @cache - def owned(): - return material_records(records) - lookup = None if records is None else lambda identity: owned().get(identity) - return _validate_result(operation, result, lookup) - - -class SourceAdapter: - def __init__(self,projection): - self.projection=projection - self.session=projection.session - self.db,self.access=self.session.db,self.session.access - self.sources,self.materials,self.data=projection.sources,projection.materials,projection.data - - @staticmethod - def _mutation_identity(expected_checkpoint,key): - _identity(expected_checkpoint,SHA,'expected checkpoint') - _identity(key,REQUEST_KEY,'request key') - - @staticmethod - def _snapshot_reference(snapshot): - _closed(snapshot,('origin_catalog_id','project_id','manifest_sha256')) - for field in ('origin_catalog_id','project_id'):_identity(snapshot[field],ID,field) - _identity(snapshot['manifest_sha256'],SHA,'manifest SHA256') - return snapshot - - def _read_registration(self,body): - _closed(body,('root','manifest_path')) - root=_identity(body['root'],NAME,'root alias');path=relative_path(body['manifest_path']) - entry=self.sources.roots.get(root) - if entry is None:raise WorkbenchError('Choose a currently configured source root.','unavailable',409) - self.access.require('inspect','projects.open') - try: - actual=os.fstat(entry[0]);identity=[actual.st_dev,actual.st_ino] - if identity!=entry[1]:raise WorkbenchError('Configured source root identity changed.','stale',409) - # Reuse Projects' anchored before/after identity and no-follow read. - raw=self.sources._read_snapshot_file(dict(root=root,root_identity=identity),path,MANIFEST_LIMIT) - except OSError: - raise WorkbenchError('Current source manifest is unavailable.','unavailable',409) from None - value=manifest(decode(raw)) - saved=dict(id=value['id'],name=value['name'],root=root,root_identity=identity, - manifest_path=path,manifest_sha256=hashlib.sha256(raw).hexdigest(),manifest=value) - return decode_retained_snapshot(saved),raw - - def _bind(self,saved,raw): - if not self.db.in_transaction:raise RuntimeError('Source binding requires its outer SQL transaction.') - if self.sources._read_snapshot_file(saved,saved['manifest_path'],MANIFEST_LIMIT)!=raw: - raise WorkbenchError('Source manifest changed before current registration.','stale',409) - row=self.db.execute('SELECT document FROM workspace_projects WHERE id=%s',(saved['id'],)).fetchone() - if row is not None: - prior=decode_retained_snapshot(decode(row[0].encode())) - if prior['manifest_sha256']!=saved['manifest_sha256']: - raise WorkbenchError('This source ID already names a different current snapshot.','conflict',409) - if prior==saved:return - # Explicit same-manifest registration can renew a moved root. The - # original canonical registration facts are never rewritten. - self.db.execute('UPDATE workspace_projects SET document=%s WHERE id=%s',(encode(saved),saved['id'])) - else:self.db.execute('INSERT INTO workspace_projects VALUES(%s,%s)',(saved['id'],encode(saved))) - - def register(self,body,expected_checkpoint,key): - self._mutation_identity(expected_checkpoint,key) - with self.access.lock,self.session.lock,self.db.lock: - actor=self.access.require('edit','source.register',owner_only=True) - self.session.check_current() - saved,raw=self._read_registration(body) - origin=self.session.summary['project_id'];invoked=False - def invoke(): - nonlocal invoked - invoked=True;self._bind(saved,raw) - return dict(_summary(saved,origin),actor_id=actor.actor_id,registered_at=timestamp()) - def capture(result,before): - record=self.materials.capture_snapshot(saved,raw,catalog_id=origin) - return dict(records=[*self.materials.declarations(record),record],result=result) - result=self.session.mutate('source.register',saved['id'],copy.deepcopy(body),expected_checkpoint,key,invoke,capture) - if any(result[field]!=value for field,value in _summary(saved,origin).items()): - raise WorkbenchError('Current source manifest differs from the acknowledged registration.','stale',409) - if not invoked: - # Replay itself does not run invoke. This explicit owner action - # may renew a verified current binding after a cold Open; it is - # not a new historical registration event or canonical write. - try: - with self.db.transaction():self._bind(saved,raw) - except WorkbenchError:raise - except BaseException as error: - self.session.available=False - failure=ProjectSessionError('Source registration was acknowledged; reopen before binding its current root.', - 'projection_unavailable',503) - snapshot=self.session.snapshot - failure.receipt=dict(committed=True,selected=True,project_id=snapshot['project_id'], - checkpoint=snapshot['checkpoint'],request_key=key) - raise failure from error - return result - - def references(self,snapshotref,offset=0,limit=40): - snapshot=self._snapshot_reference(snapshotref) - if type(offset) is not int or not 0<=offset<=2**53-1 or type(limit) is not int or not 1<=limit<=40: - raise WorkbenchError('Invalid bounded source reference page.') - with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','source.references',snapshot['project_id']) - self.session.check_current() - historical=self.materials.snapshot_ref(snapshot['project_id'],snapshot['manifest_sha256'], - catalog_id=snapshot['origin_catalog_id']) - try:current=self.sources._saved(snapshot['project_id'],extensions=True) - except WorkbenchError as error: - if error.code=='not_found': - raise WorkbenchError('Explicitly register a current source root before inspecting its references.','unavailable',409) from None - raise - # Check current binding and exact retained selection before reading - # even the live manifest. Historical root/path fields grant nothing. - if not self.sources._bound(current): - raise WorkbenchError('The selected source has no current root binding.','unavailable',409) - if current['manifest_sha256']!=snapshot['manifest_sha256'] or current['manifest']!=historical['manifest']: - raise WorkbenchError('Current source registration differs from the selected snapshot.','stale',409) - self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) - selected=current['manifest']['data'][offset:offset+limit];budget=[INSPECTION_LIMIT] - items=[self.sources._inspect(current['root'],copy.deepcopy(ref),budget,DATA_LIMIT) for ref in selected] - self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) - self.session.check_current();self.access.require('inspect','source.references',snapshot['project_id']) - return dict(items=items,total=len(current['manifest']['data']),offset=offset,limit=limit) - - def import_scientific(self,body,expected_checkpoint,key): - self._mutation_identity(expected_checkpoint,key) - _closed(body,('snapshot','reference_id','expected_sha256','document')) - snapshot=self._snapshot_reference(body['snapshot']) - _identity(body['reference_id'],NAME,'data reference') - _identity(body['expected_sha256'],SHA,'data SHA256') - if type(body['document']) is not dict:raise WorkbenchError('Scientific import metadata must be an object.') - with self.access.lock,self.session.lock,self.db.lock: - self.access.require('edit','source.import_scientific',snapshot['project_id']) - self.session.check_current() - raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],DATA_LIMIT) - # ProjectData owns the one canonical mutation and native parser; - # current rooted reads above never become an imported capability. - return self.data.import_scientific(raw,copy.deepcopy(body['document']),source,expected_checkpoint,key) - - def _capture_reference(self,snapshot,reference_id,expected_sha256,maximum_bytes): - """One rooted capture policy shared by the two fixed native consumers. - - Caller holds current access/session/database locks and authorizes the - action before any source read. Returned paths remain historical facts. - """ - historical=self.materials.snapshot_ref(snapshot['project_id'],snapshot['manifest_sha256'], - catalog_id=snapshot['origin_catalog_id']) - current,_raw=self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) - if current['manifest']!=historical['manifest']: - raise WorkbenchError('Current source differs from the selected retained snapshot.','stale',409) - raw,source=self.sources.read_snapshot_data_reference(snapshot['project_id'],reference_id, - expected_sha256,maximum_bytes) - if source['manifest_sha256']!=snapshot['manifest_sha256']: - raise WorkbenchError('Current source snapshot changed during capture.','stale',409) - self.session.check_current() - return raw,source - - def import_measurements(self,body,expected_checkpoint,key): - self._mutation_identity(expected_checkpoint,key) - _closed(body,('snapshot','reference_id','expected_sha256')) - snapshot=self._snapshot_reference(body['snapshot']) - _identity(body['reference_id'],NAME,'data reference') - _identity(body['expected_sha256'],SHA,'data SHA256') - from retained_measurements import MAX_BYTES - with self.access.lock,self.session.lock,self.db.lock: - self.access.require('edit','source.import_measurements',snapshot['project_id']) - self.session.check_current() - raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],MAX_BYTES) - source=dict(source,origin_catalog_id=snapshot['origin_catalog_id']) - self.access.require('edit','source.import_measurements',snapshot['project_id']) - return self.projection.measurements.import_packet(raw,source,expected_checkpoint,key) diff --git a/project_store.py b/project_store.py deleted file mode 100644 index 826a5ab7..00000000 --- a/project_store.py +++ /dev/null @@ -1,1286 +0,0 @@ -"""Canonical POSIX-local project checkpoints; domain codecs own document meaning. - -Only the inventory in .tuldok/project.json confers authority. Immutable revisions -are durable before its atomic replacement. The directory flock excludes cooperating -writers by physical root identity, including alternate spellings of the same root. -Uncooperative checkout/edit operations are forbidden while open and detected before -publication. An uncertain publication is reconciled by reopening and its receipt. -""" -import copy -import ctypes -from contextlib import contextmanager -import errno -import fcntl -import hashlib -import json -import math -import os -from pathlib import Path -import re -import stat -import threading -import uuid - -FORMAT = 'tuldok_project_v1' -GITIGNORE = b'/artifacts/\n/.tuldok-local/\n/.venv/\n/__pycache__/\n/PGDATA/\n/output/\n' -MAX_RECORD_BYTES = 40 * 1024 * 1024 -# A retained scientific runtime can declare tens of thousands of small files. -# Their index is bounded separately from the actual record/payload byte budgets. -MAX_MANIFEST_BYTES = 64 * 1024 * 1024 -# Coordinated internal codecs; adding a family requires its owning decoder. -SUPPORTED_FORMATS = { - 'workbench_text': 'workbench_text_v1', - 'workbench_record': 'workbench_record_v1', - 'workbench_response': 'workbench_response_v1', - 'workbench_preference': 'workbench_preference_v1', - 'saved_selection': 'saved_selection_v1', - 'saved_search': 'saved_search_v1', - 'dataset_release': 'dataset_release_v1', - 'dataset_exposure': 'dataset_exposure_v1', - 'model_definition': 'model_definition_history_v1', - 'pipeline_graph': 'pipeline_graph_history_v1', - 'recipe_definition': 'recipe_definition_history_v1', - 'model_material': 'retained_model_material_v1', - 'recipe_material': 'retained_recipe_material_v1', - 'project_snapshot': 'retained_project_snapshot_v1', - 'measurement_source': 'project_measurement_source_v1', - 'runtime_material': 'retained_runtime_material_v1', - 'study_history': 'project_study_history_v1', - 'trial_history': 'project_trial_history_v1', - 'run_history': 'project_run_history_v1', - 'run_observation': 'project_run_observation_v1', - 'native_execution_history': 'native_execution_history_v1', - 'candidate_history': 'project_candidate_history_v1', - 'project_artifact': 'project_artifact_v1', - 'project_operation': 'project_operation_v1', - 'project_fork': 'project_fork_v1', - 'project_composition': 'project_composition_v1', - 'project_file': 'project_file_v1', - 'project_extraction': 'project_extraction_v1', - 'generation_history': 'project_generation_history_v1', - 'grounded_history': 'project_grounded_history_v1', - 'caption_proposal_history': 'project_caption_proposal_history_v1', - 'text_proposal_history': 'project_text_proposal_history_v1', - # Store-only: one shared retained runtime closure for compact Runs - # (project_record_dedupe); never visible to domain owners. - 'run_runtime_closure': 'project_run_runtime_closure_v1', -} -MAX_RECORDS = 100_000 -# Scientific history can retain original runtime/file observations in hundreds -# of Runs. Bound the whole local checkpoint separately from each document. -MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 -IDENTIFIER = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$') -REQUEST_KEY = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') -DOMAIN_FORMAT = re.compile(r'^[a-z][a-z0-9_]{0,95}_v([1-9][0-9]*)$') -SHA256 = re.compile(r'^[a-f0-9]{64}$') -DIR_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC -FILE_FLAGS = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC - - -class ProjectStoreError(ValueError): - def __init__(self, message, code='invalid', status=400): - super().__init__(message) - self.code, self.status = code, status - - -def _fail(message, code='invalid', status=400): - raise ProjectStoreError(message, code, status) - - -def _sha(raw): - return hashlib.sha256(raw).hexdigest() - - -def _json_value(value, depth=0): - if depth > 64: - _fail('Project JSON exceeds its nesting limit.') - if value is None or type(value) is bool: - return - if type(value) is int: - if not -(2 ** 63) <= value < 2 ** 63: - _fail('Project JSON integer exceeds signed 64-bit range.') - return - if type(value) is float and math.isfinite(value): - return - if type(value) is str: - if not value.isascii(): - try: - value.encode('utf-8') - except UnicodeEncodeError: - _fail('Project JSON contains an invalid Unicode surrogate.') - return - if type(value) is list: - for item in value: - _json_value(item, depth + 1) - return - if type(value) is dict and all(type(key) is str for key in value): - for key, item in value.items(): - _json_value(key, depth + 1) - _json_value(item, depth + 1) - return - _fail('Project value must contain finite JSON values.') - - -def _encode(value): - _json_value(value) - return (json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, - allow_nan=False) + '\n').encode('utf-8') - - -def copy_validated_json(value): - """Own another tree of already validated JSON without Python graph copying. - - Canonical Store/Session values contain only JSON scalars, lists and string - keyed dictionaries. This is a copy operation inside that proof boundary, - never an input decoder or a substitute for freshness/codec validation. - """ - return json.loads(json.dumps(value, ensure_ascii=True, allow_nan=False, - separators=(',', ':'))) - - -def _manifest_raw(manifest): - raw = _encode(manifest) - if len(raw) > MAX_MANIFEST_BYTES: - _fail('Project checkpoint exceeds its byte limit.') - return raw - - -def _pairs(pairs): - result = {} - for key, value in pairs: - if key in result: - _fail('Project JSON contains duplicate object keys.') - result[key] = value - return result - - -def _decode(raw): - try: - value = json.loads(raw.decode('utf-8'), object_pairs_hook=_pairs, - parse_constant=lambda _: _fail('Project JSON contains a nonfinite number.')) - if _encode(value) != raw: - _fail('Project JSON must use canonical UTF-8, sorted keys, two-space indentation and LF.') - return value - except ProjectStoreError: - raise - except (ValueError, RecursionError, OverflowError) as exc: - raise ProjectStoreError('Malformed project JSON.') from exc - - -def _keys(value, keys, label): - if type(value) is not dict or set(value) != set(keys): - _fail(f'{label} has invalid or unknown fields.') - - -def _identifier(value, label): - if type(value) is not str or not IDENTIFIER.fullmatch(value) or value in ('.', '..'): - _fail(f'{label} must be a bounded path-free identifier.') - - -def _digest(value, label): - if type(value) is not str or not SHA256.fullmatch(value): - _fail(f'{label} must be a SHA256 digest.') - - -def _domain_format(value, family): - match = DOMAIN_FORMAT.fullmatch(value) if type(value) is str else None - if not match: - _fail('Domain document requires a versioned format.') - if value != SUPPORTED_FORMATS[family] and not ( - family == 'project_operation' and value == 'project_operation_v2' - or family == 'project_file' and value == 'project_file_v2' - or family == 'run_history' and value == 'project_run_history_v2'): - _fail('Unsupported domain document format.', 'unsupported', 422) - - -def _compact(value): - # CPython's C encoder is used only without indent; it distinguishes 1, 1.0, - # True and 0.0/-0.0, which Python == equality does not. - return json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False, - separators=(',', ':')) - - -def _same_json(candidate, verified): - """Exact JSON value equality: same types, float bits and key sets.""" - if candidate is verified: - return True - if candidate != verified: - return False - try: - return _compact(candidate) == _compact(verified) - except (TypeError, ValueError, RecursionError): - return False - - -def _prepared_records(records, unchanged=None): - """Validate and canonically encode a complete candidate record set. - - unchanged maps (family, id) -> (verified current document, inventory entry). - A candidate document equal to its verified current revision reuses that - revision's known canonical byte count and digest instead of re-encoding it; - its raw slot is None and the caller must not rewrite it. - """ - if type(records) is not list or len(records) > MAX_RECORDS: - _fail('Project records must be a bounded list.') - unchanged = unchanged or {} - result, seen, total = [], set(), 0 - for record in records: - _keys(record, ('family', 'id', 'document'), 'Project record') - family, identity, document = (record[k] for k in ('family', 'id', 'document')) - _identifier(family, 'Record family') - if family not in SUPPORTED_FORMATS: - _fail('Unsupported project record family.', 'unsupported', 422) - _identifier(identity, 'Record ID') - if (family, identity) in seen: - _fail('Duplicate project record identity.') - seen.add((family, identity)) - if type(document) is not dict: - _fail('Domain document must be an object.') - _domain_format(document.get('format'), family) - current = unchanged.get((family, identity)) - if current is not None and _same_json(document, current[0]) \ - and current[1]['format'] == document['format']: - # Exactly equal JSON values have identical canonical bytes; share the - # already verified current document (never handed out by alias). - total += current[1]['bytes'] - if total > MAX_TOTAL_BYTES: - _fail('Project records exceed the synchronous byte limit.') - result.append((family, identity, current[0], None, current[1])) - continue - document = copy.deepcopy(document) - raw = _encode(document) - total += len(raw) - if len(raw) > MAX_RECORD_BYTES or total > MAX_TOTAL_BYTES: - _fail('Project records exceed the synchronous byte limit.') - result.append((family, identity, document, raw, None)) - return sorted(result, key=lambda item: (item[0], item[1])) - - -_NESTED_DOCUMENT = b'\n' + b' ' * 4 - - -class _RecordsDigest: - """Incremental SHA256 of _encode([dict(family, id, document), ...]). - - add() takes each record's own canonical _encode(document) bytes, in record - order. Canonical JSON never contains a literal newline inside a string, so - a document nested two levels deep differs from its standalone encoding only - by four extra spaces after each newline. The result is byte-identical to - hashing the complete re-encoded list, without re-encoding any document. - """ - def __init__(self): - self._digest = hashlib.sha256() - self._first = True - - def add(self, family, identity, raw): - if not raw.endswith(b'\n'): - _fail('Canonical record bytes must end with LF.') - self._digest.update(b'[\n {\n "document": ' if self._first else b',\n {\n "document": ') - self._first = False - self._digest.update(raw[:-1].replace(b'\n', _NESTED_DOCUMENT)) - self._digest.update(b',\n "family": ') - self._digest.update(json.dumps(family, ensure_ascii=False).encode('utf-8')) - self._digest.update(b',\n "id": ') - self._digest.update(json.dumps(identity, ensure_ascii=False).encode('utf-8')) - self._digest.update(b'\n }') - - def hexdigest(self): - final = self._digest.copy() - final.update(b'[]\n' if self._first else b'\n]\n') - return final.hexdigest() - - -def envelope_digest(family, identity, raw): - """SHA256 of _encode(dict(family, id, document)) from the document's canonical bytes. - - Canonical JSON has no literal newline inside strings, so the document nested - one level deep differs from its standalone encoding only by two spaces after - each newline. - """ - if not raw.endswith(b'\n'): - _fail('Canonical record bytes must end with LF.') - digest = hashlib.sha256(b'{\n "document": ') - digest.update(raw[:-1].replace(b'\n', b'\n ')) - digest.update(b',\n "family": ' + json.dumps(family, ensure_ascii=False).encode('utf-8')) - digest.update(b',\n "id": ' + json.dumps(identity, ensure_ascii=False).encode('utf-8') + b'\n}\n') - return digest.hexdigest() - - -def _records_digest(rows): - """Digest of (family, id, canonical raw) rows; see _RecordsDigest.""" - digest = _RecordsDigest() - for family, identity, raw in rows: - digest.add(family, identity, raw) - return digest.hexdigest() - - -def _records(records): - """Complete canonical encoding as (family, id, document, raw) tuples.""" - return [(family, identity, document, raw) - for family, identity, document, raw, _entry in _prepared_records(records)] - - -def _physical_candidates(records, logical, physical): - """Physical envelopes for a logical candidate record set. - - A candidate equal to its current logical document reuses the current - physical document (so its verified revision is kept). Changed Runs are - compacted to a shared runtime closure when exactly reversible; every - referenced closure is included once. Returns (physical candidates, logical - result in record order) or (records, None) for malformed input, which - _prepared_records then rejects with its usual errors. - """ - from project_record_dedupe import CLOSURE_FAMILY, RUN_FAMILY, compact, is_compact - if type(records) is not list or any(type(item) is not dict or set(item) != {'family', 'id', 'document'} - or type(item['family']) is not str or type(item['id']) is not str - for item in records): - return records, None - if any(item['family'] == CLOSURE_FAMILY for item in records): - _fail('Runtime closures are store-owned records.', 'unsupported', 422) - current_logical = {(item['family'], item['id']): item['document'] for item in logical} - current_physical = {(item['family'], item['id']): item['document'] for item in physical} - closures = {item['id']: item['document'] for item in physical if item['family'] == CLOSURE_FAMILY} - candidates, logical_out, referenced = [], [], set() - for item in records: - key = (item['family'], item['id']) - document = item['document'] - if key in current_logical and _same_json(document, current_logical[key]): - stored = current_physical[key] - candidates.append(dict(family=key[0], id=key[1], document=stored)) - logical_out.append(dict(family=key[0], id=key[1], document=current_logical[key])) - if is_compact(key[0], stored): - referenced.add(stored['runtime_closure']['id']) - continue - owned = copy.deepcopy(document) - logical_out.append(dict(family=key[0], id=key[1], document=owned)) - compacted = compact(owned) if key[0] == RUN_FAMILY else None - if compacted is not None: - reduced, closure = compacted - identity = reduced['runtime_closure']['id'] - closures.setdefault(identity, closure) - referenced.add(identity) - candidates.append(dict(family=key[0], id=key[1], document=reduced)) - else: - candidates.append(dict(family=key[0], id=key[1], document=owned)) - candidates.extend(dict(family=CLOSURE_FAMILY, id=identity, document=closures[identity]) for identity in sorted(referenced)) - logical_out.sort(key=lambda item: (item['family'], item['id'])) - return candidates, logical_out - - -def _request(key, digest): - if type(key) is not str or not REQUEST_KEY.fullmatch(key): - _fail('Request key must contain 1–128 supported characters.') - _digest(digest, 'Request body digest') - - -def checked_manifest(raw): - """Decode and check a project checkpoint without reading any record file. - - Verifies format, identity, the checkpoint digest, every inventory entry's - shape, ordering, rooted path and byte bounds, membership roots and receipt - structure. Record bytes and the latest receipt's records digest are checked - by whoever reads the records. - """ - manifest = _decode(raw) - if type(manifest) is not dict: - _fail('Project checkpoint must be an object.') - if manifest.get('format') != FORMAT: - if type(manifest.get('format')) is str and re.fullmatch(r'tuldok_project_v[1-9][0-9]*', manifest['format']): - _fail('Unsupported project format.', 'unsupported', 422) - _fail('Invalid project format.') - _keys(manifest, ('format', 'project_id', 'name', 'checkpoint', 'roots', 'inventory', 'receipts'), 'Project checkpoint') - identity = manifest['project_id'] - if type(identity) is not str or not re.fullmatch('[a-f0-9]{32}', identity): - _fail('Invalid project UUID.') - if type(manifest['name']) is not str or not manifest['name'].strip() or len(manifest['name']) > 200: - _fail('Invalid project display name.') - _digest(manifest['checkpoint'], 'Checkpoint') - core = {k: v for k, v in manifest.items() if k != 'checkpoint'} - if _sha(_encode(core)) != manifest['checkpoint']: - _fail('Project checkpoint digest does not match its contents.') - inventory = manifest['inventory'] - if type(inventory) is not list or len(inventory) > MAX_RECORDS: - _fail('Invalid project inventory.') - roots, total, previous = [], 0, None - for entry in inventory: - _keys(entry, ('family', 'id', 'format', 'namespace', 'path', 'bytes', 'sha256'), 'Inventory entry') - family, record_id, digest = entry['family'], entry['id'], entry['sha256'] - _identifier(family, 'Record family') - if family not in SUPPORTED_FORMATS: - _fail('Unsupported project record family.', 'unsupported', 422) - _identifier(record_id, 'Record ID') - _digest(digest, 'Record digest') - _domain_format(entry['format'], family) - key = (family, record_id) - if previous is not None and key <= previous: - _fail('Inventory must have unique sorted record identities.') - previous = key - if entry['namespace'] != identity or entry['path'] != f'records/{family}/{record_id}/{digest}.json': - _fail('Invalid inventory namespace or rooted record path.') - if type(entry['bytes']) is not int or not 0 < entry['bytes'] <= MAX_RECORD_BYTES: - _fail('Invalid inventory record byte count.') - total += entry['bytes'] - if total > MAX_TOTAL_BYTES: - _fail('Project records exceed their total byte limit.') - roots.append(dict(family=family, id=record_id)) - if manifest['roots'] != roots: - _fail('Project membership roots disagree with the inventory.') - if type(manifest['receipts']) is not list or not manifest['receipts']: - _fail('Project requires operation receipts.') - seen = set() - for receipt in manifest['receipts']: - _keys(receipt, ('request_key', 'request_sha256', 'records_sha256'), 'Operation receipt') - _request(receipt['request_key'], receipt['request_sha256']) - _digest(receipt['records_sha256'], 'Receipt records digest') - if receipt['request_key'] in seen: - _fail('Duplicate operation receipt key.') - seen.add(receipt['request_key']) - return manifest - - -def _path_string(path): - try: - value = os.fspath(path) - except TypeError as exc: - raise ProjectStoreError('Project path must be text.') from exc - if type(value) is not str: - _fail('Project path must be text.') - return value - - -def _open_directory_path(path): - """Walk every component without following symlinks, retaining the final fd.""" - try: - raw = _path_string(path) - if type(raw) is not str or '\x00' in raw or '..' in raw.split('/'): - _fail('Unsafe project root path.') - absolute = os.path.abspath(raw) - fd = os.open('/', DIR_FLAGS) - try: - for component in absolute.split('/'): - if not component: - continue - next_fd = os.open(component, DIR_FLAGS, dir_fd=fd) - os.close(fd) - fd = next_fd - return absolute, fd - except BaseException: - os.close(fd) - raise - except OSError as exc: - _io_error(exc, 'resolve project root') - - -def _relative_parts(relative): - if type(relative) is not str or not relative or len(relative) > 4096 or '\x00' in relative: - _fail('Invalid relative project directory.') - parts = relative.split('/') - if any(part in ('', '.', '..') for part in parts): - _fail('Project directory must remain beneath its configured root.') - return parts - - -def _walk(root_fd, parts): - fd = os.dup(root_fd) - try: - for component in parts: - next_fd = _directory(fd, component) - os.close(fd) - fd = next_fd - return fd - except BaseException: - os.close(fd) - raise - - -def _verify_origin(root_fd, root_path): - absolute, check = _open_directory_path(root_path) - try: - if _physical(check) != _physical(root_fd): - _fail('Configured project root changed.', 'stale', 409) - finally: - os.close(check) - return absolute - - -@contextmanager -def anchored_directory(trusted_root_fd, relative='', *, root_path): - """Borrow a rooted no-follow directory capability; caller retains root fd.""" - parts = _relative_parts(relative) if relative else [] - descriptor = None - try: - _verify_origin(trusted_root_fd, root_path) - descriptor = _walk(trusted_root_fd, parts) - yield descriptor - _verify_origin(trusted_root_fd, root_path) - current = _walk(trusted_root_fd, parts) - try: - if _physical(current) != _physical(descriptor): - _fail('Anchored directory changed during use.', 'stale', 409) - finally: - os.close(current) - except OSError as exc: - _io_error(exc, 'access anchored directory') - finally: - if descriptor is not None: - os.close(descriptor) - - -def _directory_inside(candidate_fd, ancestor_fd): - """Compare physical directory ancestry without following path aliases.""" - current = os.dup(candidate_fd) - try: - ancestor = _physical(ancestor_fd) - while True: - if _physical(current) == ancestor: - return True - parent = os.open('..', DIR_FLAGS, dir_fd=current) - if _physical(parent) == _physical(current): - os.close(parent) - return False - os.close(current) - current = parent - finally: - os.close(current) - - -def assert_separate_project_control(trusted_root_fd, relative, *, root_path, control_path): - """Reject portable project/control overlap before creating any destination. - - The caller retains its configured root handle. Existing targets must be - physically disjoint from control in both directions; for an absent leaf, - its existing anchored parent must be outside control. A broad root may - still contain disjoint project and control siblings. - """ - parts = _relative_parts(relative) - control_fd = target_fd = None - try: - _, control_fd = _open_directory_path(control_path) - with anchored_directory(trusted_root_fd, '/'.join(parts[:-1]), root_path=root_path) as parent_fd: - try: - target_fd = _directory(parent_fd, parts[-1]) - except FileNotFoundError: - overlap = _directory_inside(parent_fd, control_fd) - else: - overlap = (_directory_inside(target_fd, control_fd) or - _directory_inside(control_fd, target_fd)) - if overlap: - _fail('Keep the private workspace control directory separate from portable project directories.', 'conflict', 409) - if target_fd is not None: - current = _directory(parent_fd, parts[-1]) - try: - if _physical(current) != _physical(target_fd): - _fail('Project directory changed during control separation check.', 'stale', 409) - finally: - os.close(current) - _verify_origin(control_fd, control_path) - except OSError as error: - _io_error(error, 'verify project/control separation') - finally: - if target_fd is not None: - os.close(target_fd) - if control_fd is not None: - os.close(control_fd) - - -def _rename_noreplace(parent_fd, staging, destination): - # Linux's RENAME_NOREPLACE provides the required atomic no-overwrite promise. - libc = ctypes.CDLL(None, use_errno=True) - rename = getattr(libc, 'renameat2', None) - if rename is None: - _fail('Atomic no-replace publication is unsupported.', 'unsupported', 422) - rename.argtypes = (ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint) - rename.restype = ctypes.c_int - if rename(parent_fd, os.fsencode(staging), parent_fd, os.fsencode(destination), 1): - number = ctypes.get_errno() - if number in (errno.ENOSYS, errno.EINVAL, errno.EOPNOTSUPP): - _fail('Filesystem lacks atomic no-replace publication.', 'unsupported', 422) - raise OSError(number, os.strerror(number)) - - -def _publish_directory(parent_fd, staging, destination): - _rename_noreplace(parent_fd, staging, destination) - - -def _io_error(exc, operation, unknown=False): - if unknown: - raise ProjectStoreError(f'Project publication outcome is unknown ({operation}); reopen and reconcile the request.', - 'unknown', 503) from exc - if exc.errno in (errno.ELOOP, errno.ENOTDIR, errno.ENXIO): - raise ProjectStoreError(f'Unsafe filesystem object during {operation}.') from exc - raise ProjectStoreError(f'Cannot {operation}.', 'unavailable', 503) from exc - - -def _physical(fd): - info = os.fstat(fd) - return info.st_dev, info.st_ino - - -def _regular(fd): - info = os.fstat(fd) - if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: - _fail('Project files must be ordinary files with no hard-link aliases.') - return info - - -def _read(directory, name, maximum): - fd = os.open(name, FILE_FLAGS, dir_fd=directory) - try: - info = _regular(fd) - if info.st_size > maximum: - _fail('Project file exceeds its byte limit.') - chunks, size = [], 0 - while True: - chunk = os.read(fd, min(1024 * 1024, maximum + 1 - size)) - if not chunk: - break - chunks.append(chunk) - size += len(chunk) - if size > maximum: - _fail('Project file exceeds its byte limit.') - after = _regular(fd) - if (info.st_size, info.st_mtime_ns, info.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns): - _fail('Project file changed while reading.', 'stale', 409) - return b''.join(chunks) - finally: - os.close(fd) - - -def _sync_file(directory, name): - fd = os.open(name, FILE_FLAGS, dir_fd=directory) - try: - _regular(fd) - os.fsync(fd) - finally: - os.close(fd) - - -def _directory(parent, name, create=False): - if create: - try: - os.mkdir(name, 0o700, dir_fd=parent) - os.fsync(parent) - except FileExistsError: - pass - return os.open(name, DIR_FLAGS, dir_fd=parent) - - -def _write_file(directory, name, raw): - fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, - 0o600, dir_fd=directory) - try: - _regular(fd) - remaining = memoryview(raw) - while remaining: - written = os.write(fd, remaining) - if not written: - raise OSError(errno.EIO, 'short project write') - remaining = remaining[written:] - os.fsync(fd) - finally: - os.close(fd) - - -class ProjectStore: - """Own one writable physical project root until close(). - - commit replaces the complete current record set. Retained unreferenced - revisions are never pruned. Callers validate domain semantics before commit - and after Open; this owner validates only the generic JSON/inventory contract. - """ - def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path, canonical_digests=None): - self.path = Path(path) - # Optional private set of digests already proven canonical; see _load. - self._canonical_digests = canonical_digests - self._root_fd = root_fd - self._origin_fd = origin_fd - self._relative_parts = relative_parts - self._origin_path = Path(origin_path) - self._meta_fd = None - self._manifest = None - self._read_cache = None - self._mutex = threading.RLock() - self._unknown = False - self._folder_mode = False - try: - fcntl.flock(root_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError as exc: - os.close(root_fd) - os.close(origin_fd) - self._origin_fd = None - self._root_fd = None - if exc.errno in (errno.EACCES, errno.EAGAIN): - raise ProjectStoreError('Project already has a writer.', 'busy', 409) from exc - _io_error(exc, 'lock project root') - - @classmethod - def create(cls, path, name, request_key, request_sha256): - raw = _path_string(path) - if type(raw) is not str or '\x00' in raw or '..' in raw.split('/'): - _fail('Unsafe project destination.') - absolute = os.path.abspath(raw) - parent, leaf = os.path.split(absolute) - _, parent_fd = _open_directory_path(parent) - try: - return cls.create_at(parent_fd, leaf, name, request_key, request_sha256, root_path=parent) - finally: - os.close(parent_fd) - - @classmethod - def create_at(cls, trusted_root_fd, relative, name, request_key, request_sha256, *, root_path): - """Create under a configured root capability; duplicate its fd for store lifetime.""" - _request(request_key, request_sha256) - if type(name) is not str or not name.strip() or len(name) > 200: - _fail('Project name must contain 1–200 characters.') - _json_value(name) - parts = _relative_parts(relative) - absolute = _verify_origin(trusted_root_fd, root_path) - origin_fd = os.dup(trusted_root_fd) - parent_fd = None - store = None - published = False - try: - parent_fd = _walk(origin_fd, parts[:-1]) - # Existing destinations, including symlinks, are never merged or replaced. - try: - os.stat(parts[-1], dir_fd=parent_fd, follow_symlinks=False) - except FileNotFoundError: - pass - else: - _fail('Project destination already exists.', 'conflict', 409) - staging = '.tuldok-new-' + uuid.uuid4().hex - os.mkdir(staging, 0o700, dir_fd=parent_fd) - root_fd = _directory(parent_fd, staging) - owned_origin, origin_fd = origin_fd, None - store = cls(Path(absolute).joinpath(*parts), root_fd, owned_origin, - parts[:-1] + [staging], absolute) - store._meta_fd = _directory(root_fd, '.tuldok', create=True) - records_fd = _directory(store._meta_fd, 'records', create=True) - os.close(records_fd) - _write_file(root_fd, '.gitignore', GITIGNORE) - os.fsync(root_fd) - manifest = dict(format=FORMAT, project_id=uuid.uuid4().hex, name=name, - roots=[], inventory=[], receipts=[dict(request_key=request_key, - request_sha256=request_sha256, records_sha256=_sha(_encode([])))]) - manifest['checkpoint'] = _sha(_encode(manifest)) - store._publish(manifest, creating=True) - store._manifest = manifest - store._binding() - _publish_directory(parent_fd, staging, parts[-1]) - published = True - store._relative_parts = parts - os.fsync(parent_fd) - store._binding() - return store - except FileExistsError as exc: - if store is not None: - store.close() - raise ProjectStoreError('Project destination already exists.', 'conflict', 409) from exc - except ProjectStoreError as exc: - if store is not None: - store.close() - if published: - raise ProjectStoreError('Project destination was published but its binding changed; reopen and reconcile the request.', - 'unknown', 503) from exc - raise - except OSError as exc: - if store is not None: - store.close() - _io_error(exc, 'create project', unknown=published) - except BaseException: - if store is not None: - store.close() - raise - finally: - if parent_fd is not None: - os.close(parent_fd) - if origin_fd is not None: - os.close(origin_fd) - - @classmethod - def open(cls, path, *, canonical_digests=None): - raw = _path_string(path) - if type(raw) is not str or '\x00' in raw or '..' in raw.split('/'): - _fail('Unsafe project root path.') - absolute = os.path.abspath(raw) - parent, leaf = os.path.split(absolute) - _, parent_fd = _open_directory_path(parent) - try: - return cls.open_at(parent_fd, leaf, root_path=parent, canonical_digests=canonical_digests) - finally: - os.close(parent_fd) - - @classmethod - def open_at(cls, trusted_root_fd, relative, *, root_path, canonical_digests=None): - """Open a descendant without following any configured-root or child alias.""" - parts = _relative_parts(relative) - absolute = _verify_origin(trusted_root_fd, root_path) - origin_fd = os.dup(trusted_root_fd) - store = None - try: - fd = _walk(origin_fd, parts) - owned_origin, origin_fd = origin_fd, None - store = cls(Path(absolute).joinpath(*parts), fd, owned_origin, parts, absolute, - canonical_digests=canonical_digests) - store._meta_fd = _directory(fd, '.tuldok') - store._manifest, _ = store._load() - return store - except OSError as exc: - if store is not None: - store.close() - _io_error(exc, 'open project') - except BaseException: - if store is not None: - store.close() - raise - finally: - if origin_fd is not None: - os.close(origin_fd) - - # ---- Folder mode: the external project as an import source/export target. - # A folder-mode Store holds the writer lock and root binding but never - # loads the record set; the database working copy owns current records. - - @classmethod - def open_folder_at(cls, trusted_root_fd, relative, *, root_path): - """Bind an existing external project folder without reading its records.""" - parts = _relative_parts(relative) - absolute = _verify_origin(trusted_root_fd, root_path) - origin_fd = os.dup(trusted_root_fd) - store = None - try: - fd = _walk(origin_fd, parts) - owned_origin, origin_fd = origin_fd, None - store = cls(Path(absolute).joinpath(*parts), fd, owned_origin, parts, absolute) - store._meta_fd = _directory(fd, '.tuldok') - store._folder_mode = True - return store - except OSError as exc: - if store is not None: - store.close() - _io_error(exc, 'open project folder') - except BaseException: - if store is not None: - store.close() - raise - finally: - if origin_fd is not None: - os.close(origin_fd) - - def folder_checkpoint(self): - """(raw bytes, checked manifest) of the folder's current project.json.""" - with self._mutex: - self._binding() - try: - raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) - except OSError as exc: - _io_error(exc, 'read project checkpoint') - return raw, checked_manifest(raw) - - def folder_record(self, entry): - """One inventory entry's exact bytes after length/SHA256 verification.""" - with self._mutex: - try: - raw = self._record_file(entry['family'], entry['id'], entry['sha256']) - except OSError as exc: - _io_error(exc, 'read project record') - if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: - _fail('Canonical record bytes changed or are corrupt.') - return raw - - def folder_has_record(self, entry): - """Whether the content-addressed revision file exists with its byte length.""" - with self._mutex: - fds = [] - try: - parent = self._meta_fd - for part in ('records', entry['family'], entry['id']): - parent = _directory(parent, part) - fds.append(parent) - info = os.stat(entry['sha256'] + '.json', dir_fd=parent, follow_symlinks=False) - return stat.S_ISREG(info.st_mode) and info.st_size == entry['bytes'] - except FileNotFoundError: - return False - except OSError as exc: - _io_error(exc, 'inspect project record') - finally: - for fd in reversed(fds): - os.close(fd) - - def folder_write_record(self, family, record_id, digest, raw): - """Durably publish one immutable revision; existing bytes must be identical.""" - with self._mutex: - try: - written = self._record_file(family, record_id, digest, raw) - except OSError as exc: - _io_error(exc, 'write project record') - if written != raw: - _fail('Immutable project revision already has different bytes.') - - def folder_publish(self, raw, expected_raw): - """Atomically replace project.json only if it still has expected_raw. - - expected_raw None means the folder has no checkpoint yet. Refuses to - overwrite a checkpoint someone else published since import/export. - """ - if len(raw) > MAX_MANIFEST_BYTES: - _fail('Project checkpoint exceeds its byte limit.') - with self._mutex: - temporary = '.checkpoint-' + uuid.uuid4().hex + '.tmp' - replaced = False - try: - _write_file(self._meta_fd, temporary, raw) - self._binding() - try: - current = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) - except FileNotFoundError: - current = None - if current != expected_raw: - os.unlink(temporary, dir_fd=self._meta_fd) - _fail('The external project changed since Tuldok last imported or saved it.', 'conflict', 409) - os.replace(temporary, 'project.json', src_dir_fd=self._meta_fd, dst_dir_fd=self._meta_fd) - replaced = True - os.fsync(self._meta_fd) - os.fsync(self._root_fd) - except OSError as exc: - self._unknown = replaced - _io_error(exc, 'publish checkpoint', unknown=replaced) - - def _binding(self): - if self._root_fd is None: - _fail('Project store is closed.', 'unavailable', 409) - if self._unknown: - _fail('Reopen project to reconcile an unknown publication.', 'unknown', 503) - _verify_origin(self._origin_fd, self._origin_path) - check = _walk(self._origin_fd, self._relative_parts) - try: - if _physical(check) != _physical(self._root_fd): - _fail('Project root changed; close and reopen.', 'stale', 409) - meta = _directory(check, '.tuldok') - try: - if _physical(meta) != _physical(self._meta_fd): - _fail('Project metadata directory changed.', 'stale', 409) - finally: - os.close(meta) - finally: - os.close(check) - - @contextmanager - def root_directory(self, *, verify_after=False): - """Borrow this project's physical root while its writer binding is pinned.""" - if type(verify_after) is not bool: - _fail('Root read verification must be an explicit boolean.') - with self._mutex: - before = self._current()[0]['checkpoint'] - descriptor = os.dup(self._root_fd) - try: - yield descriptor - self._binding() - if verify_after and self._current()[0]['checkpoint'] != before: - _fail('Project checkpoint changed during the bounded root read.', 'stale', 409) - finally: - os.close(descriptor) - - def _load(self): - self._binding() - raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) - cached = self._read_cache - if cached is not None and raw == cached[0]: - # Record files are content-addressed by their inventory digest and - # are never rewritten, and every one was length/SHA256 verified when - # this Store decoded it. Byte-identical checkpoint bytes therefore - # name the same verified record set; re-hashing every record on each - # freshness check is redundant. On-disk corruption after that point - # is still refused by Save (verify_records) and by every reopen. - self._binding() - return cached[1], cached[2] - manifest = checked_manifest(raw) - loaded = [] - records_digest = _RecordsDigest() - known = self._canonical_digests - proven = [] - for entry in manifest['inventory']: - family, record_id, digest = entry['family'], entry['id'], entry['sha256'] - record_raw = self._verified_record_bytes(entry) - if known is not None and digest in known: - # These exact bytes (named by their verified SHA256) were already - # proven to be their value's canonical encoding, which excludes - # duplicate keys and non-finite numbers; a plain parse is exact. - document = json.loads(record_raw) - else: - document = _decode(record_raw) - proven.append(digest) - if type(document) is not dict or document.get('format') != entry['format']: - _fail('Domain document format disagrees with its inventory.') - records_digest.add(family, record_id, record_raw) - loaded.append(dict(family=family, id=record_id, document=document)) - # _decode proved each raw revision is its document's canonical encoding. - if manifest['receipts'][-1]['records_sha256'] != records_digest.hexdigest(): - _fail('Latest operation receipt disagrees with the current record set.') - # A second pointer read detects checkout/replacement during closure validation. - if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: - _fail('Project checkpoint changed while reading.', 'stale', 409) - self._binding() - if known is not None and proven: - known.add_many(proven) - # Domain owners see logical records: shared runtime closures hidden and - # compact Runs expanded to their exact original documents. - from project_record_dedupe import logical_records - logical = logical_records(loaded) - self._read_cache = (raw, manifest, logical, loaded) - return manifest, logical - - def _verified_record_bytes(self, entry): - raw = self._record_file(entry['family'], entry['id'], entry['sha256']) - if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: - _fail('Canonical record bytes changed or are corrupt.', - 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) - return raw - - def _record_file(self, family, record_id, digest, raw=None, sync=False): - fds = [] - try: - parent = self._meta_fd - for part in ('records', family, record_id): - parent = _directory(parent, part, create=raw is not None) - fds.append(parent) - name = digest + '.json' - if raw is not None: - try: - existing = _read(parent, name, MAX_RECORD_BYTES) - except FileNotFoundError: - temporary = '.record-' + uuid.uuid4().hex + '.tmp' - _write_file(parent, temporary, raw) - try: - _rename_noreplace(parent, temporary, name) - except FileExistsError: - if _read(parent, name, MAX_RECORD_BYTES) != raw: - _fail('Immutable project revision already has different bytes.') - os.unlink(temporary, dir_fd=parent) - else: - if existing != raw: - _fail('Immutable project revision already has different bytes.') - # Existing revisions can come from a copied project or an - # interrupted earlier commit. Retain all ordinary sync checks; - # only the disposable write/rename/unlink cycle is avoided. - sync = True - if sync: - _sync_file(parent, name) - for fd in reversed(fds): - os.fsync(fd) - os.fsync(self._meta_fd) - return _read(parent, name, MAX_RECORD_BYTES) - finally: - for fd in reversed(fds): - os.close(fd) - - def _publish(self, manifest, creating=False): - raw = _manifest_raw(manifest) - temporary = '.checkpoint-' + uuid.uuid4().hex + '.tmp' - replaced = False - try: - _write_file(self._meta_fd, temporary, raw) - self._binding() - if not creating: - current, _ = self._load() - if current != self._manifest: - _fail('Project checkpoint changed before publication.', 'stale', 409) - # Creation owns a new exclusive directory; commit owns its root flock. - if creating: - _rename_noreplace(self._meta_fd, temporary, 'project.json') - replaced = True - else: - os.replace(temporary, 'project.json', src_dir_fd=self._meta_fd, dst_dir_fd=self._meta_fd) - replaced = True - os.fsync(self._meta_fd) - os.fsync(self._root_fd) - except OSError as exc: - self._unknown = replaced - _io_error(exc, 'publish checkpoint', unknown=replaced) - return raw - # Temporary/orphan files remain inert after failures and process interruption. - - def _snapshot(self, manifest, records): - return dict(project_id=manifest['project_id'], name=manifest['name'], - checkpoint=manifest['checkpoint'], records=copy_validated_json(records)) - - def _current(self): - try: - manifest, records = self._load() - if manifest != self._manifest: - _fail('Project checkpoint changed; close and reopen.', 'stale', 409) - return manifest, records - except OSError as exc: - _io_error(exc, 'read project snapshot') - - def verify_records(self): - """Re-read and SHA256-check every referenced canonical revision on disk. - - Ordinary freshness checks compare checkpoint bytes only; this explicit - pass detects later same-length corruption of already-decoded revisions. - """ - with self._mutex: - manifest, _records = self._current() - try: - for entry in manifest['inventory']: - self._verified_record_bytes(entry) - except OSError as exc: - _io_error(exc, 'verify project records') - if self._current()[0] != manifest: - _fail('Project checkpoint changed during verification.', 'stale', 409) - - def summary(self): - """Return scalar metadata after the checkpoint freshness check.""" - with self._mutex: - manifest, _records = self._current() - return {key: manifest[key] for key in ('project_id', 'name', 'checkpoint')} - - def snapshot(self): - with self._mutex: - manifest, records = self._current() - return self._snapshot(manifest, records) - - def reconcile(self, request_key, request_sha256): - """Inspect retry identity before a domain mutation; never repeat effects.""" - _request(request_key, request_sha256) - with self._mutex: - snapshot = self.snapshot() - for receipt in self._manifest['receipts']: - if receipt['request_key'] == request_key: - if receipt['request_sha256'] != request_sha256: - _fail('Request key was already used with different input.', 'conflict', 409) - return snapshot - return None - - def commit(self, records, expected_checkpoint, request_key, request_sha256): - _request(request_key, request_sha256) - _digest(expected_checkpoint, 'Expected checkpoint') - with self._mutex: - try: - current, loaded = self._load() - if current != self._manifest: - _fail('Project checkpoint changed; close and reopen.', 'stale', 409) - physical = self._read_cache[3] - # Unchanged records keep their verified canonical revision: no - # re-encoding, rewrite or read-back of identical immutable bytes. - entries = {(entry['family'], entry['id']): entry for entry in current['inventory']} - unchanged = {(item['family'], item['id']): (item['document'], entries[(item['family'], item['id'])]) - for item in physical} - candidates, logical_out = _physical_candidates(records, loaded, physical) - prepared = _prepared_records(candidates, unchanged) - if logical_out is None: - logical_out = [dict(family=f, id=i, document=d) for f, i, d, _, _ in prepared] - digest_rows = _RecordsDigest() - for family, identity, _document, raw, entry in prepared: - if raw is None: - raw = self._verified_record_bytes(entry) - digest_rows.add(family, identity, raw) - records_digest = digest_rows.hexdigest() - for receipt in current['receipts']: - if receipt['request_key'] == request_key: - if receipt['request_sha256'] != request_sha256 or receipt['records_sha256'] != records_digest: - _fail('Request key was already used with different input.', 'conflict', 409) - return self._snapshot(current, loaded) - if expected_checkpoint != current['checkpoint']: - _fail('Expected project checkpoint is stale.', 'stale', 409) - manifest = copy.deepcopy(current) - inventory = [] - for family, identity, document, raw, entry in prepared: - if raw is None: - inventory.append(copy.deepcopy(entry)) - continue - digest = _sha(raw) - inventory.append(dict(family=family, id=identity, format=document['format'], - namespace=current['project_id'], path=f'records/{family}/{identity}/{digest}.json', - bytes=len(raw), sha256=digest)) - manifest['inventory'] = inventory - manifest['roots'] = [dict(family=f, id=i) for f, i, _, _, _ in prepared] - manifest['receipts'].append(dict(request_key=request_key, - request_sha256=request_sha256, records_sha256=records_digest)) - del manifest['checkpoint'] - manifest['checkpoint'] = _sha(_encode(manifest)) - # Refuse before record staging. The final publication also - # rechecks this bound after the immutable files are durable. - _manifest_raw(manifest) - for family, identity, document, raw, _entry in prepared: - if raw is not None: - self._record_file(family, identity, _sha(raw), raw) - published = self._publish(manifest) - self._manifest = manifest - if self._canonical_digests is not None: - # Freshly written revisions are canonical by construction. - self._canonical_digests.add_many( - [_sha(raw) for _f, _i, _d, raw, _e in prepared if raw is not None]) - # Every committed revision is either freshly written and read - # back byte-for-byte or an unchanged verified revision. - self._read_cache = (published, manifest, logical_out, - [dict(family=f, id=i, document=d) for f, i, d, _, _ in prepared]) - return self._snapshot(manifest, logical_out) - except OSError as exc: - _io_error(exc, 'commit project records') - - def save(self, expected_checkpoint): - _digest(expected_checkpoint, 'Expected checkpoint') - with self._mutex: - snapshot = self.snapshot() - if expected_checkpoint != snapshot['checkpoint']: - _fail('Expected project checkpoint is stale.', 'stale', 409) - try: - # Save is the explicit durability and integrity boundary: sync and - # re-verify every referenced canonical revision's actual bytes. - for entry in self._manifest['inventory']: - raw = self._record_file(entry['family'], entry['id'], entry['sha256'], sync=True) - if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: - _fail('Canonical record bytes changed or are corrupt.', 'stale', 409) - _sync_file(self._meta_fd, 'project.json') - os.fsync(self._meta_fd) - os.fsync(self._root_fd) - # Imported/copied roots may not yet have a durable directory entry. - parent_fd = _walk(self._origin_fd, self._relative_parts[:-1]) - try: - os.fsync(parent_fd) - finally: - os.close(parent_fd) - if self.snapshot() != snapshot: - _fail('Project changed during Save.', 'stale', 409) - except OSError as exc: - _io_error(exc, 'save project checkpoint') - return snapshot - - def recover_checkpoint(self): - """Explicitly re-read and sync this retained physical writer binding. - - Execution callers must first fence dispatch and drain actual local - custody. This method resolves only filesystem publication uncertainty; - it does not select a SQL outcome or authorize any work to resume. - """ - with self._mutex: - self._unknown = False - try: - manifest, records = self._load() - self._manifest = manifest - return self.save(self._snapshot(manifest, records)['checkpoint']) - except BaseException: - self._unknown = True - raise - - def close(self): - with self._mutex: - self._read_cache = None - if self._meta_fd is not None: - os.close(self._meta_fd) - self._meta_fd = None - if self._root_fd is not None: - # Closing releases the flock. Explicit unlock would release it in - # an inherited descriptor shared by a forked process too. - os.close(self._root_fd) - self._root_fd = None - if self._origin_fd is not None: - os.close(self._origin_fd) - self._origin_fd = None - - def __enter__(self): - return self - - def __exit__(self, *exc): - self.close() diff --git a/project_study_execution.py b/project_study_execution.py deleted file mode 100644 index af7ed930..00000000 --- a/project_study_execution.py +++ /dev/null @@ -1,811 +0,0 @@ -"""Current Study execution composed with exact inert project accounting. - -Only authoring metadata is reconstructed. Historical reservations and Runs stay -in canonical history; they contribute charges but never become worker, grant, -admission, replay, or cancellation rows in this physical runtime. -""" -import copy -from datetime import datetime, timezone -import hashlib -import json -import time -from types import SimpleNamespace - -from experiments import (Experiments, budget, request_key, validate_retained_reservation) -from model_definitions import natural, text -from pipeline import closed, identifier, digest -from project_history import (FAMILY_FORMATS, _audit, _sha, envelope, - decode_records as history_records) -from project_material import ProjectMaterial, MaterialSnapshot -from project_session import operation_records -from workbench import WorkbenchError, encode, timestamp -from project_session import session_operations, session_records - -PREPARATIONS={'study_execution.prepare_package':'studies.prepare_package', - 'study_execution.prepare_verifier':'studies.prepare_verifier'} -INTENTS={operation+'_intent':native for operation,native in PREPARATIONS.items()} -ABANDON='study_execution.abandon_preparation' -OPERATIONS=frozenset(('study_execution.admit','study_execution.stop','study_execution.approve',ABANDON,*PREPARATIONS,*INTENTS)) -GRANT_FIELDS=('id','catalog_id','approver_actor_id','approver_credential_id','consumer_actor_id', - 'material_sha256','created_at','expires_at','revoked_at') -RESERVATION_FIELDS=('id','study_id','admission_epoch','request_key','actor_id','trial_id','parent_run_id', - 'run_id','request_sha256','material_sha256','envelope','state','settled') - - -def _invalid(message,code='invalid'): - raise WorkbenchError(message,code,409) - - -def _facts(records): - return history_records(records,materials=MaterialSnapshot(records)) - - -def _intent_key(operation,key): - return 'candidate-'+digest(dict(key=key,operation=operation))+'-intent' - - -def _preparation_body(intent,key): - body=dict(request_key=key,revision=intent['study_revision'],candidate_id=intent['candidate']['id'], - root=intent['root'],directory=intent['directory'],runtime_snapshot_sha256=intent['runtime_snapshot_sha256']) - if 'package_run_id' in intent: - body.update(package_run_id=intent['package_run_id'],package_sha256=intent['package_sha256']) - return body - - -def _preparation_reference(records,study_id,key): - receipts=operation_records(records) - matches=[(operation,receipts[_intent_key(operation,key)]) for operation in PREPARATIONS - if _intent_key(operation,key) in receipts] - if len(matches)!=1:_invalid('Preparation requires one exact retained intent.','unavailable') - operation,receipt=matches[0] - if receipt['operation']!=operation+'_intent' or receipt['result']['study_id']!=study_id: - _invalid('Preparation intent belongs to another operation or Study.','conflict') - intent=receipt['result'];complete=receipts.get(key) - if complete is None: - # The native final Trial transaction publishes its exact scientific - # receipt before the outer transport acknowledgement. That fact closes - # preparation even when the later wrapper receipt was never published. - native_sha=digest(dict(study_id=study_id,body=_preparation_body(intent,key))) - native=[] - for record in records: - if record['family']!='study_history':continue - doc=record['document'] - if doc['origin_catalog_id']!=intent['origin_catalog_id'] or doc['study']['id']!=study_id:continue - native.extend(value for value in doc['operations'] if value['actor_id']==intent['actor_id'] - and value['operation_id']==PREPARATIONS[operation] and value['request_key']==key) - if len(native)>1:_invalid('Preparation has duplicate native completion facts.') - if native: - if native[0]['request_sha256']!=native_sha:_invalid('Native preparation completion differs from its exact intent.','conflict') - validate_operation_result(operation,native[0]['result'],records) - complete=dict(operation=operation,result=copy.deepcopy(native[0]['result']),native_receipt=True) - return operation,intent,complete - - -def _validate_abandon(result,records): - closed(result,('format','origin_catalog_id','study_id','preparation_key','preparation_operation', - 'input_sha256','candidate','actor_id','reason','observed_checkpoint','occurred_at','state')) - if result['format']!='project_candidate_preparation_abandon_v1' or result['state']!='abandoned': - _invalid('Unsupported candidate preparation abandonment.','unsupported') - for field in ('origin_catalog_id','study_id','actor_id'):identifier(result[field]) - request_key(result['preparation_key']);text(result['reason'],'Abandonment reason',1000) - for field in ('input_sha256','observed_checkpoint'):_sha(result[field]) - if result['preparation_operation'] not in PREPARATIONS:_invalid('Unknown preparation operation.') - from model_definitions import reference - reference(result['candidate'],'candidate',result['origin_catalog_id']) - if datetime.fromisoformat(result['occurred_at']).utcoffset() is None:_invalid('Abandonment time requires an offset.') - if records is not None: - operation,intent,complete=_preparation_reference(records,result['study_id'],result['preparation_key']) - if (operation!=result['preparation_operation'] or intent['origin_catalog_id']!=result['origin_catalog_id'] - or intent['candidate']!=result['candidate'] or intent['request_sha256']!=result['input_sha256']): - _invalid('Abandonment differs from its exact retained preparation intent.') - if complete is not None:_invalid('Completed scientific preparation cannot be abandoned.') - return True - - -def _validate_preparation(operation,result,records): - from candidate_packages import NAME,decode_preparation_receipt - from experiments import validate_retained_trial - from model_definitions import reference - intent=operation in INTENTS;native=INTENTS[operation] if intent else PREPARATIONS[operation] - if intent: - fields=('format','origin_catalog_id','study_id','study_revision','candidate','actor_id', - 'request_sha256','root','directory','runtime_snapshot_sha256') - if native=='studies.prepare_verifier':fields+=('package_run_id','package_sha256') - closed(result,fields) - if result['format']!='project_candidate_preparation_intent_v1':_invalid('Unsupported candidate preparation intent.','unsupported') - for field in ('origin_catalog_id','study_id','actor_id'):identifier(result[field]) - natural(result['study_revision'],'Study revision',1) - for field in ('request_sha256','runtime_snapshot_sha256'):_sha(result[field]) - reference(result['candidate'],'candidate',result['origin_catalog_id']) - for field in ('root','directory'): - if type(result[field]) is not str or NAME.fullmatch(result[field]) is None:_invalid('Invalid preparation location.') - if native=='studies.prepare_verifier':identifier(result['package_run_id']);_sha(result['package_sha256']) - else: - origin=result['trial']['material']['catalog_id'] - decode_preparation_receipt(native,result,catalog_id=origin, - validate_trial=lambda value:validate_retained_trial(value,catalog_id=origin)) - if records is None:return True - facts=_facts(records) - if intent: - origin=result['origin_catalog_id'];candidate=facts.candidates[(origin,result['candidate']['id'])]['candidate'] - if candidate['study_id']!=result['study_id'] or digest(candidate)!=result['candidate']['sha256']: - _invalid('Preparation intent differs from its immutable candidate.') - study=facts.studies[(origin,result['study_id'])] - if not any(value['revision']==result['study_revision'] for value in study['revisions']): - _invalid('Preparation intent lacks its exact Study revision.') - if native=='studies.prepare_verifier': - run=facts.runs[(origin,result['package_run_id'])]['run'] - if not any(item['name']=='package.zip' and item['sha256']==result['package_sha256'] for item in run['artifacts']): - _invalid('Verifier intent differs from the retained package bytes.') - else: - from project_history import validate_operation_result as validate_history_result - validate_history_result(native,result,records) - trial=result['trial'];origin=trial['material']['catalog_id'] - candidate=facts.candidates[(origin,result['candidate_id'])]['candidate'] - if candidate['study_id']!=trial['study_id']:_invalid('Preparation candidate belongs to another Study.') - materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check_current=lambda:None)) - composition=materials.recipe(trial['material'])['material'] - if (composition['model']!=result['model'] or composition['recipe_id']!=result['graph']['id'] - or composition['recipe_revision']!=result['graph']['revision']): - _invalid('Preparation model or graph differs from its retained composition.') - descriptor=materials.runtime(composition['runtime'])['descriptor'] - if descriptor is None or descriptor['manifest_sha256']!=result['runtime_snapshot_sha256']: - _invalid('Preparation runtime differs from its retained composition.') - return True - - -def validate_operation_result(operation,result,records=None): - """Closed scientific acknowledgements; none grants current authority.""" - if operation not in OPERATIONS:return False - try: - if operation==ABANDON:return _validate_abandon(result,records) - if operation in PREPARATIONS or operation in INTENTS: - return _validate_preparation(operation,result,records) - if operation=='study_execution.admit': - closed(result,('format','origin_catalog_id','study_id','trial_id','run_id','reservation')) - if result['format']!='project_study_admission_v1':_invalid('Unsupported Study admission receipt.','unsupported') - for key in ('origin_catalog_id','study_id','trial_id','run_id'):identifier(result[key]) - reservation=result['reservation'];closed(reservation,RESERVATION_FIELDS) - for key in ('id','study_id','actor_id','trial_id','run_id'):identifier(reservation[key]) - if reservation['parent_run_id'] is not None:identifier(reservation['parent_run_id']) - natural(reservation['admission_epoch'],'Admission epoch');request_key(reservation['request_key']) - for key in ('request_sha256','material_sha256'):_sha(reservation[key]) - budget(reservation['envelope']) - if reservation['state'] not in ('reserved','running','settled','unknown'):_invalid('Invalid reservation observation.') - if reservation['settled'] is not None:budget(reservation['settled']) - if any(result[key]!=reservation[key] for key in ('study_id','trial_id','run_id')): - _invalid('Study admission references differ from their reservation.') - elif operation=='study_execution.stop': - closed(result,('format','origin_catalog_id','study_id','receipt','study_revision','state','open')) - if result['format']!='project_study_stop_v1':_invalid('Unsupported Study stop receipt.','unsupported') - identifier(result['origin_catalog_id']);identifier(result['study_id']);natural(result['study_revision'],'Study revision',1) - receipt=result['receipt'];closed(receipt,('study_id','stop_receipt_id','admission_epoch','run_ids')) - identifier(receipt['study_id']);identifier(receipt['stop_receipt_id']);natural(receipt['admission_epoch'],'Stop epoch') - if (receipt['study_id']!=result['study_id'] or result['open'] is not False - or result['state'] not in ('stopping','stopped','unknown') - or type(receipt['run_ids']) is not list or len(receipt['run_ids'])>10000):_invalid('Invalid Study epoch closure.') - for identity in receipt['run_ids']:identifier(identity) - if len(set(receipt['run_ids']))!=len(receipt['run_ids']):_invalid('Duplicate stopped Run identity.') - else: - closed(result,('format','origin_catalog_id','study_id','trial_id','approval_id','material_sha256', - 'consumer_actor_id','approver_actor_id','created_at','expires_at')) - if result['format']!='project_study_approval_v1':_invalid('Unsupported Study approval observation.','unsupported') - identifier(result['study_id']);identifier(result['trial_id']) - from project_execution import validate_operation_result as validate_execution_result - base={key:result[key] for key in ('origin_catalog_id','approval_id','material_sha256', - 'consumer_actor_id','approver_actor_id','created_at','expires_at')} - validate_execution_result('runs.approve_external',dict(format='project_execution_approval_v1',**base)) - if records is None:return True - facts=_facts(records) - if operation=='study_execution.admit': - origin=result['origin_catalog_id'];reservation=result['reservation'] - saved=facts.reservations.get((origin,result['run_id'])) - if saved is None or any(saved[key]!=reservation[key] for key in RESERVATION_FIELDS if key not in ('state','settled')): - _invalid('Admission receipt has no exact canonical reservation.') - trial=facts.trials[(origin,result['trial_id'])]['trial'];run=facts.runs[(origin,result['run_id'])]['run'] - view=SimpleNamespace(snapshot={'records':records},check_current=lambda:None) - validate_retained_reservation(reservation,trial=trial, - composition=ProjectMaterial(view).recipe(trial['material']),run=run) - elif operation=='study_execution.stop': - doc=facts.studies[(result['origin_catalog_id'],result['study_id'])];receipt=result['receipt'] - matches=[value for value in doc['revisions'] if value['revision']==result['study_revision']] - if (len(matches)!=1 or matches[0]['stop_receipt_id']!=receipt['stop_receipt_id'] - or matches[0]['admission_epoch']!=receipt['admission_epoch'] - or matches[0]['state']!=result['state'] or matches[0]['open']!=result['open']): - _invalid('Stop receipt has no exact retained epoch closure.') - if not set(receipt['run_ids'])<={value['run_id'] for value in doc['reservations']}: - _invalid('Stop receipt targets an unrelated Run.') - else: - origin=result['origin_catalog_id'];doc=facts.studies[(origin,result['study_id'])] - trial=facts.trials[(origin,result['trial_id'])]['trial'] - if trial['study_id']!=result['study_id'] or trial['material']['sha256']!=result['material_sha256']: - _invalid('Approval observation differs from its exact retained Trial.') - expected={key:result[key] for key in ('trial_id','material_sha256','approval_id','consumer_actor_id','created_at','expires_at')} - matches=[fact for fact in doc['audit'] if fact['action']=='studies.approve' - and fact['target']==result['study_id'] and fact['actor_id']==result['approver_actor_id'] - and fact['details']==expected] - if len(matches)!=1:_invalid('Approval receipt has no exact historical Study approval observation.') - return True - except WorkbenchError:raise - except (KeyError,TypeError,ValueError,AttributeError,OverflowError): - _invalid('Malformed current Study operation receipt.') - - -class _LocalRunsReference: - """Refer only to an explicitly activated local owner; never construct one.""" - def __init__(self,execution):self.execution=execution - def __getattr__(self,name):return getattr(self.execution.runs,name) - - -class _PreparationRuns: - """Scientific inputs only; this object cannot allocate or recover Runs.""" - def __init__(self,adapter):self.adapter=adapter;self.access=adapter.session.access - def _load(self,run_id): - history=self.adapter.history;history._authorize('runs.get',run_id) - facts=history._facts();doc=history._record(facts.runs,run_id,self.adapter.origin) - history._transitive(facts,doc);history._disclose(doc['run'],'unknown') - return copy.deepcopy(doc['run']) - @staticmethod - def _external(run):return run.get('execution_kind')=='external_recipe' - @staticmethod - def _extension(run): - from recipe_contract import is_evidence - return run.get('execution_kind')=='external_recipe' and is_evidence(run['material']['recipe']) - def artifact(self,run_id,name): - raw=self.adapter.history.artifact(run_id,name,origin_catalog_id=self.adapter.origin, - maximum_bytes=512*1024**2) - return raw,'application/octet-stream' - def model_evidence(self,run_id,selectors,*,contract_version=2): - if contract_version!=2:_invalid('Candidate preparation requires model evidence version 2.','unsupported') - return self.adapter.history.evidence(run_id,selectors,origin_catalog_id=self.adapter.origin) - def capture_artifact(self,*args,**kwargs):return self.adapter.history.capture_artifact(*args,**kwargs) - - -class _PreparationProjects: - def __init__(self,adapter,phase): - self.adapter,self.phase=adapter,phase;self.source=adapter.projection.sources - self.access=adapter.access - def __getattr__(self,name):return getattr(self.source,name) - def open(self,body): - return self.adapter.projection.source_adapter.register(body, - self.adapter.session.summary['checkpoint'],self.phase+'-source') - - -class _PreparationModels: - def __init__(self,adapter,phase):self.adapter,self.phase=adapter,phase - def material(self,ref,*,verify=False):return self.adapter.materials.model(ref,verify=verify) - def read_blob(self,ref,item):return self.adapter.materials.blob(ref,item) - def _read_blob(self,ref,item,maximum=40*1024**2):return self.adapter.materials.blob(ref,item,maximum) - def create(self,body): - adapter=self.adapter - return adapter.materials.mutate_definition('model_definition','models.create',None,body, - adapter.session.summary['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) - def publish(self,identity,body): - adapter=self.adapter - return adapter.materials.publish_model(adapter.projection.models,identity,body, - adapter.session.summary['checkpoint'],self.phase+'-publish') - - -class _PreparationPipelines: - def __init__(self,adapter,phase,context): - from pipeline import Pipelines - self.adapter,self.phase=adapter,phase;self.owner=Pipelines(context) - self.material_path=self.owner.material_path - def get(self,*args,**kwargs):return self.owner.get(*args,**kwargs) - def recipe_material(self,ref,*,verify=False):return self.adapter.materials.recipe(ref,verify=verify) - def create_external(self,body): - adapter=self.adapter - return adapter.materials.mutate_definition('pipeline_graph','graphs.create_external',None,body, - adapter.session.summary['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) - def prepare_external_material(self,*args,**kwargs):return self.owner.prepare_external_material(*args,**kwargs) - def _retain_material(self,prepared): - adapter=self.adapter;value=self.owner._retain_material(prepared) - record=adapter.materials.capture_recipe_material(value,self.owner.material_path) - adapter._package_records.extend([record,*adapter.materials.declarations(record), - *adapter.materials.capture_recipe_dependencies(record,adapter.session.snapshot['records'])]) - return value - - -class ProjectStudyExecution: - def __init__(self,projection,execution): - self.projection,self.execution=projection,execution - self.session,self.db,self.history=projection.session,projection.db,projection.history - self.materials=projection.materials;self.access=execution.runtime_access - self.origin=self.session.summary['project_id'] - self._dirty=set();self._revisions={};self._epochs={};self._pending_epochs={} - self._package_records=[] - self.context=copy.copy(projection.context) - self.context.access,self.context.runs=self.access,_LocalRunsReference(execution) - self.context.project_study_execution=self - self.context.project_execution=execution - self.studies=Experiments(self.context) - execution.study_execution=self - projection.study_execution=self - self.history.study_execution=self - - def _canonical(self,records=None): - if records is None and callable(getattr(self.session,'derived',None)): - from project_history import checkpoint_facts - return checkpoint_facts(self.session,self.materials) - records=self.session.snapshot['records'] if records is None else records - return history_records(records,materials=self.materials.read_view(records)) - - def _authoring_facts(self,records=None): - """Studies and Trials of the bound checkpoint for SQL authoring. - - For a checkpoint whose complete validation is on record, only the Study - and Trial documents are needed and collected (no Run history); this is - exactly the studies/trials part of the complete decode. - """ - if records is None: - validated=getattr(self.session,'validated_checkpoint',None) - if validated is not None and validated==self.session.summary['checkpoint']: - from project_history import assemble_validated_facts - return assemble_validated_facts(session_records(self.session,'study_history','trial_history'),owned=True) - return self._canonical(records) - - def restore_authoring(self,records=None): - """Reconstruct intent only, under the current project guard.""" - with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check_current();facts=self._authoring_facts(records) - with self.db.transaction(): - for (origin,identity),doc in sorted(facts.studies.items()): - if origin!=self.origin:continue - for item in doc['policies']: - ref=item['ref'] - self.db.execute('INSERT INTO model_selection_policies VALUES(%s,%s,%s,%s) ON CONFLICT(id,revision) DO UPDATE SET sha256=EXCLUDED.sha256,document=EXCLUDED.document', - (ref['id'],ref['revision'],ref['sha256'],encode(item['policy']))) - value=doc['study'] - self.db.execute('INSERT INTO model_studies VALUES(%s,%s,%s,%s) ON CONFLICT(id) DO UPDATE SET revision=EXCLUDED.revision,admission_epoch=EXCLUDED.admission_epoch,document=EXCLUDED.document', - (identity,value['revision'],value['admission_epoch'],encode(value))) - for member in doc['memberships']: - self.db.execute('INSERT INTO model_study_memberships VALUES(%s,%s,%s) ON CONFLICT(study_id,revision) DO UPDATE SET document=EXCLUDED.document', - (identity,member['revision'],encode({'trial_ids':member['trial_ids']}))) - for (origin,identity),doc in sorted(facts.trials.items()): - if origin!=self.origin:continue - value=doc['trial'] - self.db.execute('INSERT INTO model_trials VALUES(%s,%s,%s,%s,%s) ON CONFLICT(id) DO UPDATE SET document=EXCLUDED.document', - (identity,value['study_id'],value['material']['sha256'],value['replicate'],encode(value))) - for (origin,identity),doc in sorted(facts.trials.items()): - if origin!=self.origin or doc['lineage'] is None:continue - value=doc['lineage'];intent=value['intent'] - # Native current authoring only supports same-project SQL - # parent FKs; foreign history remains in the canonical view. - if any(ref is not None and ref['catalog_id']!=self.origin for ref in (intent['parent'],intent['baseline'])):continue - self.db.execute('INSERT INTO model_trial_lineage VALUES(%s,%s,%s,%s) ON CONFLICT(trial_id) DO UPDATE SET document=EXCLUDED.document', - (identity,(intent['parent'] or {}).get('trial_id'),(intent['baseline'] or {}).get('trial_id'),encode(value))) - return None - - refresh_authoring=restore_authoring - - def local_run_ids(self): - return {row[0] for row in self.db.execute('SELECT run_id FROM model_attempt_reservations')} - - def historical_reservations(self,study_id): - doc=self._canonical().studies.get((self.origin,study_id)) - if doc is None:return [] - local=self.local_run_ids() - return [copy.deepcopy(value) for value in doc['reservations'] if value['run_id'] not in local] - - def reject_historical_request(self,study_id,actor,key,sha): - for value in self.historical_reservations(study_id): - if value['request_key']==key: - _invalid('This request identifies an inert historical attempt; use a fresh key for new execution.','conflict') - - def run_fact(self,run_id): - identifier(run_id) - if self.db.execute('SELECT id FROM workspace_runs WHERE id=%s',(run_id,)).fetchone() is not None: - return copy.deepcopy(self.execution.runs._load(run_id)) - value=self._canonical().runs.get((self.origin,run_id)) - if value is None:_invalid('Retained Run observation is unavailable.','not_found') - return copy.deepcopy(value['run']) - - def lineage_fact(self,trial_id): - value=self._canonical().trials.get((self.origin,trial_id)) - return copy.deepcopy(value['lineage']) if value is not None else None - - def epoch_current(self,study): - return study['open'] and self._epochs.get(study['id'])==study['admission_epoch'] - - def require_current_epoch(self,study): - if not self.epoch_current(study): - _invalid('Explicitly continue this Study in the current runtime before admission.','unavailable') - - def external_blocked(self): - owner=getattr(self.execution,'_runs',None) - return bool(owner is not None and owner._external_blocked) - - def mark_study(self,study_id): - identifier(study_id) - if not self.db.in_transaction:raise RuntimeError('Study publication requires its owning SQL transaction.') - value=self.studies._study(study_id) - self._dirty.add(study_id) - versions=self._revisions.setdefault(study_id,{}) - prior=versions.get(value['revision']) - if prior is not None and prior!=value:_invalid('A Study revision changed inside its publication transaction.') - versions[value['revision']]=copy.deepcopy(value) - - def _operation_study(self,operation,result): - if operation in ('studies.create','studies.save','studies.continue'):return result['id'] - if operation in ('studies.prepare_trial','studies.prepare_branch'):return result['study_id'] - if operation in ('studies.branch_trial','studies.stop'):return result['study_id'] - if operation=='studies.select_candidate':return result['candidate']['study_id'] - if operation in ('studies.prepare_package','studies.prepare_verifier'):return result['trial']['study_id'] - if operation=='studies.record_lineage':return self.studies.trial(result['trial']['trial_id'])['study_id'] - _invalid('Unclassified native Study operation result.','unsupported') - - def mark_operation(self,operation,result): - self.mark_study(self._operation_study(operation,result)) - - def settle_runs(self,run_ids): - if not run_ids:return - if not self.db.in_transaction:raise RuntimeError('Reservation settlement must share Run publication.') - studies={row['study_id'] for row in self.db.execute('SELECT study_id,run_id FROM model_attempt_reservations') if row['run_id'] in run_ids} - if studies: - with self.access._reconcile(): - for identity in sorted(studies):self.studies._reconcile_local(identity) - - @staticmethod - def _audit_fact(row): - return dict(actor_id=row['actor_id'],action=row['action'],target=json.loads(row['target_json']), - details=json.loads(row['details_json']),result=row['result'], - occurred_at=datetime.fromtimestamp(row['created_at'],timezone.utc).isoformat()) - - def capture_dirty(self,before_records): - # Execution has already qualified this observation against current - # local custody. These finite metadata reads must not ask the child - # continuation to acquire unrelated Study/policy capabilities. - with self.access._reconcile(): - return self._capture_dirty(before_records) - - def _capture_dirty(self,before_records): - if not self._dirty:return copy.deepcopy(self._package_records) - facts=self._canonical(before_records);records=copy.deepcopy(self._package_records) - audit_rows=list(self.db.execute('SELECT actor_id,action,target_json,details_json,result,created_at FROM access_audit ORDER BY created_at,id')) - operation_rows=list(self.db.execute('SELECT actor_id,operation_id,request_key,request_sha256,document FROM model_operation_receipts ORDER BY actor_id,operation_id,request_key')) - for study_id in sorted(self._dirty): - old=facts.studies.get((self.origin,study_id));value=self.studies._study(study_id) - doc=copy.deepcopy(old) if old is not None else dict(format=FAMILY_FORMATS['study_history'],origin_catalog_id=self.origin, - study=value,memberships=[],policies=[],reservations=[],operations=[],revisions=[],audit=[],revision_history_complete=True) - revisions={item['revision']:item for item in doc['revisions']} - for revision,item in self._revisions.get(study_id,{}).items(): - if revision in revisions and revisions[revision]!=item:_invalid('Published Study revision cannot change.') - revisions[revision]=copy.deepcopy(item) - if value['revision'] in revisions and revisions[value['revision']]!=value:_invalid('Current Study differs from captured revision.') - revisions[value['revision']]=copy.deepcopy(value) - doc['study']=copy.deepcopy(value);doc['revisions']=[revisions[key] for key in sorted(revisions)] - if old is None and min(revisions)!=1:doc['revision_history_complete']=False - doc['memberships']=[dict(revision=row['revision'],trial_ids=json.loads(row['document'])['trial_ids']) - for row in self.db.execute('SELECT revision,document FROM model_study_memberships WHERE study_id=%s ORDER BY revision',(study_id,))] - policies={item['ref']['sha256']:item for item in doc['policies']} - ref=value['policy'];policies[ref['sha256']]=dict(ref=copy.deepcopy(ref),policy=self.studies.policy(ref)) - doc['policies']=[policies[key] for key in sorted(policies)] - reservations={item['run_id']:item for item in doc['reservations']} - reservations.update((item['run_id'],item) for item in self.studies._local_reservations(study_id)) - doc['reservations']=sorted(reservations.values(),key=lambda item:item['id']) - operations={(item['actor_id'],item['operation_id'],item['request_key']):item for item in doc['operations']} - for row in operation_rows: - result=json.loads(row['document']) - if self._operation_study(row['operation_id'],result)!=study_id:continue - item=dict(actor_id=row['actor_id'],operation_id=row['operation_id'],request_key=row['request_key'], - request_sha256=row['request_sha256'],result=result) - key=item['actor_id'],item['operation_id'],item['request_key'] - if key in operations and operations[key]!=item:_invalid('A retained native operation fact changed.') - operations[key]=item - doc['operations']=[operations[key] for key in sorted(operations)] - targets={study_id,*value['trial_ids'],*(item['run_id'] for item in doc['reservations'])} - candidates=[] - for row in self.db.execute('SELECT id,document FROM model_candidates WHERE study_id=%s',(study_id,)): - candidate=json.loads(row['document']);targets.add(candidate['id']) - candidates.append(envelope('candidate_history',self.origin,candidate['id'],dict( - format=FAMILY_FORMATS['candidate_history'],origin_catalog_id=self.origin,candidate=candidate))) - for row in audit_rows: - if not (row['action'].startswith('studies.') or row['action'].startswith('runs.')):continue - fact=self._audit_fact(row) - if fact['target'] not in targets:continue - _audit([fact]) - if fact not in doc['audit']:doc['audit'].append(fact) - records.append(envelope('study_history',self.origin,study_id,doc));records.extend(candidates) - for trial_id in value['trial_ids']: - trial=self.studies.trial(trial_id) - row=self.db.execute('SELECT document FROM model_trial_lineage WHERE trial_id=%s',(trial_id,)).fetchone() - prior=facts.trials.get((self.origin,trial_id));lineage=json.loads(row[0]) if row is not None else (prior['lineage'] if prior else None) - records.append(envelope('trial_history',self.origin,trial_id,dict(format=FAMILY_FORMATS['trial_history'], - origin_catalog_id=self.origin,trial=trial,lineage=lineage))) - return records - - def abandon_pending(self): - """Discard only unpublished projection bookkeeping during guarded repair. - - This cannot activate an imported epoch or recreate any execution right. - The root recovery owner separately qualifies custody and exact selection. - """ - self._dirty.clear();self._revisions.clear();self._pending_epochs.clear() - self._package_records.clear() - - def outcome(self,outcome): - if outcome not in ('committed','rolled_back','unknown'):raise ValueError('Unsupported Study SQL outcome.') - if outcome=='unknown':return - if outcome=='committed':self._epochs.update(self._pending_epochs) - self._dirty.clear();self._revisions.clear();self._pending_epochs.clear() - self._package_records.clear() - - def _mutate(self,operation,study_id,body,expected_checkpoint,key,invoke,scope): - return self.execution.mutate(operation,study_id,body,expected_checkpoint,key,invoke, - lambda result,before:dict(records=[],result=result),scope=scope) - - def continue_study(self,study_id,body,expected_checkpoint,key): - closed(body,('revision','admission_epoch'));request_key(key) - self.access.require('execute','studies.continue',study_id,owner_only=True) - with self.session.access.lock,self.session.lock,self.db.lock: - self.refresh_authoring() - prior=session_operations(self.session).get(key) - if prior is not None and not self.epoch_current(self.studies._study(study_id)): - _invalid('A historical Continue receipt cannot activate this runtime; use a fresh key.','unavailable') - def invoke(): - value=self.studies.continue_study(study_id,dict(body,request_key=key)) - self._pending_epochs[study_id]=value['admission_epoch'] - return value - return self._mutate('studies.continue',study_id,body,expected_checkpoint,key,invoke,'execute') - - def _current_grant(self,value,approver): - row=self.db.execute('SELECT * FROM access_execution_grants WHERE id=%s',(value['approval_id'],)).fetchone() - if row is None:_invalid('This approval is historical; approve explicitly with a fresh key.','unavailable') - grant=dict(row);closed(grant,GRANT_FIELDS) - expected=dict(origin_catalog_id=grant['catalog_id'],approval_id=grant['id'], - **{key:grant[key] for key in ('material_sha256','consumer_actor_id','approver_actor_id','created_at','expires_at')}) - if (any(value[key]!=item for key,item in expected.items()) or grant['approver_actor_id']!=approver - or grant['revoked_at'] is not None or grant['expires_at']<=int(time.time())): - _invalid('An exact current local execution approval is unavailable.','unavailable') - return grant - - def _materialize_trial(self,study_id,trial_id,expected_sha256=None): - """Explicit preparation reconstructs only the current private caches.""" - study=self.studies._study(study_id);trial=self.studies.trial(trial_id) - if trial['study_id']!=study_id or trial_id not in study['trial_ids']: - _invalid('Prepare an exact member Trial.') - if expected_sha256 is not None and trial['material']['sha256']!=expected_sha256: - _invalid('Composed material differs. Review the exact Trial.','stale') - value=self.materials.recipe(trial['material']);material=value['material'] - snapshot=json.loads(self.materials.blob(trial['material'],material['recipe_document'],2*1024**2)) - if snapshot['format']!='builtin_recipe_snapshot_v1': - # Current permission is independent from the original input proof. - # Verify every protected producer before rebuilding captured bytes. - for item in snapshot['preflight']['material']['inputs'].values(): - if item.get('kind')=='run_artifact': - source=item['source'] - self.history.capture_artifact(source['run_id'],source['artifact_name'],source['measured_sha256'], - origin_catalog_id=source['catalog_id']) - self.materials.materialize_projection(material['model'],self.projection.models.path) - self.materials.materialize_projection(trial['material'],self.projection.pipelines.material_path) - return trial - - def approve(self,study_id,body,expected_checkpoint,key): - closed(body,('trial_id','expected_material_sha256','consumer_actor_id'),('ttl_seconds',));request_key(key) - current=self.access.require('approve-execution','studies.approve',study_id,owner_only=True) - effective=dict(body,approver_actor_id=current.actor_id) - with self.session.access.lock,self.session.lock,self.db.lock: - self.refresh_authoring();prior=session_operations(self.session).get(key) - if prior is not None: - if prior['operation']!='study_execution.approve':_invalid('Request key has another operation.','conflict') - validate_operation_result(prior['operation'],prior['result'],self.session.snapshot['records']) - self._current_grant(prior['result'],current.actor_id) - else: - if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') - self._materialize_trial(study_id,body['trial_id'],body['expected_material_sha256']) - def invoke(): - grant=self.studies.approve(study_id,body) - return dict(format='project_study_approval_v1',origin_catalog_id=grant['catalog_id'], - study_id=study_id,trial_id=body['trial_id'],approval_id=grant['id'], - **{key:grant[key] for key in ('material_sha256','consumer_actor_id','approver_actor_id','created_at','expires_at')}) - value=self._mutate('study_execution.approve',study_id,effective,expected_checkpoint,key,invoke,'approve-execution') - grant=self._current_grant(value,current.actor_id) - return dict(format='project_current_execution_approval_v1',current_local=True,grant=grant) - - def admit(self,study_id,body,expected_checkpoint,key): - closed(body,('revision','admission_epoch','trial_id'),('approval_id','parent_run_id'));request_key(key) - authority=self.access.require_external_submission() - with self.session.access.lock,self.session.lock,self.db.lock: - builtin=None - self.session.check_current();self.refresh_authoring() - prior=session_operations(self.session).get(key) - if prior is not None: - result=prior['result'] - if (prior['operation']!='study_execution.admit' or self.db.execute( - 'SELECT run_id FROM model_attempt_reservations WHERE run_id=%s',(result.get('run_id'),)).fetchone() is None): - _invalid('A historical admission receipt cannot create a current local attempt.','unavailable') - else: - if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') - self.require_current_epoch(self.studies._study(study_id)) - self.reject_historical_request(study_id,authority.actor_id,key,None) - trial=self._materialize_trial(study_id,body['trial_id']) - from pipeline import BUILTIN_TRAINERS - material=self.materials.recipe(trial['material']) - if material['material']['trainer']['id'] in BUILTIN_TRAINERS: - self.access.require('execute','studies.admit_builtin',study_id,owner_only=True) - from project_native_execution import native_admission_wall_seconds - if material['material']['resource_envelope']['admitted_wall_seconds']<native_admission_wall_seconds(): - _invalid('Retained native material has an insufficient custody budget; explicitly prepare a new material.','stale') - owner=self.execution._local_owner() - from project_native_preparation import preparation_owner - preparation=preparation_owner(owner.pipelines) - if preparation is None:_invalid('Current native preparation is unavailable.','unavailable') - builtin=(preparation,owner.pipelines,trial['material']) - def invoke(): - self.execution._local_owner() - result=self.studies.admit(study_id,dict(body,request_key=key));reservation=result['reservation'] - return dict(format='project_study_admission_v1',origin_catalog_id=self.origin, - study_id=study_id,trial_id=reservation['trial_id'],run_id=reservation['run_id'],reservation=reservation) - # The submission policy above permits an approved agent. The native - # admission consumes its exact current grant atomically; execute - # here would incorrectly impose the host's owner-only policy. - if builtin is not None: - preparation,pipelines,ref=builtin - # Qualification may launch the explicit probe. It finishes - # before Session opens the finite admission transaction. - with preparation.admission(pipelines,ref,expected_checkpoint,key): - return self._mutate('study_execution.admit',study_id,body,self.session.summary['checkpoint'],key,invoke,'inspect') - return self._mutate('study_execution.admit',study_id,body,expected_checkpoint,key,invoke,'inspect') - - def stop(self,study_id,body,expected_checkpoint,key): - closed(body,('revision',));request_key(key) - self.access.require('cancel','studies.stop',study_id) - invoked=[] - with self.session.access.lock,self.session.lock,self.db.lock: - self.refresh_authoring() - def invoke(): - receipt=self.studies._close_admission(study_id,dict(body,request_key=key));value=self.studies._study(study_id) - invoked.extend(receipt['run_ids']) - return dict(format='project_study_stop_v1',origin_catalog_id=self.origin,study_id=study_id, - receipt=receipt,study_revision=value['revision'],state=value['state'],open=value['open']) - result=self._mutate('study_execution.stop',study_id,body,expected_checkpoint,key,invoke,'cancel') - # Only acknowledged canonical closure permits this current-runtime - # cancellation phase. Receipt replay does not repeat effects. - for run_id in invoked: - cancel_key='study-stop-'+hashlib.sha256((key+':'+run_id).encode()).hexdigest() - try:self.execution.cancel_run(run_id,self.session.summary['checkpoint'],cancel_key) - except BaseException as error: - error.study_stop_receipt=copy.deepcopy(result) - raise - return result - - def candidate(self,candidate_id): - value=self.history.candidate(candidate_id,origin_catalog_id=self.origin,verify=True) - return {key:value[key] for key in ('ref','candidate','run_id','checkpoint')} - - def _pending_preparation(self,study_id,body,expected_checkpoint): - identifier(study_id);request_key(body['preparation_key']);_sha(expected_checkpoint) - self.access.require('cancel','studies.reconcile_preparation',study_id,owner_only=True) - # The acknowledged intent is present even when a later Store/SQL ACK - # is uncertain. Only the qualified execution guard may recover HEAD. - snapshot=self.session.snapshot - if snapshot['checkpoint']!=expected_checkpoint:_invalid('Expected project checkpoint changed.','stale_project') - operation,intent,complete=_preparation_reference(snapshot['records'],study_id,body['preparation_key']) - validate_operation_result(operation+'_intent',intent,snapshot['records']) - if intent['origin_catalog_id']!=self.origin:_invalid('Historical origin cannot settle a current writer.','unavailable') - if complete is not None and complete['operation']!=operation:_invalid('Preparation key has another operation.','conflict') - return operation,intent,complete - - def reconcile_preparation(self,study_id,body,expected_checkpoint): - """Acknowledge a completed publication without replaying its writer.""" - closed(body,('preparation_key',)) - with self.session.access.lock,self.session.lock,self.db.lock: - operation,intent,complete=self._pending_preparation(study_id,body,expected_checkpoint) - with self.execution.package_reconciliation(body['preparation_key'],intent['request_sha256'], - intent['candidate']['sha256'],expected_checkpoint) as guard: - operation,current_intent,complete=_preparation_reference(self.session.snapshot['records'],study_id,body['preparation_key']) - if current_intent!=intent:_invalid('Preparation intent changed during guarded reconciliation.','conflict') - if complete is None:_invalid('No final preparation exists; explicitly abandon the stopped partial writer.','unavailable') - if complete['operation']!=operation:_invalid('Preparation key has another operation.','conflict') - validate_operation_result(operation,complete['result'],self.session.snapshot['records']) - if complete.get('native_receipt'): - effective=_preparation_body(intent,body['preparation_key']) - effective.pop('request_key');effective['actor_id']=intent['actor_id'] - result=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],body['preparation_key'], - lambda:copy.deepcopy(complete['result']),'cancel') - else:result=copy.deepcopy(complete['result']) - guard.confirmed() - return result - - def abandon_preparation(self,study_id,body,expected_checkpoint,key): - """Record a stopped writer decision; preserve every byte and phase fact.""" - closed(body,('preparation_key','reason'));request_key(key);text(body['reason'],'Abandonment reason',1000) - if key==body['preparation_key']:_invalid('Abandonment requires a separate request key.') - current=self.access.require('cancel','studies.abandon_preparation',study_id,owner_only=True) - effective=dict(body,actor_id=current.actor_id) - with self.session.access.lock,self.session.lock,self.db.lock: - operation,intent,complete=self._pending_preparation(study_id,body,expected_checkpoint) - if complete is not None:_invalid('Preparation is complete; reconcile its acknowledgement without abandoning science.','conflict') - with self.execution.package_reconciliation(body['preparation_key'],intent['request_sha256'], - intent['candidate']['sha256'],expected_checkpoint) as guard: - operation,current_intent,complete=_preparation_reference(self.session.snapshot['records'],study_id,body['preparation_key']) - if current_intent!=intent:_invalid('Preparation intent changed during guarded reconciliation.','conflict') - if complete is not None:_invalid('Preparation is complete; reconcile its acknowledgement without abandoning science.','conflict') - observed_checkpoint=self.session.summary['checkpoint'] - def invoke(): - actor=self.access.require('cancel','studies.abandon_preparation',study_id,owner_only=True) - if actor.actor_id!=current.actor_id:_invalid('Abandonment caller changed.','conflict') - return dict(format='project_candidate_preparation_abandon_v1',origin_catalog_id=self.origin, - study_id=study_id,preparation_key=body['preparation_key'],preparation_operation=operation, - input_sha256=intent['request_sha256'],candidate=copy.deepcopy(intent['candidate']), - actor_id=actor.actor_id,reason=body['reason'],observed_checkpoint=observed_checkpoint, - occurred_at=timestamp(),state='abandoned') - value=self._mutate(ABANDON,study_id,effective,observed_checkpoint,key,invoke,'cancel') - guard.confirmed() - return value - - def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): - from candidate_packages import NAME,_prepare_operation - from runtime_snapshots import RuntimeSnapshots - from model_definitions import checked_json - fields=('revision','candidate_id','root','directory','runtime_snapshot_sha256') - if kind=='verify':fields+=('package_run_id','package_sha256') - closed(body,fields);body=checked_json(body);request_key(key);identifier(study_id) - natural(body['revision'],'Study revision',1);identifier(body['candidate_id']);_sha(body['runtime_snapshot_sha256']) - for field in ('root','directory'): - if type(body[field]) is not str or NAME.fullmatch(body[field]) is None:_invalid('Invalid candidate preparation location.') - if kind=='verify':identifier(body['package_run_id']);_sha(body['package_sha256']) - operation='study_execution.prepare_package' if kind=='package' else 'study_execution.prepare_verifier' - native=PREPARATIONS[operation] - actor=self.access.require('edit',native,study_id,owner_only=True) - effective=dict(body,actor_id=actor.actor_id) - input_sha=digest(dict(study_id=study_id,body=effective)) - intent_key=_intent_key(operation,key);phase=intent_key.removesuffix('-intent') - with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check_current() - prior=session_operations(self.session).get(key) - if prior is not None: - # Current authority is checked above. Historical completion - # acknowledges science and cannot repeat filesystem effects. - return self._mutate(operation,study_id,effective,expected_checkpoint,key, - lambda:(_ for _ in ()).throw(RuntimeError('Completed preparation replay invoked effects.')),'edit') - if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') - receipts=session_operations(self.session) - if any(_intent_key(other,key) in receipts for other in PREPARATIONS if other!=operation): - _invalid('Preparation key already identifies another preparation operation.','conflict') - previous_intent=receipts.get(intent_key) - if previous_intent is not None: - if previous_intent['result'].get('request_sha256')!=input_sha: - _invalid('Candidate preparation key has different inputs or actor.','conflict') - _invalid('Candidate preparation has an incomplete retained intent; reconcile it before a new explicit attempt.','unavailable') - self.refresh_authoring();self.studies._expected(self.studies._study(study_id),body) - candidate=self.candidate(body['candidate_id']) - if candidate['candidate']['study_id']!=study_id:_invalid('Candidate belongs to another Study.') - if body['root'] not in self.projection.sources.roots:_invalid('Choose a currently configured source root.','unavailable') - with self.execution.runtime_inspection(): - snapshot=RuntimeSnapshots(self.db,self.access,self.db.path).get(body['runtime_snapshot_sha256']) - if snapshot['catalog_id']!=self.origin:_invalid('Register this runtime under the current project.','unavailable') - runtime_ref=dict(kind='runtime',catalog_id=self.origin,id=digest(snapshot),sha256=digest(snapshot)) - retained=self.materials.runtime(runtime_ref,verify=True) - if retained['descriptor']!=snapshot:_invalid('Current runtime differs from retained preparation material.','stale') - intent=dict(format='project_candidate_preparation_intent_v1',origin_catalog_id=self.origin, - study_id=study_id,study_revision=body['revision'],candidate=copy.deepcopy(candidate['ref']), - actor_id=actor.actor_id,request_sha256=input_sha,root=body['root'],directory=body['directory'], - runtime_snapshot_sha256=body['runtime_snapshot_sha256']) - if kind=='verify':intent.update({field:body[field] for field in ('package_run_id','package_sha256')}) - with self.execution.package_effect(key,input_sha,candidate['ref']['sha256']) as guard: - self._mutate(operation+'_intent',study_id,effective,expected_checkpoint,intent_key, - lambda:copy.deepcopy(intent),'edit') - projects=_PreparationProjects(self,phase);models=_PreparationModels(self,phase) - context=copy.copy(self.projection.context) - context.projects,context.models,context.runs=projects,models,_PreparationRuns(self) - pipelines=_PreparationPipelines(self,phase,context);context.pipelines=pipelines - execution_context=copy.copy(context);execution_context.access=self.access - execution_context.project_study_execution=self;execution_context.project_execution=self.execution - studies=Experiments(execution_context) - publication=SimpleNamespace(runtime_snapshot=snapshot,before_write=guard.publication_started, - read_recipe_blob=self.materials.blob) - result=_prepare_operation(studies,study_id,dict(body,request_key=key),kind,publication=publication) - value=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],key, - lambda:copy.deepcopy(result),'edit') - guard.confirmed() - return value - - def prepare_package(self,study_id,body,expected_checkpoint,key): - return self._prepare_candidate(study_id,body,expected_checkpoint,key,'package') - - def prepare_verifier(self,study_id,body,expected_checkpoint,key): - return self._prepare_candidate(study_id,body,expected_checkpoint,key,'verify') - - def assert_quiescent(self): - if self._dirty or self._pending_epochs or self._package_records:_invalid('Study publication is still pending.','busy') diff --git a/project_text.py b/project_text.py deleted file mode 100644 index fc778528..00000000 --- a/project_text.py +++ /dev/null @@ -1,375 +0,0 @@ -"""M1 text documents: canonical files own facts; Workbench owns text semantics. - -Only import and annotation are admitted by this slice. Reconstruction never -constructs Dataset, Jobs, Runs, or imports Access tables. The document decoder is -the closed workbench_text_v1 persisted contract; new families require their owner. -""" -import copy -from datetime import datetime -import hashlib -import re -import threading -from types import SimpleNamespace -import unicodedata - -from workbench import (Workbench, WorkbenchError, IDENTIFIER, MAX_TEXT, encode, - strings, text_value, timestamp, validate_annotation) - - -class ProjectTextError(WorkbenchError): - pass - - -def _invalid(message): - raise ProjectTextError(message, 'invalid', 400) - - -def _sha(value): - return hashlib.sha256(value.encode('utf-8')).hexdigest() - - -def _shape(value, fields, name): - if type(value) is not dict or set(value) != set(fields): - _invalid('Invalid ' + name + ' fields.') - - -def _time(value): - if type(value) is not str or len(value) > 64: - _invalid('Invalid text timestamp.') - try: - parsed = datetime.fromisoformat(value) - if parsed.utcoffset() is None: - raise ValueError() - return parsed - except (ValueError, OverflowError): - _invalid('Invalid text timestamp.') - - -SNAPSHOT_FIELDS = set('id kind name text content_hash pixel_hash groups parents provenance task annotation review revision created_at updated_at width height source_revision source_sha256 book_id session_id source_split corner_annotation source_available source_lineage_known'.split()) -IMMUTABLE_FIELDS = set('id kind name text content_hash pixel_hash parents provenance created_at width height source_revision source_sha256 book_id session_id source_split corner_annotation source_available source_lineage_known'.split()) -AUDIT_ACTIONS = {'workbench.import_asset', 'workbench.save', 'workbench.approve_review'} -SHA256 = re.compile(r'^[a-f0-9]{64}$') -REQUEST_KEY = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$') - - -def _snapshot(value, identity, original): - _shape(value, SNAPSHOT_FIELDS, 'text snapshot') - if value['id'] != identity or value['kind'] != 'text': - _invalid('Text snapshot identity mismatch.') - if type(value['revision']) is not int or value['revision'] < 1: - _invalid('Invalid text revision.') - if type(value['source_revision']) is not int or value['source_revision'] != 1: - _invalid('Invalid text source revision.') - if any(value[key] is not None for key in ('pixel_hash', 'width', 'height', 'book_id', 'session_id', 'corner_annotation')): - _invalid('Text snapshot contains non-text source fields.') - if value['source_available'] is not True or value['source_lineage_known'] is not True or value['source_split'] != 'unassigned': - _invalid('Invalid text source state.') - if text_value(value['name'], 'Name') != value['name']: - _invalid('Text name must retain canonical spelling.') - normalized = unicodedata.normalize('NFC', original.replace('\r\n', '\n').replace('\r', '\n')) - if value['text'] != normalized or value['content_hash'] != _sha(normalized) or value['source_sha256'] != _sha(original): - _invalid('Text source, normalization or hash mismatch.') - for key in ('groups', 'parents'): - if strings(value[key], key) != value[key] or (key == 'groups' and not value[key]): - _invalid('Invalid text ' + key + '.') - if any(not IDENTIFIER.fullmatch(parent) for parent in value['parents']): - _invalid('Invalid text parent identity.') - origin = value['provenance'] - _shape(origin, {'method', 'rights', 'normalization'}, 'text provenance') - if origin['method'] != 'import' or origin['normalization'] != 'NFC; LF newlines' or text_value(origin['rights'], 'Rights', 1000) != origin['rights']: - _invalid('Unsupported text provenance.') - if value['task'] not in ('text_classification', 'text_entities', 'text_corpus') or value['review'] not in ('draft', 'human_reviewed'): - _invalid('Invalid text task or review.') - if value['annotation'] is None: - if value['revision'] != 1 or value['task'] != 'text_classification' or value['review'] != 'draft': - _invalid('Missing text annotation after import.') - elif validate_annotation(value['task'], value['annotation'], value) != value['annotation']: - _invalid('Text annotation is not canonical.') - if _time(value['updated_at']) < _time(value['created_at']): - _invalid('Text update precedes creation.') - - -def decode_records(records): - """Validate complete text closure before any SQL write; return an owned copy.""" - if type(records) is not list or len(records) > 100_000: - _invalid('Project text records must be a bounded list.') - documents, operations, hashes = {}, set(), set() - for envelope in records: - _shape(envelope, {'family', 'id', 'document'}, 'text envelope') - if envelope['family'] != 'workbench_text': - raise ProjectTextError('Unsupported project record family.', 'unsupported', 409) - identity, document = envelope['id'], envelope['document'] - if type(identity) is not str or not IDENTIFIER.fullmatch(identity) or identity in documents: - _invalid('Invalid or duplicate text identity.') - if type(document) is not dict: - _invalid('Text document must be an object.') - if document.get('format') != 'workbench_text_v1': - raise ProjectTextError('Unsupported text document format.', 'unsupported', 409) - _shape(document, {'format', 'original_text', 'current', 'history', 'audit', 'operations'}, 'text document') - original = document['original_text'] - text_value(original, 'Original text', MAX_TEXT) - history = document['history'] - if type(history) is not list or not history: - _invalid('Text history must contain every revision.') - previous = None - for revision, state in enumerate(history, 1): - _snapshot(state, identity, original) - if state['revision'] != revision: - _invalid('Text history revisions must be contiguous.') - if previous and (any(state[key] != previous[key] for key in IMMUTABLE_FIELDS) or _time(state['updated_at']) < _time(previous['updated_at'])): - _invalid('Text history changes immutable source or reverses time.') - previous = state - _snapshot(document['current'], identity, original) - if document['current'] != history[-1] or history[0]['annotation'] is not None: - _invalid('Text current value or initial history is inconsistent.') - if document['current']['content_hash'] in hashes: - _invalid('Duplicate canonical text.') - hashes.add(document['current']['content_hash']) - audit, receipts = document['audit'], document['operations'] - if type(audit) is not list or type(receipts) is not list or len(audit) != len(history) or len(receipts) != len(history): - _invalid('Every text revision requires its audit and operation facts.') - for revision, (fact, receipt, state) in enumerate(zip(audit, receipts, history), 1): - _shape(fact, {'action', 'target', 'details', 'result', 'actor_id', 'occurred_at'}, 'text audit') - expected_action = 'workbench.import_asset' if revision == 1 else 'workbench.approve_review' if state['review'] == 'human_reviewed' else 'workbench.save' - details = {'revision': revision, 'source_revision': 1, 'content_sha256': _sha(encode(state))} - if revision > 1: - details.update(before_revision=revision - 1, before_source_revision=1) - _shape(fact['details'], details, 'text audit details') - if any(type(fact['details'][key]) is not type(expected) for key, expected in details.items()): - _invalid('Invalid text audit detail types.') - if fact['action'] != expected_action or fact['target'] != identity or fact['details'] != details or fact['result'] != 'success': - _invalid('Invalid text audit evidence.') - if type(fact['actor_id']) is not str or not IDENTIFIER.fullmatch(fact['actor_id']): - _invalid('Invalid historical text actor identity.') - if _time(fact['occurred_at']) < _time(state['updated_at']): - _invalid('Text audit precedes its mutation.') - _shape(receipt, {'request_key', 'request_sha256', 'action', 'target', 'revision'}, 'text operation') - key, digest = receipt['request_key'], receipt['request_sha256'] - if type(key) is not str or not REQUEST_KEY.fullmatch(key) or key in operations or type(digest) is not str or not SHA256.fullmatch(digest): - _invalid('Invalid or duplicate text operation identity.') - if receipt['target'] != identity or type(receipt['revision']) is not int or receipt['revision'] != revision or receipt['action'] != ('import_text' if revision == 1 else 'annotate'): - _invalid('Text operation revision or target mismatch.') - operations.add(key) - documents[identity] = document - # Parents are authored source lineage. They must exist, precede children, - # and form a closed acyclic graph; no foreign family can supply authority. - for identity, document in documents.items(): - current = document['current'] - for parent in current['parents']: - if parent not in documents or _time(documents[parent]['current']['created_at']) > _time(current['created_at']): - _invalid('Text parent reference is missing or newer than its child.') - visiting, visited = set(), set() - for identity in documents: - stack = [(identity, False)] - while stack: - node, leaving = stack.pop() - if leaving: - visiting.remove(node) - visited.add(node) - elif node not in visited: - if node in visiting: - _invalid('Text parent lineage contains a cycle.') - visiting.add(node) - stack.append((node, True)) - stack.extend((parent, False) for parent in documents[node]['current']['parents']) - return copy.deepcopy(records) - - -class _HostAccess: - """Current authorization stays at the installation; only inert facts travel.""" - def __init__(self, host): - self.host, self.facts = host, None - - def require(self, *args, **kwargs): - return self.host.require(*args, **kwargs) - - def denial(self, *args, **kwargs): - return self.host.denial(*args, **kwargs) - - def audit(self, action, target=None, details=None, result='success'): - if self.facts is None or action not in AUDIT_ACTIONS or result != 'success': - raise ProjectTextError('This project operation is unavailable in M1.', 'unavailable', 409) - context = self.host.require('edit', action, target) - self.facts.append(dict(action=action, target=target, details=copy.deepcopy(details or {}), - result=result, actor_id=context.actor_id, occurred_at=timestamp())) - - -class ProjectText: - def __init__(self, store, db, access): - self.store, self.db, self.access = store, db, access - self._lock = threading.RLock() - self._authority = _HostAccess(access) - inert = SimpleNamespace(db=db, lock=db.lock, path=db.path, access=self._authority) - self._workbench = Workbench(inert) - self._snapshot = None - self.available = False - - @property - def checkpoint(self): - return self._snapshot['checkpoint'] if self._snapshot else None - - def _authorize(self, scope, action, target=None): - self.access.require(scope, action, target) - - def _bound(self): - if not self.available: - raise ProjectTextError('Project projection requires reconstruction.', 'projection_unavailable', 503) - current = self.store.snapshot() - if current['checkpoint'] != self.checkpoint: - raise ProjectTextError('Project changed outside this projection. Reload it.', 'stale_project', 409) - return current - - def _tag(self, value, request_key=None): - result = dict(value, project_id=self._snapshot['project_id'], checkpoint=self.checkpoint) - if request_key is not None: - result['request_key'] = request_key - return result - - def restore(self, snapshot=None): - """Replace only this disposable text projection after full validation.""" - with self._lock, self.db.lock: - if self.db.in_transaction: - raise ProjectTextError('Reconstruction must own its SQL transaction.', 'unavailable', 409) - current = self.store.snapshot() - if snapshot is not None and snapshot != current: - raise ProjectTextError('Restore snapshot is stale.', 'stale_project', 409) - records = decode_records(current['records']) - self.available = False - with self.db.transaction(): - if self.db.execute("SELECT 1 FROM workbench_records WHERE kind<>'text' LIMIT 1").fetchone() or self.db.execute('SELECT 1 FROM samples LIMIT 1').fetchone(): - raise ProjectTextError('M1 requires an isolated text projection.', 'unavailable', 409) - self.db.execute('DELETE FROM workbench_history') - self.db.execute('DELETE FROM workbench_rights_notes') - self.db.execute('DELETE FROM workbench_target_proposals') - self.db.execute('DELETE FROM workbench_records') - for envelope in records: - doc, row = envelope['document'], envelope['document']['current'] - self.db.execute('''INSERT INTO workbench_records - (id,kind,name,text,original_text,content_hash,pixel_hash,groups_json,parents_json,provenance_json,task,annotation_json,review,revision,created_at,updated_at) - VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)''', - (row['id'], 'text', row['name'], row['text'], doc['original_text'], row['content_hash'], None, - encode(row['groups']), encode(row['parents']), encode(row['provenance']), row['task'], - encode(row['annotation']) if row['annotation'] is not None else None, row['review'], row['revision'], row['created_at'], row['updated_at'])) - for state in doc['history']: - self._workbench._history(state) - if self.store.summary()['checkpoint'] != current['checkpoint']: - raise ProjectTextError('Project changed during reconstruction.', 'stale_project', 409) - self._snapshot, self.available = current, True - return copy.deepcopy(current) - - def query(self, options=None): - with self.access.lock, self._lock, self.db.lock: - self._authorize('inspect', 'workbench.query') - self._bound() - result = self._workbench.query(options or {}) - self._bound() - return self._tag(result) - - def records(self): - """Full current values for the controller's bounded public projection. - - Canonical values come from one verified snapshot, avoiding pagination - truncation and repeated full inventory reads for each individual row. - """ - with self.access.lock, self._lock, self.db.lock: - self._authorize('inspect', 'workbench.query') - current = self._bound() - return self._tag({'items': [envelope['document']['current'] for envelope in current['records']]}) - - def get(self, record_id): - with self.access.lock, self._lock, self.db.lock: - self._authorize('inspect', 'workbench.get', record_id) - self._bound() - result = self._workbench.get(record_id) - self._bound() - return self._tag(result) - - def history(self, record_id): - with self.access.lock, self._lock, self.db.lock: - self._authorize('inspect', 'workbench.history', record_id) - self._bound() - result = self._workbench.history(record_id) - self._bound() - return [self._tag(state) for state in result] - - def import_text(self, body, expected_checkpoint, request_key): - if type(body) is not dict or set(body) - {'kind', 'text', 'name', 'groups', 'parents', 'rights'} or body.get('kind', 'text') != 'text': - raise ProjectTextError('Only text import is available in M1.', 'unavailable', 409) - return self._mutate('import_text', None, body, expected_checkpoint, request_key) - - def annotate(self, record_id, body, expected_checkpoint, request_key): - _shape(body, {'revision', 'source_revision', 'task', 'annotation', 'groups', 'review'}, 'annotation request') - if body['task'] not in ('text_classification', 'text_entities', 'text_corpus') or body['review'] not in ('draft', 'human_reviewed'): - raise ProjectTextError('This annotation operation is unavailable in M1.', 'unavailable', 409) - return self._mutate('annotate', record_id, body, expected_checkpoint, request_key) - - def _mutate(self, action, record_id, body, expected_checkpoint, request_key): - # Current owner permission cannot be revoked between authorization and - # canonical publication. Match the controller's host-before-projection - # lock order, including standalone callers of this domain adapter. - with self.access.lock: - return self._mutate_authorized(action, record_id, body, expected_checkpoint, request_key) - - def _mutate_authorized(self, action, record_id, body, expected_checkpoint, request_key): - self._authorize('edit', 'workbench.import_asset' if action == 'import_text' else 'workbench.save', record_id) - if type(request_key) is not str or not REQUEST_KEY.fullmatch(request_key): - _invalid('Supply a bounded operation request key.') - body = copy.deepcopy(body) - try: - digest = _sha(encode(dict(action=action, target=record_id, body=body))) - except (ValueError, TypeError, UnicodeError): - _invalid('Project mutation must contain valid finite JSON.') - with self._lock, self.db.lock: - if self.db.in_transaction: - raise ProjectTextError('Project mutation must own its SQL transaction.', 'unavailable', 409) - current = self._bound() - replay = self.store.reconcile(request_key, digest) - if replay is not None: - for envelope in current['records']: - doc = envelope['document'] - for receipt in doc['operations']: - if receipt['request_key'] == request_key and receipt['request_sha256'] == digest: - return self._tag(doc['history'][receipt['revision'] - 1], request_key) - _invalid('Project operation receipt has no text result.') - if expected_checkpoint != self.checkpoint: - raise ProjectTextError('Expected project checkpoint changed.', 'conflict', 409) - committed = None - self._authority.facts = [] - try: - with self.db.transaction(): - result = (self._workbench.import_asset(dict(body, kind='text')) if action == 'import_text' - else self._workbench.save(record_id, body)) - identity = result['id'] - records = copy.deepcopy(current['records']) - envelope = next((item for item in records if item['id'] == identity), None) - if envelope is None: - original = self.db.execute('SELECT original_text FROM workbench_records WHERE id=%s', (identity,)).fetchone()[0] - envelope = dict(family='workbench_text', id=identity, - document=dict(format='workbench_text_v1', original_text=original, current=None, history=[], audit=[], operations=[])) - records.append(envelope) - document = envelope['document'] - document['current'] = result - document['history'].append(result) - document['audit'].extend(self._authority.facts) - document['operations'].append(dict(request_key=request_key, request_sha256=digest, action=action, - target=identity, revision=result['revision'])) - records = decode_records(records) - # Canonical publication is inside the outer SQL transaction, - # so Workbench savepoints cannot acknowledge DB-only work. - committed = self.store.commit(records, self.checkpoint, request_key, digest) - self._snapshot = committed - return self._tag(result, request_key) - except BaseException as error: - if committed is not None: - self._snapshot, self.available = committed, False - failure = ProjectTextError('Canonical edit committed; reconstruct the unavailable SQL projection.', 'projection_unavailable', 503) - failure.committed, failure.project_id = True, committed['project_id'] - failure.checkpoint, failure.request_key = committed['checkpoint'], request_key - failure.receipt = dict(committed=True, selected=True, project_id=committed['project_id'], - checkpoint=committed['checkpoint'], request_key=request_key) - raise failure from error - if getattr(error, 'code', None) in ('unknown', 'outcome_unknown'): - self.available = False - raise - finally: - self._authority.facts = None diff --git a/project_transfer.py b/project_transfer.py deleted file mode 100644 index 2fb1d3ec..00000000 --- a/project_transfer.py +++ /dev/null @@ -1,783 +0,0 @@ -"""Explicit, report-first extraction of a stopped legacy catalog. - -Preview creates no domain supervisors and writes no source files. Only the -currently owned controller can stop and hand over a live stopped-source lease. -The private backup remains a legacy backup, never a portable authority dump. -""" -import copy -from contextlib import contextmanager -import hashlib -import json -import os -from pathlib import Path -import re -import stat -import threading -import time -import uuid -import weakref - -import psycopg -from psycopg import sql - -from metadata import Metadata, SCHEMA_VERSION, row_factory, _key -from project_store import (DIR_FLAGS, FILE_FLAGS, ProjectStore, - _rename_noreplace, _encode) -from project_artifacts import ProjectArtifacts, validate_declaration -from workbench import WorkbenchError, timestamp - -UUID=re.compile(r'^[a-f0-9]{32}$') -SHA=re.compile(r'^[a-f0-9]{64}$') -ALIAS=re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$') -ROOT_FAMILIES={ - 'records':'workbench_record','responses':'workbench_response','preferences':'workbench_preference', - 'selections':'saved_selection','searches':'saved_search','graphs':'pipeline_graph', - 'recipe_drafts':'recipe_definition','model_definitions':'model_definition','studies':'study_history', - 'runs':'run_history','releases':'dataset_release','exposures':'dataset_exposure', - 'model_materials':'model_material','recipe_materials':'recipe_material','runtime_materials':'runtime_material'} -NAMESPACED={'pipeline_graph','recipe_definition','model_definition','study_history','trial_history', - 'run_history','candidate_history','model_material','recipe_material','runtime_material','project_snapshot', - 'generation_history','grounded_history','caption_proposal_history','text_proposal_history'} -PRODUCERS={'generation':'generation_history','grounded':'grounded_history', - 'caption':'caption_proposal_history','text_proposal':'text_proposal_history'} -RUNTIME_NAMES=('generation_jobs','grounded_jobs','caption_proposals','text_classification_proposals','workspace_runs') -_LEASE_TOKEN=object() -_ISSUED_STOPPED=weakref.WeakSet() -MAX_SOURCE_FILE=40*1024*1024 - - -def fail(message,code='invalid',status=400):raise WorkbenchError(message,code,status) -def closed(value,fields,label): - if type(value) is not dict or set(value)!=set(fields):fail('Invalid '+label+' fields.') -def digest(raw):return hashlib.sha256(raw).hexdigest() -def canonical(value):return _encode(value) -def sha(value): - if type(value) is not str or not SHA.fullmatch(value):fail('Invalid extraction digest.') -def identity(value): - if type(value) is not str or not UUID.fullmatch(value):fail('Invalid extraction identity.') -def leaf(value): - if type(value) is not str or not 1<=len(value)<=200 or value in ('.','..') or any(c in value for c in '/\\') or any(ord(c)<32 for c in value):fail('Choose one exact destination directory name.') - return value - -def parts(value): - if type(value) is not str or len(value)>512 or '\\' in value or any(ord(c)<32 for c in value):fail('Invalid source-relative path.') - result=value.split('/') - if not 1<=len(result)<=16 or any(p in ('','.','..') for p in result):fail('Source reference must remain inside its configured root.') - return result - - -def _fd_identity(fd): - value=os.fstat(fd) - return value.st_dev,value.st_ino - - -@contextmanager -def _file(root,relative,maximum): - path=parts(relative);parent=os.dup(root);descriptor=None - try: - for name in path[:-1]: - child=os.open(name,DIR_FLAGS,dir_fd=parent);os.close(parent);parent=child - descriptor=os.open(path[-1],FILE_FLAGS,dir_fd=parent) - before=os.fstat(descriptor) - if not stat.S_ISREG(before.st_mode) or before.st_nlink!=1 or before.st_size>maximum:fail('Source payload is unsafe or exceeds its owner bound.','unavailable',409) - yield descriptor,before - after=os.fstat(descriptor);current=os.stat(path[-1],dir_fd=parent,follow_symlinks=False) - key=lambda v:(v.st_dev,v.st_ino,v.st_mode,v.st_nlink,v.st_size,v.st_mtime_ns,v.st_ctime_ns) - if key(before)!=key(after) or key(after)!=key(current):fail('Source payload changed during capture.','stale_source',409) - except OSError:fail('Source payload is missing or unsafe.','unavailable',409) - finally: - if descriptor is not None:os.close(descriptor) - os.close(parent) - - -def _read(root,relative,maximum): - with _file(root,relative,maximum) as (fd,info): - chunks=[];remaining=maximum+1 - while remaining: - raw=os.read(fd,min(65536,remaining)) - if not raw:break - chunks.append(raw);remaining-=len(raw) - result=b''.join(chunks) - if len(result)!=info.st_size or len(result)>maximum:fail('Source payload changed or exceeds its owner bound.','unavailable',409) - return result - - -class ReadonlyCatalog: - """The exact schema proof, without Metadata construction or source writes.""" - _signature=Metadata._signature - validate_schema=Metadata.validate_schema - def __init__(self,dsn,schema,catalog_id,path,*,exclude_peers=False): - identity(catalog_id) - if schema!='catalog_'+catalog_id:fail('Only a current legacy catalog can be extracted.','unsupported',409) - self.dsn,self.schema,self.dataset_id,self.path=dsn,schema,catalog_id,Path(path) - self.schema_version=SCHEMA_VERSION;self.lock=threading.RLock() - self.connection=psycopg.connect(dsn,autocommit=True,row_factory=row_factory) - try: - self.connection.execute(sql.SQL('SET search_path TO {}').format(sql.Identifier(schema))) - self.connection.execute("SET statement_timeout='60000'") - self.connection.execute("SET lock_timeout='1000'") - if exclude_peers: - for key in [_key(schema),*(_key(schema+':worker:'+name) for name in RUNTIME_NAMES)]: - if not self.connection.execute('SELECT pg_try_advisory_lock(%s)',(key,)).fetchone()[0]: - fail('A peer acquired legacy catalog ownership after stop.','busy',409) - self.connection.execute('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY') - self.validate_schema() - except BaseException:self.connection.close();raise - def execute(self,query,parameters=None): - # PostgreSQL READ ONLY is the enforcement boundary, including CTEs. - return self.connection.execute(query,parameters) - @property - def tables(self): - from metadata import tables_for_version - return tables_for_version(self.schema_version) - def close(self):self.connection.close() - - -class CapturedArtifacts: - """Private verified evidence custody; no preview file publication.""" - def __init__(self):self.payloads={};self.declarations={} - def publish_bytes(self,raw,declaration,*,maximum_bytes=None): - declaration=validate_declaration(declaration) - if type(raw) is not bytes or len(raw)!=declaration['bytes'] or digest(raw)!=declaration['sha256']:fail('Captured source bytes differ from their declaration.') - if maximum_bytes is not None and len(raw)>maximum_bytes:fail('Captured source exceeds its owner bound.','too_large',413) - self._add(declaration,raw);return copy.deepcopy(declaration) - def publish_fd(self,descriptor,declaration,*,maximum_bytes=None): - declaration=validate_declaration(declaration);info=os.fstat(descriptor) - if not stat.S_ISREG(info.st_mode) or info.st_size!=declaration['bytes'] or maximum_bytes is not None and info.st_size>maximum_bytes:fail('Captured source descriptor differs.') - hasher=hashlib.sha256();offset=0 - while offset<info.st_size: - raw=os.pread(descriptor,min(1024*1024,info.st_size-offset),offset) - if not raw:fail('Captured source descriptor was truncated.') - hasher.update(raw);offset+=len(raw) - after=os.fstat(descriptor) - if hasher.hexdigest()!=declaration['sha256'] or (info.st_size,info.st_mtime_ns,info.st_ctime_ns)!=(after.st_size,after.st_mtime_ns,after.st_ctime_ns):fail('Captured source descriptor changed.') - old=self.payloads.get(declaration['sha256']) - if old is None:self._add(declaration,os.dup(descriptor)) - return copy.deepcopy(declaration) - def _add(self,declaration,payload): - key=declaration['sha256'];prior=self.declarations.get(key) - if prior is not None and prior!=declaration:fail('Captured payload declarations conflict.') - if key not in self.payloads:self.payloads[key]=payload - self.declarations[key]=copy.deepcopy(declaration) - def read_blob(self,declaration,*,maximum_bytes=MAX_SOURCE_FILE): - key=declaration['sha256'];value=self.declarations.get(key) - if value is None or value['bytes']!=declaration['bytes'] or value['bytes']>maximum_bytes:fail('Captured source bytes are unavailable.','unavailable',409) - payload=self.payloads[key] - raw=payload if type(payload) is bytes else os.pread(payload,value['bytes'],0) - if len(raw)!=value['bytes'] or digest(raw)!=key:fail('Captured source changed.','stale_source',409) - return raw - def status(self,declaration): - value=validate_declaration(declaration);actual=self.declarations.get(value['sha256']);present=value['sha256'] in self.payloads - state='missing' if not present else 'verified' if actual==value else 'corrupt' - return dict(state=state,expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=actual['sha256'] if present else None,observed_bytes=actual['bytes'] if present else None) - inspect=status - @contextmanager - def open_verified(self,declaration,*,maximum_bytes=None): - value=validate_declaration(declaration);payload=self.payloads.get(value['sha256']) - if payload is None or self.declarations.get(value['sha256'])!=value:fail('Unique source payload is missing or differs.','unavailable',409) - if maximum_bytes is not None and value['bytes']>maximum_bytes:fail('Captured payload exceeds its owner bound.','too_large',413) - if type(payload) is bytes: - import tempfile - with tempfile.TemporaryFile() as temporary: - temporary.write(payload);temporary.flush();temporary.seek(0) - yield temporary.fileno() - else: - self.publish_fd(payload,value,maximum_bytes=maximum_bytes) - duplicate=os.dup(payload) - try:yield duplicate - finally:os.close(duplicate) - def close(self): - for value in self.payloads.values(): - if type(value) is int:os.close(value) - self.payloads.clear() - - -class SourceView: - def __init__(self,db,root_fd,root_path,roots,*,actor_id,captured_at=None): - self.db,self.root_fd,self.root_path=db,root_fd,Path(root_path) - self.roots=roots;self.actor_id=actor_id;self.captured_at=captured_at or timestamp() - self.artifacts=CapturedArtifacts();self._closed=False;self._payload_locations={};self._authorize=None - self.snapshot=dict(project_id=db.dataset_id,checkpoint='0'*64,records=[]) - self.fingerprint=None;self.backup_receipt=None;self.external_inventory=[];self._observed_roots={} - def require(self): - if self._authorize is not None:self._authorize() - if self._closed or self.db.connection.closed:fail('Source snapshot lease is closed.','unavailable',409) - current=os.open(self.root_path,DIR_FLAGS) - try: - if _fd_identity(current)!=_fd_identity(self.root_fd):fail('Source catalog root changed.','stale_source',409) - finally:os.close(current) - def read_catalog(self,relative,maximum_bytes=MAX_SOURCE_FILE): - self.require();return _read(self.root_fd,relative,maximum_bytes) - def read_root(self,alias,relative,maximum_bytes=MAX_SOURCE_FILE): - self.require();entry=self.roots.get(alias) - if entry is None:fail('An explicit source root binding is required.','unavailable',409) - raw=_read(entry[0],relative,maximum_bytes) - key=(alias,relative);observation=(len(raw),digest(raw)) - if key in self._observed_roots and self._observed_roots[key]!=observation:fail('External source bytes changed after capture.','stale_source',409) - self._observed_roots[key]=observation - fact=dict(root=alias,relative=relative,sha256=observation[1],bytes=observation[0]) - if fact not in self.external_inventory:self.external_inventory.append(fact) - return raw - def resolve_payload(self,sha256,size): - sha(sha256) - if sha256 in self.artifacts.declarations:return self.artifacts.read_blob(dict(sha256=sha256,bytes=size)) - location=self._payload_locations.get(sha256) - if location is None or location['bytes']!=size:fail('An explicit external payload inventory binding is required.','unavailable',409) - raw=self.read_root(location['root'],location['relative'],MAX_SOURCE_FILE) - if len(raw)!=size or digest(raw)!=sha256:fail('External source payload differs from its declaration.') - return raw - def read_bound_payload(self,sha256,*,maximum_bytes=MAX_SOURCE_FILE): - """Read exact operator-bound bytes; None means no explicit binding. - - Historical paths or root identities never create this capability. A - changed, oversized or unavailable bound file remains an error, even if - another working file exists at the original historical location. - """ - self.require();sha(sha256) - location=self._payload_locations.get(sha256) - if location is None:return None - if location['bytes']>maximum_bytes:fail('Bound original source exceeds its owner bound.','too_large',413) - return self.resolve_payload(sha256,location['bytes']) - @contextmanager - def payload_fd(self,sha256,size): - sha(sha256) - location=self._payload_locations.get(sha256) - if location is None or location['bytes']!=size:fail('An explicit payload descriptor binding is required.','unavailable',409) - with _file(self.roots[location['root']][0],location['relative'],size) as (fd,info): - declaration=dict(format='project_artifact_v1',sha256=sha256,bytes=size) - self.artifacts.publish_fd(fd,declaration,maximum_bytes=size) - yield fd - - def bind_payload(self,sha256,size,root,relative): - # Internal capability owner call, never inferred from a historical path. - self.require();sha(sha256) - if type(size) is not int or not 0<=size<=2**63-1 or root not in self.roots:fail('Invalid external payload binding.') - with _file(self.roots[root][0],relative,max(size,1)) as (fd,info): - if info.st_size!=size:fail('External payload binding differs.') - self.artifacts.publish_fd(fd,dict(format='project_artifact_v1',sha256=sha256,bytes=size),maximum_bytes=size) - self._observed_roots[(root,relative)]=(size,sha256) - self._payload_locations[sha256]=dict(root=root,relative=relative,bytes=size) - fact=dict(root=root,relative=relative,sha256=sha256,bytes=size) - if fact not in self.external_inventory:self.external_inventory.append(fact) - def close(self): - if self._closed:return - self._closed=True;self.artifacts.close();self.db.close();os.close(self.root_fd) - for fd,_ in self.roots.values():os.close(fd) - self.roots.clear() - def __enter__(self):self.require();return self - def __exit__(self,*exc):self.close() - - -class StoppedSourceLease(SourceView): - def __init__(self,*args,_token=None,**kwargs): - if _token is not _LEASE_TOKEN:fail('A stopped source must be issued by its current trusted owner.','forbidden',403) - super().__init__(*args,**kwargs) - def require(self): - super().require() - if self not in _ISSUED_STOPPED or self.backup_receipt is None:fail('Verified stopped backup is unavailable.','unavailable',409) - - -def preview_source(dataset): - """Read a currently opened source without constructing or stopping owners.""" - actor=dataset.access.require('inspect','project_transfer.preview') - root=os.open(dataset.path,DIR_FLAGS);roots={} - try: - for alias,(fd,value) in dataset.projects.roots.items():roots[alias]=(os.dup(fd),list(value)) - view=ReadonlyCatalog(dataset.db.dsn,dataset.db.schema,dataset.db.dataset_id,dataset.path) - source=SourceView(view,root,dataset.path,roots,actor_id=actor.actor_id) - source._authorize=lambda:dataset.access.require('inspect','project_transfer.preview') - return source - except BaseException: - os.close(root) - for fd,_ in roots.values():os.close(fd) - raise - - -def _snapshot(view): - from backup_metadata import read_catalog,read_sequences,summaries - tables=read_catalog(view.db);sequences=read_sequences(view.db) - return tables,sequences,digest(canonical(dict(tables=summaries(tables),sequences=sequences))) - - -def _verified_backup(source,parent_fd,directory): - from backup_metadata import tree_manifest,copy_tree,tree_modes,summaries - leaf(directory) - from project_fork import _inside - if _inside(parent_fd,source.root_fd):fail('Backup cannot overlap its source.','conflict',409) - stage='.tuldok-extraction-backup-'+uuid.uuid4().hex - os.mkdir(stage,0o700,dir_fd=parent_fd) - path=Path('/proc/self/fd')/str(parent_fd)/stage - tables,sequences,fingerprint=_snapshot(source) - before=tree_manifest(source.root_path) - copy_tree(source.root_path,path/'catalog',before,deduplicate=True) - metadata=dict(format_version=1,schema_version=source.db.schema_version,dataset_id=source.db.dataset_id,tables=tables,sequences=sequences) - from backup_metadata import write_json,file_hash - write_json(path/'metadata.json',metadata) - receipt=dict(format_version=1,kind='postgresql',schema_version=source.db.schema_version,dataset_id=source.db.dataset_id, - postgresql_server_version=source.db.connection.info.server_version,driver='psycopg '+psycopg.__version__, - restore_privileges='CONNECT and CREATE on a new PostgreSQL database; catalog schema must be absent', - metadata_sha256=file_hash(path/'metadata.json'),files=before,tables=summaries(tables),sequences=sequences) - write_json(path/'backup-receipt.json',receipt) - source.require() - if before!=tree_manifest(source.root_path) or before!=tree_manifest(path/'catalog') or tree_modes(source.root_path,before)!=tree_modes(path/'catalog',before):fail('Source changed while capturing its stopped backup.','stale_source',409) - directory_fd=os.open(stage,DIR_FLAGS,dir_fd=parent_fd) - try:os.fsync(directory_fd) - finally:os.close(directory_fd) - _rename_noreplace(parent_fd,stage,directory);os.fsync(parent_fd) - return receipt,fingerprint - - -def stop_source(dataset,*,backup_parent_fd,backup_directory,external_roots=None,controller_guard): - """Only the already running, explicitly gated owner may mint this lease.""" - from app import Dataset, _LegacyExtractionGuard - if not isinstance(dataset,Dataset) or type(controller_guard) is not _LegacyExtractionGuard or dataset._extraction_guard is not controller_guard: - fail('An exclusive current-controller stop guard is required.','forbidden',403) - controller_guard.require(dataset) - actor=dataset.access.require('admin','project_transfer.stop_source',owner_only=True) - leaf(backup_directory) - if not stat.S_ISDIR(os.fstat(backup_parent_fd).st_mode):fail('A backup parent capability is required.') - try:os.stat(backup_directory,dir_fd=backup_parent_fd,follow_symlinks=False) - except FileNotFoundError:pass - else:fail('Stopped backup destination already exists.','conflict',409) - if any(not dataset.db.runtime_owned(name) for name in RUNTIME_NAMES):fail('A peer owns a legacy runtime; stop its actual owner first.','busy',409) - if dataset.runs._blocked or dataset.runs._external_blocked:fail('Legacy worker cessation remains unknown.','unknown',409) - root=os.open(dataset.path,DIR_FLAGS);roots={};source=None - dsn,schema,catalog_id=dataset.db.dsn,dataset.db.schema,dataset.db.dataset_id - try: - for alias,(fd,value) in (external_roots if external_roots is not None else dataset.projects.roots).items(): - if type(alias) is not str or not ALIAS.fullmatch(alias) or list(_fd_identity(fd))!=list(value):fail('Invalid explicit external root capability.') - roots[alias]=(os.dup(fd),list(value)) - with dataset.image_requests.lock:pending=list(dataset.image_requests.active.items()) - for request_id,operation in pending:operation.cancel() - deadline=time.monotonic()+60 - for request_id,operation in pending: - if not operation.done.wait(max(0,deadline-time.monotonic())):fail('An owned image request did not drain.','unknown',409) - controller_guard.require(dataset) - # Existing owner close may change lifecycle observations. The preservation - # baseline is captured afterwards, never before this explicit stop. - dataset.close_for_project_extraction(deadline=deadline) - for owner in (dataset.generation_jobs,dataset.grounded,dataset.caption_proposals,dataset.text_classification_proposals): - worker=getattr(owner,'worker',None) - if worker is not None and worker.is_alive():fail('A legacy producer did not drain.','unknown',409) - if dataset.runs._blocked or dataset.runs._external_blocked:fail('Legacy process cessation is unknown.','unknown',409) - with dataset.image_requests.lock: - if dataset.image_requests.active:fail('An image request remains active.','unknown',409) - controller_guard.require(dataset) - db=ReadonlyCatalog(dsn,schema,catalog_id,dataset.path,exclude_peers=True) - # Base view permits backup verification before the stopped lease is minted. - source=SourceView(db,root,dataset.path,roots,actor_id=actor.actor_id) - import recipe_process - for row in db.execute('SELECT document FROM workspace_runs'): - run=json.loads(row[0]);custody=run.get('custody_identity') - if custody is None:continue - if run.get('state')=='completed': - raw=source.read_catalog('workspace-runs/'+run['id']+'/custody-receipt.json',16384) - receipt=recipe_process.validate_receipt(raw,custody) - else:receipt=dataset.runs._external_drain_receipt(run) - if receipt is None or not receipt['drained']:fail('An external custodian has no exact drained receipt.','unknown',409) - receipt,fingerprint=_verified_backup(source,backup_parent_fd,backup_directory) - source.backup_receipt=receipt;source.fingerprint=fingerprint;source.snapshot['checkpoint']=fingerprint - source.__class__=StoppedSourceLease - _ISSUED_STOPPED.add(source) - source.require() - return source - except BaseException: - if source is not None:source.close() - else: - os.close(root) - for fd,_ in roots.values():os.close(fd) - raise - - -def normalize_selection(selection): - closed(selection,{'project_snapshot','roots','decisions'},'extraction selection') - snapshot=selection['project_snapshot'];closed(snapshot,{'id','manifest_sha256'},'selected project snapshot') - identity(snapshot['id']);sha(snapshot['manifest_sha256']) - roots=selection['roots'];known=set(ROOT_FAMILIES)|{'producer_jobs','working_files'} - if type(roots) is not dict or set(roots)-known:fail('Unsupported extraction root family.') - normalized={} - for name in ROOT_FAMILIES: - values=roots.get(name,[]) - if type(values) is not list or len(values)>10000 or any(type(v) is not str for v in values) or len(set(values))!=len(values):fail('Extraction roots must be bounded unique identifiers.') - for value in values: - sha(value) if name in ('releases','model_materials','recipe_materials','runtime_materials') else identity(value) - normalized[name]=sorted(values) - jobs=roots.get('producer_jobs',[]) - if type(jobs) is not list or len(jobs)>10000:fail('Invalid producer selection.') - seen=set();normalized_jobs=[] - for value in jobs: - closed(value,{'kind','id'},'producer selection');identity(value['id']) - if value['kind'] not in PRODUCERS or (value['kind'],value['id']) in seen:fail('Unsupported or duplicate producer root.') - seen.add((value['kind'],value['id']));normalized_jobs.append(copy.deepcopy(value)) - normalized['producer_jobs']=sorted(normalized_jobs,key=lambda v:(v['kind'],v['id'])) - files=roots.get('working_files',[]) - if type(files) is not list or len(files)>1000:fail('Invalid working-file selection.') - from project_files import _path - seen=set();normalized_files=[] - for value in files: - closed(value,{'kind','path'},'working-file selection');_path(value['kind'],value['path']) - if (value['kind'],value['path']) in seen:fail('Duplicate working-file selection.') - seen.add((value['kind'],value['path']));normalized_files.append(copy.deepcopy(value)) - normalized['working_files']=sorted(normalized_files,key=lambda v:(v['kind'],v['path'])) - decisions=selection['decisions'] - if type(decisions) is not list or len(decisions)>10000:fail('Invalid ownership decisions.') - allowed=set(ROOT_FAMILIES.values())|NAMESPACED|{'project_artifact','project_file','trial_history','candidate_history'} - fixed=[];seen=set() - for value in decisions: - closed(value,{'family','id','include','reason'},'ownership decision') - if value['family'] not in allowed or type(value['id']) is not str or not 1<=len(value['id'])<=128 or type(value['include']) is not bool or type(value['reason']) is not str or not 1<=len(value['reason'])<=1000 or (value['family'],value['id']) in seen:fail('Invalid explicit ownership decision.') - seen.add((value['family'],value['id']));fixed.append(copy.deepcopy(value)) - return dict(project_snapshot=copy.deepcopy(snapshot),roots=normalized,decisions=sorted(fixed,key=lambda v:(v['family'],v['id']))) - - -def _seeds(selection,catalog_id): - selected={('project_snapshot',catalog_id+'.'+selection['project_snapshot']['manifest_sha256'])} - for root,family in ROOT_FAMILIES.items(): - for value in selection['roots'][root]:selected.add((family,catalog_id+'.'+value if family in NAMESPACED else value)) - for value in selection['roots']['producer_jobs']:selected.add((PRODUCERS[value['kind']],catalog_id+'.'+value['id'])) - from project_files import file_id - for value in selection['roots']['working_files']:selected.add(('project_file',file_id(catalog_id,value['kind'],value['path']))) - for value in selection['decisions']: - if value['include']:selected.add((value['family'],value['id'])) - return selected - - -def _capture(source,selection): - from project_extraction import capture_legacy - captured=capture_legacy(source,selection) - closed(captured,{'records','inventory','dependencies','blockers','working_files','warnings'},'legacy capture') - for field in captured: - if type(captured[field]) is not list:fail('Legacy capture fields must be finite lists.') - return captured - - -def _validate_candidate(records,project_id,artifacts): - from project_projection import ProjectProjection - from types import SimpleNamespace - ProjectProjection.validate_records(SimpleNamespace(project_id=project_id,artifacts=artifacts),records,project_id=project_id) - - -class ProjectTransfer: - def __init__(self,source): - if not isinstance(source,SourceView):fail('A pinned read-only source capability is required.','forbidden',403) - self.source=source;self._reports={};self.lock=threading.RLock() - - def extract(self,selection,expected_report_sha256,destination_parent_fd,directory,name,request_key,*,opened_store=None): - with self.lock:return _extract(self,selection,expected_report_sha256,destination_parent_fd,directory,name,request_key,opened_store=opened_store) - - def preview(self,selection): - with self.lock:return self._preview(selection) - - def _preview(self,selection): - source=self.source;source.require();selection=normalize_selection(selection) - if source.fingerprint is None: - _,_,source.fingerprint=_snapshot(source);source.snapshot['checkpoint']=source.fingerprint - captured=_capture(source,selection) - inventory={} - for item in captured['inventory']: - closed(item,{'family','id','revision'},'legacy inventory item') - key=item['family'],item['id'] - if key in inventory:fail('Legacy inventory repeats an identity.') - inventory[key]=item - required=_seeds(selection,source.db.dataset_id);direct=set(required);edges={} - for edge in captured['dependencies']: - closed(edge,{'source','target','reason'},'legacy dependency') - for field in ('source','target'):closed(edge[field],{'family','id'},'dependency locator') - edges.setdefault((edge['source']['family'],edge['source']['id']),[]).append((edge['target']['family'],edge['target']['id'])) - pending=list(required) - while pending: - for key in edges.get(pending.pop(),[]): - if key not in required:required.add(key);pending.append(key) - excluded={(v['family'],v['id']) for v in selection['decisions'] if not v['include']} - blockers=[copy.deepcopy(v) for v in captured['blockers'] if (v['family'],v['id']) in required] - for family,record_id in sorted(required-inventory.keys()):blockers.append(dict(family=family,id=record_id,code='missing_dependency',detail='Selected or required source fact is unavailable.')) - for family,record_id in sorted(required&excluded):blockers.append(dict(family=family,id=record_id,code='required_dependency_excluded',detail='Required closure cannot be excluded.')) - declared={(r['family'],r['id']):r for r in captured['records']} - records=[copy.deepcopy(declared[k]) for k in sorted(required) if k in declared] - artifact_report=[] - artifact_policies={} - for edge in captured['dependencies']: - target=(edge['target']['family'],edge['target']['id']) - parent=(edge['source']['family'],edge['source']['id']) - if parent in required and target[0]=='project_artifact': - policy='regenerable' if edge['reason'].startswith('regenerable:') else 'unique' - if artifact_policies.get(target[1])!='unique':artifact_policies[target[1]]=policy - for record in records: - if record['family']=='project_artifact': - state=source.artifacts.status(record['document'])['state'] - preservation=artifact_policies.get(record['id'],'unique') - artifact_report.append(dict(sha256=record['id'],bytes=record['document']['bytes'],preservation=preservation,state=state)) - if state!='verified' and preservation=='unique':blockers.append(dict(family='project_artifact',id=record['id'],code='missing_unique_payload',detail='Original required bytes cannot be regenerated during extraction.')) - file_observations=[] - from project_files import file_id - for value in captured['working_files']: - closed(value,{'kind','path','sha256','bytes','text'},'observed working file') - if ('project_file',file_id(source.db.dataset_id,value['kind'],value['path'])) in required:file_observations.append(copy.deepcopy(value)) - if not blockers: - try:_validate_candidate(records,source.db.dataset_id,source.artifacts) - except WorkbenchError as error:blockers.append(dict(family='closure',id=source.db.dataset_id,code=getattr(error,'code','invalid'),detail=str(error))) - if not isinstance(source,StoppedSourceLease):blockers.append(dict(family='source',id=source.db.dataset_id,code='cessation_proof_unavailable',detail='The current trusted owner must explicitly stop and capture this source.')) - report=dict(format='project_transfer_report_v1',source=dict(catalog_id=source.db.dataset_id,schema_version=source.db.schema_version, - catalog_signature=source.db._signature(),snapshot_sha256=source.fingerprint, - backup_receipt_sha256=digest(canonical(source.backup_receipt)) if source.backup_receipt is not None else None, - external_inventory_sha256=digest(canonical(sorted(source.external_inventory,key=lambda v:(v['root'],v['relative']))))), - selection=selection,included=[copy.deepcopy(inventory[k]) for k in sorted(required&inventory.keys())], - required=[dict(family=f,id=i) for f,i in sorted(required-direct)], - unassigned=[copy.deepcopy(inventory[k]) for k in sorted(inventory.keys()-required-excluded)], - excluded=[dict(family=f,id=i) for f,i in sorted(excluded)],blockers=blockers, - warnings=[copy.deepcopy(v) for v in captured['warnings'] if (v['family'],v['id']) in required], - artifacts=artifact_report,working_files=[{k:v[k] for k in ('kind','path','sha256','bytes')} for v in file_observations],can_extract=not blockers) - report_hash=digest(canonical(report));report['report_sha256']=report_hash - self._reports[report_hash]=dict(report=copy.deepcopy(report),records=records,working_files=file_observations) - source.require();return report - - -FAMILY='project_extraction' -FORMAT='project_extraction_v1' -SOURCE_FIELDS={'catalog_id','schema_version','catalog_signature','backup_receipt_sha256','external_inventory_sha256','snapshot_sha256'} - - -def decode_records(records): - from project_store import SUPPORTED_FORMATS - from project_files import file_id - indexed={(r['family'],r['id']):r for r in records} - counts={} - for r in records:counts[r['family']]=counts.get(r['family'],0)+1 - seen=set();result=[] - for envelope in records: - if envelope.get('family')!=FAMILY:continue - closed(envelope,{'family','id','document'},'extraction envelope');doc=envelope['document'] - closed(doc,{'format','source','selection','report_sha256','request_key','input_sha256','destination_project_id','working_files','counts','artifacts'},'extraction manifest') - if doc['format']!=FORMAT:fail('Unsupported extraction manifest format.','unsupported',409) - identity(envelope['id']);identity(doc['destination_project_id']) - if envelope['id']!=doc['destination_project_id'] or envelope['id'] in seen:fail('Duplicate or mismatched extraction identity.') - seen.add(envelope['id']);source=doc['source'];closed(source,SOURCE_FIELDS,'extraction source') - identity(source['catalog_id']) - if source['catalog_id']==doc['destination_project_id'] or type(source['schema_version']) is not int or source['schema_version']!=SCHEMA_VERSION:fail('Extraction source namespace or schema differs.') - for field in SOURCE_FIELDS-{'catalog_id','schema_version'}:sha(source[field]) - sha(doc['report_sha256']);sha(doc['input_sha256']) - overlay=indexed.get(('project_fork',doc['destination_project_id'])) - if overlay is None:fail('Extraction lacks its independent identity overlay.') - overlay=overlay['document'] - if overlay['source']!=dict(project_id=source['catalog_id'],checkpoint=source['snapshot_sha256']) or overlay['request_key']!=doc['request_key'] or overlay['artifacts']!=doc['artifacts']:fail('Extraction manifest differs from its retained identity transition.') - if digest(canonical(_input(source,doc['selection'],doc['report_sha256'],overlay['directory'],overlay['name'],doc['request_key'])))!=doc['input_sha256']:fail('Extraction input digest differs from its exact publication request.') - registered=indexed.get(('project_snapshot',source['catalog_id']+'.'+doc['selection']['project_snapshot']['manifest_sha256'])) - if registered is None or registered['document']['value']['id']!=doc['selection']['project_snapshot']['id']:fail('Extraction lacks its exact selected registered snapshot.') - if normalize_selection(doc['selection'])!=doc['selection']:fail('Extraction selection is not normalized.') - from project_store import REQUEST_KEY - if type(doc['request_key']) is not str or not REQUEST_KEY.fullmatch(doc['request_key']):fail('Invalid extraction request key.') - if type(doc['counts']) is not dict or any(f not in SUPPORTED_FORMATS or type(n) is not int or n<1 or counts.get(f,0)<n for f,n in doc['counts'].items()):fail('Extraction population counts differ from retained facts.') - if any(doc['counts'].get(f,0)<1 for f in (FAMILY,'project_fork','project_snapshot')):fail('Extraction population lacks its required source and publication facts.') - closed(doc['artifacts'],{'copied','missing_regenerable'},'extraction artifact report') - listed=set() - for field in ('copied','missing_regenerable'): - values=doc['artifacts'][field] - if type(values) is not list or values!=sorted(set(values)):fail('Invalid extraction artifact inventory.') - for value in values: - sha(value) - if value in listed or ('project_artifact',value) not in indexed:fail('Extraction artifact lacks its intrinsic declaration.') - listed.add(value) - if type(doc['working_files']) is not list:fail('Invalid extracted working-file observations.') - paths=set() - for value in doc['working_files']: - closed(value,{'kind','path','sha256','bytes','source_catalog_id','source_snapshot_sha256','destination_id','adopted_at','actor_id'},'extracted working-file observation') - if value['source_catalog_id']!=source['catalog_id'] or value['source_snapshot_sha256']!=source['snapshot_sha256']:fail('Working-file source observation differs.') - sha(value['sha256']);sha(value['destination_id']);identity(value['actor_id']) - if value['destination_id']!=file_id(doc['destination_project_id'],value['kind'],value['path']) or (value['kind'],value['path']) in paths:fail('Extracted working-file identity differs.') - paths.add((value['kind'],value['path'])) - target=indexed.get(('project_file',value['destination_id'])) - if target is None:fail('Extracted working-file adoption is missing.') - initial=target['document']['history'][0] - if target['document']['origin'] is not None or initial['revision']!=1 or any(initial[k]!=value[k] for k in ('sha256','bytes','adopted_at','actor_id')): - fail('Extracted working file is not its exact authorized first adoption.') - result.append(copy.deepcopy(envelope)) - return result - - -def _assert_unchanged(source): - source.require() - from backup_metadata import tree_manifest - if source.backup_receipt is not None and tree_manifest(source.root_path)!=source.backup_receipt['files']:fail('Stopped source files changed after the backup baseline.','stale_source',409) - for (alias,relative),(size,expected_sha) in source._observed_roots.items(): - try: - with _file(source.roots[alias][0],relative,max(size,1)) as (fd,info): - hasher=hashlib.sha256();offset=0 - while offset<info.st_size: - block=os.pread(fd,min(1024*1024,info.st_size-offset),offset) - if not block:fail('External source bytes changed after capture.','stale_source',409) - hasher.update(block);offset+=len(block) - if info.st_size!=size or hasher.hexdigest()!=expected_sha:fail('External source bytes changed after capture.','stale_source',409) - except WorkbenchError:fail('External source bytes changed after capture.','stale_source',409) - fresh=ReadonlyCatalog(source.db.dsn,source.db.schema,source.db.dataset_id,source.root_path) - try: - from types import SimpleNamespace - _,_,actual=_snapshot(SimpleNamespace(db=fresh)) - finally:fresh.close() - if actual!=source.fingerprint:fail('Stopped source SQL changed after its backup baseline.','stale_source',409) - - -class _VerificationAccess: - """Ephemeral read-only verification authority from the explicit stop lease.""" - def __init__(self,source):self.source=source;self.lock=threading.RLock() - def require(self,scope,action,target=None,**kw): - self.source.require() - if scope!='inspect':fail('Cold extraction verification cannot admit mutations.','forbidden',403) - from types import SimpleNamespace - return SimpleNamespace(actor_id=self.source.actor_id) - def identity(self):return dict(actor_id=self.source.actor_id,kind='verification') - def denial(self,*args,**kw):pass - - -def _cold_verify(store,source): - """Fresh physical SQL projection and all fixed domain decoders, no jobs.""" - from metadata import Metadata - from project_session import ProjectSession - from project_data import ProjectData - from project_material import ProjectMaterial - from project_history import ProjectHistory - from project_producers import ProjectProducers - snapshot=store.snapshot();path=store.path/'.tuldok-local'/('extraction-verify-'+uuid.uuid4().hex) - db=Metadata(path,dsn=source.db.dsn,dataset_id=snapshot['project_id'],projection_id=uuid.uuid4().hex) - artifacts=ProjectArtifacts(store) - try: - authority=_VerificationAccess(source) - session=ProjectSession(store,db,authority,artifacts,validate_records=lambda records:_validate_candidate(records,snapshot['project_id'],artifacts)) - producers=ProjectProducers(session);producers.restore(snapshot) - data=ProjectData(session,generation_entries=producers.generation_jobs);data.restore() - materials=ProjectMaterial(session);materials.restore() - history=ProjectHistory(session,materials=materials) - history._facts() - from project_fork import active_records - active=active_records(snapshot['records'],project_id=snapshot['project_id']) - expected=sum(r['family'] in ('workbench_record','workbench_text') for r in active) - actual=db.execute('SELECT count(*) FROM workbench_records').fetchone()[0] - if actual!=expected:fail('Fresh data reconstruction lost active membership.') - if store.snapshot()!=snapshot:fail('Cold reconstruction changed canonical extraction facts.') - return dict(checkpoint=snapshot['checkpoint'],active_data_records=actual) - finally: - db.close();artifacts.close() - # Only this task-created disposable projection is removed; canonical - # files and the stopped legacy schema are never cleanup targets. - with psycopg.connect(source.db.dsn,autocommit=True) as cleanup: - cleanup.execute(sql.SQL('DROP SCHEMA {} CASCADE').format(sql.Identifier(db.schema))) - - -def _input(source,selection,report_sha,directory,name,key): - return dict(source=source,selection=selection,report_sha256=report_sha,directory=directory,name=name,request_key=key) - - -def _receipt(snapshot,manifest,verification,*,replayed=False): - return dict(format='project_extraction_receipt_v1',project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint'], - request_key=manifest['request_key'],input_sha256=manifest['input_sha256'],report_sha256=manifest['report_sha256'], - source=copy.deepcopy(manifest['source']),artifacts=copy.deepcopy(manifest['artifacts']),verification=copy.deepcopy(verification), - committed=True,selected=False,replayed=replayed) - - -def _extract(self,selection,expected_report_sha256,destination_parent_fd,directory,name,request_key,*,opened_store=None): - source=self.source - if type(source) is not StoppedSourceLease:fail('Extraction requires its opaque trusted stopped-source lease.','forbidden',403) - source.require();selection=normalize_selection(selection);sha(expected_report_sha256);leaf(directory) - from project_store import REQUEST_KEY - if type(request_key) is not str or not REQUEST_KEY.fullmatch(request_key) or type(name) is not str or not name.strip() or len(name)>200:fail('Invalid extraction request identity.') - report=self.preview(selection) - if report['report_sha256']!=expected_report_sha256:fail('The extraction report changed; review the exact source closure again.','stale_source',409) - if not report['can_extract']: - error=WorkbenchError('Selected legacy project has unresolved extraction blockers.','extraction_blocked',409);error.report=report;raise error - cache=self._reports[expected_report_sha256];records=copy.deepcopy(cache['records']) - _assert_unchanged(source) - source_ref=report['source'];request_digest=digest(canonical(_input(source_ref,selection,expected_report_sha256,directory,name,request_key))) - from project_fork import (_inside,ARCHIVE_FAMILIES,_digest as fork_digest,_input as fork_input) - if _inside(destination_parent_fd,source.root_fd):fail('Extraction destination cannot overlap its source.','conflict',409) - root_path=Path(os.readlink('/proc/self/fd/'+str(destination_parent_fd))) - try:existing=os.stat(directory,dir_fd=destination_parent_fd,follow_symlinks=False) - except FileNotFoundError:existing=None - def replay(opened): - snapshot=opened.reconcile(request_key,request_digest) - if snapshot is None:fail('Extraction destination belongs to another request.','conflict',409) - manifest=next((r['document'] for r in snapshot['records'] if r['family']==FAMILY and r['id']==snapshot['project_id']),None) - if manifest is None or manifest['input_sha256']!=request_digest:fail('Extraction receipt is unavailable or differs.') - with ProjectArtifacts(opened) as current_artifacts: - _validate_candidate(snapshot['records'],snapshot['project_id'],current_artifacts) - verification=_cold_verify(opened,source);_assert_unchanged(source) - return _receipt(snapshot,manifest,verification,replayed=True) - if opened_store is not None: - # Trusted controller-only capability. Its caller pins workspace, - # authority, selected session and SQL ownership through this replay. - from project_working_store import WorkingProjectStore - if not isinstance(opened_store,(ProjectStore,WorkingProjectStore)) or existing is None: - fail('Current selected destination is unavailable.','stale',409) - with opened_store.root_directory() as owned_fd: - try:target_fd=os.open(directory,DIR_FLAGS,dir_fd=destination_parent_fd) - except OSError:fail('Selected destination binding changed.','stale',409) - try: - if _fd_identity(target_fd)!=_fd_identity(owned_fd):fail('Selected store differs from the extraction destination.','conflict',409) - finally:os.close(target_fd) - return replay(opened_store) - if existing is not None: - with ProjectStore.open_at(destination_parent_fd,directory,root_path=root_path) as opened: - return replay(opened) - stage_name='.tuldok-extraction-'+uuid.uuid4().hex - store=None;artifacts=None;committed=None;published=False;stage_identity=None - try: - store=ProjectStore.create_at(destination_parent_fd,stage_name,name,'extraction-stage.'+digest(request_key.encode())[:32],request_digest,root_path=root_path) - with store.root_directory() as owned_fd:stage_identity=_fd_identity(owned_fd) - destination=store.summary()['project_id'];created_at=timestamp() - from project_data_fork import fork_records as fork_data - from project_material_fork import fork_records as fork_material - original=copy.deepcopy(records);mappings=[] - allocator=lambda *args:uuid.uuid4().hex - for builder in (fork_data,fork_material): - made=builder(original,source.db.dataset_id,source.fingerprint,destination,id_factory=allocator,created_at=created_at) - records.extend(made['records']);mappings.extend(made['mapping']) - working=[] - from project_files import file_id,materialize_captured - for observed in cache['working_files']: - file_identity=file_id(destination,observed['kind'],observed['path']) - version=dict(revision=1,sha256=observed['sha256'],bytes=observed['bytes'],text=observed['text'],adopted_at=created_at,actor_id=source.actor_id) - records.append(dict(family='project_file',id=file_identity,document=dict(format='project_file_v1',project_id=destination,kind=observed['kind'],path=observed['path'],current=copy.deepcopy(version),history=[version],origin=None))) - working.append(dict(kind=observed['kind'],path=observed['path'],sha256=observed['sha256'],bytes=observed['bytes'],source_catalog_id=source.db.dataset_id,source_snapshot_sha256=source.fingerprint,destination_id=file_identity,adopted_at=created_at,actor_id=source.actor_id)) - declarations={r['id']:r['document'] for r in records if r['family']=='project_artifact'} - copied=sorted(v['sha256'] for v in report['artifacts'] if v['state']=='verified') - missing_regenerable=sorted(v['sha256'] for v in report['artifacts'] if v['state']!='verified' and v['preservation']=='regenerable') - fork_source=dict(project_id=source.db.dataset_id,checkpoint=source.fingerprint) - overlay=dict(format='project_fork_v1',source=fork_source,destination_project_id=destination, - request_key=request_key,request_sha256=fork_digest(fork_input(fork_source,directory,name,request_key)),directory=directory,name=name, - mapping=sorted(mappings,key=lambda m:(m['family'],m['source_id'],m['source_revision'] or 0,m['destination_id'])), - archived=sorted([dict(family=r['family'],id=r['id']) for r in original if r['family'] in ARCHIVE_FAMILIES],key=lambda v:(v['family'],v['id'])), - artifacts=dict(copied=copied,missing_regenerable=missing_regenerable)) - records.append(dict(family='project_fork',id=destination,document=overlay)) - counts={FAMILY:1} - for r in records:counts[r['family']]=counts.get(r['family'],0)+1 - manifest=dict(format=FORMAT,source=copy.deepcopy(source_ref),selection=selection,report_sha256=expected_report_sha256,request_key=request_key,input_sha256=request_digest,destination_project_id=destination,working_files=working,counts=counts,artifacts=dict(copied=copied,missing_regenerable=missing_regenerable)) - records.append(dict(family=FAMILY,id=destination,document=manifest)) - artifacts=ProjectArtifacts(store) - for value in declarations.values(): - if value['sha256'] in missing_regenerable:continue - with source.artifacts.open_verified(value,maximum_bytes=value['bytes']) as fd:artifacts.publish_fd(fd,value,maximum_bytes=value['bytes']) - with store.root_directory() as fd:materialize_captured(records,destination,fd) - _validate_candidate(records,destination,artifacts) - committed=store.commit(records,store.summary()['checkpoint'],request_key,request_digest) - verification=_cold_verify(store,source) - store.save(committed['checkpoint']);_assert_unchanged(source) - _rename_noreplace(destination_parent_fd,stage_name,directory);published=True;os.fsync(destination_parent_fd) - _assert_unchanged(source) - return _receipt(committed,manifest,verification) - except BaseException as error: - if not published and store is not None and committed is not None: - try: - final=os.stat(directory,dir_fd=destination_parent_fd,follow_symlinks=False) - published=(final.st_dev,final.st_ino)==stage_identity - except FileNotFoundError:pass - except OSError: - failure=WorkbenchError('Extraction publication outcome cannot be inspected.','unknown',503) - failure.staging_directory=stage_name - raise failure from error - if published: - failure=WorkbenchError('Extraction was published; acknowledgement requires reconciliation.','unknown',503) - failure.receipt=dict(committed=True,selected=False,project_id=committed['project_id'],checkpoint=committed['checkpoint'],request_key=request_key,directory=directory) - raise failure from error - error.staging_directory=stage_name - raise - finally: - if artifacts is not None:artifacts.close() - if store is not None:store.close() diff --git a/project_transfer_controller.py b/project_transfer_controller.py deleted file mode 100644 index 3ece224e..00000000 --- a/project_transfer_controller.py +++ /dev/null @@ -1,309 +0,0 @@ -"""Explicit process-local legacy extraction and independent workspace selection. - -The live Dataset is supplied by its existing controller. Stopping never imports -its credentials into the separate control host. Only that Dataset can issue a -StoppedSourceLease; requests carry a controller binding and report hashes, never -serialized cessation evidence or filesystem descriptors. -""" -import copy -from contextlib import ExitStack -import hashlib -import os -from pathlib import Path -import re -import threading -import uuid - -from project_store import _open_directory_path, _verify_origin, REQUEST_KEY -from project_transfer import ProjectTransfer, preview_source, normalize_selection, canonical, leaf, sha -from project_fork import _inside -from workbench import WorkbenchError - -SECTIONS = frozenset(('included', 'required', 'unassigned', 'excluded', 'blockers', 'warnings', 'artifacts', 'working_files')) -OPERATIONS = frozenset('project_transfer.' + name for name in ('status', 'preview', 'report_page', 'stop', 'extract')) -MAX_RESPONSE = 1024 * 1024 - - -def fail(message, code='invalid', status=400): - raise WorkbenchError(message, code, status) - - -def _identity(descriptor): - value = os.fstat(descriptor) - return value.st_dev, value.st_ino - - -def _bounded(value): - if len(canonical(value)) > MAX_RESPONSE: - fail('Extraction controller response exceeds its bounded interface.', 'too_large', 413) - return value - - -def _receipt_summary(value): - if value.get('format') != 'project_extraction_receipt_v1': - fields = ('committed', 'selected', 'project_id', 'checkpoint', 'request_key', 'directory') - return {key: copy.deepcopy(value[key]) for key in fields if key in value} - return dict(format='project_transfer_receipt_v1', committed=value['committed'], - project_id=value['project_id'], checkpoint=value['checkpoint'], request_key=value['request_key'], - input_sha256=value['input_sha256'], report_sha256=value['report_sha256'], source=copy.deepcopy(value['source']), - artifacts=dict(copied_count=len(value['artifacts']['copied']), missing_regenerable_count=len(value['artifacts']['missing_regenerable'])), - verification=copy.deepcopy(value['verification']), replayed=value['replayed']) - - -class ProjectTransferController: - def __init__(self, dataset, control_workspace, roots=None): - from app import Dataset - from project_workspace import ProjectWorkspace - if not isinstance(dataset, Dataset) or not isinstance(control_workspace, ProjectWorkspace): - fail('Extraction requires an existing legacy controller and a separate control workspace.', 'forbidden', 403) - if dataset.db.dataset_id == control_workspace.db.dataset_id or dataset.db is control_workspace.db: - fail('The control host must have independent current local authority.', 'forbidden', 403) - self.dataset, self.workspace = dataset, control_workspace - self._lock = threading.RLock() - self._binding = uuid.uuid4().hex - self._phase = 'source_live' - self._closed = False - self._roots = {} - self._source = self._transfer = self._report = self._selection = None - self._backup = self._destination = self._last_receipt = self._failure = None - configured = roots if roots is not None else {alias: value[0] for alias, value in control_workspace._roots.items()} - try: - if type(configured) is not dict or not 1 <= len(configured) <= 100: - fail('Configure one to 100 extraction publication roots.') - for alias, path in configured.items(): - if type(alias) is not str or not re.fullmatch('[A-Za-z0-9][A-Za-z0-9_.-]{0,63}', alias): - fail('Invalid extraction publication root alias.') - absolute, descriptor = _open_directory_path(path) - self._roots[alias] = (Path(absolute), descriptor) - # Publication parents may contain the source as a sibling, but may - # never be inside it. Nonexistent leaf targets are checked at Stop. - _, source_fd = _open_directory_path(dataset.path) - try: - if any(_inside(descriptor, source_fd) for _, descriptor in self._roots.values()): - fail('Extraction publication roots cannot be inside the source catalog.', 'conflict', 409) - finally: - os.close(source_fd) - except BaseException: - self.close() - raise - - @property - def control_workspace(self): - return self.workspace - - @property - def phase(self): - """Trusted root dispatch uses this only to choose request authentication.""" - return self._phase - - @property - def access(self): - return self.dataset.access if self._phase == 'source_live' else self.workspace.access - - def _authorize(self, action): - if self._closed: - fail('Extraction controller is closed.', 'unavailable', 503) - self.access.require('admin', 'project_transfer.' + action, owner_only=True) - - def _bound(self, binding): - if type(binding) is not str or binding != self._binding: - fail('Extraction controller changed. Refresh its process binding.', 'stale', 409) - - def _root(self, alias): - if type(alias) is not str or alias not in self._roots: - fail('Choose a configured extraction publication root.') - path, descriptor = self._roots[alias] - _verify_origin(descriptor, path) - return descriptor - - def _target(self, root, directory, *, may_exist=False, selectable=False): - leaf(directory) - descriptor = self._root(root) - if selectable: - configured = self.workspace._root(root, directory) - if _identity(configured[1]) != _identity(descriptor): - fail('The destination root must also be configured in the control workspace.', 'conflict', 409) - try: - os.stat(directory, dir_fd=descriptor, follow_symlinks=False) - except FileNotFoundError: - pass - else: - if not may_exist: - fail('Extraction publication destination already exists.', 'conflict', 409) - return descriptor - - def _summary(self): - if self._report is None: - return None - return dict(report_sha256=self._report['report_sha256'], - selection_sha256=hashlib.sha256(canonical(self._selection)).hexdigest(), - source=copy.deepcopy(self._report['source']), can_extract=self._report['can_extract'], - counts={section: len(self._report[section]) for section in sorted(SECTIONS)}) - - def _state(self): - actions = ['status'] - if self._phase in ('source_live', 'source_stopped'): - actions += ['preview', 'report_page'] - if self._phase == 'source_live' and self._report is not None: - actions.append('stop') - if self._phase == 'source_stopped' and self._report is not None: - actions.append('extract') - return _bounded(dict(format='project_transfer_state_v1', binding=self._binding, - phase=self._phase, source_catalog_id=self.dataset.db.dataset_id, - control_catalog_id=self.workspace.db.dataset_id, roots=sorted(self._roots), - available_actions=actions, report=self._summary(), backup=copy.deepcopy(self._backup), - destination=copy.deepcopy(self._destination), failure=copy.deepcopy(self._failure), - last_receipt=copy.deepcopy(self._last_receipt))) - - def status(self): - with self._lock: - self._authorize('status') - return self._state() - - def preview(self, binding, selection): - with self._lock: - self._authorize('preview');self._bound(binding) - if self._phase not in ('source_live', 'source_stopped'): - fail('The original controller remains fenced after an unsuccessful stop.', 'source_fenced', 409) - normalized = normalize_selection(selection) - if len(canonical(normalized)) > MAX_RESPONSE: - fail('Extraction selection exceeds the request bound.', 'too_large', 413) - if self._phase == 'source_live': - if self._source is not None: - self._source.close() - self._source = self._transfer = self._report = self._selection = None - self._source = preview_source(self.dataset) - # Retain only the reviewed report, rather than accumulating a - # stopped transfer's full report cache across selection changes. - self._selection = self._report = None - self._transfer = ProjectTransfer(self._source) - report = self._transfer.preview(normalized) - self._selection, self._report = normalized, report - self._failure = None - return self._state() - - def report_page(self, binding, report_sha256, section, offset=0, limit=40): - with self._lock: - self._authorize('report_page');self._bound(binding);sha(report_sha256) - if self._report is None or report_sha256 != self._report['report_sha256']: - fail('The extraction report changed. Review the current report.', 'stale', 409) - if type(section) is not str or section not in SECTIONS or type(offset) is not int or not 0 <= offset <= 1000000 or type(limit) is not int or not 1 <= limit <= 40: - fail('Choose a known report section and bounded page.') - values = self._report[section] - return _bounded(dict(format='project_transfer_report_page_v1', binding=self._binding, - report_sha256=report_sha256, section=section, items=copy.deepcopy(values[offset:offset + limit]), - offset=offset, limit=limit, total=len(values))) - - def stop(self, binding, report_sha256, backup_root, backup_directory, destination_root, directory): - with self._lock: - self._authorize('stop');self._bound(binding);sha(report_sha256) - if self._phase != 'source_live' or self._report is None or self._report['report_sha256'] != report_sha256: - fail('Review this live source before its explicit stop.', 'stale', 409) - backup_fd = self._target(backup_root, backup_directory) - destination_fd = self._target(destination_root, directory, selectable=True) - if _identity(backup_fd) == _identity(destination_fd) and backup_directory == directory: - fail('Stopped backup and extracted project require different destinations.', 'conflict', 409) - if self._source is not None: - self._source.close();self._source = self._transfer = None - # After any stop attempt, source authentication may be closed and - # ordinary legacy routes remain fenced. No DTO can mint a lease. - self._phase = 'source_fenced';self._binding = uuid.uuid4().hex - self._report = None - try: - source = self.dataset.stop_for_project_extraction(backup_fd, backup_directory) - except BaseException as error: - # Native authorization can fail before it installs the source - # request fence. Do not claim a stopped/fenced source or switch - # authentication in that no-effect case. - if not self.dataset._extraction_stopping: - self._phase = 'source_live' - self._failure = dict(action='stop', code=getattr(error, 'code', 'unknown'), detail=str(error)[:1000]) - raise - self._source, self._transfer = source, ProjectTransfer(source) - self._phase = 'source_stopped' - self._backup = dict(root=backup_root, directory=backup_directory, - receipt_sha256=hashlib.sha256(canonical(source.backup_receipt)).hexdigest()) - self._destination = dict(root=destination_root, directory=directory) - self._failure = None - return self._state() - - def extract(self, binding, report_sha256, destination_root, directory, name, request_key): - with self._lock: - self._authorize('extract');self._bound(binding);sha(report_sha256) - if self._phase != 'source_stopped' or self._transfer is None: - fail('A trusted stopped source is required before extraction.', 'source_fenced', 409) - if self._report is None or report_sha256 != self._report['report_sha256']: - fail('Review the exact stopped-source report before extraction.', 'stale', 409) - if type(name) is not str or not name.strip() or len(name) > 200 or type(request_key) is not str or not REQUEST_KEY.fullmatch(request_key): - fail('Choose an extraction name and exact request key.') - destination_fd = self._target(destination_root, directory, may_exist=True, selectable=True) - if self._backup is not None and _identity(destination_fd) == _identity(self._root(self._backup['root'])) and directory == self._backup['directory']: - fail('Extraction cannot replace its stopped backup.', 'conflict', 409) - # Pin selection before authority, matching Workspace's lock order. - # An already selected destination owns its Store writer lock; only - # its actual pinned Store can reconcile that exact destination. - try: - with self.workspace._mutex, self.workspace.access.lock, ExitStack() as guards: - self._authorize('extract') - opened = None - if self.workspace._selection == (destination_root, directory) and self.workspace._store is not None: - projection = self.workspace._projection - if projection is None or self.workspace._recovery is not None: - fail('Recover the selected workspace before reconciling extraction.', 'projection_unavailable', 503) - guards.enter_context(projection.session.lock) - guards.enter_context(projection.db.lock) - projection.session.check_current() - opened = self.workspace._store - receipt = self._transfer.extract(self._selection, report_sha256, destination_fd, directory, name, request_key, opened_store=opened) - except BaseException as error: - self._failure = dict(action='extract', code=getattr(error, 'code', 'unknown'), detail=str(error)[:1000]) - if getattr(error, 'receipt', None) is not None: - self._last_receipt = _receipt_summary(error.receipt) - if getattr(error, 'staging_directory', None) is not None: - self._failure['staging_directory'] = error.staging_directory - raise - self._last_receipt = _receipt_summary(receipt) - self._destination = dict(root=destination_root, directory=directory) - self._failure = None - selected, state, selection_error = False, None, None - try: - state = self.workspace.dispatch('project_files.status', {}) - arguments = dict(binding=state['binding'], expected_checkpoint=state['project']['checkpoint'] if state['project'] else None, - root=destination_root, directory=directory, request_key='extract-open.' + hashlib.sha256(request_key.encode()).hexdigest()[:32]) - state = self.workspace.dispatch('project_files.open', arguments) - if state['project'] is None or state['project']['project_id'] != receipt['project_id']: - fail('Workspace selected a different project.', 'project_selection_unavailable', 503) - selected = True - except BaseException as error: - selection_error = dict(code=getattr(error, 'code', 'unknown'), detail=str(error)[:1000]) - return _bounded(dict(format='project_transfer_extraction_result_v1', binding=self._binding, - receipt=_receipt_summary(receipt), selection=dict(selected=selected, state=state, error=selection_error))) - - def dispatch(self, name, arguments): - """The root transport admits only these five named, closed methods.""" - fields = { - 'project_transfer.status': set(), - 'project_transfer.preview': {'binding', 'selection'}, - 'project_transfer.report_page': {'binding', 'report_sha256', 'section', 'offset', 'limit'}, - 'project_transfer.stop': {'binding', 'report_sha256', 'backup_root', 'backup_directory', 'destination_root', 'directory'}, - 'project_transfer.extract': {'binding', 'report_sha256', 'destination_root', 'directory', 'name', 'request_key'}} - if type(name) is not str or name not in OPERATIONS or type(arguments) is not dict or set(arguments) - fields[name]: - fail('Unsupported extraction controller action.', 'unsupported', 409) - required = fields[name] - ({'offset', 'limit'} if name == 'project_transfer.report_page' else set()) - if not required <= set(arguments): - fail('Missing extraction controller arguments.') - if name == 'project_transfer.status':return self.status() - if name == 'project_transfer.preview':return self.preview(arguments['binding'], arguments['selection']) - if name == 'project_transfer.report_page':return self.report_page(arguments['binding'], arguments['report_sha256'], arguments['section'], arguments.get('offset', 0), arguments.get('limit', 40)) - if name == 'project_transfer.stop':return self.stop(arguments['binding'], arguments['report_sha256'], arguments['backup_root'], arguments['backup_directory'], arguments['destination_root'], arguments['directory']) - return self.extract(arguments['binding'], arguments['report_sha256'], arguments['destination_root'], arguments['directory'], arguments['name'], arguments['request_key']) - - def close(self): - with self._lock: - if self._closed:return - self._closed = True - try: - if self._source is not None:self._source.close() - finally: - for _, descriptor in self._roots.values():os.close(descriptor) - self._roots.clear() diff --git a/project_validation_cache.py b/project_validation_cache.py deleted file mode 100644 index 4d06760e..00000000 --- a/project_validation_cache.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Remember complete domain validation of exact project checkpoints. - -A project checkpoint digest covers its manifest, which names every canonical -record revision by length and SHA256. Records are verified against those -digests before use, so (project ID, checkpoint) identifies the exact record set. -Complete cross-domain validation of that set is a pure function of those -records and the validating source code, so an earlier success can be reused -when both are identical. - -Markers live only in the private local workspace control directory; a portable -project folder never carries validation authority. The validator fingerprint -covers every Tuldok Python module and extension schema plus the optional -validator package versions, so any code or dependency change revalidates. -""" -import hashlib -import importlib.metadata -import json -import os -from pathlib import Path -import re -import sys -import threading -import uuid - -FORMAT = 'project_validation_marker_v1' -_HEX32 = re.compile(r'^[a-f0-9]{32}$') -_HEX64 = re.compile(r'^[a-f0-9]{64}$') -_SOURCE = Path(__file__).resolve().parent -_fingerprint = None -_fingerprint_lock = threading.Lock() - - -def _package_version(name): - try: - return importlib.metadata.version(name) - except importlib.metadata.PackageNotFoundError: - return None - - -def validator_fingerprint(): - """SHA256 over validating source bytes, schemas and optional validators.""" - global _fingerprint - with _fingerprint_lock: - if _fingerprint is None: - digest = hashlib.sha256(b'tuldok-validator-fingerprint-v1\0') - paths = sorted(path for path in _SOURCE.glob('*.py')) - schemas = _SOURCE / 'extension_schemas' - if schemas.is_dir(): - paths += sorted(path for path in schemas.rglob('*') if path.is_file()) - for path in paths: - digest.update(str(path.relative_to(_SOURCE)).encode('utf-8') + b'\0') - digest.update(hashlib.sha256(path.read_bytes()).digest()) - environment = dict(python=sys.version, jsonschema=_package_version('jsonschema'), - referencing=_package_version('referencing')) - digest.update(json.dumps(environment, sort_keys=True).encode('utf-8')) - _fingerprint = digest.hexdigest() - return _fingerprint - - -class ProjectValidationCache: - """Private per-workspace markers of fully validated project checkpoints.""" - def __init__(self, control_path, *, fingerprint=None): - self.path = Path(control_path) / '.project-validation' - self.fingerprint = fingerprint or validator_fingerprint() - - def _marker(self, project_id, checkpoint): - if (type(project_id) is not str or not _HEX32.fullmatch(project_id) - or type(checkpoint) is not str or not _HEX64.fullmatch(checkpoint)): - raise ValueError('Invalid project validation identity.') - name = hashlib.sha256(f'{project_id}:{checkpoint}:{self.fingerprint}'.encode()).hexdigest() - return self.path / f'{name}.json', dict(format=FORMAT, project_id=project_id, - checkpoint=checkpoint, fingerprint=self.fingerprint) - - def known(self, project_id, checkpoint): - path, expected = self._marker(project_id, checkpoint) - try: - with open(path, 'rb') as stream: - return json.loads(stream.read(4096)) == expected - except (OSError, ValueError): - return False - - def remember(self, project_id, checkpoint): - """Best effort: a missing marker only means the next Open revalidates.""" - path, marker = self._marker(project_id, checkpoint) - try: - self.path.mkdir(mode=0o700, exist_ok=True) - temporary = self.path / f'.{uuid.uuid4().hex}.tmp' - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) - try: - os.write(descriptor, json.dumps(marker, sort_keys=True).encode('utf-8')) - os.fsync(descriptor) - finally: - os.close(descriptor) - os.replace(temporary, path) - except OSError: - pass - - -class CanonicalDigestCache: - """Private set of record digests whose bytes were proven canonical JSON. - - Canonicality is a property of the bytes alone, and record files are named - by their SHA256, so a proof for one digest holds for every project that - contains those exact bytes. The set is versioned by the canonical codec's - own source, so any change to the encoding rules starts a fresh set. - """ - def __init__(self, control_path): - import inspect - import project_store - codec = ''.join(inspect.getsource(function) for function in ( - project_store._json_value, project_store._encode, project_store._pairs, project_store._decode)) - version = hashlib.sha256((codec + json.__version__).encode('utf-8')).hexdigest()[:32] - self.path = Path(control_path) / '.canonical-digests' / f'{version}.bin' - self._known = None - self._lock = threading.Lock() - - def _load(self): - if self._known is None: - known = set() - try: - with open(self.path, 'rb') as stream: - raw = stream.read() - if len(raw) % 32 == 0: - known = {raw[index:index + 32] for index in range(0, len(raw), 32)} - except OSError: - pass - self._known = known - return self._known - - def __contains__(self, digest): - with self._lock: - return bytes.fromhex(digest) in self._load() - - def add_many(self, digests): - """Best effort: a lost update only means those records are re-proven.""" - with self._lock: - known = self._load() - fresh = {bytes.fromhex(digest) for digest in digests} - known - if not fresh: - return - known |= fresh - try: - self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - temporary = self.path.parent / f'.{uuid.uuid4().hex}.tmp' - descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) - try: - remaining = memoryview(b''.join(sorted(known))) - while remaining: - remaining = remaining[os.write(descriptor, remaining):] - os.fsync(descriptor) - finally: - os.close(descriptor) - os.replace(temporary, self.path) - except OSError: - pass diff --git a/project_validation_delta.py b/project_validation_delta.py deleted file mode 100644 index d63814c5..00000000 --- a/project_validation_delta.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Edit-scoped validation: re-check only what a mutation can affect. - -A mutation's candidate is the previous, completely validated checkpoint plus -changed/added records (no deletions) and its new operation receipt. Each -validation component of validate_project_records is a pure function of the -documents and identities of a known set of record families (see the audit in -docs/plans/database-working-store.md, stage 4). A component whose input -families contain no changed record has the same outcome as for the previous -checkpoint, so it is skipped. Old operation receipts are re-checked only when -records they bind to changed or a new receipt shares their key space. - -Anything not classified here is always validated. TULDOK_VALIDATION_CROSSCHECK=1 -additionally runs complete validation and requires the same outcome. -""" -import contextvars -from contextlib import contextmanager -from dataclasses import dataclass - -from project_data import FAMILIES as _DATA -from project_history import FAMILY_FORMATS as _HISTORY -from project_material import FAMILIES as _MATERIAL -from project_producers import FAMILIES as _PRODUCERS - -DATA = frozenset(_DATA) -HISTORY = frozenset(_HISTORY) -MATERIAL = frozenset(_MATERIAL) -PRODUCERS = frozenset(_PRODUCERS) - -# Families whose documents or identities each component reads. -COMPONENTS = { - 'data': DATA | {'workbench_text', 'project_artifact', 'project_fork'} | PRODUCERS, - 'fork': frozenset({'project_fork'}), - 'composition': frozenset({'project_composition', 'project_fork', 'run_observation'}), - 'extraction': frozenset({'project_extraction', 'project_fork', 'project_snapshot', 'project_file', 'project_artifact'}), - 'files': frozenset({'project_file', 'project_fork'}), - 'native': frozenset({'native_execution_history', 'project_artifact', 'run_history'}), - 'material': MATERIAL | {'project_fork', 'project_artifact'}, - 'producers': PRODUCERS | {'project_artifact', 'workbench_record', 'workbench_text'}, - 'history': HISTORY | MATERIAL | {'project_artifact', 'dataset_exposure', 'pipeline_graph'}, - 'measurements': frozenset({'measurement_source', 'project_artifact', 'project_snapshot'}), - 'sources': frozenset({'project_snapshot'}), -} - -# Operation-receipt prefixes -> families their result binds to. A prefix -# mapped to an empty set binds only immutable or append-only facts. -RECEIPT_BINDINGS = { - 'data.': DATA | {'workbench_text'}, - 'models.': frozenset(), 'graphs.': frozenset(), 'recipes.': frozenset(), - 'studies.': HISTORY | {'pipeline_graph', 'project_artifact'}, - 'history.disclose': HISTORY | {'pipeline_graph', 'project_artifact'}, - 'source.': frozenset(), - 'measurements.': frozenset({'measurement_source'}), - 'runs.': frozenset({'run_history'}), - 'producers.': frozenset(), - 'files.': frozenset(), - 'study_execution.': HISTORY | MATERIAL, - 'runtime.': frozenset({'runtime_material', 'project_artifact'}), - 'native_preparation.': MATERIAL | {'pipeline_graph', 'dataset_release'}, - 'artifacts.resolve': frozenset(), -} -# A new receipt in these key spaces can change the meaning of older ones. -SHARED_KEY_SPACES = ('study_execution.', 'runtime.') -# Receipts these prefixes claim are also validated inside history decoding. -HISTORY_RECEIPTS = ('studies.', 'history.disclose') - - -def _prefix(operation): - return next((prefix for prefix in RECEIPT_BINDINGS if operation.startswith(prefix)), None) - - -@dataclass(frozen=True) -class ValidationDelta: - """Identities a mutation changed relative to its validated base checkpoint.""" - changed: frozenset - operations: frozenset = frozenset() # operation names of new/changed receipts - added: frozenset = frozenset() # changed identities absent from the base - - @property - def families(self): - # A newly added artifact declaration cannot change any component: an - # unchanged record referencing an undeclared blob already failed base - # validation. New declarations are still checked by the claim loop. - return frozenset(family for family, identity in self.changed - if not (family == 'project_artifact' and (family, identity) in self.added)) - - def touches(self, component): - return bool(self.families & COMPONENTS[component]) - - def history_receipt_added(self): - return any(operation.startswith(HISTORY_RECEIPTS) for operation in self.operations) - - def receipt_needed(self, operation, identity): - if identity in self.changed: - return True - prefix = _prefix(operation) - if prefix is None: - return True - if self.families & RECEIPT_BINDINGS[prefix]: - return True - if prefix in SHARED_KEY_SPACES and any(op.startswith(prefix) for op in self.operations): - return True - return False - - -_DELTA = contextvars.ContextVar('validation_delta', default=None) - - -@contextmanager -def edit_delta(delta): - """Validate the enclosed candidate relative to its validated base checkpoint.""" - token = _DELTA.set(delta) - try: - yield - finally: - _DELTA.reset(token) - - -def current_delta(): - return _DELTA.get() diff --git a/project_working_store.py b/project_working_store.py deleted file mode 100644 index 03d35f5e..00000000 --- a/project_working_store.py +++ /dev/null @@ -1,634 +0,0 @@ -"""Persistent PostgreSQL working copy of one external project folder. - -The database holds the current record set of an opened project and is kept -across sessions. The external folder (`.tuldok/project.json` plus -content-addressed `records/`, tracked in Git) is an import source, read only -when its checkpoint changed since Tuldok last imported or saved it, and an -export target written only by save(). See docs/plans/database-working-store.md. - -Records are kept exactly as their canonical bytes with their SHA256, so the -working copy has the same manifest, checkpoint digest and receipts as the file -format and save() writes byte-identical files. This store keeps no record state -in memory: inventory, receipts and bodies are read from PostgreSQL, so an outer -transaction rollback can never leave a stale in-process copy. - -Interface parity with ProjectStore (summary, snapshot, commit, save, -reconcile, root_directory, verify_records, recover_checkpoint, close, path) -lets Session, owners, artifacts and custody use either store. -""" -from contextlib import contextmanager, nullcontext -import json -import os -from pathlib import Path -import re -import threading -import uuid - -import psycopg -from psycopg import sql - -from metadata import _key, connection_info -from project_record_dedupe import CLOSURE_FAMILY, RUN_FAMILY, compact, expand, is_compact -from project_store import (FORMAT, MAX_RECORD_BYTES, MAX_RECORDS, MAX_TOTAL_BYTES, SUPPORTED_FORMATS, - ProjectStoreError, _RecordsDigest, _decode, _digest, _domain_format, _encode, - _fail, _identifier, _keys, _manifest_raw, _request, _sha, envelope_digest) - -WORKING_ID = re.compile(r'^[a-f0-9]{32}$') -_TABLES = ( - '''CREATE TABLE IF NOT EXISTS {schema}.state ( - singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), - project_id text NOT NULL, name text NOT NULL, checkpoint text NOT NULL, - receipts bytea NOT NULL, external_sha256 text, dirty boolean NOT NULL)''', - '''CREATE TABLE IF NOT EXISTS {schema}.records ( - family text COLLATE "C" NOT NULL, id text COLLATE "C" NOT NULL, - format text NOT NULL, bytes integer NOT NULL, sha256 text NOT NULL, - raw bytea NOT NULL, PRIMARY KEY (family, id))''', -) -_MIGRATIONS = ( - # SHA256 of the canonical {family, id, document} envelope (see envelope_digest). - 'ALTER TABLE {schema}.records ADD COLUMN IF NOT EXISTS envelope_sha256 text', - # Shared runtime closure referenced by a compact Run row (project_record_dedupe). - 'ALTER TABLE {schema}.records ADD COLUMN IF NOT EXISTS closure_id text', -) -_ORDER = sql.SQL('ORDER BY family COLLATE "C", id COLLATE "C"') - - -class WorkingProjectStore: - """One exclusive owner of a project's database working copy.""" - - def __init__(self, folder, working_id, *, dsn=None, canonical_digests=None): - if type(working_id) is not str or not WORKING_ID.fullmatch(working_id): - raise ValueError('Invalid working copy identity.') - self._folder = folder - self.working_id = working_id - self.schema = 'working_' + working_id - self._canonical_digests = canonical_digests - self._mutex = threading.RLock() - self._db = None - self._closed = False - try: - self._own = psycopg.connect(connection_info(dsn), autocommit=True) - except psycopg.Error: - raise ProjectStoreError('Local PostgreSQL is unavailable.', 'unavailable', 503) from None - try: - # One Tuldok process owns a working copy at a time. - if not self._own.execute('SELECT pg_try_advisory_lock(%s)', (_key(self.schema),)).fetchone()[0]: - raise ProjectStoreError('This project is already open in another Tuldok session.', 'busy', 409) - with self._own.transaction(): - self._own.execute(sql.SQL('CREATE SCHEMA IF NOT EXISTS {}').format(sql.Identifier(self.schema))) - for statement in (*_TABLES, *_MIGRATIONS): - self._own.execute(sql.SQL(statement).format(schema=sql.Identifier(self.schema))) - except BaseException: - self._own.close() - raise - - # ---- Connection: own until bound to the projection, then shared so - # record commits are atomic with the projection's SQL writes. - - def bind(self, db): - """Use the projection database's connection and transactions.""" - with self._locked(): - self._db = db - - @contextmanager - def _locked(self): - """Store ownership in one global order: projection lock, then this mutex. - - A bound projection connection is shared with Session/owner threads that - hold the projection lock across their transactions, so store work never - takes its own mutex first and then waits for that lock. - """ - if self._db is not None: - with self._catalog_lock(), self._mutex: - yield - else: - with self._mutex: - yield - - def _catalog_lock(self): - # The projection may install a composite host -> session -> catalog - # lock; the store is innermost, so it takes only the catalog lock. - lock = self._db.lock - return getattr(lock, 'projection', lock) - - def _joined(self): - # Join the projection only inside its open transaction (Session.mutate, - # background publication), so record and projection rows commit - # atomically. Other commits use this store's own connection and never - # fire the projection's commit hooks. - return self._db is not None and self._db.in_transaction - - def _connection(self): - return self._db.connection if self._joined() else self._own - - def _guard(self): - # A bound projection connection is shared; its catalog lock serializes - # every command and transaction on it across threads. - return self._catalog_lock() if self._joined() else nullcontext() - - @contextmanager - def _transaction(self): - if self._joined(): - with self._catalog_lock(), self._db.transaction(): - yield - else: - with self._own.transaction(): - yield - - def _execute(self, statement, parameters=None): - if self._closed: - _fail('Project store is closed.', 'unavailable', 409) - with self._guard(): - return self._connection().execute(statement, parameters) - - def _q(self, text): - return sql.SQL(text).format(state=sql.Identifier(self.schema, 'state'), - records=sql.Identifier(self.schema, 'records'), order=_ORDER) - - # ---- Open: attach, import or refuse. - - @classmethod - def open(cls, folder, working_id, *, dsn=None, canonical_digests=None): - """Attach the working copy, importing the folder only if it changed.""" - store = cls(folder, working_id, dsn=dsn, canonical_digests=canonical_digests) - try: - raw, manifest = folder.folder_checkpoint() - external = _sha(raw) - state = store._state(required=False) - if state is not None and state['project_id'] != manifest['project_id']: - _fail('This working copy belongs to a different project.', 'conflict', 409) - if state is None or state['external_sha256'] != external: - if state is not None and state['dirty']: - _fail('The external project changed while this workspace has unsaved changes. ' - 'Save over it or discard the working copy and import again.', 'conflict', 409) - store._import(raw, manifest) - return store - except BaseException: - store.close() - raise - - def _state(self, *, required=True): - row = self._execute(self._q('SELECT project_id, name, checkpoint, receipts, external_sha256, dirty FROM {state}')).fetchone() - if row is None: - if required: - _fail('Project working copy is missing.', 'unavailable', 409) - return None - return dict(project_id=row[0], name=row[1], checkpoint=row[2], receipts=json.loads(bytes(row[3])), - external_sha256=row[4], dirty=row[5]) - - def _import(self, raw, manifest): - """Bring the working copy to the folder's checkpoint, reading only new revisions.""" - with self._locked(), self._transaction(): - existing = {(row[0], row[1]): row[2] for row in - self._execute(self._q('SELECT family, id, sha256 FROM {records}'))} - wanted = {} - proven = [] - known = self._canonical_digests - for entry in manifest['inventory']: - identity = (entry['family'], entry['id']) - wanted[identity] = entry - if existing.get(identity) == entry['sha256']: - continue - body = self._folder.folder_record(entry) - if known is not None and entry['sha256'] in known: - document = json.loads(body) - else: - document = _decode(body) - proven.append(entry['sha256']) - if type(document) is not dict or document.get('format') != entry['format']: - _fail('Domain document format disagrees with its inventory.') - compact_run = is_compact(entry['family'], document) - self._execute(self._q( - 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256, closure_id) ' - 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ' - 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' - 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256, ' - 'closure_id=EXCLUDED.closure_id'), - (entry['family'], entry['id'], entry['format'], entry['bytes'], entry['sha256'], body, - # A compact Run's logical envelope digest is filled from its expansion. - None if compact_run or entry['family'] == CLOSURE_FAMILY else envelope_digest(entry['family'], entry['id'], body), - document['runtime_closure']['id'] if compact_run else None)) - for family, record_id in set(existing) - set(wanted): - self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, record_id)) - if self._records_digest() != manifest['receipts'][-1]['records_sha256']: - _fail('Latest operation receipt disagrees with the current record set.') - self._write_state(manifest, external_sha256=_sha(raw), dirty=False) - if known is not None and proven: - known.add_many(proven) - - def _write_state(self, manifest, *, external_sha256, dirty): - receipts = _encode(manifest['receipts']) - self._execute(self._q('DELETE FROM {state}')) - self._execute(self._q('INSERT INTO {state} (project_id, name, checkpoint, receipts, external_sha256, dirty) ' - 'VALUES (%s,%s,%s,%s,%s,%s)'), - (manifest['project_id'], manifest['name'], manifest['checkpoint'], receipts, - external_sha256, dirty)) - - def _rows(self, statement, parameters=None): - """Stream (family, id, raw) rows through a server-side cursor. - - Bodies are fetched in small batches, never the whole record set at once. - """ - if self._closed: - _fail('Project store is closed.', 'unavailable', 409) - with self._guard(): - connection = self._connection() - idle = connection.info.transaction_status == psycopg.pq.TransactionStatus.IDLE - with (connection.transaction() if idle else nullcontext()), \ - connection.cursor(name='working_' + uuid.uuid4().hex) as cursor: - cursor.itersize = 64 - cursor.execute(statement, parameters) - for row in cursor: - yield row[0], row[1], bytes(row[2]) - - def _records_digest(self, replacements=None, identities=None): - """Streamed receipt digest over the stored (or a candidate) record set.""" - digest = _RecordsDigest() - replacements = replacements or {} - if identities is None: - for family, record_id, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}')): - digest.add(family, record_id, raw) - return digest.hexdigest() - stored = self._rows(self._q('SELECT family, id, raw FROM {records} {order}')) - pending = next(stored, None) - for identity in identities: - while pending is not None and (pending[0], pending[1]) < identity: - pending = next(stored, None) - if identity in replacements: - digest.add(identity[0], identity[1], replacements[identity]) - elif pending is not None and (pending[0], pending[1]) == identity: - digest.add(identity[0], identity[1], pending[2]) - else: - _fail('Unchanged candidate record is missing from the working copy.') - return digest.hexdigest() - - # ---- Logical documents: closures hidden, compact Runs expanded. - - def _closure(self, identity): - cache = self.__dict__.setdefault('_closures', {}) - if identity not in cache: - row = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), - (CLOSURE_FAMILY, identity)).fetchone() - if row is None: - _fail('Compact Run references a missing runtime closure.') - cache[identity] = json.loads(bytes(row[0])) - return cache[identity] - - def _logical(self, family, raw): - document = json.loads(raw) - if is_compact(family, document): - document = expand(document, self._closure(document['runtime_closure']['id'])) - return document - - # ---- Manifest identity, identical to the file format. - - def _inventory(self): - return [(row[0], row[1], row[2], row[3], row[4]) for row in - self._execute(self._q('SELECT family, id, format, bytes, sha256 FROM {records} {order}'))] - - @staticmethod - def _manifest(state, inventory): - manifest = dict(format=FORMAT, project_id=state['project_id'], name=state['name'], - roots=[dict(family=f, id=i) for f, i, _fmt, _b, _s in inventory], - inventory=[dict(family=f, id=i, format=fmt, namespace=state['project_id'], - path=f'records/{f}/{i}/{sha}.json', bytes=size, sha256=sha) - for f, i, fmt, size, sha in inventory], - receipts=state['receipts']) - manifest['checkpoint'] = _sha(_encode(manifest)) - return manifest - - # ---- ProjectStore interface. - - @property - def path(self): - return self._folder.path - - @property - def _root_fd(self): - """The external folder's locked physical root (read-only).""" - return self._folder._root_fd - - def summary(self): - with self._locked(): - state = self._state() - return {key: state[key] for key in ('project_id', 'name', 'checkpoint')} - - def manifest_raw(self): - """Canonical project.json bytes of the working copy's current checkpoint.""" - with self._locked(): - return _manifest_raw(self._manifest(self._state(), self._inventory())) - - def dirty(self): - """Whether the working copy has changes not yet saved to the folder.""" - with self._locked(): - return self._state()['dirty'] - - def snapshot(self): - """Complete independent record set (costly for large projects).""" - with self._locked(): - state = self._state() - records = [dict(family=f, id=i, document=self._logical(f, raw)) for f, i, raw in self._rows( - self._q('SELECT family, id, raw FROM {records} WHERE family <> %s {order}'), (CLOSURE_FAMILY,))] - return dict(project_id=state['project_id'], name=state['name'], - checkpoint=state['checkpoint'], records=records) - - def records(self, *families): - """Independent decoded envelopes of the selected families, in record order.""" - if not families or any(type(family) is not str for family in families): - raise TypeError('Select one or more record families by name.') - with self._locked(): - selected = [family for family in families if family != CLOSURE_FAMILY] - return [dict(family=f, id=i, document=self._logical(f, raw)) for f, i, raw in self._rows( - self._q('SELECT family, id, raw FROM {records} WHERE family = ANY(%s) {order}'), (selected,))] - - def envelope_digests(self): - """(family, id) -> canonical envelope SHA256 for every stored revision. - - Rows imported before this column existed are filled once from their - stored canonical bytes. - """ - with self._locked(): - missing = self._execute(self._q('SELECT count(*) FROM {records} WHERE envelope_sha256 IS NULL AND family <> %s'), - (CLOSURE_FAMILY,)).fetchone()[0] - if missing: - with self._transaction(): - # Logical envelopes: a compact Run hashes its exact expansion. - digests = [(envelope_digest(family, record_id, _encode(self._logical(family, raw)) - if family == RUN_FAMILY else raw), family, record_id) - for family, record_id, raw in self._rows(self._q( - 'SELECT family, id, raw FROM {records} WHERE envelope_sha256 IS NULL AND family <> %s {order}'), - (CLOSURE_FAMILY,))] - with self._guard(): - with self._connection().cursor() as cursor: - cursor.executemany(self._q('UPDATE {records} SET envelope_sha256=%s WHERE family=%s AND id=%s'), - digests) - return {(row[0], row[1]): row[2] for row in - self._execute(self._q('SELECT family, id, envelope_sha256 FROM {records} WHERE family <> %s'), - (CLOSURE_FAMILY,))} - - def records_view(self, *families): - """Every record identity in order; documents decoded only for selected families. - - Unselected envelopes carry document None. Fork and membership - validators need every identity but only read their own families. - """ - if not families or any(type(family) is not str for family in families): - raise TypeError('Select one or more record families by name.') - with self._locked(): - connection = self._connection() - idle = connection.info.transaction_status == psycopg.pq.TransactionStatus.IDLE - result = [] - with (connection.transaction() if idle else nullcontext()), \ - connection.cursor(name='working_' + uuid.uuid4().hex) as cursor: - cursor.itersize = 256 - cursor.execute(self._q('SELECT family, id, CASE WHEN family = ANY(%s) THEN raw END ' - 'FROM {records} WHERE family <> %s {order}'), (list(families), CLOSURE_FAMILY)) - rows = [(family, record_id, None if raw is None else bytes(raw)) for family, record_id, raw in cursor] - for family, record_id, raw in rows: - result.append(dict(family=family, id=record_id, - document=None if raw is None else self._logical(family, raw))) - return result - - def record(self, family, record_id): - if family == CLOSURE_FAMILY: - return None - with self._locked(): - row = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), - (family, record_id)).fetchone() - return None if row is None else dict(family=family, id=record_id, document=self._logical(family, bytes(row[0]))) - - def reconcile(self, request_key, request_sha256): - _request(request_key, request_sha256) - with self._locked(): - for receipt in self._state()['receipts']: - if receipt['request_key'] == request_key: - if receipt['request_sha256'] != request_sha256: - _fail('Request key was already used with different input.', 'conflict', 409) - return self.snapshot() - return None - - def commit(self, records, expected_checkpoint, request_key, request_sha256, *, changed=None): - """Replace the complete current record set in the working copy. - - changed, when given, names every (family, id) whose document may differ - from the stored revision; all other candidate records are the caller's - unmodified copies of stored revisions (Session.mutate constructs them - so). Without it every candidate is compared exactly with its revision. - Returns the committed snapshot, sharing the caller's record list. - """ - _request(request_key, request_sha256) - _digest(expected_checkpoint, 'Expected checkpoint') - if type(records) is not list or len(records) > MAX_RECORDS: - _fail('Project records must be a bounded list.') - with self._locked(), self._transaction(): - state = self._state() - inventory = {(f, i): (fmt, size, sha) for f, i, fmt, size, sha in self._inventory()} - candidates = {} - for record in records: - _keys(record, ('family', 'id', 'document'), 'Project record') - family, identity, document = record['family'], record['id'], record['document'] - _identifier(family, 'Record family') - if family not in SUPPORTED_FORMATS: - _fail('Unsupported project record family.', 'unsupported', 422) - if family == CLOSURE_FAMILY: - _fail('Runtime closures are store-owned records.', 'unsupported', 422) - _identifier(identity, 'Record ID') - if (family, identity) in candidates: - _fail('Duplicate project record identity.') - if type(document) is not dict: - _fail('Domain document must be an object.') - _domain_format(document.get('format'), family) - candidates[(family, identity)] = document - replacements, entries, total = {}, {}, 0 - closure_rows = {row[1]: row[2] for row in self._execute(self._q( - 'SELECT family, id, closure_id FROM {records} WHERE closure_id IS NOT NULL'))} - new_closures, row_closures, logical_raw, referenced = {}, {}, {}, set() - for identity in sorted(candidates): - document = candidates[identity] - current = inventory.get(identity) - if current is not None and changed is not None and identity not in changed \ - and current[0] in (document['format'], 'project_run_history_v2'): - entries[identity] = current - total += current[1] - if identity[1] in closure_rows and identity[0] == RUN_FAMILY: - referenced.add(closure_rows[identity[1]]) - continue - physical = document - if (identity[0] == RUN_FAMILY and changed is None and current is not None - and current[0] == document['format'] and _sha(_encode(document)) == current[2]): - # An unchanged uncompacted Run keeps its verified revision (as in - # ProjectStore); compact_run_closures migrates explicitly. - entries[identity] = current - total += current[1] - continue - if identity[0] == RUN_FAMILY: - compacted = compact(document) - if compacted is not None: - physical, closure = compacted - closure_id = physical['runtime_closure']['id'] - new_closures.setdefault(closure_id, closure) - referenced.add(closure_id) - row_closures[identity] = closure_id - logical_raw[identity] = _encode(document) - raw = _encode(physical) - if len(raw) > MAX_RECORD_BYTES: - _fail('Project records exceed the synchronous byte limit.') - digest = _sha(raw) - if current is None or current[2] != digest: - replacements[identity] = raw - entries[identity] = (physical['format'], len(raw), digest) - total += len(raw) - if total > MAX_TOTAL_BYTES: - _fail('Project records exceed the synchronous byte limit.') - # Exactly the closures referenced by stored compact Runs are kept. - for closure_id in sorted(referenced): - identity = (CLOSURE_FAMILY, closure_id) - if identity in inventory: - entries[identity] = inventory[identity] - total += inventory[identity][1] - continue - raw = _encode(new_closures[closure_id]) - replacements[identity] = raw - entries[identity] = (new_closures[closure_id]['format'], len(raw), _sha(raw)) - total += len(raw) - if total > MAX_TOTAL_BYTES: - _fail('Project records exceed the synchronous byte limit.') - records_digest = self._records_digest(replacements, sorted(entries)) - for receipt in state['receipts']: - if receipt['request_key'] == request_key: - if receipt['request_sha256'] != request_sha256 or receipt['records_sha256'] != records_digest: - _fail('Request key was already used with different input.', 'conflict', 409) - return self.snapshot() - if expected_checkpoint != state['checkpoint']: - _fail('Expected project checkpoint is stale.', 'stale', 409) - receipts = state['receipts'] + [dict(request_key=request_key, request_sha256=request_sha256, - records_sha256=records_digest)] - manifest = self._manifest(dict(state, receipts=receipts), - [(f, i, *entries[(f, i)]) for f, i in sorted(entries)]) - _manifest_raw(manifest) - for (family, identity), raw in replacements.items(): - fmt, size, digest = entries[(family, identity)] - key = (family, identity) - envelope = (None if family == CLOSURE_FAMILY else - envelope_digest(family, identity, logical_raw.get(key, raw))) - self._execute(self._q( - 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256, closure_id) ' - 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ' - 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' - 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256, ' - 'closure_id=EXCLUDED.closure_id'), - (family, identity, fmt, size, digest, raw, envelope, row_closures.get(key))) - for family, identity in set(inventory) - set(entries): - self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, identity)) - self._write_state(manifest, external_sha256=state['external_sha256'], dirty=True) - if self._canonical_digests is not None and replacements: - self._canonical_digests.add_many([_sha(raw) for raw in replacements.values()]) - return dict(project_id=state['project_id'], name=state['name'], checkpoint=manifest['checkpoint'], - records=[dict(family=f, id=i, document=candidates[(f, i)]) for f, i in sorted(candidates)]) - - def compact_run_closures(self, request_key, request_sha256): - """Format migration: store every compactable Run against a shared closure. - - The logical record set is unchanged (each compaction is checked to expand - to the exact original bytes); only the physical records and checkpoint - change. Returns (previous checkpoint, committed snapshot summary). - """ - with self._locked(): - state = self._state() - uncompacted = {(row[0], row[1]) for row in self._execute(self._q( - "SELECT family, id FROM {records} WHERE family=%s AND format='project_run_history_v1'"), (RUN_FAMILY,))} - snapshot = self.snapshot() - committed = self.commit(snapshot['records'], state['checkpoint'], request_key, request_sha256, - changed=uncompacted) - return state['checkpoint'], {key: committed[key] for key in ('project_id', 'name', 'checkpoint')} - - def save(self, expected_checkpoint): - """Export the working copy to the external folder (the only folder write).""" - _digest(expected_checkpoint, 'Expected checkpoint') - with self._locked(): - state = self._state() - if expected_checkpoint != state['checkpoint']: - _fail('Expected project checkpoint is stale.', 'stale', 409) - try: - external_raw, _external = self._folder.folder_checkpoint() - except ProjectStoreError: - external_raw = None - if (state['external_sha256'] is not None and - (external_raw is None or _sha(external_raw) != state['external_sha256'])): - _fail('The external project changed since Tuldok last imported or saved it.', 'conflict', 409) - manifest = self._manifest(state, self._inventory()) - if manifest['checkpoint'] != state['checkpoint']: - _fail('Working copy checkpoint disagrees with its records.', 'invalid', 409) - raw = _manifest_raw(manifest) - if state['external_sha256'] != _sha(raw): - for entry in manifest['inventory']: - if not self._folder.folder_has_record(entry): - body = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), - (entry['family'], entry['id'])).fetchone() - body = bytes(body[0]) - if len(body) != entry['bytes'] or _sha(body) != entry['sha256']: - _fail('Working copy record bytes are corrupt.', 'invalid', 409) - self._folder.folder_write_record(entry['family'], entry['id'], entry['sha256'], body) - self._folder.folder_publish(raw, external_raw) - with self._transaction(): - self._write_state(manifest, external_sha256=_sha(raw), dirty=False) - return {key: state[key] for key in ('project_id', 'name', 'checkpoint')} - - def verify_records(self): - """Re-check every stored revision's bytes against its SHA256 and length.""" - with self._locked(): - inventory = {(f, i): (size, sha) for f, i, _fmt, size, sha in self._inventory()} - for family, record_id, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}')): - size, digest = inventory[(family, record_id)] - if len(raw) != size or _sha(raw) != digest: - _fail('Canonical record bytes changed or are corrupt.', 'stale', 409) - - def recover_checkpoint(self): - """Database commits are atomic; return the committed record set.""" - return self.snapshot() - - @contextmanager - def root_directory(self, *, verify_after=False): - """Borrow the external folder's physical root (payload artifacts live there).""" - if type(verify_after) is not bool: - _fail('Root read verification must be an explicit boolean.') - with self._locked(): - before = self._state()['checkpoint'] - descriptor = os.dup(self._folder._root_fd) - try: - yield descriptor - self._folder._binding() - if verify_after and self._state()['checkpoint'] != before: - _fail('Project checkpoint changed during the bounded root read.', 'stale', 409) - finally: - os.close(descriptor) - - def close(self): - # The projection (and its lock's connection) may already be closed. - with self._mutex: - if self._closed: - return - self._closed = True - try: - self._folder.close() - finally: - self._own.close() - - def __enter__(self): - return self - - def __exit__(self, *exc): - self.close() - - -def registered_schemas(control_path): - """Working-copy schemas registered by a workspace control directory.""" - schemas = [] - for path in sorted(Path(control_path).glob('.project-working/*.json')): - try: - identity = json.loads(path.read_bytes()).get('working_id') - except (OSError, ValueError, AttributeError): - continue - if type(identity) is str and WORKING_ID.fullmatch(identity): - schemas.append('working_' + identity) - return schemas diff --git a/project_workspace.py b/project_workspace.py deleted file mode 100644 index b6f61256..00000000 --- a/project_workspace.py +++ /dev/null @@ -1,763 +0,0 @@ -"""One inert external-project workspace and its installation-local authority. - -Only explicit project-files operations enter this owner. Legacy domain routes -are unavailable in this mode until their canonical publication is implemented. -The installation catalog holds local credentials; it is never portable content. -""" -import copy -from contextlib import ExitStack -import hashlib -import json -import os -from pathlib import Path -import re -import threading -import uuid - -from access import Access -import agent_api -from metadata import Metadata -from project_store import ProjectStore, ProjectStoreError -from project_projection import ProjectProjection -from project_custody import ProjectCustody -from project_validation_cache import CanonicalDigestCache, ProjectValidationCache -from project_working_store import WORKING_ID, WorkingProjectStore -from workbench import WorkbenchError - - -PUBLIC_RECORD_FIELDS = ('id', 'kind', 'name', 'text', 'groups', 'parents', 'provenance', - 'task', 'annotation', 'review', 'revision', 'source_revision', - 'created_at', 'updated_at') -SUMMARY_FIELDS = ('id', 'kind', 'name', 'task', 'review', 'revision', 'source_revision', - 'created_at', 'updated_at') -PROJECT_ACTIONS = ('new', 'open', 'save', 'close', 'query', 'get', 'import_text', 'annotate') -# Set only after the complete finite dispatcher is integrated. A capability is -# a promise that the route reaches canonical publication, not an API wishlist. -M2_ENABLED = True -M3_ENABLED = True -M4_ENABLED = True -_RECOVERY_OPERATIONS = frozenset(('project_files.execution_reconcile_current', - 'project_files.native_reconcile', - 'project_files.runtime_capture', 'project_files.runtime_abandon', - 'project_files.execution_study_abandon_preparation', 'project_files.execution_study_reconcile_preparation')) -_DRAINING_OPERATIONS = _RECOVERY_OPERATIONS | frozenset(('project_files.execution_producer_cancel', - 'project_files.execution_study_stop')) -DIR_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC - - -def public_record(value): - """Project owned values into the closed public contract; do not infer policy.""" - return {key: copy.deepcopy(value[key]) for key in PUBLIC_RECORD_FIELDS} - - -def _digest(value): - return hashlib.sha256(agent_api.encoded(value)).hexdigest() - - -def http_arguments(method, target, body=None): - """Use the fixed operation contracts for identical HTTP and SDK routing.""" - names = ['project_files.status', *('project_files.' + action for action in PROJECT_ACTIONS)] - names.extend(sorted(agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | agent_api.PROJECT_FILES_M4)) - operations = {name: agent_api.operation(name) for name in names} - # A literal import route wins over a record-id placeholder. Contract - # validation still rejects an unexpected path/body/query representation. - ordered = sorted(operations.items(), key=lambda item: (item[1]['path'].count('{'), item[0])) - for name, operation in ordered: - if agent_api._route_matches(operation, method, target): - selected = (name, 1, agent_api.contract(name, 1)['sha256']) - return name, agent_api.validate_selected(selected, method, target, body) - raise WorkbenchError('Unknown project-file operation.', 'unsupported', 404) - - -class ProjectWorkspace: - project_files_mode = True - - def __init__(self, control_path, roots, *, dsn=None, runtime_sources=None): - from runtime_snapshots import configuration - configured = {} - for alias, config in (runtime_sources or {}).items(): - if (type(alias) is not str or not re.fullmatch('[A-Za-z0-9][A-Za-z0-9_.-]{0,63}', alias) - or len(configured) >= 100): - raise WorkbenchError('Configure at most 100 named current runtime sources.') - configured[alias] = copy.deepcopy(configuration(config)) - self.path = Path(control_path).resolve() - self.db = Metadata(self.path, dsn=dsn) - self.lock = self.db.lock - self.access = Access(self.db, self.path) - self.custody = ProjectCustody(self.path, self.db.dataset_id) - self.validation_cache = ProjectValidationCache(self.path) - self.canonical_digests = CanonicalDigestCache(self.path) - self._retained_recovery = None - self._dsn = dsn - self._runtime_sources = configured - self._mutex = threading.RLock() - self._roots = {} - self._store = self._text = self._projection = None - self._selection = None - self._last_project = None - self._recovery = None - self._binding = uuid.uuid4().hex - self._closed = False - self._receipts = {} - try: - if not roots or len(roots) > 100: - raise WorkbenchError('Configure between one and 100 local project-file roots.') - for alias, path in roots.items(): - if type(alias) is not str or not re.fullmatch('[A-Za-z0-9][A-Za-z0-9_.-]{0,63}', alias): - raise WorkbenchError('Invalid local project-file root alias.') - location = Path(path).expanduser().resolve(strict=True) - descriptor = os.open(location, DIR_FLAGS) - self._roots[alias] = (location, descriptor) - except BaseException: - self.close() - raise - - def _ensure_open(self): - if self._closed: - raise WorkbenchError('Project workspace is closed.', 'unavailable', 503) - - def _recovery_required(self, reason, request_key=None): - if self._recovery is None and self._selection is not None and self._last_project is not None: - self._recovery = dict(action='open', root=self._selection[0], directory=self._selection[1], - reason=reason, request_key=request_key) - - def _state(self): - self._ensure_open() - project = None - if self._recovery is None and self._store is not None: - try: - snapshot = self._store.summary() - except ProjectStoreError as error: - if error.code not in ('unknown', 'outcome_unknown', 'stale', 'stale_project', - 'invalid', 'unsupported', 'unavailable'): - raise - self._recovery_required('unknown_publication' if error.code in ('unknown', 'outcome_unknown') - else 'projection_unavailable') - else: - if self._text is not None and not self._text.available: - self._recovery_required('projection_unavailable') - else: - project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} - self._last_project = copy.deepcopy(project) - if self._recovery is not None: - return dict(format='project_files_state_v1', binding=self._binding, state='recovery_required', - roots=sorted(self._roots), project=copy.deepcopy(self._last_project), - available_actions=['open'], recovery=copy.deepcopy(self._recovery)) - return dict(format='project_files_state_v1', binding=self._binding, - state='open' if project is not None else 'no_project_open', - roots=sorted(self._roots), project=project, - available_actions=list(PROJECT_ACTIONS if project is not None else ('new', 'open'))) - - def status(self): - with self._mutex, self.access.lock: - self.access.require('inspect', 'project_files.status') - return self._state() - - def capabilities(self): - value = agent_api.capabilities(self.db.dataset_id, profile='project') - for operation in value['operations']: - if operation['id'].startswith('project_transfer.'): - operation['availability'] = ('available' if getattr(self, 'transfer_controller', None) - is not None else 'unavailable') - continue - if operation['id'] in agent_api.PROJECT_FILES_M2 and not M2_ENABLED: - operation['availability'] = 'unavailable' - if operation['id'] in agent_api.PROJECT_FILES_M3 and not M3_ENABLED: - operation['availability'] = 'unavailable' - if operation['id'] in agent_api.PROJECT_FILES_M4 and not M4_ENABLED: - operation['availability'] = 'unavailable' - if not operation['id'].startswith('project_files.') and operation['id'] not in ('context.get', 'identity.get'): - operation['availability'] = 'unavailable' - return value - - def _bound(self, arguments, *, may_be_empty=False, check_checkpoint=True, allow_recovery=False): - self._ensure_open() - if arguments['binding'] != self._binding: - raise WorkbenchError('The project workspace changed. Refresh its binding before acting.', - 'stale_project', 409) - if self._recovery is not None: - if not allow_recovery: - raise WorkbenchError('Explicitly reopen the current project to reconcile its canonical outcome.', - 'projection_unavailable', 503) - if arguments.get('expected_checkpoint') != self._last_project['checkpoint']: - raise WorkbenchError('Recovery requires the last acknowledged checkpoint.', 'stale_project', 409) - return - if self._store is None: - if not may_be_empty: - raise WorkbenchError('Open an external project first.', 'unavailable', 409) - if arguments.get('expected_checkpoint') is not None: - raise WorkbenchError('The empty workspace has no checkpoint.', 'stale_project', 409) - return - snapshot = self._store.summary() - expected = arguments.get('expected_checkpoint', arguments.get('checkpoint')) - if check_checkpoint and expected != snapshot['checkpoint']: - raise WorkbenchError('The project checkpoint changed. Refresh before acting.', 'stale_project', 409) - - def _root(self, alias, relative): - if alias not in self._roots: - raise WorkbenchError('Choose a configured local project-file root.', 'unavailable', 409) - # Store create_at/open_at retains this handle through all actual use. - if (type(relative) is not str or not relative or len(relative) > 512 or - relative.startswith('/') or '\\' in relative or '\x00' in relative or - any(part in ('', '.', '..') for part in relative.split('/')) or - len(relative.split('/')) > 16): - raise WorkbenchError('Use a project directory relative to its configured root, without traversal.') - root_path, root_fd = self._roots[alias] - from project_store import assert_separate_project_control - assert_separate_project_control(root_fd, relative, root_path=root_path, control_path=self.path) - return self._roots[alias] - - def _working_identity(self, alias, directory, project_id): - """The persistent working copy for one project folder (private registry).""" - key = hashlib.sha256(f'{alias}\0{directory}'.encode('utf-8')).hexdigest() - registry = self.path / '.project-working' - path = registry / f'{key}.json' - try: - value = json.loads(path.read_bytes()) - if (value.get('project_id') == project_id and type(value.get('working_id')) is str - and WORKING_ID.fullmatch(value['working_id'])): - return value['working_id'] - except (OSError, ValueError, AttributeError): - pass - working_id = uuid.uuid4().hex - registry.mkdir(mode=0o700, exist_ok=True) - temporary = registry / f'.{uuid.uuid4().hex}.tmp' - temporary.write_bytes(json.dumps(dict(format='project_working_registration_v1', root=alias, - directory=directory, project_id=project_id, working_id=working_id), sort_keys=True).encode('utf-8')) - os.chmod(temporary, 0o600) - os.replace(temporary, path) - return working_id - - def _open_working(self, root_fd, root_path, alias, directory): - """Attach the folder's working copy, importing only if the folder changed.""" - folder = ProjectStore.open_folder_at(root_fd, directory, root_path=root_path) - try: - _raw, manifest = folder.folder_checkpoint() - working_id = self._working_identity(alias, directory, manifest['project_id']) - return WorkingProjectStore.open(folder, working_id, dsn=self._dsn, - canonical_digests=self.canonical_digests) - except BaseException: - folder.close() - raise - - def _projection_for(self, store): - identity = uuid.uuid4().hex - path = self.path / '.project-projections' / identity - snapshot = store.summary() - projection = Metadata(path, dsn=self._dsn, dataset_id=snapshot['project_id'], projection_id=identity) - if hasattr(store, 'bind'): - # Record commits share the projection's transactions. - store.bind(projection) - try: - owner = ProjectProjection(store, projection, self.access, - {alias: value[0] for alias, value in self._roots.items()}, - custody=self.custody.for_projection(store, identity), - runtime_sources=self._runtime_sources, - validation_cache=self.validation_cache) - owner.pending_original = self.custody.pending(store) - if owner.pending_original: - owner.session.available = False - return owner, owner.text - except BaseException: - projection.close() - # Disposable failed projections remain identified for diagnosis; - # no authoritative bytes or an uncertain outcome is deleted. - raise - - def _retained_projection_for(self, store, projection_id): - if type(projection_id) is not str or not re.fullmatch('[a-f0-9]{32}', projection_id): - raise WorkbenchError('Invalid original local projection identity.', 'unavailable', 409) - db = Metadata(self.path / '.project-projections' / projection_id, dsn=self._dsn, - create=False, dataset_id=store.summary()['project_id'], projection_id=projection_id) - if hasattr(store, 'bind'): - store.bind(db) - try: - return ProjectProjection(store, db, self.access, - {alias: value[0] for alias, value in self._roots.items()}, - custody=self.custody.for_projection(store, projection_id), restore=False, - runtime_sources=self._runtime_sources, validation_cache=self.validation_cache) - except BaseException: - db.close() - raise - - def _detach(self): - """Release only a project whose current local effects have drained.""" - if self._projection is not None: - self._projection.execution.assert_quiescent() - # Closing keeps unsaved work in the database working copy; only an - # explicit Save writes the external folder. - if self._projection is not None: - self._projection.close() - if self._store is not None: - self._store.close() - self._store = self._text = self._projection = None - self._selection = None - self._last_project = self._recovery = None - - def _recover_current(self, arguments): - self._bound(arguments, allow_recovery=True) - if (arguments['root'], arguments['directory']) != self._selection: - raise WorkbenchError('Recover the current project before switching destinations.', 'projection_unavailable', 503) - if self._projection is not None and self._projection.pending_original: - raise WorkbenchError('Explicitly reconcile original-host custody before reopening this project.', - 'projection_unavailable', 503) - if self._projection is not None and self._projection.execution.has_local_custody: - # Retain actual local custody until the explicit execution recovery - # boundary proves cessation; imported history can never do so. - self._projection.execution.assert_quiescent() - root_path, root_fd = self._root(*self._selection) - # Keep the old projection and generation until a fresh inert projection - # is ready. Releasing the uncertain writer lock is necessary to reread it. - if self._store is not None: - self._store.close() - self._store = None - store = projection = None - try: - store = self._open_working(root_fd, root_path, *self._selection) - snapshot = store.summary() - if snapshot['project_id'] != self._last_project['project_id']: - raise WorkbenchError('Recovery destination has a different project identity.', 'stale_project', 409) - # Database commits are atomic: reattaching the working copy - # reconciles the outcome without writing the external folder. - projection, text = self._projection_for(store) - snapshot = store.summary() - if self._projection is not None: - self._projection.close() - self._store, self._projection, self._text = store, projection, text - self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} - self._recovery = None - self._binding = uuid.uuid4().hex - return self._state() - except BaseException: - if projection is not None and projection is not self._projection: - projection.close() - if store is not None and store is not self._store: - store.close() - raise - - def _select(self, action, arguments): - if self._recovery is not None: - if action != 'open': - raise WorkbenchError('Reopen the current project before creating another.', 'projection_unavailable', 503) - return self._recover_current(arguments) - self._bound(arguments, may_be_empty=True) - root_path, root_fd = self._root(arguments['root'], arguments['directory']) - selection = (arguments['root'], arguments['directory']) - if action == 'open' and selection == self._selection: - return self._state() - if self._projection is not None: - self._projection.execution.assert_quiescent() - # Validate and construct the incoming inert projection before releasing - # the existing binding. An invalid Open cannot strand current custody. - store = None - projection = None - created = None - try: - if action == 'new': - request = {key: arguments[key] for key in ('root', 'directory', 'name', 'request_key')} - digest = _digest(request) - reconciled = False - try: - store = ProjectStore.create_at(root_fd, arguments['directory'], arguments['name'], - arguments['request_key'], digest, root_path=root_path) - except ProjectStoreError as error: - if error.code not in ('conflict', 'unknown', 'outcome_unknown'): - raise - # A lost creation reply or restart may leave a complete, - # unselected destination. Only its exact receipt authorizes - # treating it as this New; unrelated folders remain conflicts. - try: - store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path, canonical_digests=self.canonical_digests) - except ProjectStoreError: - raise error - if store.reconcile(arguments['request_key'], digest) is None: - store.close() - raise error - reconciled = True - snapshot = store.summary() - if reconciled: - store.save(snapshot['checkpoint']) - created = snapshot - # The new folder becomes this workspace's working copy. - store.close() - store = self._open_working(root_fd, root_path, arguments['root'], arguments['directory']) - else: - store = self._open_working(root_fd, root_path, arguments['root'], arguments['directory']) - projection, text = self._projection_for(store) - self._detach() - self._store, self._projection, self._text = store, projection, text - self._selection = selection - snapshot = store.summary() - self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} - self._recovery = None - self._binding = uuid.uuid4().hex - return self._state() - except BaseException as error: - if projection is not None and projection is not self._projection: - projection.close() - if store is not None and store is not self._store: - store.close() - if created is not None and self._store is not store: - failure = WorkbenchError('Project was created but could not be selected; retry this exact New to reconcile its destination.', - 'project_selection_unavailable', 503) - failure.committed, failure.selected = True, False - failure.project_id, failure.checkpoint = created['project_id'], created['checkpoint'] - failure.request_key = arguments['request_key'] - failure.root, failure.directory = arguments['root'], arguments['directory'] - failure.receipt = dict(committed=True, selected=False, project_id=created['project_id'], - checkpoint=created['checkpoint'], request_key=arguments['request_key'], - root=arguments['root'], directory=arguments['directory']) - raise failure from error - raise - - def _record_result(self, record, *, before_revision=None, history_limit=1): - snapshot = self._store.summary() - history = [item for item in self._text.history(record['id']) - if item['revision'] <= record['revision'] and - (before_revision is None or item['revision'] < before_revision)] - history.sort(key=lambda item: item['revision'], reverse=True) - page = history[:history_limit] - self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} - return dict(binding=self._binding, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint'], - record=public_record(record), history=[public_record(item) for item in page], - next_history_before_revision=page[-1]['revision'] if len(history) > len(page) else None) - - def _save_as(self, arguments): - return self._publish_independent(arguments) - - def _compose(self, arguments): - return self._publish_independent(arguments, composed=True) - - def _publish_independent(self, arguments, *, composed=False): - """Publish a new independent project, then select a fresh inert cache.""" - from functools import partial - from project_fork import publish_fork - from project_fork_composition import build_candidate - from workbench import timestamp - self._bound(arguments) - root_path, root_fd = self._root(arguments['root'], arguments['directory']) - source = self._projection - source.execution.assert_quiescent() - source.files.require_clean() - published = None - incoming = projection = None - retained = ExitStack() - try: - if composed: - from project_composition import publish_composition - from project_artifacts import ProjectArtifacts - others=[];identities={self._store.summary()['project_id']} - for selected in arguments['sources']: - if selected['project_id'] in identities: - raise WorkbenchError('Composition source identity repeats.','conflict',409) - identities.add(selected['project_id']) - directory,descriptor=self._root(selected['root'],selected['directory']) - store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory, canonical_digests=self.canonical_digests)) - actual=store.summary() - if actual['project_id']!=selected['project_id'] or actual['checkpoint']!=selected['checkpoint']: - raise WorkbenchError('Selected composition source checkpoint differs.','stale_project',409) - artifacts=retained.enter_context(ProjectArtifacts(store)) - others.append(dict(store=store,artifacts=artifacts,checkpoint=selected['checkpoint'])) - published=publish_composition(self._store,source.artifacts,others,root_fd,arguments['directory'], - root_path=root_path,name=arguments['name'],expected_checkpoint=arguments['expected_checkpoint'], - request_key=arguments['request_key'],quiescent=True,created_at=timestamp(), - id_factory=lambda *_:uuid.uuid4().hex, - validate_candidate=lambda records,identity,artifacts:source.validate_records(records,project_id=identity,artifacts=artifacts)) - else: - published = publish_fork(self._store, source.artifacts, root_fd, arguments['directory'], - root_path=root_path, name=arguments['name'], - expected_checkpoint=arguments['expected_checkpoint'], request_key=arguments['request_key'], - quiescent=True, build_candidate=partial(build_candidate, created_at=timestamp(), - id_factory=lambda _family, _source_id, _revision: uuid.uuid4().hex), - validate_candidate=lambda records, identity: source.validate_records(records, project_id=identity)) - # A filesystem editor can act outside the workspace lock. Leave the - # complete fork unselected if its source working files changed. - source.files.require_clean() - incoming = self._open_working(root_fd, root_path, arguments['root'], arguments['directory']) - projection, text = self._projection_for(incoming) - snapshot, overlay = published['snapshot'], published['fork'] - self._detach() - self._store, self._projection, self._text = incoming, projection, text - self._selection = arguments['root'], arguments['directory'] - self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} - self._recovery = None - self._binding = uuid.uuid4().hex - summary = dict(source_project_id=overlay['source']['project_id'], - source_checkpoint=overlay['source']['checkpoint'], project_id=snapshot['project_id'], - checkpoint=snapshot['checkpoint'], root=arguments['root'], directory=arguments['directory'], - request_key=arguments['request_key'], reconciled=published['reconciled'], - mapping_count=len(overlay['mapping']), copied_count=len(overlay['artifacts']['copied']), - missing_regenerable_count=len(overlay['artifacts']['missing_regenerable'])) - result=dict(state=self._state(), fork=summary) - if composed: - result['sources']=[{key:item[key] for key in ('project_id','checkpoint')} for item in published['composition']['sources']] - return result - except BaseException as error: - if projection is not None and projection is not self._projection: - projection.close() - if incoming is not None and incoming is not self._store: - incoming.close() - receipt = getattr(error, 'receipt', None) - if published is not None: - snapshot = published['snapshot'] - receipt = dict(committed=True, selected=False, project_id=snapshot['project_id'], - checkpoint=snapshot['checkpoint'], request_key=arguments['request_key'], - root=arguments['root'], directory=arguments['directory']) - elif receipt is not None: - receipt = dict(receipt, root=arguments['root']) - if receipt is not None: - failure = WorkbenchError('Independent project was published but could not be selected; reconcile this exact request.', - 'project_selection_unavailable', 503) - failure.receipt = receipt - raise failure from error - raise - finally: - retained.close() - - def dispatch(self, name, arguments, *, selected_version=None): - """Pin workspace -> current host authority -> projection through return. - - An old request cannot observe a new Dataset alias. Revocation cannot - complete before an already-authorized canonical publication finishes. - """ - agent_api.validate_arguments(name, arguments) - if not name.startswith('project_files.'): - raise WorkbenchError('This domain is unavailable in external text-project mode.', 'unavailable', 409) - if name == 'project_files.execution_health': - return self._execution_health(arguments, selected_version=selected_version) - if name in _DRAINING_OPERATIONS: - return self._dispatch_draining(name, arguments) - action = name.removeprefix('project_files.') - with self._mutex, self.access.lock: - self._ensure_open() - operation = agent_api.operation(name) - for scope in operation['scopes']: - if scope == 'execute' and action in ('execution_submit_external', 'execution_study_admit'): - self.access.require_external_submission() - else: - self.access.require(scope, name, owner_only=action in ('new', 'open', 'save', 'save_as', 'compose', 'close')) - if selected_version == 1 and action in ('get', 'annotate'): - self._bound(arguments, check_checkpoint=action == 'get') - record = self._projection.data._require_record(arguments['record_id']) - if record['document']['current']['provenance'].get('method') != 'import': - raise WorkbenchError('This retained text provenance requires project text contract version 2.', - 'unsupported', 409) - if name in agent_api.PROJECT_FILES_M4 and not M4_ENABLED: - raise WorkbenchError('This project capability is not yet available.', 'unavailable', 409) - if name in (agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | agent_api.PROJECT_FILES_M4) - {'project_files.save_as','project_files.compose'}: - if ((name in agent_api.PROJECT_FILES_M2 and not M2_ENABLED) - or (name in agent_api.PROJECT_FILES_M3 and not M3_ENABLED) - or (name in agent_api.PROJECT_FILES_M4 and not M4_ENABLED)): - raise WorkbenchError('This project capability is not yet available.', 'unavailable', 409) - self._bound(arguments, check_checkpoint=not operation['side_effects']) - from project_dispatch import dispatch - try: - value = dispatch(self._projection, name, arguments) - if (name=='project_files.measurement_compare' - and value['value']['observation_count']>agent_api.measurement_comparison_row_limit(selected_version)): - raise WorkbenchError('Comparison exceeds the selected API version observation budget. Select comparison contract version 2 for larger complete populations.', - 'too_large',413) - if name == 'project_files.execution_health' and selected_version == 1: - value = {key: item for key, item in value.items() if key != 'native_phases'} - except BaseException as error: - if getattr(error, 'code', None) in ('unknown', 'outcome_unknown', 'projection_unavailable'): - self._recovery_required('projection_unavailable', arguments.get('request_key')) - raise - snapshot = self._store.summary() - self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} - result = dict(binding=self._binding, project_id=snapshot['project_id'], - checkpoint=snapshot['checkpoint'], value=value) - elif action == 'status': - result = self._state() - else: - # Lifecycle receipts are process-binding scoped. Domain edit - # receipts are canonical and survive any originating DB loss. - key = (name, arguments.get('binding'), arguments.get('request_key')) - signature = _digest(arguments) - prior = self._receipts.get(key) if arguments.get('request_key') else None - if prior is not None: - if prior[0] != signature: - raise WorkbenchError('Request key already has different input.', 'conflict', 409) - prior_state = prior[1]['state'] if action in ('save_as','compose') else prior[1] - if prior_state != self._state(): - raise WorkbenchError('This lifecycle receipt belongs to an obsolete workspace state.', 'stale_project', 409) - result = copy.deepcopy(prior[1]) - else: - try: - if action in ('new', 'open'): - result = self._select(action, arguments) - elif action == 'save': - self._bound(arguments) - self._store.save(arguments['expected_checkpoint']) - result = self._state() - elif action == 'save_as': - result = self._save_as(arguments) - elif action == 'compose': - result = self._compose(arguments) - elif action == 'close': - self._bound(arguments) - self._detach() - self._binding = uuid.uuid4().hex - result = self._state() - elif action == 'query': - self._bound(arguments) - value = self._text.records() - items = sorted(value['items'], key=lambda item: item['id']) - if arguments.get('after_id') is not None: - items = [item for item in items if item['id'] > arguments['after_id']] - page = items[:arguments.get('limit', 40)] - result = dict(binding=self._binding, project_id=value['project_id'], checkpoint=value['checkpoint'], - items=[{key: copy.deepcopy(item[key]) for key in SUMMARY_FIELDS} for item in page], - next_after_id=page[-1]['id'] if len(items) > len(page) else None) - elif action == 'get': - self._bound(arguments) - result = self._record_result(self._text.get(arguments['record_id']), - before_revision=arguments.get('before_revision'), history_limit=arguments.get('history_limit', 1)) - elif action in ('import_text', 'annotate'): - # Canonical domain receipts reconcile before CAS. - self._bound(arguments, check_checkpoint=False) - if action == 'import_text': - record = self._text.import_text(arguments['asset'], arguments['expected_checkpoint'], arguments['request_key']) - else: - record = self._text.annotate(arguments['record_id'], arguments['annotation'], - arguments['expected_checkpoint'], arguments['request_key']) - result = self._record_result(record) - else: - raise WorkbenchError('Unsupported project-file operation.', 'unsupported', 400) - except BaseException as error: - code = getattr(error, 'code', None) - if action not in ('save_as','compose') and code in ('unknown', 'outcome_unknown', 'projection_unavailable'): - reason = 'unknown_publication' if code in ('unknown', 'outcome_unknown') else 'projection_unavailable' - self._recovery_required(reason, arguments.get('request_key')) - raise - if arguments.get('request_key') and action in ('new', 'open', 'save', 'save_as', 'compose', 'close'): - self._receipts[key] = (signature, copy.deepcopy(result)) - if len(agent_api.encoded(result)) > agent_api.RESPONSE_LIMIT: - raise WorkbenchError('This project result exceeds the synchronous response bound; canonical work is retained.', - 'too_large', 413) - agent_api.validate_output(name, {'status': 200, 'result': result},version=selected_version) - return result - - def _dispatch_draining(self, name, arguments): - """Keep workspace selection pinned while descendants drain without SQL locks.""" - with self._mutex: - with self.access.lock: - self._ensure_open() - if not M3_ENABLED: - raise WorkbenchError('This project capability is not yet available.', 'unavailable', 409) - operation = agent_api.operation(name) - for scope in operation['scopes']: - self.access.require(scope, name, owner_only=name in _RECOVERY_OPERATIONS) - recovery_operation = name in _RECOVERY_OPERATIONS - if recovery_operation and self._projection is not None and not self._projection.session.available: - self._recovery_required('projection_unavailable', arguments.get('request_key')) - self._bound(arguments, check_checkpoint=recovery_operation and name != 'project_files.runtime_capture', - allow_recovery=recovery_operation) - projection = self._projection - recovering = self._recovery is not None or not projection.session.available - from project_dispatch import dispatch - try: - incoming = projection - if recovery_operation and projection.pending_original: - if len(projection.pending_original) != 1: - raise WorkbenchError('More than one original local custody binding needs explicit review.', - 'local_custody_unavailable', 409) - binding = projection.pending_original[0] - if binding['project_id'] != projection.session.project_id: - raise WorkbenchError('Restore the original project identity before reconciling this physical folder custody.', - 'local_custody_unavailable', 409) - if self._retained_recovery is None: - self._retained_recovery = self._retained_projection_for(self._store, binding['projection_id']) - incoming = self._retained_recovery - if incoming.db.projection_id != binding['projection_id']: - raise WorkbenchError('The retained physical recovery owner changed.', 'local_custody_unavailable', 409) - recovering = True - if incoming is not projection and name == 'project_files.execution_reconcile_current': - value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.summary['checkpoint']) - else: - internal = arguments - if recovery_operation and recovering and name != 'project_files.execution_reconcile_current': - # Public CAS above uses the last acknowledged workspace - # view. Only this qualified local owner sees its exact - # selected/attempted checkpoint for finite recovery. - internal = dict(arguments, expected_checkpoint=incoming.session.summary['checkpoint']) - if name == 'project_files.runtime_capture' and recovering: - options = {'reserve_bytes': internal['reserve_bytes']} if 'reserve_bytes' in internal else {} - value = incoming.runtime.recover_capture({'source': internal['source']}, - internal['expected_inventory_sha256'], internal['expected_checkpoint'], - internal['request_key'], **options) - else: - value = dispatch(incoming, name, internal) - if recovery_operation: - if not incoming.session.available or incoming.custody is not None and incoming.custody.has_pending: - raise WorkbenchError('Original writer custody remains unresolved.', 'outcome_unknown', 409) - if incoming is not projection: - incoming.execution.assert_quiescent() - projection.close() - projection = self._projection = incoming - self._text = incoming.text - self._retained_recovery = None - # Physical cache repair retains this same selected - # project and workspace generation; it is not Open. - except BaseException as error: - if getattr(error, 'code', None) in ('unknown', 'outcome_unknown', 'projection_unavailable'): - self._recovery_required('projection_unavailable', arguments.get('request_key')) - raise - with self.access.lock, projection.session.lock, projection.db.lock: - snapshot = self._store.summary() - self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} - if name in _RECOVERY_OPERATIONS: - self._recovery = None - result = dict(binding=self._binding, project_id=snapshot['project_id'], - checkpoint=snapshot['checkpoint'], value=value) - if len(agent_api.encoded(result)) > agent_api.RESPONSE_LIMIT: - raise WorkbenchError('Project drainage result exceeds its response bound.', 'too_large', 413) - agent_api.validate_output(name, {'status': 200, 'result': result}) - return result - - def _execution_health(self, arguments, *, selected_version=None): - """Read only indexed current-local custody, including a fenced owner.""" - name='project_files.execution_health' - with self._mutex,self.access.lock: - self._ensure_open() - if not M3_ENABLED:raise WorkbenchError('This project capability is not yet available.','unavailable',409) - self.access.require('inspect',name) - if self._projection is not None and not self._projection.session.available: - self._recovery_required('projection_unavailable') - self._bound(dict(arguments,expected_checkpoint=arguments['checkpoint']),allow_recovery=True) - incoming=self._projection - if incoming.pending_original: - if len(incoming.pending_original)!=1 or incoming.pending_original[0]['project_id']!=incoming.session.project_id: - raise WorkbenchError('Original local custody requires its exact project identity.','local_custody_unavailable',409) - identity=incoming.pending_original[0]['projection_id'] - if self._retained_recovery is None: - self._retained_recovery=self._retained_projection_for(self._store,identity) - incoming=self._retained_recovery - if incoming.db.projection_id!=identity: - raise WorkbenchError('Original physical custody changed.','local_custody_unavailable',409) - value=incoming.execution.health(arguments.get('offset',0),arguments.get('limit',40)) - if selected_version==1:value={key:item for key,item in value.items() if key!='native_phases'} - result=dict(binding=self._binding,project_id=incoming.session.project_id, - checkpoint=arguments['checkpoint'],value=value) - if len(agent_api.encoded(result))>agent_api.RESPONSE_LIMIT: - raise WorkbenchError('Current custody health exceeds its response bound.','too_large',413) - agent_api.validate_output(name,dict(status=200,result=result),version=selected_version) - return result - - def close(self): - with self._mutex: - if self._closed: - return - # Shutdown always releases handles, including a stale/unknown store; - # actual local workers drain before any database handle is closed. - if self._projection is not None: - self._projection.execution.shutdown() - self._projection.close() - if self._store is not None: - self._store.close() - self._projection = self._text = self._store = None - for _path, fd in self._roots.values(): - os.close(fd) - self._roots.clear() - self.db.close() - self._closed = True diff --git a/pumas_gateway_descriptor.py b/pumas_gateway_descriptor.py index eb25b9f1..c6469b53 100644 --- a/pumas_gateway_descriptor.py +++ b/pumas_gateway_descriptor.py @@ -1,12 +1,24 @@ -"""Decode the public Pumas PR51 HTTP advertisement; no owner authentication.""" +"""Decode the public Pumas HTTP advertisement (`GET /.well-known/pumas`); no owner authentication. + +The advertisement grows additively. Unknown fields and new entries in open lists +(protocols, schemas, capabilities, compiled features) are ignored: they are neither +errors nor retained in the decoded advertisement. Missing required fields, wrong +types, unsupported schema versions and malformed endpoints are refused with a +reason naming the field. +""" import hashlib import ipaddress import json from urllib.parse import urlsplit -SOURCE_COMMIT = '80f06ab17f9eea639fee143c86319ca9b5e1a21c' +# Pumas Library tag v0.8.0, whose `HttpServiceDescription` and `PumasBuildInfo` this decodes. +SOURCE_COMMIT = '69e32a2208c5c2d840f4adafec03350669fcb5bc' +# Transport bound. Open lists have no entry-count cap of their own. MAX_DESCRIPTION_BYTES = 64 * 1024 -MAX_ITEMS = 64 + + +class Rejected(ValueError): + """A Pumas advertisement that was read and refused. The message is the reason.""" def endpoint(value): @@ -28,60 +40,82 @@ def endpoint(value): return f'http://{address}:{port}' -def _text(value, limit=4096): +def _fail(where, problem): + raise Rejected(' '.join(filter(None, ('Pumas advertisement', where, problem))) + '.') + + +def _text(value, where, limit=4096): if not isinstance(value, str) or not value or len(value) > limit: - raise ValueError('Invalid Pumas descriptor text.') - value.encode('utf-8', errors='strict') + _fail(where, f'must be a nonempty string of at most {limit} characters') + try: + value.encode('utf-8', errors='strict') + except UnicodeError: + _fail(where, 'must be valid Unicode text') -def _uint(value): +def _uint(value, where): if type(value) is not int or not 0 <= value <= 4294967295: - raise ValueError('Invalid Pumas descriptor integer.') + _fail(where, 'must be an unsigned 32-bit integer') -def _object(value, required, optional=()): - if type(value) is not dict or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): - raise ValueError('Unsupported Pumas descriptor fields.') +def _version(value, where): + if type(value) is not int: + _fail(where, 'must be an integer') + if value != 1: + raise Rejected(f'Pumas advertisement {where} {value} is unsupported; Tuldok supports version 1.') -def _list(value): - if type(value) is not list or len(value) > MAX_ITEMS: - raise ValueError('Invalid Pumas descriptor list.') +def _object(value, where, required, optional=()): + """Open shape: every required field present. Returns only the known fields.""" + if type(value) is not dict: + _fail(where, 'must be an object') + missing = [name for name in required if name not in value] + if missing: + _fail(where, 'is missing required field(s) ' + ', '.join(missing)) + known = set(required) | set(optional) + return {name: item for name, item in value.items() if name in known} -def _strings(value): - _list(value) - for item in value: - _text(item, 512) +def _list(value, where): + if type(value) is not list: + _fail(where, 'must be a list') + return value -def _protocols(value): - _list(value) - for item in value: - _object(item, ('name', 'versions')) - _text(item['name'], 512) - _list(item['versions']) - for version in item['versions']: - _uint(version) +def _strings(value, where): + for index, item in enumerate(_list(value, where)): + _text(item, f'{where}[{index}]', 512) + return list(value) -def _build(value): - _object(value, ('build_info_schema_version', 'component', 'package_version', 'compiled_features', 'protocols', 'schemas'), - ('build_id', 'source_revision', 'target')) - if type(value['build_info_schema_version']) is not int or value['build_info_schema_version'] != 1: - raise ValueError('Unsupported Pumas build-info schema.') +def _protocols(value, where): + result = [] + for index, item in enumerate(_list(value, where)): + item = _object(item, f'{where}[{index}]', ('name', 'versions')) + _text(item['name'], f'{where}[{index}].name', 512) + for position, version in enumerate(_list(item['versions'], f'{where}[{index}].versions')): + _uint(version, f'{where}[{index}].versions[{position}]') + result.append(item) + return result + + +def _build(value, where): + value = _object(value, where, ('build_info_schema_version', 'component', 'package_version', 'compiled_features', + 'protocols', 'schemas'), ('build_id', 'source_revision', 'target')) + _version(value['build_info_schema_version'], where + '.build_info_schema_version') for name in ('component', 'package_version', 'build_id', 'source_revision', 'target'): - if name in value and value[name] is not None: - _text(value[name]) - elif name in ('component', 'package_version'): - raise ValueError('Missing Pumas build identity.') - _strings(value['compiled_features']) - _protocols(value['protocols']) - _list(value['schemas']) - for item in value['schemas']: - _object(item, ('name', 'version')) - _text(item['name'], 512) - _uint(item['version']) + if name in ('component', 'package_version') or value.get(name) is not None: + _text(value[name], f'{where}.{name}') + value['compiled_features'] = _strings(value['compiled_features'], where + '.compiled_features') + value['protocols'] = _protocols(value['protocols'], where + '.protocols') + schemas = [] + for index, item in enumerate(_list(value['schemas'], where + '.schemas')): + item = _object(item, f'{where}.schemas[{index}]', ('name', 'version')) + _text(item['name'], f'{where}.schemas[{index}].name', 512) + _uint(item['version'], f'{where}.schemas[{index}].version') + schemas.append(item) + value['schemas'] = schemas + return value def _pairs(pairs): @@ -102,29 +136,34 @@ def decode(raw): raise ValueError('Pumas descriptor exceeds the 64 KiB observation limit.') try: value = json.loads(raw.decode('utf-8'), object_pairs_hook=_pairs, parse_constant=_constant) - _object(value, ('advertisement_schema_version', 'service_generation', 'instance', 'endpoint', 'build_info')) - if type(value['advertisement_schema_version']) is not int or value['advertisement_schema_version'] != 1: - raise ValueError('Unsupported Pumas HTTP advertisement schema.') - _text(value['service_generation'], 128) - endpoint(value['endpoint']) - instance = value['instance'] - _object(instance, ('discovery_schema_version', 'registry_library_id', 'library_root', 'generation', 'pumas_version', - 'protocols', 'capabilities', 'model_ref_schema_version', 'selector_schema_version'), ('build_info',)) - if type(instance['discovery_schema_version']) is not int or instance['discovery_schema_version'] != 1: - raise ValueError('Unsupported Pumas discovery schema.') - for name in ('registry_library_id', 'library_root', 'generation', 'pumas_version'): - _text(instance[name]) - for name in ('model_ref_schema_version', 'selector_schema_version'): - _uint(instance[name]) - _protocols(instance['protocols']) - _strings(instance['capabilities']) - if instance.get('build_info') is not None: - _build(instance['build_info']) - build = value['build_info'] - _build(build) - if (not any(p['name'] == 'pumas.local-http' and 1 in p['versions'] for p in build['protocols']) - or not any(s['name'] == 'pumas.http-advertisement' and s['version'] == 1 for s in build['schemas'])): - raise ValueError('Unsupported Pumas HTTP protocol/schema advertisement.') except (UnicodeError, RecursionError) as error: raise ValueError('Invalid Pumas descriptor encoding or nesting.') from error + if type(value) is not dict or 'advertisement_schema_version' not in value: + raise ValueError('Not a Pumas HTTP advertisement: advertisement_schema_version is missing.') + value = _object(value, '', ('advertisement_schema_version', 'service_generation', 'instance', 'endpoint', + 'build_info')) + _version(value['advertisement_schema_version'], 'advertisement_schema_version') + _text(value['service_generation'], 'service_generation', 256) + try: + endpoint(value['endpoint']) + except ValueError as error: + raise Rejected(f'Pumas advertisement endpoint is invalid: {error}') from None + instance = value['instance'] = _object( + value['instance'], 'instance', ('discovery_schema_version', 'registry_library_id', 'library_root', 'generation', + 'pumas_version', 'protocols', 'capabilities', 'model_ref_schema_version', + 'selector_schema_version'), ('build_info',)) + _version(instance['discovery_schema_version'], 'instance.discovery_schema_version') + for name in ('registry_library_id', 'library_root', 'generation', 'pumas_version'): + _text(instance[name], 'instance.' + name) + for name in ('model_ref_schema_version', 'selector_schema_version'): + _uint(instance[name], 'instance.' + name) + instance['protocols'] = _protocols(instance['protocols'], 'instance.protocols') + instance['capabilities'] = _strings(instance['capabilities'], 'instance.capabilities') + if instance.get('build_info') is not None: + instance['build_info'] = _build(instance['build_info'], 'instance.build_info') + build = value['build_info'] = _build(value['build_info'], 'build_info') + if not any(p['name'] == 'pumas.local-http' and 1 in p['versions'] for p in build['protocols']): + raise Rejected('Pumas advertisement build_info does not advertise protocol pumas.local-http version 1.') + if not any(s['name'] == 'pumas.http-advertisement' and s['version'] == 1 for s in build['schemas']): + raise Rejected('Pumas advertisement build_info does not advertise schema pumas.http-advertisement version 1.') return {'advertisement': value, 'observed_sha256': hashlib.sha256(raw).hexdigest()} diff --git a/pumas_operations.py b/pumas_operations.py index 89d345bc..86e174d5 100644 --- a/pumas_operations.py +++ b/pumas_operations.py @@ -1,32 +1,63 @@ -"""Finite PR54 selected-model operations. Discovery budgets never time generation.""" +"""Finite Pumas typed model operations, as released in Pumas Library 0.8.0 (contract version 1). + +Pumas is reached only as an HTTP endpoint. Discovery budgets never time +generation. Responses grow additively: unknown response fields, capabilities, +formats, modalities, option names, availability reasons and error codes are +ignored; only what Tuldok uses is validated and selected by name. Requests send +only what the selected serving declared. +""" import hashlib import http.client import json +import math import re import socket import threading import time from urllib.parse import urlencode, urlsplit +import ai_http +from ai_codex import CodexError import gateway_discovery from pumas_gateway_descriptor import endpoint from workbench import WorkbenchError, encode PROTOCOL = 'pumas_typed_v1' -SOURCE_COMMIT = '40c5cbfed67a6f0e862a1197bb5105363d67bdb1' -MAX_CAPABILITIES = 65536 +# Pumas Library tag v0.8.0: the released contract this module follows. +SOURCE_COMMIT = '69e32a2208c5c2d840f4adafec03350669fcb5bc' +MAX_CAPABILITIES = 1024 * 1024 +MAX_SERVED_MODELS = 8 * 1024 * 1024 +MAX_SERVED_ENTRIES = 4096 MAX_TEXT_RESPONSE = 256 * 1024 MAX_IMAGE_JSON = 12 * 1024 * 1024 MAX_DIMENSION = 2048 MAX_PIXELS = 4194304 -CAPABILITIES = {'chat_generation': 'chat_generation', 'text_generation': 'text_generation', - 'text_embedding': 'text_embedding', 'image_generation': 'text_to_image', - 'image_to_text': 'image_to_text', - 'audio_transcription': 'speech_to_text', 'audio_classification': 'audio_classification'} -ERROR_CODES = {'invalid_request', 'unsupported_contract', 'model_not_found', 'ambiguous_model', - 'ambiguous_operation', 'unsupported_modality', - 'capability_unavailable', 'provider_failure', 'invalid_provider_result', - 'request_limit', 'response_limit', 'transport_lost'} +MAX_MESSAGE = 600 +# The result kind of each output Tuldok asks for. +RESULT_KINDS = {'text': 'text', 'png_base64': 'image', 'decisions': 'decisions', 'embeddings_float32': 'embeddings'} +# Capabilities Tuldok sends, with their fixed contract-version-1 semantic task. +USED_CAPABILITIES = {'chat_generation': 'chat_generation', 'image_generation': 'text_to_image', + 'image_to_text': 'image_to_text'} +# Option bounds Tuldok compares against. Other advertised options are ignored. +USED_OPTIONS = {'max_tokens', 'temperature', 'top_p', 'width', 'height', 'seed', 'input_characters', + 'image_bytes', 'image_pixels', 'image_count', 'total_image_bytes', 'message_count', 'content_parts'} +# The 0.8.0 release vocabulary. Values outside these sets are ignored, never an error. +CAPABILITIES = ('chat_generation', 'text_generation', 'text_embedding', 'image_generation', 'image_to_text', + 'multimodal_embedding', 'decision', 'rerank', 'audio_transcription', 'audio_classification') +INPUT_FORMATS = ('messages_text', 'messages_image', 'messages_multimodal', 'text', 'text_batch', 'png_base64', + 'jpeg_base64', 'pcm_s16le', 'pcm_f32le', 'content_batch', 'decision', 'rerank') +OUTPUT_FORMATS = ('text', 'embeddings_float32', 'png_base64', 'decisions', 'relevance_scores', 'labels') +MODALITIES = ('text', 'image', 'audio', 'video', 'json', 'vector') +OPTION_NAMES = ('max_tokens', 'temperature', 'top_p', 'dimensions', 'input_count', 'input_characters', 'width', + 'height', 'seed', 'max_output_tokens', 'image_bytes', 'image_pixels', 'image_count', + 'total_image_bytes', 'message_count', 'content_parts', 'input_tokens', 'top_n', 'audio_count', + 'total_audio_bytes', 'audio_seconds', 'sample_rate') +# Request `input.kind` to the descriptor input formats that accept it (the release's resolver). +# `messages` with image parts is the same request as `image_messages`; ask with that kind. +INPUT_KINDS = {'text': ('text',), 'text_batch': ('text_batch',), 'messages': ('messages_text',), + 'image_messages': ('messages_image',), 'image': ('png_base64', 'jpeg_base64'), + 'content_batch': ('content_batch',), 'decision': ('decision',), 'rerank': ('rerank',), + 'audio': ('pcm_s16le', 'pcm_f32le')} class OperationError(WorkbenchError): @@ -56,10 +87,71 @@ def constant(_): def object_fields(value, required, optional=()): + """Closed shape for request-side DTOs that Tuldok itself builds.""" if type(value) is not dict or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): raise ValueError('Unsupported typed Pumas fields.') +def response_fields(value, required): + """Open shape for Pumas responses: required fields present, additive fields ignored.""" + if type(value) is not dict or not set(required) <= value.keys(): + raise ValueError('Typed Pumas response is missing required fields.') + + +def _positive_int(value): + return type(value) is int and value > 0 + + +def message_text(value): + """Bounded single-line display of Pumas' human-readable reason (never parsed).""" + if not isinstance(value, str): + return '' + value = ' '.join(''.join(c if c.isprintable() else ' ' for c in value).split()) + return value[:MAX_MESSAGE] + ('…' if len(value) > MAX_MESSAGE else '') + + +def pre_handler(value): + """True for a refusal made before the request reached its handler: `{"error":{"code","message"}}`.""" + return (type(value) is dict and type(value.get('error')) is dict and 'contract_version' not in value + and 'request_id' not in value and 'outcome' not in value['error']) + + +def typed_error(value): + """Decode `{contract_version, request_id, error:{code, outcome, message}}` or return None. + + `outcome` is authoritative. Only `not_admitted` proves no provider effect; + any other value is treated as `unknown`. A pre-handler refusal has no + outcome and never reached a handler, so it is `not_admitted`. Unknown codes + are kept verbatim. + """ + if type(value) is not dict or type(value.get('error')) is not dict: + return None + error = value['error'] + code = error.get('code') + if not isinstance(code, str) or not code or len(code) > 200: + return None + outcome = 'not_admitted' if error.get('outcome') == 'not_admitted' or pre_handler(value) else 'unknown' + return {'code': code, 'outcome': outcome, 'message': message_text(error.get('message'))} + + +def error_message(error, prefix='Pumas'): + detail = (': ' + error['message']) if error['message'] else '.' + if error['outcome'] == 'not_admitted': + return f"{prefix} refused the request ({error['code']}; not admitted, no provider effect){detail}" + return f"{prefix} reported {error['code']}; provider outcome unknown, no automatic replay{detail}" + + +def http_error(status, raw): + """Raise a readable ValueError for a non-200 typed discovery response.""" + try: + error = typed_error(strict_json(raw)) + except ValueError: + error = None + if error is None: + raise ValueError(f'Pumas returned HTTP {status} without a typed error body.') + raise ValueError(error_message(error)) + + def alias(value): if (not isinstance(value, str) or not value or value != value.strip() or len(value.encode('utf-8')) > 256 or any(ord(c) < 32 or ord(c) == 127 for c in value)): @@ -89,50 +181,69 @@ def configuration(body, *, image=False): return {'protocol': PROTOCOL, 'profile': profile(body['profile'])} +def _bound_number(value): + return type(value) in (int, float) and -1e308 <= value <= 1e308 + + +def _descriptor(item): + """Validate one descriptor Tuldok uses. Extra fields and enum values are ignored.""" + response_fields(item, ('capability', 'semantic_task', 'input_formats', 'output_formats', 'availability', 'option_bounds')) + name = item['capability'] + if item['semantic_task'] != USED_CAPABILITIES[name]: + raise ValueError('Pumas reports a different semantic task for '+name+' within contract version 1.') + for field in ('input_formats', 'output_formats'): + if type(item[field]) is not list or any(type(value) is not str for value in item[field]): + raise ValueError('Invalid typed Pumas '+field.replace('_', ' ')+'.') + availability = item['availability'] + if (type(availability) is not dict or type(availability.get('state')) is not str + or availability.get('reason') is not None and type(availability['reason']) is not str): + raise ValueError('Invalid typed Pumas availability.') + bounds = item['option_bounds'] + if type(bounds) is not list: + raise ValueError('Invalid typed Pumas option bounds.') + seen = set() + for bound in bounds: + if type(bound) is not dict or type(bound.get('option')) is not str or bound['option'] not in USED_OPTIONS: + continue + response_fields(bound, ('option', 'minimum', 'maximum')) + if bound['option'] in seen: + raise ValueError('Pumas advertises the '+bound['option']+' bound more than once.') + seen.add(bound['option']) + if not _bound_number(bound['minimum']) or not _bound_number(bound['maximum']) or bound['minimum'] > bound['maximum']: + raise ValueError('Invalid typed Pumas option interval.') + + def validate_manifest(raw): + """Validate the fields Tuldok uses from `GET /v1/capabilities`. + + Additive fields (for example `context_tokens`), unused capabilities such as + `multimodal_embedding` or `decision`, and unknown formats, reasons and option + names are accepted and ignored. The raw observation is retained unchanged. + """ value = strict_json(raw) - object_fields(value, ('supported_contract_versions', 'model', 'profile', 'max_request_bytes', - 'max_response_bytes', 'max_stream_event_bytes', 'capabilities')) + response_fields(value, ('supported_contract_versions', 'model', 'profile', 'max_request_bytes', + 'max_response_bytes', 'capabilities')) versions = value['supported_contract_versions'] - if type(versions) is not list or not 1 <= len(versions) <= 16 or any(type(v) is not int for v in versions) or 1 not in versions: + if type(versions) is not list or not any(type(version) is int and version == 1 for version in versions): raise ValueError('Pumas typed contract version 1 is unavailable.') alias(value['model']) if profile(value['profile']) is None: raise ValueError('Pumas must resolve an exact profile.') - for key, maximum in [('max_request_bytes', 32*1024*1024), ('max_response_bytes', 32*1024*1024), ('max_stream_event_bytes', 256*1024)]: - if type(value[key]) is not int or not 0 < value[key] <= maximum: + for key in ('max_request_bytes', 'max_response_bytes'): + if not _positive_int(value[key]): raise ValueError('Unsupported typed Pumas byte bounds.') entries = value['capabilities'] - if type(entries) is not list or len(entries) > len(CAPABILITIES): + if type(entries) is not list: raise ValueError('Invalid typed Pumas capability list.') seen = set() for item in entries: - object_fields(item, ('capability', 'semantic_task', 'input_formats', 'output_formats', 'streaming', 'availability', 'option_bounds')) - name = item['capability'] - if type(name) is not str or name not in CAPABILITIES or name in seen or item['semantic_task'] != CAPABILITIES[name] or type(item['streaming']) is not bool: - raise ValueError('Unsupported typed Pumas semantic capability.') + name = item.get('capability') if type(item) is dict else None + if type(name) is not str or name not in USED_CAPABILITIES: + continue + if name in seen: + raise ValueError('Pumas advertises '+name+' more than once for one serving.') seen.add(name) - for field, allowed in [('input_formats', {'messages_text', 'text', 'text_batch', 'pcm_s16le', 'pcm_f32le', 'png_base64', 'jpeg_base64', 'messages_image'}), - ('output_formats', {'text', 'embeddings_float32', 'png_base64', 'labels'})]: - items = item[field] - if type(items) is not list or not 1 <= len(items) <= len(allowed) or any(type(v) is not str or v not in allowed for v in items) or len(set(items)) != len(items): - raise ValueError('Unsupported typed Pumas format.') - availability = item['availability'] - object_fields(availability, ('state',), ('reason',)) - if availability not in [{'state': 'available'}] and (availability.get('state') != 'unavailable' or - type(availability.get('reason')) is not str or availability.get('reason') not in {'unsupported_adapter', 'unqualified_audio_runtime', 'unknown_model_task', 'model_task_mismatch', 'runtime_unavailable'}): - raise ValueError('Invalid typed Pumas availability.') - bounds = item['option_bounds'] - if type(bounds) is not list or len(bounds) > 13: - raise ValueError('Invalid typed Pumas option bounds.') - options = set() - for bound in bounds: - object_fields(bound, ('option', 'minimum', 'maximum')) - if type(bound['option']) is not str or bound['option'] not in {'max_tokens', 'temperature', 'top_p', 'dimensions', 'input_count', 'input_characters', 'width', 'height', 'seed', 'max_output_tokens', 'image_bytes', 'image_pixels', 'image_count'} or bound['option'] in options: - raise ValueError('Unsupported typed Pumas option.') - options.add(bound['option']) - if any(type(bound[k]) not in (int, float) or not -1e308 <= bound[k] <= 1e308 for k in ('minimum', 'maximum')) or bound['minimum'] > bound['maximum']: - raise ValueError('Invalid typed Pumas option interval.') + _descriptor(item) return {'capabilities': value, 'observed_sha256': hashlib.sha256(raw).hexdigest()} @@ -144,7 +255,8 @@ def capabilities(base, model, exact_profile=None): query['profile'] = exact_profile try: observed = gateway_discovery.get_json(parts.hostname, parts.port, '/v1/capabilities?' + urlencode(query), - time.monotonic()+3, max_response=MAX_CAPABILITIES, decode=validate_manifest, require_complete=True) + time.monotonic()+3, max_response=MAX_CAPABILITIES, decode=validate_manifest, require_complete=True, + error=http_error) except (ValueError, OSError, http.client.HTTPException) as error: raise OperationError('Selected-model capabilities unavailable: '+str(error)) from None manifest = observed['capabilities'] @@ -153,37 +265,270 @@ def capabilities(base, model, exact_profile=None): return observed +class _RouteMissing(ValueError): + pass + + +def _listing(aliases): + rows = [] + for name, profiles in aliases.items(): + required = len(profiles) > 1 + label = name + (' (profiles: ' + ', '.join(profiles) + '; choose one)' if required else '') + rows.append({'id': name, 'name': label, 'profiles': profiles, 'profile_required': required}) + return rows + + +def _served_models(raw): + """`GET /v1/served-models`: one entry per served (model, profile) pair.""" + value = strict_json(raw) + response_fields(value, ('models',)) + if type(value['models']) is not list: + raise ValueError('Invalid Pumas served-model listing.') + aliases = {} + for item in value['models']: + response_fields(item, ('model', 'profile')) + selected = profile(item['profile']) + if selected is None: + raise ValueError('Pumas served-model entry has no exact profile.') + profiles = aliases.setdefault(alias(item['model']), []) + if selected not in profiles: + profiles.append(selected) + return {'models': _listing(aliases), 'source': '/v1/served-models', + 'qualification': 'Served (model, profile) pairs. Inspect the selected model capabilities before admission.'} + + +def _openai_models(raw): + value = strict_json(raw) + entries = value.get('data') if type(value) is dict else None + if type(entries) is not list: + raise ValueError('Invalid bounded Pumas served model listing.') + aliases = {} + for item in entries: + if type(item) is not dict: + raise ValueError('Invalid model listing entry.') + profiles = aliases.setdefault(alias(item.get('id')), []) + # v0.8 lists each ID once with `profiles`; earlier gateways omit it. + listed = item.get('profiles') + for selected in listed if type(listed) is list else (): + if isinstance(selected, str) and re.fullmatch(r'[A-Za-z0-9_.-]{1,128}', selected) and selected not in profiles: + profiles.append(selected) + return {'models': _listing(aliases), 'source': '/v1/models', + 'qualification': 'Serving aliases only. Inspect the selected model capabilities before admission.'} + + def models(base): + """Prefer the typed served-model listing; fall back to `/v1/models` on gateways without it.""" parts = urlsplit(endpoint(base)) - def decode(raw): - value = strict_json(raw) - entries = value.get('data') if type(value) is dict else None - if type(entries) is not list or len(entries) > 512: - raise ValueError('Invalid bounded Pumas served model listing.') - names = [] - for item in entries: - if type(item) is not dict: - raise ValueError('Invalid model listing entry.') - name = alias(item.get('id')) - # PR54 emits one catalog row per served profile and omits profile - # in these rows. Exact-profile capabilities remain admission authority. - if name not in names: - names.append(name) - return {'models': [{'id': name, 'name': name} for name in names], - 'qualification': 'Serving aliases only. Inspect the selected model capabilities before admission.'} + def missing(status, raw): + if status == 404: + raise _RouteMissing() + http_error(status, raw) + try: + return gateway_discovery.get_json(parts.hostname, parts.port, '/v1/served-models', time.monotonic()+3, + max_response=MAX_SERVED_MODELS, decode=_served_models, require_complete=True, error=missing) + except _RouteMissing: + pass return gateway_discovery.get_json(parts.hostname, parts.port, '/v1/models', time.monotonic()+3, - max_response=1024*1024, decode=decode, require_complete=True) + max_response=1024*1024, decode=_openai_models, require_complete=True, error=http_error) + + +def _known(values, vocabulary, where): + """The known members of a declared list, in order; unknown strings are ignored.""" + if type(values) is not list or any(type(value) is not str for value in values): + raise ValueError(where + ' must be a list of strings.') + return [value for value in dict.fromkeys(values) if value in vocabulary] + + +def _text_field(value, where, limit): + if not isinstance(value, str) or not value.strip() or len(value) > limit: + raise ValueError(f'{where} must be a nonempty string of at most {limit} characters.') + return value -def require(observed, name, input_format, output_format, options): - item = next((row for row in observed['capabilities']['capabilities'] if row['capability'] == name), None) - if (item is None or item['availability']['state'] != 'available' or input_format not in item['input_formats'] - or output_format not in item['output_formats']): - raise OperationError('Selected model does not currently advertise the required '+name+' capability.') - bounds = {row['option']: row for row in item['option_bounds']} +def _served_descriptor(item, where): + """One release capability descriptor, reduced to the vocabulary Tuldok knows.""" + response_fields(item, ('capability', 'semantic_task', 'input_formats', 'output_formats', 'input_modalities', + 'output_modalities', 'streaming', 'availability', 'option_bounds')) + if type(item['semantic_task']) is not str or type(item['streaming']) is not bool: + raise ValueError(where + ' has an invalid semantic task or streaming flag.') + availability = item['availability'] + if type(availability) is not dict or type(availability.get('state')) is not str: + raise ValueError(where + ' has an invalid availability.') + reason = availability.get('reason') + if reason is not None and type(reason) is not str: + raise ValueError(where + ' has an invalid availability reason.') + bounds = {} + if type(item['option_bounds']) is not list: + raise ValueError(where + ' option bounds must be a list.') + for bound in item['option_bounds']: + if type(bound) is not dict or bound.get('option') not in OPTION_NAMES: + continue + response_fields(bound, ('option', 'minimum', 'maximum')) + if (bound['option'] in bounds or not _bound_number(bound['minimum']) or not _bound_number(bound['maximum']) + or bound['minimum'] > bound['maximum']): + raise ValueError(where + ' has an invalid or repeated ' + bound['option'] + ' bound.') + bounds[bound['option']] = {'minimum': bound['minimum'], 'maximum': bound['maximum']} + available = availability['state'] == 'available' + return {'capability': item['capability'], 'semantic_task': item['semantic_task'], + 'input_formats': _known(item['input_formats'], INPUT_FORMATS, where + ' input formats'), + 'output_formats': _known(item['output_formats'], OUTPUT_FORMATS, where + ' output formats'), + 'input_modalities': _known(item['input_modalities'], MODALITIES, where + ' input modalities'), + 'output_modalities': _known(item['output_modalities'], MODALITIES, where + ' output modalities'), + 'streaming': item['streaming'], + # An unknown reason is kept as display text, never interpreted. + 'availability': {'state': 'available'} if available else + {'state': 'unavailable', 'reason': message_text(reason) or None}, + 'option_bounds': bounds} + + +def _served_entry(item, where): + response_fields(item, ('model', 'profile', 'model_id', 'provider', 'load_state', 'profile_required', + 'modalities', 'capabilities')) + selected = profile(item['profile']) + if selected is None: + raise ValueError(where + ' has no exact profile.') + context = item.get('context_tokens') + if context is not None and (type(context) is not int or not 0 < context <= 4294967295): + raise ValueError(where + ' context_tokens must be a positive 32-bit integer.') + if type(item['profile_required']) is not bool: + raise ValueError(where + ' profile_required must be true or false.') + modalities = item['modalities'] + response_fields(modalities, ('input', 'output')) + if type(item['capabilities']) is not list: + raise ValueError(where + ' capabilities must be a list.') + capabilities = [] + for index, row in enumerate(item['capabilities']): + if type(row) is not dict or row.get('capability') not in CAPABILITIES: + continue + if any(seen['capability'] == row['capability'] for seen in capabilities): + raise ValueError(where + ' lists ' + row['capability'] + ' more than once.') + capabilities.append(_served_descriptor(row, f'{where} capability {index}')) + return {'model': alias(item['model']), 'profile': selected, 'profile_required': item['profile_required'], + 'model_id': _text_field(item['model_id'], where + ' model_id', 1024), + 'provider': _text_field(item['provider'], where + ' provider', 128), + 'load_state': _text_field(item['load_state'], where + ' load_state', 128), + 'context_tokens': context, + 'modalities': {'input': _known(modalities['input'], MODALITIES, where + ' input modalities'), + 'output': _known(modalities['output'], MODALITIES, where + ' output modalities')}, + 'capabilities': capabilities} + + +def decode_served_models(raw): + """Validate a `GET /v1/served-models` body into the served (model, profile) pairs. + + Refuses a body that is not contract version 1, lacks a required field or + has a malformed entry, naming the entry. Capabilities, formats, modalities + and option names outside the release vocabulary, and unknown fields, are + dropped without error. + """ + if len(raw) > MAX_SERVED_MODELS: + raise ValueError(f'The served-model listing exceeds {MAX_SERVED_MODELS} bytes.') + value = strict_json(raw) + response_fields(value, ('supported_contract_versions', 'max_request_bytes', 'max_response_bytes', 'models')) + versions = value['supported_contract_versions'] + if type(versions) is not list or not any(type(version) is int and version == 1 for version in versions): + raise ValueError('Pumas typed contract version 1 is unavailable.') + for key in ('max_request_bytes', 'max_response_bytes'): + if not _positive_int(value[key]): + raise ValueError('Unsupported typed Pumas byte bounds.') + if type(value['models']) is not list or len(value['models']) > MAX_SERVED_ENTRIES: + raise ValueError(f'The served-model listing must be a list of at most {MAX_SERVED_ENTRIES} entries.') + models, pairs = [], set() + for index, item in enumerate(value['models']): + try: + entry = _served_entry(item, f'Served model {index}') + except ValueError as error: + message = str(error) + raise ValueError(message if message.startswith('Served model') else f'Served model {index}: {message}') from None + if (entry['model'], entry['profile']) in pairs: + raise ValueError(f"Served model {index} repeats {entry['model']} with profile {entry['profile']}.") + pairs.add((entry['model'], entry['profile'])) + models.append(entry) + return {'max_request_bytes': value['max_request_bytes'], 'max_response_bytes': value['max_response_bytes'], + 'models': models} + + +def served_models(address, cancel_event=None): + """What a Pumas endpoint serves: each (model, profile) with its capabilities, from `GET /v1/served-models`. + + Independent of the catalog. Reads at most `MAX_SERVED_MODELS` bytes within + 15 seconds and never retries. Raises `OperationError` with the reason. + """ + base = ai_http.validate_url('llamacpp', address) + try: + with ai_http.request(base, '/v1/served-models', timeout=15, label='Pumas endpoint', + cancel_event=cancel_event) as (response, transport, deadline): + raw = bytearray() + while not response.isclosed(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError() + transport.settimeout(remaining) + chunk = response.read1(65536) + if not chunk: + break + raw.extend(chunk) + if len(raw) > MAX_SERVED_MODELS: + raise ValueError(f'The served-model listing exceeds {MAX_SERVED_MODELS} bytes.') + if response.length not in (None, 0): + raise ValueError('The served-model listing ended before its declared length.') + return decode_served_models(bytes(raw)) + except (CodexError, TimeoutError, OSError, http.client.HTTPException, ValueError) as error: + raise OperationError('Pumas served models unavailable: ' + (str(error) or 'the endpoint timed out.')) from None + + +def available_capabilities(served, input_kind, output): + """The capabilities of one served model that are available to make `output` from `input_kind`. + + `served` is one entry of `served_models()`. `input_kind` is a request + `input.kind`; `output` an output format of the release. Returns capability + names in the order the endpoint declared them. + """ + if input_kind not in INPUT_KINDS: + raise ValueError('Unknown input kind; use one of ' + ', '.join(INPUT_KINDS) + '.') + if output not in OUTPUT_FORMATS: + raise ValueError('Unknown output; use one of ' + ', '.join(OUTPUT_FORMATS) + '.') + names = [] + for row in served['capabilities']: + formats = INPUT_KINDS[input_kind] + # The release sends a `text` input to chat as one user message. + if input_kind == 'text' and row['capability'] == 'chat_generation': + formats = ('messages_text',) + if (row['availability']['state'] == 'available' and output in row['output_formats'] + and any(name in row['input_formats'] for name in formats)): + names.append(row['capability']) + return names + + +def descriptor(observed, name): + """Select a capability descriptor by name, never by position or count.""" + return next((row for row in observed['capabilities']['capabilities'] + if type(row) is dict and row.get('capability') == name), None) + + +def require(observed, name, input_format, output_format, options, advisory=()): + """Check live availability and bounds. `advisory` options are checked only when advertised.""" + item = descriptor(observed, name) + if item is None: + raise OperationError('Selected model does not advertise the '+name+' capability.') + availability = item['availability'] + if availability.get('state') != 'available': + reason = availability.get('reason') + raise OperationError('Selected model reports '+name+' unavailable' + + (' ('+message_text(reason)+')' if isinstance(reason, str) else '') + '.') + if input_format not in item['input_formats'] or output_format not in item['output_formats']: + raise OperationError('Selected model does not advertise '+input_format+' to '+output_format+' for '+name+'.') + bounds = {row['option']: row for row in item['option_bounds'] + if type(row) is dict and type(row.get('option')) is str and row['option'] in USED_OPTIONS} for key, value in options.items(): - if key not in bounds or not bounds[key]['minimum'] <= value <= bounds[key]['maximum']: - raise OperationError('Selected model does not advertise the requested '+key+' bound.') + bound = bounds.get(key) + if bound is None: + if key in advisory: + continue + raise OperationError('Selected model does not advertise a '+key+' bound for '+name+'.') + if not bound['minimum'] <= value <= bound['maximum']: + raise OperationError(f"Requested {key} {value} is outside the selected model's {name} bound " + f"{bound['minimum']:g}–{bound['maximum']:g}.") def chat_request(job, user_text, max_tokens): @@ -236,11 +581,10 @@ def watch(): try: value = result(raw, payload['request_id']) except OperationError as error: + # The typed error's `outcome` is authoritative; HTTP status is evidence only. error.provider_http_status = response.status - if error.provider_code and response.status not in (400, 413, 422, 502, 503): - error.outcome = 'unknown' raise - if value['kind'] != {'text': 'text', 'png_base64': 'image'}.get(payload['output']): + if value['kind'] != RESULT_KINDS.get(payload['output']): raise OperationError('Typed result has the wrong output kind; no replay.', 'unknown', raw=raw) if response.status != 200: raise OperationError('Unexpected typed HTTP status; provider outcome unknown, no replay.', 'unknown', raw=raw) @@ -259,35 +603,65 @@ def watch(): def result(raw, request_id): + """Decode a finite typed response. Unknown response fields are ignored.""" try: value = strict_json(raw) - object_fields(value, ('contract_version', 'request_id'), ('result', 'error')) - if type(value['contract_version']) is not int or value['contract_version'] != 1 or value['request_id'] != request_id or ('result' in value) == ('error' in value): - raise ValueError('Wrong typed operation correlation.') + if pre_handler(value): + error = typed_error(value) + if error is None: + raise ValueError('Invalid Pumas refusal.') + raise OperationError(error_message(error), error['outcome'], error['code'], raw) + if type(value) is not dict or type(value.get('contract_version')) is not int or value['contract_version'] != 1: + raise ValueError('Typed response is not contract version 1.') if 'error' in value: - error = value['error']; object_fields(error, ('code', 'outcome')) - if type(error['code']) is not str or error['code'] not in ERROR_CODES or error['outcome'] not in ('not_admitted', 'unknown'): - raise ValueError('Invalid typed operation error.') - raise OperationError('Pumas '+error['code']+'; outcome '+error['outcome']+'. No automatic replay.', error['outcome'], error['code'], raw) + error = typed_error(value) + if error is None: + detail = message_text(value['error'].get('message')) if type(value['error']) is dict else '' + raise ValueError('Invalid typed operation error' + (': ' + detail if detail else '.')) + # A null request_id means Pumas could not read the body; it is still a typed rejection. + if value.get('request_id') not in (request_id, None): + raise ValueError('Typed error belongs to another request.') + raise OperationError(error_message(error), error['outcome'], error['code'], raw) + response_fields(value, ('request_id', 'result')) + if value['request_id'] != request_id: + raise ValueError('Typed result belongs to another request.') projected = value['result'] if type(projected) is not dict: raise ValueError('Invalid typed result.') if projected.get('kind') == 'text': - object_fields(projected, ('kind', 'text', 'finish_reason')) - if not isinstance(projected['text'], str) or projected['finish_reason'] != 'stop': - raise ValueError('A complete stop-terminated text result is required.') + response_fields(projected, ('kind', 'text', 'finish_reason')) + if not isinstance(projected['text'], str): + raise ValueError('Invalid typed text result.') + if projected['finish_reason'] != 'stop': + raise ValueError('Text did not finish with stop; partial output is not applied.') projected['text'].encode('utf-8') + elif projected.get('kind') == 'decisions': + # `answers` maps each question to its typed answer; the caller reads the answers it asked for. + response_fields(projected, ('kind', 'answers')) + if type(projected['answers']) is not dict: + raise ValueError('Invalid typed decisions result.') + elif projected.get('kind') == 'embeddings': + # One finite vector per item asked, all of one length; the caller checks the count against its batch. + response_fields(projected, ('kind', 'vectors')) + vectors = projected['vectors'] + if type(vectors) is not list or not vectors or any(type(vector) is not list or not vector + for vector in vectors): + raise ValueError('Invalid typed embeddings result.') + if len({len(vector) for vector in vectors}) != 1 or not all( + type(value) in (int, float) and math.isfinite(value) for vector in vectors for value in vector): + raise ValueError('Typed embeddings must be finite vectors of one length.') elif projected.get('kind') == 'image': - object_fields(projected, ('kind', 'png_base64', 'seed')) + response_fields(projected, ('kind', 'png_base64', 'seed')) if not isinstance(projected['png_base64'], str) or type(projected['seed']) is not int or not 0 <= projected['seed'] <= 4294967295: raise ValueError('Invalid typed image result.') else: - raise ValueError('Unsupported typed result format.') + raise ValueError('Unsupported typed result kind.') return projected except OperationError: raise - except (ValueError, UnicodeError, TypeError): - raise OperationError('Malformed or incomplete typed result; provider outcome unknown, no replay.', 'unknown', raw=raw) from None + except (ValueError, UnicodeError, TypeError) as error: + raise OperationError('Malformed or incomplete typed result; provider outcome unknown, no replay. '+str(error), + 'unknown', raw=raw) from None def image_dimensions(width, height): diff --git a/pumas_vision.py b/pumas_vision.py index 65ba2f2f..81badc8a 100644 --- a/pumas_vision.py +++ b/pumas_vision.py @@ -1,4 +1,4 @@ -"""Finite Image→Text consumer pinned to combined Pumas PR68, never model qualification. +"""Finite Image→Text consumer of the Pumas 0.8.0 typed contract, never model qualification. The application sends its prepared image bytes, without URL/path authority. Pumas's production PNG/turbojpeg decoder remains the admission authority. @@ -17,7 +17,7 @@ import pumas_operations as pumas from workbench import encode -SOURCE_COMMIT = '36c2aab2c885b8416fd4e39cf8c37c510596d6e4' +SOURCE_COMMIT = pumas.SOURCE_COMMIT SCHEMA = 'pumas.model-operations.image-to-text' MAX_REQUEST = 32 * 1024 * 1024 MAX_IMAGE = 8 * 1024 * 1024 @@ -26,6 +26,10 @@ MAX_AXIS = 4096 MAX_PIXELS = 4194304 MAX_MESSAGES = MAX_PARTS = 128 +# Pumas has no fixed generation cap; the live max_tokens bound follows the served context. +MAX_TOKENS = 4294967295 +IMAGE_FACTS = ('image_count', 'image_bytes', 'image_pixels', 'width', 'height', 'total_image_bytes', + 'message_count', 'content_parts') def _container(data, encoding): @@ -142,7 +146,8 @@ def validate_input(value, *, facade=False): def request(job, input_value, *, facade=True, max_tokens=512, temperature=None, top_p=None): - if type(max_tokens) is not int or not 1 <= max_tokens <= 2048: raise ValueError('Tokens must be 1–2048.') + if type(max_tokens) is not int or not 1 <= max_tokens <= MAX_TOKENS: + raise ValueError('Tokens must be a positive integer; the selected model bounds the maximum.') options = {'kind': 'text_generation', 'max_tokens': max_tokens} for key, value, maximum in [('temperature', temperature, 2), ('top_p', top_p, 1)]: if value is not None: @@ -171,7 +176,13 @@ def prepare(job, payload, stop): payload['profile'] = observed['capabilities']['profile'] input_format, images = validate_input(payload['input'], facade='semantic_task' in payload) options = {key: value for key, value in payload['options'].items() if key != 'kind'} - pumas.require(observed, 'image_to_text', input_format, 'text', dict(options, image_count=len(images))) - for image in images: pumas.require(observed, 'image_to_text', input_format, 'text', image) + # Image facts are checked against advertised bounds; local preflight already bounds them. + facts = dict(options, image_count=len(images), total_image_bytes=sum(row['image_bytes'] for row in images)) + if payload['input']['kind'] != 'image': + messages = payload['input']['messages'] + facts.update(message_count=len(messages), content_parts=sum( + 1 if type(message['content']) is str else len(message['content']) for message in messages)) + pumas.require(observed, 'image_to_text', input_format, 'text', facts, advisory=IMAGE_FACTS) + for image in images: pumas.require(observed, 'image_to_text', input_format, 'text', image, advisory=IMAGE_FACTS) job.update(capability_observation=observed, build_observation=build, producer_contract_source=SOURCE_COMMIT) pumas.request_evidence(job, payload) diff --git a/retained_measurements.py b/retained_measurements.py deleted file mode 100644 index 4eefa6d5..00000000 --- a/retained_measurements.py +++ /dev/null @@ -1,479 +0,0 @@ -"""Finite retained scalar tables, exact source bindings and explicit missingness. - -This owner validates data representation and performs fixed selections/reductions. -Producers own metric definitions, comparison scope and scientific interpretation. -It reads no paths, runs no producer and grants no evidence/disclosure authority. -""" -import base64 -import binascii -import copy -from decimal import Decimal -import hashlib -import json -import math -import re -import struct - -from workbench import WorkbenchError - - -FORMAT = 'retained_measurements_v1' -MAX_BYTES = 32 * 1024 * 1024 -MAX_SOURCE_BYTES = 8 * 1024 * 1024 -MAX_SOURCE_TOTAL = 16 * 1024 * 1024 -MAX_RESPONSE_BYTES = 256 * 1024 -MAX_ROWS = 100000 -MAX_COMPARISON_SOURCES = 16 -MAX_COMPARISON_SUBJECTS = 256 -MAX_COMPARISON_BYTES = 128 * 1024 * 1024 -MAX_COMPARISON_ROWS = 250000 -STATES = ('measured', 'failed', 'unsupported', 'unavailable') -ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') -SHA = re.compile(r'^[a-f0-9]{64}$') - - -def fail(message, code='invalid', status=400): - raise WorkbenchError('Retained measurements: ' + message, code, status) - - -def exact(value, fields): - if type(value) is not dict or set(value) != set(fields): - fail('Object fields differ from schema 1.') - - -def text(value, maximum=240): - if (type(value) is not str or not value.strip() or len(value) > maximum - or any(ord(c) < 32 or 127 <= ord(c) <= 159 for c in value)): - fail('Bounded nonempty display text required.') - try: - value.encode('utf-8') - except UnicodeError: - fail('Text must contain valid Unicode scalars.') - return value - - -def identifier(value): - if type(value) is not str or not ID.fullmatch(value): - fail('Invalid declared identity.') - return value - - -def number(value): - return (type(value) is int and abs(value) <= 2**53-1 - or type(value) is float and math.isfinite(value)) - - -def encode(value): - return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False).encode('utf-8') - - -def digest(value): - return hashlib.sha256(encode(value)).hexdigest() - - -def decode(raw): - def pairs(items): - result = {} - for key, value in items: - if key in result: - fail('Duplicate JSON key.') - result[key] = value - return result - def floating(value): - result = float(value) - if not math.isfinite(result) or result == 0 and Decimal(value) != 0: - fail('JSON number exceeds finite binary64 range.') - return result - def constant(_value): - fail('Nonfinite JSON constants are unsupported.') - try: - value = json.loads(raw.decode('utf-8'), object_pairs_hook=pairs, - parse_float=floating, parse_constant=constant) - pending = [(value, 0)] - while pending: - item, depth = pending.pop() - if depth > 24: - fail('JSON nesting exceeds the supported bound.', 'too_large', 413) - if type(item) is dict: - pending.extend((v, depth+1) for v in item.values()) - for key in item: - key.encode('utf-8') - elif type(item) is list: - pending.extend((v, depth+1) for v in item) - elif type(item) is str: - item.encode('utf-8') - return value - except (ValueError, UnicodeError, RecursionError): - fail('Unique-key finite UTF-8 JSON required.') - - -def collection(values, maximum): - if type(values) is not list or not values: - fail('Nonempty declared collection required.') - if len(values) > maximum: - fail('Declared collection exceeds its supported bound.', 'too_large', 413) - result = {} - for value in values: - if type(value) is not dict: - fail('Declared entries must be objects.') - identity = identifier(value.get('id')) - if identity in result: - fail('Duplicate declared identity.') - result[identity] = value - return result - - -def cell(kind, value): - if value is None: - return True - if kind == 'string': - return type(value) is str and len(value) <= 256 - if kind == 'boolean': - return type(value) is bool - if kind == 'integer': - return type(value) is int and abs(value) <= 2**53-1 - return kind == 'number' and number(value) - - -class MeasurementTable: - """Own an immutable input snapshot; returned projections are independent.""" - def __init__(self, raw): - if type(raw) is not bytes or not 0 < len(raw) <= MAX_BYTES: - fail('A bounded complete byte snapshot is required.', 'too_large', 413) - self.sha256 = hashlib.sha256(raw).hexdigest() - self.bytes = len(raw) - doc = decode(raw) - exact(doc, ('format', 'schema_version', 'title', 'scope', 'dimensions', 'subjects', - 'cases', 'metrics', 'observations', 'sources')) - if doc['format'] != FORMAT or type(doc['schema_version']) is not int or doc['schema_version'] != 1: - fail('Unsupported measurement schema.', 'unsupported', 409) - text(doc['title']) - self._doc = doc - self._sources = collection(doc['sources'], 256) - self._json = {} - total = 0 - for identity, source in self._sources.items(): - exact(source, ('id', 'label', 'kind', 'bytes', 'sha256', 'data_base64')) - text(source['label']) - if source['kind'] not in ('json', 'opaque'): - fail('Unsupported retained source kind.', 'unsupported', 409) - if type(source['bytes']) is not int or not 0 < source['bytes'] <= MAX_SOURCE_BYTES: - fail('Source byte declaration exceeds its bound.', 'too_large', 413) - total += source['bytes'] - if total > MAX_SOURCE_TOTAL: - fail('Source collection exceeds its decoded byte bound.', 'too_large', 413) - if type(source['sha256']) is not str or not SHA.fullmatch(source['sha256']): - fail('Source requires an exact SHA256.') - value = source['data_base64'] - if type(value) is not str or len(value) != 4*((source['bytes']+2)//3): - fail('Source encoding differs from its byte declaration.') - try: - payload = base64.b64decode(value, validate=True) - except (ValueError, binascii.Error): - fail('Invalid source base64.') - if (len(payload) != source['bytes'] or hashlib.sha256(payload).hexdigest() != source['sha256'] - or base64.b64encode(payload).decode('ascii') != value): - fail('Retained source bytes differ from their declaration.') - if source['kind'] == 'json': - self._json[identity] = decode(payload) - scope = doc['scope'] - exact(scope, ('id', 'label', 'population', 'definition')) - identifier(scope['id']);text(scope['label']) - if scope['population'] not in ('training', 'consumed', 'independent', 'reserved', 'unknown'): - fail('Unsupported declared population.', 'unsupported', 409) - self._definition(scope, ('id', 'label', 'population')) - if type(doc['dimensions']) is not list or len(doc['dimensions']) > 32: - fail('Invalid dimension collection.') - self._dimensions = collection(doc['dimensions'], 32) if doc['dimensions'] else {} - for dimension in self._dimensions.values(): - exact(dimension, ('id', 'label', 'type', 'units', 'scope')) - text(dimension['label']) - if dimension['scope'] not in ('subject', 'case', 'observation'): - fail('Unsupported dimension owner.', 'unsupported', 409) - if dimension['type'] not in ('string', 'number', 'integer', 'boolean'): - fail('Unsupported dimension type.', 'unsupported', 409) - if dimension['units'] is not None: - text(dimension['units'], 120) - self._subjects = collection(doc['subjects'], MAX_ROWS) - self._cases = collection(doc['cases'], 4096) - self._metrics = collection(doc['metrics'], 32) - expected = len(self._subjects)*len(self._cases)*len(self._metrics) - if expected > MAX_ROWS: - fail('Complete subject/case/metric matrix exceeds its row bound.', 'too_large', 413) - for subject in self._subjects.values(): - exact(subject, ('id', 'label', 'dimensions', 'reference'));text(subject['label']);self._reference(subject['reference']) - self._dimension_values('subject',subject['dimensions']) - for case in self._cases.values(): - exact(case, ('id', 'dimensions', 'reference')) - self._dimension_values('case',case['dimensions']) - self._reference(case['reference']) - for metric in self._metrics.values(): - exact(metric, ('id', 'label', 'units', 'definition'));text(metric['label']);text(metric['units'], 120) - self._definition(metric, ('id', 'label', 'units')) - observations = doc['observations'] - if type(observations) is not list or len(observations) != expected: - fail('Every subject/case/metric needs one explicit result, including missing or failed results.') - self._observations = collection(observations, MAX_ROWS) - pairs = set() - for observation in observations: - exact(observation, ('id', 'subject_id', 'case_id', 'metric_id', 'state', 'value', 'reason', 'dimensions', 'reference')) - self._dimension_values('observation',observation['dimensions']) - identity = observation['subject_id'], observation['case_id'], observation['metric_id'] - for value in identity: - identifier(value) - if (identity in pairs or identity[0] not in self._subjects or identity[1] not in self._cases - or identity[2] not in self._metrics): - fail('Duplicate or unresolved measurement identity.') - pairs.add(identity) - if observation['state'] not in STATES: - fail('Unsupported observation state.', 'unsupported', 409) - source_value = self._reference(observation['reference']) - if observation['state'] == 'measured': - value = observation['value'] - if (not number(value) or not number(source_value) or observation['reason'] is not None - or struct.pack('!d', float(value)) != struct.pack('!d', float(source_value))): - fail('Measured scalar differs from its exact retained source value.') - else: - if observation['value'] is not None: - fail('Unmeasured results require an explicit null value.') - text(observation['reason'], 1000) - - def _dimension_values(self,scope,values): - columns={k:d for k,d in self._dimensions.items() if d['scope']==scope} - if type(values) is not dict or set(values)!=set(columns): - fail('Dimension values differ from their declared owner and columns.') - if any(not cell(columns[k]['type'],v) for k,v in values.items()): - fail('Dimension value differs from its declared type.') - - def _row_dimensions(self,row): - return {**self._subjects[row['subject_id']]['dimensions'], - **self._cases[row['case_id']]['dimensions'],**row['dimensions']} - - def _reference(self, reference, *, hashed=False): - exact(reference, ('source_id', 'pointer', 'sha256') if hashed else ('source_id', 'pointer')) - source = self._sources.get(identifier(reference['source_id'])) - if source is None: - fail('Source reference is unresolved.') - pointer = reference['pointer'] - if pointer is None and not hashed: - return None - if (type(pointer) is not str or len(pointer) > 512 or pointer and not pointer.startswith('/') - or re.search(r'~(?:[^01]|$)', pointer) or source['kind'] != 'json'): - fail('A bounded existing JSON pointer is required.') - value = self._json[source['id']] - try: - for part in pointer[1:].split('/') if pointer else (): - part = part.replace('~1', '/').replace('~0', '~') - if type(value) is dict: - value = value[part] - elif type(value) is list and re.fullmatch(r'0|[1-9][0-9]*', part): - value = value[int(part)] - else: - fail('Source pointer does not identify a retained value.') - except (KeyError, IndexError): - fail('Source pointer does not identify a retained value.') - if hashed and reference['sha256'] != digest(value): - fail('Declared definition digest differs from its retained source.') - return value - - def _definition(self, descriptor, fields): - value = self._reference(descriptor['definition'], hashed=True) - if type(value) is not dict or len(encode(value)) > 16384 or any(value.get(k) != descriptor[k] for k in fields): - fail('Definition metadata differs from its exact retained document.') - return value - - def metadata(self): - return copy.deepcopy({'format': FORMAT, 'packet_sha256': self.sha256, 'title': self._doc['title'], - 'scope': self._doc['scope'], 'scope_definition':self._definition(self._doc['scope'],('id','label','population')), - 'dimensions': self._doc['dimensions'], 'metrics': self._doc['metrics'], - 'subject_count':len(self._subjects),'case_count': len(self._cases), - 'observation_count': len(self._observations),'source_count':len(self._sources)}) - - def subjects(self, *, offset=0, limit=40): - if type(offset) is not int or not 0<=offset<=2**53-1 or type(limit) is not int or not 1<=limit<=40: - fail('Invalid subject page bounds.') - return copy.deepcopy({'items':list(self._subjects.values())[offset:offset+limit], - 'offset':offset,'limit':limit,'total':len(self._subjects)}) - - def query(self, metric_id, *, subject_ids=None, filters=None, offset=0, limit=50): - identifier(metric_id) - if metric_id not in self._metrics: - fail('Metric is unavailable.', 'not_found', 404) - selected = list(self._subjects) if subject_ids is None else subject_ids - if (type(selected) is not list or not selected or len(selected)>MAX_ROWS or any(type(v) is not str for v in selected) - or len(set(selected)) != len(selected) or any(v not in self._subjects for v in selected)): - fail('Choose existing unique subjects.') - filters = {} if filters is None else filters - if (type(filters) is not dict or any(k not in self._dimensions or not cell(self._dimensions[k]['type'],v) - for k,v in filters.items())): - fail('Filters must use declared dimensions and typed scalar values.') - if type(offset) is not int or not 0<=offset<=2**53-1 or type(limit) is not int or not 1<=limit<=100: - fail('Page bounds must be explicit finite integers.') - # One pass over observations; set membership and per-subject grouping - # replace the former subjects x rows rescans. Output is unchanged: - # rows keep packet order and ties keep the first maximal observation. - chosen = set(selected) - rows = [] - for v in self._doc['observations']: - if v['metric_id']!=metric_id or v['subject_id'] not in chosen: - continue - if filters: - dimensions = self._row_dimensions(v) - if not all(dimensions[k]==wanted for k,wanted in filters.items()): - continue - rows.append(v) - grouped = {identity:{'count':0,'states':{state:0 for state in STATES},'maximum':None} for identity in selected} - for v in rows: - own = grouped[v['subject_id']] - own['count'] += 1 - if v['state'] in own['states']: - own['states'][v['state']] += 1 - if v['state']=='measured' and (own['maximum'] is None or v['value'] > own['maximum']['value']): - own['maximum'] = v - summaries = [] - for identity in selected: - own = grouped[identity];maximum = own['maximum'] - summaries.append({'subject_id':identity,'count':own['count'],'states':own['states'], - 'maximum':None if maximum is None else {k:maximum[k] for k in ('id','case_id','value')}}) - result = {'format':'retained_measurement_query_v1','packet_sha256':self.sha256, - 'scope':copy.deepcopy(self._doc['scope']),'metric':copy.deepcopy(self._metrics[metric_id]), - 'scope_definition':copy.deepcopy(self._definition(self._doc['scope'],('id','label','population'))), - 'definition':copy.deepcopy(self._definition(self._metrics[metric_id],('id','label','units'))), - 'filters':copy.deepcopy(filters),'subject_ids':list(selected),'summary':summaries, - 'rows':[dict(copy.deepcopy(v),dimensions=copy.deepcopy(self._row_dimensions(v))) - for v in rows[offset:offset+limit]], - 'total':len(rows),'offset':offset,'limit':limit,'sampling':'none'} - if len(encode(result)) > MAX_RESPONSE_BYTES: - fail('Selected response exceeds its bound; choose a smaller page.', 'too_large', 413) - return result - - def inspect(self, observation_id): - identifier(observation_id) - observation = self._observations.get(observation_id) - if observation is None: - fail('Observation is unavailable.', 'not_found', 404) - reference = observation['reference'];source = self._sources[reference['source_id']] - return copy.deepcopy({'packet_sha256':self.sha256,'observation':observation, - 'case':self._cases[observation['case_id']],'subject':self._subjects[observation['subject_id']], - 'metric':self._metrics[observation['metric_id']],'scope':self._doc['scope'], - 'scope_definition':self._definition(self._doc['scope'],('id','label','population')), - 'case_evidence':self._inspection_reference(self._cases[observation['case_id']]['reference']), - 'scalar_evidence':self._inspection_reference(reference), - 'source':{k:v for k,v in source.items() if k!='data_base64'}}) - - def _inspection_reference(self,reference): - value=self._reference(reference) - state='opaque' if reference['pointer'] is None else 'included' - if state=='included' and len(encode(value))>16384: - state='exceeds_inspection_bound';value=None - return {'reference':copy.deepcopy(reference),'source_sha256':self._sources[reference['source_id']]['sha256'], - 'state':state,'value':value} - - -class MeasurementComparison: - """One bounded scientific comparison over immutable, source-bound tables. - - The native custody owner supplies validated locator/table pairs. Compatibility - depends on original definitions and actual case evidence, not display names. - Returned rows keep their packet locator; no observations or counts are pooled - across subjects. Pagination affects rows only, never the maximum denominator. - """ - def __init__(self, tables): - if type(tables) is not list or not 1 <= len(tables) <= MAX_COMPARISON_SOURCES: - fail('Choose 1 through 16 comparison sources.', 'too_large', 413) - self._tables = [] - self._subjects = {} - seen = set() - basis = None - observations = raw_bytes = 0 - for locator, table in tables: - exact(locator, ('origin_catalog_id', 'packet_sha256')) - if (type(locator['origin_catalog_id']) is not str - or not re.fullmatch('[a-f0-9]{32}', locator['origin_catalog_id']) - or not isinstance(table, MeasurementTable) or locator['packet_sha256'] != table.sha256): - fail('Comparison source differs from its validated packet.') - key = (locator['origin_catalog_id'], locator['packet_sha256']) - if key in seen: - fail('Comparison repeats a source.') - seen.add(key) - raw_bytes += table.bytes - if raw_bytes > MAX_COMPARISON_BYTES: - fail('Comparison exceeds 128 MiB of original packets.', 'too_large', 413) - metadata = table.metadata() - cases = [] - for case in table._cases.values(): - reference = case['reference'] - evidence = table._reference(reference) - cases.append(dict(id=case['id'], dimensions=case['dimensions'], evidence=evidence, - opaque_source_sha256=table._sources[reference['source_id']]['sha256'] - if reference['pointer'] is None else None)) - current = dict(scope=metadata['scope_definition'], dimensions=metadata['dimensions'], - metrics=[table._definition(metric, ('id','label','units')) for metric in table._metrics.values()], - cases=cases) - # Canonical source bytes distinguish Boolean/numeric values too; - # ordinary Python equality is not the scientific identity relation. - identity = digest(current) - if basis is None: - basis = identity - self._header = dict(format='retained_measurement_comparison_v1', - scope=metadata['scope'], scope_definition=metadata['scope_definition'], - dimensions=metadata['dimensions'], metrics=metadata['metrics'], - definition_source=copy.deepcopy(locator), case_population_sha256=digest(cases), - case_count=metadata['case_count']) - elif identity != basis: - fail('Comparison sources have different scope, metric, dimension or original case definitions.', 'incompatible', 409) - for subject in table._subjects.values(): - if subject['id'] in self._subjects: - fail('Comparison sources overlap a subject identity.', 'conflict', 409) - self._subjects[subject['id']] = dict(copy.deepcopy(subject), source=copy.deepcopy(locator)) - observations += metadata['observation_count'] - if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_COMPARISON_ROWS: - fail('Comparison exceeds 256 subjects or 250,000 observations.', 'too_large', 413) - self._tables.append((copy.deepcopy(locator), table)) - self._header.update(subjects=list(self._subjects.values()), observation_count=observations) - - def read(self, metric_id=None, *, subject_ids=None, filters=None, offset=0, limit=50): - result = copy.deepcopy(self._header) - result['query'] = None - if metric_id is None: - if subject_ids is not None or filters is not None or offset != 0 or limit != 50: - fail('A comparison selection requires an explicit metric.') - return self._bounded(result) - selected = list(self._subjects) if subject_ids is None else subject_ids - if (type(selected) is not list or not selected - or any(type(value) is not str or value not in self._subjects for value in selected) - or len(set(selected)) != len(selected)): - fail('Choose existing unique comparison subjects.') - if type(offset) is not int or not 0 <= offset <= 2**53-1 or type(limit) is not int or not 1 <= limit <= 100: - fail('Comparison page bounds must be finite integers.') - summaries = {} - rows = [] - total = 0 - template = None - for locator, table in self._tables: - own = [identity for identity in selected if identity in table._subjects] - if not own: - continue - page = table.query(metric_id, subject_ids=own, filters=filters, offset=0, limit=1) - if template is None: - template = page - for summary in page['summary']: - summaries[summary['subject_id']] = dict(summary, source=copy.deepcopy(locator)) - start = max(0, offset-total) - if total+page['total'] > offset and len(rows) < limit: - current = table.query(metric_id, subject_ids=own, filters=filters, - offset=start, limit=limit-len(rows)) - rows.extend(dict(row, source=copy.deepcopy(locator)) for row in current['rows']) - total += page['total'] - metric = next(item for item in self._header['metrics'] if item['id'] == metric_id) - result['query'] = dict(metric=copy.deepcopy(metric), definition=template['definition'], - filters=template['filters'], subject_ids=list(selected), - summary=[summaries[identity] for identity in selected], rows=rows, - total=total, offset=offset, limit=limit, sampling='none') - return self._bounded(result) - - @staticmethod - def _bounded(value): - if len(encode(value)) > MAX_RESPONSE_BYTES: - fail('Comparison response exceeds its bound; choose fewer sources or a smaller page.', 'too_large', 413) - return value diff --git a/scripts/generate_project_files_contracts.py b/scripts/generate_project_files_contracts.py deleted file mode 100644 index ac373e53..00000000 --- a/scripts/generate_project_files_contracts.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Derive installed browser pins/routes from the local canonical API registry.""" -import argparse -import json -from pathlib import Path -import re -import sys - -ROOT=Path(__file__).resolve().parents[1] -sys.path.insert(0,str(ROOT)) -import agent_api - - -def render(original): - names=sorted(name for name in agent_api._CONTRACTS if name.startswith('project_files.')) - tables={ - 'CONTRACTS':{name:agent_api.contract(name,1)['sha256'] for name in names}, - 'VERSIONS':{name:{str(v):agent_api.contract(name,v)['sha256'] for v in sorted(agent_api._CONTRACTS[name])} - for name in names if len(agent_api._CONTRACTS[name])>1}, - 'ROUTES':{name:{'method':agent_api.operation(name)['method'],'path':agent_api.operation(name)['path'], - 'mutation':agent_api.operation(name)['side_effects']} - for name in sorted(agent_api.PROJECT_FILES_M2|agent_api.PROJECT_FILES_M3|agent_api.PROJECT_FILES_M4)}} - for label,value in tables.items(): - pattern=r'(/\* PROJECT_FILES_'+label+r'_START \*/\s*)(.*?)(\s*/\* PROJECT_FILES_'+label+r'_END \*/)' - generated=json.dumps(value,sort_keys=True,indent=2) - original,count=re.subn(pattern,lambda m:m[1]+generated+m[3],original,flags=re.S) - if count!=1:raise ValueError('Exactly one installed '+label+' table required') - return original - - -if __name__=='__main__': - parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--check',action='store_true') - args=parser.parse_args();path=ROOT/'static/project-files.js';original=path.read_text();expected=render(original) - if args.check: - if original!=expected:raise SystemExit('Installed project-file browser contracts are stale') - elif original!=expected:path.write_text(expected) diff --git a/static/project-files.css b/static/project-files.css deleted file mode 100644 index c18e2e30..00000000 --- a/static/project-files.css +++ /dev/null @@ -1,22 +0,0 @@ -:root{font-family:system-ui,sans-serif;color:#1c2929;background:#f3f5ef;font-size:16px;line-height:1.5}*{box-sizing:border-box}body{margin:0}main{max-width:1200px;margin:auto;padding:2rem 1.5rem 4rem}header{max-width:800px;margin-bottom:2rem}h1{font-size:2.5rem;line-height:1.15;letter-spacing:-.04em}h2{margin-top:0}.eyebrow{font-size:.8rem;text-transform:uppercase;letter-spacing:.12em;color:#375957}section{padding:1.5rem;background:#fff;border:1px solid #c6d0c7;border-radius:12px}label{display:block;font-weight:600;margin:.75rem 0 .3rem}input,textarea,select,button{font:inherit}input,textarea,select{display:block;width:100%;margin-top:.3rem;padding:.6rem;border:1px solid #7d9288;border-radius:5px;color:inherit;background:#fff}textarea{resize:vertical}button{padding:.6rem .9rem;border:1px solid #315d52;border-radius:5px;background:#edf3ee;color:#183c32;cursor:pointer}button:hover:enabled{background:#dceae0}button:disabled{opacity:.55;cursor:default}button[aria-current=true]{background:#204f43;color:white}.actions{display:flex;flex-wrap:wrap;gap:.6rem;margin:1rem 0}.project-location{display:grid;grid-template-columns:1fr 2fr 2fr;gap:1rem}.workspace-grid{display:grid;grid-template-columns:1fr 1.2fr;gap:1.2rem;margin-top:1.5rem}.help{font-size:.9rem;color:#43564f;margin-top:.3rem}.scope{padding:1rem;border-left:4px solid #527467}fieldset{border:0;margin:0;padding:0}legend{font-weight:600}#records-list{padding:0;list-style:none}#records-list li{margin:.5rem 0}#records-list button{width:100%;text-align:left}#project-files-error{border:2px solid #9c2f29;background:#fff0eb;padding:1rem;color:#76241f}#project-files-status{min-height:1.5rem}#project-files-summary,#project-files-receipt,#project-files-error{overflow-wrap:anywhere}#record-history{padding-left:1.5rem}#record-history pre{white-space:pre-wrap;overflow-wrap:anywhere}#record-heading{scroll-margin-top:1rem}:focus-visible{outline:3px solid #b35208;outline-offset:3px}.skip-link{position:absolute;left:1rem;top:-5rem;padding:.7rem;background:white;z-index:3}.skip-link:focus{top:1rem}.session-bar{padding:.65rem 1.5rem;border-bottom:1px solid #c6d0c7;background:#fff}@media(max-width:760px){.workspace-grid,.project-location{grid-template-columns:1fr}main{padding:1.2rem .8rem}h1{font-size:2rem}section{padding:1rem}} - -.project-history-panel{margin-top:1.5rem}.project-fact-list{list-style:none;padding:0;max-height:42rem;overflow:auto}.project-fact-list>li{padding:.65rem 0;border-bottom:1px solid #dce4de}.project-fact-list button{width:100%;text-align:left}.project-facts{overflow-wrap:anywhere}.project-facts dl{margin:.4rem 0 .8rem;border-left:2px solid #dce4de;padding-left:.7rem}.project-facts dt{font-weight:600;text-transform:capitalize}.project-facts dd{margin:0 0 .6rem;white-space:pre-wrap;max-height:18rem;overflow:auto}.project-facts details{margin:.5rem 0}.project-facts summary{cursor:pointer;font-weight:600}.project-history-panel form{margin:1rem 0}.project-history-panel legend{padding-top:.5rem} -/* History rows share the detail viewer's hash wrapping and intrinsic width. */ -#project-detail-history-list{overflow-wrap:anywhere} -.check-row{display:flex;gap:.6rem;align-items:flex-start}.check-row input{width:auto;flex:none;margin-top:.35rem} - -#project-working-summary,#project-save-as-summary,#project-fork-summary,#project-working-list,#project-working-history-list,#project-fork-list,#project-intents-list,#project-intent-detail{overflow-wrap:anywhere;min-width:0}#project-intent-detail{white-space:pre-wrap;max-height:32rem;overflow:auto}#project-working-text{white-space:pre;overflow:auto} - -#project-runtime-files,#project-runtime-summary,#project-runtime-pending,#project-runtime-layout,#project-runtime-status{overflow-wrap:anywhere;min-width:0}#project-runtime-layout,#project-runtime-status{white-space:pre-wrap;max-height:32rem;overflow:auto} - -#project-candidate-preparation-result,#project-candidate-preparation-status{overflow-wrap:anywhere;min-width:0}#project-candidate-preparation-result{white-space:pre-wrap;max-height:32rem;overflow:auto} - -#project-portfolio-source,#project-portfolio-detail,#project-portfolio-count{overflow-wrap:anywhere;min-width:0} -#project-portfolio-detail p{white-space:pre-wrap;max-width:85ch} -#project-portfolio-detail pre{white-space:pre-wrap;overflow-wrap:anywhere;max-height:32rem;overflow:auto} -#project-portfolio-detail details{margin:1rem 0} -#project-portfolio-detail h4{margin-bottom:.25rem} -#project-portfolio-detail h3{scroll-margin-top:1rem} -#project-portfolio-filter-form{display:grid;grid-template-columns:1fr 2fr 1fr;gap:.7rem;align-items:end} -#project-portfolio-filter-form button{justify-self:start} -@media(max-width:760px){#project-portfolio-filter-form{grid-template-columns:1fr}} diff --git a/static/project-files.html b/static/project-files.html deleted file mode 100644 index 52f1ccca..00000000 --- a/static/project-files.html +++ /dev/null @@ -1,347 +0,0 @@ -<!doctype html> -<html lang="en"> -<head> - <meta charset="utf-8"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <title>Projects · Tuldok - - - - - - - - - - - - - - - - - - - - - -
-

Tuldok / external projects

Project workspace

-

Create or open a project folder to work with data, models and experiment history.

-
-

Reading project status…

- - -
-

Project file

-

No project status available.

-
- - - -
-

Use a relative directory within this root. New creates a new directory; Open reads an existing project. Close the current project before opening another.

-
- - - - - - -
-

No unsubmitted edits.

-
- -
-

Prepare a Candidate package or verifier

-

Choose a current Study, Candidate and captured CPU runtime. Preparation saves an inert Trial and exact sources; approval and Run admission remain separate actions.

-
Explicit Candidate preparation - - - - - - - -


-      
Resolve this installation's pending preparation -

These actions require the original current local custody marker. Imported intent or completion history does not authorize reconciliation. Neither action deletes staging files or starts a Run.

- - - - -
-
-
-

Capture a current CPU runtime

-

An operator configures the source aliases on this installation. Inspect the selected source, review its exact inventory, then explicitly capture it. These controls copy bytes without executing Python, installing packages or granting permission to run a project.

- - - -

    -
    Capture reviewed inventory - - - -

    - -
    Abandon this installation's stopped partial capture -

    This requires its original current local custody marker. The decision preserves every partial file, retained intent and scientific fact; imported history does not authorize it.

    - - - -
    -
    Inspect retained capture facts - -

    This outcome is historical. “Captured” does not establish that a current local runtime is present or ready. Reopening a project does not capture or provision it.

    
    -    
    -
    -

    Save As an independent project

    -

    Use a new directory. Historical results keep their original identities; editable definitions receive new identities. Changed working files and unresolved current work must be resolved first.

    -
    - Fork destination - - - -

    -
    -
    -

    Scripts and notes

    -

    Inspect files already adopted into this project, or explicitly adopt exact current UTF-8 bytes. Captured scripts are displayed as text and never run by these controls.

    -
    -
      -
      Adopt current file bytes - - - - - -

      This picker calculates a digest in your browser. Adoption reads the existing file at the relative project path; it does not upload or replace it.

      -
      -
      Adopt a missing note as a deletion -

      Inspect and select a known missing note first. This action records its absence and keeps all captured revisions.

      - - -
      -

      Select a captured file revision

      -

      - -
        -
        -
        -

        Fork origins and bindings

        -
        -

            -

            Select an unresolved binding

            Review its original parameters and binding fields below. Recreate a current definition with the named model or graph controls; this history never rebinds or grants execution.

            -
            
            -    
            -

            Authored edits are saved to the project before acknowledgement. Historical materials and artifacts remain inspectable when payloads are missing. Execution requires explicit current local permission through available controls. Independent Save As and explicit working-file adoption are available when their negotiated controls are enabled. Migration remains unavailable.

            -
            -
            -

            Author text

            -
            - New text record - - - -

            Comma-separated group IDs; keep at least one. These preserve source grouping.

            - - -
            -

            Text records

            -
              - -
              -
              -

              Select a text record

              -

              - -

              Source text is immutable. Submit a new record to author different text.

              -
              - Annotation and human review - - - -

              Enter one classification label, up to 80 characters.

              - - - -
              -

              Saved annotation history

              -
                - -
                -
                - - -
                -

                Project data and scientific history

                -

                Inspect saved records, models, Studies, Runs and their exact artifact availability. Opening a project never starts computation or restores permission to disclose held-out data.

                -
                - - - -
                - -
                -

                -
                -
                  -
                  -

                  Choose a saved record

                  - -
                  - -
                  -
                    - - - - - - - -
                    -
                    -
                    -
                    -

                    Add data and saved intent

                    -
                    -
                    Import an image or audio source - - - - - - -
                    -
                    Save the entered data search -

                    Save the search text and data kind entered above. This records intent; it does not freeze a dataset.

                    - -
                    -
                    -
                    -
                    -

                    Create a model definition

                    -

                    Choose an available template from this installation. Saving a definition and publishing its source material never start training.

                    - -
                    New model - -
                    -
                    -
                    -

                    Plan a Study

                    -

                    Save scientific selection criteria and finite budget metadata. Continue and admit prepared Trials explicitly through current Study execution controls when available.

                    -
                    New Study - - -

                    Use the exact saved metric definition from retained evidence. This does not create or infer a metric.

                    -
                    - -
                    - -
                    -
                    -
                    -

                    Import declared scientific data

                    -

                    Choose a local source manifest, then one declared data reference. Tuldok reads the exact existing bytes without executing project code.

                    -
                    Register a local source -

                    The local root selected at the top of this page applies.

                    - -
                    -
                    Import a declared reference - - -
                    -
                    -
                    -

                    Current local execution

                    -

                    Review the exact current recipe before granting permission. Approvals apply to this installation and actor. Saved historical admissions and approvals never restart execution when a project is opened.

                    -
                    -

                    -
                    Review current recipe - -
                    -
                    Approve reviewed material locally - - - - -
                    -
                    Start explicitly approved execution

                    No current local approval selected.

                    -
                    Cancel a current local Run
                    -
                    -
                    -

                    Current native CPU execution

                    -

                    These explicit owner actions use this installation's operator-selected Python environment. Installed packages remain operator trust; a successful probe does not establish model quality or a complete software closure. Opening saved history never probes or starts a Run.

                    - -
                    Validate with a current native probe - - - -
                    -
                    Start the validated native Run - - -
                    -
                    Retry a current native attempt - - - -
                    -
                    Reconcile an original current phase - - -
                    - -

                    Use “Refresh current custody” above to inspect original local phase IDs. Select a phase below, then explicitly reconcile it.

                      -

                      -
                      -
                      -

                      Current Study execution

                      -

                      Select a Study in the scientific history view, then load its current epoch and prepared Trials. Closing an epoch saves cancellation intent; it does not prove that its child processes have ceased.

                      -

                      -
                      Open a current epoch
                      - -
                      Approve this Trial locally
                      -
                      Admit a Trial under the current budget
                      -
                      Close the current epoch
                      -
                      -
                      -

                      Current model producers

                      -

                      These actions contact the model endpoint you choose now. Captured requests and results remain historical facts; opening them never contacts a model or resumes a queue.

                      -

                      -
                      Start an explicit model request - -

                      Select an image or text record in saved data first.

                      - - - - - - - - - -
                      -

                      -
                      Continue a current image queue
                      -
                      Review a caption or class proposal
                      -
                      Review a grounded candidate
                      -
                      Cancel this current local producer
                      -
                      -
                      - - diff --git a/static/project-files.js b/static/project-files.js deleted file mode 100644 index 3146719e..00000000 --- a/static/project-files.js +++ /dev/null @@ -1,1956 +0,0 @@ -'use strict'; -// One page owns this projection. The server owns project state and every edit. -(()=>{ - const $=id=>document.getElementById(id), base='/api/workspace/project-files'; - const actions=['new','open','save','close','query','get','import_text','annotate']; - const tasks=['text_classification','text_entities','text_corpus']; - // Generated from agent_api; freshness is checked by the client contract tests. - const pins = /* PROJECT_FILES_CONTRACTS_START */ -{ - "project_files.annotate": "70feae2ad14c49c4ae95b5bb69b4a50fbf93191cde840544cf89b6cbe8dc5c58", - "project_files.artifact_resolve": "0df477e294b4e0729a707d7dfb6038414fc354e1ae50155eaacfa82b3a55f51f", - "project_files.artifact_status": "8dd0343f5f882c96cf023f8274734ca57175ae986e82b3bb1188cb5864784834", - "project_files.artifacts": "20a4031d189058721d09b8bf694ea9b1583eea410afd381f58fe6c70d0a0c339", - "project_files.candidate": "c519ff04bc06d5e538d36b7f278862d8f282d0e7b97dd7f262444c384a50f854", - "project_files.candidate_select": "f043be6923760a208ca50d02a1ff4b60eadf09d48a034acdb18749806a537080", - "project_files.close": "95d0099ec2a378a1bbb65676799dd3baae2f36b838301f7aa165b116fba5edf6", - "project_files.compose": "b1dcbbd4ad7ebe17b17e11238f99a93408da24d0d411527e187900cc0f089105", - "project_files.data_annotate": "289270f5cb946765c5ced22fbbd18765bb55743145014c81cf54d7a6ba3025ea", - "project_files.data_availability": "92332c04973a4fa45d47e384717152ebc4a9ca08e7eb487669688ccd2ed42cbb", - "project_files.data_get": "0d19505995cbd8c12e3938cd89cf05955a2a63488a220711d29dd24b27c0c8c8", - "project_files.data_history": "b9e9e13389befe5f531d33d4d4d893649cb076f3eff98aa6ee85a4c46f03148a", - "project_files.data_import_audio": "51b511a068bed6ec024bd70bf08a48a23a94906362d54e0f3eb1223e325c237e", - "project_files.data_import_image": "e9f86ce62d29d267090c7955e0deb084f6ad6a936d1418baf5d807aa6d84ea3e", - "project_files.data_import_mesh": "840d6e1d6ab326d144b63e275d60d304277b5d4867c9c170c6dde5e117b46107", - "project_files.data_import_pointcloud": "c171b1342a5844fc1862947e20ae675395b8c3e24b44bff1179e0e8e6df70f53", - "project_files.data_import_scientific": "a39ce46384bb2f0c3cdb9a6bbf881316ce166a26208fc41b4a88bb307af32d63", - "project_files.data_import_sequence": "da8b91c9fd76d951fddc33acf6cca56287e0d7a29319b46032629b777af1f1ee", - "project_files.data_import_text": "c2db90c749154cb5489e13494d5a4404fdaf5a14895679fe2075fdfdc39e899d", - "project_files.data_import_volume": "c6e509d797970afd7da9e2069c29736e290ada8e7034326b31e1cb707aa572e2", - "project_files.data_preference_delete": "900538c7d8f74f04ed93387a864c8235b4162902345f6864b8d325527f63da5f", - "project_files.data_preference_history": "5ee7d60ce5e4aa09ea3496e9212b3404820d619cc2d47425faa94a3506d9ad07", - "project_files.data_preference_save": "eacdaa8b32cac556f8a591c4ca3f1191d83182623bd425a5619e1c6920c17782", - "project_files.data_preferences": "35b9bff0cba8bf453d446a8b62a6d003cc49f6abb8a12b55bfb76a112c3f2067", - "project_files.data_query": "0a7585ed5c41ed28eee2505e95773741a7af82c74b5265b8ed4ebf2beb72e394", - "project_files.data_response_history": "1bd46a162efae7213ff7c8a7b77dfe843e952a629fd7c24af3b5298e04140224", - "project_files.data_response_save": "fa5b92afe529520fe8206d629bd88085a68f4f9b399cd40dd05fd3948d3592f0", - "project_files.data_responses": "3e0c009244cad04fca4356cf3647bfd2b42cfc0d6a2ac31bb8de25cdb3c60ec8", - "project_files.data_rights": "ead16cca68b03533daa81bf06148e1918ce40dbc823ee060bd9908581f788653", - "project_files.data_scientific": "c954a680440fdb822a0f895f7e14b90e05dca82535478b1271dfb33cfbfdada9", - "project_files.data_source_delete": "457618e618cca92a71b668ffb9f68b0db87e51c417c2e5f41dcde76d71a23985", - "project_files.data_source_save": "0d50c45d445b2ce1014a696c3bc95965f0b548ded939f4a650f985f34e1ce849", - "project_files.definitions": "3516fcf8764382780c7da699218a05ff1f811634656cc81722ef547f21159698", - "project_files.dependencies": "615e173c6e7cb811dc5e51ef6c368975b4dd06de14d6cf69fbd8c4bb01e3f9e4", - "project_files.evidence": "440d7dba608d633a4a42479018360ef2fa6609a47148e7edfdf5246f84c020ef", - "project_files.execution_approve_external": "b35cec1a8bc5faad9e437ee04b6fb3bc9d575e8b7c4dd0444401fa8f551fa301", - "project_files.execution_cancel": "5fff1a68bfaeefcf426c573f4f128e76c96163171f31580922be3178080ffcd0", - "project_files.execution_generation_resume": "cfadef988f4510fcea1e64e1a8a4ca60794bec26a2d1c116b35e678971cc33ae", - "project_files.execution_grounded_review": "de971104eebe46acb67edfb77f1c0df98e8302528ef446ad51c772c13e67458b", - "project_files.execution_health": "c3b1d0017f5a49cf69c3a4c5c33e1d3db9846404d12627563d97ef40de7404a5", - "project_files.execution_preflight_external": "f754d712ac7da823375408fbcec63f7006c972a6e0885d69bae6772f2b850fea", - "project_files.execution_producer_cancel": "af114f7ed941f80ec565577863a6fffc4ec1c7d7069af6715350e8489526fd61", - "project_files.execution_producer_decide": "38bd7d1a1a052d62f6fc74cc16fcaf9e3c243f97dc3c39e38cdef03ca272a5d8", - "project_files.execution_producer_health": "297c6160a07cef60a8d4a50a690805f30a75b7e1bf6543e6da3a7befcb133b7e", - "project_files.execution_producer_start": "e7ee70a06346026980ba0fbf808b6d61ff3e7ac57a6619d72cd1b8b77e9662f7", - "project_files.execution_reconcile_current": "5ed3f998f053fd6333804e604f758c58e1b862c6f92f4de4272495916cef77de", - "project_files.execution_study_abandon_preparation": "650e7f85ea3ff476448422077ecc51fc06e270e975967886d5a6c726aee5af73", - "project_files.execution_study_admit": "381cacfa7582db321520cbc0d0e0be6bd187e1e3ff98cdbde371244b5994adba", - "project_files.execution_study_approve": "0a49e0d1b8e789a96969b73565f127d5cbb2c6cc39fea40fff33029f2ddbd7e2", - "project_files.execution_study_continue": "d421a59a95fb2a6b8d4e65d652000a13177054d34bafff3d0a3e30eeae00fb75", - "project_files.execution_study_prepare_package": "6687819b4cad1fa89ca1ec874422ea04f52cbc3685691dc3c612f122933c89b2", - "project_files.execution_study_prepare_verifier": "1c28024ab764c1f5747f17b90e73a95fb1666be4f94af040f376ec6241e12421", - "project_files.execution_study_reconcile_preparation": "c5a301fd6e3159de684f5700d557ccd13dd704e617d4779015e9b5ca93d76075", - "project_files.execution_study_stop": "84d103019c80b8e7bd3b59e0259c3947de4f4f48cc05c3076cf16112d8671911", - "project_files.execution_submit_external": "fc33d4eac346c1f08423ccb5d593db43b561124629d53f2a8896ff9838db99f3", - "project_files.expose_holdout": "d69478ed54267ee1f1966f423f412a32f90b95a55997c6dc7d8348e3cdd78fd7", - "project_files.file_adopt": "a190326d5c54296f8359be594996908fa28acbbd125168821c3fa1f520ef9b07", - "project_files.file_adopt_deletion": "7b6d661856a614d1b83d8011d1ffc7f9e2619853495faf9250888739f880685c", - "project_files.file_get": "a22c27e2fd890ffcb479e0aa8ea40a0cb32d06d1e5750b970b40f27915fe4d24", - "project_files.file_history": "c83477f7b0f8616c96ad79cebaf0eb3887fe289567f5d5ceb1d1502bebed7423", - "project_files.files_adopt": "62215260bf40bfa5db2521f908413bb6dfe47656fb8ea30b38039d637e5daf51", - "project_files.files_inspect": "0c2ff3ed93109ea19b5c92ffa24ac8f67e4bcfa3fe386921e984b5dfa3bfe7eb", - "project_files.fork_info": "958bc1aa269b5459c7e2d9c6bcb6725c06f7055502d73c49a95330cc4065aea4", - "project_files.get": "6b0f4e16a00aa119b86cbb5493be0bac945ff08a5e63773a8f2c2f9296658d5b", - "project_files.graph_clone": "8f9b460530f0f136900734345fc338a1993ffdaddf5f4d90faf352c39f397514", - "project_files.graph_create": "8fcb6d573b849ac03b5ee0ae192f274c11b9b5303451410ee46013f4cb0d83dc", - "project_files.graph_create_external": "93d88731f66062c9d0ac15d38897e731e25e39df7f9b141e5f3277f2d07b4205", - "project_files.graph_get": "77eea46e63758f0ca172cd23595cf5a42fe9820dbc82f3aefb7f5b6c22dc23d9", - "project_files.graph_save": "5a9745e4041cda6be46f63b83cb1f89d102d10c8810e41261a326cf0ef6821f4", - "project_files.graphs": "4892c990da89bc193d408537761e2e2db19694840ee652e64cba64a3bf57dbdb", - "project_files.import_text": "0f168320abb858892a2724b0b59862475bec653d1670ce754d61f2d82c912538", - "project_files.lineage": "750c173b42b113b6b82542fe4d82f8652cf07e1efe5cafde4c3e776d6afc3683", - "project_files.lineage_record": "16fd6cbd20bcd3743e4ac889dc6c398e714152f57d3c65cbd013f047c4267708", - "project_files.material_intent": "79d1ee6c3d8bc1313f2e51ff1d7d4ea489f5f3e82a6fd2e00ed93e1f7613bfbd", - "project_files.material_intent_capabilities": "86e90e2d0f094beb1c277a680da9a35e28d78c827390ac1c4e51fc3003c1ce32", - "project_files.material_intents": "6431fd2780d706c77d2c4dd82f15794017368fd3bade28ac62bbdff5c8b0c547", - "project_files.materials": "92a6a6a1750fed89aef9574ab6e82be2c6c6351bfa89fbfcb1f3866d4ea3cb13", - "project_files.measurement_compare": "abbb5e406a3c95155e90f08e8a071f330425190b25275f0f6512de08b8afb929", - "project_files.measurement_import": "c8e500396138fc60a31fc916c441bb95454f410231a6508609a7ebba5cd31f18", - "project_files.measurement_inspect": "fcbe9846ca49c3da4713b54a3cb4a8237189d8eea5799f275d33992ef268914f", - "project_files.measurement_metadata": "5a92ed5259f8876a3c5b8fd1886a9e0f44f0aea8e53d7f63902ac765ddac0de7", - "project_files.measurement_query": "65b1456a6870dd6d4b7499669777a2c66157142ccc27cac1864a995664220e1c", - "project_files.measurement_sources": "7805e9c5a6b2e2ab566ca134c18697116c89e08a1991b5192b0b5c152a34041d", - "project_files.measurement_subjects": "37bd3f717c7158e34b9c221bfa1dbce79490b9d55248803d13dab2ea17b35ea5", - "project_files.model_create": "34cf790fb2a3355ccab7c3a6c7174ced8dae861812d9479582fd8d5a9212745c", - "project_files.model_get": "95cc198ab6f084b596324f7251803ad4670a9f8e786e1a9639f1b9ed041798c3", - "project_files.model_material": "46afde2b06e0e5b08565f12ec86d2d7ec43ef6b63d97c86c264fc9c2f451f20b", - "project_files.model_publish": "97c0ec706fd663d322764a2dd72bd920e7e7ac8defaee2d23624502ed239fc62", - "project_files.model_save": "a90f8d985578191dee672824c9178559452f8b0dd759f74ed91337a96ca670e8", - "project_files.models": "92e13621050d7f229a4785ed6845df2f191a9e037bcdf377fa7fb0ba796aebe3", - "project_files.native_preflight": "494c907dac56a3126d26f064ec20bb980f918bded8e7f727c1fdaaa118059b13", - "project_files.native_reconcile": "74299207459fbd08d23ad7e8c1db205a4297f014415fd463c9857ba60dda4c7e", - "project_files.native_retry": "489655b49dadc184d95d6455ad2662bd5e7fd2cde2170fd303a92b2de77471c1", - "project_files.native_submit": "82468f35f04306fd4f39a0cc94a487fc05ae739e28f24f0be49ed35644639643", - "project_files.new": "672373f8ddc0b89fea15165798aab97cf4ac2cdf678a04d96839512cc8976b6f", - "project_files.open": "fd4b9f97c03cd7050981cfafd1177bff8c86351bb603b8f7980aaa149d2b0368", - "project_files.package_state": "6f0369eafab3b8ed1db71ffb12e7d5a2bbcf400b46315d46ba1e2fb13e63baf7", - "project_files.policies": "629b61ae9bd9c65d04cd3e89fb50fb62dd455b9f75bcaf9db08b7890bb5516f5", - "project_files.policy": "5bdb86600f2ec387fde721b22fdd0afa3ed47660126847c2908c1bb9de0421cb", - "project_files.producer": "6187a9eb1e32cb3bcf13d283492b1ad909c32753aa5ed97fbc12a9708523d9e3", - "project_files.producers": "65b2f4dff355c3bf5a023bf4f9dc116e302eda85ad90e6afaa2f21f0aa0667b5", - "project_files.progression": "4b70964ce2509f3526b2915cb5bd1ec086f7f553d82d8cf43c87d524cc3db7f3", - "project_files.progression_origins": "380b9ae9931faf5f161bcbd0e689dd93525be22ae2d9b00a524ce48d377d8c95", - "project_files.publication_result": "10b4a5f3d4f37343322327024ddbedf5b6b69672c785e208ec48534d7b40c867", - "project_files.query": "9a8b0d3a99adf5d7d53b62261dfac83e8b3ea44c10cb2f6c7a9c3d79422fef08", - "project_files.recipe_create": "53a6760f03675d4d8a8d3a0ffd5b56689975667bbc136b1edf8a206307c18651", - "project_files.recipe_get": "23837c51032b7c7ae6a365616f66cab9ae0d191108027291ad49898b2c421e2a", - "project_files.recipe_material": "c384acbf2333d7dd360991d72c8e1e218dc3ad3291af76e09f76efe25ea69cc0", - "project_files.recipe_prepare": "e91344f3032bf3e6abf50e5f5399252439498bce999c1f732322fd70a5eda1cf", - "project_files.recipe_save": "7fb9708aaba280ae437ee2554ecdaef3f724f3a7f4e55b49e54e1ef186691a7e", - "project_files.recipes": "3217c2033f60242c661c6689d6f9f634ea9d3a9b188a9465edcdc0bea572bc6c", - "project_files.release_freeze": "93a1b5b3c6bf11f63c92571113ce536743e7fd07b0b77ae8aee6e909230245df", - "project_files.release_preview": "0b6e2f0c2d7e3503fa29391fc20d27f058d0d3ad61e4547aee9845633ba6c5d8", - "project_files.retained_analysis": "05959a0e49a623318d6d70ded4f86df5df5d9bf29c411fcfcd1d9c0148b650cc", - "project_files.retained_analysis_export": "fa5d97e094484da053a1374131040deefdac429ec8360d671f57b12353d69359", - "project_files.run": "3cd94d46587596c36da4d1187b4648442cf6f216d2972daa496b7879e294ddec", - "project_files.run_document_segment": "bd6599cb89b106ba0be4141fc5d38ad5e07b2c57c5b24e1fcf91802ba9ff9006", - "project_files.runs": "cab46b11c5376ba641a717ceb1ce2bbe3693b7ac264e41192849f8522eb5d61d", - "project_files.runtime_abandon": "21086c2a92f23a8d42085678a7b99a69897bacb5e84341d482b0d2e9d6b69df8", - "project_files.runtime_capture": "2d787e49e7f911820d4341b33b4738219fa773a783bb2fb5a674d05dbfbb173a", - "project_files.runtime_inspect": "ece28c002151d36b9a415d4b7692536318765d1a7f742e3fd1f60c53b3a28204", - "project_files.runtime_material": "da48c927442878373434e93bb97fc65fc7056bd8e5a4b97e8a18627eb768cb07", - "project_files.runtime_sources": "6b94cbb5c712fbbea5645cbb64f8fd8b7f72c28042ddeacd96aaaad828a81460", - "project_files.runtime_status": "80931340bab620ca52c05277bc0380c97b7e9191a808312caedabc30f4ab7e96", - "project_files.save": "38572e376db36f81582dc6419fded35ff9d7c9185f5af2cdaafd74c94790f90a", - "project_files.save_as": "2b0558049ca0055a9682bcda6eb81ce9f48358b7cf804b5d499067a5594e2d33", - "project_files.search_create": "ae5c51158d849cda199cc007c42ea7a1cb62cd15a7ca8957bd32df813548e942", - "project_files.search_delete": "518037dbe9464fd327df53f7801a3c1ba30bb822467c091a2bc684542a58495c", - "project_files.search_rename": "78ca2aed2cd34c278ee1a9858dc76ebf8d39111de36bd48ada43f0407af5ff70", - "project_files.searches": "061be81df13158c18c2a124419da07db9564b44c239c8ca80cbf65a7ad26c1bd", - "project_files.selection_create": "87f60aeb83a497e7c6adab524f3ddc4776a6820497aa676f50ab3e1104b62ba2", - "project_files.selection_delete": "cb021e6e1d7b4df1db1e4ca614067c3b26820737eca92dbc7168b3ff45b0894f", - "project_files.selection_get": "c99e320815d54b60d0ddb3179a0ee62fa5eca938ac71011813296ce93bb085b0", - "project_files.selection_rename": "7e3fe9853cc593e4c0464162b3c9137208500650a125a46704f1e38d49b6f5a1", - "project_files.selections": "fd9cc6a94144a349232e99aa1ad4890cb7cb4b023de8bf4e8249374aa29ec39d", - "project_files.snapshot": "1eacdfeb7843bc432d62c6a94890dc07f618074faecb5e22589df9c9377e5030", - "project_files.source_references": "36a6d5b97efd63268d393b456e3bfec45cc696335acf652eff503b64a4ca0869", - "project_files.source_register": "04390ea427cb9958f5a0a1118de5cf8f91434d12e9c241621cf11465d804c64e", - "project_files.status": "1d0c4b64caaab8369d4addefd72e53cc8c7d0be4c07729fa04b358febe643b6a", - "project_files.studies": "5c45e0938cef51c271d1e82b28e878bc831f5a9272773440e39022074e6e87b6", - "project_files.study": "b268a20042efe9c8bc260f74309688dc7bbbd60486d30a33621a68e1a92fa3d7", - "project_files.study_create": "8d2aac7c073a6a7b3ebb41227b5b466cf03606625404f93a59b325038a4ad7a1", - "project_files.study_save": "5bf3fb814588fd7d9c35836e9d964a64481b3cc4169caf76cad6ce776ff0fe15", - "project_files.trial": "79afa4bed6d0907f3d4116cc04cb784e94e5a24ab89725ec24f5d7bce70e869f", - "project_files.trial_prepare": "e1f6973536fd4c356c4a92fc674d6766f7df0d8a113b8ee71bb05d0adc40b04f" -} -/* PROJECT_FILES_CONTRACTS_END */; - // Generated fixed operation routes from agent_api; remote schemas are never executed. - const versionPins = /* PROJECT_FILES_VERSIONS_START */ -{ - "project_files.annotate": { - "1": "70feae2ad14c49c4ae95b5bb69b4a50fbf93191cde840544cf89b6cbe8dc5c58", - "2": "2ae39dd621161500061bc8f57bb465e90fed04cfaf921e0dc368f82bd50cd107" - }, - "project_files.execution_health": { - "1": "c3b1d0017f5a49cf69c3a4c5c33e1d3db9846404d12627563d97ef40de7404a5", - "2": "6b7a834881da36960c0a0ee393c0c26e0d918b857d30791e8e5af79c2ae4a73e" - }, - "project_files.get": { - "1": "6b0f4e16a00aa119b86cbb5493be0bac945ff08a5e63773a8f2c2f9296658d5b", - "2": "fdc0858fc03a0866bb743b04251ea5f9a3c1e6be1b211366db7e0652166736ea" - }, - "project_files.measurement_compare": { - "1": "abbb5e406a3c95155e90f08e8a071f330425190b25275f0f6512de08b8afb929", - "2": "d2de9763a9109d0e4599f50e038a3fda0e6bf7c9c60684434825d0108019bc6f" - } -} -/* PROJECT_FILES_VERSIONS_END */; - let selectedVersions=new Map(); - const selectedHeader=operation=>operation+'/'+selectedVersions.get(operation)+'/'+(versionPins[operation]?.[selectedVersions.get(operation)]??pins[operation]); - const m2Operations = /* PROJECT_FILES_ROUTES_START */ -{ - "project_files.artifact_resolve": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/artifacts/resolve" - }, - "project_files.artifact_status": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/artifacts/status" - }, - "project_files.artifacts": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/artifacts" - }, - "project_files.candidate": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/candidates/{candidate_id}" - }, - "project_files.candidate_select": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/studies/{study_id}/candidates" - }, - "project_files.compose": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/compose" - }, - "project_files.data_annotate": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/{record_id}" - }, - "project_files.data_availability": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/availability" - }, - "project_files.data_get": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}" - }, - "project_files.data_history": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/history" - }, - "project_files.data_import_audio": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-audio" - }, - "project_files.data_import_image": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-image" - }, - "project_files.data_import_mesh": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-mesh" - }, - "project_files.data_import_pointcloud": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-pointcloud" - }, - "project_files.data_import_scientific": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-scientific" - }, - "project_files.data_import_sequence": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-sequence" - }, - "project_files.data_import_text": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-text" - }, - "project_files.data_import_volume": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-volume" - }, - "project_files.data_preference_delete": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/preferences/delete" - }, - "project_files.data_preference_history": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/preference-history" - }, - "project_files.data_preference_save": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/preferences" - }, - "project_files.data_preferences": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/preferences" - }, - "project_files.data_query": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data" - }, - "project_files.data_response_history": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/response-history" - }, - "project_files.data_response_save": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/responses" - }, - "project_files.data_responses": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/responses" - }, - "project_files.data_rights": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/{record_id}/rights" - }, - "project_files.data_scientific": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/scientific" - }, - "project_files.data_source_delete": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/{record_id}/source/delete" - }, - "project_files.data_source_save": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/{record_id}/source" - }, - "project_files.definitions": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/definitions" - }, - "project_files.dependencies": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/dependencies" - }, - "project_files.evidence": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/evidence" - }, - "project_files.execution_approve_external": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/approvals" - }, - "project_files.execution_cancel": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/runs/{run_id}/cancel" - }, - "project_files.execution_generation_resume": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/generation/{producer_id}/resume" - }, - "project_files.execution_grounded_review": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/grounded/{producer_id}/review" - }, - "project_files.execution_health": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/execution" - }, - "project_files.execution_preflight_external": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/execution/preflight" - }, - "project_files.execution_producer_cancel": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/producers/{producer_id}/cancel" - }, - "project_files.execution_producer_decide": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/producers/{producer_id}/decide" - }, - "project_files.execution_producer_health": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/execution/producers/health" - }, - "project_files.execution_producer_start": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/producers" - }, - "project_files.execution_reconcile_current": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/reconcile-current" - }, - "project_files.execution_study_abandon_preparation": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/abandon" - }, - "project_files.execution_study_admit": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/runs" - }, - "project_files.execution_study_approve": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/approvals" - }, - "project_files.execution_study_continue": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/continue" - }, - "project_files.execution_study_prepare_package": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/packages" - }, - "project_files.execution_study_prepare_verifier": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/verifiers" - }, - "project_files.execution_study_reconcile_preparation": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/reconcile" - }, - "project_files.execution_study_stop": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/stop" - }, - "project_files.execution_submit_external": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/runs" - }, - "project_files.expose_holdout": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/runs/{run_id}/disclose" - }, - "project_files.file_adopt": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/files/adopt" - }, - "project_files.file_adopt_deletion": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/files/adopt-deletion" - }, - "project_files.file_get": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/files/{file_id}" - }, - "project_files.file_history": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/files/{file_id}/history" - }, - "project_files.files_adopt": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/files/adopt-many" - }, - "project_files.files_inspect": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/files" - }, - "project_files.fork_info": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/fork" - }, - "project_files.graph_clone": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/graphs/{graph_id}/clone" - }, - "project_files.graph_create": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/graphs" - }, - "project_files.graph_create_external": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/graphs/external" - }, - "project_files.graph_get": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/graphs/{graph_id}" - }, - "project_files.graph_save": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/graphs/{graph_id}" - }, - "project_files.graphs": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/graphs" - }, - "project_files.lineage": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/lineage" - }, - "project_files.lineage_record": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/lineage" - }, - "project_files.material_intent": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/material-intents/{family}/{definition_id}" - }, - "project_files.material_intent_capabilities": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/material-intents/capabilities" - }, - "project_files.material_intents": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/material-intents" - }, - "project_files.materials": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/materials" - }, - "project_files.measurement_compare": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/measurements/compare" - }, - "project_files.measurement_import": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/measurements/import" - }, - "project_files.measurement_inspect": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/measurements/{packet_sha256}/observations/{observation_id}" - }, - "project_files.measurement_metadata": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/measurements/{packet_sha256}" - }, - "project_files.measurement_query": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/measurements/{packet_sha256}/query" - }, - "project_files.measurement_sources": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/measurements" - }, - "project_files.measurement_subjects": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/measurements/{packet_sha256}/subjects" - }, - "project_files.model_create": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/models" - }, - "project_files.model_get": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/models/{model_id}" - }, - "project_files.model_material": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/materials/model" - }, - "project_files.model_publish": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/models/{model_id}/publish" - }, - "project_files.model_save": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/models/{model_id}" - }, - "project_files.models": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/models" - }, - "project_files.native_preflight": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/native/preflight" - }, - "project_files.native_reconcile": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/native/phases/{phase_id}/reconcile" - }, - "project_files.native_retry": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/native/runs/{run_id}/retry" - }, - "project_files.native_submit": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/native/runs" - }, - "project_files.package_state": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/candidates/{candidate_id}/package-state" - }, - "project_files.policies": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/policies" - }, - "project_files.policy": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/policies/get" - }, - "project_files.producer": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/producers/{origin_catalog_id}/{producer_id}" - }, - "project_files.producers": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/producers" - }, - "project_files.progression": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/progression" - }, - "project_files.progression_origins": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/progression/origins" - }, - "project_files.publication_result": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/publication-result" - }, - "project_files.recipe_create": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/recipes" - }, - "project_files.recipe_get": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/recipes/{recipe_id}" - }, - "project_files.recipe_material": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/materials/recipe" - }, - "project_files.recipe_prepare": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/recipes/{graph_id}/prepare" - }, - "project_files.recipe_save": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/recipes/{recipe_id}" - }, - "project_files.recipes": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/recipes" - }, - "project_files.release_freeze": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/releases/freeze" - }, - "project_files.release_preview": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/releases/preview" - }, - "project_files.retained_analysis": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/analysis" - }, - "project_files.retained_analysis_export": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/analysis/export" - }, - "project_files.run": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}" - }, - "project_files.run_document_segment": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/document-segment" - }, - "project_files.runs": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs" - }, - "project_files.runtime_abandon": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/runtime/abandon" - }, - "project_files.runtime_capture": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/runtime/capture" - }, - "project_files.runtime_inspect": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runtime/inspect" - }, - "project_files.runtime_material": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/materials/runtime" - }, - "project_files.runtime_sources": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runtime/sources" - }, - "project_files.runtime_status": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runtime/status" - }, - "project_files.save_as": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/save-as" - }, - "project_files.search_create": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/searches" - }, - "project_files.search_delete": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/searches/{search_id}/delete" - }, - "project_files.search_rename": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/searches/{search_id}" - }, - "project_files.searches": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/searches" - }, - "project_files.selection_create": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/selections" - }, - "project_files.selection_delete": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/selections/{selection_id}/delete" - }, - "project_files.selection_get": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/selections/{selection_id}" - }, - "project_files.selection_rename": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/selections/{selection_id}" - }, - "project_files.selections": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/selections" - }, - "project_files.snapshot": { - "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/materials/snapshot" - }, - "project_files.source_references": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/sources/{project_id}/references" - }, - "project_files.source_register": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/sources/register" - }, - "project_files.studies": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/studies" - }, - "project_files.study": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/studies/{study_id}" - }, - "project_files.study_create": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/studies" - }, - "project_files.study_save": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/studies/{study_id}" - }, - "project_files.trial": { - "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/trials/{trial_id}" - }, - "project_files.trial_prepare": { - "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/studies/{study_id}/trials" - } -} -/* PROJECT_FILES_ROUTES_END */; - let selectedContracts=new Set(); - let state=null, selected=null, history=[], summaries=[], nextRecord=null, nextHistory=null, epoch=0, busy=false, blocked=false; - let importDirty=false, annotationDirty=false, m2Dirty=false, pending=null; - let registeredSource=null,modelTemplates=[],templateBinding=null; - let runtimeInspection=null,runtimePending=null,runtimeSourceOffset=0,runtimeSourceTotal=0,runtimeFileOffset=0,runtimeFileTotal=0; - let forkPending=null,workingSelection=null,workingDeletion=null,workingOffset=0,workingTotal=0,workingHistoryOffset=0,workingHistoryTotal=0,intentsOffset=0,intentsTotal=0,forkOffset=0,forkTotal=0,intentCapsOffset=0,intentCapsTotal=0; - let nativeGraphs=[],nativePrepared=null,nativePending=null,nativeOffset=0,nativeTotal=0; - let executionGraphs=[],executionPrepared=null,executionApproval=null,executionBinding=null,executionOffset=0,executionTotal=0,executionStudy=null,executionTrials=[],studyApproval=null,executionGraphOffset=0,executionGraphTotal=0,studyTrialOffset=0,studyTrialTotal=0,producerSource=null,producerCurrent=null,producerHealthOffset=0,producerHealthTotal=0; - let explored=null,exploreOffset=0,exploreTotal=0,historyOffset=0,historyTotal=0; - const m2DirtyForms=new Set(); - const m2PagedActions=new Set(['data_query','data_history','data_responses','data_response_history', - 'data_preferences','data_preference_history','models','graphs','recipes','studies','runs', - 'lineage','policies','artifacts','materials','selections','searches','producers','source_references','execution_producer_health','files_inspect','file_history','material_intents','material_intent_capabilities','runtime_sources']); - const m2HistoryActions=new Set(['data_history','data_response_history','data_preference_history']); - const dirty=()=>importDirty||annotationDirty||m2Dirty; - const checkpoint=()=>state?.project?.checkpoint??null; - let scientificEpoch=null; - const projectScientific=window.ScientificFieldInspector({ - root:$('scientific-inspection'), - isCurrent:record=>scientificEpoch===epoch&&!blocked&&state?.state==='open'&&explored?.category==='data_query'&&explored.id===record.id&&explored.value.content_hash===record.content_hash&&explored.checkpoint===checkpoint(), - async readSlice(record,selection,signal){ - const result=await m2Request('data_scientific',{record_id:record.id,content_hash:record.content_hash,...selection},epoch,{signal}); - if(result===null)throw new DOMException('Project selection changed.','AbortError'); - return result; - } - }); - function clearScientific(){scientificEpoch=null;projectScientific.clear();} - const projectProgression=window.ProjectExperimentProgression({ - root:$('project-progression'), - async read(action,args,signal){ - if(!m2Allowed(action))throw Error('History inspection is unavailable for the selected project.'); - const value=await m2Request(action,args,epoch,{signal}); - if(value===null)throw new DOMException('Project selection changed.','AbortError'); - return value; - }, - inspectStudy:(origin,id)=>run(ticket=>inspectItem('studies',{id},origin,ticket),{clean:true}), - inspectTrial:(origin,id)=>run(ticket=>inspectItem('trials',{id},origin,ticket),{clean:true}), - inspectRun:(origin,id)=>run(ticket=>inspectItem('runs',{id},origin,ticket),{clean:true}), - onContextFocusLost:()=>$('project-files-status').focus() - }); - window.addEventListener('pagehide',()=>projectProgression.clear()); - const projectPortfolio=window.ProjectModelPortfolio({ - root:$('project-model-portfolio'), - async read(action,args,signal){ - if(!m2Allowed(action))throw Error('Model organization inspection is unavailable for the selected project.'); - const value=await m2Request(action,args,epoch,{signal}); - if(value===null)throw new DOMException('Project selection changed.','AbortError'); - return value; - }, - onContextFocusLost:()=>$('project-files-status').focus() - }); - window.addEventListener('pagehide',()=>projectPortfolio.clear()); - const projectMeasurements=window.ProjectMeasurements({ - root:$('project-measurements'), - async read(action,args,signal){ - if(!m2Allowed(action))throw Error('Measurement inspection is unavailable for the selected project.'); - const value=await m2Request(action,args,epoch,{signal}); - if(value===null)throw new DOMException('Project selection changed.','AbortError'); - return value; - },onContextFocusLost:()=>$('project-files-status').focus() - }); - window.addEventListener('pagehide',()=>projectMeasurements.clear()); - window.addEventListener('pagehide',clearScientific); - const exact=(value,keys)=>value!==null&&typeof value==='object'&&!Array.isArray(value)&&Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); - const str=(value,max,empty=false)=>typeof value==='string'&&Array.from(value).length<=max&&(empty||/\S/u.test(value)); - const id=value=>typeof value==='string'&&/^[a-f0-9]{32}$/.test(value); - const sha=value=>typeof value==='string'&&/^[a-f0-9]{64}$/.test(value); - const texts=(value,max,min=0,length=120)=>Array.isArray(value)&&value.length>=min&&value.length<=max&&new Set(value).size===value.length&&value.every(item=>str(item,length)); - const positive=value=>Number.isSafeInteger(value)&&value>0; - function annotationOK(task,value,nullable=false){ - if(value===null)return nullable; - if(task==='text_classification')return exact(value,['label'])&&str(value.label,80); - if(task==='text_corpus')return exact(value,['note'])&&str(value.note,4000); - return exact(value,['spans'])&&Array.isArray(value.spans)&&value.spans.length<=500&&value.spans.every(span=>exact(span,['label','start','end'])&&str(span.label,80)&&Number.isSafeInteger(span.start)&&span.start>=0&&positive(span.end)&&span.end>span.start); - } - function provenanceOK(value){ - if(exact(value,['method','rights','normalization'])&&value.method==='import'&&str(value.rights,1000)&&value.normalization==='NFC; LF newlines')return true; - const origin=value?.origin; - return exact(value,['method','rights','origin'])&&value.method==='project_fork'&&str(value.rights,1000)&&exact(origin,['project_id','checkpoint','family','id','revision','source_revision'])&&id(origin.project_id)&&sha(origin.checkpoint)&&['workbench_text','workbench_record'].includes(origin.family)&&id(origin.id)&&positive(origin.revision)&&positive(origin.source_revision); - } - function recordOK(value){ - const fields=['id','kind','name','text','groups','parents','provenance','task','annotation','review','revision','source_revision','created_at','updated_at']; - return exact(value,fields)&&id(value.id)&&value.kind==='text'&&str(value.name,200)&&str(value.text,200000)&&texts(value.groups,30,1)&&texts(value.parents,30)&&provenanceOK(value.provenance)&&tasks.includes(value.task)&&annotationOK(value.task,value.annotation,true)&&['draft','human_reviewed'].includes(value.review)&&positive(value.revision)&&value.source_revision===1&&str(value.created_at,80)&&str(value.updated_at,80); - } - function projectOK(value){return exact(value,['project_id','name','checkpoint'])&&id(value.project_id)&&str(value.name,200)&&sha(value.checkpoint);} - function summaryOK(value){ - return exact(value,['id','kind','name','task','review','revision','source_revision','created_at','updated_at'])&&id(value.id)&&value.kind==='text'&&str(value.name,200)&&tasks.includes(value.task)&&['draft','human_reviewed'].includes(value.review)&&positive(value.revision)&&value.source_revision===1&&str(value.created_at,80)&&str(value.updated_at,80); - } - function receiptOK(value){ - const fields=['committed','selected','project_id','checkpoint','request_key']; - return exact(value,value?.selected===false?[...fields,'root','directory']:fields)&&value.committed===true&&typeof value.selected==='boolean'&&id(value.project_id)&&sha(value.checkpoint)&&typeof value.request_key==='string'&&/^[A-Za-z0-9][A-Za-z0-9_.:-]{7,127}$/.test(value.request_key)&&(value.selected||(str(value.root,160)&&str(value.directory,512))); - } - function stateOK(value){ - const fields=['format','binding','state','roots','project','available_actions']; - if(!exact(value,value?.state==='recovery_required'?[...fields,'recovery']:fields)||value.format!=='project_files_state_v1'||!id(value.binding)||!texts(value.roots,100,0,160)||!Array.isArray(value.available_actions)||value.available_actions.length>8||new Set(value.available_actions).size!==value.available_actions.length||!value.available_actions.every(action=>actions.includes(action)))return false; - if(value.state==='no_project_open')return value.project===null; - if(!projectOK(value.project))return false; - if(value.state==='open')return true; - const recovery=value.recovery; - return value.state==='recovery_required'&&exact(recovery,['action','root','directory','reason','request_key'])&&recovery.action==='open'&&str(recovery.root,160)&&str(recovery.directory,512)&&['unknown_publication','projection_unavailable'].includes(recovery.reason)&&(recovery.request_key===null||typeof recovery.request_key==='string'&&/^[A-Za-z0-9][A-Za-z0-9_.:-]{7,127}$/.test(recovery.request_key))&&value.available_actions.length===1&&value.available_actions[0]==='open'; - } - function boundOK(value,kind){ - const fields=kind==='query'?['binding','project_id','checkpoint','items','next_after_id']:['binding','project_id','checkpoint','record','history','next_history_before_revision']; - if(!exact(value,fields)||!id(value.binding)||!id(value.project_id)||!sha(value.checkpoint))return false; - if(kind==='query')return Array.isArray(value.items)&&value.items.length<=100&&value.items.every(summaryOK)&&(value.next_after_id===null||id(value.next_after_id)&&value.items.at(-1)?.id===value.next_after_id)&&value.items.every((item,index)=>index===0||item.id>value.items[index-1].id); - return recordOK(value.record)&&Array.isArray(value.history)&&value.history.length<=4&&value.history.every(recordOK)&&value.history.every((item,index)=>item.id===value.record.id&&item.revision<=value.record.revision&&(index===0||item.revision!busy&&!blocked&&selectedContracts.has('project_files.'+action)&&state?.available_actions.includes(action); - for(const action of ['new','open','save','close'])$('project-files-'+action).disabled=!allowed(action); - $('project-files-refresh').disabled=busy; - $('project-files-root').disabled=busy||state?.state==='recovery_required'; - $('project-files-directory').disabled=busy||state?.state==='recovery_required'; - $('project-files-name').disabled=busy||state?.state==='recovery_required'; - $('text-import-fields').disabled=!open||!allowed('import_text'); - $('annotation-fields').disabled=!selected||!allowed('annotate'); - $('import-text').disabled=!allowed('import_text'); - $('annotation-save').disabled=!selected||!allowed('annotate'); - $('project-files-discard').disabled=busy||!dirty(); - $('records-more').hidden=nextRecord===null;$('records-more').disabled=!allowed('query'); - $('record-history-more').hidden=nextHistory===null;$('record-history-more').disabled=!selected||!allowed('get'); - $('project-files-open').textContent=state?.state==='recovery_required'?'Reopen to reconcile':'Open project'; - for(const button of $('records-list').querySelectorAll('button'))button.disabled=busy||blocked; - $('project-files-dirty').textContent=dirty()?'This tab has unsent or unacknowledged edits. After an error, inspect saved state before retrying; explicitly discard these edits before closing. Unsent annotations must be resolved before selecting a different record.':'No unsubmitted edits.'; - m2Controls();m4Controls();runtimeControls();candidateControls(); - if(window.TuldokAuth&&!window.TuldokAuth.canReview())$('record-review').value='draft'; - } - function showState(){ - if(executionBinding!==state.binding){nativeGraphs=[];nativePrepared=null;nativePending=null;nativeOffset=nativeTotal=0;$('project-native-graph').replaceChildren();$('project-native-phases').replaceChildren();executionBinding=state.binding;executionPrepared=null;executionApproval=null;executionStudy=null;studyApproval=null;executionGraphs=[];executionTrials=[];producerSource=null;producerCurrent=null;$('project-execution-grant').textContent='No current local approval selected.';} - const root=$('project-files-root'), previous=root.value;root.replaceChildren(); - for(const alias of state.roots){const option=document.createElement('option');option.value=alias;option.textContent=alias;root.append(option);} - if(state.roots.includes(previous))root.value=previous; - const destination=$('project-save-as-root'),oldDestination=destination.value;destination.replaceChildren(); - for(const alias of state.roots){const option=document.createElement('option');option.value=alias;option.textContent=alias;destination.append(option);} - if(state.roots.includes(oldDestination))destination.value=oldDestination; - const candidateRoot=$('project-candidate-root'),oldCandidateRoot=candidateRoot.value;candidateRoot.replaceChildren();for(const alias of state.roots){const option=document.createElement('option');option.value=alias;option.textContent=alias;candidateRoot.append(option);}if(state.roots.includes(oldCandidateRoot))candidateRoot.value=oldCandidateRoot; - const recovery=state.state==='recovery_required'; - $('project-files-summary').textContent=state.project?`${state.project.name} · ${state.project.project_id} · ${recovery?'last acknowledged':'saved'} checkpoint ${state.project.checkpoint}${recovery?' · Reopen this project to reconcile its saved state.':''}`:'No project open.'; - if(recovery){root.value=state.recovery.root;$('project-files-directory').value=state.recovery.directory;} - controls(); - } - function annotationHelp(){ - const task=$('annotation-task').value; - $('annotation-label').textContent=task==='text_classification'?'Class label':task==='text_corpus'?'Corpus note':'Entity spans (JSON)'; - $('annotation-help').textContent=task==='text_classification'?'Enter one classification label, up to 80 characters.':task==='text_corpus'?'Enter a human-authored corpus note, up to 4,000 characters.':'Enter {"spans":[{"label":"person","start":0,"end":4}]}. Offsets count Unicode code points; an empty spans array records a negative.'; - $('annotation-value').maxLength=task==='text_classification'?80:task==='text_corpus'?4000:200000; - } - function showRecord(focus=false){ - $('record-heading').textContent=selected?selected.name:'Select a text record'; - $('record-summary').textContent=selected?`Revision ${selected.revision} · ${selected.review} · source revision ${selected.source_revision}`:''; - $('record-text').value=selected?.text??''; - $('annotation-task').value=selected?.task??'text_classification'; - const value=selected?.annotation; - $('annotation-value').value=value?(selected.task==='text_entities'?JSON.stringify(value,null,2):value.label??value.note):''; - $('annotation-groups').value=selected?.groups.join(', ')??''; - $('record-review').value=selected?.review??'draft'; - annotationHelp(); - showHistory();controls();if(focus)$('record-heading').focus(); - } - function showHistory(){ - const list=$('record-history');list.replaceChildren(); - for(const snapshot of history){const item=document.createElement('li'),title=document.createElement('p'),body=document.createElement('pre');title.textContent=`Revision ${snapshot.revision} · ${snapshot.review} · ${snapshot.task}`;body.textContent=JSON.stringify(snapshot.annotation,null,2);item.append(title,body);list.append(item);} - controls(); - } - function showRecords(items){ - const list=$('records-list');list.replaceChildren(); - if(!items.length){const item=document.createElement('li');item.textContent='No text records yet.';list.append(item);} - for(const record of items){const item=document.createElement('li'),button=document.createElement('button');button.type='button';button.dataset.recordId=record.id;button.textContent=`${record.name} · ${record.review} · revision ${record.revision}`;button.setAttribute('aria-current',String(record.id===selected?.id));button.onclick=()=>select(record.id);item.append(button);list.append(item);} - controls(); - } - function needClean(){if(!dirty())return true;error('This tab has unsent or unacknowledged edits. Inspect saved state after an error, then submit or choose “Discard unsubmitted edits” before this action.');return false;} - async function discover(ticket){ - let response,value; - try{response=await window.TuldokAuth.fetch('/api/agent/capabilities',{cache:'no-store'});value=await response.json();} - catch(_){const failure=Error('Project contracts unavailable. Refresh status before continuing.');failure.code='transport';throw failure;} - if(ticket!==epoch)return false; - if(!response.ok){const failure=Error(value?.error??'Project contracts unavailable. Sign in and refresh status.');failure.status=response.status;throw failure;} - if(!exact(value,['format','envelope_version','catalog_id','operations','limits'])||value.format!=='agent_capabilities_v1'||value.envelope_version!==1||!id(value.catalog_id)||!Array.isArray(value.operations)||value.operations.length>256||JSON.stringify(value).length>1048576||!exact(value.limits,['request_bytes','response_bytes','max_operations'])||value.limits.request_bytes!==1048576||value.limits.response_bytes!==16777216||value.limits.max_operations!==256)throw Error('Unsupported project capability envelope.'); - const identity=window.TuldokAuth.identity?.(); - if(identity?.catalog_id&&identity.catalog_id!==value.catalog_id)throw Error('Project capability catalog differs from the current session.'); - const names=new Set(),chosen=new Set(),versionsChosen=new Map(); - for(const operation of value.operations){ - if(!exact(operation,['id','contracts','availability'])||typeof operation.id!=='string'||!/^[a-z][a-z0-9_.]{0,79}$/.test(operation.id)||names.has(operation.id)||!['available','unavailable','unsupported'].includes(operation.availability)||!Array.isArray(operation.contracts)||!operation.contracts.length||operation.contracts.length>4)throw Error('Unsupported operation capability.'); - names.add(operation.id);const versions=new Set(); - for(const contract of operation.contracts){ - if(!exact(contract,['version','sha256','input_schema','output_schema','scopes','authority','effects','retry','bounds'])||!positive(contract.version)||versions.has(contract.version)||!sha(contract.sha256)||!contract.input_schema||typeof contract.input_schema!=='object'||Array.isArray(contract.input_schema)||!contract.output_schema||typeof contract.output_schema!=='object'||Array.isArray(contract.output_schema)||!texts(contract.scopes,16,0,40)||!['agent','owner'].includes(contract.authority)||!['read','mutation'].includes(contract.effects)||!['read','cas','receipt','never'].includes(contract.retry)||!exact(contract.bounds,['request_bytes','response_bytes','inline_decoded_bytes','json_depth'])||!Object.values(contract.bounds).every(positive))throw Error('Unsupported operation contract.'); - versions.add(contract.version); - // Remote schemas are opaque discovery data; only installed local pins - // select behavior. Every request lets the owner verify this exact pin. - const local=versionPins[operation.id]??(Object.hasOwn(pins,operation.id)?{'1':pins[operation.id]}:{}); - if(operation.availability==='available'&&local[contract.version]===contract.sha256&&contract.version>(versionsChosen.get(operation.id)??0)){chosen.add(operation.id);versionsChosen.set(operation.id,contract.version);} - } - } - selectedContracts=chosen;selectedVersions=versionsChosen; - if(!chosen.has('project_files.status')){const failure=Error('No compatible project status contract is available.');failure.code='contract_mismatch';throw failure;} - return true; - } - async function request(path,{body,kind='state',ticket=epoch,read=false,recordId}={}){ - const binding=state?.binding, expected=checkpoint(), project=state?.project?.project_id; - const action=path===''?'status':path.startsWith('/records?')?'query':path.startsWith('/records/')?(body===undefined?'get':'annotate'):path.slice(1).replaceAll('-','_'); - const operation='project_files.'+action; - if(!selectedContracts.has(operation)){const failure=Error('No compatible contract is available for '+action+'. Refresh project status.');failure.code='contract_mismatch';throw failure;} - let response,value; - try{ - response=await window.TuldokAuth.fetch(base+path,{method:body===undefined?'GET':'POST',headers:{'Tuldok-Operation-Contract':selectedHeader(operation),...(body===undefined?{}:{'Content-Type':'application/json'})},body:body===undefined?undefined:JSON.stringify(body),cache:'no-store'}); - value=await response.json(); - }catch(_){const failure=Error(body===undefined?'Project response unavailable. Refresh status to recover.':'Acknowledgement unavailable; the edit may already be saved. Keep your edits and refresh status before retrying.');failure.code='transport';throw failure;} - if(ticket!==epoch){message('A superseded project response was ignored. The current view is unchanged.');return null;} - if(!response.ok){const fields=['error',...(Object.hasOwn(value??{},'code')?['code']:[]),...(Object.hasOwn(value??{},'receipt')?['receipt']:[])];const valid=exact(value,fields)&&str(value.error,1000)&&(!Object.hasOwn(value,'code')||str(value.code,120))&&(!Object.hasOwn(value,'receipt')||receiptOK(value.receipt));const failure=Error(valid?`${value.code??'error'}: ${value.error}`:'Invalid project error response. Refresh status.');failure.code=valid?(value.code??'http_error'):'transport';failure.status=response.status;if(valid&&value.receipt)failure.receipt=value.receipt;throw failure;} - if(!(kind==='state'?stateOK(value):kind==='fork'?forkOK(value,body):boundOK(value,kind))){const failure=Error('Unsupported project response. The current state is unknown; refresh status before continuing.');failure.code='transport';throw failure;} - if(kind==='record'&&selectedVersions.get(operation)===1&&[value.record,...value.history].some(record=>record.provenance.method!=='import')){const failure=Error('Version 1 cannot represent retained fork provenance. Refresh contracts.');failure.code='transport';throw failure;} - if(kind!=='state'&&kind!=='fork'&&(value.binding!==binding||value.project_id!==project||(read&&value.checkpoint!==expected)||(recordId&&value.record?.id!==recordId))){const failure=Error('stale: Response belongs to a different project or checkpoint. Your unsubmitted edits remain here. Refresh status.');failure.code='stale';throw failure;} - return value; - } - async function run(work,{clean=false}={}){ - if(busy)return;if(clean&&!needClean())return; - const ticket=++epoch;busy=true;clearError();controls(); - let nextFocus=null; - try{nextFocus=await work(ticket);}catch(failure){ - if(ticket===epoch){ - if(failure.status===409||failure.status>=500||['stale','conflict','binding_mismatch','transport','projection_unavailable','stale_binding','stale_checkpoint','too_large'].includes(failure.code))blocked=true; - if(failure.receipt){const receipt=failure.receipt;$('project-files-receipt').hidden=false;$('project-files-receipt').textContent=`Saved to project ${receipt.project_id} at checkpoint ${receipt.checkpoint}; request ${receipt.request_key}. ${receipt.selected?'Refresh status and reopen to restore the local view.':'The project was created but is not selected. Refresh status, then Open its directory.'}`;if(!receipt.selected){$('project-files-root').value=receipt.root;$('project-files-directory').value=receipt.directory;}} - error(failure.message); - } - }finally{if(ticket===epoch){busy=false;controls();if(nextFocus)nextFocus.focus();}} - } - async function query(ticket,append=false){ - if(state?.state!=='open'){summaries=[];nextRecord=null;showRecords([]);return;} - const cursor=append?nextRecord:null,parameters={binding:state.binding,checkpoint:checkpoint()}; - if(cursor!==null)parameters.after_id=cursor; - const result=await request('/records?'+new URLSearchParams(parameters),{kind:'query',ticket,read:true}); - if(result){if(cursor!==null&&result.items.some(item=>item.id<=cursor))throw Error('Invalid record-page continuation. Refresh status.');summaries=append?[...summaries,...result.items]:result.items;nextRecord=result.next_after_id;showRecords(summaries);} - } - function olderHistory(){return run(async ticket=>{ - if(!selected||nextHistory===null)return; - const before=nextHistory; - const result=await request('/records/'+selected.id+'?'+new URLSearchParams({binding:state.binding,checkpoint:checkpoint(),before_revision:before}),{kind:'record',ticket,read:true,recordId:selected.id}); - if(!result)return; - if(result.record.revision!==selected.revision||result.history.some(item=>item.revision>=before))throw Error('Invalid history-page continuation. Refresh status.'); - history=[...history,...result.history];nextHistory=result.next_history_before_revision;showHistory();message('Older saved annotation history loaded.');return nextHistory===null?$('record-history-title'):$('record-history-more'); - });} - function refresh(){return run(async ticket=>{ - if(!await discover(ticket))return; - const value=await request('',{ticket});if(!value)return; - const changed=state?.binding!==value.binding||checkpoint()!==value.project?.checkpoint; - state=value;blocked=false; - if(changed){m4Reset();clearExploration();selected=null;history=[];nextHistory=null;if(!annotationDirty)showRecord();else{$('record-heading').textContent='Unsubmitted annotation from a prior checkpoint';$('record-summary').textContent='Project changed. Your unsubmitted annotation remains below; discard it before selecting a saved record.';$('record-text').value='';}} - showState();await query(ticket);message(dirty()?'Project status refreshed. Unsubmitted edits remain in this tab.':'Project status refreshed.'); - });} - function select(recordId){if(annotationDirty){error('The annotation editor has unsent or unacknowledged edits. Submit or explicitly discard that annotation before selecting a saved record.');return;}return run(async ticket=>{ - const result=await request('/records/'+recordId+'?'+new URLSearchParams({binding:state.binding,checkpoint:checkpoint()}),{kind:'record',ticket,read:true,recordId}); - if(!result)return;selected=result.record;history=result.history;nextHistory=result.next_history_before_revision;annotationDirty=false;showRecord(true);await query(ticket);message('Saved text and annotation history loaded.'); - });} - function key(){return crypto.randomUUID().replaceAll('-','');} - function mutationBody(path,fields){ - const body={binding:state.binding,expected_checkpoint:checkpoint(),...fields}; - const signature=JSON.stringify({path,body}); - if(!pending||pending.signature!==signature)pending={signature,key:key()}; - return {...body,request_key:pending.key}; - } - function lifecycle(action){const recovering=action==='open'&&state?.state==='recovery_required';return run(async ticket=>{ - const fields={}; - if(['new','open'].includes(action)){ - if(!$('project-files-root').value||!$('project-files-directory').value.trim())throw Error('Choose a local root and relative project directory.'); - fields.root=$('project-files-root').value;fields.directory=$('project-files-directory').value; - if(action==='new'){if(!$('project-files-name').value.trim())throw Error('Enter a name for the new project.');fields.name=$('project-files-name').value;} - } - const value=await request('/'+action,{ticket,body:mutationBody('/'+action,fields)});if(!value)return; - state=value;pending=null;if(action!=='save'){m4Reset();clearExploration();selected=null;history=[];nextHistory=null;}showState();if(action!=='save'&&!annotationDirty)showRecord();else if(recovering&&annotationDirty){$('record-heading').textContent='Preserved unsubmitted annotation';$('record-text').value='';}await query(ticket);message(action==='save'?'Project checkpoint is saved.':action==='close'?'Project closed.':`Project ${action==='new'?'created':'opened'}.`); - return action==='save'?$('project-files-save'):(state.state==='open'?$('text-name'):$('project-files-directory')); - },{clean:!recovering});} - function groups(value){const result=value.split(',').map(item=>item.trim()).filter(Boolean);if(!texts(result,30,1))throw Error('Supply 1–30 distinct protected group IDs, each up to 120 characters.');return result;} - async function acceptRecord(value,ticket){ - state={...state,project:{...state.project,checkpoint:value.checkpoint}};selected=value.record;history=value.history;nextHistory=value.next_history_before_revision;pending=null;annotationDirty=false;showState();showRecord(true);await query(ticket); - } - $('text-import-form').onsubmit=event=>{ - event.preventDefault();if(annotationDirty){error('Submit or discard the current annotation before creating another text record.');return;} - run(async ticket=>{ - const asset={name:$('text-name').value,text:$('text-content').value,groups:groups($('text-groups').value),rights:$('text-rights').value}; - if(!str(asset.name,200)||!str(asset.text,200000)||!str(asset.rights,1000))throw Error('Enter a name, text and rights note within the displayed limits.'); - const value=await request('/import-text',{kind:'record',ticket,body:mutationBody('/import-text',{asset})});if(!value)return; - importDirty=false;$('text-name').value='';$('text-content').value='';await acceptRecord(value,ticket);message('Text submitted and saved to the project.'); - }); - }; - $('annotation-form').onsubmit=event=>{ - event.preventDefault();run(async ticket=>{ - if(!selected)throw Error('Select a saved text record first.'); - const task=$('annotation-task').value,raw=$('annotation-value').value; - let annotation;if(task==='text_entities'){try{annotation=JSON.parse(raw);}catch(_){throw Error('Entity spans must be valid JSON.');}}else annotation=task==='text_corpus'?{note:raw}:{label:raw}; - if(!annotationOK(task,annotation))throw Error('The annotation does not match this task. Follow the instructions above the editor.'); - const body={revision:selected.revision,source_revision:selected.source_revision,task,annotation,groups:groups($('annotation-groups').value),review:$('record-review').value}; - const value=await request('/records/'+selected.id,{kind:'record',ticket,recordId:selected.id,body:mutationBody('/records/'+selected.id,{annotation:body})});if(!value)return; - await acceptRecord(value,ticket);message('Annotation submitted and saved to the project.'); - }); - }; - function projectValue(value,depth=0){ - const box=document.createElement('div'); - if(value===null||typeof value!=='object'){box.textContent=value===null?'Not recorded':String(value);return box;} - if(depth>5){box.textContent='Further saved details available through the project API.';return box;} - const entries=Array.isArray(value)?value.map((item,index)=>[String(index+1),item]):Object.entries(value); - const list=document.createElement('dl'); - for(const [name,item] of entries){ - const term=document.createElement('dt'),definition=document.createElement('dd');term.textContent=name.replaceAll('_',' '); - if(item!==null&&typeof item==='object'){ - const details=document.createElement('details'),summary=document.createElement('summary');summary.textContent=Array.isArray(item)?`${item.length} saved items`:'Saved details';details.append(summary,projectValue(item,depth+1));definition.append(details); - }else definition.textContent=item===null?'Not recorded':String(item); - list.append(term,definition); - } - box.append(list);return box; - } - const m2Allowed=action=>!busy&&!blocked&&state?.state==='open'&&selectedContracts.has('project_files.'+action); - function m2Controls(){ - projectPortfolio.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), - project_id:state?.project?.project_id,enabled:m2Allowed('files_inspect')&&m2Allowed('file_get')}); - projectProgression.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), - project_id:state?.project?.project_id,enabled:m2Allowed('progression')&&m2Allowed('progression_origins')}); - projectMeasurements.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), - enabled:['measurement_sources','measurement_compare','measurement_inspect'].every(m2Allowed)}); - if(blocked||state?.state!=='open'||scientificEpoch!==null&&(scientificEpoch!==epoch||explored?.checkpoint!==checkpoint()))clearScientific(); - const action=$('project-explore-kind').value; - $('project-explore-load').disabled=!m2Allowed(action); - $('project-explore-more').disabled=!m2Allowed(action); - for(const [field,operation] of [['project-source-fields','data_import_'+$('project-source-kind').value],['project-search-fields','search_create'],['project-selection-fields','selection_create'],['project-rights-fields','data_rights'],['project-data-note-fields','data_annotate']])$(field).disabled=!m2Allowed(operation)||(['project-selection-fields','project-rights-fields','project-data-note-fields'].includes(field)&&(!explored||explored.checkpoint!==checkpoint())); - $('project-definitions-load').disabled=!m2Allowed('definitions'); - $('project-model-create-fields').disabled=!modelTemplates.length||templateBinding!==state?.binding||!m2Allowed('model_create'); - $('project-model-publish').disabled=!explored||explored.category!=='models'||explored.checkpoint!==checkpoint()||!m2Allowed('model_publish'); - $('project-study-create-fields').disabled=!m2Allowed('study_create'); - $('project-register-fields').disabled=!m2Allowed('source_register'); - $('project-source-references-more').disabled=!m2Allowed('source_references'); - $('project-scientific-fields').disabled=!registeredSource||registeredSource.binding!==state?.binding||!m2Allowed('data_import_scientific'); - executionControls(); - $('project-artifact-status').disabled=!explored||explored.category!=='artifacts'||!m2Allowed('artifact_status'); - $('project-artifact-resolve-fields').disabled=!explored||explored.category!=='artifacts'||!m2Allowed('artifact_resolve'); - $('project-metadata-fields').disabled=!explored||explored.checkpoint!==checkpoint()||!m2Allowed(explored.saveAction); - for(const name of ['project-detail-history','project-detail-availability','project-run-evidence','project-run-document','project-run-disclose','project-detail-history-more'])$(name).disabled=busy||blocked||state?.state!=='open'; - for(const button of $('project-explore-list').querySelectorAll('button'))button.disabled=busy||blocked; - } - function clearExploration(){ - clearScientific(); - explored=null;exploreOffset=0;exploreTotal=0;historyOffset=0;historyTotal=0; - $('project-explore-list').replaceChildren();$('project-detail-facts').replaceChildren();$('project-detail-history-list').replaceChildren(); - $('project-detail-title').textContent='Choose a saved record';$('project-explore-summary').textContent=''; - for(const name of ['project-metadata-form','project-data-note-form','project-rights-form','project-selection-form','project-detail-history','project-detail-history-more','project-detail-availability','project-run-evidence','project-run-document','project-run-disclose','project-disclosure-label','project-model-publish','project-artifact-status','project-artifact-resolve-form','project-explore-more'])if(!m2DirtyForms.has(name))$(name).hidden=true; - if(m2Dirty)$('project-detail-title').textContent='Preserved unsubmitted metadata; discard it before selecting a saved record.'; - } - async function m4Response(action,value,args,project){ - const invalid=()=>{const failure=Error('Saved file or fork response differs from its selected identity, revision or exact bytes.');failure.code='transport';throw failure;}; - const natural=value=>Number.isSafeInteger(value)&&value>=0; - const safeText=(value,max,empty=false)=>str(value,max,empty)&&!value.includes('\0')&&!Array.from(value).some(ch=>ch.length===1&&ch.charCodeAt(0)>=0xd800&&ch.charCodeAt(0)<=0xdfff); - const filePath=(kind,path)=>['script','note'].includes(kind)&&safeText(path,512)&&path.startsWith((kind==='script'?'scripts':'notes')+'/')&&!path.includes('\\')&&path.split('/').every(part=>part!==''&&part!=='.'&&part!=='..')&&!/[\x00-\x1f]/.test(path); - const shaBytes=async raw=>Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',raw)),n=>n.toString(16).padStart(2,'0')).join(''); - const fileID=async value=>shaBytes(new TextEncoder().encode(project+'\0'+value.kind+'\0'+value.path)); - const metadata=value=>exact(value,['revision','sha256','bytes','adopted_at','actor_id'])&&positive(value.revision)&&sha(value.sha256)&&natural(value.bytes)&&value.bytes<=1048576&&safeText(value.adopted_at,80)&&id(value.actor_id); - const deletedMetadata=value=>exact(value,['revision','deleted','sha256','bytes','adopted_at','actor_id'])&&positive(value.revision)&&value.revision>=2&&value.deleted===true&&value.sha256===null&&value.bytes===0&&safeText(value.adopted_at,80)&&id(value.actor_id); - const families=['model_definition','pipeline_graph','recipe_definition']; - const fields=['task','specification','source_selector','preprocessing_policy','output_policy','recipe_source','inputs','runtime','release','nodes','data','evaluation_profile','model','trainer']; - const reselection=value=>Array.isArray(value)&&value.length<=14&&new Set(value).size===value.length&&value.every(field=>fields.includes(field)); - const origin=value=>exact(value,['project_id','checkpoint','family','id','revision','created_at'])&&id(value.project_id)&&sha(value.checkpoint)&&families.includes(value.family)&&safeText(value.id,512)&&positive(value.revision)&&safeText(value.created_at,80); - if(['file_get','file_adopt','file_adopt_deletion'].includes(action)){ - const keys=action==='file_get'?['id','project_id','kind','path','version']:['id','project_id','kind','path','revision','sha256','bytes','adopted_at','changed',...(action==='file_adopt_deletion'?['deleted']:[])]; - if(!exact(value,keys)||!sha(value.id)||value.project_id!==project||!filePath(value.kind,value.path)||value.id!==await fileID(value))invalid(); - if(action==='file_get'){ - const v=value.version,deleted=v?.deleted===true,meta=Object.fromEntries(Object.entries(v??{}).filter(([k])=>k!=='text')); - if(deleted){if(value.kind!=='note'||!exact(v,['revision','deleted','sha256','bytes','adopted_at','actor_id','text'])||!deletedMetadata(meta)||v.text!==null)invalid();} - else{if(!exact(v,['revision','sha256','bytes','adopted_at','actor_id','text'])||!metadata(meta)||typeof v.text!=='string'||Array.from(v.text).some(ch=>ch.length===1&&ch.charCodeAt(0)>=0xd800&&ch.charCodeAt(0)<=0xdfff))invalid(); - const bytes=new TextEncoder().encode(v.text);if(bytes.length>1048576||bytes.length!==v.bytes||await shaBytes(bytes)!==v.sha256)invalid();} - if(value.id!==args.file_id||(args.revision!==undefined&&args.revision!==v.revision))invalid(); - }else if(action==='file_adopt_deletion'){ - if(value.kind!=='note'||value.path!==args.path||value.deleted!==true||value.sha256!==null||value.bytes!==0||value.changed!==true||value.revision!==args.expected_revision+1||!safeText(value.adopted_at,80))invalid(); - }else if(value.kind!==args.kind||value.path!==args.path||value.sha256!==args.expected_sha256||!positive(value.revision)||typeof value.changed!=='boolean'||value.revision!==args.expected_revision+Number(value.changed)||!natural(value.bytes)||value.bytes>1048576||!safeText(value.adopted_at,80))invalid(); - } - if(action==='file_history'&&(!exact(value,['id','items','total','offset','limit'])||value.id!==args.file_id||!value.items.every(item=>metadata(item)||deletedMetadata(item))))invalid(); - if(action==='files_inspect')for(const item of value.items){ - if(!exact(item,['id','kind','path','revision','expected_sha256','expected_bytes','observed_sha256','observed_bytes','state','reason'])||!filePath(item.kind,item.path)||item.id!==await fileID(item)||!positive(item.revision)||!(item.expected_sha256===null||sha(item.expected_sha256))||!natural(item.expected_bytes)||item.expected_bytes>1048576||!(item.observed_sha256===null||sha(item.observed_sha256))||!(item.observed_bytes===null||natural(item.observed_bytes)&&item.observed_bytes<=1048576)||!['unchanged','changed','missing','invalid','deleted'].includes(item.state)||!(item.reason===null||['missing','unsafe_path','too_large','invalid_utf8','unreadable','unstable'].includes(item.reason)))invalid(); - if(item.expected_sha256===null&&(item.kind!=='note'||item.expected_bytes!==0||['unchanged','missing'].includes(item.state)))invalid(); - if(item.state==='deleted'&&(item.expected_sha256!==null||item.observed_sha256!==null||item.observed_bytes!==null||item.reason!==null))invalid(); - if(item.state==='unchanged'&&(item.observed_sha256!==item.expected_sha256||item.observed_bytes!==item.expected_bytes||item.reason!==null))invalid(); - } - if(action==='material_intents')for(const item of value.items)if(!exact(item,['family','id','catalog_id','definition_id','name','revision','required_reselection','fork_origin'])||!families.includes(item.family)||item.catalog_id!==project||!id(item.definition_id)||item.id!==project+'.'+item.definition_id||!safeText(item.name,200)||item.revision!==1||!reselection(item.required_reselection)||!origin(item.fork_origin)||item.fork_origin.family!==item.family)invalid(); - if(action==='material_intent'){ - const current=value.current; - if(!exact(value,['family','id','current','fork_origin'])||value.family!==args.family||value.id!==project+'.'+args.definition_id||!exact(current,['format','catalog_id','id','revision','name','definition_family','definition','required_reselection'])||current.format!=='project_material_fork_intent_v1'||current.catalog_id!==project||current.id!==args.definition_id||current.revision!==1||!safeText(current.name,200)||current.definition_family!==args.family||current.definition===null||typeof current.definition!=='object'||Array.isArray(current.definition)||!reselection(current.required_reselection)||!origin(value.fork_origin)||value.fork_origin.family!==args.family)invalid(); - } - if(action==='material_intent_capabilities')for(const item of value.items)if(!exact(item,['family','id','requires_reselection'])||!families.includes(item.family)||typeof item.id!=='string'||!item.id.startsWith(project+'.')||!id(item.id.slice(33))||!reselection(item.requires_reselection))invalid(); - if(action==='fork_info'){ - const page=value.mapping,offset=args.offset??0,limit=args.limit??40; - if(!exact(value,['source','project_id','mapping','archived_count','copied_count','missing_regenerable_count'])||value.project_id!==project||!(value.source===null||exact(value.source,['project_id','checkpoint'])&&id(value.source.project_id)&&sha(value.source.checkpoint))||!['archived_count','copied_count','missing_regenerable_count'].every(k=>natural(value[k]))||!exact(page,['items','total','offset','limit'])||!natural(page.total)||page.offset!==offset||page.limit!==limit||!Array.isArray(page.items)||page.items.length!==Math.min(limit,Math.max(0,page.total-offset)))invalid(); - for(const item of page.items)if(!exact(item,['family','source_id','source_revision','destination_id'])||!safeText(item.family,80)||!safeText(item.source_id,512)||!(item.source_revision===null||positive(item.source_revision))||!safeText(item.destination_id,512))invalid(); - } - } - async function m2Request(action,arguments_,ticket,{signal}={}){ - const operation='project_files.'+action,definition=m2Operations[operation]; - if(!definition||!selectedContracts.has(operation))throw Error('This operation is unavailable for the selected project.'); - const binding=state.binding,cp=checkpoint(),project=state.project.project_id; - let values={binding,[definition.mutation?'expected_checkpoint':'checkpoint']:cp,...arguments_}; - let path=definition.path; - for(const [name,value] of Object.entries(values))path=path.replace('{'+name+'}',encodeURIComponent(value)); - const remaining=Object.fromEntries(Object.entries(values).filter(([name])=>!definition.path.includes('{'+name+'}'))); - const options={method:definition.method,cache:'no-store',signal,headers:{'Tuldok-Operation-Contract':selectedHeader(operation)}}; - if(definition.method==='GET')path+='?'+new URLSearchParams(remaining); - else{ - if(definition.mutation&&!['execution_reconcile_current','execution_study_reconcile_preparation','native_reconcile'].includes(action)){ - let prepared; - if(action==='runtime_capture'){ - const signature=JSON.stringify(arguments_); - if(runtimePending&&(runtimePending.signature!==signature||runtimePending.body.binding!==binding))throw Error('Inspect the pending runtime capture outcome, then keep its exact request or explicitly discard the capture form before changing source, inventory or reserve.'); - if(!runtimePending)runtimePending={signature,body:mutationBody(path,arguments_)}; - prepared=runtimePending.body;$('project-runtime-status-key').value=prepared.request_key;$('project-runtime-abandon-key').value=prepared.request_key; - $('project-runtime-pending').textContent=`Pending capture ${prepared.request_key}: source ${prepared.source}, inventory ${prepared.expected_inventory_sha256}, reserve ${prepared.reserve_bytes??6442450944} bytes. Retry uses these exact fields. Preserve the key until its retained outcome is known.`; - Object.assign(remaining,prepared); - }else if(['native_preflight','native_submit','native_retry'].includes(action)){ - const signature=JSON.stringify({action,arguments_}); - if(nativePending&&(nativePending.signature!==signature||nativePending.body.binding!==binding))throw Error('Keep the exact pending native request until its outcome is known, or explicitly discard unsubmitted fields before choosing a different action.'); - if(!nativePending)nativePending={action,signature,body:mutationBody(path,arguments_)}; - prepared=nativePending.body;Object.assign(remaining,Object.fromEntries(Object.entries(prepared).filter(([field])=>!definition.path.includes('{'+field+'}')))); - $('project-native-pending').textContent=`Pending ${action}: request ${prepared.request_key}. Keep this exact key and fields after a lost acknowledgement. No automatic retry is performed.`; - }else prepared=mutationBody(path,arguments_); - values={...values,...prepared};remaining.request_key=prepared.request_key; - } - if(['execution_study_prepare_package','execution_study_prepare_verifier'].includes(action)){$('project-candidate-recovery-study').value=arguments_.study_id;$('project-candidate-recovery-key').value=values.request_key;$('project-candidate-preparation-status').textContent=`Pending preparation ${values.request_key}. Preserve this key until its outcome is known.`;} - options.headers['Content-Type']='application/json';options.body=JSON.stringify(remaining); - if(new TextEncoder().encode(options.body).length>1048576)throw Error('This source exceeds the synchronous import limit. Choose a smaller source.'); - } - let response,value; - try{response=await window.TuldokAuth.fetch(path,options);value=await response.json();} - catch(problem){if(problem.name==='AbortError')throw problem;const failure=Error(definition.mutation?'Acknowledgement unavailable; this change may already be saved. Keep your edits and refresh project status.':'Saved history is unavailable. Refresh project status.');failure.code='transport';throw failure;} - if(ticket!==epoch)return null; - if(!response.ok){ - const failure=Error(str(value?.error,1000)?value.error:'Unsupported project error response.');failure.code=value?.code??'transport';failure.status=response.status; - if(value?.receipt){if(!receiptOK(value.receipt)||value.receipt.request_key!==values.request_key||value.receipt.project_id!==project||!value.receipt.selected){const invalid=Error('Project receipt belongs to a different request.');invalid.code='transport';throw invalid;}failure.receipt=value.receipt;} - throw failure; - } - if(!exact(value,['binding','project_id','checkpoint','value'])||value.binding!==binding||value.project_id!==project||!sha(value.checkpoint)||(!definition.mutation&&value.checkpoint!==cp)||!value.value||typeof value.value!=='object'||Array.isArray(value.value)){ - const failure=Error('Saved response belongs to a different project or checkpoint. Refresh project status.');failure.code='transport';throw failure; - } - if(m2PagedActions.has(action)){ - const page=value.value,limit=arguments_.limit??(m2HistoryActions.has(action)?1:40),offset=arguments_.offset??0; - if(!Array.isArray(page.items)||page.items.length>limit||page.offset!==offset||page.limit!==limit||!Number.isSafeInteger(page.total)||page.total<0||page.items.length!==Math.min(limit,Math.max(0,page.total-offset))){const failure=Error('Saved page differs from its requested continuation.');failure.code='transport';throw failure;} - } - if(['files_inspect','file_get','file_history','file_adopt','file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info'].includes(action)){await m4Response(action,value.value,arguments_,project);if(ticket!==epoch)return null;} - if(action.startsWith('execution_')){ - const result=value.value,invalid=()=>{const failure=Error('Current execution response differs from the selected project, material or action.');failure.code='transport';throw failure;}; - if(Object.hasOwn(result,'origin_catalog_id')&&result.origin_catalog_id!==project)invalid(); - if(['execution_approve_external','execution_study_approve'].includes(action)){ - const grant=result.grant,expected=arguments_.expected_digest??arguments_.expected_material_sha256; - if(result.format!=='project_current_execution_approval_v1'||result.current_local!==true||!id(grant?.id)||grant.catalog_id!==project||grant.consumer_actor_id!==arguments_.consumer_actor_id||grant.material_sha256!==expected||grant.revoked_at!==null||!Number.isSafeInteger(grant.created_at)||!Number.isSafeInteger(grant.expires_at)||grant.expires_at<=grant.created_at)invalid(); - } - if(action==='execution_cancel'&&result.run_id!==arguments_.run_id)invalid(); - if(['execution_study_admit','execution_study_stop'].includes(action)&&result.study_id!==arguments_.study_id)invalid(); - if(action==='execution_study_admit'&&(result.trial_id!==arguments_.trial_id||result.reservation?.trial_id!==result.trial_id||result.reservation?.run_id!==result.run_id))invalid(); - const actions={execution_producer_start:'start',execution_generation_resume:'resume',execution_grounded_review:'review',execution_producer_decide:'decide',execution_producer_cancel:'cancel'}; - if(Object.hasOwn(actions,action)){ - const family=arguments_.family??(action==='execution_generation_resume'?'generation_history':'grounded_history'); - if(result.format!=='project_producer_action_v1'||result.family!==family||result.action!==actions[action]||!id(result.producer_id)||(arguments_.producer_id&&result.producer_id!==arguments_.producer_id)||(arguments_.document?.request_id&&result.producer_id!==arguments_.document.request_id))invalid(); - } - } - if(definition.mutation){state={...state,project:{...state.project,checkpoint:value.checkpoint}};pending=null;showState();} - return value.value; - } - async function explore(ticket,more=false){ - const action=$('project-explore-kind').value,offset=more?exploreOffset:0,args={offset,limit:40}; - if(['policies','lineage'].includes(action))args.origin_catalog_id=state.project.project_id; - if(action==='producers')args.family=$('project-producer-family').value; - if(action==='data_query'){if($('project-data-search').value)args.q=$('project-data-search').value;if($('project-data-kind').value)args.kind=$('project-data-kind').value;} - const value=await m2Request(action,args,ticket);if(!value)return; - if(!more)$('project-explore-list').replaceChildren(); - for(const item of value.items){ - const row=document.createElement('li'),button=document.createElement('button');button.type='button'; - const fact=item.study??item.run??item.current??item.value??item,origin=item.origin_catalog_id??fact.ref?.catalog_id??state.project.project_id; - button.textContent=[fact.name??fact.id??item.sha256??item.id??'Saved record',fact.kind??fact.format??item.family??'',fact.state??fact.review??item.availability?.status??'',fact.revision?`revision ${fact.revision}`:''].filter(Boolean).join(' · '); - button.onclick=()=>run(ticket=>inspectItem(action,item,origin,ticket),{clean:true});row.append(button);$('project-explore-list').append(row); - } - exploreOffset=value.offset+value.items.length;exploreTotal=value.total;$('project-explore-more').hidden=exploreOffset>=exploreTotal; - $('project-explore-summary').textContent=`Showing ${value.total?offset+1:0}–${exploreOffset} of ${exploreTotal} saved records.`; - } - async function inspectItem(category,item,origin,ticket){ - const fact=item.study??item.run??item.current??item.value??item; - const entry={category,origin,id:fact.id??item.id,value:fact,checkpoint:checkpoint()}; - const request={data_query:['data_get','record_id'],models:['model_get','model_id'],graphs:['graph_get','graph_id'],recipes:['recipe_get','recipe_id'],studies:['study','study_id'],trials:['trial','trial_id'],runs:['run','run_id'],selections:['selection_get','selection_id']}[category]; - let detail=fact; - if(category==='producers'){detail=await m2Request('producer',{origin_catalog_id:origin,producer_id:entry.id,family:$('project-producer-family').value},ticket);if(!detail)return;} - if(request){const args={[request[1]]:entry.id};if(['studies','trials','runs'].includes(category))args.origin_catalog_id=origin;detail=await m2Request(request[0],args,ticket);if(!detail)return;} - clearExplorationDetail();entry.value=detail.study??detail;entry.saveAction={models:'model_save',graphs:'graph_save',recipes:'recipe_save',studies:'study_save'}[category];explored=entry; - $('project-detail-title').textContent=entry.value.name??`${category==='runs'?'Historical Run':category==='trials'?'Historical Trial':'Saved record'} ${entry.id??''}`; - $('project-detail-facts').append(projectValue(detail)); - if(category==='artifacts'){entry.artifactSha=detail.sha256;$('project-artifact-status').hidden=false;artifactResolutionForm(detail.availability?.state);} - if(category==='models')$('project-model-publish').hidden=false; - if(entry.saveAction&&origin===state.project.project_id){$('project-metadata-form').hidden=false;$('project-metadata-name').value=entry.value.name??'';} - if(category==='data_query'){ - if(entry.value.kind==='field_sequence'){scientificEpoch=ticket;projectScientific.show(entry.value);} - $('project-detail-history').hidden=false;$('project-detail-availability').hidden=false;$('project-rights-form').hidden=false;$('project-selection-form').hidden=false; - $('project-rights-note').value=entry.value.provenance?.rights_note_correction?.note??entry.value.provenance?.rights??'unknown'; - const key=entry.value.task?.endsWith('_classification')?'label':entry.value.task==='image_caption'?'caption':entry.value.task==='text_corpus'||['sequence','mesh','pointcloud','volume','field_sequence'].includes(entry.value.kind)?'note':null; - if(key){entry.annotationKey=key;$('project-data-note-form').hidden=false;$('project-data-note').value=entry.value.annotation?.[key]??'';$('project-data-note-help').textContent=`${entry.value.task.replaceAll('_',' ')} · save a ${key==='label'?'class label':key}.`;$('project-data-note').maxLength=key==='label'?80:4000;$('project-data-review').value=entry.value.review==='human_reviewed'?'human_reviewed':'draft';} - } - if(category==='runs'){entry.document=detail.document;$('project-run-document').hidden=!detail.document;$('project-run-evidence').hidden=false;$('project-run-disclose').hidden=!selectedContracts.has('project_files.expose_holdout');$('project-disclosure-label').hidden=$('project-run-disclose').hidden;} - controls();$('project-detail-title').focus(); - } - function clearExplorationDetail(){ - clearScientific(); - explored=null;historyOffset=0;historyTotal=0;$('project-detail-facts').replaceChildren();$('project-detail-history-list').replaceChildren(); - for(const name of ['project-metadata-form','project-data-note-form','project-rights-form','project-selection-form','project-detail-history','project-detail-history-more','project-detail-availability','project-run-evidence','project-run-document','project-run-disclose','project-disclosure-label','project-model-publish','project-artifact-status','project-artifact-resolve-form'])$(name).hidden=true; - } - async function dataHistory(ticket,more=false){ - if(!explored||explored.category!=='data_query')throw Error('Select saved data first.'); - const value=await m2Request('data_history',{record_id:explored.id,offset:more?historyOffset:0,limit:1},ticket);if(!value)return; - if(!more)$('project-detail-history-list').replaceChildren(); - for(const snapshot of value.items){const row=document.createElement('li');row.append(projectValue(snapshot));$('project-detail-history-list').append(row);} - historyOffset=value.offset+value.items.length;historyTotal=value.total;$('project-detail-history-more').hidden=historyOffset>=historyTotal; - message(`Saved revisions loaded: ${historyOffset} of ${historyTotal}.`); - } - async function metadataSaved(ticket,action,args){ - const value=await m2Request(action,args,ticket);if(!value)return; - const form={model_save:'project-metadata-form',graph_save:'project-metadata-form',recipe_save:'project-metadata-form',study_save:'project-metadata-form',data_annotate:'project-data-note-form',data_rights:'project-rights-form',selection_create:'project-selection-form',search_create:'project-search-form',data_import_image:'project-source-form',data_import_audio:'project-source-form',model_create:'project-model-create-form',study_create:'project-study-create-form',data_import_scientific:'project-scientific-form'}[action]; - if(form)for(const field of $(form).querySelectorAll('input'))if(!['number','file'].includes(field.type))field.value=''; - m2DirtyForms.delete(form);m2Dirty=m2DirtyForms.size>0; - if(!m2Dirty)clearExplorationDetail();await query(ticket);message('Change saved to the project. Load saved records to inspect the new revision.');controls(); - } - $('project-explore-load').onclick=()=>run(ticket=>explore(ticket),{clean:true}); - $('project-explore-more').onclick=()=>run(ticket=>explore(ticket,true),{clean:true}); - $('project-detail-history').onclick=()=>run(ticket=>dataHistory(ticket)); - $('project-detail-history-more').onclick=()=>run(ticket=>dataHistory(ticket,true)); - function artifactResolutionForm(availability){ - $('project-artifact-resolve-form').hidden=availability!=='missing'; - const roots=$('project-artifact-root'),previous=roots.value;roots.replaceChildren(); - for(const root of state.roots){const option=document.createElement('option');option.value=root;option.textContent=root;roots.append(option);} - if(state.roots.includes(previous))roots.value=previous; - } - async function artifactStatus(ticket){ - if(!explored?.artifactSha)throw Error('Select a saved artifact first.'); - const value=await m2Request('artifact_status',{sha256:explored.artifactSha},ticket);if(!value)return; - $('project-detail-facts').append(projectValue(value));artifactResolutionForm(value.state);controls(); - } - $('project-artifact-status').onclick=()=>run(artifactStatus); - $('project-artifact-resolve-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - if(!explored?.artifactSha)throw Error('Select a missing artifact first.'); - const expected=explored.artifactSha,value=await m2Request('artifact_resolve',{sha256:expected,root:$('project-artifact-root').value,path:$('project-artifact-path').value},ticket);if(!value)return; - if(value.format!=='project_artifact_resolution_v1'||value.sha256!==expected)throw Error('Artifact resolution observation differs from this artifact.'); - m2DirtyForms.delete('project-artifact-resolve-form');m2Dirty=m2DirtyForms.size>0;$('project-artifact-path').value=''; - $('project-detail-facts').append(projectValue(value));await artifactStatus(ticket);message('Artifact resolution observation saved. Current availability refreshed.'); - });}; - for(const field of ['project-artifact-root','project-artifact-path'])$(field).addEventListener('input',()=>{m2DirtyForms.add('project-artifact-resolve-form');m2Dirty=true;controls();}); - $('project-detail-availability').onclick=()=>run(async ticket=>{const value=await m2Request('data_availability',{record_id:explored.id},ticket);if(value)$('project-detail-facts').append(projectValue(value));}); - $('project-run-evidence').onclick=()=>run(async ticket=>{const value=await m2Request('evidence',{run_id:explored.id,origin_catalog_id:explored.origin},ticket);if(value){$('project-detail-facts').append(projectValue(value));message('Retained evidence read through current disclosure and integrity checks.');}}); - $('project-run-disclose').onclick=()=>run(async ticket=>{const value=await m2Request('expose_holdout',{run_id:explored.id,origin_catalog_id:explored.origin,reason:$('project-disclosure-reason').value},ticket);if(value){$('project-detail-facts').append(projectValue(value));message('Held-out disclosure allowed for this Run on this installation.');}},{clean:true}); - $('project-metadata-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const value=explored.value,name=$('project-metadata-name').value;if(!str(name,120))throw Error('Enter a name up to 120 characters.'); - const args=explored.category==='studies'?{study_id:explored.id,revision:value.revision,name,limits:value.limits,policy:value.policy}:{[{models:'model_id',graphs:'graph_id',recipes:'recipe_id'}[explored.category]]:explored.id,revision:value.revision,document:{...value,name}}; - await metadataSaved(ticket,explored.saveAction,args); - });}; - $('project-data-note-form').onsubmit=event=>{event.preventDefault();run(ticket=>metadataSaved(ticket,'data_annotate',{record_id:explored.id,revision:explored.value.revision,source_revision:explored.value.source_revision,task:explored.value.task,annotation:{[explored.annotationKey]:$('project-data-note').value},groups:explored.value.groups,review:$('project-data-review').value}));}; - $('project-rights-form').onsubmit=event=>{event.preventDefault();run(ticket=>metadataSaved(ticket,'data_rights',{record_id:explored.id,revision:explored.value.revision,source_revision:explored.value.source_revision,note:$('project-rights-note').value}));}; - $('project-selection-form').onsubmit=event=>{event.preventDefault();run(ticket=>metadataSaved(ticket,'selection_create',{document:{name:$('project-selection-name').value,items:[{id:explored.id,revision:explored.value.revision,source_revision:explored.value.source_revision}]}}));}; - $('project-search-form').onsubmit=event=>{event.preventDefault();run(ticket=>metadataSaved(ticket,'search_create',{document:{name:$('project-search-name').value,criteria:{q:$('project-data-search').value,kind:$('project-data-kind').value,review:'',sort:'',task:'',label:'',group:'',rights:''}}}));}; - $('project-source-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const file=$('project-source-file').files?.[0];if(!file||file.size>524288)throw Error('Choose a source file of at most 512 KiB.'); - const bytes=new Uint8Array(await file.arrayBuffer());let raw='';for(const byte of bytes)raw+=String.fromCharCode(byte); - if(ticket!==epoch)return; - const kind=$('project-source-kind').value,args={name:$('project-source-name').value,groups:groups($('project-source-groups').value),rights:$('project-source-rights').value,[kind==='audio'?'wav':'image']:btoa(raw)}; - await metadataSaved(ticket,'data_import_'+kind,args);$('project-source-file').value='';$('project-source-name').value=''; - });}; - for(const field of ['project-metadata-name','project-data-note','project-data-review','project-rights-note','project-selection-name','project-search-name','project-source-file','project-source-name','project-source-groups','project-source-rights'])$(field).addEventListener('input',()=>{const form=field.startsWith('project-source-')?'project-source-form':field.startsWith('project-metadata-')?'project-metadata-form':field.startsWith('project-data-')?'project-data-note-form':field.startsWith('project-rights-')?'project-rights-form':field.startsWith('project-selection-')?'project-selection-form':'project-search-form';m2DirtyForms.add(form);m2Dirty=true;controls();}); - $('project-source-kind').addEventListener('change',()=>{m2DirtyForms.add('project-source-form');m2Dirty=true;controls();}); - $('project-explore-kind').addEventListener('change',controls); - $('project-definitions-load').onclick=()=>run(async ticket=>{ - const value=await m2Request('definitions',{},ticket);if(!value)return; - if(!Array.isArray(value.recipes))throw Error('No model template definitions are available.'); - modelTemplates=value.recipes.filter(item=>item.model_template).map(item=>item.model_template);templateBinding=state.binding; - $('project-model-template').replaceChildren(); - for(const [index,template] of modelTemplates.entries()){const option=document.createElement('option');option.value=String(index);option.textContent=template.name;$('project-model-template').append(option);} - controls();message(modelTemplates.length?'Available model templates loaded.':'This installation has no model templates.'); - },{clean:true}); - $('project-model-create-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - if(templateBinding!==state.binding)throw Error('Load model templates for this project first.'); - const template=modelTemplates[Number($('project-model-template').value)];if(!template)throw Error('Choose a model template.'); - const document=JSON.parse(JSON.stringify(template));document.name=$('project-model-create-name').value; - await metadataSaved(ticket,'model_create',document); - });}; - $('project-model-publish').onclick=()=>run(ticket=>metadataSaved(ticket,'model_publish',{model_id:explored.id,revision:explored.value.revision}),{clean:true}); - for(const field of ['project-model-template','project-model-create-name'])$(field).addEventListener('input',()=>{m2DirtyForms.add('project-model-create-form');m2Dirty=true;controls();}); - $('project-study-create-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const limits={};for(const [key,field] of [['attempt_slots','attempts'],['admitted_wall_seconds','seconds'],['retained_bytes','retained'],['peak_staging_bytes','staging']]){const value=Number($('project-study-'+field).value);if(!Number.isSafeInteger(value)||value<1)throw Error('Study budget values must be positive whole numbers.');limits[key]=value;} - const metric=$('project-study-metric').value;if(!sha(metric))throw Error('Use the exact SHA256 of a saved metric definition.'); - await metadataSaved(ticket,'study_create',{name:$('project-study-create-name').value,limits,policy:{format:'selection_policy_v1',metric_definition_sha256:metric,checkpoint_rule:$('project-study-checkpoint').value,replicate_reduce:$('project-study-replicates').value,required_capabilities:[],numerical_acceptance:$('project-study-numerical').value,tie_break:'trial_id'}}); - });}; - function scientificReferenceHash(reference){ - if(reference.status!=='available'||!sha(reference.observed_sha256))return null; - if(Object.hasOwn(reference,'sha256'))return sha(reference.sha256)&&reference.integrity==='verified'&&reference.observed_sha256===reference.sha256?reference.observed_sha256:null; - return reference.integrity==='unverified'?reference.observed_sha256:null; - } - async function sourceReferences(ticket,more){ - if(!registeredSource||registeredSource.binding!==state.binding)throw Error('Register this source for the current project binding first.'); - const {origin_catalog_id,project_id,manifest_sha256}=registeredSource; - const value=await m2Request('source_references',{origin_catalog_id,project_id,manifest_sha256,offset:more?registeredSource.next_offset:0,limit:40},ticket);if(!value)return; - if(!more){registeredSource.references=[];$('project-scientific-reference').replaceChildren();} - for(const reference of value.items){ - registeredSource.references.push(reference);const observed=scientificReferenceHash(reference),option=document.createElement('option');option.value=reference.id;option.disabled=observed===null; - const integrity=observed?(Object.hasOwn(reference,'sha256')?'declared digest verified':'observed digest unverified; not pinned in the manifest'):(reference.integrity==='mismatch'?'declared digest mismatch':reference.status??'unavailable'); - option.textContent=`${reference.label??reference.id} · ${integrity}`;$('project-scientific-reference').append(option); - } - registeredSource.next_offset=value.offset+value.items.length;$('project-source-references-more').hidden=registeredSource.next_offset>=value.total; - } - $('project-source-references-more').onclick=()=>run(ticket=>sourceReferences(ticket,true)); - $('project-run-document').onclick=()=>run(async ticket=>{ - const descriptor=explored?.document;if(!descriptor||!sha(descriptor.sha256)||!Number.isSafeInteger(descriptor.bytes)||descriptor.bytes<0||descriptor.bytes>64*1024*1024)throw Error('This retained document cannot be downloaded in this browser. Use its exact project API segments.'); - const pieces=[];let offset=0; - do{ - const limit=262144,value=await m2Request('run_document_segment',{run_id:explored.id,origin_catalog_id:explored.origin,expected_sha256:descriptor.sha256,offset,limit},ticket);if(!value)return; - if(value.format!=='project_run_document_segment_v1'||value.origin_catalog_id!==explored.origin||value.run_id!==explored.id||value.sha256!==descriptor.sha256||value.bytes!==descriptor.bytes||value.offset!==offset||value.limit!==limit||typeof value.base64!=='string')throw Error('Retained document segment differs from the selected document.'); - const raw=atob(value.base64),bytes=Uint8Array.from(raw,char=>char.charCodeAt(0)); - if(bytes.length!==Math.min(limit,descriptor.bytes-offset)||value.next_offset!==(offset+bytes.lengthbyte.toString(16).padStart(2,'0')).join('')!==descriptor.sha256)throw Error('Retained document integrity differs; no download was created.'); - const link=document.createElement('a'),url=URL.createObjectURL(blob);link.href=url;link.download='retained-run-'+explored.id+'.json';link.click();URL.revokeObjectURL(url);message('Exact retained Run document downloaded and verified.'); - }); - $('project-register-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const value=await m2Request('source_register',{root:$('project-files-root').value,manifest_path:$('project-register-path').value},ticket);if(!value)return; - if(!id(value.project_id)||!id(value.origin_catalog_id)||!sha(value.manifest_sha256))throw Error('The registered source identity is unavailable.'); - registeredSource={binding:state.binding,origin_catalog_id:value.origin_catalog_id,project_id:value.project_id,manifest_sha256:value.manifest_sha256,references:[],next_offset:0}; - await sourceReferences(ticket,false); - m2DirtyForms.delete('project-register-form');m2Dirty=m2DirtyForms.size>0;controls();message('Source manifest saved. Select one exact declared data reference to import.'); - });}; - $('project-scientific-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - if(!registeredSource||registeredSource.binding!==state.binding)throw Error('Register a local source for this project binding first.'); - const reference=registeredSource.references.find(item=>item.id===$('project-scientific-reference').value),observed=reference&&scientificReferenceHash(reference);if(!observed)throw Error('Choose an available data reference with a current observed digest. Declared digests must match and be verified.'); - const {origin_catalog_id,project_id,manifest_sha256}=registeredSource; - await metadataSaved(ticket,'data_import_scientific',{snapshot:{origin_catalog_id,project_id,manifest_sha256},reference_id:reference.id,expected_sha256:observed,document:{name:$('project-scientific-name').value,groups:groups($('project-scientific-groups').value),rights:$('project-scientific-rights').value}}); - });}; - for(const [form,fields] of [['project-study-create-form',['project-study-create-name','project-study-metric','project-study-attempts','project-study-seconds','project-study-retained','project-study-staging','project-study-checkpoint','project-study-replicates','project-study-numerical']],['project-register-form',['project-register-path']],['project-scientific-form',['project-scientific-reference','project-scientific-name','project-scientific-groups','project-scientific-rights']]])for(const field of fields)$(field).addEventListener('input',()=>{m2DirtyForms.add(form);m2Dirty=true;controls();}); - function executionControls(){ - const prepared=!!executionPrepared&&executionPrepared.binding===state?.binding,approved=!!executionApproval&&executionApproval.binding===state?.binding; - producerControls();nativeControls(); - $('project-execution-reconcile').disabled=busy||blocked||!state?.project||!['open','recovery_required'].includes(state.state)||!selectedContracts.has('project_files.execution_reconcile_current'); - $('project-execution-graphs-more').disabled=!m2Allowed('graphs');$('project-execution-graphs-more').hidden=executionGraphOffset>=executionGraphTotal; - $('project-study-trials-more').disabled=!executionStudy||!m2Allowed('study');$('project-study-trials-more').hidden=studyTrialOffset>=studyTrialTotal; - $('project-execution-health').disabled=!preparationRecoveryAllowed('execution_health'); - $('project-execution-health-more').disabled=!preparationRecoveryAllowed('execution_health');$('project-execution-health-more').hidden=executionOffset>=executionTotal; - $('project-execution-graphs').disabled=!m2Allowed('graphs'); - $('project-execution-preflight-fields').disabled=!executionGraphs.length||!m2Allowed('execution_preflight_external'); - $('project-execution-approval-fields').disabled=!prepared||!m2Allowed('execution_approve_external'); - $('project-execution-submit-fields').disabled=!prepared||!approved||!m2Allowed('execution_submit_external'); - $('project-execution-cancel-fields').disabled=!m2Allowed('execution_cancel'); - $('project-study-execution-load').disabled=explored?.category!=='studies'||!m2Allowed('study'); - $('project-study-continue-fields').disabled=!executionStudy||!m2Allowed('execution_study_continue'); - $('project-study-approval-fields').disabled=!executionStudy||!executionTrials.length||!m2Allowed('execution_study_approve'); - $('project-study-admit-fields').disabled=!executionStudy||!studyApproval||!m2Allowed('execution_study_admit'); - $('project-study-stop-fields').disabled=!executionStudy||!m2Allowed('execution_study_stop'); - } - function executionFact(value){$('project-execution-facts').replaceChildren(projectValue(value));} - function nativePhaseOK(value,phaseId=null){ - return exact(value,['format','origin_catalog_id','phase_id','phase','run_id','state','spec_sha256','receipt_sha256'])&&value.format==='project_native_phase_v1'&&value.origin_catalog_id===state.project.project_id&&id(value.phase_id)&&(!phaseId||value.phase_id===phaseId)&&['probe','train','verify'].includes(value.phase)&&(value.run_id===null||id(value.run_id))&&['prepared','ready','go_intent','completed','failed','unknown'].includes(value.state)&&sha(value.spec_sha256)&&(value.receipt_sha256===null||sha(value.receipt_sha256)); - } - function nativeValidationOK(value,revision){ - const fields=['status','valid','errors','validation_token','graph_revision','execution_sha256','release','runtime']; - if(!value||typeof value!=='object'||Array.isArray(value)||!fields.every(field=>Object.hasOwn(value,field))||!Object.keys(value).every(field=>fields.includes(field)||field==='selection_policy')||!str(value.status,120)||typeof value.valid!=='boolean'||!Array.isArray(value.errors)||value.errors.length>100||!positive(value.graph_revision)||value.graph_revision!==revision||!sha(value.execution_sha256)||!['release','runtime'].every(field=>value[field]===null||typeof value[field]==='object'&&!Array.isArray(value[field])))return false; - if(!value.errors.every(error=>exact(error,['path','message','code'])&&Object.values(error).every(part=>str(part,2000,true))))return false; - return value.valid?value.status==='valid'&&value.errors.length===0&&sha(value.validation_token)&&value.release!==null&&value.runtime!==null&&value.runtime.available===true:value.errors.length>0&&value.status===value.errors[0].code&&value.validation_token===null; - } - function nativeAdmissionOK(value){return exact(value,['format','origin_catalog_id','run_id','request_sha256'])&&value.format==='project_run_admission_v1'&&value.origin_catalog_id===state.project.project_id&&id(value.run_id)&&sha(value.request_sha256);} - function nativeControls(){ - $('project-native-graphs').disabled=!m2Allowed('graphs');$('project-native-more').disabled=!m2Allowed('graphs');$('project-native-more').hidden=nativeOffset>=nativeTotal; - $('project-native-preflight-fields').disabled=!nativeGraphs.length||!m2Allowed('native_preflight'); - $('project-native-submit-fields').disabled=!nativePrepared||nativePrepared.binding!==state?.binding||nativePrepared.checkpoint!==checkpoint()||!m2Allowed('native_submit'); - $('project-native-retry-fields').disabled=!m2Allowed('native_retry'); - $('project-native-reconcile-fields').disabled=!preparationRecoveryAllowed('native_reconcile'); - $('project-native-pending-retry').disabled=!nativePending||nativePending.body.binding!==state?.binding||!preparationRecoveryAllowed(nativePending.action); - } - async function loadNativeGraphs(ticket,more=false){ - const value=await m2Request('graphs',{offset:more?nativeOffset:0,limit:40},ticket);if(!value)return; - if(!more){nativeGraphs=[];nativePrepared=null;$('project-native-graph').replaceChildren();} - for(const graph of value.items.filter(graph=>graph.recipe!=='external_recipe')){if(!id(graph.id)||!positive(graph.revision))throw Error('Saved native graph has invalid identity.');nativeGraphs.push(graph);const option=document.createElement('option');option.value=graph.id;option.textContent=`${graph.name??graph.id} · ${graph.recipe??'native'} · revision ${graph.revision}`;$('project-native-graph').append(option);} - nativeOffset=value.offset+value.items.length;nativeTotal=value.total;controls();message('Saved graph metadata loaded. No native probe or Run started.'); - } - $('project-native-graphs').onclick=()=>run(ticket=>loadNativeGraphs(ticket));$('project-native-more').onclick=()=>run(ticket=>loadNativeGraphs(ticket,true)); - $('project-native-graph').addEventListener('change',()=>{nativePrepared=null;$('project-native-probe-confirm').checked=false;$('project-native-submit-confirm').checked=false;controls();}); - $('project-native-preflight-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const graph=nativeGraphs.find(value=>value.id===$('project-native-graph').value);if(!graph||!$('project-native-probe-confirm').checked)throw Error('Select a saved native graph and explicitly authorize its current CPU probe.'); - nativePrepared=null;controls(); - const value=await m2Request('native_preflight',{graph_id:graph.id,revision:graph.revision},ticket);if(!value)return; - if(!nativeValidationOK(value,graph.revision))throw Error('Native validation differs from the selected saved graph.'); - nativePending=null;nativePrepared=value.valid?{binding:state.binding,checkpoint:checkpoint(),graph_id:graph.id,revision:graph.revision,validation_token:value.validation_token}:null; - executionFormSaved('project-native-preflight-form');$('project-native-probe-confirm').checked=false;$('project-native-submit-confirm').checked=false;$('project-native-facts').replaceChildren(projectValue(value));$('project-native-pending').textContent='Native validation acknowledgement saved.';controls();message(value.valid?'Native validation succeeded. Starting a Run requires a separate explicit action.':'Native validation reported errors. No Run was started.'); - });}; - $('project-native-submit-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - if(!nativePrepared||nativePrepared.binding!==state.binding||nativePrepared.checkpoint!==checkpoint()||!$('project-native-submit-confirm').checked)throw Error('Validate this current graph and separately authorize starting its Run.'); - const value=await m2Request('native_submit',{graph_id:nativePrepared.graph_id,graph_revision:nativePrepared.revision,validation_token:nativePrepared.validation_token},ticket);if(!value)return;if(!nativeAdmissionOK(value))throw Error('Native admission differs from the selected project.'); - nativePending=null;nativePrepared=null;$('project-native-run-id').value=value.run_id;$('project-execution-run-id').value=value.run_id;$('project-native-submit-confirm').checked=false;executionFormSaved('project-native-submit-form');$('project-native-facts').replaceChildren(projectValue(value));$('project-native-pending').textContent='Native admission acknowledged. Refresh current custody to inspect execution.';controls();message($('project-native-pending').textContent); - });}; - $('project-native-retry-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const run_id=$('project-native-run-id').value;if(!id(run_id)||!$('project-native-retry-confirm').checked)throw Error('Enter a current native Run ID and explicitly authorize a new attempt.'); - const value=await m2Request('native_retry',{run_id},ticket);if(!value)return;if(!nativeAdmissionOK(value)||value.run_id===run_id)throw Error('Native retry admission differs from the selected project.'); - nativePending=null;$('project-native-run-id').value=value.run_id;$('project-execution-run-id').value=value.run_id;$('project-native-retry-confirm').checked=false;executionFormSaved('project-native-retry-form');$('project-native-facts').replaceChildren(projectValue(value));$('project-native-pending').textContent='Native retry admission acknowledged. A saved admission is not proof of completion.';controls();message($('project-native-pending').textContent); - });}; - $('project-native-pending-retry').onclick=()=>run(async ticket=>{ - if(!nativePending||nativePending.body.binding!==state.binding)throw Error('This tab has no exact current pending native request.'); - const saved=nativePending,{binding,expected_checkpoint,request_key,...args}=saved.body;if(saved.action==='native_preflight'){nativePrepared=null;controls();}const value=await m2Request(saved.action,args,ticket);if(!value)return; - if(saved.action==='native_preflight'){ - if(!nativeValidationOK(value,args.revision))throw Error('Native validation differs from the original saved graph.'); - nativePrepared=value.valid?{binding:state.binding,checkpoint:checkpoint(),graph_id:args.graph_id,revision:args.revision,validation_token:value.validation_token}:null; - executionFormSaved('project-native-preflight-form');$('project-native-probe-confirm').checked=false; - }else{ - if(!nativeAdmissionOK(value)||saved.action==='native_retry'&&value.run_id===args.run_id)throw Error('Native acknowledgement differs from the selected project.');nativePrepared=null;$('project-native-run-id').value=value.run_id;$('project-execution-run-id').value=value.run_id; - const kind=saved.action==='native_submit'?'submit':'retry';executionFormSaved('project-native-'+kind+'-form');$('project-native-'+kind+'-confirm').checked=false; - } - nativePending=null;$('project-native-facts').replaceChildren(projectValue(value));$('project-native-pending').textContent='Exact pending native request acknowledged. No new request key was created.';controls();message($('project-native-pending').textContent); - }); - $('project-native-reconcile-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const phase_id=$('project-native-phase-id').value;if(!id(phase_id))throw Error('Enter the original current local phase ID.'); - const value=await m2Request('native_reconcile',{phase_id},ticket);if(!value)return;if(!nativePhaseOK(value,phase_id))throw Error('Native reconciliation differs from the original phase.'); - executionFormSaved('project-native-reconcile-form');$('project-native-facts').replaceChildren(projectValue(value));message('Original current native phase observed. Reconciliation does not start or resume it. Refresh project status before another action.'); - });}; - for(const [form,fields] of [['project-native-preflight-form',['project-native-probe-confirm']],['project-native-submit-form',['project-native-submit-confirm']],['project-native-retry-form',['project-native-run-id','project-native-retry-confirm']],['project-native-reconcile-form',['project-native-phase-id']]])for(const field of fields)$(field).addEventListener(field.endsWith('confirm')?'change':'input',()=>{m2DirtyForms.add(form);m2Dirty=true;controls();}); - function localConsumer(){const actor=window.TuldokAuth.identity?.()?.actor_id;return id(actor)?actor:'';} - async function currentHealth(ticket,more=false){ - const value=await m2Request('execution_health',{offset:more?executionOffset:0,limit:40},ticket);if(!value)return; - if(value.format!=='project_execution_health_v1'||!Array.isArray(value.active_run_ids)||typeof value.drain_required!=='boolean'||value.offset!==(more?executionOffset:0)||value.limit!==40||value.active_run_ids.length!==Math.min(40,Math.max(0,value.active_count-value.offset)))throw Error('Unsupported current custody observation.'); - let nativeTotal=0; - if(selectedVersions.get('project_files.execution_health')>=2){ - const page=value.native_phases,ids=page?.items?.map(item=>item.phase_id); - if(!page||!exact(page,['items','total','offset','limit'])||!Array.isArray(page.items)||!Number.isSafeInteger(page.total)||page.total<0||page.offset!==(more?executionOffset:0)||page.limit!==40||page.items.length!==Math.min(40,Math.max(0,page.total-page.offset))||!page.items.every(item=>nativePhaseOK(item))||ids.join()!==[...new Set(ids)].sort().join())throw Error('Current native phases differ from the selected project or continuation.'); - nativeTotal=page.total;if(!more)$('project-native-phases').replaceChildren(); - for(const phase of page.items){const li=document.createElement('li'),button=document.createElement('button');button.type='button';button.textContent=`${phase.phase_id} · ${phase.phase} · ${phase.state}`;button.onclick=()=>{$('project-native-phase-id').value=phase.phase_id;$('project-native-phase-id').focus();};li.append(button);$('project-native-phases').append(li);} - }else if(Object.hasOwn(value,'native_phases'))throw Error('Version 1 cannot represent native phase custody. Refresh contracts.'); - executionOffset=(more?executionOffset:0)+40;executionTotal=Math.max(value.active_count,nativeTotal);executionFact(value); - $('project-execution-summary').textContent=`${value.active_count} current active or unresolved Runs; ${nativeTotal} original current native phases. Local owner ${value.local_owner_active?'active':'inactive'}. ${value.drain_required?'Explicit reconciliation must drain current custody before closing or extracting.':value.available?'The project projection is available.':'The current outcome needs explicit reconciliation.'}`; - if(value.active_run_ids.length)$('project-execution-run-id').value=value.active_run_ids[0];controls(); - } - $('project-execution-health').onclick=()=>run(ticket=>currentHealth(ticket)); - $('project-execution-health-more').onclick=()=>run(ticket=>currentHealth(ticket,true)); - $('project-execution-reconcile').onclick=()=>run(async ticket=>{const value=await m2Request('execution_reconcile_current',{},ticket);if(!value)return;executionApproval=null;studyApproval=null;executionFact(value);controls();message(value.available?'Current local custody reconciled and drained. No imported admission was started.':'Current reconciliation remains unavailable; custody is still fenced.');}); - async function loadExecutionGraphs(ticket,more=false){ - const value=await m2Request('graphs',{offset:more?executionGraphOffset:0,limit:40},ticket);if(!value)return; - if(!more){executionGraphs=[];$('project-execution-graph').replaceChildren();}const graphs=value.items.filter(graph=>graph.recipe==='external_recipe');executionGraphs.push(...graphs);executionGraphOffset=value.offset+value.items.length;executionGraphTotal=value.total;executionPrepared=null;executionApproval=null; - for(const graph of graphs){const option=document.createElement('option');option.value=graph.id;option.textContent=`${graph.name} · revision ${graph.revision}`;$('project-execution-graph').append(option);} - $('project-execution-consumer').value=localConsumer();$('project-study-consumer').value=localConsumer();controls();message(executionGraphs.length?'Select an external graph and inspect its exact current material.':'No external graphs on this page. Create one through the named external graph operation.'); - } - $('project-execution-graphs').onclick=()=>run(ticket=>loadExecutionGraphs(ticket),{clean:true}); - $('project-execution-graphs-more').onclick=()=>run(ticket=>loadExecutionGraphs(ticket,true),{clean:true}); - $('project-execution-graph').addEventListener('change',()=>{executionPrepared=null;executionApproval=null;$('project-execution-grant').textContent='No current local approval selected.';controls();}); - $('project-execution-preflight-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const graph=executionGraphs.find(value=>value.id===$('project-execution-graph').value);if(!graph)throw Error('Select a saved external graph.'); - const value=await m2Request('execution_preflight_external',{graph_id:graph.id,revision:graph.revision},ticket);if(!value)return; - if(!sha(value.material_sha256)||value.graph_revision!==graph.revision)throw Error('Current material does not match the selected graph revision.'); - executionPrepared={binding:state.binding,graph_id:graph.id,graph_revision:graph.revision,expected_digest:value.material_sha256};executionApproval=null;$('project-execution-confirm').checked=false;executionFact(value);controls(); - message('Exact current material inspected. Review its sources, inputs and runtime before approving. No Run has been admitted.'); - });}; - $('project-execution-approval-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - if(!executionPrepared||executionPrepared.binding!==state.binding)throw Error('Inspect the exact current material first.'); - if(!$('project-execution-confirm').checked)throw Error('Explicitly approve the reviewed material before granting permission.'); - const consumer=$('project-execution-consumer').value,ttl=Number($('project-execution-ttl').value);if(!id(consumer)||!Number.isInteger(ttl)||ttl<60||ttl>3600)throw Error('Choose the current execution actor and an approval lifetime of 60–3600 seconds.'); - const {binding,...prepared}=executionPrepared,value=await m2Request('execution_approve_external',{...prepared,consumer_actor_id:consumer,ttl_seconds:ttl},ticket);if(!value)return; - if(value.format!=='project_current_execution_approval_v1'||value.current_local!==true||value.grant?.material_sha256!==prepared.expected_digest||value.grant.consumer_actor_id!==consumer||!id(value.grant.id))throw Error('Current local approval does not match the reviewed request.'); - executionFormSaved('project-execution-approval-form');executionApproval={binding:state.binding,grant:value.grant};$('project-execution-grant').textContent=`Current local approval ${value.grant.id} for actor ${consumer}, expires ${new Date(value.grant.expires_at*1000).toLocaleString()}.`; - controls();message('Current local approval granted. Starting the Run is a separate explicit action.'); - });}; - $('project-execution-submit-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - if(executionPrepared?.binding!==state.binding||executionApproval?.binding!==state.binding)throw Error('Review and approve the current material on this installation first.'); - const {binding,...prepared}=executionPrepared,value=await m2Request('execution_submit_external',{...prepared,approval_id:executionApproval.grant.id},ticket);if(!value)return; - $('project-execution-run-id').value=value.run_id;executionApproval=null;executionFact(value);controls();message(`Admission saved for Run ${value.run_id}. This acknowledges durable intent; refresh current custody to inspect execution.`); - });}; - $('project-execution-cancel-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const run_id=$('project-execution-run-id').value;if(!id(run_id))throw Error('Choose a current local Run ID.'); - const value=await m2Request('execution_cancel',{run_id},ticket);if(!value)return;executionFact(value);message('Cancellation intent saved. Refresh current custody to establish whether execution has ceased.'); - });}; - $('project-study-execution-load').onclick=()=>run(async ticket=>{ - if(explored?.category!=='studies'||explored.origin!==state.project.project_id)throw Error('Select a current-namespace Study in the scientific history view.'); - const value=await m2Request('study',{study_id:explored.id,origin_catalog_id:explored.origin,offset:0,limit:40},ticket);if(!value)return; - executionStudy=value.study;studyApproval=null;executionTrials=value.trials.items;studyTrialOffset=value.trials.offset+value.trials.items.length;studyTrialTotal=value.trials.total;$('project-study-execution-trial').replaceChildren(); - for(const trial of executionTrials){const option=document.createElement('option');option.value=trial.id;option.textContent=trial.id;$('project-study-execution-trial').append(option);} - $('project-study-consumer').value=localConsumer();$('project-study-execution-summary').textContent=`${executionStudy.name} · revision ${executionStudy.revision} · epoch ${executionStudy.admission_epoch} · ${executionStudy.state}. ${value.trials.total} prepared Trials.`;controls(); - },{clean:true}); - $('project-study-trials-more').onclick=()=>run(async ticket=>{const value=await m2Request('study',{study_id:executionStudy.id,origin_catalog_id:state.project.project_id,revision:executionStudy.revision,offset:studyTrialOffset,limit:40},ticket);if(!value)return;for(const trial of value.trials.items){executionTrials.push(trial);const option=document.createElement('option');option.value=trial.id;option.textContent=trial.id;$('project-study-execution-trial').append(option);}studyTrialOffset=value.trials.offset+value.trials.items.length;studyTrialTotal=value.trials.total;controls();}); - $('project-study-execution-trial').addEventListener('change',()=>{studyApproval=null;controls();}); - $('project-study-continue-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const value=await m2Request('execution_study_continue',{study_id:executionStudy.id,revision:executionStudy.revision,admission_epoch:executionStudy.admission_epoch},ticket);if(!value)return;executionStudy=value;controls();$('project-study-execution-summary').textContent=`Current epoch ${value.admission_epoch} · revision ${value.revision} · ${value.state}.`;message('Current Study epoch saved. No Trial has been admitted by this action.'); - });}; - $('project-study-approval-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const trial_id=$('project-study-execution-trial').value,consumer_actor_id=$('project-study-consumer').value;if(!id(trial_id)||!id(consumer_actor_id))throw Error('Select a prepared Trial and current execution actor.'); - const trial=await m2Request('trial',{trial_id,origin_catalog_id:state.project.project_id},ticket);if(!trial)return; - const value=await m2Request('execution_study_approve',{study_id:executionStudy.id,trial_id,expected_material_sha256:trial.material.sha256,consumer_actor_id},ticket);if(!value)return;executionFormSaved('project-study-approval-form');studyApproval={binding:state.binding,trial_id,grant:value.grant};executionFact(value);controls();message('Exact Trial material approved locally. Admission is a separate explicit action.'); - });}; - $('project-study-admit-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - if(studyApproval?.binding!==state.binding||studyApproval.trial_id!==$('project-study-execution-trial').value)throw Error('Approve this exact Trial under current local authority first.'); - const value=await m2Request('execution_study_admit',{study_id:executionStudy.id,revision:executionStudy.revision,admission_epoch:executionStudy.admission_epoch,trial_id:studyApproval.trial_id,approval_id:studyApproval.grant.id},ticket);if(!value)return; - $('project-execution-run-id').value=value.run_id;studyApproval=null;executionStudy=null;executionFact(value);controls();message('Trial admission and budget reservation saved. Reload the Study before another epoch action; refresh current custody for execution status.'); - });}; - $('project-study-stop-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const value=await m2Request('execution_study_stop',{study_id:executionStudy.id,revision:executionStudy.revision},ticket);if(!value)return;executionStudy=null;studyApproval=null;executionFact(value);controls();message('Finite Study epoch closure saved and cancellation requested. Current child custody must still prove cessation.'); - });}; - function producerControls(){ - const family=$('project-producer-task').value,current=!!producerCurrent&&producerCurrent.binding===state?.binding; - $('project-producer-health').disabled=!m2Allowed('execution_producer_health');$('project-producer-health-more').disabled=!m2Allowed('execution_producer_health');$('project-producer-health-more').hidden=producerHealthOffset>=producerHealthTotal; - $('project-producer-start-fields').disabled=!m2Allowed('execution_producer_start'); - $('project-producer-source-load').disabled=explored?.category!=='data_query'||!m2Allowed('data_get'); - $('project-producer-current-load').disabled=explored?.category!=='producers'||explored.origin!==state?.project?.project_id||!m2Allowed('producer'); - $('project-producer-label-fields').hidden=family!=='text_proposal_history'; - $('project-producer-count-fields').hidden=!['generation_history','grounded_history'].includes(family); - $('project-producer-generation-fields').hidden=family!=='generation_history'; - $('project-producer-resume-fields').disabled=!current||producerCurrent.family!=='generation_history'||!m2Allowed('execution_generation_resume'); - $('project-producer-review-fields').disabled=!current||producerCurrent.family!=='grounded_history'||!m2Allowed('execution_grounded_review'); - $('project-producer-decision-fields').disabled=!current||!['caption_proposal_history','text_proposal_history'].includes(producerCurrent.family)||!m2Allowed('execution_producer_decide'); - $('project-producer-cancel-fields').disabled=!current||!m2Allowed('execution_producer_cancel'); - } - async function producerHealth(ticket,more=false){const value=await m2Request('execution_producer_health',{offset:more?producerHealthOffset:0,limit:40},ticket);if(!value)return;producerHealthOffset=value.offset+value.items.length;producerHealthTotal=value.total;executionFact(value);$('project-producer-health-summary').textContent=`${value.total} current local producer admissions. ${value.fenced?'Current custody is fenced.':'Inspect each current worker and provider outcome before closing.'}`;controls();} - $('project-producer-health').onclick=()=>run(ticket=>producerHealth(ticket)); - $('project-producer-health-more').onclick=()=>run(ticket=>producerHealth(ticket,true)); - function executionFormSaved(form){m2DirtyForms.delete(form);m2Dirty=m2DirtyForms.size>0;} - $('project-producer-task').addEventListener('change',()=>{producerSource=null;$('project-producer-source-summary').textContent='Select matching saved data as source.';controls();}); - $('project-producer-source-load').onclick=()=>run(async ticket=>{ - if(explored?.category!=='data_query')throw Error('Select saved data in the scientific history view.'); - const value=await m2Request('data_get',{record_id:explored.id},ticket);if(!value)return; - const family=$('project-producer-task').value,kind=family==='caption_proposal_history'?'image':'text'; - if(family==='generation_history'||value.kind!==kind)throw Error(`This task requires selected ${kind} data.`); - producerSource={binding:state.binding,source_id:value.id,revision:value.revision,source_revision:value.source_revision};$('project-producer-source-summary').textContent=`${value.name} · revision ${value.revision} · source revision ${value.source_revision}`;controls(); - }); - $('project-producer-start-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const family=$('project-producer-task').value;if(!$('project-producer-confirm').checked)throw Error('Explicitly authorize this model request.'); - const instruction=$('project-producer-instruction').value,seed=Number($('project-producer-seed').value); - if(!Number.isInteger(seed)||seed<0||seed>4294967295)throw Error('Use a nonnegative 32-bit seed.'); - const document={server_url:$('project-producer-url').value,model:$('project-producer-model').value,seed}; - if($('project-producer-transport').value==='typed'){document.protocol='pumas_typed_v1';document.profile=$('project-producer-profile').value||null;if(family!=='generation_history')document.seed=null;} - if(family==='generation_history'){ - Object.assign(document,{prompt:instruction,count:Number($('project-producer-count').value),strategy:$('project-producer-strategy').value,session_id:$('project-producer-session').value,width:Number($('project-producer-width').value),height:Number($('project-producer-height').value)}); - if(document.strategy==='varied'){document.prompt_url=$('project-producer-prompt-url').value;document.prompt_model=$('project-producer-prompt-model').value;} - }else{ - if(producerSource?.binding!==state.binding)throw Error('Load exact saved source revisions for this project first.'); - const {binding,...source}=producerSource;Object.assign(document,source,{instruction}); - if(family==='grounded_history')document.count=Number($('project-producer-count').value);else document.request_id=key(); - if(family==='text_proposal_history')document.labels=$('project-producer-labels').value.split('\n').filter(value=>value!==''); - } - const value=await m2Request('execution_producer_start',{family,document},ticket);if(!value)return; - executionFormSaved('project-producer-start-form');$('project-producer-confirm').checked=false;producerCurrent=null;executionFact(value);controls();message(`Producer admission saved: ${value.producer_id}. Read its saved history for captured results and current observations.`); - });}; - $('project-producer-current-load').onclick=()=>run(async ticket=>{ - if(explored?.category!=='producers'||explored.origin!==state.project.project_id)throw Error('Select current-namespace producer history first.'); - const family=explored.value.family,value=await m2Request('producer',{family,producer_id:explored.id,origin_catalog_id:explored.origin,entries_offset:0,entries_limit:40},ticket);if(!value)return; - producerCurrent={binding:state.binding,family,producer_id:explored.id,job:value.job};$('project-producer-current-summary').textContent=`${family.replaceAll('_',' ')} · ${explored.id} · ${value.status}. Retained history alone does not grant current authority.`; - $('project-producer-candidate').replaceChildren();for(const candidate of value.job.candidates??[]){const option=document.createElement('option');option.value=candidate.id;option.textContent=`${candidate.text} · ${candidate.status}`;$('project-producer-candidate').append(option);}controls(); - },{clean:true}); - async function producerAction(ticket,action,document){ - if(producerCurrent?.binding!==state.binding)throw Error('Load a current-namespace producer first.'); - const {family,producer_id}=producerCurrent,args={producer_id};if(!['execution_generation_resume','execution_grounded_review'].includes(action))args.family=family;if(document)args.document=document; - const value=await m2Request(action,args,ticket);if(!value)return; - for(const form of ['project-producer-review-form','project-producer-decision-form'])executionFormSaved(form); - producerCurrent=null;executionFact(value);controls();message(action==='execution_producer_cancel'?'Cancellation intent saved. Refresh current producer custody; a retained acknowledgement does not prove cessation.':'Scientific producer action saved. Reload its history before another action.'); - } - $('project-producer-resume-form').onsubmit=event=>{event.preventDefault();run(ticket=>producerAction(ticket,'execution_generation_resume'));}; - $('project-producer-cancel-form').onsubmit=event=>{event.preventDefault();run(ticket=>producerAction(ticket,'execution_producer_cancel'));}; - $('project-producer-decision-form').onsubmit=event=>{event.preventDefault();run(ticket=>{ - const document={revision:producerCurrent.job.revision,decision:$('project-producer-decision').value};if(producerCurrent.family==='text_proposal_history'&&document.decision==='apply_draft')document.labels=producerCurrent.job.config.labels;return producerAction(ticket,'execution_producer_decide',document); - });}; - $('project-producer-review-form').onsubmit=event=>{event.preventDefault();run(ticket=>producerAction(ticket,'execution_grounded_review',{revision:producerCurrent.job.revision,candidate_id:$('project-producer-candidate').value,decision:$('project-producer-review-decision').value,note:$('project-producer-note').value}));}; - - const preparationRecoveryAllowed=action=>!busy&&!!state?.project&&['open','recovery_required'].includes(state.state)&&selectedContracts.has('project_files.'+action); - function candidateControls(){ - const kind=$('project-candidate-kind').value||'package';$('project-candidate-verifier-fields').hidden=kind!=='verifier'; - $('project-candidate-prepare-fields').disabled=!m2Allowed('execution_study_prepare_'+kind); - const reconcile=preparationRecoveryAllowed('execution_study_reconcile_preparation'),abandon=preparationRecoveryAllowed('execution_study_abandon_preparation'); - $('project-candidate-recovery-fields').disabled=!(reconcile||abandon);$('project-candidate-reconcile').disabled=!reconcile;$('project-candidate-abandon').disabled=!abandon; - } - function candidatePreparedOK(value,study,candidate=null,runtime=null){ - const keys=['state','candidate_id','trial','graph','model','runtime_snapshot_sha256',...(Object.hasOwn(value??{},'package_run_id')?['package_run_id','package_sha256']:[])]; - return exact(value,keys)&&value.state==='prepared'&&id(value.candidate_id)&&(!candidate||value.candidate_id===candidate)&&sha(value.runtime_snapshot_sha256)&&(!runtime||value.runtime_snapshot_sha256===runtime)&&value.trial?.study_id===study&&id(value.trial?.id)&&value.trial.material?.catalog_id===state.project.project_id&&value.model?.catalog_id===state.project.project_id&&id(value.graph?.id)&&positive(value.graph?.revision)&&(!Object.hasOwn(value,'package_run_id')||id(value.package_run_id)&&sha(value.package_sha256)); - } - $('project-candidate-prepare-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const study_id=$('project-candidate-study').value,candidate_id=$('project-candidate-id').value,revision=Number($('project-candidate-revision').value),runtime_snapshot_sha256=$('project-candidate-runtime').value,root=$('project-candidate-root').value,directory=$('project-candidate-directory').value,kind=$('project-candidate-kind').value; - if(!id(study_id)||!id(candidate_id)||!positive(revision)||!sha(runtime_snapshot_sha256)||!root||!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,119}$/.test(directory)||!['package','verifier'].includes(kind))throw Error('Choose exact current Study/Candidate identities, revision, publication directory and captured runtime digest.'); - const args={study_id,candidate_id,revision,root,directory,runtime_snapshot_sha256}; - if(kind==='verifier'){args.package_run_id=$('project-candidate-package-run').value;args.package_sha256=$('project-candidate-package-sha').value;if(!id(args.package_run_id)||!sha(args.package_sha256))throw Error('Choose the exact completed package Run and SHA256.');} - const value=await m2Request('execution_study_prepare_'+kind,args,ticket);if(!value)return; - if(!candidatePreparedOK(value,study_id,candidate_id,runtime_snapshot_sha256)||kind==='verifier'&&(value.package_run_id!==args.package_run_id||value.package_sha256!==args.package_sha256))throw Error('Preparation result differs from its current Study, Candidate or captured runtime.'); - executionFormSaved('project-candidate-prepare-form');$('project-candidate-preparation-result').textContent=JSON.stringify(value,null,2);$('project-candidate-preparation-status').textContent='Preparation acknowledged. This saved Trial still requires current local approval and a separate explicit admission.';message('Candidate preparation saved. No Run was launched.'); - });}; - async function candidateRecover(ticket,abandon){ - const study_id=$('project-candidate-recovery-study').value,preparation_key=$('project-candidate-recovery-key').value; - if(!id(study_id)||!/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(preparation_key))throw Error('Choose the exact current Study and original preparation request key.'); - const action='execution_study_'+(abandon?'abandon':'reconcile')+'_preparation';if(!state?.project||!['open','recovery_required'].includes(state.state)||!selectedContracts.has('project_files.'+action))throw Error('Current preparation recovery is unavailable.'); - const args={study_id,preparation_key};if(abandon){args.reason=$('project-candidate-abandon-reason').value;if(!str(args.reason,1000))throw Error('Enter an explicit reason before abandoning the stopped partial preparation.');} - const value=await m2Request(action,args,ticket);if(!value)return; - if(abandon){if(value.format!=='project_candidate_preparation_abandon_v1'||value.state!=='abandoned'||value.origin_catalog_id!==state.project.project_id||value.study_id!==study_id||value.preparation_key!==preparation_key||value.reason!==args.reason||!sha(value.input_sha256)||!sha(value.observed_checkpoint)||value.candidate?.catalog_id!==state.project.project_id)throw Error('Abandonment differs from the exact current preparation intent.');executionFormSaved('project-candidate-recovery-form');} - else if(!candidatePreparedOK(value,study_id))throw Error('Reconciled preparation belongs to another Study.'); - executionFormSaved('project-candidate-prepare-form');$('project-candidate-preparation-result').textContent=JSON.stringify(value,null,2);$('project-candidate-preparation-status').textContent=abandon?'Stopped partial preparation abandoned; original staging and scientific facts remain. Refresh project status.':'Final preparation acknowledged; no writer was replayed and no Run was launched. Refresh project status.';message($('project-candidate-preparation-status').textContent); - } - $('project-candidate-reconcile').onclick=()=>run(ticket=>candidateRecover(ticket,false));$('project-candidate-abandon').onclick=()=>run(ticket=>candidateRecover(ticket,true)); - for(const [form,fields] of [['project-candidate-prepare-form',['project-candidate-study','project-candidate-revision','project-candidate-id','project-candidate-kind','project-candidate-root','project-candidate-directory','project-candidate-runtime','project-candidate-package-run','project-candidate-package-sha']],['project-candidate-recovery-form',['project-candidate-recovery-study','project-candidate-recovery-key','project-candidate-abandon-reason']]])for(const field of fields)$(field).addEventListener(field==='project-candidate-kind'?'change':'input',()=>{m2DirtyForms.add(form);m2Dirty=true;controls();}); - - function runtimeSummaryOK(value,complete=false){ - const keys=['format','project_id','manifest_sha256','file_count','total_bytes','actor_id','observed_at','request_key',...(complete?['runtime_ref']:[])]; - return exact(value,keys)&&value.format===(complete?'project_runtime_capture_v1':'project_runtime_capture_intent_v1')&&id(value.project_id)&&sha(value.manifest_sha256)&&positive(value.file_count)&&value.file_count<=40000&&Number.isSafeInteger(value.total_bytes)&&value.total_bytes>=0&&value.total_bytes<=2147483648&&id(value.actor_id)&&str(value.observed_at,80)&&typeof value.request_key==='string'&&/^[A-Za-z0-9][A-Za-z0-9_.:-]{7,127}$/.test(value.request_key)&&(!complete||exact(value.runtime_ref,['kind','catalog_id','id','sha256'])&&value.runtime_ref.kind==='runtime'&&value.runtime_ref.catalog_id===value.project_id&&sha(value.runtime_ref.id)&&value.runtime_ref.id===value.runtime_ref.sha256); - } - function runtimeControls(){ - $('project-runtime-sources').disabled=!m2Allowed('runtime_sources');$('project-runtime-sources-more').disabled=!m2Allowed('runtime_sources'); - $('project-runtime-inspect').disabled=!$('project-runtime-source').value||!m2Allowed('runtime_inspect'); - $('project-runtime-source').disabled=!m2Allowed('runtime_inspect'); - $('project-runtime-files-more').disabled=!runtimeInspection||runtimeInspection.binding!==state?.binding||!m2Allowed('runtime_inspect'); - $('project-runtime-capture-fields').disabled=!runtimeInspection||runtimeInspection.binding!==state?.binding||!m2Allowed('runtime_capture'); - $('project-runtime-capture').disabled=$('project-runtime-capture-fields').disabled||!$('project-runtime-confirm').checked; - $('project-runtime-status-fields').disabled=!preparationRecoveryAllowed('runtime_status'); - $('project-runtime-abandon-fields').disabled=!preparationRecoveryAllowed('runtime_abandon'); - $('project-runtime-retry').disabled=!runtimePending||runtimePending.body.binding!==state?.binding||!preparationRecoveryAllowed('runtime_capture'); - } - async function runtimeSources(ticket,more=false){ - const offset=more?runtimeSourceOffset:0,value=await m2Request('runtime_sources',{offset,limit:40},ticket);if(!value)return; - if(!value.items.every(item=>exact(item,['source'])&&typeof item.source==='string'&&/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/.test(item.source)))throw Error('Unsupported configured runtime source page.'); - if(!more){$('project-runtime-source').replaceChildren();runtimeInspection=null;$('project-runtime-confirm').checked=false;} - for(const item of value.items){const option=document.createElement('option');option.value=item.source;option.textContent=item.source;$('project-runtime-source').append(option);} - runtimeSourceOffset=offset+value.items.length;runtimeSourceTotal=value.total;$('project-runtime-sources-more').hidden=runtimeSourceOffset>=runtimeSourceTotal;controls(); - message(value.total?'Choose a current source and explicitly inspect it. No runtime has been captured.':'No current runtime sources are configured on this installation.'); - } - async function runtimeInspect(ticket,more=false){ - const source=$('project-runtime-source').value;if(!source)throw Error('Choose a configured source alias.'); - const offset=more?runtimeFileOffset:0,value=await m2Request('runtime_inspect',{source,offset,limit:40},ticket);if(!value)return; - const page=value.files; - if(!exact(value,['format','source','inventory_sha256','profile','python_version','file_count','total_bytes','manifest_bytes','free_bytes','packages','launch','files'])||value.format!=='project_runtime_inspection_v1'||value.source!==source||!sha(value.inventory_sha256)||value.profile!=='captured_cpu_python_v1'||value.python_version!=='3.12'||!positive(value.file_count)||value.file_count>40000||!positive(value.total_bytes)||value.total_bytes>2147483648||!positive(value.manifest_bytes)||value.manifest_bytes>16777216||!Number.isSafeInteger(value.free_bytes)||value.free_bytes<0||!Array.isArray(value.packages)||value.packages.length<2||value.packages.length>256||!value.packages.every(item=>exact(item,['name','version'])&&str(item.name,128)&&str(item.version,128))||!exact(page,['items','total','offset','limit'])||page.total!==value.file_count||page.offset!==offset||page.limit!==40||!Array.isArray(page.items)||page.items.length!==Math.min(40,Math.max(0,page.total-offset)))throw Error('Runtime inventory differs from the reviewed source or continuation.'); - if(!page.items.every(item=>exact(item,['path','bytes','sha256','mode','kind'])&&str(item.path,1024)&&Number.isSafeInteger(item.bytes)&&item.bytes>=0&&item.bytes<=536870912&&sha(item.sha256)&&[256,320].includes(item.mode)&&['data','elf'].includes(item.kind)))throw Error('Unsupported runtime file inventory.'); - if(more&&(!runtimeInspection||runtimeInspection.binding!==state.binding||runtimeInspection.source!==source||runtimeInspection.inventory_sha256!==value.inventory_sha256))throw Error('Current runtime inventory changed between pages. Inspect it again before capture.'); - if(!more){runtimeInspection={binding:state.binding,source,inventory_sha256:value.inventory_sha256};$('project-runtime-files').replaceChildren();$('project-runtime-confirm').checked=false;} - $('project-runtime-summary').textContent=`Current configured source ${source} · ${value.inventory_sha256} · ${value.file_count} files · ${value.total_bytes} bytes · ${value.free_bytes} bytes currently free. This observation does not capture a runtime.`; - $('project-runtime-layout').textContent=JSON.stringify({python_version:value.python_version,packages:value.packages,launch:value.launch},null,2); - for(const item of page.items){const li=document.createElement('li');li.textContent=`${item.path} · ${item.kind} · ${item.bytes} bytes · ${item.sha256}`;$('project-runtime-files').append(li);} - runtimeFileOffset=offset+page.items.length;runtimeFileTotal=page.total;$('project-runtime-files-more').hidden=runtimeFileOffset>=runtimeFileTotal;controls(); - } - $('project-runtime-sources').onclick=()=>run(ticket=>runtimeSources(ticket));$('project-runtime-sources-more').onclick=()=>run(ticket=>runtimeSources(ticket,true)); - $('project-runtime-inspect').onclick=()=>run(ticket=>runtimeInspect(ticket));$('project-runtime-files-more').onclick=()=>run(ticket=>runtimeInspect(ticket,true)); - $('project-runtime-source').addEventListener('change',()=>{runtimeInspection=null;$('project-runtime-confirm').checked=false;controls();}); - function runtimeAbandonmentOK(value,key,reason=null){ - return exact(value,['format','project_id','capture_key','input_sha256','manifest_sha256','actor_id','reason','observed_checkpoint','observed_at'])&&value.format==='project_runtime_capture_abandon_v1'&&value.project_id===state.project.project_id&&value.capture_key===key&&sha(value.input_sha256)&&sha(value.manifest_sha256)&&id(value.actor_id)&&str(value.reason,1000)&&(reason===null||value.reason===reason)&&sha(value.observed_checkpoint)&&str(value.observed_at,80); - } - async function captureRuntime(ticket,retry=false){ - let args; - if(retry){ - if(!runtimePending||runtimePending.body.binding!==state.binding)throw Error('This tab has no exact current pending capture request. Inspect retained status before choosing a new action.'); - const {binding,expected_checkpoint,request_key,...saved}=runtimePending.body;args=saved; - }else{ - if(runtimeInspection?.binding!==state.binding||!$('project-runtime-confirm').checked)throw Error('Inspect and explicitly confirm the current source inventory before capture.'); - const reserve_bytes=Number($('project-runtime-reserve').value);if(!Number.isSafeInteger(reserve_bytes)||reserve_bytes<0)throw Error('Choose a nonnegative safe integer disk reserve.'); - args={source:runtimeInspection.source,expected_inventory_sha256:runtimeInspection.inventory_sha256,reserve_bytes}; - } - const value=await m2Request('runtime_capture',args,ticket);if(!value)return; - if(!runtimeSummaryOK(value,true)||value.project_id!==state.project.project_id||value.manifest_sha256!==args.expected_inventory_sha256||value.request_key!==runtimePending?.body.request_key)throw Error('Runtime capture acknowledgement differs from the reviewed inventory or pending request.'); - $('project-runtime-status-key').value=value.request_key;runtimePending=null; - if(Number($('project-runtime-reserve').value)===args.reserve_bytes)executionFormSaved('project-runtime-capture-form');$('project-runtime-confirm').checked=false; - $('project-runtime-pending').textContent=`Capture request ${value.request_key} acknowledged. Retained capture facts do not grant permission to execute.`;$('project-runtime-status').textContent=JSON.stringify(value,null,2);controls();message('Reviewed runtime bytes captured locally and scientific metadata saved. No project execution started.'); - } - $('project-runtime-capture-form').onsubmit=event=>{event.preventDefault();run(ticket=>captureRuntime(ticket));}; - $('project-runtime-retry').onclick=()=>run(ticket=>captureRuntime(ticket,true)); - $('project-runtime-abandon-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const capture_key=$('project-runtime-abandon-key').value,reason=$('project-runtime-abandon-reason').value; - if(!/^[A-Za-z0-9][A-Za-z0-9_.:-]{7,127}$/.test(capture_key)||!str(reason,1000))throw Error('Choose the exact current capture key and an explicit abandonment reason.'); - const value=await m2Request('runtime_abandon',{capture_key,reason},ticket);if(!value)return; - if(!runtimeAbandonmentOK(value,capture_key,reason))throw Error('Runtime abandonment differs from the current exact capture and reason.'); - if(runtimePending?.body.request_key===capture_key){if(Number($('project-runtime-reserve').value)===runtimePending.body.reserve_bytes)executionFormSaved('project-runtime-capture-form');runtimePending=null;} - executionFormSaved('project-runtime-abandon-form');$('project-runtime-status-key').value=capture_key;$('project-runtime-status').textContent=JSON.stringify(value,null,2);$('project-runtime-pending').textContent='Stopped partial capture abandoned. Original partial bytes and scientific intent remain. Refresh project status.';message($('project-runtime-pending').textContent);controls(); - });}; - $('project-runtime-status-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const capture_key=$('project-runtime-status-key').value;if(!/^[A-Za-z0-9][A-Za-z0-9_.:-]{7,127}$/.test(capture_key))throw Error('Choose the exact retained capture request key.'); - const value=await m2Request('runtime_status',{capture_key},ticket);if(!value)return; - const abandoned=value?.state==='abandoned'; - if(!exact(value,abandoned?['state','intent','result','abandonment']:['state','intent','result'])||!['absent','intent_recorded','captured','abandoned'].includes(value.state)||value.state!==(abandoned?'abandoned':value.result?'captured':value.intent?'intent_recorded':'absent')||value.result&&!value.intent||value.intent!==null&&!runtimeSummaryOK(value.intent)||value.result!==null&&!runtimeSummaryOK(value.result,true)||[value.intent,value.result].some(item=>item!==null&&(item.project_id!==state.project.project_id||item.request_key!==capture_key))||value.result&&['project_id','manifest_sha256','file_count','total_bytes','request_key'].some(field=>value.intent[field]!==value.result[field])||abandoned&&(!value.intent||value.result!==null||!runtimeAbandonmentOK(value.abandonment,capture_key)||value.abandonment.manifest_sha256!==value.intent.manifest_sha256))throw Error('Retained runtime status differs from the selected capture request.'); - $('project-runtime-status').textContent=JSON.stringify(value,null,2);message('Retained capture outcome loaded. Current local readiness was not inspected.'); - });}; - for(const field of ['project-runtime-abandon-key','project-runtime-abandon-reason'])$(field).addEventListener('input',()=>{m2DirtyForms.add('project-runtime-abandon-form');m2Dirty=true;controls();}); - for(const field of ['project-runtime-reserve','project-runtime-confirm'])$(field).addEventListener(field.endsWith('confirm')?'change':'input',()=>{m2DirtyForms.add('project-runtime-capture-form');m2Dirty=true;controls();}); - - function m4Controls(){ - $('project-save-as-fields').disabled=!m2Allowed('save_as'); - $('project-working-load').disabled=!m2Allowed('files_inspect');$('project-working-more').disabled=!m2Allowed('files_inspect'); - $('project-working-adopt-fields').disabled=!m2Allowed('file_adopt'); - $('project-working-delete-fields').disabled=!m2Allowed('file_adopt_deletion')||!workingDeletion||workingDeletion.binding!==state?.binding||workingDeletion.checkpoint!==checkpoint(); - $('project-working-history').disabled=!workingSelection||workingSelection.binding!==state?.binding||!m2Allowed('file_history'); - $('project-working-history-more').disabled=!m2Allowed('file_history'); - for(const [button,action] of [['project-fork-load','fork_info'],['project-fork-more','fork_info'],['project-intents-load','material_intents'],['project-intents-more','material_intents'],['project-intent-capabilities','material_intent_capabilities'],['project-intent-capabilities-more','material_intent_capabilities']])$(button).disabled=!m2Allowed(action); - } - function m4Reset(){ - runtimeInspection=null;runtimeSourceOffset=runtimeSourceTotal=runtimeFileOffset=runtimeFileTotal=0;$('project-runtime-source').replaceChildren();$('project-runtime-files').replaceChildren();$('project-runtime-layout').textContent='';$('project-runtime-summary').textContent='';$('project-runtime-confirm').checked=false; - workingSelection=null;clearWorkingDeletion();workingOffset=workingTotal=workingHistoryOffset=workingHistoryTotal=intentsOffset=intentsTotal=forkOffset=forkTotal=intentCapsOffset=intentCapsTotal=0; - for(const name of ['project-working-list','project-working-history-list','project-fork-list','project-intents-list'])$(name).replaceChildren(); - $('project-working-text').value='';$('project-working-summary').textContent='';$('project-intent-detail').textContent=''; - for(const name of ['project-working-more','project-working-history-more','project-fork-more','project-intents-more','project-intent-capabilities-more'])$(name).hidden=true; - } - function forkOK(value,body){ - if(!exact(value,['state','fork'])||!stateOK(value.state)||value.state.state!=='open')return false; - const f=value.fork; - return exact(f,['source_project_id','source_checkpoint','project_id','checkpoint','root','directory','request_key','reconciled','mapping_count','copied_count','missing_regenerable_count'])&& - id(f.source_project_id)&&id(f.project_id)&&f.source_project_id!==f.project_id&&sha(f.source_checkpoint)&&sha(f.checkpoint)&& - f.source_checkpoint===body.expected_checkpoint&&f.source_project_id===forkPending?.source_project_id&& - ['root','directory','request_key'].every(field=>f[field]===body[field])&&typeof f.reconciled==='boolean'&& - ['mapping_count','copied_count','missing_regenerable_count'].every(field=>Number.isSafeInteger(f[field])&&f[field]>=0)&& - value.state.binding!==body.binding&&value.state.project.project_id===f.project_id&&value.state.project.checkpoint===f.checkpoint&&value.state.project.name===body.name; - } - $('project-save-as-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - if(importDirty||annotationDirty||[...m2DirtyForms].some(form=>form!=='project-save-as-form'))throw Error('Submit or discard other unsubmitted edits before Save As.'); - const fields={root:$('project-save-as-root').value,directory:$('project-save-as-directory').value,name:$('project-save-as-name').value.trim()},signature=JSON.stringify(fields); - if(!fields.root||!fields.directory||!fields.name)throw Error('Choose a new root, relative directory and project name.'); - if(forkPending&&forkPending.signature!==signature)throw Error('Inspect the pending Save As destination, then keep its exact request or explicitly discard this form before choosing another destination.'); - if(!forkPending)forkPending={signature,source_project_id:state.project.project_id,body:mutationBody('/save-as',fields)}; - const result=await request('/save-as',{body:forkPending.body,ticket,kind:'fork'});if(!result)return; - state=result.state;forkPending=null;pending=null;executionFormSaved('project-save-as-form');m4Reset();clearExploration();selected=null;history=[];nextHistory=null;showRecord();showState(); - const f=result.fork;$('project-save-as-summary').textContent=`Independent project ${f.project_id}; ${f.mapping_count} editable mappings, ${f.copied_count} copied payloads, ${f.missing_regenerable_count} missing regenerable payloads. Source checkpoint ${f.source_checkpoint} is retained as its origin.`; - await query(ticket);message('Independent project created and selected. Review its fork origins and binding readiness.');return $('project-fork-load'); - });}; - async function inspectWorking(ticket,more=false){ - const offset=more?workingOffset:0,value=await m2Request('files_inspect',{offset,limit:40},ticket);if(!value)return; - clearWorkingDeletion();if(!more)$('project-working-list').replaceChildren(); - for(const item of value.items){const li=document.createElement('li'),button=document.createElement('button');button.type='button';button.textContent=`${item.path} · ${item.state} · captured revision ${item.revision}`;button.onclick=()=>run(async next=>{ - const version=await m2Request('file_get',{file_id:item.id},next);if(!version)return;showWorking(version); - if(item.kind==='note'&&item.state==='missing'&&sha(item.expected_sha256)){workingDeletion={binding:state.binding,checkpoint:checkpoint(),path:item.path,revision:item.revision,sha256:item.expected_sha256};$('project-working-delete-summary').textContent=`Missing note ${item.path} · captured revision ${item.revision} · SHA256 ${item.expected_sha256}. Earlier text will remain in revision history.`;} - controls();$('project-working-kind').value=item.kind;$('project-working-path').value=item.path;$('project-working-revision').value=String(item.revision);$('project-working-sha').value=item.observed_sha256??''; - message(item.state==='unchanged'?'Captured bytes match the current file.':'Current file differs or is unavailable. Review its exact current digest before adoption.');return $('project-working-heading'); - },{clean:true});li.append(button);$('project-working-list').append(li);} - workingOffset=offset+value.items.length;workingTotal=value.total;$('project-working-more').hidden=workingOffset>=workingTotal; - } - function clearWorkingDeletion(){ - workingDeletion=null;$('project-working-delete-confirm').checked=false;executionFormSaved('project-working-delete-form'); - $('project-working-delete-summary').textContent='Inspect and select a known missing note first. This action records its absence and keeps all captured revisions.'; - } - function showWorking(value){ - clearWorkingDeletion(); - workingSelection={id:value.id,binding:state.binding};$('project-working-heading').textContent=value.path; - $('project-working-summary').textContent=value.version.deleted===true?`Captured revision ${value.version.revision} · note deletion retained. No text is present in this revision; select an earlier revision to read its captured text.`:`Captured revision ${value.version.revision} · ${value.version.sha256} · ${value.version.bytes} bytes${value.version.bytes===0?' · empty file':''}.`; - $('project-working-text').value=value.version.deleted===true?'':value.version.text;controls(); - } - async function workingHistory(ticket,more=false){ - if(!workingSelection||workingSelection.binding!==state.binding)throw Error('Select a file from the current project first.'); - const offset=more?workingHistoryOffset:0,value=await m2Request('file_history',{file_id:workingSelection.id,offset,limit:40},ticket);if(!value)return; - if(value.id!==workingSelection.id)throw Error('Working-file history belongs to another file.'); - if(!more)$('project-working-history-list').replaceChildren(); - for(const version of value.items){const li=document.createElement('li'),button=document.createElement('button');button.type='button';button.textContent=`Revision ${version.revision} · ${version.deleted===true?'note deletion':version.sha256} · ${version.adopted_at}`;button.onclick=()=>run(async next=>{const result=await m2Request('file_get',{file_id:workingSelection.id,revision:version.revision},next);if(result)showWorking(result);return $('project-working-heading');});li.append(button);$('project-working-history-list').append(li);} - workingHistoryOffset=offset+value.items.length;workingHistoryTotal=value.total;$('project-working-history-more').hidden=workingHistoryOffset>=workingHistoryTotal; - } - $('project-working-local').addEventListener('change',()=>run(async()=>{ - const file=$('project-working-local').files?.[0];if(!file)return;if(file.size>1048576)throw Error('Choose a UTF-8 file of at most 1 MiB.'); - const bytes=await file.arrayBuffer();new TextDecoder('utf-8',{fatal:true}).decode(bytes); - $('project-working-sha').value=Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',bytes)),value=>value.toString(16).padStart(2,'0')).join(''); - m2DirtyForms.add('project-working-adopt-form');m2Dirty=true;message('Matching local file digest calculated. Review the server project path and expected adopted revision before adoption.'); - })); - $('project-working-adopt-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const args={kind:$('project-working-kind').value,path:$('project-working-path').value,expected_revision:Number($('project-working-revision').value),expected_sha256:$('project-working-sha').value}; - const result=await m2Request('file_adopt',args,ticket);if(!result)return;executionFormSaved('project-working-adopt-form');$('project-working-revision').value=String(result.revision); - await inspectWorking(ticket);const saved=await m2Request('file_get',{file_id:result.id,revision:result.revision},ticket);if(saved)showWorking(saved);message('Exact working-file bytes adopted. The captured text remains inert.');return $('project-working-heading'); - });}; - $('project-working-delete-confirm').addEventListener('change',()=>{m2DirtyForms.add('project-working-delete-form');m2Dirty=true;controls();}); - $('project-working-delete-form').onsubmit=event=>{event.preventDefault();run(async ticket=>{ - const selected=workingDeletion;if(!selected||selected.binding!==state.binding||selected.checkpoint!==checkpoint()||!$('project-working-delete-confirm').checked)throw Error('Inspect and select a known missing note, then explicitly confirm its deletion revision.'); - const result=await m2Request('file_adopt_deletion',{path:selected.path,expected_revision:selected.revision,expected_sha256:selected.sha256},ticket);if(!result)return; - executionFormSaved('project-working-delete-form');clearWorkingDeletion();await inspectWorking(ticket); - const saved=await m2Request('file_get',{file_id:result.id,revision:result.revision},ticket);if(saved)showWorking(saved); - message('Note deletion retained. Earlier captured text remains available in revision history.');return $('project-working-heading'); - });}; - async function forkInfo(ticket,more=false){ - const offset=more?forkOffset:0,value=await m2Request('fork_info',{offset,limit:40},ticket);if(!value)return; - const page=value.mapping;if(!exact(page,['items','total','offset','limit'])||page.offset!==offset||page.limit!==40||!Array.isArray(page.items)||page.items.length!==Math.min(40,Math.max(0,page.total-offset)))throw Error('Fork mapping continuation differs.'); - if(!more)$('project-fork-list').replaceChildren();$('project-fork-summary').textContent=value.source?`Fork origin ${value.source.project_id} at ${value.source.checkpoint}; ${value.archived_count} archived records, ${value.copied_count} copied payloads, ${value.missing_regenerable_count} missing regenerable payloads.`:'This project has no independent-fork origin overlay.'; - for(const mapping of page.items){const li=document.createElement('li');li.textContent=`${mapping.family}: ${mapping.source_id} revision ${mapping.source_revision??'none'} → ${mapping.destination_id}`;$('project-fork-list').append(li);} - forkOffset=offset+page.items.length;forkTotal=page.total;$('project-fork-more').hidden=forkOffset>=forkTotal; - } - async function intentList(ticket,more=false,capabilities=false){ - const offset=more?(capabilities?intentCapsOffset:intentsOffset):0,action=capabilities?'material_intent_capabilities':'material_intents'; - const value=await m2Request(action,{offset,limit:40},ticket);if(!value)return;if(!more)$('project-intents-list').replaceChildren(); - for(const item of value.items){const li=document.createElement('li'),fields=capabilities?item.requires_reselection:item.required_reselection; - if(capabilities)li.textContent=`${item.family} · ${item.id} · ${fields.length?'Reselect '+fields.join(', '):'Current editable definition; execution still requires current permission.'}`; - else{const button=document.createElement('button');button.type='button';button.textContent=`${item.name} · ${item.family} · reselect ${fields.join(', ')}`;button.onclick=()=>run(async next=>{const detail=await m2Request('material_intent',{family:item.family,definition_id:item.definition_id},next);if(!detail)return;$('project-intent-title').textContent=detail.current.name;$('project-intent-detail').textContent=JSON.stringify(detail,null,2);return $('project-intent-title');});li.append(button);} - $('project-intents-list').append(li);} - if(capabilities){intentCapsOffset=offset+value.items.length;intentCapsTotal=value.total;$('project-intent-capabilities-more').hidden=intentCapsOffset>=intentCapsTotal;} - else{intentsOffset=offset+value.items.length;intentsTotal=value.total;$('project-intents-more').hidden=intentsOffset>=intentsTotal;} - } - $('project-working-load').onclick=()=>run(ticket=>inspectWorking(ticket),{clean:true});$('project-working-more').onclick=()=>run(ticket=>inspectWorking(ticket,true)); - $('project-working-history').onclick=()=>run(ticket=>workingHistory(ticket));$('project-working-history-more').onclick=()=>run(ticket=>workingHistory(ticket,true)); - $('project-fork-load').onclick=()=>run(ticket=>forkInfo(ticket));$('project-fork-more').onclick=()=>run(ticket=>forkInfo(ticket,true)); - $('project-intents-load').onclick=()=>run(ticket=>intentList(ticket));$('project-intents-more').onclick=()=>run(ticket=>intentList(ticket,true)); - $('project-intent-capabilities').onclick=()=>run(ticket=>intentList(ticket,false,true));$('project-intent-capabilities-more').onclick=()=>run(ticket=>intentList(ticket,true,true)); - for(const [form,fields] of [['project-save-as-form',['project-save-as-directory','project-save-as-name','project-save-as-root']],['project-working-adopt-form',['project-working-path','project-working-revision','project-working-sha','project-working-kind']]])for(const field of fields)$(field).addEventListener('input',()=>{m2DirtyForms.add(form);m2Dirty=true;controls();}); - - for(const [form,fields] of [['project-producer-start-form',['project-producer-url','project-producer-model','project-producer-instruction','project-producer-labels','project-producer-seed','project-producer-profile','project-producer-session','project-producer-width','project-producer-height','project-producer-count','project-producer-prompt-url','project-producer-prompt-model']],['project-producer-review-form',['project-producer-note']],['project-execution-approval-form',['project-execution-consumer','project-execution-ttl']],['project-study-approval-form',['project-study-consumer']]])for(const field of fields)$(field).addEventListener('input',()=>{m2DirtyForms.add(form);m2Dirty=true;controls();}); - for(const action of ['new','open','save','close'])$('project-files-'+action).onclick=()=>lifecycle(action); - $('project-files-refresh').onclick=refresh; - $('records-more').onclick=()=>run(async ticket=>{await query(ticket,true);message('More text records loaded.');return nextRecord===null?$('records-title'):$('records-more');}); - $('record-history-more').onclick=olderHistory; - $('project-files-discard').onclick=()=>{importDirty=false;annotationDirty=false;m2Dirty=false;m2DirtyForms.clear();clearWorkingDeletion();nativePending=null;nativePrepared=null;for(const field of ['project-native-probe-confirm','project-native-submit-confirm','project-native-retry-confirm'])$(field).checked=false;forkPending=null;runtimePending=null;runtimeInspection=null;for(const field of ['project-candidate-study','project-candidate-id','project-candidate-directory','project-candidate-runtime','project-candidate-package-run','project-candidate-package-sha','project-candidate-recovery-study','project-candidate-recovery-key','project-candidate-abandon-reason','project-runtime-abandon-key','project-runtime-abandon-reason'])$(field).value='';$('project-runtime-confirm').checked=false;$('project-runtime-pending').textContent='';for(const field of ['project-save-as-directory','project-save-as-name','project-working-path','project-working-sha','project-working-local'])$(field).value='';$('project-working-revision').value='0';executionApproval=null;studyApproval=null;producerCurrent=null;for(const field of ['project-producer-url','project-producer-model','project-producer-instruction','project-producer-labels','project-producer-note','project-producer-profile','project-producer-session','project-producer-prompt-url','project-producer-prompt-model'])$(field).value='';$('project-producer-confirm').checked=false;$('project-execution-confirm').checked=false;$('project-artifact-path').value='';for(const form of ['project-model-create-form','project-study-create-form','project-register-form','project-scientific-form'])for(const field of $(form).querySelectorAll('input'))field.value=field.defaultValue??'';clearExplorationDetail();$('project-source-file').value='';$('project-source-name').value='';$('project-search-name').value='';$('project-selection-name').value='';$('text-name').value='';$('text-content').value='';$('text-groups').value='project';$('text-rights').value='unknown';showRecord();clearError();message('Unsubmitted edits discarded. Saved project records are unchanged.');}; - for(const field of ['text-name','text-content','text-groups','text-rights'])$(field).addEventListener('input',()=>{importDirty=true;controls();}); - for(const field of ['annotation-value','annotation-groups','record-review','annotation-task'])$(field).addEventListener(field==='annotation-task'||field==='record-review'?'change':'input',()=>{annotationDirty=true;annotationHelp();controls();}); - window.addEventListener('beforeunload',event=>{if(dirty()){event.preventDefault();event.returnValue='';}}); - window.addEventListener('tuldok-session',()=>{++epoch;busy=false;blocked=true;selectedContracts=new Set();controls();message('Session changed. Refreshing project status; your unsubmitted edits remain here.');refresh();}); - controls();refresh(); -})(); diff --git a/static/project-measurements.css b/static/project-measurements.css deleted file mode 100644 index d1ba5ff3..00000000 --- a/static/project-measurements.css +++ /dev/null @@ -1,14 +0,0 @@ -#project-measurements{overflow-wrap:anywhere;min-width:0} -#project-measurements section{margin-top:1rem;padding:1rem} -#project-measurements table{width:100%;border-collapse:collapse;margin:1rem 0} -#project-measurements caption{text-align:left;font-weight:600;padding:.5rem 0} -#project-measurements th,#project-measurements td{border-bottom:1px solid #c6d0c7;padding:.6rem;text-align:left;vertical-align:top} -#project-measurements pre{white-space:pre-wrap;max-height:28rem;overflow:auto;font-size:.9rem} -#project-measurements dl{margin:.5rem 0}#project-measurements dt{font-weight:600}#project-measurements dd{margin:0 0 .5rem} -#project-measurements details{margin:.75rem 0}#project-measurements summary{cursor:pointer;font-weight:600} -#project-measurements-filters{display:grid;grid-template-columns:repeat(auto-fit,minmax(240px,1fr));gap:1rem;margin:1rem 0} -#project-measurements-filters fieldset{border:1px solid #c6d0c7;padding:.75rem} -#project-measurements-chart{min-height:5rem;min-width:0;width:100%} -#project-measurements-chart .vega-embed{width:100%} -#project-measurements-maxima,#project-measurements-rows{overflow:auto;min-width:0} -@media(max-width:760px){#project-measurements-filters{grid-template-columns:1fr}#project-measurements table{min-width:34rem}} diff --git a/static/project-measurements.js b/static/project-measurements.js deleted file mode 100644 index 64e3f9c0..00000000 --- a/static/project-measurements.js +++ /dev/null @@ -1,323 +0,0 @@ -/* Native scalar selection/inspection. The host owns current access and reads. */ -(() => { - 'use strict'; - const STATES = ['measured', 'failed', 'unsupported', 'unavailable']; - const hash = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); - const identity = value => typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(value); - const natural = value => Number.isSafeInteger(value) && value >= 0; - const finite = value => typeof value === 'number' && Number.isFinite(value); - const display = value => value === null ? 'Unknown' : Object.is(value, -0) ? '-0' : String(value); - const node = (tag, text) => { const value = document.createElement(tag); if (text !== undefined) value.textContent = text; return value; }; - const button = (text, invoke) => { const value = node('button', text); value.type = 'button'; value.onclick = invoke; return value; }; - const sourceKey = source => source.origin_catalog_id + '.' + source.id; - const spec = {path: '/static/retained-measurement-maxima.vl.json', sha256: 'c9440bbe90d215afb15d47a5f12c5aefcb20dc1d3ddba19d72fe5ba3adb1cd92'}; - function page(value, offset, limit, items) { - if (!value || value.offset !== offset || value.limit !== limit || !natural(value.total) || !Array.isArray(items) - || items.length !== Math.min(limit, Math.max(0, value.total - offset))) throw Error('Measurement page differs from the selected continuation.'); - } - function jsonDetails(title, value, open = false) { - const details = node('details'); details.open = open; - details.append(node('summary', title), node('pre', JSON.stringify(value, null, 2))); return details; - } - window.ProjectMeasurements = function({root, read, onContextFocusLost = () => {}}) { - root.innerHTML = '

                      Retained measurements

                      Compare exact saved scalars and inspect their original cases. Maxima use every matching case; failed and unavailable observations remain counted.

                      '; - const $ = name => root.querySelector('#project-measurements-' + name); - let context = null, generation = 0, plotGeneration = 0, pending, busy = false, pendingFocus = false; - let sources = [], sourceOffset = 0, sourceTotal = 0, selected = null, descriptors = null, metadata = null; - let subjects = [], subjectOffset = 0, subjectTotal = 0, filters = new Map(), applied = null, query = null, selectedSubjects = new Set(); - const renderer = window.RetainedChart.mount($('chart'), spec, {onError: error => { - const ownsFocus = root.contains(document.activeElement); clearResults(); $('status').textContent = error.message; controls(); - if (ownsFocus) $('compare').focus(); - }}); - function clearInspection() { $('inspection').replaceChildren(); $('inspection').hidden = true; } - function clearResults() { - ++plotGeneration; renderer.clear(); query = null; applied = null; - $('results').hidden = true; $('maxima').replaceChildren(); $('rows').replaceChildren(); clearInspection(); - } - function controls() { - for (const control of root.querySelectorAll('input,select,button')) control.disabled = !context?.enabled || busy; - $('sources-prev').hidden = sourceOffset === 0; $('sources-next').hidden = sourceOffset + sources.length >= sourceTotal; - $('source').disabled ||= !sources.length; $('source-all').disabled ||= !sources.length || sources.length > 16; - $('load').disabled ||= !$('source').selectedOptions.length || $('source').selectedOptions.length > 16; - $('subjects-prev').hidden = subjectOffset === 0; $('subjects-next').hidden = subjectOffset + 40 >= subjectTotal; - $('compare').disabled ||= !metadata || !selectedSubjects.size; - $('subjects-all').disabled ||= !metadata; $('subjects-clear').disabled ||= !metadata; - $('prev').disabled ||= !query || query.offset === 0; - $('next').disabled ||= !query || query.offset + query.rows.length >= query.total; - root.setAttribute('aria-busy', String(busy)); - } - function clearSource() { - selected = descriptors = metadata = null; subjects = []; selectedSubjects.clear(); subjectOffset = subjectTotal = 0; filters.clear(); - $('form').hidden = true; $('context').replaceChildren(); $('filters').replaceChildren(); - $('metric').replaceChildren(); $('subjects').replaceChildren(); clearResults(); - } - function reset() { - ++generation; pending?.abort(); pending = undefined; busy = false; pendingFocus = false; - clearSource(); sources = []; sourceOffset = sourceTotal = 0; - $('source').replaceChildren(); $('source-page').textContent = ''; $('status').textContent = ''; controls(); - } - const ownsFocus = () => root.contains(document.activeElement) - || busy && pendingFocus && document.activeElement === document.body; - async function perform(work) { - if (!context?.enabled || busy) return; - const ticket = ++generation, authority = context.key; - pending = new AbortController(); const signal = pending.signal; - const active = () => ticket === generation && context?.key === authority && context.enabled && !signal.aborted; - pendingFocus = root.contains(document.activeElement); busy = true; controls(); - let focus; - try { focus = await work(active, signal, authority); } - catch (error) { - if (active()) { - const ownedFocus = ownsFocus(); reset(); $('status').textContent = error.message; - if (ownedFocus) $('sources').focus(); - } - } finally { if (active()) { busy = false; pending = undefined; pendingFocus = false; controls(); focus?.focus(); } } - } - async function loadSources(offset = 0) { - clearSource(); - return perform(async(active, signal) => { - const value = await read('measurement_sources', {offset, limit: 40}, signal); if (!active()) return; - page(value, offset, 40, value.items); - const seen = new Set(); - for (const item of value.items) { - const key = sourceKey(item); - if (!hash(item.id) || !/^[a-f0-9]{32}$/.test(item.origin_catalog_id) || seen.has(key) - || item.blob?.sha256 !== item.id || typeof item.title !== 'string') throw Error('Invalid retained measurement source.'); - seen.add(key); - } - sources = value.items; sourceOffset = offset; sourceTotal = value.total; $('source').replaceChildren(); - for (const [index, item] of sources.entries()) { - const option = node('option', item.title + ' · ' + item.id.slice(0, 12) + (item.availability?.state === 'missing' ? ' · missing packet' : '')); - option.value = sourceKey(item); option.selected = index === 0; $('source').append(option); - } - $('source-page').textContent = `${sources.length ? offset + 1 : 0}–${offset + sources.length} of ${sourceTotal} sources`; - $('status').textContent = sources.length ? 'Choose compatible sources and load their exact retained metadata.' : 'No retained measurement sources in this project.'; - return $('source'); - }); - } - const same = (left, right) => { - if (Object.is(left, right)) return true; - if (!left || !right || typeof left !== 'object' || typeof right !== 'object' || Array.isArray(left) !== Array.isArray(right)) return false; - const keys = Object.keys(left); - return keys.length === Object.keys(right).length && keys.every(key => Object.hasOwn(right, key) && same(left[key], right[key])); - }; - const locatorKey = value => value.origin_catalog_id + '.' + value.packet_sha256; - const selection = () => ({sources:selected.map(item => ({origin_catalog_id:item.origin_catalog_id, packet_sha256:item.id}))}); - function checkComparison(response) { - if (!response || !Array.isArray(response.sources) || response.sources.length !== selected.length - || new TextEncoder().encode(JSON.stringify(response)).length > 256 * 1024) throw Error('Comparison differs from selected sources.'); - for (const [index, source] of response.sources.entries()) { - const expected = selected[index]; - if (source.origin_catalog_id !== expected.origin_catalog_id || source.id !== expected.id - || source.title !== expected.title || source.blob?.sha256 !== source.id || source.blob.bytes !== expected.blob.bytes - || source.source?.measured_sha256 !== source.id) throw Error('Comparison source identity changed.'); - } - if (descriptors && !same(descriptors, response.sources)) throw Error('Original comparison capture facts changed.'); - const value = response.value, keys = selected.map(sourceKey); - if (value?.format !== 'retained_measurement_comparison_v1' || !hash(value.case_population_sha256) - || !Array.isArray(value.subjects) || !value.subjects.length || value.subjects.length > 256 - || !natural(value.case_count) || !value.case_count || !natural(value.observation_count) - || !Array.isArray(value.metrics) || !value.metrics.length || value.metrics.length > 32 - || !Array.isArray(value.dimensions) || value.dimensions.length > 32 - || value.observation_count !== value.subjects.length * value.case_count * value.metrics.length - || locatorKey(value.definition_source) !== keys[0] - || value.subjects.some(item => !identity(item.id) || typeof item.label !== 'string' || !keys.includes(locatorKey(item.source))) - || new Set(value.subjects.map(item => item.id)).size !== value.subjects.length) throw Error('Invalid comparison metadata.'); - if (metadata && ['scope','scope_definition','dimensions','metrics','definition_source','case_population_sha256','case_count','observation_count','subjects'].some(key => !same(value[key], metadata[key]))) throw Error('Comparison population changed.'); - return value; - } - function checkRead(response, source) { - const original = descriptors.find(item => sourceKey(item) === locatorKey(source)); - if (!original || response?.source?.origin_catalog_id !== source.origin_catalog_id || response.source.id !== source.packet_sha256 - || response.source.blob?.sha256 !== source.packet_sha256 || !same(response.source, original) - || new TextEncoder().encode(JSON.stringify(response)).length > 256 * 1024) throw Error('Observation differs from its original packet.'); - return response.value; - } - function subjectPage(offset) { - if (!metadata || !natural(offset) || offset > subjectTotal) throw Error('Invalid comparison subject page.'); - subjectOffset = offset; $('subjects').replaceChildren(); - const items = subjects.slice(offset, offset + 40); - for (const item of items) { - const option = node('option', item.label + ' · ' + item.id); option.value = item.id; - option.selected = selectedSubjects.has(item.id); $('subjects').append(option); - } - subjectSelectionStatus(); - return $('subjects'); - } - function subjectSelectionStatus() { - const count = Math.min(40, subjectTotal - subjectOffset); - $('subject-page').textContent = `Subjects ${count ? subjectOffset + 1 : 0}–${subjectOffset + count} of ${subjectTotal}. ${selectedSubjects.size} selected across all pages; selection is retained when paging.`; - } - function dimensions() { - filters.clear(); $('filters').replaceChildren(); - for (const [index, dimension] of metadata.dimensions.entries()) { - const field = node('fieldset'), legend = node('legend', `${dimension.label}${dimension.units ? ' · ' + dimension.units : ''} (${dimension.scope})`); - const mode = node('select'); mode.id = 'project-measurements-filter-mode-' + index; - for (const [value, label] of [['any','Any recorded value'],['value','Exact value'],['unknown','Unknown only']]) { const option = node('option', label); option.value = value; mode.append(option); } - const label = node('label', 'Match'); label.htmlFor = mode.id; label.append(mode); - let input; - if (dimension.type === 'boolean') { input = node('select'); for (const value of ['true','false']) { const option = node('option', value); option.value = value; input.append(option); } } - else { input = node('input'); input.type = dimension.type === 'string' ? 'text' : 'number'; input.maxLength = 256; input.step = dimension.type === 'integer' ? '1' : 'any'; } - input.id = 'project-measurements-filter-value-' + index; - const valueLabel = node('label', 'Exact recorded value'); valueLabel.htmlFor = input.id; valueLabel.append(input); - field.append(legend, label, valueLabel); $('filters').append(field); filters.set(dimension.id, {dimension, mode, input}); - mode.onchange = invalidate; input.oninput = invalidate; - } - } - function filterValues() { - const values = {}; - for (const [id, {dimension, mode, input}] of filters) { - if (mode.value === 'any') continue; - if (mode.value === 'unknown') { values[id] = null; continue; } - if (mode.value !== 'value' || !input.value.length) throw Error('Enter an exact filter value or choose Any recorded value.'); - const value = dimension.type === 'boolean' ? input.value === 'true' : dimension.type === 'string' ? input.value : Number(input.value); - if ((dimension.type === 'number' && !finite(value)) || (dimension.type === 'integer' && !Number.isSafeInteger(value))) throw Error('Filter values must match the declared numeric type.'); - values[id] = value; - } - return values; - } - function invalidate() { - ++generation; pending?.abort(); pending = undefined; busy = false; clearResults(); - $('status').textContent = 'Selection edited. Compare metric to apply it.'; controls(); - } - function attributes(title, values) { - const details = node('details'); details.append(node('summary', title)); - const list = node('dl'); - for (const [key, value] of Object.entries(values)) { - const dimension = metadata.dimensions.find(item => item.id === key); - list.append(node('dt', dimension?.label ?? key), node('dd', display(value) + (dimension?.units ? ' ' + dimension.units : ''))); - } - details.append(list); return details; - } - function table(caption, headings) { - const result = node('table'), row = node('tr'), head = node('thead'), body = node('tbody'); - for (const text of headings) { const cell = node('th', text); cell.scope = 'col'; row.append(cell); } - head.append(row); result.append(node('caption', caption), head, body); return {result, body}; - } - async function inspect(observation_id, source) { - clearInspection(); - return perform(async(active, signal) => { - const value = checkRead(await read('measurement_inspect', {...source, observation_id}, signal), source); if (!active()) return; - if (value.packet_sha256 !== source.packet_sha256 || value.observation?.id !== observation_id - || value.observation.metric_id !== applied?.metric_id || !applied.subject_ids.includes(value.observation.subject_id) - || !same(subjects.find(item => item.id === value.observation.subject_id)?.source, source)) throw Error('Observation differs from the selected comparison.'); - const heading = node('h3', value.subject.label + ' · ' + value.case.id); heading.tabIndex = -1; - const observation = value.observation; - $('inspection').append(heading, node('p', `${value.metric.label} · ${value.metric.units} · ${observation.state} · ${observation.value === null ? observation.reason : display(observation.value)}`), - attributes('Case attributes, including grid density and domain cell counts', value.case.dimensions), - attributes('Subject execution attributes', value.subject.dimensions), attributes('Observation work attributes', observation.dimensions), - jsonDetails('Original case evidence', value.case_evidence, true), jsonDetails('Original scalar evidence', value.scalar_evidence, true), - jsonDetails('Exact observation, metric, source and scope', value)); - $('inspection').hidden = false; heading.focus(); $('status').textContent = 'Exact retained observation and original source evidence.'; - }); - } - async function showQuery(value, request, active, signal, authority) { - if (!value || value.metric?.id !== request.metric_id - || !same(value.metric, metadata.metrics.find(metric => metric.id === request.metric_id)) - || JSON.stringify(value.subject_ids) !== JSON.stringify(request.subject_ids) - || !value.filters || Object.keys(value.filters).length !== Object.keys(request.filters).length - || Object.entries(request.filters).some(([key, wanted]) => !Object.hasOwn(value.filters, key) || value.filters[key] !== wanted) - || !Array.isArray(value.summary) || value.summary.length !== request.subject_ids.length || value.sampling !== 'none') throw Error('Measurement comparison differs from its requested selection.'); - page(value, request.offset, request.limit, value.rows); - if (value.summary.reduce((count, summary) => count + summary.count, 0) !== value.total) throw Error('Comparison denominator differs from its complete summaries.'); - const maxima = table('Maximum over all matching cases, including every observation status', ['Subject / execution attributes','Maximum','Measured','Failed','Unsupported','Unavailable']); - const points = [], inspections = new Map(); - for (const [index, summary] of value.summary.entries()) { - if (summary.subject_id !== request.subject_ids[index] || !natural(summary.count) || STATES.some(state => !natural(summary.states?.[state])) - || STATES.reduce((sum, state) => sum + summary.states[state], 0) !== summary.count - || (summary.maximum === null) !== (summary.states.measured === 0)) throw Error('Invalid complete measurement summary.'); - const subject = subjects.find(item => item.id === summary.subject_id); if (!subject || !same(subject.source, summary.source)) throw Error('Subject differs from its original comparison packet.'); - const row = node('tr'), label = node('td'); label.append(node('strong', subject.label), attributes('Execution attributes', subject.dimensions)); - const maximum = node('td'); - if (summary.maximum === null) maximum.textContent = 'No measured value'; - else { - const point = summary.maximum; - if (!finite(point.value) || !identity(point.id) || !identity(point.case_id)) throw Error('Invalid retained maximum.'); - maximum.append(button(display(point.value) + ' · inspect maximum for ' + subject.label, () => void inspect(point.id, summary.source))); - const token = JSON.stringify([summary.source.origin_catalog_id, summary.source.packet_sha256, point.id]); - inspections.set(token, {id:point.id, source:summary.source}); - points.push({subject_id:subject.id, value:point.value, observation_id:token}); - } - row.append(label, maximum, ...STATES.map(state => node('td', String(summary.states[state])))); maxima.body.append(row); - } - const observations = table('Exact observations on this page', ['Subject','Case','Value / status','Inspection']); - for (const observation of value.rows) { - if (!request.subject_ids.includes(observation.subject_id) || observation.metric_id !== request.metric_id || !identity(observation.id) - || !STATES.includes(observation.state) || (observation.state === 'measured' ? !finite(observation.value) : observation.value !== null)) throw Error('Invalid retained observation row.'); - const subject = subjects.find(item => item.id === observation.subject_id), row = node('tr'), detail = node('td'); - if (!subject || !same(subject.source, observation.source)) throw Error('Observation belongs to another packet.'); - detail.append(button('Inspect ' + observation.id, () => void inspect(observation.id, observation.source)), attributes('Case and execution attributes', observation.dimensions)); - row.append(node('td', subject.label), node('td', observation.case_id), node('td', observation.value === null ? observation.state + ': ' + observation.reason : display(observation.value)), detail); observations.body.append(row); - } - query = value; applied = structuredClone(request); $('results').hidden = false; - $('heading').textContent = value.metric.label + ' · ' + value.metric.units; - $('counts').textContent = `${request.subject_ids.length} selected subjects of ${subjectTotal}; ${value.total} matching observations. Showing ${value.rows.length ? value.offset + 1 : 0}–${value.offset + value.rows.length}. Maxima use every matching case, independent of this page.`; - $('maxima').replaceChildren(maxima.result); $('rows').replaceChildren(observations.result); - const plotTicket = ++plotGeneration; - if (points.length) await renderer.render({maxima:points}, {signal, - isCurrent: () => context?.enabled && context.key === authority && plotTicket === plotGeneration, - onSignal:{measurement_select: selection => { const id = selection?.observation_id?.[0]; const original = inspections.get(id); if (original) void inspect(original.id, original.source); }}}); - else { renderer.clear(); $('chart').append(node('p', 'No measured values for this selection. Missing and failed observations remain counted below.')); } - if (active()) { $('heading').focus(); $('status').textContent = 'Exact retained comparison. Metric and scope definitions are available above.'; } - } - function compare(offset = 0, saved = null) { - return perform(async(active, signal, authority) => { - const request = saved ? {...saved, offset} : {metric_id:$('metric').value, - subject_ids:subjects.filter(item => selectedSubjects.has(item.id)).map(item => item.id), filters:filterValues(), offset, limit:50}; - if (!request.subject_ids.length) throw Error('Choose at least one subject.'); - clearResults(); - const value = checkComparison(await read('measurement_compare', {...selection(), ...request}, signal)).query; if (!active()) return; - await showQuery(value, request, active, signal, authority); - }); - } - $('sources').onclick = () => void loadSources(); - $('sources-prev').onclick = () => void loadSources(Math.max(0, sourceOffset - 40)); - $('sources-next').onclick = () => void loadSources(sourceOffset + 40); - $('source').onchange = () => { invalidate(); clearSource(); $('status').textContent = 'Load the selected compatible sources.'; controls(); }; - $('source-form').onsubmit = event => { - event.preventDefault(); - void perform(async(active, signal) => { - const chosen = [...$('source').selectedOptions].map(option => sources.find(item => sourceKey(item) === option.value)); - clearSource(); selected = chosen; - if (!selected.length || selected.length > 16 || selected.some(item => !item)) throw Error('Choose 1 through 16 retained sources.'); - const response = await read('measurement_compare', selection(), signal); if (!active()) return; - metadata = checkComparison(response); descriptors = response.sources; - if (metadata.query !== null) throw Error('Metadata reply unexpectedly contains a query.'); - subjects = metadata.subjects; subjectTotal = subjects.length; selectedSubjects = new Set(subjects.map(item => item.id)); - $('context').append(node('h3', `Comparison of ${selected.length} retained sources`), node('p', metadata.scope.label + ' · population: ' + metadata.scope.population), - node('p', typeof metadata.scope_definition.task === 'string' ? metadata.scope_definition.task : 'Interpret results using the retained scope definition.'), - jsonDetails('Exact compatible scope, metric definitions and original packet sources', {metadata, sources:descriptors})); - for (const metric of metadata.metrics) { const option = node('option', metric.label + ' · ' + metric.units); option.value = metric.id; $('metric').append(option); } - dimensions(); subjectPage(0); if (!active()) return; - $('form').hidden = false; $('status').textContent = 'Choose a metric, subjects and optional exact attribute filters.'; return $('metric'); - }); - }; - $('source-all').onclick = () => { - for (const option of $('source').options) option.selected = true; - $('source').onchange(); - }; - $('metric').onchange = invalidate; - $('subjects').onchange = () => { - for (const option of $('subjects').options) { - if (option.selected) selectedSubjects.add(option.value); else selectedSubjects.delete(option.value); - } - invalidate(); subjectSelectionStatus(); controls(); - }; - for (const [name, all] of [['subjects-all',true],['subjects-clear',false]]) $(name).onclick = () => { - selectedSubjects = new Set(all ? subjects.map(item => item.id) : []); - invalidate(); subjectPage(subjectOffset); controls(); - }; - for (const [name, direction] of [['subjects-prev',-1],['subjects-next',1]]) $(name).onclick = () => { - void perform(() => subjectPage(subjectOffset + direction * 40)); - }; - $('form').onsubmit = event => { event.preventDefault(); void compare(); }; - $('prev').onclick = () => void compare(Math.max(0, query.offset - query.limit), applied); - $('next').onclick = () => void compare(query.offset + query.limit, applied); - controls(); - return {setContext(value) { - if (value.key !== context?.key || context?.enabled && !value.enabled) { const ownedFocus = ownsFocus(); context = value; reset(); if (ownedFocus) onContextFocusLost(); } - else context = value; - root.hidden = !value.enabled; controls(); - }, clear() { context = null; reset(); root.hidden = true; }}; - }; -})(); diff --git a/static/project-model-portfolio.js b/static/project-model-portfolio.js deleted file mode 100644 index 4f1b7fa3..00000000 --- a/static/project-model-portfolio.js +++ /dev/null @@ -1,188 +0,0 @@ -'use strict'; -// Authored organization is a retained note, not Trial lineage or qualification. -// This module owns its document contract, finite view and cancellable reads. -(() => { - const fields=['method','purpose','experiment','outcome','lesson','reproduction']; - const exact=(value,keys)=>value&&typeof value==='object'&&!Array.isArray(value)&& - Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); - const text=(value,limit)=>typeof value==='string'&&value.length>0&&value.length<=limit; - const identity=value=>text(value,160)&&/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value); - const sha=value=>typeof value==='string'&&/^[a-f0-9]{64}$/.test(value); - const natural=value=>Number.isSafeInteger(value)&&value>=0; - function fail(message,code='invalid'){throw Object.assign(Error(message),{code});} - function noteRef(value){ - if(!exact(value,['file_id','path','revision','sha256'])||!sha(value.file_id)||!sha(value.sha256)|| - !natural(value.revision)||value.revision<1||!text(value.path,512)||!value.path.startsWith('notes/')|| - value.path.split('/').some(part=>!part||part==='.'||part==='..'))fail('Invalid captured model-note reference.'); - } - function decode(raw){ - if(typeof raw!=='string'||new TextEncoder().encode(raw).length>1048576)fail('Organization note exceeds its retained text limit.'); - let value;try{value=JSON.parse(raw);}catch{fail('Organization note is not valid JSON.');} - if(value?.format!=='project_model_portfolio_v1')fail('Unsupported model-organization document format.','unsupported'); - if(!exact(value,['format','project_id','title','description','authorship','groups','models','relationships'])|| - !/^[a-f0-9]{32}$/.test(value.project_id)||!text(value.title,200)||!text(value.description,8000)|| - value.authorship!=='retrospective'||!Array.isArray(value.groups)||value.groups.length>64|| - !Array.isArray(value.models)||!value.models.length||value.models.length>1000|| - !Array.isArray(value.relationships)||value.relationships.length>2000)fail('Invalid model-organization document.'); - const groups=new Map(),models=new Map(); - for(const group of value.groups){ - if(!exact(group,['id','label','parent','description'])||!identity(group.id)||groups.has(group.id)|| - !text(group.label,200)||!text(group.description,8000)||group.parent!==null&&!identity(group.parent))fail('Invalid or duplicate organization group.'); - groups.set(group.id,group); - } - for(const group of groups.values()){ - const visited=new Set([group.id]);let parent=group.parent; - while(parent!==null){if(!groups.has(parent))fail('Organization group parent is unavailable.'); - if(visited.has(parent))fail('Organization grouping contains a cycle.');visited.add(parent);parent=groups.get(parent).parent;} - } - for(const model of value.models){ - if(!exact(model,['id','label','group_id','aliases','review','card','summary'])||!identity(model.id)||models.has(model.id)|| - !text(model.label,500)||!groups.has(model.group_id)||!Array.isArray(model.aliases)||model.aliases.length>128|| - new Set(model.aliases).size!==model.aliases.length||model.aliases.some(alias=>!text(alias,500))|| - !['source_reviewed','review_pending'].includes(model.review))fail('Invalid or duplicate organized model.'); - noteRef(model.card); - if(model.review==='source_reviewed'){ - if(!exact(model.summary,fields)||fields.some(field=>!text(model.summary[field],16384)))fail('Reviewed model requires all six authored summary fields.'); - }else if(model.summary!==null)fail('Pending review cannot expose a reviewed summary.'); - models.set(model.id,model); - } - const pairs=new Set(); - for(const relation of value.relationships){ - if(!exact(relation,['from','to','kind','explanation','evidence_model','excerpt'])|| - !models.has(relation.from)||!models.has(relation.to)||relation.from===relation.to|| - !['comparison','parameter_derivation'].includes(relation.kind)||!text(relation.explanation,8000)|| - !models.has(relation.evidence_model)||!text(relation.excerpt,16384))fail('Invalid authored model relationship.'); - const key=JSON.stringify([relation.from,relation.to,relation.kind]);if(pairs.has(key))fail('Duplicate authored model relationship.');pairs.add(key); - } - const parents=new Map(),complete=new Set(),visiting=new Set(); - for(const relation of value.relationships)if(relation.kind==='parameter_derivation'){ - if(!parents.has(relation.from))parents.set(relation.from,[]);parents.get(relation.from).push(relation.to); - } - function visit(id){if(complete.has(id))return;if(visiting.has(id))fail('Authored parameter derivation contains a cycle.'); - visiting.add(id);for(const parent of parents.get(id)||[])visit(parent);visiting.delete(id);complete.add(id);} - for(const id of parents.keys())visit(id); - return value; - } - function checkCard(document,model,result){ - const ref=model.card,version=result?.version; - if(result?.project_id!==document.project_id||result.id!==ref.file_id||result.path!==ref.path||result.kind!=='note'|| - version?.deleted===true||version?.revision!==ref.revision||version.sha256!==ref.sha256||typeof version.text!=='string') - fail('Model card differs from its exact retained organization reference.'); - if(model.summary&&fields.some(field=>!version.text.includes(model.summary[field])))fail('Authored summary differs from the captured model card.'); - for(const relation of document.relationships)if(relation.evidence_model===model.id&&!version.text.includes(relation.excerpt)) - fail('Authored relationship excerpt differs from its captured model card.'); - return version.text; - } - window.ProjectModelPortfolioDocument=Object.freeze({decode,checkCard}); - const node=(tag,value)=>{const element=document.createElement(tag);if(value!==undefined)element.textContent=value;return element;}; - window.ProjectModelPortfolio=function({root,read,onContextFocusLost=()=>{}}){ - root.innerHTML='

                      Authored model organization

                      Browse a retained guide to models and their methods. Grouping and retrospective comparison notes are separate from recorded Trial ancestry and scientific qualification.

                      '; - const $=name=>root.querySelector('#project-portfolio-'+name); - let context=null,generation=0,pending=null,busy=false,notes=new Map(),noteOffset=0,noteTotal=0; - let guide=null,groups=new Map(),models=new Map(),applied={group:'',search:'',review:''},offset=0,selected=''; - const pageSize=20; - function controls(){ - const enabled=!!context?.enabled; - for(const name of ['notes','notes-more','note','load','group','search','review','prev','next'])$(name).disabled=!enabled||busy; - $('note').disabled ||= !notes.size;$('load').disabled ||= !notes.has($('note').value); - $('notes-more').hidden=noteOffset>=noteTotal;$('notes-more').disabled ||= noteOffset>=noteTotal; - $('prev').disabled ||= !guide||offset===0;$('next').disabled ||= !guide||offset+pageSize>=filtered().length; - for(const button of root.querySelectorAll('#project-portfolio-filter-form button,[data-lineage-node],#project-portfolio-detail button'))button.disabled=!enabled||busy; - root.setAttribute('aria-busy',String(busy)); - } - function clearGuide(){guide=null;groups.clear();models.clear();selected='';offset=0;applied={group:'',search:'',review:''};$('content').hidden=true;$('detail').replaceChildren();$('graph').replaceChildren();$('search').value='';$('review').value='';$('group').replaceChildren();} - function reset(){++generation;pending?.abort();pending=null;busy=false;notes.clear();noteOffset=noteTotal=0;$('note').replaceChildren();$('note-count').textContent='';$('status').textContent='';clearGuide();controls();} - async function perform(work){ - if(!context?.enabled||busy)return; - const ticket=++generation,key=context.key;pending=new AbortController();const signal=pending.signal;busy=true;controls(); - const active=()=>ticket===generation&&context?.enabled&&context.key===key&&!signal.aborted; - try{await work(active,signal);}catch(error){if(active()){$('detail').replaceChildren();$('status').textContent=(error.code?error.code+': ':'')+error.message;$('status').focus();}} - finally{if(active()){busy=false;pending=null;controls();}} - } - async function findNotes(more=false){ - return perform(async(active,signal)=>{ - const requested=more?noteOffset:0,value=await read('files_inspect',{offset:requested,limit:40},signal);if(!active())return; - if(value?.offset!==requested||value.limit!==40||!natural(value.total)||!Array.isArray(value.items)|| - value.items.length!==Math.min(40,Math.max(0,value.total-requested)))fail('Organization note page differs from its continuation.'); - if(!more){clearGuide();notes.clear();$('note').replaceChildren();} - for(const item of value.items)if(item.kind==='note'&&item.path?.endsWith('.portfolio.json')){ - if(!sha(item.id)||!natural(item.revision)||item.revision<1||!sha(item.expected_sha256)||notes.has(item.id))fail('Invalid organization note selection.'); - notes.set(item.id,structuredClone(item));const option=node('option',item.path+' · revision '+item.revision);option.value=item.id;$('note').append(option); - } - noteOffset=requested+value.items.length;noteTotal=value.total; - $('note-count').textContent=`${notes.size} organization notes found; inspected ${noteOffset} of ${noteTotal} adopted files.`; - $('status').textContent=notes.size?'Choose a retained organization note.':'No organization note on these pages. Adopt a .portfolio.json note or inspect the next page.'; - }); - } - function inGroup(model,wanted){let id=model.group_id;while(id!==null){if(id===wanted)return true;id=groups.get(id).parent;}return false;} - function filtered(){if(!guide)return [];return guide.models.filter(model=>(!applied.group||inGroup(model,applied.group))&& - (!applied.review||model.review===applied.review)&&(!applied.search||[model.label,...model.aliases].some(value=>value.toLocaleLowerCase().includes(applied.search))));} - function render(){ - if(!guide)return; - const focused=$('graph').contains(document.activeElement)?document.activeElement.id:null; - const matching=filtered(),shown=matching.slice(offset,offset+pageSize),included=new Set(); - for(const model of shown){let id=model.group_id;while(id!==null){included.add(id);id=groups.get(id).parent;}} - const graphNodes=[...included].map(id=>{const group=groups.get(id);return {id:'group/'+id,title:group.label,recordedAt:'',parent:group.parent===null?null:'group/'+group.parent,baseline:null,lines:['Authored method group']};}); - graphNodes.push(...shown.map(model=>({id:'model/'+model.id,title:model.label,recordedAt:'',parent:'group/'+model.group_id,baseline:null, - lines:[model.aliases.slice(0,3).join(' · ')||model.id,model.review==='source_reviewed'?'Source reviewed · authored summary':'Source review pending']}))); - $('graph').innerHTML=window.WorkspaceExperimentGraph.render(graphNodes,{selected:selected?'model/'+selected:'',zoom:1}).html; - for(const button of $('graph').querySelectorAll('[data-lineage-node]')){ - const value=button.dataset.lineageNode; - button.onclick=()=>{if(value.startsWith('model/'))void selectModel(value.slice(6));else{$('group').value=value.slice(6);applyFilters();}}; - } - $('count').textContent=`${guide.models.length} models in this guide; ${guide.models.filter(model=>model.review==='source_reviewed').length} source reviewed. ${matching.length} match the applied filters. Showing ${shown.length?offset+1:0}–${offset+shown.length}.`; - controls(); - if(focused){const replacement=$('graph').querySelector('#'+CSS.escape(focused)); - if(replacement)replacement.focus({preventScroll:true});else{$('count').tabIndex=-1;$('count').focus();}} - } - function applyFilters(){if(!guide||busy)return;applied={group:$('group').value,search:$('search').value.trim().toLocaleLowerCase(),review:$('review').value}; - if(applied.group&&!groups.has(applied.group))return;offset=0;selected='';$('detail').replaceChildren();render();$('status').textContent='Applied model filters.';} - async function loadGuide(){ - const choice=notes.get($('note').value);if(!choice)return; - return perform(async(active,signal)=>{ - clearGuide();const value=await read('file_get',{file_id:choice.id,revision:choice.revision},signal);if(!active())return; - if(value?.id!==choice.id||value.project_id!==context.project_id||value.kind!=='note'||value.path!==choice.path|| - value.version?.revision!==choice.revision||value.version.sha256!==choice.expected_sha256||value.version.deleted===true)fail('Organization note differs from the selected captured revision.'); - const parsed=decode(value.version.text); - if(parsed.project_id!==context.project_id)fail('This guide belongs to another project; re-author its captured note bindings.','unsupported'); - guide=parsed;groups=new Map(guide.groups.map(group=>[group.id,group]));models=new Map(guide.models.map(model=>[model.id,model])); - $('title').textContent=guide.title;$('description').textContent=guide.description; - $('source').textContent=`Retrospective authored guide · ${choice.path} · revision ${choice.revision} · SHA256 ${choice.expected_sha256}`; - const all=node('option','All groups');all.value='';$('group').append(all); - for(const group of guide.groups){const option=node('option',group.label);option.value=group.id;$('group').append(option);} - $('content').hidden=false;render();$('status').textContent='Loaded retained authored model organization.'; - }); - } - async function selectModel(id){ - const model=models.get(id);if(!model||busy)return; - return perform(async(active,signal)=>{ - if(!filtered().some(item=>item.id===id)){ - applied={group:'',search:'',review:''};$('group').value='';$('search').value='';$('review').value=''; - } - offset=Math.floor(filtered().findIndex(item=>item.id===id)/pageSize)*pageSize; - selected=id;$('detail').replaceChildren();render(); - const value=await read('file_get',{file_id:model.card.file_id,revision:model.card.revision},signal);if(!active())return; - const captured=checkCard(guide,model,value),heading=node('h3',model.label);heading.id='project-portfolio-model-heading';heading.tabIndex=-1;$('detail').append(heading); - $('detail').append(node('p',(model.aliases.join(' · ')||model.id)+' · '+(model.review==='source_reviewed'?'Source reviewed':'Source review pending'))); - if(model.summary){const labels={method:'Method',purpose:'Purpose',experiment:'Training and experiment',outcome:'Observed outcome',lesson:'Lesson',reproduction:'Reproduction'}; - for(const field of fields){$('detail').append(node('h4',labels[field]),node('p',model.summary[field]));}} - else $('detail').append(node('p','A source-reviewed interpretation has not been recorded for this fit. Its original captured card remains available below.')); - const relations=guide.relationships.filter(relation=>relation.from===id||relation.to===id); - if(relations.length){$('detail').append(node('h4','Authored model relationships')); - for(const relation of relations){const box=node('div'),other=models.get(relation.from===id?relation.to:relation.from),button=node('button', - (relation.kind==='comparison'?'Comparison: ':relation.from===id?'Parameters from: ':'Derived model: ')+other.label);button.type='button';button.onclick=()=>void selectModel(other.id); - box.append(button,node('p',relation.explanation));const evidence=node('details');evidence.append(node('summary','Exact retained relationship excerpt'),node('p',relation.excerpt)); - const source=node('button','Read evidence card: '+models.get(relation.evidence_model).label);source.type='button';source.onclick=()=>void selectModel(relation.evidence_model);evidence.append(source);box.append(evidence);$('detail').append(box);}} - const raw=node('details'),summary=node('summary','Exact captured card and original receipts');raw.append(summary,node('p',`${model.card.path} · revision ${model.card.revision} · SHA256 ${model.card.sha256}`),node('pre',captured));$('detail').append(raw); - $('status').textContent='Exact retained model summary and captured evidence.';heading.focus(); - }); - } - $('notes').onclick=()=>void findNotes();$('notes-more').onclick=()=>void findNotes(true); - $('note').onchange=()=>{clearGuide();controls();}; - $('source-form').onsubmit=event=>{event.preventDefault();void loadGuide();}; - $('filter-form').onsubmit=event=>{event.preventDefault();applyFilters();}; - for(const [name,direction] of [['prev',-1],['next',1]])$(name).onclick=()=>{if(busy||!guide)return;offset=Math.max(0,offset+direction*pageSize);selected='';$('detail').replaceChildren();render();}; - controls(); - return {setContext(value){if(value.key!==context?.key||context?.enabled&&!value.enabled){const owned=root.contains(document.activeElement);context=value;reset();if(owned)onContextFocusLost();}else context=value;root.hidden=!value.enabled;controls();},clear(){context=null;reset();root.hidden=true;}}; - }; -})(); diff --git a/static/project-progression.js b/static/project-progression.js deleted file mode 100644 index ae2fa97e..00000000 --- a/static/project-progression.js +++ /dev/null @@ -1,117 +0,0 @@ -'use strict'; -// The native host owns binding/checkpoint/credential changes and exact reads. -// This controller owns source choice, its finite graph window and pending reads. -(() => { - const id=value=>typeof value==='string'&&/^[a-f0-9]{32}$/.test(value); - const natural=value=>Number.isSafeInteger(value)&&value>=0; - const short=value=>String(value).slice(0,12); - const key=(origin,trial)=>origin+'.'+trial; - const element=(tag,text)=>{const e=document.createElement(tag);if(text!==undefined)e.textContent=text;return e;}; - window.ProjectExperimentProgression=function({root,read,inspectStudy,inspectTrial,inspectRun,onContextFocusLost=()=>{}}) { - root.innerHTML='

                      Experiment progression

                      Explore recorded relationships across saved Studies. Solid edges show derivation; dashed edges show comparison baselines. Run states are saved outcomes.

                      '; - const $=name=>root.querySelector('#project-progression-'+name); - let context=null,generation=0,pending=null,busy=false,origins=[],originsNext=0,originsTotal=0; - let nodes=new Map(),origin='',appliedCohort=null,next=0,total=0,related=new Map(),selected='',zoom=1; - function clearGraph(){nodes.clear();related.clear();origin='';appliedCohort=null;next=0;total=0;selected='';$('graph').replaceChildren();$('detail').replaceChildren();$('summary').textContent='';} - function controls(){ - const enabled=!!context?.enabled; - $('origins').disabled=!enabled||busy;$('origins-more').disabled=!enabled||busy; - $('origins-more').hidden=originsNext>=originsTotal; - $('origin').disabled=!enabled||busy||!origins.length;$('cohort').disabled=!enabled||busy; - $('load').disabled=!enabled||busy||!$('origin').value; - $('more').hidden=next>=total;$('more').disabled=!enabled||busy||nodes.size>=100; - for(const name of ['zoom-out','zoom-in','zoom-reset'])$(name).disabled=!enabled||busy||!nodes.size; - for(const button of root.querySelectorAll('[data-lineage-node],#project-progression-detail button'))button.disabled=!enabled||busy; - root.setAttribute('aria-busy',String(busy)); - } - function reset(){ - ++generation;pending?.abort();pending=null;busy=false;clearGraph();origins=[];originsNext=0;originsTotal=0; - const option=element('option','Choose a history source');option.value='';$('origin').replaceChildren(option); - $('cohort').value='';$('status').textContent='';controls(); - } - async function perform(work){ - if(!context?.enabled||busy)return; - const ticket=++generation,authority=context.key;pending=new AbortController();const signal=pending.signal;busy=true;controls(); - const active=()=>ticket===generation&&context?.key===authority&&!signal.aborted; - try{await work(active,signal);}catch(error){if(active()){const ownedFocus=root.contains(document.activeElement);reset();$('status').textContent=error.message;if(ownedFocus)$('origins').focus();}} - finally{if(active()){busy=false;pending=null;controls();}} - } - function page(value,offset,limit,items){ - if(!value||value.offset!==offset||value.limit!==limit||!natural(value.total)||!Array.isArray(items)||items.length!==Math.min(limit,Math.max(0,value.total-offset)))throw Error('History page differs from its selected continuation.'); - } - async function loadOrigins(append=false){ - return perform(async(active,signal)=>{ - const offset=append?originsNext:0,value=await read('progression_origins',{offset,limit:40},signal);if(!active())return; - page(value,offset,40,value.items); - const seen=new Set(append?origins.map(item=>item.origin_catalog_id):[]); - for(const item of value.items){if(!id(item.origin_catalog_id)||seen.has(item.origin_catalog_id)||!natural(item.study_count)||!natural(item.trial_count))throw Error('Invalid history source.');seen.add(item.origin_catalog_id);} - if(!append){clearGraph();origins=[];const option=element('option','Choose a history source');option.value='';$('origin').replaceChildren(option);} - origins.push(...value.items);originsNext=offset+value.items.length;originsTotal=value.total; - for(const item of value.items){const option=element('option',(item.origin_catalog_id===context.project_id?'This project':'Imported catalog '+short(item.origin_catalog_id))+' · '+item.study_count+' Studies · '+item.trial_count+' Trials');option.value=item.origin_catalog_id;$('origin').append(option);} - $('status').textContent=origins.length?'Choose a history source to load its Trials.':'No recorded Study history in this project.'; - if(!append)$('origin').focus(); - }); - } - function validNode(node,source){ - const trial=node?.trial,lineage=node?.lineage; - if(!id(trial?.id)||!id(trial?.study_id)||trial.study_id!==node.study?.id||typeof node.study.name!=='string'|| - trial.material?.catalog_id!==source||!natural(node.attempt_count)||!Array.isArray(node.runs)||node.runs.length>5||node.runs.length>node.attempt_count||node.runs_truncated!==(node.attempt_count>node.runs.length)|| - node.runs.some(run=>!id(run.id)||typeof run.state!=='string'||Object.hasOwn(run,'validation')))throw Error('Invalid saved Trial overview.'); - if(lineage!==null&&(lineage?.trial?.catalog_id!==source||lineage.trial.trial_id!==trial.id||lineage.trial.material_sha256!==trial.material.sha256))throw Error('Trial ancestry belongs to different material.'); - } - async function loadGraph(append=false){ - const requestedOrigin=append?origin:$('origin').value,cohort=append?appliedCohort:($('cohort').value||null),offset=append?next:0; - if(!id(requestedOrigin)||append&&nodes.size>=100)return; - return perform(async(active,signal)=>{ - const args={origin_catalog_id:requestedOrigin,offset,limit:20,...(cohort===null?{}:{cohort})}; - const value=await read('progression',args,signal);if(!active())return; - page(value,offset,20,value.nodes); - if(value.origin_catalog_id!==requestedOrigin||value.evidence!=='saved_metadata_only'||value.historical!==true||value.execution_available!==false||value.cohort!==cohort||!Array.isArray(value.related)||value.related.length>40)throw Error('History response differs from the selected source.'); - const seen=new Set(append?nodes.keys():[]); - for(const node of value.nodes){validNode(node,requestedOrigin);const identity=key(requestedOrigin,node.trial.id);if(seen.has(identity))throw Error('Duplicate Trial in history page.');seen.add(identity);} - for(const ref of value.related)if(!id(ref.catalog_id)||!id(ref.trial_id)||typeof ref.material_sha256!=='string'||!/^[a-f0-9]{64}$/.test(ref.material_sha256))throw Error('Invalid related Trial identity.'); - if(seen.size>100)throw Error('The graph window holds 100 Trials; choose a recorded cohort for another window.'); - if(!append){clearGraph();origin=requestedOrigin;appliedCohort=cohort;} - for(const node of value.nodes)nodes.set(key(origin,node.trial.id),structuredClone(node)); - for(const ref of value.related)related.set(key(ref.catalog_id,ref.trial_id),structuredClone(ref)); - next=offset+value.nodes.length;total=value.total;render(); - $('status').textContent=nodes.size?'Loaded '+nodes.size+' recorded Trials.':'No Trials match this source and recorded cohort.'; - }); - } - function detail(){ - const node=nodes.get(selected);$('detail').replaceChildren();if(!node)return; - const add=(tag,text)=>{const e=element(tag,text);$('detail').append(e);return e;}; - add('h3','Trial '+short(node.trial.id));add('p',node.study.name); - if(node.lineage){const intent=node.lineage.intent;add('p',intent.hypothesis);add('p',intent.change_summary);} - else add('p','Historical ancestry and intent are unknown. No relationships are inferred.'); - for(const kind of ['parent','baseline']){ - const ref=node.lineage?.intent[kind],label=kind==='parent'?'Parent':'Comparison baseline'; - if(ref){const button=add('button',label+' Trial '+short(ref.trial_id));button.type='button';button.onclick=()=>inspectTrial(ref.catalog_id,ref.trial_id);} - else add('p',label+': none declared.'); - } - for(const [label,inspect,identity] of [['Inspect Study',inspectStudy,node.study.id],['Inspect Trial',inspectTrial,node.trial.id]]){const button=add('button',label);button.type='button';button.onclick=()=>inspect(origin,identity);} - add('h4','Saved Run outcomes'); - for(const run of node.runs){const button=add('button','Inspect Run '+short(run.id)+' · saved '+run.state);button.type='button';button.onclick=()=>inspectRun(origin,run.id);} - if(!node.attempt_count)add('p','No recorded admitted attempt.'); - if(node.runs_truncated)add('p','Showing five of '+node.attempt_count+' recorded attempts.'); - const facts=add('details'),summary=element('summary','Exact original references and recorded intent');facts.append(summary,element('pre',JSON.stringify({origin_catalog_id:origin,trial:node.trial,lineage:node.lineage},null,2))); - } - function render(){ - const focused=$('graph').contains(document.activeElement)?document.activeElement.id:null; - const graph=window.WorkspaceExperimentGraph.render([...nodes.values()].map(node=>({id:key(origin,node.trial.id),title:node.study.name,recordedAt:node.lineage?.recorded_at||'', - parent:node.lineage?.intent.parent?key(node.lineage.intent.parent.catalog_id,node.lineage.intent.parent.trial_id):null, - baseline:node.lineage?.intent.baseline?key(node.lineage.intent.baseline.catalog_id,node.lineage.intent.baseline.trial_id):null, - lines:[short(node.trial.id)+' · '+(node.lineage?.intent.ancestry||'ancestry unknown'),node.lineage?.intent.change_summary||'Intent not recorded',(node.runs[0]?.state||'prepared')+' · '+node.attempt_count+' recorded attempts']})),{selected,zoom}); - $('graph').innerHTML=graph.html; - for(const button of $('graph').querySelectorAll('[data-lineage-node]')){button.disabled=busy;button.onclick=()=>{selected=button.dataset.lineageNode;render();$('status').textContent='Selected Trial '+short(nodes.get(selected).trial.id)+'.';};} - const outside=[...related.keys()].filter(identity=>!nodes.has(identity)).length; - $('summary').textContent=nodes.size+' of '+total+' Trials loaded · '+(appliedCohort===null?'all recorded cohorts':'applied cohort: '+appliedCohort)+' · '+outside+' related identities outside this graph window.'; - detail();if(focused)$('graph').querySelector('#'+CSS.escape(focused))?.focus({preventScroll:true}); - } - $('origins').onclick=()=>loadOrigins();$('origins-more').onclick=()=>loadOrigins(true); - $('form').onsubmit=event=>{event.preventDefault();loadGraph();};$('more').onclick=()=>loadGraph(true); - $('origin').onchange=()=>{++generation;pending?.abort();pending=null;busy=false;clearGraph();$('status').textContent='Load the selected history source.';controls();}; - for(const [name,delta] of [['zoom-in',0.2],['zoom-out',-0.2],['zoom-reset',null]])$(name).onclick=()=>{zoom=delta===null?1:Math.max(0.6,Math.min(1.6,zoom+delta));render();}; - return {setContext(value){if(value.key!==context?.key){const ownedFocus=root.contains(document.activeElement);context=value;reset();if(ownedFocus)onContextFocusLost();}else context=value;root.hidden=!value.enabled;controls();},clear(){context=null;reset();root.hidden=true;}}; - }; -})(); diff --git a/static/project-transfer.css b/static/project-transfer.css deleted file mode 100644 index c7acbc75..00000000 --- a/static/project-transfer.css +++ /dev/null @@ -1,2 +0,0 @@ -:root{color-scheme:light dark;--bg:#f5f6f8;--panel:#fff;--ink:#172333;--muted:#4c5b6c;--line:#b9c3cf;--accent:#1648a2;--notice:#fff3d5;--error:#8a1720} -*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:16px/1.55 system-ui,sans-serif}main{max-width:1080px;margin:0 auto;padding:2rem 1.2rem 5rem}header{max-width:800px;margin-bottom:1.7rem}.eyebrow{font-size:.85rem;text-transform:uppercase;letter-spacing:.08em;color:var(--muted)}h1{line-height:1.15;font-size:clamp(2rem,4vw,2.8rem)}h2{font-size:1.35rem;margin-top:0}h3{font-size:1.05rem}p{overflow-wrap:anywhere}section{background:var(--panel);border:1px solid var(--line);border-radius:.6rem;padding:1.3rem;margin:1.2rem 0}fieldset{border:0;padding:0;margin:0}legend{font-weight:650;margin-bottom:.7rem}label{display:flex;flex-direction:column;gap:.35rem;margin:.7rem 0;font-weight:550}input,select,textarea,button{font:inherit}input,select,textarea{max-width:100%;width:100%;padding:.55rem;border:1px solid var(--line);border-radius:.25rem;background:var(--panel);color:var(--ink)}textarea{resize:vertical}button{padding:.55rem .8rem;margin:.3rem .35rem .3rem 0;cursor:pointer;border:1px solid var(--accent);border-radius:.3rem;color:var(--accent);background:var(--panel)}button:disabled{opacity:.5;cursor:default}a{color:var(--accent)}.grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:.2rem 1rem}.actions{align-self:end}.help{font-size:.9rem;color:var(--muted)}.check{display:flex;flex-direction:row;align-items:flex-start;gap:.65rem;font-weight:400}.check input{width:auto;flex:0 0 auto;margin-top:.35rem}.facts{padding-left:1.4rem;overflow-wrap:anywhere}.facts li{margin:.65rem 0}.facts li p{margin:.15rem 0}.facts button{font-size:.85rem;padding:.15rem .4rem}.notice{padding:.8rem;border-left:4px solid #9c6800;background:var(--notice);overflow-wrap:anywhere}#transfer-error{padding:1rem;border:2px solid var(--error);color:var(--error);background:var(--panel);overflow-wrap:anywhere}#transfer-receipt,#report-summary,#transfer-context{overflow-wrap:anywhere}details{margin:1rem 0;border-top:1px solid var(--line);padding-top:.7rem}summary{cursor:pointer;font-weight:600}:focus-visible{outline:3px solid var(--accent);outline-offset:3px}.skip-link{position:absolute;left:1rem;top:-5rem;padding:.5rem;background:var(--panel);z-index:10}.skip-link:focus{top:.5rem}[hidden]{display:none!important}@media(max-width:650px){main{padding:1rem .7rem 3rem}.grid{grid-template-columns:1fr}section{padding:1rem}button{min-height:44px}h1{font-size:2rem}}@media(prefers-color-scheme:dark){:root{--bg:#10171f;--panel:#19232e;--ink:#ecf0f5;--muted:#b7c5d5;--line:#647487;--accent:#99bbff;--notice:#362d18;--error:#ffabb2}} diff --git a/static/project-transfer.html b/static/project-transfer.html deleted file mode 100644 index 504a2c44..00000000 --- a/static/project-transfer.html +++ /dev/null @@ -1,91 +0,0 @@ - - - - - Extract a legacy project · Tuldok - - - - - -
                      -

                      Tuldok / legacy catalog

                      Extract an independent project

                      -

                      Choose one registered project and the global records that belong with it. Review required dependencies before explicitly stopping the source and publishing a new project directory.

                      -

                      Preview reads the source. Extraction preserves its verified post-stop baseline. Scripts stay inert; these controls do not run computations or restore old permissions.

                      -
                      -

                      Reading extraction status…

                      - -
                      -

                      Current controller

                      Sign in to inspect this controller.

                      - - -
                      -
                      -

                      1. Choose the project and source facts

                      -
                      Explicit source selection -
                      -
                      -

                      Global data, Studies and Runs are not assigned to a project by their names or location. Enter the exact IDs you choose; review additional dependencies and unassigned facts in the report.

                      -
                      -
                      - -
                      Add producer observations or working files -
                      -
                      -
                      -
                      -

                      Selected working files become new UTF-8 adoptions attributed to this extraction. No earlier adoption is invented and no script is executed.

                      -
                      -
                      Explicit ownership decisions -
                      -
                      -

                      A required dependency cannot be excluded while claiming a complete extraction.

                      -
                      -
                      Load a prepared selection document - -

                      This imports only the named project snapshot, finite source families and ownership decisions. It does not accept commands, credentials or execution options. Review its summary before preview.

                      -
                      -

                      Selection summary

                        -

                        No selection has been previewed.

                        - -
                        -
                        -
                        -

                        2. Review dependency closure

                        Preview a selection to see its report.

                        -
                        -
                        -

                          - -
                          -
                          -

                          3. Stop the source and capture its backup

                          -

                          Stopping fences new source requests and drains the existing owners. A failed or unknown stop keeps the source fenced and cannot authorize extraction. Keep the private backup separate from the destination.

                          -
                          Explicit stop destinations -
                          -
                          - - -
                          -
                          - -
                          -

                          5. Publish the reviewed project

                          -
                          Independent destination -

                          No stopped destination is confirmed.

                          - -

                          Keep this exact key and destination when reconciling an uncertain outcome. Refreshing status never retries publication.

                          - - -
                          - -

                          Retained outcome

                          No extraction has been acknowledged.

                          - -

                          -
                          -
                          - diff --git a/static/project-transfer.js b/static/project-transfer.js deleted file mode 100644 index 58d00e5c..00000000 --- a/static/project-transfer.js +++ /dev/null @@ -1,203 +0,0 @@ -'use strict'; -// Five installed controller operations; discovery never executes remote schemas. -(()=>{ - const families={records:'Data records',responses:'Responses',preferences:'Comparative judgments',selections:'Saved selections',searches:'Saved searches',graphs:'Pipeline graphs',recipe_drafts:'Recipe definitions',model_definitions:'Model definitions',studies:'Studies',runs:'Historical Runs',releases:'Frozen releases',exposures:'Exposure facts',model_materials:'Model materials',recipe_materials:'Recipe materials',runtime_materials:'Runtime materials'}; - const canonicalFamilies=['workbench_record','workbench_response','workbench_preference','saved_selection','saved_search','pipeline_graph','recipe_definition','model_definition','study_history','trial_history','run_history','candidate_history','dataset_release','dataset_exposure','model_material','recipe_material','runtime_material','project_snapshot','project_artifact','project_file','generation_history','grounded_history','caption_proposal_history','text_proposal_history']; - const shaRoots=new Set(['releases','model_materials','recipe_materials','runtime_materials']),producerKinds=['generation','grounded','caption','text_proposal']; - const sections=['included','required','unassigned','excluded','blockers','warnings','artifacts','working_files']; - const actions=['status','preview','report_page','stop','extract']; - const exact=(v,keys)=>v!==null&&typeof v==='object'&&!Array.isArray(v)&&Object.keys(v).length===keys.length&&keys.every(k=>Object.hasOwn(v,k)); - const text=(v,max,empty=false)=>typeof v==='string'&&Array.from(v).length<=max&&(empty||/\S/u.test(v)); - const id=v=>typeof v==='string'&&/^[a-f0-9]{32}$/.test(v),sha=v=>typeof v==='string'&&/^[a-f0-9]{64}$/.test(v); - const integer=(v,max=1000000)=>Number.isSafeInteger(v)&&v>=0&&v<=max; - const clone=v=>JSON.parse(JSON.stringify(v)); - function protocol(message){const failure=Error(message);failure.code='invalid_response';return failure;} - const leaf=v=>text(v,200)&&!/[\/\\\x00-\x1f]/.test(v)&&!['.','..'].includes(v); - const working=v=>exact(v,['kind','path'])&&['script','note'].includes(v.kind)&&text(v.path,512)&&v.path.startsWith(v.kind==='script'?'scripts/':'notes/')&&!/[\\\x00-\x1f]/.test(v.path)&&v.path.split('/').length<=16&&v.path.split('/').every(p=>p&&!['.','..'].includes(p)); - function normalizeSelection(value){ - if(!exact(value,['project_snapshot','roots','decisions'])||!exact(value.project_snapshot,['id','manifest_sha256'])||!id(value.project_snapshot.id)||!sha(value.project_snapshot.manifest_sha256))throw Error('Choose the exact registered project ID and manifest SHA256.'); - if(!value.roots||typeof value.roots!=='object'||Array.isArray(value.roots)||Object.keys(value.roots).some(k=>!Object.hasOwn(families,k)&&!['producer_jobs','working_files'].includes(k)))throw Error('The selection contains an unknown source family.'); - const roots={}; - for(const family of Object.keys(families)){ - const list=value.roots[family]??[]; - if(!Array.isArray(list)||list.length>10000||new Set(list).size!==list.length||!list.every(shaRoots.has(family)?sha:id))throw Error('Use unique exact '+(shaRoots.has(family)?'SHA256 digests':'IDs')+' for '+families[family]+'.'); - roots[family]=[...list].sort(); - } - const jobs=value.roots.producer_jobs??[],files=value.roots.working_files??[]; - if(!Array.isArray(jobs)||jobs.length>10000||!jobs.every(j=>exact(j,['kind','id'])&&producerKinds.includes(j.kind)&&id(j.id))||new Set(jobs.map(j=>j.kind+':'+j.id)).size!==jobs.length)throw Error('Choose known producer kinds and unique job IDs.'); - if(!Array.isArray(files)||files.length>1000||!files.every(working)||new Set(files.map(f=>f.kind+':'+f.path)).size!==files.length)throw Error('Working files must be unique scripts/... or notes/... paths within the selected project.'); - roots.producer_jobs=clone(jobs).sort((a,b)=>(a.kind+':'+a.id).localeCompare(b.kind+':'+b.id)); - roots.working_files=clone(files).sort((a,b)=>(a.kind+':'+a.path).localeCompare(b.kind+':'+b.path)); - if(!Array.isArray(value.decisions)||value.decisions.length>10000||!value.decisions.every(d=>exact(d,['family','id','include','reason'])&&canonicalFamilies.includes(d.family)&&text(d.id,128)&&typeof d.include==='boolean'&&text(d.reason,1000))||new Set(value.decisions.map(d=>d.family+':'+d.id)).size!==value.decisions.length)throw Error('Each ownership decision needs a known family, exact report ID and reason.'); - const result={project_snapshot:clone(value.project_snapshot),roots,decisions:clone(value.decisions).sort((a,b)=>(a.family+':'+a.id).localeCompare(b.family+':'+b.id))}; - if(new TextEncoder().encode(JSON.stringify(result)).length>1048576)throw Error('The selection exceeds one MiB.'); - return result; - } - const sourceOK=v=>exact(v,['catalog_id','schema_version','catalog_signature','snapshot_sha256','backup_receipt_sha256','external_inventory_sha256'])&&id(v.catalog_id)&&v.schema_version===10&&sha(v.catalog_signature)&&sha(v.snapshot_sha256)&&(v.backup_receipt_sha256===null||sha(v.backup_receipt_sha256))&&sha(v.external_inventory_sha256); - const failureOK=v=>v===null||(v&&typeof v==='object'&&Object.keys(v).every(k=>['action','code','detail','staging_directory'].includes(k))&&['stop','extract'].includes(v.action)&&text(v.code,120)&&text(v.detail,1000,true)&&(v.staging_directory===undefined||leaf(v.staging_directory))); - function receiptOK(v){ - if(v===null)return true; - if(exact(v,['format','committed','project_id','checkpoint','request_key','input_sha256','report_sha256','source','artifacts','verification','replayed']))return v.format==='project_transfer_receipt_v1'&&v.committed===true&&id(v.project_id)&&sha(v.checkpoint)&&text(v.request_key,128)&&sha(v.input_sha256)&&sha(v.report_sha256)&&sourceOK(v.source)&&v.source.backup_receipt_sha256!==null&&exact(v.artifacts,['copied_count','missing_regenerable_count'])&&integer(v.artifacts.copied_count)&&integer(v.artifacts.missing_regenerable_count)&&exact(v.verification,['checkpoint','active_data_records'])&&v.verification.checkpoint===v.checkpoint&&integer(v.verification.active_data_records)&&typeof v.replayed==='boolean'; - return v&&typeof v==='object'&&!Array.isArray(v)&&Object.keys(v).every(k=>['committed','selected','project_id','checkpoint','request_key','directory'].includes(k))&&v.committed===true&&typeof v.selected==='boolean'&&id(v.project_id)&&sha(v.checkpoint)&&text(v.request_key,128)&&(v.directory===undefined||leaf(v.directory)); - } - function stateOK(v){ - if(!exact(v,['format','binding','phase','source_catalog_id','control_catalog_id','roots','available_actions','report','backup','destination','failure','last_receipt'])||v.format!=='project_transfer_state_v1'||!id(v.binding)||!['source_live','source_stopped','source_fenced'].includes(v.phase)||!id(v.source_catalog_id)||!id(v.control_catalog_id)||v.source_catalog_id===v.control_catalog_id||!Array.isArray(v.roots)||!v.roots.length||v.roots.length>100||!v.roots.every(r=>typeof r==='string'&&/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/.test(r))||new Set(v.roots).size!==v.roots.length||!Array.isArray(v.available_actions)||!v.available_actions.includes('status')||!v.available_actions.every(a=>actions.includes(a))||new Set(v.available_actions).size!==v.available_actions.length||!failureOK(v.failure)||!receiptOK(v.last_receipt))return false; - if(v.report!==null&&(!exact(v.report,['report_sha256','selection_sha256','source','can_extract','counts'])||!sha(v.report.report_sha256)||!sha(v.report.selection_sha256)||!sourceOK(v.report.source)||v.report.source.catalog_id!==v.source_catalog_id||typeof v.report.can_extract!=='boolean'||!exact(v.report.counts,sections)||!Object.values(v.report.counts).every(n=>integer(n))))return false; - if(v.backup!==null&&(!exact(v.backup,['root','directory','receipt_sha256'])||!v.roots.includes(v.backup.root)||!leaf(v.backup.directory)||!sha(v.backup.receipt_sha256)))return false; - if(v.destination!==null&&(!exact(v.destination,['root','directory'])||!v.roots.includes(v.destination.root)||!leaf(v.destination.directory)))return false; - return v.phase!=='source_fenced'||v.available_actions.every(a=>a==='status'); - } - function itemOK(section,v){ - if(['included','unassigned'].includes(section))return exact(v,['family','id','revision'])&&text(v.family,64)&&text(v.id,128)&&(v.revision===null||integer(v.revision)&&v.revision>0); - if(['required','excluded'].includes(section))return exact(v,['family','id'])&&text(v.family,64)&&text(v.id,128); - if(['blockers','warnings'].includes(section))return exact(v,['family','id','code','detail'])&&text(v.family,64)&&text(v.id,128)&&text(v.code,120)&&text(v.detail,1000,true); - if(section==='artifacts')return exact(v,['sha256','bytes','preservation','state'])&&sha(v.sha256)&&integer(v.bytes,Number.MAX_SAFE_INTEGER)&&['unique','regenerable'].includes(v.preservation)&&['verified','missing','corrupt','unverified'].includes(v.state); - return section==='working_files'&&exact(v,['kind','path','sha256','bytes'])&&working({kind:v.kind,path:v.path})&&sha(v.sha256)&&integer(v.bytes,1048576); - } - function pageOK(v){return exact(v,['format','binding','report_sha256','section','items','offset','limit','total'])&&v.format==='project_transfer_report_page_v1'&&id(v.binding)&&sha(v.report_sha256)&§ions.includes(v.section)&&integer(v.offset)&&v.limit===40&&integer(v.total)&&Array.isArray(v.items)&&v.items.length<=40&&v.items.length===Math.min(40,Math.max(0,v.total-v.offset))&&v.items.every(item=>itemOK(v.section,item));} - if(typeof module==='object'&&module.exports)module.exports={normalizeSelection,stateOK,pageOK,receiptOK}; - if(typeof document==='undefined')return; - const $=name=>document.getElementById(name),base='/api/workspace/project-transfer'; - const pins=/* PROJECT_TRANSFER_CONTRACTS_START */ -{ - "project_transfer.extract": "00e0ae9a09d36279684f3614e76445ba1dca50557e55047f699a48b43f823aa8", - "project_transfer.preview": "7072d7f08763dd05e36c9be6b5aa730411948bb34c32361ec80215dd8b31bf24", - "project_transfer.report_page": "09dd19eb4299b323710181ea20a7b4dcffce0c2173eeb51a315d551e72811515", - "project_transfer.status": "190e72fe33a75105b3da034469931b4e30e646ef38af6b55ff8a3c2444725870", - "project_transfer.stop": "050519dab950115508123eef3bf9a9fe4aad40b920a6612b1b850688afec90b8" -} -/* PROJECT_TRANSFER_CONTRACTS_END */; - const routes={status:{method:'GET',path:''},preview:{method:'POST',path:'/preview'},report_page:{method:'GET',path:'/report'},stop:{method:'POST',path:'/stop'},extract:{method:'POST',path:'/extract'}}; - let state=null,contracts=new Set(),draft={roots:{},decisions:[]},busy=false,epoch=0,revision=0,reviewed=false,onlyCessation=false; - let handoff=false,freshControlAttempt=false,freshControl=false,refreshAfter=false,pending=null,lastReceipt=null,page=null,controlCatalog=null,dirty=false; - function error(message){$('transfer-error').hidden=false;$('transfer-error').textContent=message;$('transfer-error').focus();} - function message(value){$('transfer-status').textContent=value;} - function clearError(){$('transfer-error').hidden=true;$('transfer-error').textContent='';} - function currentOwner(){const value=window.TuldokAuth?.identity?.();return value?.kind==='owner'&&value.scopes?.includes('admin');} - function action(name){return currentOwner()&&contracts.has('project_transfer.'+name)&&state?.available_actions.includes(name);} - function signedControl(){return freshControl&&window.TuldokAuth?.identity?.()?.catalog_id===controlCatalog;} - function controls(){ - const live=state?.phase==='source_live',stopped=state?.phase==='source_stopped',report=state?.report; - $('selection-fields').disabled=busy||!!pending||!action('preview')||(stopped&&!signedControl()); - $('transfer-refresh').disabled=busy; - $('report-load').disabled=busy||!report||!action('report_page')||(stopped&&!signedControl()); - $('report-previous').disabled=$('report-load').disabled||!page||page.offset===0; - $('report-next').disabled=$('report-load').disabled||!page||page.offset+page.items.length>=page.total; - $('report-reviewed').disabled=busy||!reviewed||!report; - $('stop-fields').disabled=busy||!live||!action('stop')||!reviewed; - $('transfer-stop').disabled=$('stop-fields').disabled||!onlyCessation||!$('report-reviewed').checked||!$('stop-confirm').checked; - $('extract-fields').disabled=busy||!!pending||!stopped||!signedControl()||!action('extract')||!reviewed||!report?.can_extract; - $('transfer-extract').disabled=$('extract-fields').disabled||!$('report-reviewed').checked||!$('extract-confirm').checked; - $('transfer-reconcile').hidden=!pending; - $('transfer-reconcile').disabled=busy||!pending||!stopped||!signedControl()||!action('extract')||state.report?.report_sha256!==pending.report_sha256; - $('control-signin-panel').hidden=!handoff&&!(stopped&&!signedControl()); - $('control-signin').disabled=busy; - $('transfer-fence').hidden=state?.phase!=='source_fenced'; - $('transfer-fence').textContent='The source remains fenced. No stopped-source proof is available; extraction is unavailable. Retain any reported receipt or staging directory for operator review.'; - } - function invalidate(){revision++;dirty=true;reviewed=false;onlyCessation=false;page=null;$('report-reviewed').checked=false;$('stop-confirm').checked=false;$('extract-confirm').checked=false;$('report-items').replaceChildren();$('selection-dirty').textContent='Selection changed. Explicitly preview it again before any source action.';controls();} - function selection(){return normalizeSelection({project_snapshot:{id:$('snapshot-id').value.trim(),manifest_sha256:$('snapshot-sha').value.trim()},roots:clone(draft.roots),decisions:clone(draft.decisions)});} - function summarize(){ - const list=$('selection-summary');list.replaceChildren(); - for(const [key,values] of Object.entries(draft.roots))if(values.length){const li=document.createElement('li');li.textContent=`${families[key]??(key==='producer_jobs'?'Producer observations':'Working files')}: ${values.length}`;const remove=document.createElement('button');remove.type='button';remove.textContent='Remove this family';remove.onclick=()=>{delete draft.roots[key];invalidate();summarize();};li.append(remove);list.append(li);} - for(const decision of draft.decisions){const li=document.createElement('li');li.textContent=`${decision.include?'Include':'Exclude'} ${decision.family} ${decision.id}: ${decision.reason}`;const remove=document.createElement('button');remove.type='button';remove.textContent='Remove decision';remove.onclick=()=>{draft.decisions=draft.decisions.filter(d=>d!==decision);invalidate();summarize();};li.append(remove);list.append(li);} - if(!list.children.length){const li=document.createElement('li');li.textContent='No global source facts selected. Only the exact registered snapshot will be included unless you add roots.';list.append(li);} - } - function fillRoots(){for(const name of ['backup-root','destination-root']){const node=$(name),old=node.value;node.replaceChildren();for(const alias of state.roots){const option=document.createElement('option');option.value=alias;option.textContent=alias;node.append(option);}if(state.roots.includes(old))node.value=old;}} - function keepReceipt(value){if(!receiptOK(value)||value===null)return;lastReceipt=clone(value);$('transfer-receipt').textContent=`Publication recorded: project ${value.project_id}; ${value.verification?'current verified checkpoint':'recorded checkpoint'} ${value.checkpoint}; request ${value.request_key}.${value.replayed?' Exact request reconciled.':''}`;$('project-files-link').hidden=false;} - function adopt(value,{ownPreview=false}={}){ - if(!stateOK(value))throw protocol('Unsupported extraction controller response. Refresh status before continuing.'); - const changed=state?.binding!==value.binding;state=value;controlCatalog=value.control_catalog_id; - fillRoots();if(value.last_receipt)keepReceipt(value.last_receipt); - if(!ownPreview)$('selection-dirty').textContent='Controller status is current. Explicitly preview your local selection before any source effect.'; - if(changed||!ownPreview){reviewed=false;onlyCessation=false;$('report-reviewed').checked=false;page=null;$('report-items').replaceChildren();} - if(value.phase==='source_live'){handoff=false;freshControl=false;freshControlAttempt=false;} - else{handoff=!signedControl();} - $('transfer-context').textContent=`${value.phase.replaceAll('_',' ')} · source ${value.source_catalog_id} · separate control host ${value.control_catalog_id}`; - $('control-signin-status').textContent=`Control catalog: ${value.control_catalog_id}. ${signedControl()?'Fresh control sign-in verified.':'Submit a new control-host login code using the standard session dialog.'}`; - $('report-summary').textContent=value.report?`Report ${value.report.report_sha256}. Included ${value.report.counts.included}; required ${value.report.counts.required}; unassigned ${value.report.counts.unassigned}; blockers ${value.report.counts.blockers}; warnings ${value.report.counts.warnings}; artifacts ${value.report.counts.artifacts}. ${value.report.can_extract?'Complete stopped-source closure is available.':'Extraction is unavailable until every blocker is resolved.'}`:'No current report. Explicitly preview this selection.'; - if(value.destination){$('destination-root').value=value.destination.root;$('destination-directory').value=value.destination.directory;} - if(value.backup){$('backup-root').value=value.backup.root;$('backup-directory').value=value.backup.directory;} - $('extract-destination').textContent=value.destination?`Confirmed destination: ${value.destination.root} / ${value.destination.directory}.`:'Confirm the destination before stopping the source.'; - if(value.failure){$('transfer-pending').hidden=false;$('transfer-pending').textContent=`${value.failure.action}: ${value.failure.code}. ${value.failure.detail}${value.failure.staging_directory?' Retained stage: '+value.failure.staging_directory+'.':''}`;} - controls(); - } - function row(section,item){const li=document.createElement('li'); - if(section==='artifacts')li.textContent=`${item.sha256} · ${item.bytes} bytes · ${item.preservation} · ${item.state}`; - else if(section==='working_files')li.textContent=`${item.kind}: ${item.path} · ${item.bytes} bytes · ${item.sha256}`; - else{li.textContent=`${item.family} · ${item.id}${item.revision!==undefined&&item.revision!==null?' · revision '+item.revision:''}${item.code?' · '+item.code:''}`;if(item.detail){const detail=document.createElement('p');detail.textContent=item.detail;li.append(detail);}} - return li; - } - async function discover(ticket){ - const response=await window.TuldokAuth.fetch('/api/agent/capabilities',{cache:'no-store'}),value=await response.json(); - if(ticket!==epoch)return false;if(!response.ok)throw Error(value?.error??'Sign in, then refresh extraction status.'); - if(!exact(value,['format','envelope_version','catalog_id','operations','limits'])||value.format!=='agent_capabilities_v1'||value.envelope_version!==1||!id(value.catalog_id)||!Array.isArray(value.operations)||value.operations.length>256||!exact(value.limits,['request_bytes','response_bytes','max_operations'])||value.limits.request_bytes!==1048576||value.limits.response_bytes!==16777216||value.limits.max_operations!==256)throw protocol('Unsupported capability discovery response.'); - if(window.TuldokAuth.identity?.()?.catalog_id!==value.catalog_id)throw Error('Capabilities belong to a different current session.'); - const chosen=new Set(),seen=new Set(); - for(const operation of value.operations){if(!exact(operation,['id','contracts','availability'])||typeof operation.id!=='string'||seen.has(operation.id)||!Array.isArray(operation.contracts)||operation.contracts.length>4||!['available','unavailable','unsupported'].includes(operation.availability))throw Error('Unsupported capability entry.');seen.add(operation.id); - const versions=new Set();for(const contract of operation.contracts){if(!exact(contract,['version','sha256','input_schema','output_schema','scopes','authority','effects','retry','bounds'])||!Number.isSafeInteger(contract.version)||contract.version<1||versions.has(contract.version)||!sha(contract.sha256)||!contract.input_schema||typeof contract.input_schema!=='object'||!contract.output_schema||typeof contract.output_schema!=='object'||!Array.isArray(contract.scopes)||!contract.scopes.every(s=>text(s,40))||!['owner','agent'].includes(contract.authority)||!['read','mutation'].includes(contract.effects)||!['read','cas','receipt','never'].includes(contract.retry)||!exact(contract.bounds,['request_bytes','response_bytes','inline_decoded_bytes','json_depth'])||!Object.values(contract.bounds).every(v=>Number.isSafeInteger(v)&&v>0))throw protocol('Unsupported installed contract discovery frame.');versions.add(contract.version);if(contract.version===1&&operation.availability==='available'&&pins[operation.id]===contract.sha256)chosen.add(operation.id);} - } - contracts=chosen;if(!contracts.has('project_transfer.status'))throw Error('No compatible extraction status contract is available.');return true; - } - async function call(name,args,ticket,{mutation=false}={}){ - const operation='project_transfer.'+name,route=routes[name];if(!contracts.has(operation))throw Error('No compatible contract is available for '+name+'. Refresh status.'); - let response,value; - try{const query=route.method==='GET'&&Object.keys(args).length?'?'+new URLSearchParams(args):'';response=await window.TuldokAuth.fetch(base+route.path+query,{method:route.method,cache:'no-store',headers:{'Tuldok-Operation-Contract':operation+'/1/'+pins[operation],...(route.method==='POST'?{'Content-Type':'application/json'}:{})},...(route.method==='POST'?{body:JSON.stringify(args)}:{})});value=await response.json();} - catch(_){const failure=Error(mutation?'Acknowledgement unavailable. Keep this exact request and refresh status; the source may be fenced or publication may already exist.':'Response unavailable. Refresh status explicitly.');failure.code='transport';throw failure;} - if(new TextEncoder().encode(JSON.stringify(value)).length>1048576)throw Error('Extraction response exceeds its interface bound.'); - if(mutation&&value?.receipt&&receiptOK(value.receipt))keepReceipt(value.receipt); - if(ticket!==epoch){message('A session change superseded the response. Any known publication receipt has been retained. Refresh status before continuing.');return null;} - if(!response.ok){const keys=['error',...(Object.hasOwn(value??{},'code')?['code']:[]),...(Object.hasOwn(value??{},'receipt')?['receipt']:[])];const valid=exact(value,keys)&&text(value.error,1000)&&(!value.code||text(value.code,120))&&(!value.receipt||receiptOK(value.receipt));const failure=Error(valid?`${value.code??'error'}: ${value.error}`:'Unsupported error response. Retain the current request and refresh status.');failure.code=valid?(value.code??'http_error'):'transport';throw failure;} - return value; - } - async function run(work){if(busy)return;busy=true;clearError();const ticket=++epoch;controls();try{await work(ticket);}catch(failure){if(ticket===epoch){if(['invalid_response','transport','stale'].includes(failure.code)){reviewed=false;onlyCessation=false;$('report-reviewed').checked=false;if(failure.code==='invalid_response')contracts.clear();}error(failure.message);}}finally{busy=false;controls();if(refreshAfter){refreshAfter=false;void refresh();}}} - async function refresh(){return run(async ticket=>{if(!await discover(ticket))return;const value=await call('status',{},ticket);if(value){adopt(value);message('Controller status refreshed. No source or publication action was retried.');}});} - async function loadPage(ticket,offset=0,section=$('report-section').value){ - const binding=state?.binding,report=state?.report?.report_sha256;if(!report)throw Error('Preview the selection before loading its report.'); - const value=await call('report_page',{binding,report_sha256:report,section,offset,limit:40},ticket); - if(!value)return;if(!pageOK(value)||value.binding!==binding||value.report_sha256!==report||value.section!==section||value.offset!==offset||value.total!==state.report.counts[section])throw protocol('Report page is stale or unsupported. Preview and review the current report again.'); - page=value;$('report-items').replaceChildren(...value.items.map(item=>row(section,item)));$('report-page-status').textContent=`${section.replaceAll('_',' ')}: ${value.items.length?value.offset+1:0}–${value.offset+value.items.length} of ${value.total}.`; - if(section==='blockers')onlyCessation=value.total===1&&value.items.length===1&&value.items[0].code==='cessation_proof_unavailable'&&value.items[0].family==='source'&&value.items[0].id===state.source_catalog_id; - controls(); - } - function selectionAction(work){try{work();invalidate();summarize();}catch(failure){error(failure.message);}} - for(const [value,label] of Object.entries(families)){const option=document.createElement('option');option.value=value;option.textContent=label;$('selection-family').append(option);} - for(const value of canonicalFamilies){const option=document.createElement('option');option.value=value;option.textContent=value.replaceAll('_',' ');$('decision-family').append(option);} - $('selection-add').onclick=()=>selectionAction(()=>{const key=$('selection-family').value,values=$('selection-ids').value.split(/[\s,]+/u).filter(Boolean);if(!values.length||!values.every(shaRoots.has(key)?sha:id))throw Error('Enter exact '+(shaRoots.has(key)?'SHA256 digests':'32-character IDs')+' for this family.');draft.roots[key]=[...new Set([...(draft.roots[key]??[]),...values])].sort();if(draft.roots[key].length>10000)throw Error('Select at most 10,000 roots per family.');$('selection-ids').value='';}); - $('producer-add').onclick=()=>selectionAction(()=>{const value={kind:$('producer-kind').value,id:$('producer-id').value.trim()};if(!producerKinds.includes(value.kind)||!id(value.id))throw Error('Choose a known producer kind and exact job ID.');const list=draft.roots.producer_jobs??=[];if(!list.some(v=>v.kind===value.kind&&v.id===value.id))list.push(value);$('producer-id').value='';}); - $('working-add').onclick=()=>selectionAction(()=>{const value={kind:$('working-kind').value,path:$('working-path').value.trim()};if(!working(value))throw Error('Use an exact scripts/... or notes/... path without traversal.');const list=draft.roots.working_files??=[];if(!list.some(v=>v.kind===value.kind&&v.path===value.path))list.push(value);$('working-path').value='';}); - $('decision-add').onclick=()=>selectionAction(()=>{const value={family:$('decision-family').value,id:$('decision-id').value.trim(),include:$('decision-include').value==='include',reason:$('decision-reason').value};if(!canonicalFamilies.includes(value.family)||!text(value.id,128)||!text(value.reason,1000))throw Error('Enter a known report family, exact ID and reason.');draft.decisions=draft.decisions.filter(v=>!(v.family===value.family&&v.id===value.id));draft.decisions.push(value);$('decision-id').value='';$('decision-reason').value='';}); - $('selection-clear').onclick=()=>selectionAction(()=>{draft={roots:{},decisions:[]};}); - $('selection-file').onchange=async()=>{const ticket=epoch;try{const file=$('selection-file').files?.[0];if(!file)return;if(file.size>1048576)throw Error('Selection files are limited to one MiB.');const value=normalizeSelection(JSON.parse(await file.text()));if(ticket!==epoch){message('A superseded selection file was ignored. The current selection is unchanged.');return;}$('snapshot-id').value=value.project_snapshot.id;$('snapshot-sha').value=value.project_snapshot.manifest_sha256;draft={roots:value.roots,decisions:value.decisions};invalidate();summarize();message('Loaded a closed selection document. Review its friendly summary, then explicitly preview.');}catch(failure){error('Selection file refused: '+failure.message);}finally{$('selection-file').value='';}}; - for(const name of ['snapshot-id','snapshot-sha'])$(name).addEventListener('input',invalidate); - for(const name of ['report-reviewed','stop-confirm','extract-confirm'])$(name).addEventListener('change',controls); - $('transfer-refresh').onclick=refresh; - $('transfer-preview-form').onsubmit=event=>{event.preventDefault();return run(async ticket=>{const body=selection(),selectedRevision=revision;if(!await discover(ticket))return;const value=await call('preview',{binding:state.binding,selection:body},ticket);if(value&&selectedRevision===revision){adopt(value,{ownPreview:true});reviewed=true;dirty=false;onlyCessation=false;$('selection-dirty').textContent='This exact selection was previewed. Review the report before proceeding.';$('report-section').value='blockers';await loadPage(ticket,0,'blockers');message(state.phase==='source_live'?'Read-only preview complete. Review the closure before any explicit stop.':'Stopped-source preview complete. Review this exact report before extraction.');$('report-title').focus();controls();}});}; - $('report-load').onclick=()=>run(ticket=>loadPage(ticket));$('report-previous').onclick=()=>run(ticket=>loadPage(ticket,Math.max(0,(page?.offset??0)-40)));$('report-next').onclick=()=>run(ticket=>loadPage(ticket,(page?.offset??0)+(page?.items.length??0))); - $('report-section').addEventListener('change',()=>{page=null;controls();}); - $('transfer-stop-form').onsubmit=event=>{event.preventDefault();return run(async ticket=>{ - if(!reviewed||!onlyCessation||!$('report-reviewed').checked||!$('stop-confirm').checked)throw Error('Review this report and explicitly confirm source stop first.'); - const body={binding:state.binding,report_sha256:state.report.report_sha256,backup_root:$('backup-root').value,backup_directory:$('backup-directory').value.trim(),destination_root:$('destination-root').value,directory:$('destination-directory').value.trim()}; - if(!leaf(body.backup_directory)||!leaf(body.directory)||body.backup_root===body.destination_root&&body.backup_directory===body.directory)throw Error('Choose different new backup and destination directory names without separators.'); - handoff=true;freshControl=false;freshControlAttempt=false;reviewed=false;$('report-reviewed').checked=false;message('Explicit source stop is pending. No other effect is available.'); - try{const value=await call('stop',body,ticket,{mutation:true});if(value){adopt(value);message('Source stop returned. Sign in separately to the control host, then explicitly preview again.');$('control-title').focus();}} - finally{handoff=true;$('control-signin-panel').hidden=false;$('control-signin-status').textContent=`Separate control catalog: ${controlCatalog??'refresh controller status'}. Submit a new login code, then refresh status.`;window.dispatchEvent(new MessageEvent('message',{origin:location.origin,data:{type:'tuldok-session-changed'}}));} - });}; - $('control-signin').onclick=()=>{const button=$('session-open');if(button)button.click();else error('The standard session dialog is unavailable.');}; - const loginForm=$('session-form');if(loginForm)loginForm.addEventListener('submit',()=>{if(handoff||state?.phase!=='source_live')freshControlAttempt=true;},true); - const logout=$('session-logout');if(logout)logout.addEventListener('click',()=>{epoch++;contracts.clear();reviewed=false;onlyCessation=false;freshControl=false;freshControlAttempt=false;$('report-reviewed').checked=false;controls();message('Session sign-out requested. Local selection and retained receipts remain; sign in before continuing.');},true); - async function extract(ticket,body){ - pending=clone(body);$('transfer-pending').hidden=false;$('transfer-pending').textContent=`Retained request ${body.request_key} for ${body.destination_root} / ${body.directory}. No automatic retry will occur.`;controls(); - const value=await call('extract',body,ticket,{mutation:true});if(!value)return; - if(!exact(value,['format','binding','receipt','selection'])||value.format!=='project_transfer_extraction_result_v1'||value.binding!==state.binding||!receiptOK(value.receipt)||!exact(value.selection,['selected','state','error'])||typeof value.selection.selected!=='boolean')throw protocol('Unsupported extraction result. Retain its known receipt and refresh status.'); - keepReceipt(value.receipt);const selected=value.selection.selected,project=value.selection.state?.project; - if(selected&&(!project||project.project_id!==value.receipt.project_id||project.checkpoint!==value.receipt.checkpoint))throw protocol('Publication is recorded, but the selected project identity or checkpoint differs. Inspect the file workspace explicitly.'); - pending=null;reviewed=false;$('report-reviewed').checked=false;$('extract-confirm').checked=false;$('transfer-pending').hidden=true;$('selection-outcome').textContent=selected?'At acknowledgement, the verified destination was selected. The file workspace link opens the current inert metadata view.':'Publication is recorded. Selection was unavailable; inspect the file workspace and Open this directory explicitly.'; - message(value.receipt.replayed?'Exact extraction request reconciled against the current destination.':'Project published. No computation was started.');$('outcome-title').focus(); - } - $('transfer-extract-form').onsubmit=event=>{event.preventDefault();return run(async ticket=>{if(!signedControl()||!reviewed||!state?.report?.can_extract||!$('report-reviewed').checked||!$('extract-confirm').checked)throw Error('Sign in to the control host and review the stopped-source report before extraction.');const body={binding:state.binding,report_sha256:state.report.report_sha256,destination_root:state.destination?.root,directory:state.destination?.directory,name:$('destination-name').value,request_key:$('extraction-key').value.trim()};if(!text(body.name,200)||!text(body.request_key,128)||!/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(body.request_key))throw Error('Enter a project name and exact request key.');await extract(ticket,body);});}; - $('transfer-reconcile').onclick=()=>run(async ticket=>{if(!pending||!signedControl()||state?.report?.report_sha256!==pending.report_sha256)throw Error('Refresh current authorization and this exact report before an explicit reconciliation.');await extract(ticket,{...pending,binding:state.binding});}); - window.addEventListener('tuldok-session',()=>{epoch++;contracts.clear();reviewed=false;onlyCessation=false;$('report-reviewed').checked=false;$('report-items').replaceChildren();if(freshControlAttempt&&window.TuldokAuth.identity?.()?.catalog_id===controlCatalog){freshControl=true;handoff=false;message('Fresh control-host sign-in verified. Explicitly preview the stopped source again.');}controls();if(busy)refreshAfter=true;else void refresh();}); - window.addEventListener('beforeunload',event=>{if(dirty||pending){event.preventDefault();event.returnValue='';}}); - $('extraction-key').value=crypto.randomUUID().replaceAll('-','');summarize();controls();void refresh(); -})(); diff --git a/static/project-workspace.html b/static/project-workspace.html new file mode 100644 index 00000000..f9540153 --- /dev/null +++ b/static/project-workspace.html @@ -0,0 +1,55 @@ + + + + + + +Map · Tuldok + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Skip to content +
                          + t.Tuldok +

                          + +
                          +
                          +
                          +
                          +
                          +
                          + + +
                          + + diff --git a/static/project-workspace/actions.js b/static/project-workspace/actions.js new file mode 100644 index 00000000..72b46349 --- /dev/null +++ b/static/project-workspace/actions.js @@ -0,0 +1,340 @@ +'use strict'; +// The controls that change the project (spec-w3.md section 3.1), and the +// plumbing every such control shares: who may use one (the rules of +// service._allowed, from the signed-in identity), a group of buttons bound to +// one key whose state outlives a re-render (waiting for an answer, the +// two-press "Confirm" state, typed text, the last refusal), and the views +// built of them: the plan view, the conflict panel, the controls of Needs +// you and the strip's Save. A control calls one operation; while it waits it +// is disabled and says so; when the answer comes what it can have changed +// is forgotten and the page reads again (hooks.changed, set by app.js); a +// refusal is shown under the control in the server's words, with its +// problems listed. Nothing is retried on its own. Text goes in through text +// nodes only. +window.LabActions = (() => { + const P = window.TuldokProject, h = window.LabCharts.h; + + // ---- Who may use a control ------------------------------------------------ + const identity = () => (window.TuldokAuth && window.TuldokAuth.identity()) || null; + /** Whether the session satisfies an operation's who rule (inspect, edit, cancel, request, owner). */ + function may(who) { + const id = identity(); + if (!id || !['owner', 'agent'].includes(id.kind) || !Array.isArray(id.scopes)) return false; + const scopes = id.scopes; + if (who === 'inspect' || who === 'edit' || who === 'cancel') return scopes.includes(who); + if (who === 'request') return scopes.includes('inspect') && (id.kind === 'owner' || scopes.includes('execute')); + if (who === 'owner') return id.kind === 'owner' && scopes.includes('approve-execution'); + return false; + } + const isAgent = () => (identity() || {}).kind === 'agent'; + + let hooks = { changed: async () => {}, refused: async () => {}, reread: async () => {} }; + /** app.js: changed() reads everything again after an answer, refused() after a refusal, reread() on request. */ + function connect(given) { hooks = Object.assign({}, hooks, given); } + + // ---- Groups of controls ---------------------------------------------------- + // A group's state, by its key: busy (the item waiting), confirming (the item + // in its two-press state), refusal ({error, item}) and text (typed, by field). + const slots = new Map(); + const slot = key => { if (!slots.has(key)) slots.set(key, { busy: null, confirming: null, refusal: null, text: {} }); return slots.get(key); }; + const groups = key => [...document.querySelectorAll('.act-group')].filter(g => g._key === key); + const btn = (cls, text, key) => { const b = h('button', cls, text); b.type = 'button'; if (key) b.dataset.key = key; return b; }; + + /** + * A group of controls bound to key: an element that draws its items from + * the group's state whenever it changes and on every re-render. An item is + * { id, label, cls, who, op, args(slot) -> arguments (or run(slot), a step + * of the page that throws to refuse; with local, nothing is read again + * after it), busy (the words while + * waiting), confirm (words: the item takes two presses), input ({name, + * label, placeholder, max, always}: a field shown always, or in the + * confirm state), done(result), extra(error) (more under a refusal), + * hidden } or a plain control { id, label, cls, onClick }. Items whose who + * the session does not satisfy are not drawn; a group with none is empty. + * options.refusalIn: the element that shows the refusal instead of the group. + */ + function group(key, items, options = {}) { + const el = h('div', 'act-group' + (options.inline ? ' inline' : '')); + el._key = key; el._items = items; el._options = options; el.dataset.group = key; + fill(el); + return el; + } + const shown = el => el._items.filter(i => !i.hidden && (!i.who || may(i.who))); + function fill(el) { + const key = el._key, s = slot(key), items = shown(el), opts = el._options; + el.textContent = ''; + if (!items.length) { el.hidden = true; return; } + el.hidden = false; + const row = h('div', 'actions'); + const confirming = s.confirming && items.find(i => i.id === s.confirming); + if (confirming) { + const it = confirming; + if (it.input && !it.input.always) el.appendChild(field(key, it.input, s)); + el.appendChild(h('p', 'confirm-words', it.confirm)); + const ok = btn('btn primary act', s.busy ? (it.busy || 'Waiting…') : 'Confirm', key + ':' + it.id + ':confirm'); + ok.disabled = !!s.busy; if (s.busy) ok.setAttribute('aria-busy', 'true'); + ok.addEventListener('click', () => press(key, it, true)); + const back = btn('btn ghost act', 'Back', key + ':' + it.id + ':back'); back.disabled = !!s.busy; + back.addEventListener('click', () => leave(key)); + row.appendChild(ok); row.appendChild(back); + } else { + for (const it of items) { + if (it.input && it.input.always) row.appendChild(field(key, it.input, s)); + const b = btn((it.cls || 'btn sm') + (it.op || it.who ? ' act' : ''), s.busy === it.id ? (it.busy || 'Waiting…') : it.label, key + ':' + it.id); + if (it.title) b.title = it.title; + if (s.busy) { b.disabled = true; if (s.busy === it.id) b.setAttribute('aria-busy', 'true'); } + b.addEventListener('click', () => it.onClick ? it.onClick() : press(key, it, false)); + row.appendChild(b); + } + } + el.appendChild(row); + const host = opts.refusalIn || el; + if (opts.refusalIn) opts.refusalIn.textContent = ''; + if (s.refusal && items.some(i => i.id === s.refusal.item)) { + const it = items.find(i => i.id === s.refusal.item); + host.appendChild(refusal(s.refusal.error, it.extra ? it.extra(s.refusal.error) : null)); + } + } + function field(key, input, s) { + const l = h('label', 'act-field'); + l.appendChild(document.createTextNode(input.label)); + const area = input.lines ? h('textarea') : h('input'); + if (input.lines) area.rows = input.lines; else area.type = 'text'; + if (input.max) area.maxLength = input.max; + if (input.placeholder) area.placeholder = input.placeholder; + area.value = s.text[input.name] || ''; area.dataset.key = key + ':field:' + input.name; area.disabled = !!s.busy; + area.autocomplete = 'off'; area.spellcheck = false; + area.addEventListener('input', () => { s.text[input.name] = area.value; }); + area.addEventListener('keydown', e => { if (e.key === 'Enter' && !input.lines) { e.preventDefault(); const b = area.closest('.act-group').querySelector('button.act'); if (b && !b.disabled) b.click(); } }); + l.appendChild(area); + if (input.max && input.max <= 1000) l.appendChild(h('span', 'hint', 'at most ' + input.max + ' characters')); + return l; + } + function refill(key) { for (const g of groups(key)) fill(g); } + function focusKey(dataKey) { const e = [...document.querySelectorAll('[data-key]')].find(x => x.dataset.key === dataKey && x.offsetParent !== null); if (e) { e.focus(); return true; } return false; } + function leave(key) { const s = slot(key), was = s.confirming; s.confirming = null; refill(key); if (was) focusKey(key + ':' + was); } + /** Escape: leaves every two-press state not waiting for its answer; true when one was left. */ + function escape() { + let left = false; + for (const [key, s] of slots) if (s.confirming && !s.busy) { leave(key); left = true; } + return left; + } + async function press(key, it, confirmed) { + const s = slot(key); + if (s.busy) return; + if (it.confirm && !confirmed) { s.confirming = it.id; s.refusal = null; refill(key); focusKey(key + ':' + (it.input && !it.input.always ? 'field:' + it.input.name : it.id + ':confirm')); return; } + s.busy = it.id; s.refusal = null; refill(key); + let result, error = null; + // run(slot): a step of the page itself (editor.js), which may call operations; it throws to refuse. + try { result = it.run ? await it.run(s) : await P.call(it.op, it.args ? it.args(s) : {}); } catch (e) { error = e; } + s.busy = null; + if (error) { + s.refusal = { error, item: it.id }; refill(key); + focusKey(key + ':' + it.id) || focusKey(key + ':' + it.id + ':confirm'); + if (!it.local) await hooks.refused({ key, item: it, error }); + return; + } + s.confirming = null; s.text = {}; refill(key); + // local: nothing in the project changed, so nothing is read again. + if (it.local) { if (it.done) it.done(result); return; } + P.forgetBut(P.WATCHED); // what it can have changed is read afresh; the watched answers are read again by hooks.changed and compared + if (it.done) it.done(result); + await hooks.changed({ key, item: it, result, focus: key + ':' + it.id }); + } + + /** A refusal in the server's words: its sentence, its problems, what it names; extra under them. */ + function refusal(error, extra) { + const box = h('div', 'failure refusal'); box.setAttribute('role', 'alert'); + box.appendChild(h('p', null, error.message || String(error))); + const problems = error.problems || []; + if (problems.length) { + const ul = h('ul'); + for (const p of problems) { + const where = [p.path, p.pointer].filter(x => x && x !== '/').join(' '); + ul.appendChild(h('li', null, (p.severity === 'warning' ? 'Warning: ' : '') + (where ? where + ': ' : '') + (p.message || ''))); + } + box.appendChild(ul); + } + if (error.references && error.references.length) box.appendChild(h('p', 'small', 'Named: ' + error.references.join(', ') + '.')); + if (error.files && error.files.length) box.appendChild(h('p', 'small', 'Files: ' + error.files.join(', ') + '.')); + if (extra) box.appendChild(extra); + return box; + } + + // ---- The strip's Save and the controls of Needs you --------------------------- + const plural = (n, one, many) => n + ' ' + (n === 1 ? one : many); + /** Save, with a refusal changed-on-disk offering to read the project again. */ + const saveItem = () => ({ + id: 'save', label: 'Save', cls: 'btn sm', who: 'edit', op: 'save', busy: 'Saving…', + extra: error => { if (error.code !== 'changed-on-disk') return null; const b = btn('btn sm', 'Read the project again', 'reread'); b.addEventListener('click', () => hooks.reread()); const d = h('div', 'actions'); d.appendChild(b); return d; } + }); + /** The strip's part: how many definitions are unsaved or marked removed and how many annotations are unsaved ("3 definitions, 41 annotations unsaved"), and Save, which writes both. */ + function stripSave(status, note) { + const n = status.unsaved.length + status.removed.length, a = P.annotationsUnsaved(status); + if (!n && !a) { if (note) note.textContent = ''; return null; } + const box = h('span', 'unsaved'); + box.appendChild(h('span', 'count', [n ? plural(n, 'definition', 'definitions') : null, a ? plural(a, 'annotation', 'annotations') : null].filter(Boolean).join(', ') + ' unsaved')); + box.appendChild(group('save', [saveItem()], { inline: true, refusalIn: note })); + return box; + } + /** The controls under one item of Needs you (adapter.needs): go(screen, id) opens a place. */ + function needControls(n, go) { + const items = []; + if (n.request) { + const id = n.request.id; + items.push({ id: 'approve', label: 'Approve', cls: 'btn sm primary', who: 'owner', op: 'approve_build', args: () => ({ request: id }), busy: 'Approving…' }); + items.push(declineItem(id)); + items.push({ id: 'open', label: 'Open on Train', cls: 'btn sm ghost', onClick: () => go('train', 'request:' + id) }); + } else if (n.save) { + items.push(saveItem()); + for (const ref of n.references.slice(0, 6)) { + const where = ref.startsWith('recipes/') ? 'map' : ref.startsWith('datasets/') ? 'data' : ref.startsWith('annotations/') ? 'annotate' : null; + if (where) items.push({ id: 'open-' + ref, label: ref, cls: 'btn sm ghost', onClick: () => { if (where !== 'annotate') return go(where, ref); if (window.LabAnnotate) window.LabAnnotate.choose(ref); go(where); } }); + } + } else if (n.environment) { + const e = n.environment; + if (e.interpreter) items.push(checkItem(e.reference)); + items.push({ id: 'open', label: 'On Train', cls: 'btn sm ghost', onClick: () => go('train', 'env:' + e.reference) }); + } else if (n.run) { + items.push({ id: 'run', label: 'The run', cls: 'btn sm ghost', onClick: () => go('train', 'run:' + n.run.id) }); + items.push({ id: 'plan', label: 'Plan build', cls: 'btn sm', who: 'request', onClick: () => go('train', 'plan:' + n.reference) }); + } + if (!items.length) return null; + return group('need:' + (n.save ? 'save' : n.key), items); + } + const declineItem = id => ({ id: 'decline', label: 'Decline', cls: 'btn sm', who: 'owner', op: 'decline_build', busy: 'Declining…', + confirm: 'Decline this request; the reason is kept with it and shown to whoever asked.', + input: { name: 'reason', label: 'Reason (optional)', max: 400, lines: 2 }, + args: s => ({ request: id, reason: (s.text.reason || '').trim() }) }); + const checkItem = reference => ({ id: 'check', label: 'Check again', cls: 'btn sm', who: 'edit', op: 'check_environment', args: () => ({ reference }), busy: 'Checking…' }); + + // ---- The plan view ------------------------------------------------------------ + const ACTION = { + build: ['pending', 'Build'], built: ['good', 'Built already'], ready: ['good', 'Ready'], + 'set-up': ['warn', 'Set up first'], blocked: ['serious', 'Blocked'] + }; + const gb = x => (x >= 100 ? Math.round(x) : x >= 1 ? x.toFixed(1) : x >= 0.01 ? x.toFixed(2) : x > 0 ? x.toExponential(1) : '0') + ' GB'; + const hours = x => x >= 1 ? x.toFixed(1) + ' h' : x * 60 >= 1 ? Math.round(x * 60) + ' min' : x > 0 ? Math.max(1, Math.round(x * 3600)) + ' s' : '0 h'; + /** What one step would do, in sentences. */ + function stepWords(st) { + const out = []; + if (st.action === 'build') { + out.push({ 'not-built': 'Not built yet: it will be built.', 'out-of-date': 'Out of date: it will be built again.', failed: 'Its last run failed: it will be built again.', running: 'Running now: a build will not start it a second time.' }[st.state] || 'It will be built.'); + for (const r of st.reasons || []) out.push(r); + } else if (st.action === 'built') out.push('Built already' + (st.adopted ? ' (adopted)' : '') + (st.scratch ? ', by a run from the code as it was on disk' : '') + ': nothing to do.'); + else if (st.action === 'ready') out.push('Ready on this machine.'); + else if (st.action === 'set-up') out.push(st.state === 'failing' ? 'Its last check failed: check it or connect another interpreter on Train before building.' : 'Not set up on this machine: connect an interpreter on Train, or set it up from the command line, before building.'); + else if (st.action === 'blocked') { + if (!st.blocked_by.length) out.push(st.kind === 'benchmark' ? 'Blocked: none of its subjects would run.' : 'Blocked.'); + for (const c of st.blocked_by) out.push('Blocked: ' + (c.reference === st.reference ? 'it is' : c.reference + ' is') + ' labelled rebuild ' + c.rebuild + (c.reason ? ' (' + c.reason + ')' : '') + '.'); + if (st.blocked_by_more) out.push('And ' + plural(st.blocked_by_more, 'more cause', 'more causes') + '.'); + } + const uses = Object.entries(st.inputs || {}).map(([name, i]) => name + ' from ' + i.reference + (i.output ? ' (' + i.output + ')' : '')); + if (uses.length) out.push('Uses ' + uses.join(', ') + '.'); + if (st.environment && st.kind !== 'environment') out.push('Runs in ' + st.environment + '.'); + if (st.subjects) { + if (st.subjects.run.length) out.push('Runs the subjects ' + st.subjects.run.join(', ') + '.'); + for (const k of st.subjects.skipped) out.push('Skips ' + k.name + ': ' + k.reason + '.'); + } + if (st.action === 'build') { + const r = st.resources || {}, said = []; + if (r.gpu === true) said.push('uses the GPU'); + if (typeof r.disk_gb === 'number') said.push(gb(r.disk_gb) + ' of disk'); if (typeof r.wall_hours === 'number') said.push(hours(r.wall_hours)); + out.push(said.length ? 'Declares: ' + said.join(', ') + '.' : 'Declares no GPU, disk or time.'); + const e = st.estimate; + if (e) { const est = []; if (e.disk_bytes != null) est.push(gb(e.disk_bytes / 1e9) + ' of disk'); if (e.wall_seconds != null) est.push(hours(e.wall_seconds / 3600)); if (est.length) out.push('Estimated: ' + est.join(', ') + '.'); } + } + return out; + } + /** The totals of a plan: GPU steps, disk and time as stated, estimated or unknown. */ + function totals(plan) { + const n = plan.counts.build, out = []; + if (!n) return ['Nothing would be built, so there are no totals.']; + out.push('To build: ' + plural(n, 'step', 'steps') + ', ' + (plan.gpu_steps ? plan.gpu_steps + ' of them on the GPU' : 'none on the GPU') + '.'); + for (const [name, sum, unknown, est, estSteps, fmt] of [['Disk', plan.disk_gb, plan.disk_unknown, plan.estimated_disk_gb, plan.estimated_disk_steps, gb], ['Time', plan.wall_hours, plan.wall_unknown, plan.estimated_wall_hours, plan.estimated_wall_steps, hours]]) { + let t = name + ': ' + (unknown === n ? 'unknown, no step states it' : fmt(sum) + ' stated' + (unknown ? '; ' + unknown + ' of ' + n + ' state none, so it is more' : '')); + t += estSteps ? '; estimated ' + fmt(est) + ' for ' + plural(estSteps, 'step', 'steps') + ' that were measured.' : '; no estimate was measured.'; + out.push(t); + } + out.push('The figures are as the definitions state them; nothing was run to make this plan.'); + return out; + } + /** + * The plan view in the drawer, for sel 'plan:,' or + * 'plan:*' (everything buildable): read with `plan`, shown in sentences, + * with the scratch box and Request build. back(): leaves the view. Read + * again for the plan on screen (aside._again), it stays until the new one + * is built off the page and put in its place in one step. + */ + function planView(aside, sel, go, back) { + if (aside._for === sel && !aside._again) return; + const S = window.LabScreens, same = aside._for === sel && !!aside.firstChild, mine = S.parts.turn(aside), live = () => aside._turn === mine && aside._for === sel; + aside._for = sel; + const every = sel === 'plan:*', targets = every ? [] : sel.slice(5).split(',').filter(Boolean); + if (!same) { aside.textContent = ''; aside.appendChild(h('p', 'note', 'Reading…')); } + P.read('plan', { targets, every }).then(plan => { + if (!live()) return; + const ins = h('div', 'inspector plan-view'); + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'plan · nothing runs until it is requested')); + const title = h('h2', null, every ? 'A build of everything buildable' : 'A build of ' + plan.targets.join(', ')); title.tabIndex = -1; hd.appendChild(title); + const c = plan.counts, meta = h('div', 'ins-meta'); + meta.appendChild(window.LabScreens.stateEl(plan.buildable ? (c.build ? 'good' : 'neutral') : 'serious', plan.buildable ? (c.build ? 'can be built' : 'nothing to build') : 'cannot be built')); + hd.appendChild(meta); ins.appendChild(hd); + if (back) { const a = h('div', 'actions'); const b = btn('btn sm ghost', '← Back', 'plan-back'); b.addEventListener('click', back); a.appendChild(b); ins.appendChild(a); } + const parts = [plural(c.build, 'step', 'steps') + ' to build', c['set-up'] + ' to set up', c.blocked + ' blocked']; + if (c.ready) parts.push(plural(c.ready, 'environment', 'environments') + ' ready'); + if (c.built) parts.push(c.built + ' built already'); + ins.appendChild(h('p', 'plan-lead', parts.join(', ') + ': ' + (plan.buildable ? (every ? 'everything buildable' : plan.targets.join(', ')) + (c.build ? ' can be built.' : ' is built already.') : (plan.blocked_targets.join(', ') + (plan.blocked_targets.length === 1 ? ' cannot' : ' cannot') + ' be built.')))); + if (plan.blocked_targets.length) ins.appendChild(h('p', 'failure', 'Blocked ' + (plan.blocked_targets.length === 1 ? 'target' : 'targets') + ': ' + plan.blocked_targets.join(', ') + '. A build is refused while any target is blocked; the steps below say why.')); + const d = h('div'); d.appendChild(h('h3', null, 'Steps, in the order they would run')); + if (!plan.steps.length) d.appendChild(h('p', 'note', 'No step: nothing here is built by Tuldok.')); + const ol = h('ol', 'steps'); + for (const st of plan.steps) { + const [tone, word] = ACTION[st.action] || ['neutral', st.action]; + const li = h('li', st.action === 'build' ? 'todo' : st.action === 'blocked' ? 'blocked' : 'done'); li.dataset.step = st.reference; + li.appendChild(window.LabScreens.stateEl(tone, '')); + const t = h('span'); t.appendChild(h('span', 'title', st.number + '. ' + word + ' · ' + (st.title || st.reference))); + t.appendChild(h('span', 'sub mono', st.reference + (st.action === 'build' && st.resources && st.resources.gpu === true ? ' · GPU' : ''))); + for (const line of stepWords(st)) t.appendChild(h('span', 'sub', line)); + li.appendChild(t); ol.appendChild(li); + } + d.appendChild(ol); ins.appendChild(d); + const tt = h('div'); tt.appendChild(h('h3', null, 'What it takes')); for (const line of totals(plan)) tt.appendChild(h('p', 'small', line)); ins.appendChild(tt); + if (plan.notes.length) { const nd = h('div'); nd.appendChild(h('h3', null, 'Notes')); for (const n of plan.notes) nd.appendChild(h('p', 'small', n)); ins.appendChild(nd); } + if (may('request')) { + const key = 'request:' + sel, s = slot(key); + const f = h('div', 'form'); + const lab = h('label', 'check'); const box = h('input'); box.type = 'checkbox'; box.checked = !!s.text.scratch; box.dataset.key = key + ':scratch'; + box.addEventListener('change', () => { s.text.scratch = box.checked; }); + lab.appendChild(box); lab.appendChild(document.createTextNode(' Run from the code as it is on disk; the product is marked scratch')); + f.appendChild(lab); ins.appendChild(f); + const label = isAgent() ? 'Ask the owner for this build' : 'Request build'; + ins.appendChild(group(key, [{ id: 'request', label, cls: 'btn primary', who: 'request', op: 'request_build', busy: isAgent() ? 'Asking…' : 'Requesting…', + args: st => ({ targets, every, scratch: !!st.text.scratch }), done: r => go('train', 'request:' + r.id) }])); + ins.appendChild(h('p', 'note', isAgent() ? 'The owner approves or declines what you ask for; nothing runs before.' : 'Your request is queued at once and runs when the build worker takes it.')); + } + S.parts.putDrawer(aside, ins); + }, error => { if (live()) S.parts.putDrawer(aside, refusal(error)); }); + } + + // ---- The conflict panel -------------------------------------------------------- + /** The main area when a read was refused conflict: the server's sentence, the definitions, and the two ways out. */ + function conflictPanel(main, error) { + const sec = h('section', 'conflict'); + const t = h('h2', null, 'Files changed on disk where Tuldok holds unsaved edits'); t.tabIndex = -1; sec.appendChild(t); + sec.appendChild(h('p', null, error.message)); + const refs = error.references || []; + if (refs.length) { const ul = h('ul', 'conflict-refs'); for (const r of refs) ul.appendChild(h('li', 'mono', r)); sec.appendChild(ul); } + sec.appendChild(h('p', 'note', 'Keep my edits: the unsaved edits stay and replace those files when you save. Take the files: the unsaved edits of ' + (refs.length === 1 ? 'this definition are' : 'these definitions are') + ' dropped and the files are read as they are now.')); + const g = group('conflict', [ + { id: 'keep', label: 'Keep my edits', cls: 'btn primary', who: 'edit', op: 'resolve_conflict', args: () => ({ choice: 'keep' }), busy: 'Keeping…' }, + { id: 'discard', label: 'Take the files', cls: 'btn', who: 'edit', op: 'resolve_conflict', args: () => ({ choice: 'discard' }), busy: 'Taking the files…', + confirm: 'Drop the unsaved edits of ' + (refs.length ? refs.join(', ') : 'these definitions') + ' and take the files on disk.' } + ]); + sec.appendChild(g); + if (g.hidden) sec.appendChild(h('p', 'note', 'Someone who may edit this project has to choose; until then the project cannot be read.')); + main.appendChild(sec); + } + + return { may, isAgent, connect, group, slot, escape, refusal, stripSave, needControls, planView, conflictPanel, declineItem, checkItem, focusKey }; +})(); diff --git a/static/project-workspace/adapter.js b/static/project-workspace/adapter.js new file mode 100644 index 00000000..c101f813 --- /dev/null +++ b/static/project-workspace/adapter.js @@ -0,0 +1,304 @@ +'use strict'; +// The page's one way to the project. It calls the project's operations +// (POST /api/project/) with the contract the server publishes at +// /api/agent/capabilities (never pinned here), keeps each answer until it is +// read again, and shapes what the screens read: the map's sections, models +// and lines, the sentence of where the project stands, and what needs the +// person. It reads on its own; an operation that changes the project is +// called only by a control the person pressed (actions.js). Reading again +// (again()) compares each answer's text with the kept one, so the page can +// leave alone what did not change; what is not watched is forgotten when +// something changed (forgetBut()). A refusal is thrown as an Error carrying +// the server's code, problems, references and files. +window.TuldokProject = (() => { + // Kinds of change, by the project format's names, with the brief's labels. + const KINDS = [ + { id: 'training-policy', label: 'Training policy' }, + { id: 'architecture', label: 'Architecture' }, + { id: 'refinement', label: 'Refinement' }, + { id: 'component', label: 'Component' }, + { id: 'sweep-trial', label: 'Sweep trial' } // a trial a sweep made (spec-t7) + ]; + const UNSET = 'unset'; // the class and filter key of a model without a kind of change + const kindLabel = id => (KINDS.find(k => k.id === id) || { label: 'Kind not set' }).label; + // A model's state, as a word and the tone of its mark. + const STATES = { + built: ['built', 'good'], running: ['running', 'pending'], 'out-of-date': ['out of date', 'warn'], + failed: ['failed', 'serious'], 'not-built': ['not built', 'neutral'], 'needs-work': ['needs work', 'warn'], + 'cannot-rebuild': ['cannot be rebuilt', 'neutral'], none: ['nothing to build', 'neutral'], + ready: ['ready', 'good'], failing: ['failing', 'serious'], 'not-set-up': ['not set up', 'warn'] + }; + const stateWord = s => (STATES[s] || [s || 'unknown', 'neutral']); + const ENDED_BADLY = { failed: 'failed', 'timed-out': 'ran out of time', 'over-disk': 'went over its disk budget', unknown: 'ended in an unknown state' }; + + // ---- Calls -------------------------------------------------------------- + let contracts = null; + async function json(response) { + let value; + try { value = await response.json(); } catch (e) { throw Error('The server answered ' + response.status + ' without JSON.'); } + if (!response.ok) throw Object.assign(Error(value.error || 'The request was refused.'), { code: value.code, status: response.status, + problems: Array.isArray(value.problems) ? value.problems : [], references: Array.isArray(value.references) ? value.references : [], files: Array.isArray(value.files) ? value.files : [] }); + return value; + } + async function selection(name) { + if (!contracts) { + const manifest = await json(await window.TuldokAuth.fetch('/api/agent/capabilities', { cache: 'no-store' })); + contracts = new Map(manifest.operations.map(operation => { + const newest = operation.contracts.reduce((a, b) => (b.version > a.version ? b : a)); + return [operation.id, operation.id + '/' + newest.version + '/' + newest.sha256]; + })); + } + if (!contracts.has(name)) throw Error('This Tuldok does not offer ' + name + '.'); + return contracts.get(name); + } + async function call(name, body = {}) { + try { + const response = await window.TuldokAuth.fetch('/api/project/' + name, { + method: 'POST', cache: 'no-store', + headers: { 'Content-Type': 'application/json', 'Tuldok-Operation-Contract': await selection('project.' + name) }, + body: JSON.stringify(body) + }); + return await json(response); + } catch (error) { contracts = null; throw error; } + } + + // Answers are kept by operation and arguments until forgotten, each with its text, so a reading + // again can tell an answer that changed from one that did not. + const kept = new Map(); // key -> { answer: promise, text: the answer as JSON once it came } + const keyOf = (name, args) => name + ' ' + JSON.stringify(args); + function read(name, args = {}) { + const key = keyOf(name, args); + if (!kept.has(key)) { + const entry = { text: null }; + entry.answer = call(name, args).then(value => { entry.text = JSON.stringify(value); entry.value = value; return value; }, error => { if (kept.get(key) === entry) kept.delete(key); throw error; }); + kept.set(key, entry); + } + return kept.get(key).answer; + } + /** + * Asks again for the kept answer of each named operation (no arguments), + * asking for the first time when none is kept, and resolves to the names + * whose answer differs from the kept one; for those the kept answer is + * replaced, and an equal answer leaves the kept object as it is. When any + * of them is refused or does not arrive, the error is thrown and nothing + * kept is replaced: a reading that failed changes nothing. + */ + async function again(names) { + const got = await Promise.all(names.map(async name => { + const key = keyOf(name, {}), had = kept.get(key); + const before = had ? await had.answer.then(() => had.text, () => null) : null; + const value = await call(name, {}); + return { name, key, had, before, value, text: JSON.stringify(value) }; + })); + const changed = []; + for (const g of got) { + if (g.before !== null && g.text === g.before && kept.get(g.key) === g.had) continue; + kept.set(g.key, { answer: Promise.resolve(g.value), text: g.text }); changed.push(g.name); + } + return changed; + } + /** The kept answer of a read when it has arrived, else undefined; asks nothing (the Benchmarks tab's intervals, spec-t3 section 5.2). */ + function peek(name, args = {}) { const entry = kept.get(keyOf(name, args)); return entry && entry.text !== null ? (entry.value !== undefined ? entry.value : JSON.parse(entry.text)) : undefined; } + function forget(names) { for (const key of [...kept.keys()]) if (names.includes(key.split(' ')[0])) kept.delete(key); } + /** Forgets every kept answer whose operation is not named. */ + function forgetBut(names) { for (const key of [...kept.keys()]) if (!names.includes(key.split(' ')[0])) kept.delete(key); } + function forgetAll() { kept.clear(); } + // What the page watches: asked again on every reading ('map' on some, app.js), kept across a change. + const WATCHED = ['summary', 'status', 'build_requests', 'environments', 'runs', 'map']; + + // ---- Looking inside a product (spec-d1.md, inside.js) ----------------------------- + // The five reads of a finished run's product. Their answers are kept like any other until a reading + // finds something changed (they are not watched, so forgetBut() drops them); the slice the viewer draws + // as a slider moves is asked for each time and not kept, so the kept answers do not grow with every move. + const defined = o => Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined && v !== null)); + const product = { + records: (run, { offset, limit, kind, split } = {}) => read('records', defined({ run, offset, limit, kind, split })), + record: (run, id) => read('record', { run, id }), + /** Any file of the product, or a member of an archive in it, read as a record by its address. */ + recordAt: (run, path) => read('record', { run, path }), + inside: (run, { path, offset, limit } = {}) => read('inside', defined({ run, path: path || null, offset, limit })), + /** A slice; kept unless fresh (the viewer's slider). */ + slice: (run, { path, array, axes, index, most }, fresh = false) => (fresh ? call : read)('array_slice', defined({ run, path, array, axes, index, most })), + summary: (run, { path, array, bins }) => read('array_summary', defined({ run, path, array, bins })), + // A model's weights (spec-d2.md, weights.js): its weights files, a comparison of two, and tiles of a tensor + // of three or more axes (asked for each time the person moves through them unless kept). + weights: run => read('weights', { run }), + compare: (run, path, otherRun, otherPath) => read('weights_compare', { run, path, other_run: otherRun, other_path: otherPath }), + tiles: (run, { path, array, across, axes, index, first, most }, fresh = false) => (fresh ? call : read)('array_tiles', defined({ run, path, array, across, axes, index, first, most })), + // The fields a run kept (spec-d6.md, fields.js): its cases, and one slice of a case's native field, prediction + // and signed error with the error's figures (asked for each time the person moves through it unless kept). + fields: run => read('fields', { run }), + fieldCompare: (run, { entry, case: one, axis, index, mean_free }, fresh = false) => (fresh ? call : read)('field_compare', defined({ run, entry, case: one, axis, index, mean_free })), + // A model's structure (spec-t2.md, structure.js): its graph folded by the person's choices, a number per module + // and operation, and the matching of two graphs' modules. Kept like the rest until a reading finds a change. + graph: (run, { depth, open, close } = {}) => read('model_graph', defined({ run, depth, open: open && open.length ? open : null, close: close && close.length ? close : null })), + graphOverlay: (run, { kind, against, other_run, path, other_path }) => read('model_graph_overlay', defined({ run, kind, against, other_run, path, other_path })), + graphCompare: (run, otherRun) => read('model_graph_compare', { run, other_run: otherRun }), + /** The address of an image's or a sound's bytes (GET /api/project/file), as the record operation gives it. */ + fileUrl: (run, path) => '/api/project/file?run=' + encodeURIComponent(run) + '&path=' + encodeURIComponent(path) + }; + + // ---- Annotating (spec-d4.md, annotate.js) -------------------------------------------------- + // The three reads of annotations, kept like any other answer until a reading finds something changed (they are + // not watched, so forgetBut() drops them). The two changes (set_annotation, revert_annotations) are called by + // controls (actions.js) by their names. + const annotations = { + tasks: () => read('annotation_tasks'), + /** A page of a task's records with their annotations; state one of the counts' names, or none for all. */ + records: (task, { state, kind, offset, limit } = {}) => read('annotated_records', defined({ task, state, kind, offset, limit })), + one: (task, record) => read('annotation', { task, record }) + }; + /** How many records have an unsaved annotation, in all, from a status answer. */ + const annotationsUnsaved = status => ((status && status.annotations_unsaved) || []).reduce((n, t) => n + t.records, 0); + + // ---- The map ------------------------------------------------------------ + /** + * What the Map tab and the drawer read, from the answer of `map`: + * sections (one per group that has models, the group tree read depth + * first in authored order, then one section "Not in a group" when a model + * names no group), models in section order and, within a section, in the + * map's order (training date, then those without one) except that one + * without a date naming a parent comes right after it (placed()), inputs + * as lines, comparisons and the track with its scale. The page's drafts + * (editor.js) are models too, marked draft. + */ + function shapeMap(raw, drafts = []) { + const groups = new Map(raw.groups.map(g => [g.id, g])); + const children = new Map(), roots = []; + for (const g of raw.groups) { + if (g.parent && groups.has(g.parent)) { if (!children.has(g.parent)) children.set(g.parent, []); children.get(g.parent).push(g); } + else roots.push(g); + } + const order = [], seen = new Set(); + const visit = (g, up) => { if (seen.has(g.id)) return; seen.add(g.id); order.push({ group: g, up }); for (const c of children.get(g.id) || []) visit(c, up.concat([g.title])); }; + roots.forEach(g => visit(g, [])); + const kinds = new Set(KINDS.map(k => k.id)); + const models = raw.models.map(m => ({ + id: m.reference, title: m.title, short: m.short, description: m.description || '', + group: groups.has(m.group) ? m.group : null, kind: kinds.has(m.change) ? m.change : UNSET, + parent: m.derived_from || null, trainedAt: m.trained_at, state: m.state, adopted: m.adopted, card: m.card, + value: m.value, count: m.count, benchmarks: m.benchmarks, unsaved: !!m.unsaved, + // Never built: drawn hollow and dashed (spec-w3 section 3.2). + planned: m.state === 'not-built' && !m.adopted && !m.trained_at + })); + // A draft (editor.js) is a model of the page only, after its parent's models. + for (const d of drafts) models.push({ + id: d.reference, title: d.title, short: d.short || null, description: d.description || '', + group: groups.has(d.group) ? d.group : null, kind: kinds.has(d.change) ? d.change : UNSET, + parent: d.parent, trainedAt: null, state: 'draft', adopted: false, card: null, + value: null, count: null, benchmarks: 0, unsaved: false, planned: true, draft: true + }); + const byId = new Map(models.map(m => [m.id, m])); + for (const m of models) if (m.parent && !byId.has(m.parent)) m.parent = null; + const sections = []; + for (const { group, up } of order) { + const list = placed(models.filter(m => m.group === group.id)); + if (list.length) sections.push({ id: group.id, title: group.title, up, description: group.description, models: list }); + } + const loose = placed(models.filter(m => m.group === null)); + if (loose.length) sections.push({ id: null, title: 'Not in a group', up: [], loose: true, models: loose }); + for (const s of sections) for (const m of s.models) m.groupTitle = s.title; + const derived = new Set(raw.edges.filter(e => e.kind === 'derived').map(e => e.from + ' ' + e.to)); + const inputs = raw.edges.filter(e => e.kind === 'input' && !derived.has(e.from + ' ' + e.to) && byId.has(e.from) && byId.has(e.to)) + .map(e => ({ from: e.to, to: e.from })); // drawn from the input to the model that uses it + const comparisons = raw.comparisons.filter(c => byId.has(c.from) && byId.has(c.to)); + const track = raw.track; + const trackScale = track ? window.LabCharts.scale(models.map(m => m.value), track.metric.better) : null; + return { sections, models: sections.flatMap(s => s.models), byId, inputs, comparisons, track, trackScale, groups: raw.groups }; + } + /** + * One section's models in the map's order, except that a model without a + * date naming a parent in the same section comes right after that parent, + * after the parent's earlier such children (spec-w3 section 3.2). Linear. + */ + function placed(list) { + const here = new Set(list.map(m => m.id)), after = new Map(), moved = new Set(); + for (const m of list) if (!m.trainedAt && m.parent && m.parent !== m.id && here.has(m.parent)) { + if (!after.has(m.parent)) after.set(m.parent, []); + after.get(m.parent).push(m); moved.add(m.id); + } + if (!moved.size) return list; + const out = [], seen = new Set(); + const emit = m => { if (seen.has(m.id)) return; seen.add(m.id); out.push(m); for (const c of after.get(m.id) || []) emit(c); }; + for (const m of list) if (!moved.has(m.id)) emit(m); + for (const m of list) emit(m); // a loop of parents without dates (never stored): kept, in the map's order + return out; + } + // The drafts of the page (editor.js): drawn on the map, never sent anywhere. A draft + // whose parent is gone, or whose reference the project now has, is dropped + // (onDrop tells editor.js); a session that may not edit draws none. + let drafts = [], draftsVersion = 0, onDrop = () => {}; + function setDrafts(list, dropped) { drafts = list.slice(); draftsVersion++; if (dropped) onDrop = dropped; } + let mapRaw = null, mapShaped = null, mapKey = ''; + async function map() { + const raw = await read('map'); + const editing = !!(window.LabActions && window.LabActions.may('edit')), key = draftsVersion + ' ' + editing; + if (raw !== mapRaw || key !== mapKey) { + const refs = new Set(raw.models.map(m => m.reference)); + const gone = drafts.filter(d => !refs.has(d.parent) || refs.has(d.reference)); + const shown = editing ? drafts.filter(d => !gone.includes(d)) : []; + mapRaw = raw; mapKey = key; mapShaped = shapeMap(raw, shown); + if (gone.length) onDrop(gone); + } + return mapShaped; + } + const trackName = track => track ? track.metric.title + ' · ' + track.statistic : ''; + const better = track => track && track.metric.better === 'higher' ? 'higher is better' : 'lower is better'; + const units = track => track && track.metric.units && track.metric.units !== '1' ? ' ' + track.metric.units : ''; + + // ---- Where the project stands ------------------------------------------- + const plural = (n, one, many) => n + ' ' + (n === 1 ? one : many); + function titleOf(shaped, reference) { const m = shaped && shaped.byId.get(reference); return m ? (m.short ? m.short + ' · ' : '') + m.title : reference; } + /** + * {tone, word, text}: the one sentence of the status strip. Reads the map's + * model states and, once read, the runs and build requests. + */ + function standing(shaped, runs, requests) { + const models = shaped.models.filter(m => !m.draft), count = s => models.filter(m => s.includes(m.state)).length; + const built = count(['built']), toBuild = count(['not-built', 'out-of-date', 'failed']), cannot = count(['cannot-rebuild']), work = count(['needs-work']); + const parts = [(built ? built : 'none') + ' built', toBuild + ' to build', cannot + ' cannot be rebuilt']; + if (work) parts.push(work + (work === 1 ? ' needs' : ' need') + ' work'); + let text = plural(models.length, 'model', 'models') + ': ' + parts.join(', ') + '.'; + const running = (runs || []).filter(r => r.state === 'running'), queued = (requests || []).filter(r => r.state === 'queued' || r.state === 'running'); + let tone = 'good', word = 'Idle'; + if (running.length) { tone = 'pending'; word = 'Building'; text += ' Running ' + running.map(r => titleOf(shaped, r.reference) + (r.subject ? ' (' + r.subject + ')' : '')).join(', ') + '.'; } + else if (queued.some(r => r.state === 'running')) { tone = 'pending'; word = 'Building'; text += ' A build is between two of its steps.'; } // a request runs, none of its runs does at this moment + else if (queued.length) { tone = 'pending'; word = 'Queued'; text += ' ' + plural(queued.length, 'build request is', 'build requests are') + ' queued.'; } + else if (runs && requests) text += ' Nothing is running.'; + return { tone, word, text }; + } + + const day = iso => iso ? new Date(iso).toLocaleDateString('en-GB', { day: 'numeric', month: 'short', year: 'numeric' }) : ''; + const listed = (items, most = 3) => items.slice(0, most).join(', ') + (items.length > most ? ' and ' + (items.length - most) + ' more' : ''); + /** + * What needs the person, each a sentence with a kind ('approval', 'changed' + * or 'missing', which sets its mark): build requests awaiting approval, + * unsaved edits and files changed on disk, environments not ready, the + * definitions whose last run ended badly, a build worker stopped or elsewhere. + */ + function needs({ summary, status, requests, environments, runs, shaped }) { + const out = []; + for (const r of requests) if (r.state === 'awaiting-approval') { + const who = r.requested_by && (r.requested_by.label || r.requested_by.kind) || 'someone'; + out.push({ kind: 'approval', key: 'request:' + r.id, request: r, text: 'A build of ' + (r.every ? 'everything buildable' : listed(r.targets || [])) + ', asked by ' + who + ' on ' + day(r.requested_at) + ', awaits your approval.' }); + } + if (status.unsaved.length) out.push({ kind: 'changed', key: 'unsaved', save: true, references: status.unsaved.map(String), text: plural(status.unsaved.length, 'definition has', 'definitions have') + ' unsaved edits: ' + listed(status.unsaved.map(String)) + '.' }); + const typed = annotationsUnsaved(status); + if (typed) out.push({ kind: 'changed', key: 'annotations-unsaved', save: true, references: status.annotations_unsaved.map(t => t.task), text: plural(typed, 'annotation is', 'annotations are') + ' unsaved: ' + listed(status.annotations_unsaved.map(t => t.task + ' (' + t.records + ')')) + '.' }); + if (status.removed.length) out.push({ kind: 'changed', key: 'removed', save: true, references: status.removed.map(String), text: plural(status.removed.length, 'definition is', 'definitions are') + ' marked removed and not yet saved: ' + listed(status.removed.map(String)) + '.' }); + const disk = [].concat(status.added, status.changed, status.missing, status.annotations_changed || []).map(String); + if (disk.length) out.push({ kind: 'changed', text: plural(disk.length, 'file changed', 'files changed') + ' on disk since the project was last read: ' + listed(disk) + '.' }); + if (status.irregular.length) out.push({ kind: 'missing', text: plural(status.irregular.length, 'file is', 'files are') + ' not regular files and are not read: ' + listed(status.irregular.map(String)) + '.' }); + const results = [].concat(status.results_unsaved, status.results_changed).map(String); + if (results.length) out.push({ kind: 'changed', text: 'Results differ from their files: ' + listed(results) + '.' }); + for (const e of environments) if (e.state !== 'ready') out.push({ kind: e.state === 'failing' ? 'missing' : 'changed', key: 'env:' + e.reference, environment: e, text: 'The environment ' + e.title + (e.state === 'failing' ? ' failed its last check.' : ' is not set up on this machine.') }); + const last = new Map(); + for (const r of runs) if (!last.has(r.reference)) last.set(r.reference, r); // runs come newest first + for (const [reference, r] of last) if (ENDED_BADLY[r.state]) out.push({ kind: 'missing', key: 'run:' + r.id, run: r, reference, text: 'The last run of ' + titleOf(shaped, reference) + ' ' + ENDED_BADLY[r.state] + (r.finished_at ? ' on ' + day(r.finished_at) : '') + '.' }); + if (summary.worker === 'stopped') out.push({ kind: 'changed', text: 'The build worker is stopped: queued builds will not start.' }); + if (summary.worker === 'elsewhere') out.push({ kind: 'changed', text: 'Another Tuldok process runs the builds of this project.' }); + return out; + } + + return { KINDS, UNSET, WATCHED, kindLabel, stateWord, call, read, peek, again, forget, forgetBut, forgetAll, map, setDrafts, trackName, better, units, standing, needs, titleOf, day, product, annotations, annotationsUnsaved }; +})(); diff --git a/static/project-workspace/annotate.js b/static/project-workspace/annotate.js new file mode 100644 index 00000000..4015349d --- /dev/null +++ b/static/project-workspace/annotate.js @@ -0,0 +1,984 @@ +'use strict'; +// The Annotate tab (docs/plans/definition-first-projects/spec-d4.md section 7): a task's records as a paged grid of +// 48 with a dot for the state of each, the record in its viewer (inside.js's), and the task's form beside it with +// the decision, a note, Keep and next, Skip, Revert and, for a stale annotation, Still right. Boxes are drawn on the +// image, spans marked in the text and segments on the waveform, in Canvas 2D and the DOM with the tokens of +// base.css, by pointer and by keyboard. The New task form (in the drawer) keeps a new annotations/ +// definition as an unsaved edit, as the editor keeps a new definition; the Data tab's Import and Freeze forms +// (datasetForm) keep a new dataset the same way. Annotations are kept with set_annotation and written by Save. +// +// Updating in place (spec-w4): the tab is drawn in four parts (the bar, the grid, the viewer, the form), each built +// off the page and put in place only when what it shows differs, so a reading that changed something elsewhere +// redraws the grid and leaves the form alone. What is typed or drawn in the form is a draft kept in page memory per +// task and record until it is kept, reverted or put right: a reading never replaces a form that holds one, and +// moving to another record keeps it. The forms in the drawer are never touched by a reading on the timer or on +// focus, and keep what is typed in their group's slot (actions.js). Keys (1 to 9, Enter, the left and right +// arrows) are ignored while typing in a field. +window.LabAnnotate = (() => { + const P = window.TuldokProject, S = window.LabScreens, A = window.LabActions, C = window.LabCharts, h = C.h, V = window.LabInside.viewers; + const { sect, putInPlace, putDrawer, turn, time, bytes } = S.parts; + const PER_PAGE = 48; + const TYPES = ['review', 'class', 'tags', 'text', 'boxes', 'spans', 'segments', 'polygons', 'mask', 'depth', 'keypoints', 'ranking', 'timeseries', 'track', 'points', 'box3d']; + const TYPE_WORD = { review: 'Review', class: 'Class: one term', tags: 'Tags: any of the terms', text: 'Text: a caption or a transcript', boxes: 'Boxes on an image', spans: 'Spans of a text', segments: 'Segments of a sound', + polygons: 'Polygons on an image', mask: 'A mask of an image', depth: 'A depth map of an image', keypoints: 'Keypoints on an image', ranking: 'A ranking of candidates', timeseries: 'Stretches of a series or a sound', track: 'Tracks across frames', points: 'Point objects of a point cloud', box3d: '3-D boxes in a point cloud' }; + const NEEDS_TERMS = ['class', 'tags', 'boxes', 'spans', 'segments', 'polygons', 'mask', 'keypoints', 'ranking', 'timeseries', 'track', 'points', 'box3d']; + // The kinds of record a task describes (spec-d4, spec-t4 section 1); the others take any kind but file. + const ONLY = { boxes: ['image'], spans: ['text'], segments: ['audio'], polygons: ['image'], mask: ['image'], depth: ['image'], keypoints: ['image'], timeseries: ['audio', 'array'], track: ['array'], points: ['pointcloud'], box3d: ['pointcloud'] }; + const GENERATED = ['polygons', 'mask', 'depth', 'keypoints', 'ranking', 'timeseries', 'track', 'points', 'box3d']; // reviewed, not drawn here + const FILTERS = [['all', 'All'], ['todo', 'To do'], ['accepted', 'Accepted'], ['needs-work', 'Needs work'], ['rejected', 'Rejected'], ['stale', 'Stale']]; + const DECISIONS = [['accepted', 'Accept', 'ok'], ['needs-work', 'Needs work', 'warn'], ['rejected', 'Reject', 'bad']]; + const STATE = { none: ['neutral', 'not annotated'], draft: ['neutral', 'draft'], accepted: ['good', 'accepted'], 'needs-work': ['warn', 'needs work'], rejected: ['serious', 'rejected'] }; + const SUGGESTS = ['text', 'class', 'tags']; // the tasks a served model can suggest a value of (spec-d5 section 4) + const FORMAT = 'tuldok_definition_v1'; + const count = (n, one, many) => n.toLocaleString('en-GB') + ' ' + (n === 1 ? one : many || one + 's'); + const button = (cls, text, key, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (key) b.dataset.key = key; if (onClick) b.addEventListener('click', onClick); return b; }; + const kbd = text => h('span', 'kbd', text); + const clamp = (v, lo, hi) => Math.min(hi, Math.max(lo, v)); + const rgb = (c, a) => a == null ? 'rgb(' + c.join(',') + ')' : 'rgba(' + c.join(',') + ',' + a + ')'; + const nameOf = ref => ref.slice(ref.indexOf('/') + 1); + const copy = v => v == null ? v : JSON.parse(JSON.stringify(v)); + const textOf = doc => JSON.stringify(doc, null, 2) + '\n'; + + // ---- What is kept in this page ------------------------------------------------------------------------------------ + const ui = { task: null, filter: 'all', page: 0, sel: new Map(), selAt: 0, terms: new Map(), mode: 'annotate' }; // sel and terms: per task + const drafts = new Map(); // task + '\n' + record -> { state, value, note, dirty } + const store = new Map(); // the viewers' settings (inside.js), per record + let here = null; // the tab on screen: { main, act, root, keys, parts, t, page, rec, pick, repaint, relist } + + const draftKey = (t, rec) => t.task + '\n' + rec.id; + function emptyValue(type) { return type === 'class' || type === 'review' || GENERATED.includes(type) ? null : type === 'text' ? '' : []; } + /** A draft from the annotation as it is (not typed yet), or the one typed. */ + function draftOf(t, rec, fresh) { + const k = draftKey(t, rec), had = drafts.get(k); + if (had && (had.dirty || !fresh)) return had; + const a = rec.annotation, d = { state: a && a.state !== 'draft' ? a.state : null, value: a && a.value != null ? copy(a.value) : emptyValue(t.type), note: (a && a.note) || '', dirty: false }; + if (t.type === 'text' && typeof d.value !== 'string') d.value = ''; + if (['tags', 'boxes', 'spans', 'segments'].includes(t.type) && !Array.isArray(d.value)) d.value = []; + drafts.set(k, d); return d; + } + /** The value a draft keeps, as the task's value: null for review, an empty text or no term. */ + function valueOf(t, d) { + if (t.type === 'review') return null; + if (t.type === 'class') return d.value || null; + if (t.type === 'text') return d.value ? d.value : null; + if (t.type === 'tags') return t.vocabulary.filter(v => d.value.includes(v)); + if (t.type === 'segments') return d.value.map(s => { const o = { term: s.term, start: s.start, end: s.end }; if (s.text) o.text = s.text; return o; }); + return copy(d.value); + } + function itemOf(t, rec, d) { const it = { record: rec.id, state: d.state || 'draft', value: valueOf(t, d) }; if (d.note && d.note.trim()) it.note = d.note; return it; } + + // ---- Reading --------------------------------------------------------------------------------------------------- + /** A page of the task's records for the filter: "To do" is the records not annotated, then the drafts. */ + async function pageOf(t) { + const offset = ui.page * PER_PAGE; + if (ui.filter !== 'todo') { const r = await P.annotations.records(t.task, { state: ui.filter === 'all' ? null : ui.filter, offset, limit: PER_PAGE }); return { records: r.records, total: r.total }; } + const u = await P.annotations.records(t.task, { state: 'unannotated', offset, limit: PER_PAGE }); + const need = PER_PAGE - u.records.length, d = await P.annotations.records(t.task, { state: 'draft', offset: Math.max(0, offset - u.total), limit: Math.max(1, need) }); + return { records: need > 0 ? u.records.concat(d.records.slice(0, need)) : u.records, total: u.total + d.total }; + } + async function load() { + const tasks = (await P.annotations.tasks()).tasks; + if (!tasks.length) return { tasks }; + const t = tasks.find(x => x.task === ui.task) || tasks[0]; ui.task = t.task; + if (!t.run) return { tasks, t }; + let page = await pageOf(t); + if (!page.records.length && ui.page > 0 && page.total) { ui.page = Math.floor((page.total - 1) / PER_PAGE); page = await pageOf(t); } + let id = ui.sel.get(t.task); + if (!page.records.some(r => r.id === id)) id = page.records.length ? page.records[ui.selAt === -1 ? page.records.length - 1 : 0].id : null; + ui.selAt = 0; if (id) ui.sel.set(t.task, id); else ui.sel.delete(t.task); + // The models that can suggest (spec-d5 section 4), for a session that may annotate; none when they cannot be read. + const offers = A.may('edit') && SUGGESTS.includes(t.type) ? await P.read('suggestion_models', { task: t.task }).then(r => r.kinds || {}, () => ({})) : {}; + // The task's annotation sets (spec-t4 section 6): Review is offered when an annotator made one. + const sets = await P.read('annotation_sets', { task: t.task }).then(r => r.sets, () => []); + return { tasks, t, page, rec: page.records.find(r => r.id === id) || null, offers, sets }; + } + + // ---- The tab ----------------------------------------------------------------------------------------------------- + /** The screen (app.js): the tab in main; the drawer holds the New task form when it is selected. */ + function annotate(main, aside, state, act) { + draw(main, act); + if (state.sel.annotate === 'new-task') { taskForm(aside, act); return true; } + aside.textContent = ''; aside._for = null; + return false; + } + /** A person's own move (a record, a page, a filter, a task): drawn again at once from what is kept. */ + const redraw = () => { if (here && here.root && here.root.isConnected) draw(here.main, here.act); }; + + /** + * Draws the tab: "Reading…" only when it is not on screen; otherwise each part whose content differs is built off + * the page and put in place, the others are left alone. A form holding a draft of the record it shows is kept. + */ + async function draw(main, act) { + const view = main._view, mine = main._paint = (main._paint || 0) + 1; + const shown = !!(here && here.root && main.contains(here.root) && main._painted === view); + if (!shown) { main.textContent = ''; main.appendChild(h('p', 'note', 'Reading…')); here = { main, act, root: null, keys: {}, parts: {}, pick: -1 }; } + here.act = act; + main.setAttribute('aria-busy', 'true'); + let got; try { got = await load(); } catch (error) { got = { error }; } + if (main._paint !== mine || main._view !== view) return; + const reviewing = !got.error && got.t && got.t.run && ui.mode === 'review' && reviewable(got.sets); + const kind = got.error ? 'error' : !got.tasks.length ? 'none' : !got.t.run ? 'unbuilt' : reviewing ? 'review' : 'records'; + const t = got.t || null, rec = got.rec || null, may = A.may('edit'); + const keys = { shape: kind, bar: kind === 'error' ? 'error' : JSON.stringify([got.tasks.map(x => [x.task, x.title, x.type]), t && [t.task, t.run, t.records, t.counts, t.dataset], ui.filter, may, ui.mode, reviewable(got.sets)]) }; + if (kind === 'records') { + keys.list = JSON.stringify([t.task, t.run, ui.filter, ui.page, got.page.total, got.page.records.map(r => [r.id, r.kind, r.path, r.preview || null, r.in_product, r.annotation && [r.annotation.state, r.annotation.stale, r.annotation.unsaved]])]); + keys.view = rec ? JSON.stringify([t.task, t.type, t.run, rec.id, rec.kind, rec.in_product]) : 'none'; // its place in the list is set in place (place()) + keys.form = rec ? JSON.stringify([t.task, t.type, t.vocabulary, rec.id, rec.kind, rec.in_product, rec.annotation, may, (got.offers || {})[rec.kind] || []]) : 'none'; + const same = here.rec && rec && here.t && here.t.task === t.task && here.rec.id === rec.id; + if (same && here.keys.form && keys.form !== here.keys.form && (drafts.get(draftKey(t, rec)) || {}).dirty) keys.form = here.keys.form; // what is typed stays + } else keys.msg = JSON.stringify([kind, t && [t.task, t.dataset], may, got.error ? got.error.message : null]); + // What changed is read before anything is put in place, so that it is put in one step. + const build = {}; + try { + if (keys.view && keys.view !== here.keys.view) { if (!here.rec || !rec || here.rec.id !== rec.id) here.pick = -1; build.view = rec ? await buildView(t, rec, got.page) : h('p', 'empty', 'Choose a record.'); } + if (keys.form && keys.form !== here.keys.form && rec && rec.annotation && rec.annotation.stale) build.detail = await P.annotations.one(t.task, rec.id).catch(() => null); + } catch (error) { build.view = h('div', 'iv'); build.view.appendChild(A.refusal(error)); } + if (main._paint !== mine || main._view !== view) return; + main.removeAttribute('aria-busy'); + here.t = t; here.page = got.page || null; here.rec = rec; here.tasks = got.tasks; here.offers = got.offers || {}; here.sets = got.sets || []; + const fresh = !here.root || keys.shape !== here.keys.shape; + if (fresh) { + const root = h('div', 'annotate'); root.dataset.shape = kind; + here.parts = { bar: h('div', 'ann-barbox') }; root.appendChild(here.parts.bar); + if (kind === 'review') { here.parts.review = h('div', 'ann-review'); root.appendChild(here.parts.review); } + else if (kind === 'records') { const body = h('div', 'ann-body'); for (const p of ['list', 'view', 'form']) { here.parts[p] = h('div', 'ann-' + p); body.appendChild(here.parts[p]); } here.parts.view.classList.add('viewer'); here.parts.view.setAttribute('aria-label', 'The record'); root.appendChild(body); } + else { here.parts.msg = h('div', 'ann-msg'); root.appendChild(here.parts.msg); } + here.keys = { shape: kind }; + here.root = root; + } + const put = (name, make) => { + if (here.keys[name] === keys[name]) return; + here.keys[name] = keys[name]; + const content = make(); if (fresh) here.parts[name].replaceChildren(content); else putInPlace(here.parts[name], content, main); + }; + put('bar', () => kind === 'error' ? h('div') : buildBar(got.tasks, t, got.sets)); + if (kind === 'review') { + if (fresh) putInPlace(main, here.root); + main._painted = view; + await window.LabReview.draw(here.parts.review, t, got.sets, here.act); + return; + } + if (kind === 'records') { + put('list', () => buildList(t, got.page)); + put('view', () => { const hooks = build.view._hooks || {}; here.repaint = hooks.repaint || null; here.textOf = hooks.textOf || null; here.play = hooks.play || null; return build.view; }); + put('form', () => rec ? buildForm(t, rec, build.detail) : h('p', 'empty', t.records ? 'No record matches the filter.' : 'The product of ' + t.dataset + ' holds no records.')); + // The selection is set in place: the grid is not drawn again for it. + for (const b of here.parts.list.querySelectorAll('.rec')) { const on = String(!!rec && b.dataset.id === rec.id); if (b.getAttribute('aria-pressed') !== on) b.setAttribute('aria-pressed', on); } + if (rec) place(here.parts.view, got.page, rec); + if (here.repaint) here.repaint(); + } else put('msg', () => buildMessage(kind, t, got.error)); + if (fresh) putInPlace(main, here.root); + main._painted = view; + if (here.focusAfter) { const k = here.focusAfter; here.focusAfter = null; A.focusKey(k); } + if (here.focusSel) { here.focusSel = false; const b = here.root.querySelector('.rec[aria-pressed="true"]'); if (b) b.focus(); } + } + + function buildMessage(kind, t, error) { + const box = h('div', 'ann-empty'); + if (kind === 'error') { box.appendChild(A.refusal(error)); return box; } + if (kind === 'none') { + box.appendChild(h('p', 'empty', 'This project has no annotation task. A task says what is recorded about the records of one dataset: a decision, a class, tags, a text, boxes, spans or segments.')); + if (!A.may('edit')) box.appendChild(h('p', 'note', 'Making a task needs the right to edit.')); + return box; + } + box.appendChild(h('p', 'empty', t.dataset + ' has no product on this machine, so ' + t.task + ' has no records to annotate yet. Build the dataset first.')); + box.appendChild(A.group('annotate:plan', [{ id: 'plan', label: 'Plan a build of ' + t.dataset, cls: 'btn', who: 'request', onClick: () => here.act.go('train', 'plan:' + t.dataset) }])); + return box; + } + + // ---- The bar ----------------------------------------------------------------------------------------------------- + /** Review needs a set made by an annotator besides the kept set. */ + const reviewable = sets => (sets || []).some(x => x.set !== 'kept'); + function buildBar(tasks, t, sets) { + const bar = h('div', 'ann-bar'); + if (tasks.length) { + const l = h('label', 'ann-task'); l.appendChild(document.createTextNode('Task')); + const s = h('select'); s.dataset.key = 'ann:task'; + for (const x of tasks) { const o = h('option', null, x.title + ' · ' + x.type); o.value = x.task; o.selected = !!t && x.task === t.task; s.appendChild(o); } + s.addEventListener('change', () => { ui.task = s.value; ui.page = 0; here.pick = -1; redraw(); }); + l.appendChild(s); bar.appendChild(l); + } + bar.appendChild(A.group('annotate:new', [{ id: 'new', label: 'New task', cls: 'btn sm', who: 'edit', onClick: () => here.act.select('new-task') }], { inline: true })); + if (t && t.run && reviewable(sets)) { + const seg = h('div', 'seg ann-mode'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'Annotate or review the annotators\' sets'); + for (const [id, label] of [['annotate', 'Annotate'], ['review', 'Review']]) { const b = button(null, label, 'ann:mode-' + id, () => { if (ui.mode !== id) { ui.mode = id; redraw(); } }); b.setAttribute('aria-pressed', String(ui.mode === id)); seg.appendChild(b); } + bar.appendChild(seg); + } + if (t && t.run && ui.mode === 'review' && reviewable(sets)) return bar; + if (t && t.run) { + const c = t.counts, n = { all: null, todo: c.unannotated + c.draft, accepted: c.accepted, 'needs-work': c['needs-work'], rejected: c.rejected, stale: c.stale }; + const seg = h('div', 'seg ann-filter'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'Show the records'); + for (const [id, label] of FILTERS) { const b = button(null, label + (n[id] != null ? ' · ' + n[id].toLocaleString('en-GB') : ''), 'ann:filter-' + id, () => { if (ui.filter !== id) { ui.filter = id; ui.page = 0; redraw(); } }); b.setAttribute('aria-pressed', String(ui.filter === id)); seg.appendChild(b); } + bar.appendChild(seg); + const decided = Math.min(t.records, c.accepted + c['needs-work'] + c.rejected); + const pr = h('span', 'ann-progress'); const bg = h('i'), fill = h('b'); fill.style.width = (t.records ? Math.round(100 * decided / t.records) : 0) + '%'; bg.appendChild(fill); pr.appendChild(bg); + pr.appendChild(document.createTextNode(decided.toLocaleString('en-GB') + ' of ' + count(t.records, 'record') + ' decided')); + bg.setAttribute('role', 'progressbar'); bg.setAttribute('aria-valuemin', '0'); bg.setAttribute('aria-valuemax', String(t.records)); bg.setAttribute('aria-valuenow', String(decided)); bg.setAttribute('aria-label', 'Decided'); + bar.appendChild(pr); + const keys = h('span', 'ann-keys small faint'); + const say = (...parts) => { for (const p of parts) keys.appendChild(typeof p === 'string' ? document.createTextNode(p) : p); }; + say(kbd('←'), ' ', kbd('→'), ' move'); + if (['class', 'tags'].includes(t.type) && t.vocabulary.length) say(' · ', kbd('1'), '–', kbd(String(Math.min(9, t.vocabulary.length))), ' term'); + say(' · ', kbd('Enter'), ' keep and next'); + bar.appendChild(keys); + } + return bar; + } + + // ---- The grid ---------------------------------------------------------------------------------------------------- + const marked = a => !a ? 'none' : a.state; + function buildList(t, page) { + const box = h('div'); + const first = ui.page * PER_PAGE + 1, last = ui.page * PER_PAGE + page.records.length; + box.appendChild(sect('Records', page.total ? first.toLocaleString('en-GB') + '–' + last.toLocaleString('en-GB') + ' of ' + page.total.toLocaleString('en-GB') : '0')); + if (!page.total) { box.appendChild(h('p', 'empty', ui.filter === 'all' ? 'The product holds no records.' : 'No record is ' + FILTERS.find(f => f[0] === ui.filter)[1].toLowerCase() + '.')); return box; } + const grid = h('div', 'rec-grid ann-grid'); grid.setAttribute('role', 'group'); grid.setAttribute('aria-label', 'Records ' + first + ' to ' + last + ' of ' + page.total + '; the left and right arrows move the selection'); + for (const r of page.records) grid.appendChild(tile(t, r)); + box.appendChild(grid); + const pages = Math.ceil(page.total / PER_PAGE); + if (pages > 1) { + const pg = h('div', 'pager'); + const pb = button('btn sm', 'Previous', 'ann:page-prev', () => turnPage(ui.page - 1, 0)); pb.disabled = ui.page === 0; + const nb = button('btn sm', 'Next', 'ann:page-next', () => turnPage(ui.page + 1, 0)); nb.disabled = ui.page + 1 >= pages; + pg.appendChild(pb); pg.appendChild(nb); pg.appendChild(h('span', null, 'page ' + (ui.page + 1) + ' of ' + pages)); box.appendChild(pg); + } + return box; + } + function turnPage(n, at) { ui.page = n; ui.sel.delete(ui.task); ui.selAt = at; redraw(); } + function tile(t, r) { + const a = r.annotation, st = marked(a), [, word] = STATE[st]; + const b = h('button', 'rec ann-rec ' + (r.kind || 'gone')); b.type = 'button'; b.dataset.id = r.id; b.dataset.key = 'ann-rec-' + r.id; b.dataset.state = st; + if (a && a.stale) b.dataset.stale = '1'; if (a && a.unsaved) b.dataset.unsaved = '1'; + b.setAttribute('aria-pressed', 'false'); b.title = r.id; + b.setAttribute('aria-label', (r.kind ? (V.WORD[r.kind] || r.kind) : 'Gone') + ' ' + r.id + ', ' + word + (a && a.stale ? ', stale' : '') + (a && a.unsaved ? ', unsaved' : '')); + const th = h('span', 'th'); + if (!r.in_product) th.appendChild(V.glyph('refused')); + else if (r.kind === 'text' && r.preview != null) th.appendChild(h('span', 'pre', r.preview.slice(0, 160))); + else th.appendChild(V.glyph(r.kind)); + if (r.in_product && r.kind === 'image' && r.path) { const img = h('img'); img.alt = ''; img.loading = 'lazy'; img.decoding = 'async'; img.src = P.product.fileUrl(t.run, r.path); img.addEventListener('load', () => { const g = th.querySelector('svg'); if (g) g.remove(); }, { once: true }); img.addEventListener('error', () => img.remove(), { once: true }); th.appendChild(img); } + const dot = h('i', 'badge ' + st); dot.setAttribute('aria-hidden', 'true'); th.appendChild(dot); + const id = h('span', 'id'); id.appendChild(h('bdi', null, r.id)); + const line = h('span', 'st'); line.appendChild(document.createTextNode(word)); + if (a && a.stale) line.appendChild(h('b', 'stale', 'stale')); if (a && a.unsaved) line.appendChild(h('b', 'unsaved', 'unsaved')); + b.appendChild(th); b.appendChild(id); b.appendChild(line); + b.addEventListener('click', () => select(r.id)); + return b; + } + function select(id) { if (!here || !here.t) return; ui.sel.set(here.t.task, id); redraw(); } + /** Moves the selection by d without keeping anything, across pages; on keep, done() moves on the same way. */ + function move(d, focusTile) { + if (!here || !here.page || !here.rec) return; + const list = here.page.records, i = list.findIndex(r => r.id === here.rec.id), n = i + d, pages = Math.ceil(here.page.total / PER_PAGE); + here.focusSel = !!focusTile; + if (n >= 0 && n < list.length) { ui.sel.set(here.t.task, list[n].id); return true; } + if (n >= list.length && ui.page + 1 < pages) { ui.page++; ui.sel.delete(ui.task); ui.selAt = 0; return true; } + if (n < 0 && ui.page > 0) { ui.page--; ui.sel.delete(ui.task); ui.selAt = -1; return true; } + here.focusSel = false; return false; + } + + // ---- The viewer --------------------------------------------------------------------------------------------------- + function heading(id, line) { const hd = h('header', 'iv-head'); hd.appendChild(h('h3', 'mono', id)); if (line) hd.appendChild(h('p', 'small muted', line)); return hd; } + /** Sets the record's place in the list into the viewer's Previous, Next and "3 of 48", in place: the viewer is not drawn again for it. */ + function place(view, page, rec) { + const nav = view.querySelector('.ann-nav'); if (!nav) return; + const at = page.records.indexOf(rec), pages = Math.ceil(page.total / PER_PAGE), [prev, where, next] = nav.children; + const text = (ui.page * PER_PAGE + at + 1).toLocaleString('en-GB') + ' of ' + page.total.toLocaleString('en-GB'); + if (where.textContent !== text) where.textContent = text; + const p = at <= 0 && ui.page === 0, n = at >= page.records.length - 1 && ui.page + 1 >= pages; + if (prev.disabled !== p) prev.disabled = p; if (next.disabled !== n) next.disabled = n; + } + async function buildView(t, rec, page) { + const el = h('div', 'iv ann-iv'); + const nav = h('div', 'ann-nav'); + nav.appendChild(button('btn sm', '← Previous', 'ann:prev', () => { if (move(-1)) redraw(); })); nav.appendChild(h('span', 'small muted')); nav.appendChild(button('btn sm', 'Next →', 'ann:next', () => { if (move(1)) redraw(); })); + el.appendChild(nav); place(el, page, rec); + el._hooks = {}; + if (!rec.in_product) { el.appendChild(heading(rec.id, 'Not in the current product')); el.appendChild(h('p', 'empty', 'This record is not in the current product of ' + t.dataset + ': it is gone, so its annotation is stale.')); return el; } + const got = await P.product.record(t.run, rec.id); + if (t.type === 'boxes' && rec.kind === 'image') boxView(el, got, t, rec); + else if (t.type === 'spans' && rec.kind === 'text') spanView(el, got, t, rec); + else if (t.type === 'segments' && rec.kind === 'audio') segmentView(el, got, t, rec); + else if (GENERATED.includes(t.type) && (draftOf(t, rec).value != null) && (el.appendChild(heading(rec.id, 'The kept annotation drawn over the record')), window.LabReview.overlay(el, got, t, draftOf(t, rec).value))) { /* drawn */ } + else await V.record(el, got, t.run, store); + return el; + } + const cur = () => here && here.t && here.rec ? draftOf(here.t, here.rec) : null; + /** The boxes, spans or segments of a record's draft (the one typed, or the annotation's). */ + const itemsOf = (t, rec) => { const d = draftOf(t, rec); if (!Array.isArray(d.value)) d.value = []; return d.value; }; + const newTerm = t => { const v = ui.terms.get(t.task); return t.vocabulary.includes(v) ? v : t.vocabulary[0]; }; + const fontOf = px => px + 'px ' + getComputedStyle(document.body).fontFamily; + /** Something drawn or typed changed: the draft is marked, and the drawing and the list follow. */ + function touch(list = true) { + const d = cur(); if (!d) return; d.dirty = true; + const m = here.root && here.root.querySelector('.ann-form .umark'); if (m && m.textContent !== 'not kept yet') { m.textContent = 'not kept yet'; m.classList.add('typed'); } + if (d.from) markSuggestion(d); + if (list && here.relist) here.relist(); + if (here.repaint) here.repaint(); + } + function pickItem(i) { here.pick = i; if (here.relist) here.relist(); if (here.repaint) here.repaint(); } + /** The keys of a surface (boxes, segments): N adds one, [ and ] choose, Delete removes, Escape lets go; the rest is the surface's. */ + function surfaceKeys(e, items, add, step) { + const list = items(); let used = true; + if (e.key === 'n' || e.key === 'N') { list.push(add()); here.pick = list.length - 1; touch(); } + else if (e.key === ']' || e.key === '[') { if (list.length) pickItem(((here.pick < 0 ? (e.key === ']' ? -1 : 0) : here.pick) + (e.key === ']' ? 1 : -1) + list.length) % list.length); } + else if ((e.key === 'Delete' || e.key === 'Backspace') && list[here.pick]) { list.splice(here.pick, 1); here.pick = -1; touch(); } + else if (e.key === 'Escape' && here.pick >= 0) { e.stopPropagation(); pickItem(-1); } + else if (e.key.startsWith('Arrow') && list[here.pick]) { step(list[here.pick], e.key, e.shiftKey, e.altKey); touch(); } + else used = false; + if (used) e.preventDefault(); + } + + // Boxes, drawn on the image. Coordinates are pixels of the image, whatever its size on screen. + function boxView(el, got, t, rec) { + const r = got.record, k = got.content || {}, may = A.may('edit'), items = () => itemsOf(t, rec); + el.appendChild(heading(r.id, ['Image', k.width && k.height ? k.width + ' × ' + k.height + ' pixels' : null, bytes(r.bytes)].filter(Boolean).join(' · '))); + const stage = h('div', 'ann-stage'), wrap = h('div', 'ann-imgwrap'), img = h('img'), cv = h('canvas', 'ann-surface'); + img.src = r.url; img.alt = ''; img.draggable = false; + cv.tabIndex = 0; cv.dataset.key = 'ann:surface'; cv.setAttribute('role', 'application'); + cv.setAttribute('aria-label', 'Image ' + r.id + ' with its boxes. ' + (may ? 'Drag to draw a box, drag a box to move it, a corner to resize it. N adds a box, [ and ] choose one, the arrows move it, Shift and the arrows resize it, Delete removes it.' : '')); + const said = h('p', 'sr-only'); said.setAttribute('aria-live', 'polite'); + wrap.appendChild(img); wrap.appendChild(cv); stage.appendChild(wrap); el.appendChild(stage); el.appendChild(said); + if (may) el.appendChild(h('p', 'small faint ann-help', 'Drag on the image to draw a box; drag a box to move it, a corner to resize it. With the image focused: N adds a box, [ and ] choose one, the arrows move it (Alt: by 10), Shift and the arrows resize it, Delete removes it.')); + let nw = k.width || 0, nh = k.height || 0, frame = 0, drag = null; + const layout = () => { + if (!wrap.isConnected) return; + if (!nw || !nh) { if (!img.naturalWidth) return; nw = img.naturalWidth; nh = img.naturalHeight; } + const avail = stage.clientWidth; if (!avail) return; + const s = Math.min(avail / nw, Math.max(180, innerHeight * 0.55) / nh), w = Math.max(1, Math.floor(nw * s)), hh = Math.max(1, Math.floor(nh * s)); + if (wrap.style.width !== w + 'px' || wrap.style.height !== hh + 'px') { wrap.style.width = w + 'px'; wrap.style.height = hh + 'px'; } + wrap.classList.toggle('pixels', s >= 2); + wrap.dataset.image = nw + 'x' + nh; + paint(); + }; + const paint = () => { + frame = 0; + const W0 = cv.clientWidth, H0 = cv.clientHeight; if (!W0 || !H0 || !nw) return; + const dpr = window.devicePixelRatio || 1, W = Math.round(W0 * dpr), H = Math.round(H0 * dpr); + if (cv.width !== W) cv.width = W; if (cv.height !== H) cv.height = H; + const ctx = cv.getContext('2d'); ctx.clearRect(0, 0, W, H); + const sx = W / nw, sy = H / nh, list = items(), mark = V.token('--mark'), acc = V.token('--accent'), halo = V.token('--surface'), ink = V.token('--accent-ink'); + ctx.font = fontOf(Math.round(11.5 * dpr)); ctx.textBaseline = 'top'; + list.forEach((b, i) => { + const on = i === here.pick, x = b.x * sx, y = b.y * sy, w = b.w * sx, hh = b.h * sy, c = on ? acc : mark; + ctx.fillStyle = rgb(c, on ? 0.16 : 0.08); ctx.fillRect(x, y, w, hh); + ctx.lineWidth = (on ? 4 : 3) * dpr; ctx.strokeStyle = rgb(halo, 0.9); ctx.strokeRect(x, y, w, hh); + ctx.lineWidth = (on ? 2 : 1.5) * dpr; ctx.strokeStyle = rgb(c); ctx.strokeRect(x, y, w, hh); + const label = (i + 1) + ' ' + b.term, tw = ctx.measureText(label).width + 8 * dpr, th = 16 * dpr, ty = y >= th ? y - th : y; + ctx.fillStyle = rgb(c); ctx.fillRect(x, ty, Math.min(tw, Math.max(tw, w)), th); ctx.fillStyle = rgb(ink); ctx.fillText(label, x + 4 * dpr, ty + 2 * dpr); + if (on) for (const [cx, cy] of [[x, y], [x + w, y], [x, y + hh], [x + w, y + hh]]) { const s = 7 * dpr; ctx.fillStyle = rgb(halo); ctx.fillRect(cx - s / 2, cy - s / 2, s, s); ctx.strokeStyle = rgb(acc); ctx.lineWidth = 1.5 * dpr; ctx.strokeRect(cx - s / 2, cy - s / 2, s, s); } + }); + const b = list[here.pick]; const text = b ? 'Box ' + (here.pick + 1) + ', ' + b.term + ', at ' + b.x + ', ' + b.y + ', ' + b.w + ' by ' + b.h + ' pixels' : count(list.length, 'box', 'boxes'); + if (said.textContent !== text) said.textContent = text; + }; + const later = () => { if (!frame) frame = requestAnimationFrame(paint); }; + el._hooks.repaint = later; + img.addEventListener('load', layout); + const ro = new ResizeObserver(() => { if (!stage.isConnected) { if (stage._seen) ro.disconnect(); return; } stage._seen = true; layout(); }); + ro.observe(stage); + V.paints(cv, later); // the theme + const at = e => { const q = cv.getBoundingClientRect(); return { x: clamp((e.clientX - q.left) / q.width * nw, 0, nw), y: clamp((e.clientY - q.top) / q.height * nh, 0, nh) }; }; + const corners = b => [['nw', b.x, b.y], ['ne', b.x + b.w, b.y], ['sw', b.x, b.y + b.h], ['se', b.x + b.w, b.y + b.h]]; + const fitBox = b => { b.x = clamp(Math.round(b.x), 0, nw - 1); b.y = clamp(Math.round(b.y), 0, nh - 1); b.w = clamp(Math.round(b.w), 1, nw - b.x); b.h = clamp(Math.round(b.h), 1, nh - b.y); return b; }; + cv.addEventListener('pointerdown', e => { + if (e.button !== 0 || !may || !nw) return; + e.preventDefault(); cv.focus({ preventScroll: true }); + const p = at(e), list = items(), tol = 8 * nw / cv.clientWidth; + let hit = null; + const order = list.map((_, i) => i).reverse(); if (here.pick >= 0 && list[here.pick]) order.unshift(here.pick); + for (const i of order) { for (const [corner, cx, cy] of corners(list[i])) if (Math.abs(p.x - cx) <= tol && Math.abs(p.y - cy) <= tol) { hit = { mode: 'resize', i, corner }; break; } if (hit) break; } + if (!hit) for (const i of order) { const b = list[i]; if (p.x >= b.x && p.x <= b.x + b.w && p.y >= b.y && p.y <= b.y + b.h) { hit = { mode: 'move', i }; break; } } + if (!hit) { list.push({ term: newTerm(t), x: p.x, y: p.y, w: 0, h: 0 }); hit = { mode: 'new', i: list.length - 1 }; } + drag = Object.assign(hit, { from: p, orig: Object.assign({}, list[hit.i]), id: e.pointerId, moved: false }); + here.pick = hit.i; cv.setPointerCapture(e.pointerId); later(); + }); + cv.addEventListener('pointermove', e => { + if (!drag || e.pointerId !== drag.id) return; + const p = at(e), b = items()[drag.i], o = drag.orig; if (!b) return; + drag.moved = true; + if (drag.mode === 'new') Object.assign(b, { x: Math.min(drag.from.x, p.x), y: Math.min(drag.from.y, p.y), w: Math.abs(p.x - drag.from.x), h: Math.abs(p.y - drag.from.y) }); + else if (drag.mode === 'move') Object.assign(b, { x: clamp(o.x + p.x - drag.from.x, 0, nw - o.w), y: clamp(o.y + p.y - drag.from.y, 0, nh - o.h) }); + else { const fx = drag.corner.includes('w') ? o.x + o.w : o.x, fy = drag.corner.includes('n') ? o.y + o.h : o.y; Object.assign(b, { x: Math.min(fx, p.x), y: Math.min(fy, p.y), w: Math.abs(p.x - fx), h: Math.abs(p.y - fy) }); } + later(); + }); + const end = e => { + if (!drag || e.pointerId !== drag.id) return; + const list = items(), b = list[drag.i], d = drag; drag = null; + if (!b) return; + if (d.mode === 'new' && (b.w < 1 || b.h < 1)) { list.splice(d.i, 1); here.pick = -1; if (here.relist) here.relist(); later(); return; } // a press without a drag: nothing drawn, nothing chosen + if (!d.moved) { if (here.relist) here.relist(); later(); return; } + fitBox(b); touch(); + }; + cv.addEventListener('pointerup', end); cv.addEventListener('pointercancel', end); + cv.addEventListener('keydown', e => { + if (!may || !nw) return; + surfaceKeys(e, items, () => fitBox({ term: newTerm(t), x: nw / 4, y: nh / 4, w: Math.max(1, nw / 2), h: Math.max(1, nh / 2) }), (b, key, shift, alt) => { + const s = alt ? 10 : 1, dx = key === 'ArrowLeft' ? -s : key === 'ArrowRight' ? s : 0, dy = key === 'ArrowUp' ? -s : key === 'ArrowDown' ? s : 0; + if (shift) { b.w = clamp(b.w + dx, 1, nw - b.x); b.h = clamp(b.h + dy, 1, nh - b.y); } + else { b.x = clamp(b.x + dx, 0, nw - b.w); b.y = clamp(b.y + dy, 0, nh - b.h); } + }); + }); + } + + // Spans, marked in the text. Offsets are characters (code points) of the record's text. + function spanView(el, got, t, rec) { + const r = got.record, k = got.content || {}, text = k.text || '', may = A.may('edit'), items = () => itemsOf(t, rec); + const total = k.characters != null ? k.characters : Array.from(text).length; + el.appendChild(heading(r.id, ['Text', count(total, 'character'), k.truncated ? 'truncated: spans can be marked in the first ' + Array.from(text).length.toLocaleString('en-GB') : null].filter(Boolean).join(' · '))); + const pre = h('pre', 'iv-text ann-text'); pre.tabIndex = 0; pre.dataset.key = 'ann:text-surface'; + pre.setAttribute('aria-label', 'The text, with its spans marked' + (may ? '; select a stretch of it to mark a span' : '')); + el.appendChild(pre); + const cps = Array.from(text), u16 = [0]; for (const c of cps) u16.push(u16[u16.length - 1] + c.length); // code point -> UTF-16 offset + const toCp = u => { let lo = 0, hi = cps.length; while (lo < hi) { const m = (lo + hi + 1) >> 1; if (u16[m] <= u) lo = m; else hi = m - 1; } return lo; }; + const paint = () => { + const list = items(), cuts = new Set([0, cps.length]); + for (const s of list) { cuts.add(clamp(s.start, 0, cps.length)); cuts.add(clamp(s.end, 0, cps.length)); } + const at = [...cuts].sort((a, b) => a - b), frag = document.createDocumentFragment(); + for (let j = 0; j + 1 < at.length; j++) { + const a = at[j], b = at[j + 1], piece = cps.slice(a, b).join(''), over = list.map((s, i) => [s, i]).filter(([s]) => s.start <= a && s.end >= b); + if (!over.length) frag.appendChild(document.createTextNode(piece)); + else { const m = h('mark', 'ann-span' + (over.some(([, i]) => i === here.pick) ? ' sel' : '') + (over.length > 1 ? ' many' : ''), piece); m.dataset.i = String(over[over.length - 1][1]); m.title = over.map(([s, i]) => (i + 1) + ' ' + s.term).join(', '); frag.appendChild(m); } + for (const [s, i] of list.map((s, i) => [s, i])) if (s.end === b) { const tag = h('span', 'ann-tag' + (i === here.pick ? ' sel' : ''), (i + 1) + ' ' + s.term); tag.setAttribute('aria-hidden', 'true'); tag.contentEditable = 'false'; frag.appendChild(tag); } + } + pre.replaceChildren(frag); + }; + el._hooks.repaint = paint; + /** The UTF-16 offset in the text of a place in pre, the tags left out. */ + const offsetOf = (node, offset) => { const range = document.createRange(); range.setStart(pre, 0); range.setEnd(node, offset); const f = range.cloneContents(); for (const x of f.querySelectorAll('.ann-tag')) x.remove(); return f.textContent.length; }; + const mark = () => { + if (!may) return false; + const s = getSelection(); if (!s.rangeCount || s.isCollapsed) return false; + const range = s.getRangeAt(0); if (!pre.contains(range.startContainer) || !pre.contains(range.endContainer)) return false; + let a = toCp(offsetOf(range.startContainer, range.startOffset)), b = toCp(offsetOf(range.endContainer, range.endOffset)); if (a > b) [a, b] = [b, a]; + if (a === b) return false; + const list = items(); list.push({ term: newTerm(t), start: a, end: b }); here.pick = list.length - 1; + s.removeAllRanges(); touch(); return true; + }; + pre.addEventListener('mouseup', () => { mark(); }); + pre.addEventListener('click', e => { const m = e.target.closest && e.target.closest('mark.ann-span'); if (m && getSelection().isCollapsed) pickItem(Number(m.dataset.i)); }); + if (may) { const row = h('div', 'iv-ctl'); row.appendChild(button('btn sm', 'Mark the selected text', 'ann:mark', () => { if (!mark()) say.textContent = 'Select a stretch of the text first.'; })); const say = h('span', 'small faint', 'Select a stretch of the text to mark it as a span of the term chosen in the form.'); say.setAttribute('aria-live', 'polite'); row.appendChild(say); el.appendChild(row); } + paint(); + el._hooks.textOf = (a, b) => cps.slice(a, b).join(''); + } + + // Segments, drawn on the waveform. Times are seconds of the sound. + function segmentView(el, got, t, rec) { + const r = got.record, k = got.content || {}, may = A.may('edit'), items = () => itemsOf(t, rec); + if (k.seconds == null || !k.peaks || !k.peaks.length) { + el.appendChild(heading(r.id, 'Sound')); + el.appendChild(h('p', 'note', 'Tuldok reads a sound\'s length from a WAV only, so segments cannot be drawn on this one; its viewer is below.')); + V.record(el, got, t.run, store); return; + } + const secs = k.seconds; + el.appendChild(heading(r.id, [secs.toFixed(secs < 10 ? 2 : 1) + ' s', k.rate.toLocaleString('en-GB') + ' Hz', count(k.channels, 'channel'), bytes(r.bytes)].join(' · '))); + const cv = h('canvas', 'ann-surface ann-wave'); cv.tabIndex = 0; cv.dataset.key = 'ann:surface'; cv.setAttribute('role', 'application'); + cv.setAttribute('aria-label', 'Waveform of ' + r.id + ', ' + secs.toFixed(2) + ' seconds, with its segments. ' + (may ? 'Drag to mark a segment, drag a segment to move it, an end to move that end. N adds a segment, [ and ] choose one, the arrows move it, Shift and the arrows move its end, P plays it, Delete removes it.' : '')); + const said = h('p', 'sr-only'); said.setAttribute('aria-live', 'polite'); + const audio = h('audio'); audio.controls = true; audio.preload = 'metadata'; audio.src = r.url; audio.dataset.key = 'ann:audio'; audio.setAttribute('aria-label', 'Play ' + r.id); + el.appendChild(cv); el.appendChild(audio); el.appendChild(said); + if (may) el.appendChild(h('p', 'small faint ann-help', 'Drag on the waveform to mark a segment; drag a segment to move it, an end to move that end. With the waveform focused: N adds a segment, [ and ] choose one, the arrows move it by 0.05 s (Alt: by 0.5 s), Shift and the arrows move its end, P plays it, Delete removes it.')); + let frame = 0, drag = null, stopAt = null; + const paint = () => { + frame = 0; + const size = V.fit(cv, 120); if (!size) return; const [W, H] = size, dpr = window.devicePixelRatio || 1, ctx = cv.getContext('2d'); + ctx.clearRect(0, 0, W, H); V.wave(ctx, W, H, k.peaks); + const list = items(), mark = V.token('--k-refinement'), acc = V.token('--accent'), ink = V.token('--accent-ink'); + ctx.font = fontOf(Math.round(11.5 * dpr)); ctx.textBaseline = 'top'; + list.forEach((s, i) => { + const on = i === here.pick, x0 = s.start / secs * W, x1 = s.end / secs * W, c = on ? acc : mark; + ctx.fillStyle = rgb(c, on ? 0.26 : 0.16); ctx.fillRect(x0, 0, Math.max(1, x1 - x0), H); + ctx.fillStyle = rgb(c); ctx.fillRect(x0, 0, Math.max(1, dpr * (on ? 2 : 1)), H); ctx.fillRect(x1 - Math.max(1, dpr * (on ? 2 : 1)), 0, Math.max(1, dpr * (on ? 2 : 1)), H); + const label = (i + 1) + ' ' + s.term, tw = ctx.measureText(label).width + 8 * dpr; + ctx.fillRect(x0, 0, tw, 16 * dpr); ctx.fillStyle = rgb(ink); ctx.fillText(label, x0 + 4 * dpr, 2 * dpr); + }); + if (!audio.paused) { ctx.fillStyle = rgb(V.token('--ink')); ctx.fillRect(audio.currentTime / secs * W, 0, Math.max(1, dpr), H); } + const s = list[here.pick], text = s ? 'Segment ' + (here.pick + 1) + ', ' + s.term + ', ' + s.start.toFixed(2) + ' to ' + s.end.toFixed(2) + ' seconds' : count(list.length, 'segment'); + if (said.textContent !== text) said.textContent = text; + }; + const later = () => { if (!frame) frame = requestAnimationFrame(paint); }; + el._hooks.repaint = later; + V.paints(cv, paint); + const tick = () => { if (stopAt != null && audio.currentTime >= stopAt) { audio.pause(); stopAt = null; } later(); if (!audio.paused) requestAnimationFrame(tick); }; + audio.addEventListener('play', () => requestAnimationFrame(tick)); audio.addEventListener('pause', later); + el._hooks.play = i => { const s = items()[i]; if (!s) return; stopAt = s.end; audio.currentTime = s.start; const p = audio.play(); if (p && p.catch) p.catch(() => {}); }; + const round = v => Math.round(v * 1000) / 1000; + const at = e => { const q = cv.getBoundingClientRect(); return clamp((e.clientX - q.left) / q.width * secs, 0, secs); }; + cv.addEventListener('pointerdown', e => { + if (e.button !== 0 || !may) return; + e.preventDefault(); cv.focus({ preventScroll: true }); + const p = at(e), list = items(), tol = 6 * secs / cv.clientWidth; + let hit = null; + const order = list.map((_, i) => i).reverse(); if (here.pick >= 0 && list[here.pick]) order.unshift(here.pick); + for (const i of order) { const s = list[i]; if (Math.abs(p - s.start) <= tol) { hit = { mode: 'start', i }; break; } if (Math.abs(p - s.end) <= tol) { hit = { mode: 'end', i }; break; } } + if (!hit) for (const i of order) { const s = list[i]; if (p > s.start && p < s.end) { hit = { mode: 'move', i }; break; } } + if (!hit) { list.push({ term: newTerm(t), start: p, end: p }); hit = { mode: 'new', i: list.length - 1 }; } + drag = Object.assign(hit, { from: p, orig: Object.assign({}, list[hit.i]), id: e.pointerId, moved: false }); + here.pick = hit.i; cv.setPointerCapture(e.pointerId); later(); + }); + cv.addEventListener('pointermove', e => { + if (!drag || e.pointerId !== drag.id) return; + const p = at(e), s = items()[drag.i], o = drag.orig; if (!s) return; + drag.moved = true; + if (drag.mode === 'new') { s.start = Math.min(drag.from, p); s.end = Math.max(drag.from, p); } + else if (drag.mode === 'move') { const len = o.end - o.start, a = clamp(o.start + p - drag.from, 0, secs - len); s.start = a; s.end = a + len; } + else { const fixed = drag.mode === 'start' ? o.end : o.start; s.start = Math.min(fixed, p); s.end = Math.max(fixed, p); } + later(); + }); + const end = e => { + if (!drag || e.pointerId !== drag.id) return; + const list = items(), s = list[drag.i], d = drag; drag = null; + if (!s) return; + s.start = round(s.start); s.end = round(s.end); + if (s.end - s.start < 0.01) { if (d.mode === 'new') { list.splice(d.i, 1); here.pick = -1; if (here.relist) here.relist(); later(); return; } s.end = round(Math.min(secs, s.start + 0.01)); } + if (!d.moved) { if (here.relist) here.relist(); later(); return; } + touch(); + }; + cv.addEventListener('pointerup', end); cv.addEventListener('pointercancel', end); + cv.addEventListener('keydown', e => { + if (!may) return; + if ((e.key === 'p' || e.key === 'P') && items()[here.pick]) { e.preventDefault(); el._hooks.play(here.pick); return; } + surfaceKeys(e, items, () => { const a = round(clamp(audio.currentTime || 0, 0, Math.max(0, secs - 0.25))); return { term: newTerm(t), start: a, end: round(Math.min(secs, a + 0.25)) }; }, (s, key, shift, alt) => { + const step = alt ? 0.5 : 0.05, d = key === 'ArrowLeft' || key === 'ArrowDown' ? -step : step; + if (shift) s.end = round(clamp(s.end + d, s.start + 0.01, secs)); + else { const len = s.end - s.start; s.start = round(clamp(s.start + d, 0, secs - len)); s.end = round(s.start + len); } + }); + }); + } + + // ---- The form ---------------------------------------------------------------------------------------------------- + function buildForm(t, rec, detail) { + here.relist = null; + const d = draftOf(t, rec, true), a = rec.annotation, may = A.may('edit'), fits = !ONLY[t.type] || ONLY[t.type].includes(rec.kind); + const form = h('div', 'inspector ann-formbox'); + const hd = h('header'); hd.appendChild(h('span', 'kicker', t.title + ' · ' + t.type)); + const t2 = h('h2', 'mono', rec.id); t2.tabIndex = -1; hd.appendChild(t2); + const meta = h('div', 'ins-meta'), [tone, word] = STATE[marked(a)]; meta.appendChild(S.stateEl(tone, word)); + if (a && a.stale) meta.appendChild(h('span', 'ann-flag stale', 'stale')); + if (a && a.unsaved) meta.appendChild(h('span', 'ann-flag unsaved', 'unsaved')); + const um = h('span', 'umark' + (d.dirty ? ' typed' : ''), d.dirty ? 'not kept yet' : ''); meta.appendChild(um); + hd.appendChild(meta); + if (a) hd.appendChild(h('p', 'small faint', (a.state === 'draft' ? 'Drafted' : 'Annotated') + ' by ' + a.by + ' · ' + time(a.at))); + form.appendChild(hd); + if (a && a.stale) { + const box = h('div', 'ann-stale'); box.setAttribute('role', 'status'); + box.appendChild(h('p', null, rec.in_product ? 'Stale: the record\'s content differs from what was annotated.' : 'Stale: the record is gone from the current product of ' + t.dataset + '.')); + if (rec.in_product && detail && detail.annotation) box.appendChild(h('p', 'small mono', 'annotated ' + String(detail.annotation.sha256 || 'without a hash').slice(0, 12) + ' · now ' + String(detail.sha256 || 'not hashed').slice(0, 12))); + box.appendChild(h('p', 'small', rec.in_product ? 'The annotation as it was made is below. Still right keeps it for the content as it is now; or change it and keep it.' : 'Remove the annotation, or keep it until the record comes back.')); + form.appendChild(box); + } + if (may && rec.in_product && ((here.offers || {})[rec.kind] || []).length) { const sb = h('div', 'ann-suggestbox'); sb.appendChild(suggestBox(t, rec)); form.appendChild(sb); } + const f = h('div', 'form ann-fields'); + if (!rec.in_product) f.appendChild(h('p', 'note', 'Nothing can be set on a record that is not in the product.')); + else if (rec.kind === 'file') f.appendChild(h('p', 'note', 'A file of no kind cannot be annotated.')); + else if (!fits) f.appendChild(h('p', 'note', 'This task annotates ' + ONLY[t.type].join(' or ') + ' records; this record is ' + (V.WORD[rec.kind] || rec.kind).toLowerCase() + ', so only a decision and a note can be kept for it.')); + if (rec.in_product && rec.kind !== 'file' && fits) valueControls(f, t, rec, d, may); + // The decision. + const dl = h('div', 'ann-field'); dl.appendChild(h('h3', null, 'Decision')); + const seg = h('div', 'seg ann-decide'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'Decision'); + for (const [v, label, cls] of DECISIONS) { + const b = button(cls, label, 'ann:decide-' + v, () => { d.state = d.state === v ? null : v; for (const x of seg.children) x.setAttribute('aria-pressed', String(x.dataset.key === 'ann:decide-' + d.state)); touch(false); }); + b.setAttribute('aria-pressed', String(d.state === v)); b.disabled = !may || !rec.in_product; seg.appendChild(b); + } + dl.appendChild(seg); dl.appendChild(h('p', 'small faint', 'Without a decision, Keep and next keeps a draft.')); f.appendChild(dl); + const nl = h('label'); nl.appendChild(document.createTextNode('Note')); + const note = h('textarea'); note.rows = 2; note.maxLength = 4000; note.value = d.note; note.dataset.key = 'ann:note'; note.disabled = !may || !rec.in_product; note.placeholder = 'optional'; + note.addEventListener('input', () => { d.note = note.value; touch(false); }); nl.appendChild(note); f.appendChild(nl); + form.appendChild(f); + // Keep and next, Skip, Revert, Still right, Remove; a refusal is shown under them in the server's words. + const slot = A.slot('annotate:form'); if (slot.for !== draftKey(t, rec)) { slot.refusal = null; slot.confirming = null; slot.for = draftKey(t, rec); } + const k = draftKey(t, rec), after = () => { drafts.delete(k); }; + form.appendChild(A.group('annotate:form', [ + { id: 'keep', label: 'Keep and next', cls: 'btn primary', who: 'edit', op: 'set_annotation', busy: 'Keeping…', hidden: !rec.in_product, title: 'Enter', + args: () => Object.assign({ task: t.task, items: [itemOf(t, rec, d)] }, d.from ? { from: d.from } : {}), done: () => { after(); move(1); } }, + { id: 'skip', label: 'Skip', cls: 'btn ghost', onClick: () => { if (move(1)) redraw(); } }, + { id: 'still', label: 'Still right', cls: 'btn', who: 'edit', op: 'set_annotation', busy: 'Keeping…', hidden: !(a && a.stale && rec.in_product), + args: () => ({ task: t.task, items: [Object.assign({ record: rec.id, state: a.state, value: a.value }, a.note ? { note: a.note } : {})] }), done: after }, + { id: 'remove', label: 'Remove the annotation', cls: 'btn', who: 'edit', op: 'set_annotation', busy: 'Removing…', hidden: !(a && !rec.in_product), + confirm: 'Remove the annotation of ' + rec.id + '; Save removes its line from the file.', args: () => ({ task: t.task, items: [{ record: rec.id, remove: true }] }), done: after }, + { id: 'revert', label: 'Revert', cls: 'btn ghost', who: 'edit', op: 'revert_annotations', busy: 'Reverting…', hidden: !(a && a.unsaved), + confirm: 'Put back the saved annotation of ' + rec.id + ' in place of its unsaved one' + '; a record with no saved annotation has none again.', args: () => ({ task: t.task, records: [rec.id] }), done: after } + ])); + if (!may) form.appendChild(h('p', 'note', 'This session may look but not annotate: that needs the right to edit.')); + else form.appendChild(h('p', 'note', 'Keep and next keeps the annotation in Tuldok as an unsaved edit; Save writes every task\'s annotations to its file.')); + return form; + } + + /** The controls of the value, by task. */ + function valueControls(f, t, rec, d, may) { + if (t.type === 'review') return; + if (GENERATED.includes(t.type)) { // spec-t4: made by annotators and reviewed; the value is shown, not drawn here + const box = h('div', 'ann-field'); box.appendChild(h('h3', null, 'Value')); + box.appendChild(h('p', null, window.LabReview.summary(t, d.value))); + box.appendChild(h('p', 'small faint', here.sets && here.sets.some(x => x.set !== 'kept') ? 'This task\'s annotations are made by annotators: choose Review in the bar to compare their sets and keep one.' : 'This task\'s annotations are made by annotators (a dataset with the step annotate); the decision and the note can be kept here.')); + f.appendChild(box); return; + } + if (t.type === 'text') { + const l = h('label'); l.appendChild(document.createTextNode('Text')); + const area = h('textarea'); area.rows = 5; area.value = d.value || ''; area.dataset.key = 'ann:text'; area.disabled = !may; area.placeholder = 'a caption, a transcript or a summary'; + area.addEventListener('input', () => { d.value = area.value; touch(false); }); l.appendChild(area); f.appendChild(l); return; + } + if (t.type === 'class' || t.type === 'tags') { + const one = t.type === 'class', box = h('div', 'ann-field'); box.appendChild(h('h3', null, one ? 'Term' : 'Terms')); + const rd = h('div', 'radio ann-terms'); rd.setAttribute('role', one ? 'radiogroup' : 'group'); rd.setAttribute('aria-label', one ? 'Term' : 'Terms'); + t.vocabulary.forEach((term, i) => { + const l = h('label'), inp = h('input'); inp.type = one ? 'radio' : 'checkbox'; inp.name = 'ann-term'; inp.value = term; inp.dataset.key = 'ann:term-' + i; inp.disabled = !may; + inp.checked = one ? d.value === term : d.value.includes(term); + inp.addEventListener('change', () => { if (one) { if (inp.checked) d.value = term; } else d.value = inp.checked ? d.value.concat([term]) : d.value.filter(x => x !== term); touch(false); }); + l.appendChild(inp); l.appendChild(h('span', null, term)); if (i < 9) l.appendChild(kbd(String(i + 1))); rd.appendChild(l); + }); + box.appendChild(rd); f.appendChild(box); return; + } + // Boxes, spans and segments: the term of new ones, and the list of those drawn. + const box = h('div', 'ann-field'); + const what = { boxes: ['Boxes', 'box', 'boxes'], spans: ['Spans', 'span', 'spans'], segments: ['Segments', 'segment', 'segments'] }[t.type]; + const tl = h('label', 'ann-newterm'); tl.appendChild(document.createTextNode('Term of new ' + what[2])); + const ts = h('select'); ts.dataset.key = 'ann:new-term'; ts.disabled = !may; + for (const term of t.vocabulary) { const o = h('option', null, term); o.value = term; o.selected = term === newTerm(t); ts.appendChild(o); } + ts.addEventListener('change', () => { ui.terms.set(t.task, ts.value); }); + tl.appendChild(ts); box.appendChild(tl); + box.appendChild(h('h3', null, what[0])); + const listBox = h('div', 'ann-itembox'); box.appendChild(listBox); + const relist = () => putInPlace(listBox, itemList(t, d, what, may)); + here.relist = relist; listBox.appendChild(itemList(t, d, what, may)); + f.appendChild(box); + } + function itemList(t, d, what, may) { + const list = d.value; + if (!list.length) return h('p', 'note', 'No ' + what[2] + ' yet. ' + { boxes: 'Drag on the image to draw one.', spans: 'Select a stretch of the text to mark one.', segments: 'Drag on the waveform to mark one.' }[t.type]); + const ul = h('ul', 'ann-items'); + list.forEach((it, i) => { + const row = h('li', i === here.pick ? 'sel' : null); row.dataset.i = String(i); + const pick = button('linkish', (i + 1) + '', 'ann:item-' + i, () => { pickItem(here.pick === i ? -1 : i); }); pick.setAttribute('aria-label', 'Choose ' + what[1] + ' ' + (i + 1)); pick.setAttribute('aria-pressed', String(i === here.pick)); row.appendChild(pick); + const li = h('span', 'ann-fields'); row.appendChild(li); // the term, the place and Remove, after the number + const ts = h('select'); ts.dataset.key = 'ann:item-' + i + '-term'; ts.setAttribute('aria-label', 'Term of ' + what[1] + ' ' + (i + 1)); ts.disabled = !may; + for (const term of t.vocabulary.includes(it.term) ? t.vocabulary : [it.term].concat(t.vocabulary)) { const o = h('option', null, term); o.value = term; o.selected = term === it.term; ts.appendChild(o); } + ts.addEventListener('change', () => { it.term = ts.value; touch(); }); li.appendChild(ts); + const num = (name, label, step, lo, hi) => { + const n = h('input'); n.type = 'number'; n.step = String(step); n.value = String(it[name]); n.dataset.key = 'ann:item-' + i + '-' + name; n.setAttribute('aria-label', label + ' of ' + what[1] + ' ' + (i + 1)); n.disabled = !may; n.className = 'ann-num'; + n.addEventListener('change', () => { const v = Number(n.value); if (!Number.isFinite(v)) { n.value = String(it[name]); return; } it[name] = clamp(step === 1 ? Math.round(v) : Math.round(v * 1000) / 1000, lo(), hi()); if (it.start !== undefined && it.end <= it.start) it.end = Math.min(hi(), it.start + (step === 1 ? 1 : 0.01)); touch(); }); + return n; + }; + if (t.type === 'boxes') li.appendChild(h('span', 'mono small ann-coords', 'x ' + it.x + ', y ' + it.y + ' · ' + it.w + ' × ' + it.h)); + if (t.type === 'spans') { + const n = here.textOf ? Array.from(here.textOf(0, 1e9)).length : 1e9; + li.appendChild(num('start', 'Start', 1, () => 0, () => Math.max(0, it.end - 1))); li.appendChild(num('end', 'End', 1, () => it.start + 1, () => n)); + li.appendChild(h('q', 'ann-covered', here.textOf ? here.textOf(it.start, it.end) : '')); + } + if (t.type === 'segments') { + li.appendChild(num('start', 'Start (s)', 0.01, () => 0, () => Math.max(0, it.end - 0.01))); li.appendChild(num('end', 'End (s)', 0.01, () => it.start + 0.01, () => 1e9)); + const tx = h('input'); tx.type = 'text'; tx.value = it.text || ''; tx.placeholder = 'what is said (optional)'; tx.dataset.key = 'ann:item-' + i + '-text'; tx.setAttribute('aria-label', 'What is said in segment ' + (i + 1)); tx.disabled = !may; tx.className = 'ann-saying'; + tx.addEventListener('input', () => { it.text = tx.value; touch(false); }); li.appendChild(tx); + li.appendChild(button('btn sm', 'Play', 'ann:item-' + i + '-play', () => { if (here.play) here.play(i); })); + } + const rm = button('btn sm ghost', 'Remove', 'ann:item-' + i + '-remove', () => { list.splice(i, 1); here.pick = -1; touch(); }); rm.disabled = !may; rm.setAttribute('aria-label', 'Remove ' + what[1] + ' ' + (i + 1)); li.appendChild(rm); + ul.appendChild(row); + }); + return ul; + } + + // ---- Suggestions (spec-d5 section 4) ---------------------------------------------------------------------------- + // Suggest asks the chosen model (suggestion_models' offers for the record's kind) for a value of the task for the + // record on screen. The answer fills the record's draft with its `from`, marked as the model's suggestion until the + // person changes or keeps it; Keep sends `from` with set_annotation, so the saved line names the model. Cancel drops + // the answer when it comes (the endpoint is not told: no operation stops a suggestion). A refusal is shown beside the + // form, which is left as it was. Suggest for this page asks, one record after another, for the records on screen that + // have no annotation and no typed draft, and keeps each answer as a draft with its `from`. The model chosen is + // remembered per task and kind of record while the page is open. A reading never asks anything. + const sug = { choice: new Map(), asking: null, refusals: new Map(), batch: null, said: new Map(), turn: 0 }; + const offerKey = o => o.model + '\u0000' + o.profile; + /** The model chosen for the task and kind: the one picked while the page is open, else the first offered. */ + function chosen(t, kind) { const list = (here && here.offers || {})[kind] || [], k = sug.choice.get(t.task + '\n' + kind); return list.find(o => offerKey(o) === k) || list[0] || null; } + const fromWords = f => f.model + ' · ' + f.profile; + function suggestionLine(d) { + const same = JSON.stringify(valueOf(here.t, d)) === d.fromValue; + return [same ? 'Suggested by ' + fromWords(d.from) + ' (' + d.from.capability + '): check it, change it or keep it.' : 'Changed from the suggestion of ' + fromWords(d.from) + '; kept, the line still names it.', same]; + } + /** Sets the suggestion's mark in place: a suggestion, or one the person changed. */ + function markSuggestion(d) { + const m = here.root && here.root.querySelector('.ann-form .ann-sugmark'); if (!m) return; + const [text, same] = suggestionLine(d); + if (m.textContent !== text) m.textContent = text; + if (m.classList.contains('changed') === same) m.classList.toggle('changed', !same); + } + function suggestBox(t, rec) { + const box = h('div', 'ann-suggest'), list = here.offers[rec.kind] || [], k = draftKey(t, rec), d = drafts.get(k); + const asking = sug.asking && sug.asking.key === k ? sug.asking : null, batch = sug.batch && sug.batch.task === t.task ? sug.batch : null; + const row = h('div', 'ann-suggest-row'); + const l = h('label', 'ann-suggest-model'); l.appendChild(document.createTextNode('Suggest with')); + const sel = h('select'); sel.dataset.key = 'ann:suggest-model'; const pick = chosen(t, rec.kind); + for (const o of list) { const op = h('option', null, fromWords(o) + ' (' + o.capability + ')'); op.value = offerKey(o); op.selected = !!pick && offerKey(o) === offerKey(pick); sel.appendChild(op); } + sel.disabled = !!asking || !!batch; + sel.addEventListener('change', () => { sug.choice.set(t.task + '\n' + rec.kind, sel.value); }); + l.appendChild(sel); row.appendChild(l); + if (asking) { + const b = button('btn sm', 'Asking ' + asking.model + '…', 'ann:suggest'); b.disabled = true; b.setAttribute('aria-busy', 'true'); row.appendChild(b); + row.appendChild(button('btn sm ghost', 'Cancel', 'ann:suggest-cancel', () => { sug.asking = null; reboxed(t, rec, 'ann:suggest'); })); + } else { const b = button('btn sm', 'Suggest', 'ann:suggest', () => ask(t, rec)); b.disabled = !!batch || !!sug.asking; row.appendChild(b); } + box.appendChild(row); + if (d && d.from) { const [text, same] = suggestionLine(d); const m = h('p', 'small ann-sugmark' + (same ? '' : ' changed'), text); m.setAttribute('role', 'status'); box.appendChild(m); } + const refusal = sug.refusals.get(k); + if (refusal) { const r = A.refusal(refusal); r.classList.add('ann-sugrefusal'); r.insertBefore(h('p', 'small', 'No suggestion: the form is as it was.'), r.children[1] || null); box.appendChild(r); } + const page = h('div', 'ann-suggest-row ann-batch'); + if (batch) { + const line = h('span', 'small ann-batch-line', batchWords(batch)); line.setAttribute('role', 'status'); page.appendChild(line); + const stop = button('btn sm ghost', batch.stop ? 'Stopping…' : 'Stop', 'ann:suggest-stop', () => { batch.stop = true; reboxed(t, rec, 'ann:suggest-stop'); }); stop.disabled = batch.stop; page.appendChild(stop); + } else { + const b = button('btn sm', 'Suggest for this page', 'ann:suggest-page', () => suggestPage(t)); b.disabled = !!sug.asking; page.appendChild(b); + const said = sug.said.get(t.task); + page.appendChild(h('span', 'small faint ann-batch-line', said || 'Asks for each record on this page with no annotation, one after another, and keeps each answer as a draft.')); + } + box.appendChild(page); + return box; + } + const shows = (t, rec) => !!(here && here.root && here.root.isConnected && here.t && here.rec && here.t.task === t.task && here.rec.id === rec.id); + /** The suggestion part of the form on screen, drawn again in place; the focus goes to focus when it was lost. */ + function reboxed(t, rec, focus) { + if (!shows(t, rec)) return; + const w = here.parts.form && here.parts.form.querySelector('.ann-suggestbox'); if (!w) return; + const r = putInPlace(w, suggestBox(t, rec)); + if (focus && (!r.inside || !r.found || document.activeElement === document.body)) A.focusKey(focus); + } + const rebox = focus => { if (here && here.t && here.rec) reboxed(here.t, here.rec, focus); }; + async function ask(t, rec) { + const o = chosen(t, rec.kind); if (!o || sug.asking || sug.batch) return; + const k = draftKey(t, rec), mine = { key: k, model: o.model, n: ++sug.turn }; + sug.asking = mine; sug.refusals.delete(k); reboxed(t, rec, 'ann:suggest-cancel'); + let got = null, error = null; + try { got = await P.call('suggest_annotation', { task: t.task, record: rec.id, model: o.model, profile: o.profile }); } catch (e) { error = e; } + if (sug.asking !== mine) return; // cancelled: the answer is dropped + sug.asking = null; + if (error) { sug.refusals.set(k, error); reboxed(t, rec, 'ann:suggest'); return; } + fill(t, rec, got.suggestion); + if (shows(t, rec)) { here.keys.form = null; here.focusAfter = 'ann:suggest'; redraw(); } + } + /** The suggestion in the record's draft, with its `from`; what was decided and noted stays. */ + function fill(t, rec, s) { + const d = draftOf(t, rec); + if (t.type === 'text') d.value = typeof s.value === 'string' ? s.value : ''; + else if (t.type === 'class') d.value = typeof s.value === 'string' ? s.value : null; + else if (t.type === 'tags') d.value = Array.isArray(s.value) ? s.value.slice() : []; + d.from = s.from; d.fromValue = JSON.stringify(valueOf(t, d)); d.dirty = true; + } + const batchWords = b => 'Asking for ' + Math.max(1, b.at) + ' of ' + count(b.total, 'record') + ' · ' + b.kept + ' kept, ' + b.refused + ' refused'; + function progress(b) { const line = here && here.root && here.root.querySelector('.ann-form .ann-batch-line'); const text = batchWords(b); if (line && line.textContent !== text) line.textContent = text; } + /** Suggest for this page: each record on screen with no annotation and no typed draft, one after another, kept as a draft. */ + async function suggestPage(t) { + if (sug.batch || sug.asking || !here.page) return; + const todo = here.page.records.filter(r => r.in_product && !r.annotation && chosen(t, r.kind) && !(drafts.get(draftKey(t, r)) || {}).dirty); + const b = sug.batch = { task: t.task, total: todo.length, at: 0, kept: 0, refused: 0, first: null, stop: false }; + sug.said.delete(t.task); rebox('ann:suggest-stop'); + for (const r of todo) { + if (b.stop) break; + b.at++; progress(b); + const o = chosen(t, r.kind); + try { + const got = await P.call('suggest_annotation', { task: t.task, record: r.id, model: o.model, profile: o.profile }); + if (b.stop) break; // stopped while it was asked: the answer is dropped + await P.call('set_annotation', { task: t.task, from: got.suggestion.from, items: [{ record: r.id, state: 'draft', value: got.suggestion.value }] }); + b.kept++; + P.forgetBut(P.WATCHED); here.act.refresh(); // the grid shows the new draft + } catch (error) { b.refused++; if (!b.first) b.first = r.id + ': ' + error.message; } + progress(b); + } + sug.batch = null; + sug.said.set(t.task, (b.stop ? 'Stopped. ' : 'Done. ') + 'Kept ' + count(b.kept, 'draft') + ', ' + b.refused + ' refused, of ' + count(b.total, 'record') + ' with no annotation' + (b.first ? '. First refused: ' + b.first : '.')); + rebox('ann:suggest-page'); + } + + // ---- Keys -------------------------------------------------------------------------------------------------------- + function onKey(e) { + if (e.defaultPrevented || e.altKey || e.ctrlKey || e.metaKey || !here || !here.root || !here.root.isConnected || !here.rec) return; + const t = e.target, choice = !!(t && t.matches && t.matches('input[type="radio"], input[type="checkbox"]')); // a term or a tag: its keys are the form's but the arrows + if (t && t.closest && t.closest('#aside, #needs-panel, dialog, .session-bar, audio, .turns, .ann-surface')) return; + if (!choice && t && t.closest && t.closest('input, select, textarea, [contenteditable="true"]')) return; + if (e.key === 'ArrowLeft' || e.key === 'ArrowRight') { if (choice) return; e.preventDefault(); if (move(e.key === 'ArrowRight' ? 1 : -1, !!(t.classList && t.classList.contains('rec')))) redraw(); return; } + if (e.key === 'Enter') { if (t.closest && t.closest('button, a, summary, [role="button"]') && !t.closest('.ann-decide')) return; const b = here.parts.form && here.parts.form.querySelector('[data-key="annotate:form:keep"]'); if (b && !b.disabled) { e.preventDefault(); b.click(); } return; } + if (/^[1-9]$/.test(e.key) && ['class', 'tags'].includes(here.t.type)) { + const box = here.parts.form && here.parts.form.querySelector('[data-key="ann:term-' + (Number(e.key) - 1) + '"]'); + if (!box || box.disabled) return; + e.preventDefault(); if (here.t.type === 'class') box.checked = true; else box.checked = !box.checked; + box.dispatchEvent(new Event('change')); + } + } + window.addEventListener('keydown', onKey); + + // ---- Forms in the drawer: a new task, an import, a freeze ------------------------------------------------------- + /** A form in the drawer for sel: drawn once, and again only after a control's answer, from what its slot keeps. */ + function formDrawer(aside, sel, load, build) { + if (aside._for === sel && aside._again !== 'control') return; + const same = aside._for === sel && !!aside.firstChild, mine = turn(aside); + aside._for = sel; aside._editing = null; + if (!same) { aside.textContent = ''; aside.appendChild(h('p', 'note', 'Reading…')); } + load().then(value => { if (aside._turn !== mine || aside._for !== sel) return; const ins = h('div', 'inspector ann-new'); build(ins, value); putDrawer(aside, ins); }, + error => { if (aside._turn !== mine || aside._for !== sel) return; putDrawer(aside, A.refusal(error)); }); + } + const slug = text => text.toLowerCase().normalize('NFKD').replace(/[̀-ͯ]/g, '').replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 64).replace(/-+$/, ''); + /** A labelled field whose value lives in the slot s under name. */ + function input(s, key, name, label, { type = 'text', hint, max, placeholder, area, value, onInput } = {}) { + const l = h('label'); l.appendChild(document.createTextNode(label)); + const i = area ? h('textarea') : h('input'); if (!area) i.type = type; else i.rows = area; + if (max) i.maxLength = max; if (placeholder) i.placeholder = placeholder; i.autocomplete = 'off'; i.spellcheck = false; + i.dataset.key = key + ':field:' + name; i.value = s.text[name] != null ? s.text[name] : value || ''; + if (s.text[name] == null && value) s.text[name] = value; + i.addEventListener('input', () => { s.text[name] = i.value; if (onInput) onInput(i.value); }); + l.appendChild(i); if (hint) l.appendChild(h('span', 'hint', hint)); return l; + } + function choice(s, key, name, label, options, fallback, onChange) { + const l = h('label'); l.appendChild(document.createTextNode(label)); + const sel = h('select'); sel.dataset.key = key + ':field:' + name; + if (s.text[name] == null || !options.some(o => o[0] === s.text[name])) s.text[name] = fallback; + for (const [v, text] of options) { const o = h('option', null, text); o.value = v; o.selected = v === s.text[name]; sel.appendChild(o); } + sel.addEventListener('change', () => { s.text[name] = sel.value; if (onChange) onChange(sel.value); }); + l.appendChild(sel); return l; + } + const lines = text => (text || '').split('\n').map(x => x.trim()).filter(Boolean); + const header = (ins, kicker, title, lead) => { const hd = h('header'); hd.appendChild(h('span', 'kicker', kicker)); const t2 = h('h2', null, title); t2.tabIndex = -1; hd.appendChild(t2); ins.appendChild(hd); if (lead) ins.appendChild(h('p', 'note', lead)); }; + + /** New task: title, name, dataset, task and vocabulary; kept as a new annotations/ definition. */ + function taskForm(aside, act) { + formDrawer(aside, 'annotate:new-task', () => P.read('definitions', { kind: 'dataset' }), (ins, defs) => { + const key = 'newtask', s = A.slot(key), sets = defs.definitions.filter(d => !d.removed); + header(ins, 'new annotation task', 'A new task', 'A task says what is recorded about the records of one dataset. It is kept as a new definition, an unsaved edit, until Save writes it to annotations/.json.'); + const f = h('div', 'form'); + let nameBox = null; + f.appendChild(input(s, key, 'title', 'Title (required)', { max: 200, placeholder: 'What is recorded', onInput: () => { if (!s.text.nameTyped && nameBox) { nameBox.value = slug(s.text.title || ''); s.text.name = nameBox.value; } } })); + const nl = input(s, key, 'name', 'Name', { max: 120, hint: 'a to z, 0 to 9, . _ -; the file is annotations/.json', placeholder: 'made from the title', onInput: v => { s.text.nameTyped = !!v; } }); + nameBox = nl.querySelector('input'); if (!s.text.nameTyped) { nameBox.value = slug(s.text.title || ''); s.text.name = nameBox.value; } + f.appendChild(nl); + if (!sets.length) f.appendChild(h('p', 'failure', 'The project has no dataset: a task annotates the records of one.')); + f.appendChild(choice(s, key, 'dataset', 'Dataset', sets.map(d => [d.reference, d.title + ' · ' + d.reference]), sets.length ? sets[0].reference : '')); + const vocab = input(s, key, 'vocabulary', 'Vocabulary: one term per line', { area: 5, hint: '1 to 500 different terms of 1 to 200 characters', placeholder: 'clean\nnoisy\nbroken' }); + const points = input(s, key, 'keypoints', 'Keypoints: one name per line', { area: 4, hint: 'the points of one instance, in the order its values list them', placeholder: 'head\ntail' }); + const units = input(s, key, 'units', 'Units of depth', { max: 40, placeholder: 'm' }); + const mode = choice(s, key, 'mask', 'A pixel\'s value is', [['semantic', 'its class (semantic)'], ['instance', 'one object (instance)']], 'semantic'); + const showVocab = type => { vocab.hidden = !NEEDS_TERMS.includes(type); points.hidden = type !== 'keypoints'; units.hidden = type !== 'depth'; mode.hidden = type !== 'mask'; }; + f.appendChild(choice(s, key, 'task', 'Task', TYPES.map(x => [x, x + ' · ' + TYPE_WORD[x]]), 'class', showVocab)); + f.appendChild(vocab); f.appendChild(points); f.appendChild(units); f.appendChild(mode); showVocab(s.text.task); + ins.appendChild(f); + const reference = () => 'annotations/' + (s.text.name || '').trim(); + ins.appendChild(A.group(key, [ + { id: 'keep', label: 'Keep', cls: 'btn primary', who: 'edit', op: 'set_definition', busy: 'Keeping…', + args: st => { + const doc = { format: FORMAT, kind: 'annotation', title: (st.text.title || '').trim(), dataset: st.text.dataset, task: st.text.task }; + if (NEEDS_TERMS.includes(st.text.task)) doc.vocabulary = lines(st.text.vocabulary); + if (st.text.task === 'keypoints') doc.keypoints = lines(st.text.keypoints); + if (st.text.task === 'depth') doc.units = (st.text.units || '').trim(); + if (st.text.task === 'mask' && st.text.mask === 'instance') doc.mask = 'instance'; + return { reference: reference(), text: textOf(doc) }; + }, + done: r => { ui.task = r.reference; ui.page = 0; ui.filter = 'all'; act.select(null); } }, + { id: 'back', label: 'Cancel', cls: 'btn ghost', onClick: () => act.select(null) } + ])); + ins.appendChild(h('p', 'note', 'Keep checks the definition and keeps it in Tuldok as an unsaved edit; Save writes it to the project folder.')); + }); + } + + /** The Data tab's two forms (spec-d4 section 7): Import and Freeze, each kept as a new datasets/ definition. */ + function datasetForm(aside, which, act) { + const freeze = which === 'freeze', key = 'new' + which, sel = 'data:new:' + which; + if (!['import', 'freeze'].includes(which)) { aside.textContent = ''; aside._for = sel; aside.appendChild(h('p', 'empty', 'There is no form ' + which + ' here.')); return; } + formDrawer(aside, sel, () => Promise.all([P.read('definitions', { kind: 'dataset' }), freeze ? P.annotations.tasks() : null]), (ins, [defs, tasks]) => { + const s = A.slot(key), sets = defs.definitions.filter(d => !d.removed); + header(ins, freeze ? 'new dataset · freeze' : 'new dataset · import', freeze ? 'Freeze a dataset' : 'Import files', + freeze ? 'A dataset with splits made from another dataset\'s records and their annotations: the records the first task includes, ordered by the seed and cut by the shares. It reads the tasks\' annotations as saved and committed.' + : 'A dataset of files on this machine: each file under the source (or each line, with lines) becomes a record. A source relative to the code root is read from the commit a build uses.'); + const f = h('div', 'form'); + let nameBox = null; + f.appendChild(input(s, key, 'title', 'Title (required)', { max: 200, onInput: () => { if (!s.text.nameTyped && nameBox) { nameBox.value = slug(s.text.title || ''); s.text.name = nameBox.value; } } })); + const nl = input(s, key, 'name', 'Name', { max: 120, hint: 'the file is datasets/.json', placeholder: 'made from the title', onInput: v => { s.text.nameTyped = !!v; } }); + nameBox = nl.querySelector('input'); if (!s.text.nameTyped) { nameBox.value = slug(s.text.title || ''); s.text.name = nameBox.value; } + f.appendChild(nl); + if (!freeze) { + f.appendChild(input(s, key, 'source', 'Source (required)', { hint: 'a directory: relative to the code root, or absolute (this machine only)', placeholder: 'data/photos' })); + f.appendChild(input(s, key, 'pattern', 'Pattern', { hint: 'which files; empty: every file (**/*)', placeholder: '**/*.png' })); + f.appendChild(choice(s, key, 'kind', 'Kind', [['', 'By each file\'s extension']].concat(['text', 'image', 'audio', 'pointcloud', 'mesh', 'array', 'file'].map(k => [k, 'Every file is ' + (V.WORD[k] || k).toLowerCase()])), '')); + f.appendChild(input(s, key, 'rights', 'Rights (required)', { area: 2, max: 1000, hint: 'what may be done with the files', placeholder: 'Photographed by us; free to use.' })); + const lab = h('label', 'check'), box = h('input'); box.type = 'checkbox'; box.checked = !!s.text.lines; box.dataset.key = key + ':field:lines'; + box.addEventListener('change', () => { s.text.lines = box.checked; }); lab.appendChild(box); lab.appendChild(document.createTextNode(' Each line of a text file is a record of its own')); f.appendChild(lab); + } else { + const ofTasks = h('div', 'ann-field'); + const drawTasks = ds => { + const mine = tasks.tasks.filter(x => x.dataset === ds); + if (!Array.isArray(s.text.tasks) || s.text.tasksOf !== ds) { s.text.tasks = mine.length ? [mine[0].task] : []; s.text.tasksOf = ds; } // the dataset's first task, until the person chooses + const box = h('div'); box.appendChild(h('h3', null, 'Its tasks')); + if (!mine.length) box.appendChild(h('p', 'failure', ds ? ds + ' has no annotation task: a freeze takes its records by one.' : 'Choose the dataset first.')); + for (const m of mine) { + const lab = h('label', 'check'), c = h('input'); c.type = 'checkbox'; c.checked = s.text.tasks.includes(m.task); c.dataset.key = key + ':field:task-' + m.task; + c.addEventListener('change', () => { s.text.tasks = c.checked ? s.text.tasks.concat([m.task]) : s.text.tasks.filter(x => x !== m.task); drawTasks(ds); }); + lab.appendChild(c); lab.appendChild(document.createTextNode(' ' + m.title + ' · ' + m.task + (s.text.tasks[0] === m.task ? ' (first: its states take a record in)' : ''))); box.appendChild(lab); + } + putInPlace(ofTasks, box); + }; + const named = h('div', 'ann-field'); + // Records chosen on a map of embeddings (curation.js, spec-t5 section 5): only those are frozen. + const drawNamed = ds => { + const box = h('div'); + if (Array.isArray(s.text.records) && s.text.recordsOf === ds) { + box.appendChild(h('p', 'small frz-records', 'Only the ' + s.text.records.length.toLocaleString('en-GB') + ' records chosen on the map: ' + s.text.records.slice(0, 3).join(', ') + (s.text.records.length > 3 ? ', …' : '') + '.')); + const b = h('button', 'btn sm ghost', 'Freeze every record instead'); b.type = 'button'; b.dataset.key = key + ':field:records-clear'; + b.addEventListener('click', () => { s.text.records = null; drawNamed(ds); }); box.appendChild(b); + } + putInPlace(named, box); + }; + f.appendChild(choice(s, key, 'dataset', 'The dataset whose records are frozen', sets.map(d => [d.reference, d.title + ' · ' + d.reference]), sets.length ? sets[0].reference : '', ds => { drawTasks(ds); drawNamed(ds); })); + f.appendChild(named); drawNamed(s.text.dataset); + f.appendChild(ofTasks); drawTasks(s.text.dataset); + const two = h('div', 'two'); + two.appendChild(input(s, key, 'seed', 'Seed (required)', { type: 'number', value: '7', hint: 'an integer' })); + two.appendChild(input(s, key, 'shares', 'Shares (required)', { value: 'train 8, validation 1, test 1', hint: 'each split and its relative size' })); + f.appendChild(two); + const st = h('div', 'ann-field'); st.appendChild(h('h3', null, 'Which states of the first task take a record in')); + if (!Array.isArray(s.text.include)) s.text.include = ['accepted']; + for (const [v, label] of [['accepted', 'Accepted'], ['needs-work', 'Needs work'], ['rejected', 'Rejected'], ['draft', 'Draft']]) { + const lab = h('label', 'check'), c = h('input'); c.type = 'checkbox'; c.checked = s.text.include.includes(v); c.dataset.key = key + ':field:include-' + v; + c.addEventListener('change', () => { s.text.include = c.checked ? s.text.include.concat([v]) : s.text.include.filter(x => x !== v); }); + lab.appendChild(c); lab.appendChild(document.createTextNode(' ' + label)); st.appendChild(lab); + } + f.appendChild(st); + } + ins.appendChild(f); + const reference = () => 'datasets/' + (s.text.name || '').trim(); + ins.appendChild(A.group(key, [ + { id: 'keep', label: 'Keep', cls: 'btn primary', who: 'edit', op: 'set_definition', busy: 'Keeping…', args: st => ({ reference: reference(), text: textOf(freeze ? freezeDoc(st.text) : importDoc(st.text)) }), + done: r => { act.select(r.reference); } }, + { id: 'back', label: 'Cancel', cls: 'btn ghost', onClick: () => act.select(null) } + ])); + ins.appendChild(h('p', 'note', 'Keep checks the definition and keeps it in Tuldok as an unsaved edit; Save writes it. Building it is planned and requested as any build is.')); + }); + } + /** The definition an import writes (docs/contracts/project-folder.md, Steps Tuldok provides). */ + function importDoc(x) { + const settings = { source: (x.source || '').trim() }; + if ((x.pattern || '').trim()) settings.pattern = x.pattern.trim(); + if (x.kind) settings.kind = x.kind; + settings.rights = (x.rights || '').trim(); + if (x.lines) settings.lines = true; + return { format: FORMAT, kind: 'dataset', title: (x.title || '').trim(), run: { step: 'import', settings } }; + } + /** The definition a freeze writes; shares are read from "train 8, validation 1, test 1". */ + function freezeDoc(x) { + const shares = {}; + for (const part of String(x.shares || '').split(',').map(p => p.trim()).filter(Boolean)) { + const m = /^(\S+)\s*[ :=]\s*(\d+)$/.exec(part); + if (!m) throw Object.assign(Error('Shares are written as names and whole numbers, for example "train 8, validation 1, test 1"; "' + part + '" is not.'), { code: 'invalid' }); + shares[m[1]] = Number(m[2]); + } + const seed = Number(String(x.seed || '').trim()); + if (!Number.isInteger(seed) || String(x.seed || '').trim() === '') throw Object.assign(Error('The seed is a whole number.'), { code: 'invalid' }); + const settings = { seed, shares }; + if (Array.isArray(x.records) && x.records.length && x.recordsOf === x.dataset) settings.records = x.records.slice(); + const include = (x.include || []).slice().sort((a, b) => ['accepted', 'needs-work', 'rejected', 'draft'].indexOf(a) - ['accepted', 'needs-work', 'rejected', 'draft'].indexOf(b)); + if (!(include.length === 1 && include[0] === 'accepted')) settings.include = include; + return { format: FORMAT, kind: 'dataset', title: (x.title || '').trim(), run: { step: 'freeze', inputs: { records: x.dataset }, annotations: (x.tasks || []).slice(), settings } }; + } + + /** The Freeze form filled in with records chosen on a map (curation.js): the dataset and only those records. */ + function fillFreeze(dataset, records) { const s = A.slot('newfreeze'); s.text.dataset = dataset; s.text.records = records.slice(); s.text.recordsOf = dataset; s.text.tasks = null; } + + /** Needs you: the Annotate tab opens on this task. */ + function choose(task) { ui.task = task; ui.page = 0; ui.filter = 'all'; } + + S.annotate = annotate; + // The drawer forms' parts, for the Generate form (generate.js). + const forms = { formDrawer, input, choice, header, slug, textOf }; + return { choose, datasetForm, forms, fillFreeze, redraw }; +})(); diff --git a/static/project-workspace/app.js b/static/project-workspace/app.js new file mode 100644 index 00000000..044658dd --- /dev/null +++ b/static/project-workspace/app.js @@ -0,0 +1,333 @@ +'use strict'; +// The shell of the project workspace (the mockup's b-map.html script): tabs, +// routing by hash (#/), the status strip and Needs you, the +// drawer, and when the project is read. The first screen reads summary and +// map; the strip's other sources are read right after; a tab reads what it +// needs when it is first shown. The page reads again in one way (reread(), +// spec-w4.md): on its timer (every two seconds while a build is queued or +// running, every fifteen otherwise), when the window regains focus and after +// a control's answer (actions.js), asking again for what it watches (summary, +// status, build requests, environments, runs, and the map except on the +// timer unless a run or request ended or status changed). When no answer +// differs nothing is drawn again; only the progress of a running run (the +// strip's and its row's in Train's list) and the minute of "Read at" are set +// in place. When one does, what is not watched is forgotten and the strip, +// the tab and the open drawer are drawn again in place, keeping the scroll +// and the focus, so a model turns to built without a reload. The open run +// reads on its progress and logs on every reading. A reading that fails +// leaves the page as it is and the strip says so; a read refused conflict +// shows the conflict panel. +(() => { + const P = window.TuldokProject, S = window.LabScreens, A = window.LabActions; + const TABS = [['map', 'Map'], ['data', 'Data'], ['annotate', 'Annotate'], ['train', 'Train'], ['sweeps', 'Sweeps'], ['benchmarks', 'Benchmarks'], ['compare', 'Compare'], ['analysis', 'Analysis'], ['overview', 'Overview']]; + const POLL_MS = 2000, IDLE_MS = 15000; // how often the page asks again while something builds, and while nothing does + const ACTIVE_RUN = ['queued', 'running'], ACTIVE_REQUEST = ['queued', 'running']; + const $ = id => document.getElementById(id); + const state = { screen: 'map', sel: {}, kinds: null, query: '', zoom: 1, track: false, ui: {}, benchmark: null, cmp: [] }; + const data = { summary: null, map: null, standing: null, needs: null, status: null, readAt: '', polling: false }; + let failed = null, conflict = null, pollTimer = null, pollFast = false, states = null, opener = null; + const clock = () => new Date().toLocaleTimeString('en-GB', { hour: '2-digit', minute: '2-digit' }); + + function parse() { + const hash = location.hash.replace(/^#/, ''), cut = hash.indexOf('/'); + const screen = cut < 0 ? hash : hash.slice(0, cut), id = cut < 0 ? '' : decodeURIComponent(hash.slice(cut + 1)); + state.screen = TABS.some(t => t[0] === screen) ? screen : 'map'; + if (id) state.sel[state.screen] = id; else delete state.sel[state.screen]; + } + function go(screen, id) { + // What opened a drawer gets the focus back when it closes. + if (id && !state.sel[screen]) opener = document.activeElement && document.activeElement.dataset ? { key: document.activeElement.dataset.key || null, el: document.activeElement } : null; + const closing = !id && state.sel[screen]; + const next = '#' + screen + (id ? '/' + encodeURIComponent(id) : ''); + if (location.hash === next) { parse(); render(); } else location.hash = next; + if (closing && opener) { const o = opener; opener = null; setTimeout(() => { if (o.el && o.el.isConnected) o.el.focus(); else if (o.key) A.focusKey(o.key); }, 0); } + } + const act = { + go, select: id => go(state.screen, id || undefined), + setQuery: q => { state.query = q; render(); }, + zoom: d => { state.zoom = Math.round(Math.min(1.6, Math.max(0.5, (state.zoom || 1) + d)) * 10) / 10; render(); }, + toggleKind: (k, all) => { if (!state.kinds) state.kinds = new Set([k]); else if (state.kinds.has(k)) { state.kinds.delete(k); if (!state.kinds.size) state.kinds = null; } else { state.kinds.add(k); if (state.kinds.size === all.length) state.kinds = null; } render(); }, + toggleTrack: () => { state.track = !state.track; render(); }, + // The other tabs: their own settings (state.ui), and the benchmark and compare set, which the address keeps (?benchmark=, ?cmp=). + setUI: (k, v) => { state.ui[k] = v; if (ADDRESSED.includes(k)) address(); render(); }, + /** Keeps a setting of the tab that a view has already drawn in place (fields.js's slider): into the address, and nothing drawn again. */ + keepUI: (k, v) => { state.ui[k] = v; if (ADDRESSED.includes(k)) address(); }, + // The compare set is the page's, of up to COMPARE_MOST models from any benchmark or the map (spec-t3 section 5.1): another benchmark keeps it. + setBenchmark: name => { state.benchmark = name; address(); render(); }, + toggleCompare: id => { const i = state.cmp.indexOf(id); if (i >= 0) state.cmp.splice(i, 1); else if (state.cmp.length < COMPARE_MOST) state.cmp.push(id); address(); render(); }, + clearCompare: () => { state.cmp = []; address(); render(); }, + /** Reads everything again (a control's answer came, or a view asks for it). */ + refresh: () => changed(), + /** Draws the map again from what is read (the page's drafts changed, editor.js); the drawer is kept. */ + redraw: async () => { try { data.map = await P.map(); } catch (error) { fail(error); } strip(); render(); }, + /** Draws the drawer afresh. */ + reopen: () => { $('aside')._for = null; render(); } + }; + // The settings of the Analysis tab the address keeps, so that a reload opens it where it was (spec-d6 section 3). + // And those of the Training block of a run's drawer (spec-t1 section 6): the runs compared, the series, smoothing, log scale, across. + const ADDRESSED = ['anaBench', 'anaSubject', 'anaMetric', 'anaCase', 'anaAxis', 'anaIndex', 'anaMeanFree', 'trWith', 'trSeries', 'trSmooth', 'trLog', 'trX', 'cmpCase', 'cmpAxis', 'cmpIndex', 'cmpMeanFree', 'sweep']; // sweep: the Sweeps tab's (spec-t7) + const COMPARE_MOST = 4; + /** Writes the benchmark, the compare set and the Analysis tab's settings into the address without reloading, so the view can be shared. */ + function address() { + const q = new URLSearchParams(location.search); + if (state.benchmark) q.set('benchmark', state.benchmark); else q.delete('benchmark'); + if (state.cmp.length) q.set('cmp', state.cmp.join(',')); else q.delete('cmp'); + for (const k of ADDRESSED) { const v = state.ui[k]; if (v === null || v === undefined || v === '' || v === false) q.delete(k); else q.set(k, String(v)); } + const text = q.toString().replace(/%2C/g, ',').replace(/%2F/g, '/'); // references read as they are + history.replaceState(null, '', location.pathname + (text ? '?' + text : '') + location.hash); + } + + /** Rebuilds container with build(), giving back the focused control (by data-key), its caret and the scroll. */ + function keepFocus(container, build) { + const active = document.activeElement, key = active && container.contains(active) && active.dataset ? active.dataset.key : null; + const caret = key && typeof active.selectionStart === 'number' ? active.selectionStart : null, top = container.scrollTop; + build(); + container.scrollTop = top; + if (key) { const again = [...container.querySelectorAll('[data-key]')].find(e => e.dataset.key === key); if (again) { again.focus({ preventScroll: true }); if (caret !== null && again.setSelectionRange) again.setSelectionRange(caret, caret); } } + } + + // ---- The status strip and Needs you --------------------------------------- + // The strip is drawn by parts, each made again only when what it shows differs (spec-w4 R2): a reading + // that changed nothing touches no element of it but the text of "Read at", when its minute changes. + const parts = {}; // by name: { key, el } + const part = (name, key, make) => { const p = parts[name]; if (p && p.key === key) return p.el; const el = make(); parts[name] = { key, el }; return el; }; + const span = (cls, text) => Object.assign(document.createElement('span'), { className: cls, textContent: text }); + const setText = (node, text) => { if (node.data !== text) node.data = text; }; + function hairline(p) { + const hair = span('hair', ''), i = document.createElement('i'); i.style.width = Math.round(100 * p.completed / Math.max(1, p.total)) + '%'; hair.appendChild(i); + hair.title = p.stage + ' ' + p.completed + ' of ' + p.total; hair.setAttribute('role', 'progressbar'); hair.setAttribute('aria-valuemin', '0'); hair.setAttribute('aria-valuemax', String(p.total)); hair.setAttribute('aria-valuenow', String(p.completed)); + return hair; + } + /** The Needs you button: made once, its words, state and aria-expanded set when they differ. */ + function needsButton() { + if (!parts.needs) { + const b = document.createElement('button'); b.className = 'btn sm needs-btn'; b.type = 'button'; b.id = 'needs-btn'; b.dataset.key = 'needs-btn'; b.setAttribute('aria-controls', 'needs-panel'); + b.addEventListener('click', () => { const panel = $('needs-panel'); panel.hidden = !panel.hidden; b.setAttribute('aria-expanded', String(!panel.hidden)); fillNeeds(true); }); + parts.needs = { key: null, el: b }; + } + const b = parts.needs.el, text = 'Needs you · ' + (data.needs ? data.needs.length : '…'), open = String(!$('needs-panel').hidden); + if (b.textContent !== text) b.textContent = text; + if (b.disabled !== !data.needs) b.disabled = !data.needs; + if (b.getAttribute('aria-expanded') !== open) b.setAttribute('aria-expanded', open); + return b; + } + function strip() { + const el = $('strip'), want = []; + if (failed) { want.push(part('state', 'failed ' + !!conflict, () => S.stateEl('serious', conflict ? 'Conflict' : 'Not read'))); want.push(part('msg', 'failed ' + failed, () => span('msg', failed))); } + else if (!data.standing) want.push(part('msg', 'reading', () => span('msg', 'Reading the project…'))); + else { + const { tone, word, text } = data.standing, p = data.progress; + want.push(part('state', tone + ' ' + word, () => S.stateEl(tone, word))); + if (p) want.push(part('hair', [p.stage, p.completed, p.total].join(' '), () => hairline(p))); + const m = part('msg', 'standing', () => { const s = span('msg', ''); s.append(document.createTextNode(''), document.createTextNode('')); return s; }); + setText(m.firstChild, text + (p ? ' ' + p.stage + ': ' + p.completed + ' of ' + p.total + '.' : '')); + setText(m.lastChild, ' Read at ' + data.readAt + '.'); + want.push(m); + const n = data.status ? data.status.unsaved.length + data.status.removed.length : 0, a = P.annotationsUnsaved(data.status); + if (n || a) want.push(part('save', n + ' ' + a + ' ' + A.may('edit'), () => A.stripSave(data.status, $('strip-note')))); + else if ($('strip-note').firstChild) $('strip-note').textContent = ''; + want.push(needsButton()); + } + const now = [...el.children]; + if (now.length !== want.length || want.some((e, i) => now[i] !== e)) keepFocus(el, () => { + want.forEach((e, i) => { if (el.children[i] !== e) el.insertBefore(e, el.children[i] || null); }); + while (el.children.length > want.length) el.lastElementChild.remove(); + }); + fillNeeds(); + } + /** Places Needs you sends to: the panel closes and the focus goes to the drawer that opens. */ + function needsGo(screen, id) { const panel = $('needs-panel'); panel.hidden = true; const b = $('needs-btn'); if (b) b.setAttribute('aria-expanded', 'false'); opener = { key: 'needs-btn', el: null }; go(screen, id); } + /** Needs you, when open: drawn again when what it lists (or who may use its controls) differs, or when it is opened (force). */ + function fillNeeds(force) { + const panel = $('needs-panel'); if (panel.hidden || !data.needs) return; + const key = JSON.stringify([data.needs.map(n => [n.kind, n.key || '', n.text, n.references || null]), (window.TuldokAuth && window.TuldokAuth.identity()) || null]); + if (!force && panel._key === key) return; + panel._key = key; + keepFocus(panel, () => { panel.textContent = ''; panel.appendChild(S.needsList(data.needs, n => A.needControls(n, needsGo))); }); + } + + // ---- Rendering -------------------------------------------------------------- + /** + * Draws the tab on screen. The tab bar is built once (start()); this sets + * which link is current. The drawer is drawn again in place when app.js + * marked it (aside._again: 'data' after a reading that changed something, + * 'control' after a control's answer, which the editor and the branch form + * take too); otherwise a drawer already showing its subject is left alone. + */ + function render() { + for (const a of $('tabs').children) { const on = a.dataset.tab === state.screen; if (on !== a.hasAttribute('aria-current')) { if (on) a.setAttribute('aria-current', 'page'); else a.removeAttribute('aria-current'); } } + const main = $('main'), aside = $('aside'), drawer = $('drawer'); + const name = data.summary ? data.summary.name : ''; + document.title = TABS.find(t => t[0] === state.screen)[1] + (name ? ' · ' + name : '') + ' · Tuldok'; + if (failed || !data.map) { + main.className = 'content'; main.textContent = ''; main._tree = null; main._view = null; main._painted = null; + if (conflict) A.conflictPanel(main, conflict); + else { const p = document.createElement('p'); p.className = failed ? 'failure' : 'note'; p.textContent = failed || 'Reading the project…'; main.appendChild(p); } + drawer.classList.remove('open'); drawer.setAttribute('aria-hidden', 'true'); main.classList.remove('with-drawer'); aside.textContent = ''; aside._for = null; aside._again = false; + return; + } + const wasMap = main.classList.contains('map'), view = state.screen + '/' + (state.screen === 'overview' && state.sel.overview === 'report' ? 'report' : ''); // a selection opens the drawer and keeps the place; the report is its own view + main.className = state.screen === 'map' ? 'map' : 'content'; + if (state.screen !== 'map' && wasMap) main._tree = null; + if (main._view !== view) { main.scrollTop = 0; main.scrollLeft = 0; main._view = view; } // a new screen starts at its top + const has = S[state.screen](main, aside, state, act, data); + aside._again = false; + drawer.classList.toggle('open', !!has); drawer.setAttribute('aria-hidden', String(!has)); main.classList.toggle('with-drawer', !!has); + if (!has) aside._for = null; + } + function fail(error) { failed = error.message; conflict = error.code === 'conflict' ? error : null; } + + // ---- Reading ------------------------------------------------------------------ + const WATCH = P.WATCHED.filter(n => n !== 'map'); // asked again on every reading; the map too on some (step()) + async function first() { + try { + data.summary = await P.read('summary'); + $('projname').textContent = data.summary.name; + if (window.LabEditor) window.LabEditor.load(data.summary.project_id); // the drafts of this project, kept for this tab + data.map = await P.map(); + data.standing = P.standing(data.map, null, null); data.readAt = clock(); failed = null; conflict = null; + } catch (error) { fail(error); } + strip(); render(); + if (failed) return; + const read = await sources(); + if (read.error && read.error.code === 'conflict') { fail(read.error); strip(); render(); return; } + ended(read); await stand(read); + strip(); + if (state.screen === 'overview') render(); // it shows where the project stands and Needs you + } + /** The strip's other sources, as kept: status, build requests, environments and runs. */ + async function sources() { + try { const [status, requests, environments, runs] = await Promise.all([P.read('status'), P.read('build_requests'), P.read('environments'), P.read('runs')]); return { status, requests: requests.requests, environments: environments.environments, runs: runs.runs }; } + catch (error) { return { error }; } + } + /** Whether a run or a build request that was active is no more (or a run came and went) since the last reading. */ + function ended(read) { + if (read.error) return false; + const now = new Map(read.runs.map(r => ['run:' + r.id, r.state]).concat(read.requests.map(r => ['request:' + r.id, r.state]))); + let gone = false; + if (states) for (const [key, st] of now) { + const was = states.get(key); + if (key.startsWith('run:') ? (was === undefined || ACTIVE_RUN.includes(was)) && !ACTIVE_RUN.includes(st) : ACTIVE_REQUEST.includes(was) && !ACTIVE_REQUEST.includes(st)) gone = true; + } + states = now; + return gone; + } + /** Where the project stands, Needs you and the running run's progress, from the sources; asks again later (in two seconds while something builds). */ + async function stand(read) { + try { + if (read.error) throw read.error; + data.status = read.status; + data.standing = P.standing(data.map, read.runs, read.requests); + data.needs = P.needs({ summary: data.summary, status: read.status, requests: read.requests, environments: read.environments, runs: read.runs, shaped: data.map }); + // The latest progress of each running run: the first is the strip's, all are set into Train's rows (tabs.js's S.progress). + const running = read.runs.filter(r => r.state === 'running'); + data.progressOf = new Map(await Promise.all(running.map(r => P.call('progress', { run: r.id }).then(p => [r.id, p.event], () => [r.id, null])))); + data.progress = running.length ? data.progressOf.get(running[0].id) : null; + data.readAt = clock(); + later(read.runs.some(r => ACTIVE_RUN.includes(r.state)) || read.requests.some(r => ACTIVE_REQUEST.includes(r.state))); + } catch (error) { + unread(error); + } + } + /** A reading that failed (not a conflict): what is on screen stays, the strip says the project was not read and why, and it is asked again later. */ + function unread(error) { + data.progress = null; data.progressOf = new Map(); + data.standing = { tone: 'serious', word: 'Not read', text: P.standing(data.map, null, null).text + ' The last reading failed: ' + error.message.replace(/\.$/, '') + '.' }; + later(false); + } + /** What a reading sets in place in the main area whether or not anything changed: Overview's "Read at" and the progress of the running runs in Train's list. */ + function current() { + const at = $('main').querySelector('[data-read-at]'), text = 'Read at ' + data.readAt; + if (at && at.firstChild) setText(at.firstChild, text); + if (S.progress && data.progressOf) S.progress(data.progressOf); + } + /** Asks again later: in two seconds while something builds, otherwise in fifteen, so a request an agent makes or an edit made elsewhere shows without the window losing and regaining focus. */ + function later(busy) { + if (pollTimer && busy && !pollFast) { clearTimeout(pollTimer); pollTimer = null; } + if (!pollTimer) { pollFast = busy; pollTimer = setTimeout(() => { pollTimer = null; reread('timer'); }, busy ? POLL_MS : IDLE_MS); } + } + /** + * The one way of reading again (spec-w4 R1 to R3), for a cause: 'timer', + * 'focus' (the window regained it, or a control was refused) or 'control' + * (a control's answer, which always counts as a change). One reading at a + * time: a cause arriving meanwhile waits, the strongest of those waiting kept. + */ + const RANK = { timer: 0, focus: 1, control: 2 }; + let reading = null, waiting = null, owed = false; // owed: a control was answered and the reading after it failed, so the next one counts as a change + function reread(cause, info) { + if (!reading) { reading = step(cause, info).catch(error => console.error(error)).finally(() => { reading = null; }); return reading; } + if (!waiting) { const w = waiting = { cause, info }; w.done = reading.then(() => { waiting = null; return reread(w.cause, w.info); }); } + else if (RANK[cause] > RANK[waiting.cause] || (cause === waiting.cause && info)) Object.assign(waiting, { cause, info }); + return waiting.done; + } + /** + * One reading: asks again for what the page watches (the map too, except + * on the timer unless a run or request ended or status changed). Nothing + * changed: only the strip's parts that differ (the progress, the minute of + * "Read at") and the same two in the main area are touched. A reading that + * failed touches nothing but the strip, which says so. Something changed: what is not watched is + * forgotten, and the strip, the tab and the open drawer are drawn again in + * place (a drawer holding the editor or the branch form only after a + * control). The open run reads on (tabs.js's S.tick). + */ + async function step(cause, info) { + if (failed) { if (cause === 'timer') return; P.forgetAll(); await first(); return settleFocus(info); } + const control = cause === 'control'; + let changed; + try { + changed = await P.again(cause === 'timer' ? WATCH : P.WATCHED); + if (changed.includes('map')) data.map = await P.map(); + } catch (error) { return notRead(error, control); } + if (changed.includes('summary')) try { data.summary = await P.read('summary'); } catch (error) { /* the strip says what failed */ } + const read = await sources(); + if (read.error && read.error.code === 'conflict') return notRead(read.error, control); + if (cause === 'timer' && (ended(read) || changed.includes('status'))) { + try { if ((await P.again(['map'])).length) { changed = changed.concat(['map']); data.map = await P.map(); } } catch (error) { return notRead(error, control); } + } else if (cause !== 'timer') ended(read); + await stand(read); + const anything = control || owed || changed.length > 0; owed = false; + if (anything) P.forgetBut(P.WATCHED); + strip(); + if (anything) { $('aside')._again = control ? 'control' : 'data'; render(); } else current(); + if (S.tick) await S.tick(); + if (control) settleFocus(info); + } + /** A reading that could not be made: a conflict shows the conflict panel; anything else leaves the page as it is and says so in the strip. */ + function notRead(error, control) { + if (error.code === 'conflict') { fail(error); strip(); render(); return; } + if (control) owed = true; + unread(error); strip(); + } + /** After a control's answer: the focus stays on the control when it is still there, else goes to the drawer's heading. */ + function settleFocus(info) { + if (info && info.focus && (!document.activeElement || document.activeElement === document.body)) { + if (!A.focusKey(info.focus)) setTimeout(() => { if (document.activeElement === document.body) { const t = $('aside').querySelector('h2') || $('main'); if (t.tabIndex < 0 && t !== $('main')) t.tabIndex = -1; t.focus(); } }, 300); + } + } + const changed = info => reread('control', info); + A.connect({ changed, refused: () => reread('focus'), reread: () => reread('control') }); + + function start() { + // auth.js puts its session controls first in the body; they belong at the right of the top bar. + const session = document.querySelector('body > .session-bar'); if (session) $('top-right').appendChild(session); + S.theme.init($('theme')); + $('close').appendChild(S.icon('x')); + $('close').addEventListener('click', () => go(state.screen)); + window.addEventListener('keydown', e => { + if (e.key !== 'Escape') return; + if (A.escape()) return; + if (window.LabEditor && window.LabEditor.escape()) return; + if (!$('needs-panel').hidden) { $('needs-panel').hidden = true; const b = $('needs-btn'); if (b) { b.setAttribute('aria-expanded', 'false'); b.focus(); } return; } + if ($('drawer').classList.contains('open') && state.sel[state.screen]) go(state.screen); + }); + window.addEventListener('hashchange', () => { parse(); render(); }); + window.addEventListener('focus', () => { if (data.summary || failed) reread('focus'); }); + // Who may use a control comes from the session: draw the controls again when it changes. + window.addEventListener('tuldok-session', () => { if (failed) reread('focus'); else { $('aside')._for = null; if (data.map) act.redraw(); else { strip(); render(); } } }); + { const q = new URLSearchParams(location.search); state.benchmark = q.get('benchmark') || null; state.cmp = (q.get('cmp') || '').split(',').filter(Boolean).slice(0, COMPARE_MOST); for (const k of ADDRESSED) if (q.get(k)) state.ui[k] = q.get(k); } + // The tab bar, once (spec-w4 R5): render() sets which link is current. + for (const [id, label] of TABS) { const a = document.createElement('a'); a.href = '#' + id; a.textContent = label; a.dataset.tab = id; $('tabs').appendChild(a); } + parse(); strip(); render(); first(); + } + if (document.readyState === 'loading') document.addEventListener('DOMContentLoaded', start, { once: true }); else start(); +})(); diff --git a/static/project-workspace/base.css b/static/project-workspace/base.css new file mode 100644 index 00000000..050a810d --- /dev/null +++ b/static/project-workspace/base.css @@ -0,0 +1,1007 @@ +/* The project workspace: tokens, primitives and the shell. Started from the + chosen mockup (docs/plans/project-workspace-redesign/mockups/shared/base.css + and the style block of b-map.html); class names are the mockup's, kinds of + change are named as in the project format. The faces are vendored under + /static/vendor/ubuntu-sans/ with their licence. */ +@font-face { font-family: "Ubuntu Sans"; font-style: normal; font-weight: 100 800; font-stretch: 75% 100%; font-display: swap; src: url("/static/vendor/ubuntu-sans/UbuntuSans[wdth,wght].ttf") format("truetype"); } +@font-face { font-family: "Ubuntu Sans"; font-style: italic; font-weight: 100 800; font-stretch: 75% 100%; font-display: swap; src: url("/static/vendor/ubuntu-sans/UbuntuSans-Italic[wdth,wght].ttf") format("truetype"); } +@font-face { font-family: "Ubuntu Sans Mono"; font-style: normal; font-weight: 400 700; font-display: swap; src: url("/static/vendor/ubuntu-sans/UbuntuSansMono[wght].ttf") format("truetype"); } +@font-face { font-family: "Ubuntu Sans Mono"; font-style: italic; font-weight: 400 700; font-display: swap; src: url("/static/vendor/ubuntu-sans/UbuntuSansMono-Italic[wght].ttf") format("truetype"); } +:root { + color-scheme: light; + --page: #f3f5f7; + --surface: #ffffff; + --surface-2: #eef1f4; + --line: #dfe4ea; + --line-strong: #c5cdd6; + --ink: #1b2330; + --ink-2: #596679; + --ink-3: #8a94a3; + --accent: #2c5fb8; + --accent-ink: #ffffff; + --accent-wash: #e8eef9; + --focus: #2c5fb8; + --good: #237c60; + --warn: #b4700e; + --serious: #b7452b; + --good-wash: #e4f2ec; + --warn-wash: #fbf0dd; + --serious-wash: #f9e6e1; + --k-training-policy: #178f74; + --k-architecture: #2e6fcf; + --k-refinement: #d2652f; + --k-component: #8a94a3; + --mark: #2e6fcf; + --grid: #e6eaef; + /* The colour ramps of the viewers of a product (inside.js): sequential low to high, diverging about zero. */ + --ramp-lo: #e4ebf5; --ramp-hi: #1c5cab; + --ramp-neg: #2e6fcf; --ramp-mid: #efeeeb; --ramp-pos: #d2652f; + /* The runs drawn together on a training chart, in the order they are added (validated: CVD and contrast, light and dark). */ + --run-1: #2e6fcf; --run-2: #d2652f; --run-3: #8a4fbf; --run-4: #178f74; + --font: "Ubuntu Sans", "Noto Sans", system-ui, sans-serif; + --mono: "Ubuntu Sans Mono", "Ubuntu Mono", ui-monospace, monospace; + --radius: 4px; +} +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + color-scheme: dark; + --page: #0f1317; --surface: #171c22; --surface-2: #1f262e; + --line: #2a323c; --line-strong: #3a4552; + --ink: #e6eaf0; --ink-2: #aab3c0; --ink-3: #7c8694; + --accent: #6a9ae8; --accent-ink: #0f1317; --accent-wash: #1d2a40; --focus: #8db4f5; + --good: #4fb68f; --warn: #d9a04a; --serious: #e07a5f; + --good-wash: #15302a; --warn-wash: #342a16; --serious-wash: #3a2320; + --k-training-policy: #22a07f; --k-architecture: #4f8ae0; --k-refinement: #d96f3c; --k-component: #7c8694; + --mark: #4f8ae0; --grid: #222a33; + --ramp-lo: #2a3542; --ramp-hi: #8db4f5; --ramp-neg: #4f8ae0; --ramp-mid: #3a3a37; --ramp-pos: #e07a4a; + --run-1: #4f8ae0; --run-2: #d96f3c; --run-3: #9a72d8; --run-4: #22a07f; + } +} +:root[data-theme="dark"] { + color-scheme: dark; + --page: #0f1317; --surface: #171c22; --surface-2: #1f262e; + --line: #2a323c; --line-strong: #3a4552; + --ink: #e6eaf0; --ink-2: #aab3c0; --ink-3: #7c8694; + --accent: #6a9ae8; --accent-ink: #0f1317; --accent-wash: #1d2a40; --focus: #8db4f5; + --good: #4fb68f; --warn: #d9a04a; --serious: #e07a5f; + --good-wash: #15302a; --warn-wash: #342a16; --serious-wash: #3a2320; + --k-training-policy: #22a07f; --k-architecture: #4f8ae0; --k-refinement: #d96f3c; --k-component: #7c8694; + --mark: #4f8ae0; --grid: #222a33; + --ramp-lo: #2a3542; --ramp-hi: #8db4f5; --ramp-neg: #4f8ae0; --ramp-mid: #3a3a37; --ramp-pos: #e07a4a; + --run-1: #4f8ae0; --run-2: #d96f3c; --run-3: #9a72d8; --run-4: #22a07f; +} + +* { box-sizing: border-box; } +html, body { height: 100%; } +body { + margin: 0; background: var(--page); color: var(--ink); + font: 14px/1.5 var(--font); -webkit-font-smoothing: antialiased; +} +[hidden] { display: none !important; } +h1, h2, h3, h4 { margin: 0; font-weight: 600; letter-spacing: -0.01em; line-height: 1.25; } +h1 { font-size: 20px; } +h2 { font-size: 16px; } +h3 { font-size: 14px; } +p { margin: 0; max-width: 66ch; } +a { color: var(--accent); text-decoration: none; } +a:hover { text-decoration: underline; } +button, input, select { font: inherit; color: inherit; } +button { cursor: pointer; } +.sr-only { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; } +:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; border-radius: 2px; } +.mono { font-family: var(--mono); font-size: 12.5px; } +.num { font-variant-numeric: tabular-nums; } +.muted { color: var(--ink-2); } +.faint { color: var(--ink-3); } +.small { font-size: 12.5px; } + +/* Buttons: quiet by default. */ +.btn { + display: inline-flex; align-items: center; gap: 6px; min-height: 30px; padding: 4px 10px; + border: 1px solid var(--line-strong); border-radius: var(--radius); background: var(--surface); color: var(--ink); +} +.btn:hover { background: var(--surface-2); } +.btn.primary { background: var(--accent); border-color: var(--accent); color: var(--accent-ink); } +.btn.primary:hover { filter: brightness(1.05); } +.btn.ghost { border-color: transparent; background: transparent; } +.btn.ghost:hover { background: var(--surface-2); } +.btn[disabled] { opacity: .5; cursor: not-allowed; } +.btn svg { width: 14px; height: 14px; flex: none; } +.btn.sm { min-height: 26px; padding: 2px 8px; font-size: 13px; } +.btn[aria-pressed="true"] { background: var(--accent-wash); border-color: var(--accent); } +input[type="search"], input[type="text"], select, textarea { + min-height: 30px; padding: 4px 8px; border: 1px solid var(--line-strong); border-radius: var(--radius); + background: var(--surface); color: var(--ink); +} +input::placeholder { color: var(--ink-3); } + +/* Kind and state marks: identity by a dot and a word, never colour alone. + A model without a kind of change has a hollow dot and the words "kind not set". */ +.chip { display: inline-flex; align-items: center; gap: 6px; font-size: 12.5px; color: var(--ink-2); white-space: nowrap; } +.chip::before { content: ""; width: 8px; height: 8px; border-radius: 50%; background: var(--k-component); flex: none; } +.chip.training-policy::before { background: var(--k-training-policy); } +.chip.architecture::before { background: var(--k-architecture); } +.chip.refinement::before { background: var(--k-refinement); } +.chip.component::before { background: var(--k-component); } +.chip.unset::before { background: transparent; box-shadow: inset 0 0 0 1.5px var(--ink-3); } +.chip.unset { color: var(--ink-3); } +.state { display: inline-flex; align-items: center; gap: 6px; font-size: 12.5px; color: var(--ink-2); white-space: nowrap; } +.state svg { width: 14px; height: 14px; flex: none; } +.state.good { color: var(--good); } +.state.warn { color: var(--warn); } +.state.serious { color: var(--serious); } +.state.neutral { color: var(--ink-3); } + +/* Rows: the dense list primitive. */ +.rows { list-style: none; margin: 0; padding: 0; } +.rows > li { border-top: 1px solid var(--line); } +.rows > li:last-child { border-bottom: 1px solid var(--line); } +.row { + display: grid; align-items: center; gap: 12px; width: 100%; padding: 7px 8px; text-align: left; + border: 0; background: transparent; color: inherit; border-radius: 0; +} +button.row:hover { background: var(--surface-2); } +.row[aria-pressed="true"], .row[aria-selected="true"] { background: var(--accent-wash); box-shadow: inset 3px 0 0 var(--accent); } +.row > span { min-width: 0; } +.row > span > .title, .row > span > .sub { display: block; } +.row .title { font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.row .sub { color: var(--ink-2); font-size: 12.5px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + +/* Section heading with a quiet one-line summary. */ +.sect { display: flex; align-items: baseline; gap: 12px; margin: 0 0 10px; } +.sect h2 { flex: none; } +.sect .sum { color: var(--ink-2); font-size: 13px; } +.sect .spacer { flex: 1; } + +/* Definition list for facts. */ +.facts { display: grid; grid-template-columns: max-content 1fr; gap: 4px 16px; margin: 0; font-size: 13px; } +.facts dt { color: var(--ink-2); } +.facts dd { margin: 0; overflow-wrap: anywhere; min-width: 0; } + +/* Notes that need the user, listed not shouted. */ +.needs { list-style: none; margin: 0; padding: 0; } +.needs li { display: grid; grid-template-columns: 18px 1fr; gap: 10px; align-items: start; padding: 8px 0; border-top: 1px solid var(--line); font-size: 13.5px; } +.needs li:last-child { border-bottom: 1px solid var(--line); } +.needs svg { width: 16px; height: 16px; margin-top: 2px; } +.needs .approval svg { color: var(--accent); } +.needs .changed svg { color: var(--warn); } +.needs .missing svg { color: var(--serious); } +.needs .empty { display: block; padding: 8px 0; border: 0; } + +/* Inspector (the drawer). */ +.inspector { display: flex; flex-direction: column; gap: 18px; } +.inspector header { display: flex; flex-direction: column; gap: 6px; } +.inspector header .kicker { font-family: var(--mono); font-size: 12.5px; color: var(--ink-2); overflow-wrap: anywhere; } +.inspector h2 { font-size: 17px; overflow-wrap: anywhere; } +.inspector h3 { font-size: 13px; color: var(--ink-2); font-weight: 600; margin-bottom: 6px; } +.inspector p { font-size: 13.5px; line-height: 1.55; max-width: 60ch; } +.inspector .actions { display: flex; flex-wrap: wrap; gap: 8px; } +.inspector .note { font-size: 12.5px; color: var(--ink-3); } +.ins-meta { display: flex; gap: 12px; flex-wrap: wrap; align-items: center; } +.metrics { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 10px 16px; margin: 0; } +.metrics > div { display: flex; flex-direction: column; min-width: 0; } +.metrics dt { font-size: 12px; color: var(--ink-2); } +.metrics dd { margin: 0; font-size: 16px; font-weight: 500; font-variant-numeric: tabular-nums; overflow-wrap: anywhere; } +.metrics dd small { font-size: 12px; color: var(--ink-3); font-weight: 400; margin-left: 4px; } +.links { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 4px; } +.links button { text-align: left; padding: 6px 8px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); width: 100%; } +.links button:hover { background: var(--surface-2); } +.links .why { display: block; font-size: 12.5px; color: var(--ink-2); margin-top: 2px; } +.empty { color: var(--ink-2); font-size: 13.5px; max-width: 60ch; } + +/* Track bar: one neutral mark on the track's log scale; empty when the model has no value. */ +.mbar { display: inline-block; width: 56px; height: 6px; background: var(--grid); border-radius: 3px; overflow: hidden; vertical-align: middle; } +.mbar i { display: block; height: 100%; background: var(--mark); border-radius: 3px; } +.mbar.none { background: transparent; box-shadow: inset 0 0 0 1px var(--grid); } +.trow[aria-selected="true"] .mbar i { background: var(--accent); } + +/* Chart chrome (shared by the track and the charts of the other tabs). */ +.chart { position: relative; } +.chart svg { display: block; width: 100%; height: auto; } +.chart .axis text { fill: var(--ink-3); font-size: 11px; font-family: var(--font); } +.chart .axis line, .chart .grid line { stroke: var(--grid); stroke-width: 1; shape-rendering: crispEdges; } +.chart .axis .base { stroke: var(--line-strong); } +.chart .label { fill: var(--ink-2); font-size: 12px; paint-order: stroke; stroke: var(--surface); stroke-width: 3px; stroke-linejoin: round; } +.chart .pt { stroke: var(--surface); stroke-width: 2; } +.chart .pt.training-policy { fill: var(--k-training-policy); } +.chart .pt.architecture { fill: var(--k-architecture); } +.chart .pt.refinement { fill: var(--k-refinement); } +.chart .pt.component { fill: var(--k-component); } +.chart .pt.unset { fill: var(--surface); stroke: var(--ink-3); stroke-width: 1.5; } +.chart .pt.dim { opacity: .18; } +.chart .hit { fill: transparent; cursor: pointer; } +.chart .hit:focus-visible { outline: none; } +.chart .hit:focus-visible + .pt, .chart .hit:hover + .pt { stroke: var(--ink); } +.chart .sel { fill: none; stroke: var(--accent); stroke-width: 2; } +.chart .frontier { fill: none; stroke: var(--ink-3); stroke-width: 1.5; stroke-linejoin: round; } +.chart .line { fill: none; stroke: var(--mark); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +.chart .cross { stroke: var(--line-strong); stroke-width: 1; } +.tip { + position: absolute; z-index: 5; pointer-events: none; min-width: 160px; max-width: 280px; padding: 8px 10px; + background: var(--surface); border: 1px solid var(--line-strong); border-radius: var(--radius); font-size: 12.5px; color: var(--ink-2); + overflow-wrap: anywhere; +} +.tip b { display: block; color: var(--ink); font-size: 13px; font-weight: 600; } +.tip .v { color: var(--ink); font-weight: 500; font-variant-numeric: tabular-nums; } +.legend { display: flex; flex-wrap: wrap; gap: 14px; margin: 8px 0 0; padding: 0; list-style: none; font-size: 12.5px; color: var(--ink-2); } +.filters { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin-bottom: 14px; } +.filters .grp { display: inline-flex; gap: 4px; flex-wrap: wrap; } +.filters .pop { margin-left: auto; font-size: 12.5px; color: var(--ink-3); } +table.tv { border-collapse: collapse; width: 100%; font-size: 13px; } +table.tv th, table.tv td { text-align: left; padding: 6px 8px; border-top: 1px solid var(--line); vertical-align: top; } +table.tv th { color: var(--ink-2); font-weight: 500; } +table.tv td.n, table.tv th.n { text-align: right; font-variant-numeric: tabular-nums; } +table.tv td.ck { font-family: var(--mono); font-size: 12.5px; color: var(--ink-2); } +table.tv td.nowrap, table.tv td.ck { white-space: nowrap; } + +/* Lineage list (git-graph style). */ +.tree-list { list-style: none; margin: 0; padding: 0; } +.tree-list .tsect { position: sticky; top: 0; z-index: 2; display: grid; grid-template-columns: var(--graphw, 72px) minmax(0, 1fr) auto; gap: 12px; align-items: center; padding: 6px 8px; background: var(--surface); border-top: 1px solid var(--line); border-bottom: 1px solid var(--line); } +.tree-list .tsect h3 { font-size: 13px; font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.tree-list .tsect h3 .up { color: var(--ink-3); font-weight: 400; } +.tree-list .tsect.loose h3 { color: var(--ink-2); font-style: italic; } +.tree-list .tsect .sum { font-size: 12.5px; color: var(--ink-2); white-space: nowrap; } +.tree-list .tsect svg { display: block; height: 20px; } +.tree-list .trow { display: grid; grid-template-columns: var(--graphw, 72px) var(--ckw, 44px) minmax(150px, 1fr) minmax(0, 1.4fr) 120px 64px 64px; gap: 12px; align-items: center; width: 100%; padding: 0 8px; min-height: 30px; border: 0; background: transparent; color: inherit; text-align: left; } +.tree-list .trow:hover { background: var(--surface-2); } +.tree-list .trow[aria-selected="true"] { background: var(--accent-wash); box-shadow: inset 3px 0 0 var(--accent); } +.tree-list .trow svg { display: block; height: 30px; } +.tree-list .trow .ck { font-family: var(--mono); font-size: 12.5px; color: var(--ink-2); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.tree-list .trow .nm { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.tree-list .trow .pu { font-size: 12.5px; color: var(--ink-3); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +@media (max-width: 1100px) { .tree-list .trow { grid-template-columns: var(--graphw, 72px) var(--ckw, 44px) minmax(120px, 1fr) 120px 64px 64px; } .tree-list .trow .pu { display: none; } } +.tree-list .trow .ms { font-variant-numeric: tabular-nums; font-size: 12.5px; color: var(--ink-2); text-align: right; white-space: nowrap; } +.tree-list .trow .ms.none { color: var(--ink-3); } +.tree-list .trow.related { background: var(--surface-2); } +.tree-list .trow.walk { background: var(--accent-wash); } +.tree-list .lane0 { stroke: var(--line-strong); stroke-width: 1.5; fill: none; } +.tree-list .lanek { stroke-width: 2; fill: none; } +.tree-list .dot { stroke: var(--surface); stroke-width: 2; fill: var(--k-component); } +.tree-list .dot.training-policy { fill: var(--k-training-policy); } +.tree-list .dot.architecture { fill: var(--k-architecture); } +.tree-list .dot.refinement { fill: var(--k-refinement); } +.tree-list .dot.component { fill: var(--k-component); } +.tree-list .dot.unset { fill: var(--surface); stroke: var(--ink-3); stroke-width: 1.5; } +.tree-list .dot.sel { stroke: var(--accent); stroke-width: 2; } +.lk-training-policy { stroke: var(--k-training-policy); } .lk-architecture { stroke: var(--k-architecture); } .lk-refinement { stroke: var(--k-refinement); } .lk-component { stroke: var(--k-component); } .lk-unset { stroke: var(--ink-3); } + +/* Lineage map (horizontal canvas). */ +.tree-map { position: relative; overflow: auto; background: var(--surface); } +.tree-map .inner { transform-origin: 0 0; } +.tree-map svg { display: block; } +.tree-map .tline { fill: none; stroke: var(--mark); stroke-width: 1.5; opacity: .75; } +.tree-map .tgrid { stroke: var(--grid); stroke-width: 1; shape-rendering: crispEdges; } +.tree-map .tlabel { fill: var(--ink-3); font-size: 10px; font-family: var(--font); } +.tree-map .ttitle { fill: var(--ink-2); font-size: 11px; font-family: var(--font); } +.tree-map .tpt { stroke: var(--surface); stroke-width: 1.5; cursor: pointer; fill: var(--k-component); } +.tree-map .tpt.training-policy { fill: var(--k-training-policy); } .tree-map .tpt.architecture { fill: var(--k-architecture); } .tree-map .tpt.refinement { fill: var(--k-refinement); } .tree-map .tpt.component { fill: var(--k-component); } +.tree-map .tpt.unset { fill: var(--surface); stroke: var(--ink-3); } +.tree-map .tpt.sel { stroke: var(--accent); stroke-width: 3; } +.tree-map .tpt.dim { opacity: .2; } +.tree-map .band { fill: none; stroke: var(--line); } +.tree-map .bandlabel { fill: var(--ink-2); font-size: 12px; font-weight: 600; font-family: var(--font); } +.tree-map .bandlabel.loose { font-style: italic; font-weight: 500; } +.tree-map .lanelabel { fill: var(--ink-3); font-size: 11px; font-family: var(--font); } +.tree-map .seq { stroke: var(--line-strong); stroke-width: 1.5; fill: none; } +.tree-map .der { fill: none; opacity: .55; stroke-width: 1.5; } +.tree-map .der.cold { opacity: .18; } +.tree-map .der.hot { opacity: 1; stroke-width: 2.5; } +.tree-map .inp { fill: none; stroke: var(--ink-2); stroke-width: 1.5; } +.tree-map .cmp { stroke: var(--ink-2); stroke-width: 1.5; stroke-dasharray: 4 4; fill: none; pointer-events: stroke; cursor: pointer; } +.tree-map .cmp:hover, .tree-map .cmp:focus-visible { stroke: var(--accent); stroke-width: 2.5; outline: none; } +.tree-map .node { cursor: pointer; } +.tree-map .node circle { stroke: var(--surface); stroke-width: 2; fill: var(--k-component); } +.tree-map .node.training-policy circle { fill: var(--k-training-policy); } +.tree-map .node.architecture circle { fill: var(--k-architecture); } +.tree-map .node.refinement circle { fill: var(--k-refinement); } +.tree-map .node.component circle { fill: var(--k-component); } +.tree-map .node.unset circle { fill: var(--surface); stroke: var(--ink-3); stroke-width: 1.5; } +.tree-map .node text { fill: var(--ink-2); font-size: 10.5px; font-family: var(--mono); text-anchor: middle; pointer-events: none; } +.tree-map .node text.nl { paint-order: stroke; stroke: var(--surface); stroke-width: 3px; stroke-linejoin: round; } +.tree-map .node text.nl.off { display: none; } +.tree-map .node.sel circle { stroke: var(--accent); stroke-width: 3; } +.tree-map .node.sel text { fill: var(--ink); font-weight: 600; } +.tree-map .node:hover circle, .tree-map .node:focus-visible circle { stroke: var(--ink); } +.tree-map .node:focus, .tree-map .node:focus-visible, .tree-map .tpt:focus { outline: none; } +.tree-map .node.dim { opacity: .25; } +.tree-map .node.kin circle { stroke: var(--ink); } +.tree-map .node.walk circle { stroke: var(--accent); stroke-width: 3; } +.tree-map .hitr { fill: transparent; } +.minimap { position: absolute; left: 12px; bottom: 12px; width: 220px; height: 56px; background: var(--surface); border: 1px solid var(--line-strong); border-radius: var(--radius); cursor: pointer; z-index: 2; } +.minimap svg { width: 100%; height: 100%; display: block; } +.minimap .vp { fill: var(--accent-wash); stroke: var(--accent); stroke-width: 1; opacity: .8; } +.minimap .d { fill: var(--ink-3); } +.zoomctl { display: inline-flex; gap: 4px; } + +/* Dataset cards and run rows (the Data and Train tabs build on these). */ +.ds-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(230px, 1fr)); gap: 14px; } +.ds-card { display: block; text-align: left; padding: 0; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); color: inherit; overflow: hidden; } +.ds-card:hover { border-color: var(--line-strong); } +.ds-card[aria-pressed="true"] { border-color: var(--accent); box-shadow: inset 0 0 0 1px var(--accent); } +.ds-card .t { padding: 10px 12px 12px; display: grid; gap: 3px; } +.ds-card .t .title { font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.ds-card .t .meta { display: flex; gap: 8px; align-items: center; font-size: 12.5px; color: var(--ink-2); } +.ds-card .t .path { font-family: var(--mono); font-size: 11.5px; color: var(--ink-3); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.row.run { grid-template-columns: 44px minmax(160px, 1fr) 120px 90px 50px 100px; } +.row.run .ck { font-family: var(--mono); font-size: 12.5px; color: var(--ink-2); } +.coverage { display: flex; flex-wrap: wrap; gap: 6px 18px; font-size: 12.5px; color: var(--ink-2); margin: -6px 0 14px; } +.coverage b { color: var(--ink); font-weight: 500; } +.cases svg { display: block; width: 100%; height: 64px; } +.cases .b { fill: var(--mark); } .cases .b.sel { fill: var(--accent); } +.cases .base { stroke: var(--line-strong); } + +/* The Benchmarks, Train, Data and Analysis tabs (tabs.js), from the mockup's base.css. */ +.chart .pt { fill: var(--k-component); } +.chart .pt.other { fill: var(--surface); stroke: var(--ink-2); stroke-width: 1.5; } +.chart g:hover > .pt, .chart g:focus-within > .pt { stroke: var(--ink); } +.chart .best { fill: none; stroke: var(--good); stroke-width: 1.5; stroke-dasharray: 5 3; } +.chart .stem { stroke: var(--line-strong); stroke-width: 1; } +.chart .rk { fill: var(--ink); font-size: 12.5px; font-family: var(--font); } +.chart .rk.on { font-weight: 600; } +.chart .rv { fill: var(--ink-2); font-size: 12px; font-family: var(--font); font-variant-numeric: tabular-nums; } +.chart .hit.band { fill: transparent; } +.chart .hit.band:hover, .chart .hit.band:focus-visible { fill: var(--surface-2); } +.legend .chip.other::before { background: transparent; box-shadow: inset 0 0 0 1.5px var(--ink-2); } +.chart.cases svg { height: auto; } +.cases .b.none { fill: none; stroke: var(--ink-3); stroke-width: 1; } +.cases g:hover > .b, .cases g:focus-within > .b { fill: var(--ink); } +.chart .hit ~ * { pointer-events: none; } /* the marks over a hit area never take the pointer from it */ +.bench-chart { margin-top: 22px; } +.bench-chart h3.panel { font-size: 13px; color: var(--ink-2); margin: 14px 0 4px; } +.bench-desc { margin: -4px 0 8px; } +.bench-note { margin: 0 0 12px; } +.bench-cmp, .bench-all { margin-top: 28px; } +.tv-wrap { overflow-x: auto; max-width: 100%; } +table.tv th button { font: inherit; color: inherit; background: none; border: 0; padding: 0; cursor: pointer; text-align: inherit; } +table.tv th button[aria-sort="ascending"]::after { content: " ↑"; color: var(--ink-3); } +table.tv th button[aria-sort="descending"]::after { content: " ↓"; color: var(--ink-3); } +table.tv tr.is-sel td { background: var(--accent-wash); } +table.tv tr.rowbtn { cursor: pointer; } +table.tv tr.rowbtn:hover td { background: var(--surface-2); } +table.tv td .linkish { font: inherit; color: var(--ink); background: none; border: 0; padding: 0; text-align: left; cursor: pointer; } +table.tv td.grp { max-width: 220px; overflow: hidden; text-overflow: ellipsis; } +table.tv td.best { color: var(--good); font-weight: 600; } +table.tv.pts { margin-top: 8px; } +.cmp-chips { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; } +.cmp-chips .c { display: inline-flex; align-items: center; gap: 6px; padding: 2px 6px 2px 10px; border: 1px solid var(--line); border-radius: 999px; font-size: 12.5px; background: var(--surface); max-width: 100%; } +.cmp-chips .c i { width: 10px; height: 10px; border-radius: 50%; display: inline-block; flex: none; } +.cmp-chips .c button { border: 0; background: none; padding: 0 2px; color: var(--ink-3); font-size: 14px; line-height: 1; cursor: pointer; } +.cmp-chips .c.s1 i { background: var(--k-architecture); } .cmp-chips .c.s2 i { background: var(--k-refinement); } .cmp-chips .c.s3 i { background: var(--k-training-policy); } +table.cmpt { margin-top: 12px; } +table.cmpt .d { color: var(--ink-2); font-size: 12.5px; } +table.cmpt .d.better { color: var(--good); } table.cmpt .d.worse { color: var(--serious); } +.sect.gap, .filters.gap { margin-top: 28px; } +.row.todo { grid-template-columns: 120px minmax(0, 1fr) 140px; } +.row.request { grid-template-columns: 150px minmax(0, 1fr) 170px; } +.row.run { grid-template-columns: 120px minmax(160px, 1fr) 130px 70px 150px; } +.row.run .ck { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.row.env { grid-template-columns: 120px minmax(0, 1fr) minmax(0, 0.8fr); } +.row.env .mono { color: var(--ink-3); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; direction: rtl; text-align: left; } +.row .sub.warnline { color: var(--warn); white-space: normal; } +.row .sub.live { display: flex; align-items: center; gap: 8px; } +.row .hair { position: relative; width: 120px; height: 2px; background: var(--grid); flex: none; } +.row .hair i { position: absolute; left: 0; top: 0; bottom: 0; background: var(--accent); } +.setup { display: grid; grid-template-columns: max-content minmax(0, 1fr); gap: 6px 14px; font-size: 13px; margin: 0; } +.setup dt { color: var(--ink-2); } +.setup dd { margin: 0; overflow-wrap: anywhere; min-width: 0; } +.setup dd.pre { white-space: pre-wrap; } +.evtext { white-space: pre-wrap; overflow-wrap: anywhere; font: 12px/1.5 var(--mono); padding: 10px; margin: 0; background: var(--surface-2); border-radius: var(--radius); color: var(--ink); max-height: 360px; overflow: auto; } +.ds-card .pv { display: block; width: 100%; aspect-ratio: 16 / 9; background: var(--surface-2); border-bottom: 1px solid var(--line); } +.ds-card .pv.text { display: grid; place-items: center; color: var(--ink-3); font: 12px/1.6 var(--mono); padding: 10px; white-space: pre; overflow: hidden; text-align: left; } +.ds-card .t .meta .state { margin-left: auto; } +.ds-card .t .uses { font-size: 12px; color: var(--ink-3); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.viewer-bar { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin-bottom: 14px; } +.viewer-bar select { max-width: 420px; } +.filters select { max-width: 300px; } +.filters input[type="search"] { width: 180px; } +.ana-sum { grid-template-columns: repeat(4, minmax(0, 160px)); margin: 4px 0 10px; } +.ana-none { margin-top: 8px; } +.pager { display: flex; gap: 8px; align-items: center; font-size: 12.5px; color: var(--ink-2); margin: 10px 0; } +table.cases-t { margin-top: 8px; } +/* Kept fields (fields.js, spec-d6 section 3): three panels of equal size side by side, stacked when narrow. */ +section.fld { margin-top: 26px; display: grid; gap: 10px; min-width: 0; } +section.fld .sect { margin-bottom: 0; flex-wrap: wrap; } +.fld-head { display: grid; gap: 4px; min-width: 0; } +.fld-head p { margin: 0; overflow-wrap: anywhere; } +.fld-pick select { max-width: min(520px, 100%); min-width: 0; } +.fld-pick { max-width: 100%; } +.fld-run { color: var(--accent); } +.fld-ctl .fld-at { min-width: 4.5em; } +.fld-view { display: grid; gap: 6px; min-width: 0; } +.fld-panels { position: relative; display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 14px; min-width: 0; } +.fld-panels[aria-busy="true"] { cursor: progress; } +.fld-panel { margin: 0; display: grid; gap: 4px; min-width: 0; align-content: start; } +.fld-title { font-size: 12.5px; font-weight: 600; color: var(--ink); } +canvas.fld-cv { display: block; width: 100%; background: var(--surface-2); border-radius: 3px; cursor: crosshair; } +canvas.fld-bar { display: block; width: 100%; height: 10px; border-radius: 2px; } +.fld-where { margin: 0; } +.fld-tip { position: absolute; z-index: 3; pointer-events: none; display: grid; gap: 2px; padding: 6px 8px; min-width: 150px; font-size: 12px; background: var(--surface); color: var(--ink); border: 1px solid var(--line-strong); border-radius: var(--radius); box-shadow: 0 2px 8px rgba(0, 0, 0, .14); } +.fld-tip[hidden] { display: none; } +.fld-tip-row { display: flex; justify-content: space-between; gap: 12px; font-variant-numeric: tabular-nums; } +.fld-tip-row span:first-child { color: var(--ink-2); } +.fld-figs { display: flex; flex-wrap: wrap; gap: 10px 28px; align-items: flex-start; min-width: 0; } +.fld-figs table.tv { width: auto; min-width: 0; font-size: 12.5px; } +.fld-figs table.tv caption { text-align: left; caption-side: top; padding-bottom: 4px; } +.fld-figs th { font-weight: 400; color: var(--ink-2); text-align: left; padding-right: 16px; } +.fld-figs th.mono { overflow-wrap: anywhere; max-width: 32ch; } +.fld-figs td.n { font-variant-numeric: tabular-nums; } +@media (max-width: 760px) { .fld-panels { grid-template-columns: minmax(0, 1fr); } } +@media (max-width: 1100px) { .row.run { grid-template-columns: 90px minmax(0, 1fr) 70px 130px; } .row.run > .chip { display: none; } } + +/* ---- The shell: top bar, tabs, status strip, stage, drawer (b-map.html). ---- */ +body.shell { height: 100vh; display: grid; grid-template-rows: 48px auto auto 1fr; overflow: clip; } /* clip: not even a script or scrollIntoView can scroll the shell */ +.top { display: flex; align-items: center; gap: 22px; padding: 0 20px; border-bottom: 1px solid var(--line); background: var(--surface); min-width: 0; } +.brand { display: inline-flex; align-items: center; gap: 8px; font-weight: 600; color: var(--ink); flex: none; } +.brand:hover { text-decoration: none; } +.brand .mark { display: inline-grid; place-items: center; width: 22px; height: 22px; border-radius: 5px; background: var(--ink); color: var(--page); font-family: var(--mono); font-size: 13px; } +.top h1 { font-size: 15px; font-weight: 600; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; min-width: 0; flex: 0 1 auto; } +/* The tabs keep their size while they fit; on a narrower window the bar scrolls within itself instead of widening the page. */ +.tabs { display: flex; gap: 2px; margin-left: 8px; flex: 0 1 auto; min-width: 0; overflow-x: auto; scrollbar-width: none; } +.tabs::-webkit-scrollbar { display: none; } +.tabs a { flex: none; white-space: nowrap; } +/* A phone's width: the project's name gives way (the Overview tab names it), and the strip wraps instead of widening the page. */ +@media (max-width: 640px) { + body.shell { grid-template-rows: auto auto auto 1fr; } + .top { flex-wrap: wrap; gap: 4px 10px; padding: 6px 12px 0; min-height: 48px; } + .top h1, .top #session-identity { display: none; } /* the Overview tab names the project; Session names who is signed in */ + .top .right { gap: 6px; } + .tabs { order: 3; flex: 1 0 100%; margin: 0 -4px; padding-bottom: 4px; } /* a row of their own, scrolling */ + .strip { flex-wrap: wrap; gap: 8px; padding: 8px 12px; } + .strip .hair { display: none; } +} +.tabs a { padding: 6px 10px; border-radius: var(--radius); color: var(--ink-2); } +.tabs a:hover { background: var(--surface-2); text-decoration: none; } +.tabs a[aria-current="page"] { color: var(--ink); background: var(--accent-wash); } +.top .right { margin-left: auto; display: flex; gap: 14px; align-items: center; flex: none; } +/* The session controls of auth.js, moved into the top bar and drawn with the page's tokens. */ +.top .session-bar { display: inline-flex; align-items: center; gap: 6px; padding: 0; background: transparent; color: var(--ink-2); font: 12.5px var(--font); flex-wrap: nowrap; } +.top .session-bar button { min-height: 26px; padding: 2px 8px; font: 13px var(--font); color: var(--ink); background: transparent; border: 1px solid transparent; border-radius: var(--radius); } +.top .session-bar button:hover { background: var(--surface-2); } +#session-dialog { color: var(--ink); background: var(--surface); border: 1px solid var(--line-strong); border-radius: var(--radius); font: 14px/1.5 var(--font); } +#session-dialog::backdrop { background: rgba(15, 19, 23, .45); } +#session-dialog h2 { margin-bottom: 10px; } +#session-dialog button { min-height: 30px; padding: 4px 10px; border: 1px solid var(--line-strong); border-radius: var(--radius); background: var(--surface); color: var(--ink); } +#session-token { min-height: 30px; padding: 4px 8px; border: 1px solid var(--line-strong); border-radius: var(--radius); background: var(--surface); color: var(--ink); } +#session-status, .session-help { font: 13px/1.5 var(--font); color: var(--ink-2); } +.strip { display: flex; align-items: center; gap: 14px; padding: 8px 20px; border-bottom: 1px solid var(--line); background: var(--surface); font-size: 13px; min-width: 0; } +.strip .state { font-weight: 600; } +.strip .msg { color: var(--ink-2); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; } +.strip .needs-btn { margin-left: auto; flex: none; } +.strip .needs-btn[aria-expanded="true"] { background: var(--accent-wash); } +.strip .hair { position: relative; width: 120px; height: 2px; background: var(--grid); flex: none; } +.strip .hair i { position: absolute; left: 0; top: 0; bottom: 0; background: var(--accent); } +#needs-panel { margin: 12px 20px 0; padding: 4px 16px; background: var(--surface); border: 1px solid var(--line); border-radius: var(--radius); z-index: 4; position: absolute; right: 0; top: 0; width: min(560px, calc(100vw - 40px)); max-height: calc(100% - 24px); overflow: auto; } +#needs-panel .needs li:first-child { border-top: 0; } +#needs-panel .needs li:last-child { border-bottom: 0; } +.stage { position: relative; min-height: 0; overflow: clip; display: grid; } +/* overflow-anchor: the page keeps the main area's scroll itself (putInPlace, spec-w4 R4). The browser's scroll + anchoring, left on, moved it back to the top when a tile's picture replaced its glyph in the first second after a + dataset opened (spec-d1 As built, D1b). */ +main { overflow: auto; overflow-anchor: none; padding: 18px 20px 40px; min-height: 0; min-width: 0; } +main.map { padding: 10px 20px 12px; display: grid; grid-template-rows: auto auto minmax(0, 1fr); overflow: hidden; } +main.map .tree-host { overflow: auto; min-height: 0; border: 1px solid var(--line); border-top: 0; border-radius: 0 0 var(--radius) var(--radius); background: var(--surface); } +main.map > .filters { margin-bottom: 10px; } +main.map > .empty { margin-top: 12px; } +.tree-map { height: auto; max-height: 46vh; border: 1px solid var(--line); border-radius: var(--radius) var(--radius) 0 0; } +.tree-list .tsect { top: 0; } +main.with-drawer { padding-right: calc(min(420px, 92vw) + 20px); } +.drawer { position: absolute; top: 0; right: 0; bottom: 0; width: min(420px, 92vw); background: var(--surface); border-left: 1px solid var(--line); padding: 18px 20px 40px; overflow: auto; z-index: 3; } +.drawer { transform: translateX(100%); visibility: hidden; } +.drawer.open { transform: none; visibility: visible; } +@media (prefers-reduced-motion: no-preference) { .drawer { transition: transform .2s ease, visibility .2s; } main { transition: padding-right .2s ease; } } +.drawer .close { position: absolute; right: 10px; top: 10px; } +.drawer .close svg { width: 16px; height: 16px; } +.content { max-width: none; } +.notyet { color: var(--ink-2); font-size: 14px; } +.failure { color: var(--serious); font-size: 13.5px; max-width: 80ch; overflow-wrap: anywhere; } + +/* Overview. */ +.ov-status { margin-bottom: 28px; } +.ov-line { display: flex; gap: 12px; align-items: baseline; } +.ov-line .state { font-size: 14px; font-weight: 600; } +.ov-text { font-size: 17px; line-height: 1.4; max-width: 70ch; } +.ov-row { display: grid; grid-template-columns: minmax(0, 1.5fr) minmax(0, 1fr); gap: 32px; margin: 24px 0 8px; } +.ov-cols { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); gap: 32px; margin: 28px 0; } +.ov-history { margin-top: 24px; } +.row.ov-env { grid-template-columns: minmax(0, 1fr) auto; } +.row.ov-act { grid-template-columns: 150px minmax(0, 1fr) auto; } +table.tv.kinds td, table.tv.kinds th { white-space: nowrap; } +.report-frame { display: block; width: 100%; height: calc(100vh - 210px); min-height: 360px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); } +.report-bar { display: flex; gap: 12px; align-items: center; margin-bottom: 12px; } +@media (max-width: 1100px) { .ov-row { grid-template-columns: 1fr; } } +@media (max-width: 900px) { .ov-cols { grid-template-columns: 1fr; } .tabs a { padding: 6px 7px; } } + +/* A model's card, built as elements by markdown.js. */ +.card-md { font-size: 13.5px; line-height: 1.55; overflow-wrap: anywhere; } +.card-md > * + * { margin-top: 10px; } +.card-md h1, .card-md h2, .card-md h3, .card-md h4, .card-md h5, .card-md h6 { color: var(--ink); margin-top: 16px; } +.card-md h1 { font-size: 16px; } .card-md h2 { font-size: 15px; } .card-md h3 { font-size: 14px; color: var(--ink); margin-bottom: 0; } .card-md h4, .card-md h5, .card-md h6 { font-size: 13.5px; } +.card-md > :first-child { margin-top: 0; } +.card-md p { max-width: 66ch; } +.card-md ul, .card-md ol { margin: 0; padding-left: 20px; } +.card-md li + li { margin-top: 3px; } +.card-md code { font-family: var(--mono); font-size: 12.5px; background: var(--surface-2); border-radius: 3px; padding: 0 3px; } +.card-md pre { margin: 0; padding: 10px; background: var(--surface-2); border-radius: var(--radius); overflow: auto; font: 12.5px/1.5 var(--mono); white-space: pre; } +.card-md pre code { background: none; padding: 0; } +.card-md blockquote { margin: 0; padding-left: 12px; border-left: 3px solid var(--line-strong); color: var(--ink-2); } +.card-md hr { border: 0; border-top: 1px solid var(--line); } +.card-md .tbl { overflow-x: auto; } +.card-md table { border-collapse: collapse; font-size: 12.5px; } +.card-md th, .card-md td { padding: 4px 8px; border-top: 1px solid var(--line); text-align: left; vertical-align: top; overflow-wrap: normal; } +.card-md th { color: var(--ink-2); font-weight: 500; } +.card-md .link { color: var(--ink); text-decoration: underline dotted var(--ink-3); } +.card-md .link .href { color: var(--ink-3); font-family: var(--mono); font-size: 11.5px; } + +/* ---- Controls that change the project (actions.js, spec-w3 section 3.1). ---- */ +/* From the mockup's base.css and b-map.html: forms, radios, two columns, steps. */ +textarea { font: inherit; resize: vertical; } +.form { display: grid; gap: 10px; } +.form label { display: grid; gap: 3px; font-size: 12.5px; color: var(--ink-2); } +.form select, .form input[type="text"], .form textarea { width: 100%; font: inherit; color: var(--ink); background: var(--surface); border: 1px solid var(--line-strong); border-radius: var(--radius); padding: 5px 8px; } +.form .two { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; align-items: start; } /* a field with a hint beside one without: both start at the top */ +.form .hint, .act-field .hint { font-family: var(--mono); font-size: 11.5px; color: var(--ink-3); } +.form .radio { display: flex; flex-wrap: wrap; gap: 6px; } +.form .radio label { display: inline-flex; align-items: center; gap: 6px; padding: 4px 10px; border: 1px solid var(--line-strong); border-radius: 999px; color: var(--ink); cursor: pointer; } +.form .radio input { margin: 0; } +.form .radio label:has(input:checked) { background: var(--accent-wash); border-color: var(--accent); } +.form label.check { display: flex; align-items: flex-start; gap: 8px; color: var(--ink); font-size: 13px; } +.form label.check input { margin: 3px 0 0; flex: none; } +.steps { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 8px; } +.steps li { display: grid; grid-template-columns: 18px minmax(0, 1fr); gap: 10px; } +.steps li > .state { align-self: start; margin-top: 2px; } +.steps .title { display: block; font-weight: 500; overflow-wrap: anywhere; } +.steps .sub { display: block; font-size: 12.5px; color: var(--ink-2); overflow-wrap: anywhere; } +.steps .sub.mono { font-size: 11.5px; color: var(--ink-3); } +.steps .done .title { color: var(--ink-2); } +.steps .blocked .title { color: var(--serious); } +.act-group { display: grid; gap: 8px; min-width: 0; } +.act-group .actions { display: flex; flex-wrap: wrap; gap: 8px; align-items: flex-end; } +.act-group.inline { display: inline-flex; gap: 8px; align-items: center; } +.act-group.inline > .actions { flex-wrap: nowrap; } +.act-field { display: grid; gap: 3px; font-size: 12.5px; color: var(--ink-2); flex: 1 1 220px; min-width: 0; } +.act-field input, .act-field textarea { width: 100%; min-width: 0; } +.confirm-words { font-size: 13px; color: var(--ink); max-width: 60ch; } +.btn[aria-busy="true"] { cursor: progress; } +.refusal { display: grid; gap: 4px; padding: 8px 10px; border: 1px solid var(--serious); border-radius: var(--radius); background: var(--serious-wash); color: var(--ink); font-size: 13px; } +.refusal > p:first-child { color: var(--serious); font-weight: 500; } +.refusal ul { margin: 0; padding-left: 18px; color: var(--ink); } +.refusal .actions { margin-top: 4px; } +.rows > li.with-act { display: flex; align-items: center; gap: 8px; } +.rows > li.with-act > .row { flex: 1 1 auto; min-width: 0; } +.rows > li.with-act > .act-group { flex: none; margin-right: 8px; } +.sect .act-group { align-self: center; } +.sect .act-group .btn { white-space: nowrap; } +.hair { position: relative; display: inline-block; width: 120px; height: 2px; background: var(--grid); flex: none; vertical-align: middle; } +.hair i { position: absolute; left: 0; top: 0; bottom: 0; background: var(--accent); } +.progress-line { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; } +.progress-line .hair { width: 160px; } +.live-note { margin-top: -8px; } +.need { display: grid; gap: 8px; min-width: 0; } +.need .act-group .actions { gap: 6px; } +.strip .unsaved { display: inline-flex; align-items: center; gap: 8px; flex: none; color: var(--warn); font-size: 12.5px; white-space: nowrap; } +.strip .unsaved + .needs-btn { margin-left: 0; } +.strip .unsaved { margin-left: auto; } +.strip-note:not(:empty) { padding: 8px 20px; border-bottom: 1px solid var(--line); background: var(--surface); } +.strip-note .refusal { max-width: 900px; } +.reason { padding: 8px 10px; border-left: 3px solid var(--line-strong); background: var(--surface-2); border-radius: 0 var(--radius) var(--radius) 0; font-size: 13.5px; } +.plan-lead { font-size: 14px; color: var(--ink); } +.conflict { display: grid; gap: 14px; max-width: 760px; } +.conflict h2 { font-size: 17px; } +.conflict-refs { margin: 0; padding-left: 18px; } +.plain { list-style: none; margin: 0; padding: 0; display: grid; gap: 4px; } +.warnline { color: var(--warn); } + +/* ---- Editing and branching (editor.js, spec-w3 section 3.2). ---- */ +/* A model never built: hollow and dashed, in its kind's colour; a draft is labelled "new" (the mockup's planned model). */ +.tree-map .node.planned circle { fill: var(--surface); stroke: var(--k-component); stroke-dasharray: 3 2; stroke-width: 2; } +.tree-map .node.planned.training-policy circle { stroke: var(--k-training-policy); } +.tree-map .node.planned.architecture circle { stroke: var(--k-architecture); } +.tree-map .node.planned.refinement circle { stroke: var(--k-refinement); } +.tree-map .node.planned.unset circle { stroke: var(--ink-3); } +.tree-map .node.planned.sel circle { stroke: var(--accent); stroke-width: 3; } +.tree-map .node.draft text.nl { fill: var(--accent); font-weight: 600; } +.tree-map .unsaved-dot { fill: var(--warn); stroke: var(--surface); stroke-width: 1; } +.tree-list .dot.planned { fill: var(--surface); stroke: var(--k-component); stroke-dasharray: 3 2; stroke-width: 2; } +.tree-list .dot.planned.training-policy { stroke: var(--k-training-policy); } +.tree-list .dot.planned.architecture { stroke: var(--k-architecture); } +.tree-list .dot.planned.refinement { stroke: var(--k-refinement); } +.tree-list .dot.planned.unset { stroke: var(--ink-3); } +.tree-list .dot.planned.sel { stroke: var(--accent); } +.tree-list .trow.draft .nm, .tree-list .trow.draft .ck { color: var(--accent); } +.chip.draft { color: var(--accent); font-weight: 500; } +.chip.draft::before { background: transparent; box-shadow: none; border: 1.5px dashed var(--accent); width: 6px; height: 6px; } +/* The unsaved mark, and "not kept yet" where something typed is not kept. */ +.umark { font-size: 11.5px; color: var(--warn); font-weight: 500; white-space: nowrap; } +.umark:empty { display: none; } +.umark.typed { color: var(--accent); } +.tree-list .trow .nm .umark, .row .title .umark { margin-left: 8px; } +/* The editor. */ +.editor .seg { align-self: flex-start; } +.seg { display: inline-flex; border: 1px solid var(--line-strong); border-radius: var(--radius); overflow: hidden; } +.seg button { border: 0; background: var(--surface); padding: 4px 12px; font: inherit; font-size: 13px; color: var(--ink-2); cursor: pointer; } +.seg button + button { border-left: 1px solid var(--line-strong); } +.seg button[aria-pressed="true"] { background: var(--accent-wash); color: var(--ink); } +.code-field { display: grid; gap: 4px; min-width: 0; } +textarea.code { width: 100%; min-width: 0; font: 12px/1.5 var(--mono); white-space: pre; overflow: auto; tab-size: 2; padding: 8px; } +.form input[type="number"] { width: 100%; min-width: 0; font: inherit; color: var(--ink); background: var(--surface); border: 1px solid var(--line-strong); border-radius: var(--radius); padding: 5px 8px; min-height: 30px; } +.form .two > * { min-width: 0; } +.form .radio label.chip-radio { padding: 3px 9px; } +.form .radio label.chip-radio .chip { color: var(--ink); } +.form .derived { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; } +.form .derived-name { color: var(--ink); font-size: 13px; flex: 1 1 140px; min-width: 0; overflow-wrap: anywhere; } +.form .settings { display: grid; gap: 8px; padding-top: 6px; border-top: 1px solid var(--line); } +.form .settings h3 { margin: 0; } +.form .settings .note { font-size: 12.5px; color: var(--ink-3); overflow-wrap: anywhere; } +.problems { display: grid; gap: 4px; padding: 8px 10px; border-radius: var(--radius); font-size: 13px; } +.problems ul { margin: 0; padding-left: 18px; } +.problems.warnings { border: 1px solid var(--warn); background: var(--warn-wash); } +.picking { display: grid; gap: 6px; justify-items: start; padding: 8px 10px; border: 1px dashed var(--accent); border-radius: var(--radius); background: var(--accent-wash); } +.picking p:first-child { font-weight: 500; color: var(--ink); } +.new-steps { display: grid; gap: 10px; padding-top: 10px; border-top: 1px solid var(--line); } +.new-steps h3 { margin: 0; } +.steps-box:empty { display: none; } +/* The Overview's list of every definition, and Train's Planned. */ +.ov-defs { margin: 28px 0; } +.ov-defs h3.ov-kind { font-size: 13px; color: var(--ink-2); margin: 14px 0 4px; } +.row.def { grid-template-columns: minmax(120px, 0.7fr) minmax(0, 1.3fr) 150px 90px; } +.row.def .name { color: var(--ink-2); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.row.def > .title { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +div.row.def { cursor: default; } +@media (max-width: 900px) { .row.def { grid-template-columns: minmax(0, 1fr) minmax(0, 1fr) 110px; } .row.def .umark { display: none; } } + +/* ---- Looking inside a product (inside.js, spec-d1.md section 7); from the mockup's Data tab. ---- */ +.ds-item { display: grid; align-content: start; gap: 6px; min-width: 0; } +.ds-foot { display: flex; gap: 8px; } +.ds-back { margin-bottom: 10px; } +.ds-head { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 16px; margin-bottom: 12px; } +.ds-head h2 { font-size: 17px; overflow-wrap: anywhere; } +.ds-head .meta { display: flex; flex-wrap: wrap; gap: 4px 12px; align-items: center; font-size: 12.5px; color: var(--ink-2); } +.ds-head select { max-width: 360px; } +.inside-problems { margin: -4px 0 12px; } +.ds-body { display: grid; grid-template-columns: minmax(0, 1fr) minmax(360px, 44%); gap: 20px; align-items: start; } +@media (max-width: 1100px) { .ds-body { grid-template-columns: minmax(0, 1fr); } } +.rec-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(96px, 1fr)); gap: 8px; } +.rec { display: grid; grid-template-rows: auto auto; padding: 0; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); overflow: hidden; text-align: left; color: inherit; min-width: 0; } +.rec:hover { border-color: var(--line-strong); } +.rec[aria-pressed="true"] { border-color: var(--accent); box-shadow: inset 0 0 0 1px var(--accent); } +.rec .th { display: grid; place-items: center; aspect-ratio: 1; background: var(--surface-2); color: var(--ink-3); overflow: hidden; border-bottom: 1px solid var(--line); } +.rec .th > * { grid-area: 1 / 1; } +.rec .th svg.glyph { width: 30px; height: 30px; } +.rec .th img { width: 100%; height: 100%; object-fit: contain; } +.rec .th canvas { display: block; width: 100%; height: 100%; } +.rec .th .pre { align-self: start; justify-self: stretch; margin: 6px 7px; font: 10.5px/1.35 var(--mono); color: var(--ink-2); white-space: pre-wrap; overflow-wrap: anywhere; overflow: hidden; text-align: left; display: -webkit-box; -webkit-box-orient: vertical; -webkit-line-clamp: 6; } +.rec .id { display: block; font: 11px var(--mono); color: var(--ink-2); padding: 3px 6px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; direction: rtl; text-align: left; } +.rec.refused .th { color: var(--serious); } +.inside-list .sect { margin-bottom: 8px; } +.crumbs { display: flex; flex-wrap: wrap; gap: 2px 6px; align-items: baseline; font-size: 13px; margin: 0 0 8px; overflow-wrap: anywhere; } +.crumbs .sep { color: var(--ink-3); } +.crumbs .here { font-weight: 500; } +.crumbs .linkish { font: inherit; color: var(--accent); background: none; border: 0; padding: 0; cursor: pointer; } +.crumbs .linkish:hover { text-decoration: underline; } +.row.tree-entry { grid-template-columns: 18px minmax(0, 1fr) auto; } +.row.tree-entry svg.glyph { width: 18px; height: 18px; color: var(--ink-3); } +.viewer { position: sticky; top: 0; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); padding: 12px; display: grid; gap: 10px; min-width: 0; max-height: calc(100vh - 140px); overflow: auto; } +@media (max-width: 1100px) { .viewer { position: static; max-height: none; } } +.iv { display: grid; gap: 10px; min-width: 0; } +.iv-head { display: grid; gap: 2px; min-width: 0; } +.iv-head h3 { font-size: 13px; font-weight: 600; overflow-wrap: anywhere; } +.iv-text { margin: 0; padding: 10px; background: var(--surface-2); border-radius: var(--radius); font: 12.5px/1.5 var(--mono); color: var(--ink); white-space: pre-wrap; overflow-wrap: anywhere; max-height: 52vh; overflow: auto; } +.iv-imgbox { overflow: auto; max-height: 60vh; background: var(--surface-2); border-radius: var(--radius); } +.iv-img { display: block; width: 100%; padding: 0; border: 0; background: none; cursor: zoom-in; } +.iv-img img { display: block; width: 100%; max-height: 56vh; object-fit: contain; image-rendering: pixelated; } +.iv-img.actual { width: auto; cursor: zoom-out; } +.iv-img.actual img { width: auto; max-height: none; max-width: none; } +.iv canvas { display: block; width: 100%; background: var(--surface-2); border-radius: var(--radius); } +.iv canvas.iv-3d { cursor: grab; touch-action: none; } +.iv canvas.iv-3d.dragging { cursor: grabbing; } +.iv canvas.iv-heat { cursor: crosshair; } +.iv audio { width: 100%; } +.iv-ctl { display: flex; flex-wrap: wrap; gap: 8px 14px; align-items: center; font-size: 12.5px; color: var(--ink-2); } +.iv-field { display: inline-flex; gap: 6px; align-items: center; min-width: 0; } +.iv-field select { max-width: 240px; } +.iv-slider input[type="range"] { width: 140px; } +.iv-array { display: grid; gap: 10px; min-width: 0; } +.iv-probe { font-variant-numeric: tabular-nums; color: var(--ink); min-height: 1.5em; overflow-wrap: anywhere; } +.iv-ends { display: flex; justify-content: space-between; margin-top: 2px; } +.iv-facts { font-size: 12.5px; } +.iv-hist { display: grid; gap: 6px; } + +/* ---- A model's weights (weights.js, spec-d2 section 4): the block in the model's drawer. ---- */ +.wts { display: grid; gap: 10px; min-width: 0; } +.wts > h3 { margin-bottom: 0; } +.wts-head { display: grid; gap: 2px; min-width: 0; } +.wts-head p { margin: 0; } +.wts-path { overflow-wrap: anywhere; } +.wts-pick { display: grid; gap: 4px; font-size: 12.5px; color: var(--ink-2); } +.wts-pick select { max-width: 100%; min-width: 0; } +.wts-bar { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; } +.wts .seg { justify-self: start; max-width: 100%; flex-wrap: wrap; } +.wts .seg button { font-size: 12.5px; padding: 3px 10px; } +.wts-total, .wts-there { margin: 0; } +.wts-tablebox { min-width: 0; } +table.wts-table { table-layout: fixed; width: 100%; font-size: 12.5px; } +table.wts-table th, table.wts-table td { padding: 6px 4px; vertical-align: middle; } +table.wts-table th:first-child, table.wts-table td:first-child { padding-left: 0; } +table.wts-table th:last-child, table.wts-table td:last-child { padding-right: 0; } +table.wts-table th { font-size: 12px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +table.wts-table col.wc-n { width: 76px; } +table.wts-table col.wc-dist { width: 60px; } +table.wts-table col.wc-chg { width: 82px; } +.wts-name { min-width: 0; } +.wts-name .linkish { display: block; max-width: 100%; color: var(--ink); text-decoration: underline; text-decoration-color: var(--line-strong); text-underline-offset: 2px; } +.wts-name .linkish:hover { text-decoration-color: var(--accent); } +.wts-name .linkish[aria-expanded="true"] { color: var(--accent); text-decoration-color: var(--accent); } +.wts-lname { overflow-wrap: anywhere; } +.wts-shape { display: block; font-size: 12px; color: var(--ink-2); font-variant-numeric: tabular-nums; overflow-wrap: anywhere; } +.wts-mark { display: inline-block; margin: 3px 4px 0 0; padding: 0 6px; border-radius: 9px; font-size: 11.5px; line-height: 18px; border: 1px solid var(--warn); color: var(--warn); background: var(--warn-wash); } +.wts-mark.only { border-color: var(--line-strong); color: var(--ink-2); background: var(--surface-2); } +.wts-dist canvas.wts-spark { display: block; width: 100%; height: 22px; background: var(--surface-2); border-radius: 3px; } +.wts-dist canvas.wts-spark.refused { outline: 1px solid var(--serious); } +.wts-chg { font-variant-numeric: tabular-nums; } +.wts-cbar { display: block; height: 6px; background: var(--surface-2); border-radius: 3px; overflow: hidden; } +.wts-cbar i { display: block; height: 100%; background: var(--mark); border-radius: 3px; } +.wts-fig { display: block; margin-top: 2px; font-size: 12px; color: var(--ink); } +.wts-zero { font-size: 12px; color: var(--ink-2); font-style: italic; } +tr.wts-open > td { padding: 8px 0 14px; border-top: 0; } +tr.wts-row:has(+ tr.wts-open) > td { background: var(--accent-wash); } +.wts-tensor { display: grid; gap: 8px; min-width: 0; padding: 10px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); } +.wts-tensor p { margin: 0; } +.wts-what { overflow-wrap: anywhere; } +.wts-stats { font-size: 12.5px; } +.wts canvas.wts-hist { display: block; width: 100%; background: var(--surface-2); border-radius: var(--radius); } +.wts-images { display: grid; gap: 8px; min-width: 0; } +.wts-ctl .iv-field select { max-width: 100%; } +.wts-ctl .iv-slider input[type="range"] { width: 110px; } +.wts-pics.tiles { display: grid; grid-template-columns: repeat(auto-fill, minmax(72px, 1fr)); gap: 8px; } +.wts-pics.one canvas { display: block; width: 100%; background: var(--surface-2); border-radius: var(--radius); } +.wts-tile { margin: 0; display: grid; gap: 2px; min-width: 0; } +.wts-tile canvas { display: block; width: 100%; background: var(--surface-2); border-radius: 3px; } +.wts-tile figcaption { font-size: 11.5px; color: var(--ink-3); text-align: center; } +.wts canvas.wts-ramp { display: block; width: 100%; height: 10px; border-radius: 3px; } +.wts-legend { display: grid; gap: 2px; } +.wts-legend .wts-range { margin-top: 4px; } +.wts-pager { margin: 0; flex-wrap: wrap; } +.wts-reason { margin: 0; } + +/* ---- A model's structure (structure.js, docs/plans/training-and-curation/spec-t2.md section 4). ---- */ +.st { display: grid; gap: 10px; min-width: 0; } +.st > h3 { margin-bottom: 0; } +.st p { margin: 0; } +.st-size, .st-how, .st-reason { overflow-wrap: anywhere; } +.st-bar { display: flex; flex-wrap: wrap; gap: 8px 12px; align-items: end; min-width: 0; } +.st-pick { display: grid; gap: 4px; font-size: 12.5px; color: var(--ink-2); min-width: 0; max-width: 100%; } +.st-pick select { max-width: 100%; min-width: 0; } +.st-folds { display: flex; flex-wrap: wrap; gap: 6px; } +.st-draw { min-width: 0; } +.st-scroll { overflow: auto; max-height: 380px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface-2); } +.st-scroll svg { display: block; font-family: var(--font); } +.st-scroll.fit svg { height: auto; } +/* "Open wide": the Structure block over the page, under the top bar, at the window's width. */ +.st-overlay { position: fixed; inset: 48px 0 0 0; z-index: 60; overflow: auto; padding: 16px 20px 24px; background: var(--page); border-top: 1px solid var(--line); } +.st-overlay .st-scroll { max-height: calc(100vh - 280px); } +.st-edge { fill: none; stroke: var(--line-strong); stroke-width: 1.3; } +.st-edge.back { stroke-dasharray: 3 3; } +.st-arrowhead { fill: var(--line-strong); } +.st-node { cursor: pointer; } +.st-node:focus { outline: none; } +.st-node .st-box { fill: var(--surface); stroke: var(--line-strong); stroke-width: 1; } +.st-node.group .st-box { stroke: var(--ink-3); } +.st-node .st-inner { fill: none; stroke: var(--line-strong); stroke-dasharray: 2 2; } +.st-node.input .st-box, .st-node.output .st-box { fill: var(--accent-wash); stroke: var(--accent); } +.st-node .st-label { font-size: 12px; font-weight: 600; fill: var(--ink); } +.st-node .st-subl { font-size: 11px; fill: var(--ink-2); font-variant-numeric: tabular-nums; } +.st-node.deep .st-label, .st-node.deep .st-subl { fill: var(--surface); } +.st-node:hover .st-box { stroke: var(--accent); } +.st-node:focus-visible .st-box, .st-node.selected .st-box { stroke: var(--accent); stroke-width: 2.5; } +.st-node.mark-added .st-box, .st-node.mark-changed .st-box, .st-node.mark-inside .st-box { stroke: var(--warn); stroke-width: 2; stroke-dasharray: 5 3; } +.st-node.mark-added .st-box { stroke-dasharray: none; } +.st-badge { fill: var(--warn); } +.st-legend { display: grid; gap: 4px; } +.st-ramp { display: block; height: 10px; border-radius: 3px; background: linear-gradient(to right, var(--ramp-lo), var(--ramp-hi)); } +.st-detail { display: grid; gap: 8px; min-width: 0; padding: 10px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); } +.st-dtitle { overflow-wrap: anywhere; } +.st-facts { font-size: 12.5px; } +.st-facts dd { overflow-wrap: anywhere; } +.st-tensors { list-style: none; margin: 0; padding: 0; display: grid; gap: 4px; font-size: 12.5px; } +.st-tensors li { overflow-wrap: anywhere; } +.st .linkish { font: inherit; color: var(--accent); background: none; border: 0; padding: 0; cursor: pointer; text-align: left; text-decoration: underline; text-underline-offset: 2px; overflow-wrap: anywhere; } +.st-detail .btn { justify-self: start; } +.st-sub { font-size: 13px; color: var(--ink-2); font-weight: 600; margin: 4px 0 0; overflow-wrap: anywhere; } +.st-difflist { list-style: none; margin: 4px 0 0; padding: 0; display: grid; gap: 3px; font-size: 12.5px; } +.st-difflist li { overflow-wrap: anywhere; padding-left: 8px; border-left: 3px solid var(--warn); } +.st-difflist li.st-d-missing { border-left-color: var(--line-strong); } +.st-outline { min-width: 0; } +.st-tree { list-style: none; margin: 0; padding: 0 0 0 12px; } +.st-outline > .st-tree { padding-left: 0; } +.st-row { display: flex; flex-wrap: wrap; gap: 2px 8px; align-items: baseline; min-width: 0; padding: 1px 0; } +.st-toggle { display: inline-flex; gap: 4px; align-items: baseline; font: inherit; font-size: 12.5px; color: var(--ink); background: none; border: 0; padding: 2px 0; cursor: pointer; min-width: 0; overflow-wrap: anywhere; text-align: left; } +.st-toggle:hover { color: var(--accent); } +.st-caret { color: var(--ink-3); width: 10px; flex: none; } +.st-m-missing { border-color: var(--line-strong); color: var(--ink-2); background: var(--surface-2); } + +/* ---- Annotating (annotate.js, spec-d4.md section 7); from the mockup's Annotate tab. ---- */ +.annotate { container-type: inline-size; min-width: 0; } +.ann-bar { display: flex; flex-wrap: wrap; gap: 8px 14px; align-items: center; margin-bottom: 14px; } +.ann-bar .ann-task { display: inline-flex; gap: 6px; align-items: center; font-size: 12.5px; color: var(--ink-2); min-width: 0; } +.ann-bar .ann-task select { max-width: 340px; min-width: 0; } +.ann-filter button { white-space: nowrap; } +.ann-progress { display: inline-flex; align-items: center; gap: 8px; font-size: 12.5px; color: var(--ink-2); white-space: nowrap; font-variant-numeric: tabular-nums; } +.ann-progress i { position: relative; display: inline-block; width: 90px; height: 4px; border-radius: 2px; background: var(--grid); overflow: hidden; } +.ann-progress b { position: absolute; left: 0; top: 0; bottom: 0; background: var(--good); } +.ann-keys { white-space: nowrap; } +.kbd { display: inline-block; font: 11px/1.4 var(--font); min-width: 16px; text-align: center; border: 1px solid var(--line-strong); border-bottom-width: 2px; border-radius: 3px; padding: 0 4px; color: var(--ink-2); background: var(--surface); } +.ann-body { display: grid; grid-template-columns: minmax(220px, 0.85fr) minmax(0, 1.5fr) minmax(290px, 340px); gap: 18px; align-items: start; } +@container (max-width: 1180px) { .ann-body { grid-template-columns: minmax(200px, 0.8fr) minmax(0, 1.5fr); } .ann-list { grid-row: span 2; } .ann-form { grid-column: 2; } } +@container (max-width: 700px) { .ann-body { grid-template-columns: minmax(0, 1fr); } .ann-list { grid-row: auto; } .ann-form { grid-column: auto; } } +.ann-list, .ann-view, .ann-form { min-width: 0; } +.ann-list .sect { margin-bottom: 8px; } +.ann-grid { grid-template-columns: repeat(auto-fill, minmax(88px, 1fr)); } +.ann-rec .th { position: relative; } +.ann-rec .th .pre { margin-right: 20px; } +.ann-rec.gone .th { color: var(--serious); } +.ann-rec .st { display: flex; gap: 6px; align-items: baseline; padding: 0 6px 4px; font-size: 11px; color: var(--ink-3); white-space: nowrap; overflow: hidden; } +.ann-rec .st b { font-weight: 500; } +.ann-rec .st b.stale { color: var(--serious); } +.ann-rec .st b.unsaved { color: var(--warn); } +.ann-rec[data-stale="1"] { border-color: var(--serious); } +.badge { position: absolute; right: 5px; top: 5px; width: 9px; height: 9px; border-radius: 50%; box-shadow: 0 0 0 2px var(--surface); } +.badge.none { background: var(--surface); box-shadow: inset 0 0 0 1.5px var(--ink-3), 0 0 0 2px var(--surface); } +.badge.draft { background: var(--accent); } +.badge.accepted { background: var(--good); } +.badge.needs-work { background: var(--warn); } +.badge.rejected { background: var(--serious); } +.ann-view.viewer { position: static; max-height: none; overflow: visible; } +.ann-nav { display: flex; align-items: center; justify-content: space-between; gap: 8px; } +.ann-stage { width: 100%; min-width: 0; display: flex; justify-content: center; background: var(--surface-2); border-radius: var(--radius); } +.ann-imgwrap { position: relative; max-width: 100%; } +.ann-imgwrap img { display: block; width: 100%; height: 100%; user-select: none; -webkit-user-drag: none; } +.ann-imgwrap.pixels img { image-rendering: pixelated; } +.ann-imgwrap canvas.ann-surface { position: absolute; inset: 0; width: 100%; height: 100%; background: transparent; border-radius: 0; cursor: crosshair; touch-action: none; } +.iv canvas.ann-wave { cursor: crosshair; touch-action: none; } +.ann-surface:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; } +.ann-help { max-width: none; } +.ann-text mark.ann-span { background: color-mix(in srgb, var(--k-refinement) 26%, transparent); color: inherit; border-bottom: 2px solid var(--k-refinement); cursor: pointer; } +.ann-text mark.ann-span.many { background: color-mix(in srgb, var(--k-refinement) 42%, transparent); } +.ann-text mark.ann-span.sel { background: color-mix(in srgb, var(--accent) 30%, transparent); border-bottom-color: var(--accent); } +.ann-text .ann-tag { display: inline-block; margin: 0 2px; padding: 0 4px; border-radius: 3px; font: 10.5px/1.5 var(--font); background: var(--k-refinement); color: var(--accent-ink); vertical-align: 1px; user-select: none; -webkit-user-select: none; } +.ann-text .ann-tag.sel { background: var(--accent); } +.ann-formbox { gap: 14px; } +.ann-formbox h2.mono { font-family: var(--mono); font-size: 14px; } +.ann-flag { font-size: 11.5px; font-weight: 500; } +.ann-flag.stale { color: var(--serious); } +.ann-flag.unsaved { color: var(--warn); } +.ann-field { display: grid; gap: 6px; min-width: 0; } +.ann-field h3 { margin: 0; } +.form .radio.ann-terms label { padding: 3px 6px 3px 10px; } +.form .radio.ann-terms .kbd { margin-left: 2px; } +.ann-decide { justify-self: start; } +.seg button.ok[aria-pressed="true"] { background: var(--good-wash); color: var(--ink); box-shadow: inset 0 -2px 0 var(--good); } +.seg button.warn[aria-pressed="true"] { background: var(--warn-wash); color: var(--ink); box-shadow: inset 0 -2px 0 var(--warn); } +.seg button.bad[aria-pressed="true"] { background: var(--serious-wash); color: var(--ink); box-shadow: inset 0 -2px 0 var(--serious); } +.seg button[disabled] { cursor: not-allowed; opacity: .55; } +.ann-newterm { max-width: 260px; } +.ann-items { list-style: none; margin: 0; padding: 0; display: grid; gap: 4px; } +.ann-items li { display: grid; grid-template-columns: auto minmax(0, 1fr); gap: 6px; align-items: start; padding: 5px 6px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); min-width: 0; } +.ann-items li.sel { border-color: var(--accent); box-shadow: inset 3px 0 0 var(--accent); } +.ann-items .linkish { font: 600 12.5px var(--mono); color: var(--accent); background: none; border: 0; padding: 0 4px; cursor: pointer; min-width: 22px; } +.ann-items select { max-width: 116px; min-height: 26px; padding: 1px 4px; font-size: 13px; } +.form .ann-items input.ann-num { width: 66px; min-height: 26px; padding: 1px 4px; } +.form .ann-items input.ann-saying { flex: 1 1 140px; min-width: 0; width: auto; min-height: 26px; padding: 1px 6px; } +.ann-items .ann-coords { color: var(--ink-2); flex: 1 1 auto; } +.ann-items .ann-covered { flex: 1 1 100%; font-size: 12.5px; color: var(--ink-2); overflow-wrap: anywhere; } +.ann-items .ann-fields { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; min-width: 0; } +.ann-items .linkish { min-height: 26px; } +.ann-items .ann-fields > .btn.ghost { margin-left: auto; } +.ann-stale { display: grid; gap: 4px; padding: 8px 10px; border: 1px solid var(--serious); border-left-width: 3px; border-radius: var(--radius); background: var(--serious-wash); } +.ann-stale p:first-child { color: var(--serious); font-weight: 500; } +.ann-empty { display: grid; gap: 12px; justify-items: start; } +.ann-new .form h3 { margin: 4px 0 0; } + +/* ---- The Pumas endpoint (generate.js): on Train beside the environments, the Generate form, and Suggest (annotate.js). ---- */ +.ep-models > li.ep-model { display: grid; grid-template-columns: minmax(150px, 220px) minmax(0, 1fr) minmax(120px, 190px); gap: 12px; align-items: start; padding: 8px 8px; } +.ep-model > span { min-width: 0; } +.ep-model .title, .ep-model .sub { display: block; } +.ep-model .title { font-weight: 500; overflow-wrap: anywhere; } +.ep-model .title.mono { font-family: var(--mono); font-size: 13px; } +.ep-model .sub { color: var(--ink-2); font-size: 12.5px; overflow-wrap: anywhere; } +.ep-model .sub.mono { font-family: var(--mono); font-size: 12px; color: var(--ink-3); } +.ep-model .ep-makes b { font-weight: 500; color: var(--ink); } +.ep-model .ep-off { color: var(--ink-3); } +.ep-model .ep-load { text-align: right; } +@media (max-width: 1100px) { .ep-models > li.ep-model { grid-template-columns: minmax(120px, 170px) minmax(0, 1fr); } .ep-model .ep-load { grid-column: 2; text-align: left; } } +.row.ep-row .sub { white-space: normal; } +.ep-part { display: grid; gap: 6px; } +.ep-part .note { margin: 0; } +.ep-found { display: grid; gap: 6px; } +.ep-found .rows { margin-top: 4px; } +.ep-found .row.ep-hit { cursor: default; } +.gen-form .gen-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(150px, 1fr)); gap: 10px; align-items: start; } +.gen-form .gen-line { margin: -4px 0 0; } +.gen-form .ann-field h3 { margin: 4px 0 0; } +.gen-none { display: grid; gap: 10px; justify-items: start; } +.ann-suggest { display: grid; gap: 8px; padding: 9px 10px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface-2); min-width: 0; } +.ann-suggest-row { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; min-width: 0; } +.ann-suggest-model { display: inline-flex; flex-wrap: wrap; gap: 6px; align-items: center; font-size: 12.5px; color: var(--ink-2); min-width: 0; flex: 1 1 200px; } +.ann-suggest-model select { min-width: 0; max-width: 100%; flex: 1 1 140px; font: inherit; font-size: 13px; color: var(--ink); background: var(--surface); border: 1px solid var(--line-strong); border-radius: var(--radius); padding: 3px 6px; } +.ann-sugmark { margin: 0; padding: 2px 0 2px 8px; border-left: 3px solid var(--accent); color: var(--ink); overflow-wrap: anywhere; } +.ann-sugmark.changed { border-left-color: var(--ink-3); color: var(--ink-2); font-weight: 400; } +.ann-batch { border-top: 1px solid var(--line); padding-top: 8px; } +.ann-batch .ann-batch-line { flex: 1 1 180px; min-width: 0; overflow-wrap: anywhere; font-variant-numeric: tabular-nums; } +.ann-sugrefusal { font-size: 12.5px; } + +/* What a run's training script logged (training.js, spec-t1.md section 6), in the run's drawer. */ +.trn { display: grid; gap: 8px; min-width: 0; } +.trn[aria-busy="true"] { cursor: progress; } +.trn h4 { margin: 4px 0 0; font-size: 13px; } +.trn-at { margin: 0; color: var(--ink-2); font-variant-numeric: tabular-nums; } +.trn-series, .trn-controls { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 10px; min-width: 0; } +.trn-series .btn { max-width: 100%; overflow: hidden; text-overflow: ellipsis; } +.trn-smooth { display: inline-flex; align-items: center; gap: 6px; } +.trn-smooth input { width: 110px; } +.trn-smooth .num { min-width: 2.6em; font-variant-numeric: tabular-nums; } +.trn-runs { display: grid; gap: 6px; min-width: 0; } +.trn-runs select { max-width: 100%; min-width: 0; } +.trn-legend { list-style: none; margin: 0; padding: 0; display: grid; gap: 2px; } +.trn-legend li { display: flex; align-items: center; gap: 8px; min-width: 0; font-size: 13px; } +.trn-legend li > span:nth-child(2) { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; } +.trn-legend .trn-remove { min-height: 22px; padding: 0 6px; margin-left: auto; } +.trn-swatch { flex: none; width: 14px; height: 3px; border-radius: 2px; display: inline-block; } +.trn-fig { margin: 0; display: grid; gap: 2px; min-width: 0; } +.trn-cap { color: var(--ink-2); overflow-wrap: anywhere; } +.trn-cap b { color: var(--ink); font-weight: 600; } +.trn-plot { position: relative; min-width: 0; } +canvas.trn-chart { display: block; width: 100%; cursor: crosshair; } +.trn-note { margin: 0; } +.trn-note:empty { display: none; } +.trn-tip { position: absolute; z-index: 3; pointer-events: none; display: grid; gap: 2px; padding: 6px 8px; max-width: 100%; font-size: 12px; background: var(--surface); color: var(--ink); border: 1px solid var(--line-strong); border-radius: var(--radius); box-shadow: 0 2px 8px rgba(0, 0, 0, .14); font-variant-numeric: tabular-nums; } +.trn-tip[hidden] { display: none; } +.trn-tip-head { color: var(--ink-2); } +.trn-tip-row { display: flex; align-items: center; gap: 6px; } +.trn-hists, .trn-samples-box { display: grid; gap: 6px; min-width: 0; border-top: 1px solid var(--line); padding-top: 8px; } +.trn-strip-row { display: grid; gap: 2px; min-width: 0; } +.trn-strip-label { display: flex; flex-wrap: wrap; gap: 0 4px; min-width: 0; overflow-wrap: anywhere; } +canvas.trn-strip { display: block; width: 100%; background: var(--surface-2); border-radius: 2px; } +.trn-samples { display: flex; gap: 8px; overflow-x: auto; padding-bottom: 4px; min-width: 0; } +.trn-sample { margin: 0; flex: none; display: grid; gap: 2px; justify-items: center; } +.trn-thumb { display: block; width: 72px; height: 72px; object-fit: contain; background: var(--surface-2); border-radius: 3px; image-rendering: pixelated; } +div.trn-thumb { display: grid; place-items: center; text-align: center; padding: 4px; } +.trn-problems { color: var(--warn); margin: 0; } + +/* Curation by embeddings (curation.js, docs/plans/training-and-curation/spec-t5.md section 6): the Map mode of a + dataset opened. The categorical colours are the dataviz reference palette's eight, in its fixed order, stepped for + each surface. */ +:root { --cat-1: #2a78d6; --cat-2: #eb6834; --cat-3: #1baf7a; --cat-4: #eda100; --cat-5: #e87ba4; --cat-6: #008300; --cat-7: #6250d6; --cat-8: #e34948; } +@media (prefers-color-scheme: dark) { :root:not([data-theme="light"]) { --cat-1: #3987e5; --cat-2: #d95926; --cat-3: #199e70; --cat-4: #c98500; --cat-5: #d55181; --cat-6: #008300; --cat-7: #9085e9; --cat-8: #e66767; } } +:root[data-theme="dark"] { --cat-1: #3987e5; --cat-2: #d95926; --cat-3: #199e70; --cat-4: #c98500; --cat-5: #d55181; --cat-6: #008300; --cat-7: #9085e9; --cat-8: #e66767; } +.cur { display: grid; gap: 10px; min-width: 0; } +.cur-head { display: grid; gap: 6px; min-width: 0; } +.cur-head select { max-width: 100%; } +.cur-facts { margin: 0; overflow-wrap: anywhere; } +.cur-bar { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 12px; min-width: 0; } +.cur-pick, .cur-num { display: inline-flex; align-items: center; gap: 6px; font-size: 13px; color: var(--ink-2); } +.cur-pick select { max-width: 220px; } +.cur-num input { width: 6.5em; } +.cur-map { position: relative; min-width: 0; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); overflow: hidden; } +.cur-map canvas { display: block; width: 100%; touch-action: none; cursor: crosshair; } +.cur-map canvas:focus-visible { outline: 2px solid var(--focus); outline-offset: -2px; } +.cur-tip { position: absolute; pointer-events: none; display: grid; gap: 2px; max-width: 240px; padding: 4px 8px; font-size: 12px; background: var(--surface); color: var(--ink); border: 1px solid var(--line-strong); border-radius: var(--radius); box-shadow: 0 2px 6px rgba(0, 0, 0, .12); overflow-wrap: anywhere; } +.cur-tip span { color: var(--ink-2); } +.cur-legend > div { display: flex; flex-wrap: wrap; gap: 4px 14px; font-size: 12px; color: var(--ink-2); } +.cur-key { display: inline-flex; align-items: center; gap: 6px; } +.cur-key .sw { width: 10px; height: 10px; border-radius: 50%; flex: none; } +.cur-key .sw.thin { border-radius: 2px; background: color-mix(in srgb, var(--warn) 22%, var(--surface)); border: 1px solid var(--line-strong); } +.cur-ramp { display: inline-block; width: 80px; height: 10px; border-radius: 2px; background: linear-gradient(90deg, var(--ramp-lo), var(--ramp-hi)); vertical-align: middle; } +.cur-chosen > div { display: grid; gap: 8px; } +.cur-chosen-line { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } +.cur-form { border: 1px solid var(--line); border-radius: var(--radius); padding: 10px; background: var(--surface); } +.good-line { color: var(--good); margin: 0; } +.cur-lists > div { display: grid; gap: 8px; min-width: 0; } +.cur-tabs { flex-wrap: wrap; max-width: 100%; justify-self: start; } +.cur-controls { display: flex; flex-wrap: wrap; gap: 8px 14px; align-items: center; } +.cur-list > p { margin: 0; } +.cur-rows { max-height: 320px; overflow: auto; } +.cur-row { grid-template-columns: minmax(0, 1fr) auto; gap: 0 12px; padding: 5px 8px; } +.cur-row .title { overflow-wrap: anywhere; } +.cur-row .num { font-family: var(--mono); font-size: 12px; color: var(--ink-2); grid-row: 1 / span 2; grid-column: 2; align-self: center; } +.cur-row .sub { grid-column: 1; } +.frz-records { margin: 0 0 4px; overflow-wrap: anywhere; } diff --git a/static/project-workspace/charts.js b/static/project-workspace/charts.js new file mode 100644 index 00000000..1cbe4a5e --- /dev/null +++ b/static/project-workspace/charts.js @@ -0,0 +1,236 @@ +'use strict'; +// Chart helpers in plain SVG, shared by the map's track and the charts of the +// other tabs: element builders, a value scale fitted to the values it shows, +// number formats and the hover tip. Every value a chart draws is also written +// as text somewhere on the page; tips enhance, never gate. +window.LabCharts = (() => { + const svgNS = 'http://www.w3.org/2000/svg'; + const el = (name, attrs = {}, text) => { const n = document.createElementNS(svgNS, name); for (const k in attrs) n.setAttribute(k, attrs[k]); if (text !== undefined) n.textContent = text; return n; }; + const h = (tag, cls, text) => { const n = document.createElement(tag); if (cls) n.className = cls; if (text !== undefined) n.textContent = text; return n; }; + + /** A number in at most four significant figures, scientific below 0.01 and from 1e5. */ + function fmtSci(v) { + if (v == null || !Number.isFinite(v)) return '—'; + const a = Math.abs(v); + if (a === 0) return '0'; + if (a >= 0.01 && a < 1e5) return a >= 100 ? String(Math.round(v * 10) / 10) : v.toPrecision(3).replace(/(\.\d*?)0+$/, '$1').replace(/\.$/, ''); + const e = Math.floor(Math.log10(a)); + return (v / Math.pow(10, e)).toFixed(2) + 'e' + e; + } + const tickLabel = v => v >= 0.01 && v < 1e5 ? String(v) : '1e' + Math.round(Math.log10(v)); + + /** + * A scale for values: logarithmic over whole decades when every value is + * positive, linear from zero otherwise. at(v) is 0 at the low end and 1 at + * the high end; good(v) is the share of the scale that is better than the + * worst end, by better ('lower' or 'higher'), for bars that grow as a value + * improves. + */ + function scale(values, better = 'lower') { + const xs = values.filter(v => v != null && Number.isFinite(v)); + if (!xs.length) return null; + const positive = xs.every(v => v > 0); + let lo, hi, at, ticks; + if (positive) { + lo = Math.floor(Math.log10(Math.min(...xs))); hi = Math.ceil(Math.log10(Math.max(...xs))); + if (hi === lo) hi = lo + 1; + at = v => (Math.log10(Math.min(Math.max(v, Math.pow(10, lo)), Math.pow(10, hi))) - lo) / (hi - lo); + const step = Math.ceil((hi - lo) / 5); ticks = []; + for (let e = hi; e >= lo; e -= step) ticks.push(Math.pow(10, e)); + } else { + lo = Math.min(0, ...xs); hi = Math.max(...xs); if (hi === lo) hi = lo + 1; + at = v => (v - lo) / (hi - lo); + ticks = [hi, (hi + lo) / 2, lo]; + } + return { log: positive, at, ticks, label: positive ? tickLabel : fmtSci, good: v => better === 'higher' ? at(v) : 1 - at(v) }; + } + + /** A tip inside host, placed near (x, y) in host coordinates and kept inside it. */ + function tooltip(host) { + const t = h('div', 'tip'); t.hidden = true; host.appendChild(t); + return { + node: t, + show(x, y, rows) { + t.textContent = ''; + for (const r of rows) { + if (r.b) { t.appendChild(h('b', null, r.b)); continue; } + const line = h('div'); if (r.v) { line.appendChild(h('span', 'v', r.v)); line.appendChild(document.createTextNode(' ')); } + line.appendChild(document.createTextNode(r.l || '')); t.appendChild(line); + } + t.hidden = false; + t.style.left = Math.max(4, Math.min(x + 14, host.clientWidth - t.offsetWidth - 8)) + 'px'; + t.style.top = Math.max(4, Math.min(y + 12, host.clientHeight - t.offsetHeight - 8)) + 'px'; + }, + hide() { t.hidden = true; } + }; + } + // ---- Charts of benchmark results (the Benchmarks and Analysis tabs) -------- + // Each draws from data the page holds (the chart operation's points, or + // per-case values) at a given pixel width, so its text stays at the type + // scale whatever the window; every point can be hovered and focused, and + // Enter or a click selects it when the chart is given onSelect. + + /** A nice step (1, 2 or 5 times a power of ten) near span / count. */ + function niceStep(span, count) { const raw = span / Math.max(1, count), p = Math.pow(10, Math.floor(Math.log10(raw))), f = raw / p; return (f <= 1 ? 1 : f <= 2 ? 2 : f <= 5 ? 5 : 10) * p; } + /** + * The brief's axis: logarithmic over whole decades when every value is + * positive and they span at least two decades, linear otherwise (from zero + * when every value is positive). at(v) is 0 at the low end, 1 at the high. + */ + function axis(values) { + const xs = values.filter(v => v != null && Number.isFinite(v)); + if (!xs.length) return null; + const min = Math.min(...xs), max = Math.max(...xs); + if (min > 0 && max / min >= 100) { + const lo = Math.floor(Math.log10(min)), hi = Math.max(lo + 1, Math.ceil(Math.log10(max))), step = Math.ceil((hi - lo) / 6), ticks = []; + for (let e = lo; e <= hi; e += step) ticks.push(Math.pow(10, e)); + return { log: true, ticks, at: v => (Math.log10(Math.max(v, Math.pow(10, lo))) - lo) / (hi - lo), label: tickLabel }; + } + let lo = min >= 0 ? 0 : min, hi = max <= 0 ? 0 : max; if (hi === lo) hi = lo + (Math.abs(lo) || 1); + const step = niceStep(hi - lo, 5); lo = Math.floor(lo / step) * step; hi = Math.ceil(hi / step - 1e-9) * step; + const ticks = []; for (let v = lo; v <= hi + step / 2; v += step) ticks.push(Math.abs(v) < step / 1e6 ? 0 : v); + return { log: false, ticks, at: v => (v - lo) / (hi - lo), label: fmtSci }; + } + const axisWords = (title, a) => title + (a && a.log ? ' (log scale)' : ''); + /** Wires hover, focus, click and Enter of a focusable mark to a tip and onSelect. */ + function interactive(node, host, svg, at, rows, onSelect) { + const tip = host._tip || (host._tip = tooltip(host)); + const show = () => { const r = host.getBoundingClientRect(), s = svg.getBoundingClientRect(), [x, y] = at(); tip.show(s.left - r.left + x * s.width / svg.viewBox.baseVal.width, s.top - r.top + y * s.height / svg.viewBox.baseVal.height, rows()); }; + node.addEventListener('pointerenter', show); node.addEventListener('focus', show); + node.addEventListener('pointerleave', () => tip.hide()); node.addEventListener('blur', () => tip.hide()); + if (onSelect) { node.addEventListener('click', onSelect); node.addEventListener('keydown', e => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); onSelect(); } }); } + } + function frame(width, height, label) { const svg = el('svg', { viewBox: `0 0 ${width} ${height}`, role: 'group', 'aria-label': label }); return svg; } + function yGrid(svg, ax, Y, L, R, W) { const grid = el('g', { class: 'grid' }), labels = el('g', { class: 'axis' }); for (const v of ax.ticks) { grid.appendChild(el('line', { x1: L, y1: Y(v), x2: W - R, y2: Y(v) })); labels.appendChild(el('text', { x: L - 8, y: Y(v) + 4, 'text-anchor': 'end' }, ax.label(v))); } svg.appendChild(grid); svg.appendChild(labels); } + const textWidth = (text, size = 12) => text.length * size * 0.56; + + /** + * A scatter of points {id, name, x, y, kind, frontier, rows} with the + * frontier drawn through the points marked on it (in x order); the ends of + * the frontier are named. opts: width, height, xTitle, yTitle, xBetter, + * yBetter, selected, onSelect(id), dimOff (dim points off the frontier). + */ + function scatter(host, points, opts) { + host.textContent = ''; host.classList.add('chart'); host._tip = null; + const W = Math.max(360, opts.width), H = opts.height || 340, L = 64, R = 20, T = 22, B = 40; + const xa = axis(points.map(p => p.x)), ya = axis(points.map(p => p.y)); + const X = v => L + xa.at(v) * (W - L - R), Y = v => T + (1 - ya.at(v)) * (H - T - B); + const svg = frame(W, H, axisWords(opts.yTitle, ya) + ' against ' + axisWords(opts.xTitle, xa) + ', ' + points.length + ' points'); + yGrid(svg, ya, Y, L, R, W); + const ax = el('g', { class: 'axis' }); + xa.ticks.forEach((v, i) => { ax.appendChild(el('line', { x1: X(v), y1: H - B, x2: X(v), y2: H - B + 4, class: 'base' })); ax.appendChild(el('text', { x: X(v), y: H - B + 16, 'text-anchor': i === xa.ticks.length - 1 ? 'end' : 'middle' }, xa.label(v))); }); + ax.appendChild(el('line', { x1: L, y1: H - B, x2: W - R, y2: H - B, class: 'base' })); + ax.appendChild(el('text', { x: W - R, y: H - 4, 'text-anchor': 'end', class: 'label' }, axisWords(opts.xTitle, xa))); + ax.appendChild(el('text', { x: L, y: T - 8, class: 'label' }, axisWords(opts.yTitle, ya))); + svg.appendChild(ax); + const front = points.filter(p => p.frontier).sort((a, b) => a.x - b.x || a.y - b.y); + if (front.length > 1) svg.appendChild(el('path', { class: 'frontier', d: front.map((p, i) => (i ? 'L' : 'M') + X(p.x).toFixed(1) + ',' + Y(p.y).toFixed(1)).join(' ') })); + const order = points.slice().sort((a, b) => (a.id === opts.selected) - (b.id === opts.selected)); + for (const p of order) { + const g = el('g'), cx = X(p.x), cy = Y(p.y); + const hit = el('circle', { cx, cy, r: 11, class: 'hit', tabindex: 0, role: 'button', 'data-id': p.id, 'aria-label': p.name + ', ' + opts.yTitle + ' ' + fmtSci(p.y) + ', ' + opts.xTitle + ' ' + fmtSci(p.x) + (p.frontier ? ', on the frontier' : '') }); + g.appendChild(hit); g.appendChild(el('circle', { cx, cy, r: 6, class: 'pt ' + p.kind + ((opts.dimOff && !p.frontier) || p.dim ? ' dim' : '') })); + if (p.nsig) g.appendChild(el('circle', { cx, cy, r: 10, class: 'nsig' })); // on the frontier, not told apart from a neighbour (spec-t3) + if (p.id === opts.selected) g.appendChild(el('circle', { cx, cy, r: 9, class: 'sel' })); + interactive(hit, host, svg, () => [cx, cy], p.rows, opts.onSelect && (() => opts.onSelect(p.id))); + svg.appendChild(g); + } + if (front.length) { // name the two ends of the frontier, as the mockup does + const lowY = opts.yBetter === 'higher' ? front.reduce((a, b) => b.y > a.y ? b : a) : front.reduce((a, b) => b.y < a.y ? b : a); + const lowX = opts.xBetter === 'higher' ? front.reduce((a, b) => b.x > a.x ? b : a) : front.reduce((a, b) => b.x < a.x ? b : a); + const word = (better, title) => { const t = (better === 'higher' ? 'highest ' : 'lowest ') + title.charAt(0).toLowerCase() + title.slice(1); return t.length > 34 ? t.slice(0, 33) + '…' : t; }; + const put = (p, text, dy) => { const w = textWidth(text), right = X(p.x) + 10 + w <= W - R; svg.appendChild(el('text', { x: right ? X(p.x) + 10 : X(p.x) - 10, y: Math.min(H - B - 4, Math.max(T + 10, Y(p.y) + dy)), 'text-anchor': right ? 'start' : 'end', class: 'label' }, text)); }; + put(lowY, lowY.short + ' · ' + word(opts.yBetter, opts.yName || opts.yTitle), 4); + if (lowX !== lowY) { // on the side the frontier does not leave by, so the line does not run through the words + const at = front.indexOf(lowX), next = front[at + 1] || front[at - 1]; + put(lowX, lowX.short + ' · ' + word(opts.xBetter, opts.xName || opts.xTitle), next && Y(next.y) >= Y(lowX.y) ? -12 : 18); + } + } + host.appendChild(svg); + return { frontier: front }; + } + + /** + * Values in training order {id, name, y, best, kind, rows}: the values as a + * line of points, the best so far as a step line in the good colour. + */ + function progression(host, points, opts) { + host.textContent = ''; host.classList.add('chart'); host._tip = null; + const W = Math.max(360, opts.width), H = opts.height || 240, L = 64, R = 20, T = 22, B = 36, n = points.length; + const ya = axis(points.map(p => p.y).concat(points.map(p => p.best))); + const X = i => L + (n > 1 ? i / (n - 1) : 0.5) * (W - L - R), Y = v => T + (1 - ya.at(v)) * (H - T - B); + const svg = frame(W, H, axisWords(opts.yTitle, ya) + ' for ' + n + ' subjects in training order, with the best so far'); + yGrid(svg, ya, Y, L, R, W); + const ax = el('g', { class: 'axis' }); + ax.appendChild(el('line', { x1: L, y1: H - B, x2: W - R, y2: H - B, class: 'base' })); + const every = Math.max(1, Math.ceil(n / Math.max(1, Math.floor((W - L - R) / 90)))); + points.forEach((p, i) => { if (i % every === 0 || i === n - 1) { const t = p.short.length > 12 ? p.short.slice(0, 11) + '…' : p.short; ax.appendChild(el('text', { x: X(i), y: H - B + 16, 'text-anchor': n === 1 ? 'middle' : i === 0 ? 'start' : i === n - 1 ? 'end' : 'middle' }, t)); } }); + ax.appendChild(el('text', { x: L, y: T - 8, class: 'label' }, axisWords(opts.yTitle, ya))); + ax.appendChild(el('text', { x: W - R, y: H - 4, 'text-anchor': 'end', class: 'label' }, 'training order')); + svg.appendChild(ax); + if (n > 1) { + svg.appendChild(el('path', { class: 'line', d: points.map((p, i) => (i ? 'L' : 'M') + X(i).toFixed(1) + ',' + Y(p.y).toFixed(1)).join(' ') })); + svg.appendChild(el('path', { class: 'best', d: points.map((p, i) => (i ? 'H' + X(i).toFixed(1) + 'V' : 'M' + X(i).toFixed(1) + ',') + Y(p.best).toFixed(1)).join(' ') })); + } + points.forEach((p, i) => { + const g = el('g'), cx = X(i), cy = Y(p.y); + const hit = el('circle', { cx, cy, r: Math.max(6, Math.min(11, (W - L - R) / Math.max(1, n) / 2)), class: 'hit', tabindex: 0, role: 'button', 'data-id': p.id, 'aria-label': p.name + ', ' + opts.yTitle + ' ' + fmtSci(p.y) + ', best so far ' + fmtSci(p.best) }); + g.appendChild(hit); g.appendChild(el('circle', { cx, cy, r: n > 80 ? 3 : 4, class: 'pt ' + p.kind + (p.dim ? ' dim' : '') })); + if (p.id === opts.selected) g.appendChild(el('circle', { cx, cy, r: 8, class: 'sel' })); + interactive(hit, host, svg, () => [cx, cy], p.rows, opts.onSelect && (() => opts.onSelect(p.id))); + svg.appendChild(g); + }); + host.appendChild(svg); + } + + /** Subjects best first {id, name, y, kind, rows, lo, hi}: one row each, a dot on the value axis with a hairline from its start, and a whisker from lo to hi when given (an interval, spec-t3). */ + function ranking(host, points, opts) { + host.textContent = ''; host.classList.add('chart'); host._tip = null; + const n = points.length, RH = n > 60 ? 16 : 20, W = Math.max(360, opts.width), L = Math.min(260, Math.max(120, W * 0.24)), R = 64, T = 26, H = T + n * RH + 8; + const xa = axis(points.map(p => p.y).concat(points.map(p => p.lo), points.map(p => p.hi))), X = v => L + xa.at(v) * (W - L - R); + const svg = frame(W, H, axisWords(opts.yTitle, xa) + ' for ' + n + ' subjects, best first'); + const grid = el('g', { class: 'grid' }), ax = el('g', { class: 'axis' }); + xa.ticks.forEach((v, i) => { grid.appendChild(el('line', { x1: X(v), y1: T - 4, x2: X(v), y2: H - 8 })); ax.appendChild(el('text', { x: X(v), y: T - 10, 'text-anchor': i === xa.ticks.length - 1 && xa.ticks.length > 1 ? 'end' : 'middle' }, xa.label(v))); }); + svg.appendChild(grid); svg.appendChild(ax); + points.forEach((p, i) => { + const cy = T + i * RH + RH / 2, cx = X(p.y), g = el('g'); + const max = Math.floor((L - 14) / 6.6), t = p.name.length > max ? p.name.slice(0, max - 1) + '…' : p.name; + g.appendChild(el('text', { x: L - 10, y: cy + 4, 'text-anchor': 'end', class: 'rk' + (p.id === opts.selected ? ' on' : '') }, t)); + g.appendChild(el('line', { x1: L, y1: cy, x2: cx, y2: cy, class: 'stem' })); + const ci = p.lo != null && p.hi != null; + const hit = el('rect', { x: 0, y: cy - RH / 2, width: W, height: RH, class: 'hit band', tabindex: 0, role: 'button', 'data-id': p.id, 'aria-label': p.name + ', ' + opts.yTitle + ' ' + fmtSci(p.y) + (ci ? ', 95% interval ' + fmtSci(p.lo) + ' to ' + fmtSci(p.hi) : '') }); + g.insertBefore(hit, g.firstChild); + if (ci) { const a = X(p.lo), b = X(p.hi), w = el('g', { class: 'ci', 'data-lo': p.lo, 'data-hi': p.hi }); w.appendChild(el('line', { x1: a, y1: cy, x2: b, y2: cy })); w.appendChild(el('line', { x1: a, y1: cy - 4, x2: a, y2: cy + 4 })); w.appendChild(el('line', { x1: b, y1: cy - 4, x2: b, y2: cy + 4 })); g.appendChild(w); } + g.appendChild(el('circle', { cx, cy, r: 4.5, class: 'pt ' + p.kind + (p.dim ? ' dim' : '') })); + if (p.id === opts.selected) g.appendChild(el('circle', { cx, cy, r: 8, class: 'sel' })); + g.appendChild(el('text', { x: W - 4, y: cy + 4, 'text-anchor': 'end', class: 'rv' }, fmtSci(p.y))); + interactive(hit, host, svg, () => [cx, cy], p.rows, opts.onSelect && (() => opts.onSelect(p.id))); + svg.appendChild(g); + }); + host.appendChild(svg); + } + + /** One mark per case {label, value, reason}: a bar for a value, a hollow tick at the base for a missing one. */ + function strip(host, items, opts) { + host.textContent = ''; host.classList.add('chart', 'cases'); host._tip = null; + const W = Math.max(360, opts.width), H = opts.height || 96, L = 64, R = 12, T = 10, B = 14, n = items.length; + const ya = axis(items.map(c => c.value)) || axis([0, 1]), Y = v => T + (1 - ya.at(v)) * (H - T - B); + const svg = frame(W, H, opts.label + ', ' + n + ' cases'); + yGrid(svg, ya, Y, L, R, W); + const step = (W - L - R) / Math.max(1, n), bw = Math.max(1.5, Math.min(18, step - (step > 4 ? 2 : 0.5))), base = Y(ya.log ? ya.ticks[0] : 0); + svg.appendChild(el('line', { x1: L, y1: H - B, x2: W - R, y2: H - B, class: 'base' })); + items.forEach((c, i) => { + const x = L + i * step + (step - bw) / 2, g = el('g'); + const has = c.value != null && Number.isFinite(c.value), top = has ? Y(c.value) : H - B - 3; + const mark = has ? el('rect', { x, y: Math.min(top, base), width: bw, height: Math.max(1.5, Math.abs(base - top)), rx: Math.min(1.5, bw / 3), class: 'b' + (c.id === opts.selected ? ' sel' : '') }) + : el('rect', { x, y: H - B - 3, width: bw, height: 3, class: 'b none' }); + const hit = el('rect', { x: L + i * step, y: T, width: Math.max(step, 2), height: H - T - B, class: 'hit', tabindex: 0, role: 'button', 'aria-label': c.label + ', ' + (has ? fmtSci(c.value) : 'no value' + (c.reason ? ': ' + c.reason : '')) }); + g.appendChild(hit); g.appendChild(mark); + interactive(hit, host, svg, () => [x + bw / 2, top], () => [{ b: c.label }, has ? { v: fmtSci(c.value), l: opts.valueTitle } : { l: 'no value' + (c.reason ? ': ' + c.reason : '') }], opts.onSelect && (() => opts.onSelect(c.id))); + svg.appendChild(g); + }); + host.appendChild(svg); + } + + return { el, h, fmtSci, scale, tooltip, axis, scatter, progression, ranking, strip }; +})(); diff --git a/static/project-workspace/compare.css b/static/project-workspace/compare.css new file mode 100644 index 00000000..a3d7b431 --- /dev/null +++ b/static/project-workspace/compare.css @@ -0,0 +1,86 @@ +/* The Compare tab, parallel coordinates and intervals (docs/plans/training-and-curation/spec-t3.md section 5), + on the tokens of base.css, so light and dark follow it. */ + +/* The compare set's fourth colour, and a model's colour in its column. */ +.cmp-chips .c.s4 i { background: var(--k-component); } +.cmpv .swatch, .cmpv-colhead .swatch { width: 10px; height: 10px; border-radius: 50%; display: inline-block; flex: none; margin-right: 6px; vertical-align: -1px; } +.s1 > .swatch { background: var(--k-architecture); } .s2 > .swatch { background: var(--k-refinement); } +.s3 > .swatch { background: var(--k-training-policy); } .s4 > .swatch { background: var(--k-component); } + +/* ---- The Compare tab ---- */ +.cmpv-head { display: grid; gap: 8px; min-width: 0; } +.cmpv-head .cmp-chips .iv-field { margin-left: 8px; font-size: 12.5px; color: var(--ink-2); } +.cmpv-empty { margin-top: 14px; } +.cmpv-table { margin-top: 18px; } +table.tv.cmpv { min-width: 100%; } +table.tv.cmpv th.cmpv-col { text-align: left; min-width: 150px; vertical-align: bottom; } +table.tv.cmpv th.cmpv-row { text-align: left; font-weight: 400; color: var(--ink-2); max-width: 340px; min-width: 120px; vertical-align: top; } +table.tv.cmpv tr.cmpv-group th { text-align: left; padding-top: 16px; font-weight: 600; color: var(--ink); border-bottom: 1px solid var(--line); } +table.tv.cmpv tr.cmpv-group .sum { margin-left: 10px; font-weight: 400; color: var(--ink-3); font-size: 12px; } +table.tv.cmpv td { vertical-align: top; } +.cmpv-path { display: block; overflow-wrap: anywhere; font-size: 12px; color: var(--ink); } +.cmpv-kind { display: block; } +td.cmpv-v { overflow-wrap: anywhere; max-width: 260px; font-size: 12px; } +table.tv.cmpv td.n { text-align: left; } +.cmpv-value { display: block; font-variant-numeric: tabular-nums; } +td .ci { display: block; font-size: 11.5px; color: var(--ink-3); font-weight: 400; font-variant-numeric: tabular-nums; white-space: nowrap; } +.cmpv-n { display: block; } +td.cmpv-on { color: var(--good); } +td.cmpv-nsig { color: var(--warn); } +.cmpv-out { margin-top: 26px; display: grid; gap: 10px; min-width: 0; } +.cmpv-out .sect { margin-bottom: 0; flex-wrap: wrap; } +.cmpv-outhead { display: grid; gap: 4px; min-width: 0; } +.cmpv-outhead p { margin: 0; overflow-wrap: anywhere; } +.cmpv-outhead select { max-width: min(520px, 100%); min-width: 0; } +.cmpv-grid { position: relative; display: grid; gap: 10px; min-width: 0; } +.cmpv-grid[aria-busy="true"] { cursor: progress; } +.cmpv-cells { display: grid; grid-template-columns: repeat(var(--cols, 3), minmax(0, 1fr)); gap: 10px; min-width: 0; } +.cmpv-colhead { font-size: 12.5px; font-weight: 600; color: var(--ink); overflow-wrap: anywhere; min-width: 0; } +.cmpv-orow { display: grid; gap: 4px; min-width: 0; } +.cmpv-otitle { display: flex; flex-wrap: wrap; gap: 4px 14px; align-items: center; min-width: 0; } +.cmpv-barbox { flex: 1 1 180px; max-width: 360px; min-width: 0; } +.cmpv-cell { margin: 0; min-width: 0; } +.cmpv-missing { margin: 0; padding: 10px 0; } +.cmpv-figs dl { margin: 0; display: grid; gap: 2px; font-size: 12px; } +.cmpv-figs dl > div { display: flex; justify-content: space-between; gap: 8px; min-width: 0; } +.cmpv-figs dt { color: var(--ink-2); overflow-wrap: anywhere; } +.cmpv-figs dd { margin: 0; font-variant-numeric: tabular-nums; } +.cmpv-where { margin: 0; } + +/* ---- Intervals on the charts ---- */ +.chart .ci line { stroke: var(--ink-3); stroke-width: 1.25; } +.chart .nsig { fill: none; stroke: var(--warn); stroke-width: 1.5; stroke-dasharray: 3 2; } + +/* ---- Parallel coordinates ---- */ +.pc { margin-top: 26px; min-width: 0; } +.pc .sect { flex-wrap: wrap; } +.pc-count { margin-left: auto; color: var(--ink-2); } +.pc-draw { min-width: 0; } +.pc-svg { display: block; width: 100%; height: auto; touch-action: none; } +.pc .pcl { fill: none; stroke: var(--k-component); stroke-width: 1.4; opacity: .7; } +.pc .pcl.training-policy { stroke: var(--k-training-policy); } +.pc .pcl.architecture { stroke: var(--k-architecture); } +.pc .pcl.refinement { stroke: var(--k-refinement); } +.pc .pcl.other, .pc .pcl.unset { stroke: var(--ink-3); } +.pc .pcl.out { opacity: .08; } +.pc .pcl.on { stroke-width: 3; opacity: 1; } +.pc .pc-line { stroke: var(--line-strong); stroke-width: 1; } +.pc .pc-title { fill: var(--ink); font-size: 11.5px; font-weight: 600; font-family: var(--font); } +.pc .pc-sub { fill: var(--ink-3); font-size: 10.5px; font-family: var(--font); } +.pc .pc-tick { fill: var(--ink-2); font-size: 10.5px; font-family: var(--font); paint-order: stroke; stroke: var(--surface); stroke-width: 3px; stroke-linejoin: round; } +.pc .pc-brush { fill: var(--accent); fill-opacity: .22; stroke: var(--accent); stroke-width: 1; } +.pc .pc-hit { fill: transparent; cursor: ns-resize; } +.pc-axes { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; margin-top: 8px; } +.pc-ax { display: inline-flex; align-items: center; gap: 2px; padding: 1px 4px; border: 1px solid var(--line); border-radius: 999px; font-size: 12px; background: var(--surface); max-width: 100%; min-width: 0; } +.pc-ax.metric { border-color: var(--line-strong); } +.pc-ax .pc-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; max-width: 22ch; padding: 0 2px; } +.pc-ax button { border: 0; background: none; color: var(--ink-3); cursor: pointer; padding: 0 3px; font: inherit; line-height: 1.4; } +.pc-ax button:disabled { opacity: .3; cursor: default; } +.pc-ax button[aria-pressed="true"] { color: var(--accent); font-weight: 600; } +.pc-choose { margin-top: 8px; font-size: 12.5px; } +.pc-choose summary { cursor: pointer; color: var(--ink-2); } +.pc-list { list-style: none; margin: 6px 0 0; padding: 0; display: grid; gap: 2px; max-height: 260px; overflow: auto; } +.pc-list label { display: flex; gap: 6px; align-items: baseline; min-width: 0; } +.pc-list label .mono { overflow-wrap: anywhere; min-width: 0; } +.pc-filtered { margin: 0 0 6px; } +.chart g.pc-out > :not(.hit) { opacity: .15; } diff --git a/static/project-workspace/compare.js b/static/project-workspace/compare.js new file mode 100644 index 00000000..d0f4ad4d --- /dev/null +++ b/static/project-workspace/compare.js @@ -0,0 +1,342 @@ +'use strict'; +// The Compare tab (docs/plans/training-and-curation/spec-t3.md section 5.1), added to window.LabScreens: the page's +// compare set of two to four models, one column each. A table of who each model is (group, kind of change, the +// model it derives from, state, training date), what differs between their recipes (model_compare's differs: the +// settings, data and parent, only where they differ), and per benchmark that measures one of them its metrics with +// the tab's statistic, the 95 % interval from per-case values held here and the number of cases, the best of a row +// marked, and where each sits on the frontier of each tradeoff chart. Then the outputs: the cases the models' runs +// kept (spec-d6, fields.js's parts), matched by name; one case chosen; for each model its native field, its +// prediction and its signed error, three rows of panels with one colour scale a row (native and predicted share +// one), moved together by one control (the axis cut across, the position, "Remove the mean"). +// +// Updating in place (spec-w4): the tab is read and built like the others (tabs.js's paint) and put in place in one +// step; the person's moves through the outputs ask field_compare of every column at once (one round in flight, the +// latest wins) and draw the canvases again in place, keeping the case, axis, position and mean option in the +// address (cmpCase, cmpAxis, cmpIndex, cmpMeanFree). Choosing another case draws the tab again in place. +(() => { + const S = window.LabScreens, P = window.TuldokProject, Tree = window.LabTree, C = window.LabCharts, h = C.h; + const { sect, putInPlace, modelState } = S.parts; + const D = window.LabInside.drawing, R = P.product; + const F = () => window.LabFields.parts; + const fmt = C.fmtSci; + const STATISTICS = ['median', 'mean', 'p10', 'p90', 'min', 'max']; + const MOST = 4, CELL_MOST = 260; // models in the set; the CSS height of a panel at most + const button = (cls, text, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (onClick) b.addEventListener('click', onClick); return b; }; + const units = m => m.units && m.units !== '1' ? ' (' + m.units + ')' : ''; + const shapeText = s => s && s.length ? s.join(' × ') : '—'; + const shown = v => v === undefined ? '—' : v === null ? 'null' : typeof v === 'object' ? JSON.stringify(v) : String(v); + const KIND_WORDS = { group: 'group', change: 'kind of change', parent: 'derives from', data: 'data', setting: 'setting' }; + const ui4 = ui => ({ anaAxis: ui.cmpAxis, anaIndex: ui.cmpIndex, anaMeanFree: ui.cmpMeanFree }); + + // ---- Reading ---------------------------------------------------------------------------------------------------- + /** + * The outputs the tab shows: every model's kept cases, the cases kept by two models or more (comparable, matched by + * name), the one chosen, the person's place in it and every column's first slice (null where the model did not + * keep it or kept another shape). + */ + async function readOutputs(models, ui) { + const Fp = F(); + const kept = await Promise.all(models.map(m => Fp.keptCases(m).catch(error => ({ none: true, error })))); + const byName = new Map(); + kept.forEach((k, i) => { + if (!k || k.none) return; + for (const c of k.cases) { if (!c.comparable) continue; if (!byName.has(c.name)) byName.set(c.name, models.map(() => null)); const row = byName.get(c.name); if (!row[i]) row[i] = c; } + }); + const shared = [...byName].map(([name, cols]) => ({ name, cols, n: cols.filter(Boolean).length, split: cols.find(Boolean).split || null })) + .filter(x => x.n >= 2).sort((a, b) => b.n - a.n || a.name.localeCompare(b.name)); + if (!shared.length) return { kept, shared, chosen: null }; + const chosen = shared.find(x => x.name === ui.cmpCase) || shared[0]; + const lead = chosen.cols.find(Boolean); + const view = Fp.viewFor(lead, !ui.cmpCase || chosen.name === ui.cmpCase ? ui4(ui) : { anaMeanFree: ui.cmpMeanFree }); + const answers = await Promise.all(chosen.cols.map(c => c && sameShape(c, lead) ? R.fieldCompare(c.run, Fp.argsOf(c, view)).catch(error => ({ error })) : null)); + return { kept, shared, chosen, lead, view, answers }; + } + const sameShape = (a, b) => a.shape.length === b.shape.length && a.shape.every((n, i) => n === b.shape[i]); + + // ---- The screen --------------------------------------------------------------------------------------------------- + function compare(main, aside, state, act, data) { + const ui = state.ui, sel = state.sel.compare || null, g = S.parts.graphFor(data.map); + const models = state.cmp.filter(k => g.byId.has(k)), others = state.cmp.filter(k => !g.byId.has(k)); + const stat = STATISTICS.includes(ui.stat) ? ui.stat : 'median'; + S.paint(main, async () => { + if (models.length < 2) return { models }; + const [cmp, outputs] = await Promise.all([P.read('model_compare', { recipes: models, statistic: stat }), readOutputs(models, ui)]); + return { models, cmp, outputs }; + }, (frag, got) => { + frag.appendChild(head(g, state, act, stat, others)); + if (got.models.length < 2) { + frag.appendChild(h('p', 'empty cmpv-empty', 'Choose two to four models to compare: Compare in a model\'s drawer on the Map or on the Benchmarks tab, or the boxes of the Benchmarks table.')); + return; + } + const list = got.models.map(id => g.byId.get(id)); + frag.appendChild(table(got.cmp, list, g, act, sel)); + frag.appendChild(outputs(got.outputs, list, act, ui)); + }); + if (!sel || !g.byId.has(sel)) { aside.textContent = ''; aside._for = null; return false; } + S.inspectorModel(aside, sel, act, data, ins => S.compareActs(ins, sel, state, act, 'compare')); + const ins = aside.querySelector('.inspector'); if (ins) S.compareActs(ins, sel, state, act, 'compare'); + return true; + } + + /** The set as chips with ×, Clear, and the statistic. */ + function head(g, state, act, stat, others) { + const box = h('div', 'cmpv-head'); + box.appendChild(sect('Compare', state.cmp.length + ' of ' + MOST + ' models')); + const chips = h('div', 'cmp-chips'); + state.cmp.forEach((k, i) => { + const m = g.byId.get(k), name = m ? Tree.nameOf(m) : k.replace(/^subject:/, ''); + const c = h('span', 'c s' + (i + 1)); c.appendChild(h('i')); c.appendChild(document.createTextNode(name)); + const x = button(null, '×', () => act.toggleCompare(k)); x.setAttribute('aria-label', 'Remove ' + name); x.dataset.key = 'cmpv-x-' + k; c.appendChild(x); chips.appendChild(c); + }); + if (state.cmp.length) { const clear = button('btn sm ghost', 'Clear', () => act.clearCompare()); clear.dataset.key = 'cmpv-clear'; chips.appendChild(clear); } + const s = h('select'); s.setAttribute('aria-label', 'Statistic'); s.dataset.key = 'cmpv-stat'; + for (const v of STATISTICS) { const o = h('option', null, v.charAt(0).toUpperCase() + v.slice(1)); o.value = v; o.selected = v === stat; s.appendChild(o); } + s.addEventListener('change', () => act.setUI('stat', s.value)); + const l = h('label', 'iv-field'); l.appendChild(document.createTextNode('Statistic')); l.appendChild(s); chips.appendChild(l); + box.appendChild(chips); + if (others.length) box.appendChild(h('p', 'small muted', others.map(k => k.replace(/^subject:/, '')).join(', ') + (others.length === 1 ? ' is a benchmark subject' : ' are benchmark subjects') + ' made from no recipe on the map, and left out here.')); + return box; + } + + // ---- The table: who, what differs, the metrics and the frontier ------------------------------------------------------- + function table(cmp, list, g, act, sel) { + const sec = h('section', 'cmpv-table'); + const t = h('table', 'tv cmpv'), thead = h('thead'), hr = h('tr'); + hr.appendChild(h('th', 'cmpv-row', '')); + list.forEach((m, i) => { + const th = h('th', 'cmpv-col s' + (i + 1)); th.dataset.model = m.id; + const b = button('linkish', Tree.nameOf(m), () => act.select(m.id === sel ? null : m.id)); b.dataset.key = 'cmpv-open-' + m.id; b.title = 'Open ' + m.title; + th.appendChild(h('i', 'swatch')); th.appendChild(b); hr.appendChild(th); + }); + thead.appendChild(hr); t.appendChild(thead); + const tb = h('tbody'), n = list.length; + const group = (title, extra) => { const tr = h('tr', 'cmpv-group'); const th = h('th', null, title); th.colSpan = n + 1; if (extra) th.appendChild(h('span', 'sum', extra)); tr.appendChild(th); tb.appendChild(tr); }; + const row = (label, cells, cls, labelCls) => { const tr = h('tr', cls || null); const th = h('th', 'cmpv-row' + (labelCls ? ' ' + labelCls : '')); if (typeof label === 'string') th.textContent = label; else th.appendChild(label); tr.appendChild(th); for (const c of cells) tr.appendChild(c); tb.appendChild(tr); return tr; }; + const td = (text, cls) => h('td', cls || null, text); + // Who. + group('Who'); + row('Group', list.map(m => td(m.groupTitle || '—'))); + row('Kind of change', list.map(m => { const c = h('td'); c.appendChild(h('span', 'chip ' + m.kind, P.kindLabel(m.kind))); return c; })); + row('Derives from', list.map(m => { const p = m.parent && g.byId.get(m.parent); return td(p ? Tree.nameOf(p) : m.parent || '—'); })); + row('State', list.map(m => { const c = h('td'); c.appendChild(modelState(m.state)); return c; })); + row('Trained', list.map(m => td(m.trainedAt ? P.day(m.trainedAt) : '—', 'nowrap'))); + // What differs. + group('What differs', cmp.differs.length + (cmp.more ? ' of ' + (cmp.differs.length + cmp.more) : '') + ' paths differ · ' + cmp.same + ' the same'); + if (!cmp.differs.length) row('Nothing', list.map(() => td('the same', 'small faint'))); + for (const d of cmp.differs) { + const label = h('span'); label.appendChild(h('span', 'mono cmpv-path', d.path)); label.appendChild(h('span', 'cmpv-kind small faint', KIND_WORDS[d.kind] || d.kind)); + const tr = row(label, d.values.map((v, i) => td(d.present[i] ? shown(v) : '—', 'mono cmpv-v' + (d.present[i] ? '' : ' faint'))), 'cmpv-diff'); + tr.dataset.path = d.path; tr.dataset.kind = d.kind; + } + if (cmp.more) row('…', list.map(() => td(cmp.more + ' more not listed', 'small faint'))); + // The benchmarks. + if (!cmp.benchmarks.length) { group('Results'); row('None', list.map(() => td('no benchmark measures it', 'small faint'))); } + for (const b of cmp.benchmarks) { + group(b.title, cmp.statistic + ' · 95 % interval from ' + cmp.resamples.toLocaleString('en-GB') + ' resamples of the per-case values held here'); + for (const metric of b.metrics) { + const values = b.models.map(c => c && c.values[metric.id] && c.values[metric.id].value != null ? c.values[metric.id].value : null).filter(v => v != null); + const best = !values.length || metric.better === 'none' ? null : metric.better === 'higher' ? Math.max(...values) : Math.min(...values); + const tr = row(metric.title + units(metric), b.models.map(c => { + const v = c && c.values[metric.id]; + if (!c) return td('not a subject', 'small faint'); + if (!v || v.value == null) return td('—', 'n faint'); + const cell = h('td', 'n' + (values.length > 1 && v.value === best ? ' best' : '')); + cell.appendChild(h('span', 'cmpv-value', fmt(v.value))); + if (v.low != null) { const ci = h('span', 'ci', fmt(v.low) + ' – ' + fmt(v.high)); ci.title = 'The 95 % interval of the ' + cmp.statistic + ' from ' + cmp.resamples + ' resamples of its per-case values (seed ' + cmp.seed + ')'; cell.appendChild(ci); cell.dataset.low = String(v.low); cell.dataset.high = String(v.high); } + if (v.count != null) cell.appendChild(h('span', 'cmpv-n small faint', v.count.toLocaleString('en-GB') + ' cases')); + cell.dataset.value = String(v.value); + return cell; + }), 'cmpv-metric'); + tr.dataset.benchmark = b.benchmark; tr.dataset.metric = metric.id; + } + for (const f of b.frontiers) { + const tr = row('Frontier · ' + f.title, f.models.map((r, i) => { + if (!b.models[i]) return td('not a subject', 'small faint'); + if (!r) return td('not drawn', 'small faint'); + const words = r.on ? 'on the frontier' + (r.not_separated_from.length ? ', not separated from ' + r.not_separated_from.join(', ') : '') : 'off the frontier'; + const cell = td(words, r.on ? (r.not_separated_from.length ? 'cmpv-nsig' : 'cmpv-on') : 'faint'); + cell.dataset.on = String(r.on); cell.dataset.nsig = r.not_separated_from.join(','); + return cell; + }), 'cmpv-front'); + tr.dataset.chart = f.chart; + } + } + t.appendChild(tb); + const wrap = h('div', 'tv-wrap'); wrap.appendChild(t); sec.appendChild(wrap); + return sec; + } + + // ---- The outputs ---------------------------------------------------------------------------------------------------- + const ROWS = [['reference', 'Native', 'reference_f32'], ['predicted', 'Predicted', 'predicted_f32'], ['error', 'Signed error (predicted − native)', 'error_f32']]; + function outputs(o, list, act, ui) { + const sec = h('section', 'cmpv-out'); sec.dataset.key = 'cmpv-out'; + if (!o.chosen) { + sec.appendChild(sect('Outputs')); + const said = list.map((m, i) => { const k = o.kept[i]; return Tree.nameOf(m) + ': ' + (!k ? 'no product on this machine' : k.none ? 'kept no fields' : k.cases.filter(c => c.comparable).length + ' comparable cases'); }); + sec.appendChild(h('p', 'empty cmpv-none', 'No case is kept by two of these models. ' + said.join('; ') + '.')); + return sec; + } + const c = o.chosen, lead = o.lead, ctx = { o, list, act, view: o.view, answers: o.answers.slice(), busy: false, again: false }; + sec.appendChild(sect('Outputs', c.name + (c.split ? ' · ' + c.split : '') + ' · kept by ' + c.n + ' of ' + list.length)); + const hd = h('div', 'cmpv-outhead'); + if (o.shared.length > 1) { + const s = h('select'); s.setAttribute('aria-label', 'Case'); s.dataset.key = 'cmpv-case'; + for (const x of o.shared) { const op = h('option', null, x.name + (x.split ? ' · ' + x.split : '') + ' · ' + x.n + ' of ' + list.length); op.value = x.name; op.selected = x === c; s.appendChild(op); } + s.addEventListener('change', () => { act.keepUI('cmpAxis', null); act.keepUI('cmpIndex', null); act.setUI('cmpCase', s.value); }); + const l = h('label', 'iv-field'); l.appendChild(document.createTextNode('Case')); l.appendChild(s); hd.appendChild(l); + } else hd.appendChild(h('p', 'mono cmpv-casename', c.name)); + hd.appendChild(h('p', 'small muted', [shapeText(lead.shape), lead.dtype, 'the native field of each model\'s own run'].filter(Boolean).join(' · '))); + sec.appendChild(hd); + ctx.ctl = h('div', 'iv-ctl cmpv-ctl'); controls(ctx); sec.appendChild(ctx.ctl); + ctx.grid = grid(ctx); sec.appendChild(ctx.grid); + ctx.say = h('p', 'small muted cmpv-where'); sec.appendChild(ctx.say); + ctx.err = h('div', 'cmpv-err'); sec.appendChild(ctx.err); + redraw(ctx, false); + return sec; + } + /** The one control of every column: the axis, the sliders of the axes not drawn and "Remove the mean". */ + function controls(ctx) { + const Fp = F(), lead = ctx.o.lead, v = ctx.view, nd = lead.shape.length, names = Fp.axisNames(nd), items = []; + if (nd >= 3) { + const s = h('select'); s.setAttribute('aria-label', 'The axis cut across, for every model'); s.dataset.key = 'cmpv-axis'; + for (let a = 0; a < nd; a++) { const op = h('option', null, names[a] + ' (' + lead.shape[a] + ')'); op.value = String(a); op.selected = a === v.axis; s.appendChild(op); } + s.addEventListener('change', () => { v.axis = Number(s.value); ctx.act.keepUI('cmpAxis', String(v.axis)); controls(ctx); ask(ctx); }); + const l = h('label', 'iv-field'); l.appendChild(document.createTextNode('Cut across')); l.appendChild(s); items.push(l); + for (const a of Fp.fixedAxes(nd, v.axis)) { + const l2 = h('label', 'iv-field iv-slider'); l2.appendChild(h('span', null, a === v.axis ? 'Position on ' + names[a] : names[a])); + const r = h('input'); r.type = 'range'; r.min = '0'; r.max = String(lead.shape[a] - 1); r.value = String(v.index[a]); r.dataset.key = 'cmpv-slider-' + a; r.dataset.axis = String(a); + r.setAttribute('aria-label', 'Position on ' + names[a] + ' for every model, 0 to ' + (lead.shape[a] - 1)); + const at = h('span', 'num small', v.index[a] + ' of ' + lead.shape[a]); + r.addEventListener('input', () => { v.index[a] = Number(r.value); at.textContent = v.index[a] + ' of ' + lead.shape[a]; ctx.act.keepUI('cmpIndex', v.index.join(',')); ask(ctx); }); + l2.appendChild(r); l2.appendChild(at); items.push(l2); + } + } + const mean = button('btn sm', 'Remove the mean', () => { v.meanFree = !v.meanFree; mean.setAttribute('aria-pressed', String(v.meanFree)); ctx.act.keepUI('cmpMeanFree', v.meanFree ? '1' : null); ask(ctx); }); + mean.dataset.key = 'cmpv-mean'; mean.setAttribute('aria-pressed', String(v.meanFree)); mean.title = 'Take each field\'s mean from it before the error is formed, in every column'; + items.push(mean); + if (ctx.ctl.isConnected) { const frag = document.createDocumentFragment(); frag.append(...items); putInPlace(ctx.ctl, frag, document.getElementById('main')); } + else ctx.ctl.replaceChildren(...items); + } + /** Asks field_compare of every column for the place: one round in flight; a move meanwhile waits, the latest wins. */ + function ask(ctx) { + if (ctx.busy) { ctx.again = true; return; } + ctx.busy = true; ctx.again = false; ctx.grid.setAttribute('aria-busy', 'true'); + const Fp = F(), cols = ctx.o.chosen.cols, lead = ctx.o.lead; + Promise.all(cols.map(c => c && sameShape(c, lead) ? R.fieldCompare(c.run, Fp.argsOf(c, ctx.view), true).catch(error => ({ error })) : null)).then(answers => { + if (ctx.again || !ctx.grid.isConnected) return; + ctx.answers = answers; redraw(ctx, true); + }).finally(() => { + ctx.busy = false; + if (ctx.again && ctx.grid.isConnected) ask(ctx); else ctx.grid.removeAttribute('aria-busy'); + }); + } + + /** Three rows of panels, a column per model, a colour bar a row, the figures under each column. */ + function grid(ctx) { + const { list, o } = ctx, n = list.length, wrap = h('div', 'cmpv-grid'); wrap.style.setProperty('--cols', String(n)); + ctx.cells = {}; ctx.bars = {}; + const heads = h('div', 'cmpv-cells cmpv-heads'); + list.forEach((m, i) => { const d = h('div', 'cmpv-colhead s' + (i + 1)); d.appendChild(h('i', 'swatch')); d.appendChild(h('span', null, Tree.nameOf(m))); heads.appendChild(d); }); + wrap.appendChild(heads); + for (const [id, title] of ROWS) { + const rowEl = h('div', 'cmpv-orow'); rowEl.dataset.row = id; + const top = h('div', 'cmpv-otitle'); top.appendChild(h('span', 'fld-title', title)); + const bar = h('canvas', 'fld-bar cmpv-bar'); bar.setAttribute('aria-hidden', 'true'); const ends = h('div', 'iv-ends small faint cmpv-ends'); + const barBox = h('div', 'cmpv-barbox'); barBox.appendChild(bar); barBox.appendChild(ends); top.appendChild(barBox); + rowEl.appendChild(top); + ctx.bars[id] = { bar, ends }; D.paints(bar, () => drawBar(ctx, id)); + const cells = h('div', 'cmpv-cells'); + list.forEach((m, i) => { + const fig = h('figure', 'cmpv-cell'); fig.dataset.model = m.id; fig.dataset.row = id; + const c = o.chosen.cols[i]; + if (!c) fig.appendChild(h('p', 'small faint cmpv-missing', 'Not kept by this model.')); + else if (!sameShape(c, o.lead)) fig.appendChild(h('p', 'small faint cmpv-missing', 'Kept with another shape (' + shapeText(c.shape) + '), not drawn.')); + else { + const cv = h('canvas', 'fld-cv cmpv-cv'); cv.setAttribute('role', 'img'); cv.dataset.model = m.id; cv.dataset.row = id; fig.appendChild(cv); + ctx.cells[id + ' ' + i] = { cv, i, id }; + D.paints(cv, () => drawCell(ctx, id, i)); + cv.addEventListener('mousemove', ev => hover(ctx, id, i, ev)); cv.addEventListener('mouseleave', () => { ctx.tip.hidden = true; }); + } + cells.appendChild(fig); + }); + rowEl.appendChild(cells); wrap.appendChild(rowEl); + } + ctx.figs = h('div', 'cmpv-cells cmpv-figs'); wrap.appendChild(ctx.figs); + ctx.tip = h('div', 'fld-tip cmpv-tip'); ctx.tip.hidden = true; ctx.tip.setAttribute('role', 'status'); wrap.appendChild(ctx.tip); + return wrap; + } + /** The answers as floats, one scale a row over every column: native and predicted share one, the errors one diverging about zero. */ + function decode(ctx) { + const Fp = F(), got = ctx.answers.filter(a => a && !a.error); + const lows = got.map(a => a.range && a.range[0]).filter(v => v != null), highs = got.map(a => a.range && a.range[1]).filter(v => v != null); + ctx.range = lows.length ? [Math.min(...lows), Math.max(...highs)] : [0, 0]; + ctx.limit = Math.max(0, ...got.map(a => a.error_limit || 0)); + ctx.values = ctx.answers.map(a => a && !a.error ? { reference: D.floats(a.reference_f32), predicted: D.floats(a.predicted_f32), error: D.floats(a.error_f32) } : null); + const seq = Fp.sequential(ctx.range[0], ctx.range[1]); + ctx.scale = { reference: seq, predicted: seq, error: Fp.diverging(ctx.limit) }; + } + function redraw(ctx, draw) { + decode(ctx); + const Fp = F(), lead = ctx.o.lead, nd = lead.shape.length, names = Fp.axisNames(nd), first = ctx.answers.find(a => a && !a.error); + const where = !first ? '' : nd >= 3 ? 'through ' + first.axes.map(x => names[x]).join(' and ') + ' at ' + Fp.fixedAxes(nd, first.axis).map(x => names[x] + ' = ' + first.index[x]).join(', ') : nd === 2 ? 'the whole field' : 'the whole line'; + for (const { cv, i, id } of Object.values(ctx.cells)) { + const a = ctx.answers[i]; + cv.setAttribute('aria-label', ROWS.find(r => r[0] === id)[1] + ' of ' + Tree.nameOf(ctx.list[i]) + (a && !a.error ? ', ' + a.rows + ' by ' + a.columns + ' cells ' + where : ', not read')); + if (draw) drawCell(ctx, id, i); + } + ctx.bars.reference.ends.replaceChildren(...[fmt(ctx.range[0]), 'native and predicted, every column', fmt(ctx.range[1])].map(t => h('span', null, t))); + ctx.bars.predicted.ends.replaceChildren(...[fmt(ctx.range[0]), 'the same scale', fmt(ctx.range[1])].map(t => h('span', null, t))); + ctx.bars.error.ends.replaceChildren(...['−' + fmt(ctx.limit), '0', '+' + fmt(ctx.limit)].map(t => h('span', null, t))); + if (draw) for (const id of Object.keys(ctx.bars)) drawBar(ctx, id); + ctx.say.textContent = first ? 'A slice ' + where + (first.stride > 1 ? ', every ' + D.ordinal(first.stride) + ' row and column' : '') + ' of each model\'s field; one scale for native and predicted from ' + fmt(ctx.range[0]) + ' to ' + fmt(ctx.range[1]) + ', the errors on ±' + fmt(ctx.limit) + ' about zero' + (first.mean_free ? ', each field\'s mean removed first' : '') + '.' : ''; + const errors = ctx.answers.filter(a => a && a.error); + ctx.err.replaceChildren(...errors.slice(0, 1).map(a => window.LabActions.refusal(a.error))); + figures(ctx); + if (first) { ctx.grid.dataset.index = JSON.stringify(first.index); ctx.grid.dataset.axis = String(first.axis); ctx.grid.dataset.meanFree = String(first.mean_free); } + } + function drawCell(ctx, id, i) { + const cell = ctx.cells[id + ' ' + i], a = ctx.answers[i], vals = ctx.values && ctx.values[i]; if (!cell || !a || a.error || !vals) return; + const cv = cell.cv, w = cv.clientWidth; if (!w) return; + const cssH = a.rows === 1 ? 28 : Math.max(32, Math.min(CELL_MOST, Math.round(w * a.rows / a.columns))); + const size = D.fit(cv, cssH); if (!size) return; const gx = cv.getContext('2d'); gx.clearRect(0, 0, size[0], size[1]); + const rect = a.rows === 1 ? [0, 0, size[0], size[1]] : D.contain(a.rows, a.columns, size[0], size[1]); + cell.rect = rect.map(x => x / (window.devicePixelRatio || 1)); + D.heat(gx, vals[id], a.rows, a.columns, ctx.scale[id], rect[0], rect[1], rect[2], rect[3]); + cv.dataset.drawn = String((cv._drawn = (cv._drawn || 0) + 1)); + } + function drawBar(ctx, id) { + const b = ctx.bars[id], size = D.fit(b.bar, 10); if (!size) return; + const n = 64, vals = new Float32Array(n), m = ctx.limit || 0, range = ctx.range || [0, 0]; + for (let i = 0; i < n; i++) { const t = (i + 0.5) / n; vals[i] = id === 'error' ? -m + 2 * m * t : range[0] + (range[1] - range[0]) * t; } + D.heat(b.bar.getContext('2d'), vals, 1, n, ctx.scale[id], 0, 0, size[0], size[1]); + } + function hover(ctx, id, i, ev) { + const cell = ctx.cells[id + ' ' + i], a = ctx.answers[i]; if (!cell || !cell.rect || !a || a.error) return; + const box = cell.cv.getBoundingClientRect(), x = ev.clientX - box.left - cell.rect[0], y = ev.clientY - box.top - cell.rect[1]; + const col = Math.floor(x / cell.rect[2] * a.columns), row = Math.floor(y / cell.rect[3] * a.rows); + if (col < 0 || row < 0 || col >= a.columns || row >= a.rows) { ctx.tip.hidden = true; return; } + const k = row * a.columns + col, at = a.index.slice(), nd = ctx.o.lead.shape.length, names = F().axisNames(nd), v = ctx.values[i]; + if (nd === 1) at[0] = col * a.stride; else { at[a.axes[0]] = row * a.stride; at[a.axes[1]] = col * a.stride; } + ctx.tip.replaceChildren(h('b', null, Tree.nameOf(ctx.list[i])), h('div', 'mono fld-tip-at', nd === 3 ? at.map((q, j) => names[j] + ' ' + q).join(', ') : '(' + at.join(', ') + ')')); + for (const [key, word] of [['reference', 'Native'], ['predicted', 'Predicted'], ['error', a.mean_free ? 'Error, means removed' : 'Error']]) { const r = h('div', 'fld-tip-row'); r.dataset.value = key; r.appendChild(h('span', null, word)); r.appendChild(h('span', 'num', fmt(v[key][k]))); ctx.tip.appendChild(r); } + ctx.tip.dataset.cell = JSON.stringify(at); ctx.tip.dataset.model = ctx.list[i].id; + ctx.tip.hidden = false; + const wrap = ctx.grid.getBoundingClientRect(), tw = ctx.tip.offsetWidth; + let left = ev.clientX - wrap.left + 14; if (left + tw > wrap.width) left = Math.max(0, ev.clientX - wrap.left - tw - 14); + ctx.tip.style.left = left + 'px'; ctx.tip.style.top = (ev.clientY - wrap.top + 14) + 'px'; + } + const FIGURES = [['relative_l2', 'Relative L2'], ['mean_free_relative_l2', 'Mean-free relative L2'], ['rmse', 'RMSE'], ['max_abs_error', 'Largest |error|']]; + /** The whole-field figures under each column. */ + function figures(ctx) { + const items = ctx.answers.map((a, i) => { + const box = h('div', 'cmpv-fig'); box.dataset.model = ctx.list[i].id; + if (!a || a.error) { box.appendChild(h('p', 'small faint', a && a.error ? 'Not read.' : '—')); return box; } + const dl = h('dl', 'cmpv-figdl'); + for (const [key, word] of FIGURES) { const d = h('div'); d.dataset.fig = key; d.dataset.value = a.figures[key] == null ? '' : String(a.figures[key]); d.appendChild(h('dt', null, word)); d.appendChild(h('dd', 'num', fmt(a.figures[key]))); dl.appendChild(d); } + box.appendChild(dl); return box; + }); + ctx.figs.replaceChildren(...items); + } + + S.compare = compare; +})(); diff --git a/static/project-workspace/curation.js b/static/project-workspace/curation.js new file mode 100644 index 00000000..cb974fb7 --- /dev/null +++ b/static/project-workspace/curation.js @@ -0,0 +1,501 @@ +'use strict'; +// Curation by embeddings on the page (docs/plans/training-and-curation/spec-t5.md section 6): the Map mode of a dataset +// opened on the Data tab (inside.js draws the mode switch and the viewer; this module draws the map and its lists), and +// the Data tab's Embed form, which keeps a new dataset whose run is the step embed as an unsaved edit, as Generate does. +// +// The map is a canvas of the set's records in two dimensions (embedding_map), coloured by a choice, with the thin cells +// shaded; the wheel zooms about the pointer, Move drags the view, Box and Lasso choose points. Hovering a point shows its +// id; resting on it shows the record in inside.js's viewer; a click opens it and asks for its neighbours. The chosen +// points become a new dataset (a select definition, kept with set_definition) or fill the Freeze form (annotate.js). +// Beside the map: More like this, Duplicates, Outliers, Leakage and Thin regions, each asked for when first shown. +// +// Updating in place (spec-w4): the answers are not watched; a reading that changed something makes inside.js draw the +// opened view again, and this module draws the map again from its kept state (the set, the colour, the view, the tool, +// what is chosen and opened, each list's controls). A person's own changes redraw only the part they change. +window.LabCuration = (() => { + const P = window.TuldokProject, S = window.LabScreens, A = window.LabActions, h = window.LabCharts.h; + const { putInPlace } = S.parts; + const FORMAT = 'tuldok_definition_v1'; + const SELECT_LIMIT = 20000; // records one select names (check.py) + const CATEGORIES = 8; // categorical colours: --cat-1 to --cat-8; the rest, and "other", are --ink-3 + const NO_CODE = 65535; + const REST_MS = 300; // resting on a point this long shows its record in the viewer + const button = (cls, text, key, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (key) b.dataset.key = key; if (onClick) b.addEventListener('click', onClick); return b; }; + const count = (n, one, many) => n.toLocaleString('en-GB') + ' ' + (n === 1 ? one : many || one + 's'); + const pct = x => (x * 100).toFixed(x < 0.1 ? 1 : 0) + ' %'; + const cos = x => x.toFixed(4); + const D = () => window.LabInside.drawing; + const raw = b64 => { const s = atob(b64 || ''), u = new Uint8Array(s.length); for (let i = 0; i < s.length; i++) u[i] = s.charCodeAt(i); return u; }; + const u16 = b64 => { const u = raw(b64), d = new DataView(u.buffer), out = new Uint16Array(u.length >> 1); for (let i = 0; i < out.length; i++) out[i] = d.getUint16(i * 2, true); return out; }; + + // ---- What is kept per dataset across drawings ----------------------------------------------------------------- + const states = new Map(); + const stateOf = ref => { + if (!states.has(ref)) states.set(ref, { key: null, colour: undefined, view: { k: 1, cu: 0.5, cv: 0.5 }, tool: 'box', chosen: new Set(), opened: null, + list: 'duplicates', dup: '0.98', leak: '0.95', train: 'train', k: '5', form: false, kept: null }); + return states.get(ref); + }; + const usable = em => ((em && em.sets) || []).filter(s => !s.problem && s.records_run); + const keyOf = s => s.run + '/' + s.name; + /** The embeddings sets of a dataset (kept like any answer until a reading finds a change). */ + const sets = ref => P.read('embeddings', { dataset: ref }).catch(error => ({ error, sets: [] })); + const base = s => ({ run: s.run, name: s.name, records_run: s.records_run }); + + // ---- The Map mode ------------------------------------------------------------------------------------------------- + /** + * The map and its lists for a dataset opened in inside.js: ctx is {ref, em (the embeddings answer), act, + * show(id, run) (shows a record in the viewer), live() (whether the view is still on screen)}. + */ + function panel(ctx) { + const st = stateOf(ctx.ref), list = usable(ctx.em); + const root = h('section', 'cur'); root.setAttribute('aria-label', 'The records on a map of their embeddings'); + const broken = ((ctx.em && ctx.em.sets) || []).filter(s => s.problem); + if (!list.length) { + root.appendChild(h('p', 'empty', 'No embeddings of this dataset can be read.')); + for (const s of broken) root.appendChild(h('p', 'small warnline', s.holder + ' · ' + s.name + ': ' + s.problem)); + return root; + } + const set = list.find(s => keyOf(s) === st.key) || list[0]; st.key = keyOf(set); + const c = { ctx, st, set, root, m: null, pts: null, hover: null, rest: 0 }; + const head = h('div', 'cur-head'); + if (list.length > 1) { + const sel = h('select'); sel.setAttribute('aria-label', 'Embeddings'); sel.dataset.key = 'cur-set'; + for (const s of list) { const o = h('option', null, s.holder.replace(/^datasets\//, '') + ' · ' + s.name + ' · ' + s.model.name); o.value = keyOf(s); o.selected = keyOf(s) === st.key; sel.appendChild(o); } + sel.addEventListener('change', () => { st.key = sel.value; st.chosen = new Set(); st.opened = null; st.colour = undefined; st.view = { k: 1, cu: 0.5, cv: 0.5 }; putInPlace(root.parentNode, panel(ctx)); }); + head.appendChild(sel); + } + c.facts = h('p', 'small muted cur-facts', set.rows.toLocaleString('en-GB') + ' records · ' + set.dimensions + ' dimensions · ' + set.model.name + (set.prefix ? ' · prefix “' + set.prefix + '”' : '')); + head.appendChild(c.facts); + if (!set.current) head.appendChild(h('p', 'small warnline', 'These embeddings describe run ' + set.records_run.slice(0, 8) + ', not the dataset\'s current product: they may not match its records.')); + for (const s of broken) head.appendChild(h('p', 'small warnline', s.holder + ' · ' + s.name + ': ' + s.problem)); + const bar = h('div', 'cur-bar'); + c.colourSlot = h('span', 'cur-colour'); bar.appendChild(c.colourSlot); + const tools = h('div', 'seg'); tools.setAttribute('role', 'group'); tools.setAttribute('aria-label', 'Dragging on the map'); + for (const [tool, label] of [['move', 'Move'], ['box', 'Box'], ['lasso', 'Lasso']]) { + const b = button(null, label, 'cur-tool-' + tool, () => { st.tool = tool; for (const x of tools.children) x.setAttribute('aria-pressed', String(x === b)); }); + b.setAttribute('aria-pressed', String(st.tool === tool)); tools.appendChild(b); + } + bar.appendChild(tools); + bar.appendChild(button('btn sm ghost', 'Reset view', 'cur-reset', () => { st.view = { k: 1, cu: 0.5, cv: 0.5 }; paint(c); })); + head.appendChild(bar); + root.appendChild(head); + const box = h('div', 'cur-map'); + c.canvas = h('canvas'); c.canvas.tabIndex = 0; c.canvas.setAttribute('role', 'img'); c.canvas.dataset.key = 'cur-canvas'; + c.canvas.setAttribute('aria-label', 'Map of ' + set.rows + ' records; the arrow keys move between the nearest records, Enter opens one'); + c.tip = h('div', 'cur-tip'); c.tip.hidden = true; c.tip.setAttribute('aria-hidden', 'true'); + c.live = h('p', 'sr-only'); c.live.setAttribute('aria-live', 'polite'); + box.appendChild(c.canvas); box.appendChild(c.tip); box.appendChild(c.live); + root.appendChild(box); + c.legend = h('div', 'cur-legend'); root.appendChild(c.legend); + c.selBar = h('div', 'cur-chosen'); root.appendChild(c.selBar); + c.lists = h('div', 'cur-lists'); root.appendChild(c.lists); + c.canvas.parentNode.appendChild(h('p', 'note cur-reading', 'Reading the map…')); + wire(c); + load(c); + return root; + } + + async function load(c) { + const { st, set } = c; + let m; + try { + const first = await P.read('embedding_map', base(set)); + const colour = st.colour === undefined ? (first.colours.includes('split') ? 'split' : null) : st.colour; + st.colour = colour; + m = colour ? await P.read('embedding_map', { ...base(set), colour }) : first; + } catch (error) { + if (!c.ctx.live()) return; + putInPlace(c.root.querySelector('.cur-map'), A.refusal(error)); return; + } + if (!c.ctx.live() || !c.root.isConnected && c.m) return; + c.m = decode(m); + const reading = c.root.querySelector('.cur-reading'); if (reading) reading.remove(); + c.facts.textContent = [count(m.rows, 'record'), m.dimensions + ' dimensions', set.model.name, m.method === 'pca' ? 'principal components (' + m.explained.map(pct).join(' and ') + ' of the variance, from ' + count(m.sample, 'row') + ')' : 'the script\'s ' + m.method, + m.missing ? count(m.missing, 'row') + ' with no record in the product' : null, set.prefix ? 'prefix “' + set.prefix + '”' : null].filter(Boolean).join(' · '); + colourChoice(c); legend(c); + D().paints(c.canvas, () => paint(c)); + chosenBar(c); lists(c); + } + function decode(m) { + const x = D().floats(m.x_f32), y = D().floats(m.y_f32), at = new Map(m.ids.map((id, i) => [id, i])); + const col = m.colour ? (m.colour.type === 'category' ? { codes: u16(m.colour.codes_u16) } : { values: D().floats(m.colour.values_f32) }) : null; + return { ...m, x, y, at, col }; + } + + // ---- Colour ---------------------------------------------------------------------------------------------------------- + const WORD = c => c === 'split' ? 'Split' : c === 'kind' ? 'Kind' : c.startsWith('annotation:') ? 'Annotation · ' + c.slice(11) : 'Field · ' + c.slice(6); + function colourChoice(c) { + const l = h('label', 'cur-pick'); l.appendChild(document.createTextNode('Colour by ')); + const sel = h('select'); sel.dataset.key = 'cur-colour'; + for (const [v, t] of [['', 'nothing']].concat(c.m.colours.map(x => [x, WORD(x)]))) { const o = h('option', null, t); o.value = v; o.selected = v === (c.st.colour || ''); sel.appendChild(o); } + sel.addEventListener('change', async () => { + c.st.colour = sel.value || null; + try { c.m = decode(c.st.colour ? await P.read('embedding_map', { ...base(c.set), colour: c.st.colour }) : await P.read('embedding_map', base(c.set))); } catch (error) { putInPlace(c.legend, A.refusal(error)); return; } + legend(c); paint(c); + }); + l.appendChild(sel); putInPlace(c.colourSlot, l); + } + function palette() { + const t = D().token; const cats = []; for (let i = 1; i <= CATEGORIES; i++) cats.push(t('--cat-' + i)); + return { cats, other: t('--ink-3'), ink: t('--ink'), accent: t('--accent'), surface: t('--surface'), thin: t('--warn'), grid: t('--grid') }; + } + /** [r, g, b] of a row, or null for a row with no value. */ + function colourOf(c, pal, i) { + const col = c.m.col; + if (!col) return pal.cats[0]; + if (col.codes) { const k = col.codes[i]; if (k === NO_CODE) return null; const cat = c.m.colour.categories[k]; return cat && !cat.other && k < CATEGORIES ? pal.cats[k] : pal.other; } + const v = col.values[i]; if (!Number.isFinite(v)) return null; + return c._ramp(v); + } + function legend(c) { + const box = h('div'); const m = c.m, pal = palette(); + if (!m.colour) { box.appendChild(h('span', 'small muted', 'One colour: choose what to colour by.')); } + else if (m.colour.type === 'number') { + c._ramp = D().ramp(m.colour.low, m.colour.high); + const sw = h('span', 'cur-ramp'); sw.setAttribute('aria-hidden', 'true'); box.appendChild(h('span', 'small', WORD(m.colour.by) + ': ' + m.colour.low.toLocaleString('en-GB') + ' ')); box.appendChild(sw); box.appendChild(h('span', 'small', ' ' + m.colour.high.toLocaleString('en-GB'))); + } else { + m.colour.categories.forEach((cat, k) => { + const item = h('span', 'cur-key'); const sw = h('span', 'sw'); const rgb = !cat.other && k < CATEGORIES ? pal.cats[k] : pal.other; sw.style.background = 'rgb(' + rgb.join(',') + ')'; sw.dataset.cat = cat.other || k >= CATEGORIES ? 'other' : String(k + 1); + item.appendChild(sw); item.appendChild(h('span', null, cat.name + ' · ' + cat.count.toLocaleString('en-GB'))); box.appendChild(item); + }); + } + if (m.thin.length) { const item = h('span', 'cur-key'); const sw = h('span', 'sw thin'); item.appendChild(sw); item.appendChild(h('span', null, 'thin region')); box.appendChild(item); } + box.dataset.colour = m.colour ? m.colour.by : ''; + putInPlace(c.legend, box); + } + + // ---- Drawing -------------------------------------------------------------------------------------------------------- + const PAD = 12; + function frame(c, W, H) { + const dpr = window.devicePixelRatio || 1, p = PAD * dpr, [x0, x1, y0, y1] = c.m.bounds, v = c.st.view; + const iw = W - 2 * p, ih = H - 2 * p; + const sx = x => p + ((((x - x0) / (x1 - x0)) - v.cu) * v.k + 0.5) * iw; + const sy = y => p + (((1 - (y - y0) / (y1 - y0)) - v.cv) * v.k + 0.5) * ih; + const back = (px, py) => [((px - p) / iw - 0.5) / v.k + v.cu, ((py - p) / ih - 0.5) / v.k + v.cv]; + return { dpr, p, iw, ih, sx, sy, back }; + } + function paint(c) { + if (!c.m || !c.canvas.isConnected) return; + const wide = c.canvas.clientWidth; const size = D().fit(c.canvas, Math.max(260, Math.min(460, Math.round(wide * 0.7)))); if (!size) return; + const [W, H] = size, ctx = c.canvas.getContext('2d'), m = c.m, pal = palette(), f = frame(c, W, H); + if (m.colour && m.colour.type === 'number') c._ramp = D().ramp(m.colour.low, m.colour.high); + c.f = f; + ctx.clearRect(0, 0, W, H); + ctx.fillStyle = 'rgb(' + pal.surface.join(',') + ')'; ctx.fillRect(0, 0, W, H); + // Thin cells, washed. + const [x0, x1, y0, y1] = m.bounds, cells = m.grid.cells; + ctx.fillStyle = 'rgba(' + pal.thin.join(',') + ',0.14)'; + for (const t of m.thin) { const a = f.sx(t.x[0]), b = f.sx(t.x[1]), top = f.sy(t.y[1]), bot = f.sy(t.y[0]); ctx.fillRect(a, top, b - a, bot - top); } + void x0; void x1; void y0; void y1; void cells; + const r = Math.max(1.5, 2.4 * f.dpr), n = m.ids.length, px = new Float32Array(n), py = new Float32Array(n); + for (let i = 0; i < n; i++) { px[i] = f.sx(m.x[i]); py[i] = f.sy(m.y[i]); } + c.pts = { px, py }; + // Rows with no value first and faint, then the rest. + for (let pass = 0; pass < 2; pass++) for (let i = 0; i < n; i++) { + const rgb = colourOf(c, pal, i); if ((rgb === null) !== (pass === 0)) continue; + if (px[i] < -r || py[i] < -r || px[i] > W + r || py[i] > H + r) continue; + ctx.fillStyle = rgb === null ? 'rgba(' + pal.other.join(',') + ',0.45)' : 'rgb(' + rgb.join(',') + ')'; + ctx.beginPath(); ctx.arc(px[i], py[i], r, 0, 6.2832); ctx.fill(); + } + // What is chosen: a ring in ink. The opened record: a ring in the accent, joined to its most alike. + ctx.lineWidth = Math.max(1, f.dpr); ctx.strokeStyle = 'rgb(' + pal.ink.join(',') + ')'; + for (const id of c.st.chosen) { const i = m.at.get(id); if (i == null) continue; ctx.beginPath(); ctx.arc(px[i], py[i], r + 1.6 * f.dpr, 0, 6.2832); ctx.stroke(); } + const o = c.st.opened != null ? m.at.get(c.st.opened) : null; + if (o != null) { + ctx.strokeStyle = 'rgb(' + pal.accent.join(',') + ')'; ctx.lineWidth = 1.5 * f.dpr; + for (const nb of c.like || []) { const j = m.at.get(nb.id); if (j == null) continue; ctx.beginPath(); ctx.moveTo(px[o], py[o]); ctx.lineTo(px[j], py[j]); ctx.stroke(); } + ctx.lineWidth = 2 * f.dpr; ctx.beginPath(); ctx.arc(px[o], py[o], r + 3 * f.dpr, 0, 6.2832); ctx.stroke(); + } + if (c.drag && c.drag.tool !== 'move' && c.drag.path.length > 1) { + ctx.strokeStyle = 'rgb(' + pal.accent.join(',') + ')'; ctx.lineWidth = f.dpr; ctx.setLineDash([4 * f.dpr, 3 * f.dpr]); ctx.beginPath(); + if (c.drag.tool === 'box') { const [a, b] = [c.drag.path[0], c.drag.path[c.drag.path.length - 1]]; ctx.rect(a[0], a[1], b[0] - a[0], b[1] - a[1]); } + else { ctx.moveTo(c.drag.path[0][0], c.drag.path[0][1]); for (const q of c.drag.path) ctx.lineTo(q[0], q[1]); ctx.closePath(); } + ctx.stroke(); ctx.setLineDash([]); + } + c.canvas.dataset.drawn = String(n); + } + /** The row nearest the canvas point (device pixels) within 10 CSS pixels, or null. */ + function nearest(c, X, Y) { + if (!c.pts) return null; + const { px, py } = c.pts, lim = (10 * (window.devicePixelRatio || 1)) ** 2; let best = null, bd = lim; + for (let i = 0; i < px.length; i++) { const d = (px[i] - X) ** 2 + (py[i] - Y) ** 2; if (d < bd) { bd = d; best = i; } } + return best; + } + const at = (c, e) => { const r = c.canvas.getBoundingClientRect(), dpr = window.devicePixelRatio || 1; return [(e.clientX - r.left) * dpr, (e.clientY - r.top) * dpr]; }; + function inside(poly, x, y) { let inn = false; for (let i = 0, j = poly.length - 1; i < poly.length; j = i++) { const [xi, yi] = poly[i], [xj, yj] = poly[j]; if ((yi > y) !== (yj > y) && x < (xj - xi) * (y - yi) / (yj - yi) + xi) inn = !inn; } return inn; } + function valueWord(c, i) { + const col = c.m.col; if (!col) return ''; + if (col.codes) { const k = col.codes[i]; return k === NO_CODE ? 'no ' + WORD(c.m.colour.by).toLowerCase() : c.m.colour.categories[k].name; } + return Number.isFinite(col.values[i]) ? String(col.values[i]) : 'no value'; + } + function wire(c) { + const cv = c.canvas, st = c.st; + cv.addEventListener('wheel', e => { + if (!c.f) return; e.preventDefault(); + const [X, Y] = at(c, e), [u, v] = c.f.back(X, Y), k = Math.min(200, Math.max(1, st.view.k * Math.exp(-e.deltaY * 0.0015))); + const fx = (X - c.f.p) / c.f.iw - 0.5, fy = (Y - c.f.p) / c.f.ih - 0.5; + st.view = { k, cu: u - fx / k, cv: v - fy / k }; paint(c); + }, { passive: false }); + cv.addEventListener('pointerdown', e => { + if (!c.f || e.button !== 0) return; + cv.setPointerCapture(e.pointerId); + c.drag = { tool: e.shiftKey && st.tool === 'move' ? 'box' : st.tool, path: [at(c, e)], view: { ...st.view }, add: e.shiftKey, moved: false }; + }); + cv.addEventListener('pointermove', e => { + const [X, Y] = at(c, e); + if (c.drag) { + const d = c.drag, [sx, sy] = d.path[0]; + if (Math.abs(X - sx) + Math.abs(Y - sy) > 4) d.moved = true; + if (d.tool === 'move') st.view = { ...d.view, cu: d.view.cu - (X - sx) / c.f.iw / d.view.k, cv: d.view.cv - (Y - sy) / c.f.ih / d.view.k }; + else d.path.push([X, Y]); + paint(c); return; + } + hover(c, nearest(c, X, Y), e); + }); + const end = e => { + const d = c.drag; c.drag = null; if (!d) return; + if (!d.moved) { const i = nearest(c, ...at(c, e)); if (i != null) open(c, c.m.ids[i]); paint(c); return; } + if (d.tool !== 'move') { + const { px, py } = c.pts, chosen = d.add ? new Set(st.chosen) : new Set(); + if (d.tool === 'box') { + const [a, b] = [d.path[0], d.path[d.path.length - 1]], lx = Math.min(a[0], b[0]), hx = Math.max(a[0], b[0]), ly = Math.min(a[1], b[1]), hy = Math.max(a[1], b[1]); + for (let i = 0; i < px.length; i++) if (px[i] >= lx && px[i] <= hx && py[i] >= ly && py[i] <= hy) chosen.add(c.m.ids[i]); + } else for (let i = 0; i < px.length; i++) if (inside(d.path, px[i], py[i])) chosen.add(c.m.ids[i]); + st.chosen = chosen; st.form = false; st.kept = null; chosenBar(c); + c.live.textContent = count(chosen.size, 'record') + ' chosen.'; + } + paint(c); + }; + cv.addEventListener('pointerup', end); + cv.addEventListener('pointercancel', () => { c.drag = null; paint(c); }); + cv.addEventListener('pointerleave', () => hover(c, null)); + cv.addEventListener('keydown', e => { + if (!c.m || !c.pts) return; + const dirs = { ArrowLeft: [-1, 0], ArrowRight: [1, 0], ArrowUp: [0, -1], ArrowDown: [0, 1] }; + if (e.key === 'Enter' && c.hover != null) { e.preventDefault(); open(c, c.m.ids[c.hover]); return; } + if (e.key === 'Escape' && st.chosen.size) { st.chosen = new Set(); chosenBar(c); paint(c); return; } + const d = dirs[e.key]; if (!d) return; + e.preventDefault(); e.stopPropagation(); + const { px, py } = c.pts, from = c.hover != null ? c.hover : st.opened != null && c.m.at.has(st.opened) ? c.m.at.get(st.opened) : null; + let best = null, bd = Infinity; + for (let i = 0; i < px.length; i++) { + if (from == null) { const dd = (px[i] - c.canvas.width / 2) ** 2 + (py[i] - c.canvas.height / 2) ** 2; if (dd < bd) { bd = dd; best = i; } continue; } + if (i === from) continue; + const dx = px[i] - px[from], dy = py[i] - py[from], along = dx * d[0] + dy * d[1]; if (along <= 0) continue; + const across = Math.abs(dx * d[1] - dy * d[0]), score = along + 2 * across; if (score < bd) { bd = score; best = i; } + } + if (best != null) { hover(c, best); c.live.textContent = c.m.ids[best] + (c.m.col ? ', ' + valueWord(c, best) : '') + '.'; } + }); + } + function hover(c, i, e) { + c.hover = i; clearTimeout(c.rest); + if (i == null) { c.tip.hidden = true; return; } + const dpr = window.devicePixelRatio || 1; + c.tip.textContent = ''; c.tip.appendChild(h('b', 'mono', c.m.ids[i])); const w = valueWord(c, i); if (w) c.tip.appendChild(h('span', null, w)); + c.tip.style.left = Math.round(c.pts.px[i] / dpr + 10) + 'px'; c.tip.style.top = Math.round(c.pts.py[i] / dpr + 10) + 'px'; c.tip.hidden = false; + const id = c.m.ids[i]; + c.rest = setTimeout(() => { if (c.hover === i && c.ctx.live()) c.ctx.show(id, c.set.records_run); }, REST_MS); + void e; + } + /** Opens a record: the viewer shows it, the map rings it, More like this lists its neighbours. */ + function open(c, id) { + c.st.opened = id; c.ctx.show(id, c.set.records_run); + c.st.list = 'like'; lists(c); paint(c); + } + + // ---- What is chosen --------------------------------------------------------------------------------------------------- + function chosenBar(c) { + const st = c.st, box = h('div'), n = st.chosen.size; + if (st.kept) box.appendChild(h('p', 'small good-line', 'Kept ' + st.kept + ' as an unsaved edit; Save writes it to the project folder. Building it is planned as any build is.')); + if (!n) { box.appendChild(h('p', 'small muted', 'Drag with Box or Lasso to choose records (Shift adds to them).')); putInPlace(c.selBar, box); return; } + const line = h('div', 'cur-chosen-line'); line.appendChild(h('b', 'cur-count', count(n, 'record') + ' chosen')); + const ordered = () => c.m.ids.filter(id => st.chosen.has(id)); + if (A.may('edit')) { + line.appendChild(button('btn sm', 'New dataset from these', 'cur-new', () => { st.form = !st.form; chosenBar(c); })); + line.appendChild(button('btn sm', 'Freeze these', 'cur-freeze', () => { + if (n > SELECT_LIMIT) { st.form = false; putInPlace(c.selBar, refusalBox('At most ' + SELECT_LIMIT.toLocaleString('en-GB') + ' records can be frozen by name; ' + n.toLocaleString('en-GB') + ' are chosen.')); return; } + window.LabAnnotate.fillFreeze(c.ctx.ref, ordered()); c.ctx.act.go('data', 'new:freeze'); + })); + } + line.appendChild(button('btn sm ghost', 'Clear', 'cur-clear', () => { st.chosen = new Set(); st.form = false; chosenBar(c); paint(c); })); + box.appendChild(line); + if (st.form && A.may('edit')) box.appendChild(newForm(c, ordered)); + putInPlace(c.selBar, box); + } + const refusalBox = text => A.refusal(Object.assign(Error(text), { code: 'invalid' })); + function newForm(c, ordered) { + const F = window.LabAnnotate.forms, key = 'curnew', s = A.slot(key), f = h('div', 'form cur-form'); + let nameBox = null; + f.appendChild(F.input(s, key, 'title', 'Title (required)', { max: 200, onInput: () => { if (!s.text.nameTyped && nameBox) { nameBox.value = F.slug(s.text.title || ''); s.text.name = nameBox.value; } } })); + const nl = F.input(s, key, 'name', 'Name', { max: 120, hint: 'the file is datasets/.json', placeholder: 'made from the title', onInput: v => { s.text.nameTyped = !!v; } }); + nameBox = nl.querySelector('input'); if (!s.text.nameTyped) { nameBox.value = F.slug(s.text.title || ''); s.text.name = nameBox.value; } + f.appendChild(nl); + f.appendChild(h('p', 'small faint', 'A dataset of exactly these ' + count(c.st.chosen.size, 'record') + ' of ' + c.ctx.ref + ', in its order, each as it is there (its split and annotations kept): the step select.')); + f.appendChild(A.group(key, [ + { id: 'keep', label: 'Keep', cls: 'btn primary', who: 'edit', op: 'set_definition', busy: 'Keeping…', + args: st => { + const ids = ordered(); + if (ids.length > SELECT_LIMIT) throw Object.assign(Error('At most ' + SELECT_LIMIT.toLocaleString('en-GB') + ' records can be selected by name; ' + ids.length.toLocaleString('en-GB') + ' are chosen.'), { code: 'invalid' }); + const doc = { format: FORMAT, kind: 'dataset', title: (st.text.title || '').trim(), run: { step: 'select', inputs: { records: c.ctx.ref }, settings: { records: ids } } }; + return { reference: 'datasets/' + (st.text.name || '').trim(), text: F.textOf(doc) }; + }, + done: r => { c.st.kept = r.reference; c.st.form = false; } }, + { id: 'back', label: 'Cancel', cls: 'btn ghost', onClick: () => { c.st.form = false; chosenBar(c); } }])); + return f; + } + + // ---- The lists ------------------------------------------------------------------------------------------------------ + const LISTS = [['like', 'More like this'], ['duplicates', 'Duplicates'], ['outliers', 'Outliers'], ['leakage', 'Leakage'], ['thin', 'Thin regions']]; + function lists(c) { + const st = c.st, box = h('div'); + const seg = h('div', 'seg cur-tabs'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'Lists'); + const shown = LISTS.filter(([id]) => id !== 'leakage' || c.m.colours.includes('split')); + if (!shown.some(([id]) => id === st.list)) st.list = 'duplicates'; + for (const [id, label] of shown) { const b = button(null, label, 'cur-list-' + id, () => { if (st.list !== id) { st.list = id; lists(c); } }); b.setAttribute('aria-pressed', String(st.list === id)); seg.appendChild(b); } + box.appendChild(seg); + const body = h('div', 'cur-list'); body.dataset.list = st.list; box.appendChild(body); + putInPlace(c.lists, box); + fillList(c, body); + } + function rowButton(c, key, parts, onClick) { + const li = h('li'), b = button('row cur-row', null, key, onClick); + for (const [cls, text] of parts) b.appendChild(h('span', cls, text)); + li.appendChild(b); return li; + } + const status = r => r.complete ? 'Every one of the ' + count(r.rows, 'record') + ' was examined.' : count(r.examined, 'record') + ' of ' + r.rows.toLocaleString('en-GB') + ' were examined within the search\'s bound; what is listed is exact for those.'; + function numberControl(c, label, name, min, max, step) { + const l = h('label', 'cur-num'); l.appendChild(document.createTextNode(label + ' ')); + const i = h('input'); i.type = 'number'; i.min = String(min); i.max = String(max); i.step = String(step); i.value = c.st[name]; i.dataset.key = 'cur-' + name; + i.addEventListener('change', () => { const v = Number(i.value); if (Number.isFinite(v) && v >= min && v <= max) { c.st[name] = i.value; lists(c); } }); + l.appendChild(i); return l; + } + async function fillList(c, body) { + const st = c.st, set = c.set, controls = h('div', 'cur-controls'), ul = h('ul', 'rows cur-rows'), note = h('p', 'small muted'); + const done = () => { if (body.isConnected) { const f = document.createDocumentFragment(); f.appendChild(controls); f.appendChild(note); f.appendChild(ul); putInPlace(body, f); } }; + body.appendChild(h('p', 'note', 'Reading…')); + try { + if (st.list === 'like') { + if (st.opened == null) { note.textContent = 'Open a record (click it on the map, or a row of a list) to see the records most like it.'; c.like = []; return done(); } + const r = await P.read('embedding_neighbours', { ...base(set), record: st.opened, limit: 12 }); + c.like = r.neighbours; paint(c); + note.textContent = 'The ' + r.neighbours.length + ' records most like ' + r.record + ', by cosine.'; + for (const n of r.neighbours) ul.appendChild(rowButton(c, 'cur-like-' + n.id, [['title mono', n.id], ['sub', [n.split, n.kind].filter(Boolean).join(' · ')], ['num', cos(n.cosine)]], () => open(c, n.id))); + } else if (st.list === 'duplicates') { + controls.appendChild(numberControl(c, 'Cosine at least', 'dup', 0.5, 1, 0.001)); + const r = await P.read('embedding_duplicates', { ...base(set), threshold: Number(st.dup), limit: 200 }); + note.textContent = (r.total ? count(r.total, 'pair') + ' (' + count(r.groups, 'record') + ')' : 'No pair') + ' at or above ' + r.threshold + '. ' + status(r); + for (const p of r.pairs) ul.appendChild(rowButton(c, 'cur-dup-' + p.a + '-' + p.b, [['title mono', p.a + ' ≈ ' + p.b], ['sub', [p.a_split, p.b_split].filter(Boolean).join(' · ')], ['num', cos(p.cosine)]], () => { st.chosen = new Set([p.a, p.b]); chosenBar(c); open(c, p.a); })); + } else if (st.list === 'outliers') { + controls.appendChild(numberControl(c, 'Neighbour', 'k', 1, 50, 1)); + const r = await P.read('embedding_outliers', { ...base(set), k: Number(st.k), limit: 50 }); + note.textContent = 'Farthest from their ' + ordinal(r.k) + ' nearest record' + (r.median != null ? '; the median distance is ' + r.median.toFixed(4) : '') + '. ' + status(r); + for (const o of r.outliers) ul.appendChild(rowButton(c, 'cur-out-' + o.id, [['title mono', o.id], ['sub', 'nearest ' + o.nearest + (o.split ? ' · ' + o.split : '')], ['num', o.distance.toFixed(4)]], () => open(c, o.id))); + } else if (st.list === 'leakage') { + controls.appendChild(numberControl(c, 'Cosine at least', 'leak', 0.5, 1, 0.001)); + const splits = c.m.colour && c.m.colour.by === 'split' ? c.m.colour.categories.map(x => x.name) : [st.train]; + const l = h('label', 'cur-num'); l.appendChild(document.createTextNode('Trained on ')); const sel = h('select'); sel.dataset.key = 'cur-train'; + for (const s of new Set([st.train, ...splits])) { const o = h('option', null, s); o.value = s; o.selected = s === st.train; sel.appendChild(o); } + sel.addEventListener('change', () => { st.train = sel.value; lists(c); }); l.appendChild(sel); controls.appendChild(l); + const r = await P.read('embedding_leakage', { ...base(set), threshold: Number(st.leak), train: st.train, limit: 200 }); + note.textContent = !Object.keys(r.splits).length ? 'The records have no splits.' : (r.leaked ? count(r.leaked, 'record') + ' outside ' + r.train + ' ' + (r.leaked === 1 ? 'has' : 'have') + ' a near-copy in it' : 'No record outside ' + r.train + ' has a near-copy in it') + ' at or above ' + r.threshold + '. ' + status(r); + for (const p of r.pairs) ul.appendChild(rowButton(c, 'cur-leak-' + p.record, [['title mono', p.record + ' ≈ ' + p.near], ['sub', p.split + ' · ' + r.train], ['num', cos(p.cosine)]], () => { st.chosen = new Set([p.record, p.near]); chosenBar(c); open(c, p.record); })); + } else { + const m = c.m; + note.textContent = m.thin.length ? count(m.thin.length, 'cell') + ' of the map\'s ' + m.grid.cells + ' × ' + m.grid.cells + ' hold few records: at most a quarter of the median of the cells that hold any. A row chooses its records.' : 'No cell of the map is thin.'; + for (const t of m.thin) ul.appendChild(rowButton(c, 'cur-thin-' + t.cell.join('-'), [['title', 'Cell ' + (t.cell[1] + 1) + ', ' + (t.cell[0] + 1)], ['sub mono', t.records.slice(0, 4).join(', ') + (t.count > 4 ? ', …' : '')], ['num', String(t.count)]], () => { st.chosen = new Set(t.records); chosenBar(c); paint(c); })); + } + } catch (error) { controls.appendChild(A.refusal(error)); } + done(); + } + const ordinal = n => n + (n % 100 >= 11 && n % 100 <= 13 ? 'th' : ['th', 'st', 'nd', 'rd'][n % 10] || 'th'); + + // ---- Data: the Embed form ------------------------------------------------------------------------------------------ + /** Embed: a new datasets/ whose run is the step embed, kept as an unsaved edit. */ + function form(aside, act) { + const F = window.LabAnnotate.forms; + F.formDrawer(aside, 'data:new:embed', () => Promise.all([P.read('endpoint').catch(error => ({ error })), P.read('definitions', { kind: 'dataset' })]), (ins, [ep, defs]) => { + const key = 'newembed', s = A.slot(key), em = ep.embedder || null; + F.header(ins, 'new dataset · embed', 'Embed a dataset', + 'A vector for each text and image record of a dataset, asked of an embedding model the Pumas endpoint serves, written as embeddings Tuldok draws as a map. When the definition names no model, the machine\'s embedder is used: EmbeddingGemma 2 unless the owner set another.'); + const embedders = em ? em.embedders : []; + if (ep.error || !ep.address || !ep.answered || !embedders.length) { + const why = h('div', 'gen-none'); why.setAttribute('role', 'status'); + why.appendChild(h('p', 'failure', ep.error ? 'What the endpoint serves could not be read: ' + ep.error.message + : !ep.address ? 'No Pumas endpoint is set on this machine, so there is nothing to embed with.' + : !ep.answered ? 'The Pumas endpoint at ' + ep.address + ' does not answer: ' + (ep.problem || 'no reason was given.') + : 'The Pumas endpoint at ' + ep.address + ' serves no embedding model now.')); + const acts = h('div', 'actions'); acts.appendChild(button('btn', 'The endpoint on the Train tab', 'emb:endpoint', () => act.go('train', 'endpoint'))); acts.appendChild(button('btn ghost', 'Cancel', 'emb:cancel', () => act.select(null))); + why.appendChild(acts); ins.appendChild(why); return; + } + const sets = defs.definitions.filter(d => !d.removed), f = h('div', 'form emb-form'); + let nameBox = null; + f.appendChild(F.input(s, key, 'title', 'Title (required)', { max: 200, onInput: () => { if (!s.text.nameTyped && nameBox) { nameBox.value = F.slug(s.text.title || ''); s.text.name = nameBox.value; } } })); + const nl = F.input(s, key, 'name', 'Name', { max: 120, hint: 'the file is datasets/.json', placeholder: 'made from the title', onInput: v => { s.text.nameTyped = !!v; } }); + nameBox = nl.querySelector('input'); if (!s.text.nameTyped) { nameBox.value = F.slug(s.text.title || ''); s.text.name = nameBox.value; } + f.appendChild(nl); + if (!sets.length) f.appendChild(h('p', 'failure', 'The project has no dataset to embed.')); + else f.appendChild(F.choice(s, key, 'dataset', 'The dataset whose records are embedded', sets.map(d => [d.reference, d.title + ' · ' + d.reference]), sets[0].reference)); + const named = em.named_by === 'setting' ? 'this machine\'s embedder, ' + em.model : 'EmbeddingGemma 2'; + const resolved = em.chosen ? em.chosen.model + ' · ' + em.chosen.profile + ' (' + em.chosen.model_id + ')' : 'not served now'; + const options = [['', 'Default: ' + named + ' → ' + resolved]].concat(embedders.map(x => [x.model + '\u0000' + x.profile, x.model + ' · ' + x.profile + ' (' + x.model_id + ')'])); + f.appendChild(F.choice(s, key, 'model', 'Model', options, '')); + const line = h('p', 'small faint emb-line'); line.dataset.key = 'emb-default'; + line.textContent = em.chosen ? 'With the default, the definition names no model: a build uses ' + named + ' as this machine resolves it then (now ' + resolved + '). Choose a model to write it into the definition.' + : 'The default embedder cannot be used now: ' + (em.problem || 'it is not served.') + ' Choose a served model, or the owner sets the machine\'s embedder on the Train tab.'; + f.appendChild(line); + f.appendChild(F.input(s, key, 'prefix', 'Prefix', { max: 1000, value: em.prefix, hint: 'put before each text; EmbeddingGemma expects a task, such as “task: clustering | query: ”. Unchanged, it is not written and the machine\'s applies.' })); + const two = h('div', 'two'); + const bound = (embedders.find(x => x.model + '\u0000' + x.profile === s.text.model) || em.chosen && embedders.find(x => x.model === em.chosen.model && x.profile === em.chosen.profile) || {}).dimensions; + const dims = F.input(s, key, 'dimensions', 'Dimensions', { type: 'number', hint: bound ? 'empty: the model\'s own; ' + bound.minimum + ' to ' + bound.maximum + ' (truncated by Pumas)' : 'empty: the model\'s own; Pumas truncates to fewer' }); + dims.querySelector('input').min = '1'; dims.querySelector('input').max = String(bound ? bound.maximum : 4096); two.appendChild(dims); + const batch = F.input(s, key, 'batch', 'Batch', { type: 'number', hint: 'records a request carries; empty: 32' }); batch.querySelector('input').min = '1'; batch.querySelector('input').max = '256'; two.appendChild(batch); + f.appendChild(two); + ins.appendChild(f); + ins.appendChild(A.group(key, [ + { id: 'keep', label: 'Keep', cls: 'btn primary', who: 'edit', op: 'set_definition', busy: 'Keeping…', + args: st => ({ reference: 'datasets/' + (st.text.name || '').trim(), text: F.textOf(embedDoc(st.text, em)) }), done: r => { act.select(r.reference); } }, + { id: 'back', label: 'Cancel', cls: 'btn ghost', onClick: () => act.select(null) }])); + ins.appendChild(h('p', 'note', 'Keep checks the definition and keeps it in Tuldok as an unsaved edit; Save writes it. Building it is planned and requested as any build is.')); + }); + } + function whole(text, label, min, max) { + const raw_ = String(text == null ? '' : text).trim(); if (!raw_) return null; + const v = Number(raw_); if (!Number.isInteger(v) || v < min || v > max) throw Object.assign(Error(label + ' is a whole number from ' + min + ' to ' + max + '; "' + raw_ + '" is not.'), { code: 'invalid' }); + return v; + } + /** The definition an embed writes (docs/contracts/project-folder.md, Steps Tuldok provides). */ + function embedDoc(x, em) { + const settings = {}; + if (x.model) { const [model, profile] = x.model.split('\u0000'); settings.model = model; if (em.embedders.filter(e => e.model === model).length > 1) settings.profile = profile; } + if (x.prefix != null && x.prefix !== em.prefix) settings.prefix = x.prefix; + const d = whole(x.dimensions, 'Dimensions', 1, 4096); if (d !== null) settings.dimensions = d; + const b = whole(x.batch, 'Batch', 1, 256); if (b !== null) settings.batch = b; + return { format: FORMAT, kind: 'dataset', title: (x.title || '').trim(), run: { step: 'embed', inputs: { records: x.dataset }, settings } }; + } + + // ---- Train: the machine's embedder in the endpoint's drawer ----------------------------------------------------------- + /** The embedder part of the endpoint's drawer (generate.js): what it resolves to; for the owner, a model and prefix to set. */ + function embedderPart(ins, ep) { + const em = ep.embedder; if (!em) return; + const part = h('div', 'ep-part emb-part'); part.appendChild(h('h3', null, 'Embedder')); + const dl = h('dl', 'setup'); const put = (k, v) => { dl.appendChild(h('dt', null, k)); dl.appendChild(h('dd', null, v)); }; + put('Named by', em.named_by === 'setting' ? 'this machine: ' + em.model + (em.profile ? ' · ' + em.profile : '') : 'the default, EmbeddingGemma 2'); + put('Resolves to', em.chosen ? em.chosen.model + ' · ' + em.chosen.profile + ' (' + em.chosen.model_id + ')' : 'nothing now'); + put('Prefix', em.prefix ? '“' + em.prefix + '”' : 'none'); + part.appendChild(dl); + if (em.problem && ep.address) part.appendChild(h('p', 'small warnline', em.problem)); + if (A.may('owner')) { + const key = 'endpoint:embedder', s = A.slot(key), F = window.LabAnnotate.forms, f = h('div', 'form'); + const options = [['', 'EmbeddingGemma 2 (the default)']].concat(em.embedders.map(x => [x.model + '\u0000' + x.profile, x.model + ' · ' + x.profile + ' (' + x.model_id + ')'])); + if (s.text.model == null) s.text.model = em.model ? em.model + '\u0000' + (em.profile || (em.chosen ? em.chosen.profile : '')) : ''; + f.appendChild(F.choice(s, key, 'model', 'Embed with, when a definition names no model', options, '')); + f.appendChild(F.input(s, key, 'prefix', 'Prefix', { max: 1000, value: em.prefix })); + f.appendChild(A.group(key, [{ id: 'set', label: 'Set the embedder', cls: 'btn', who: 'owner', op: 'set_embedder', busy: 'Checking…', + args: st => { const [model, profile] = (st.text.model || '').split('\u0000'); const several = model && em.embedders.filter(e => e.model === model).length > 1; return { model: model || null, profile: several ? profile : null, prefix: st.text.prefix == null ? em.prefix : st.text.prefix }; } }])); + part.appendChild(f); + } + ins.appendChild(part); + } + + return { sets, usable, panel, form, embedderPart }; +})(); diff --git a/static/project-workspace/editor.js b/static/project-workspace/editor.js new file mode 100644 index 00000000..ee438f7c --- /dev/null +++ b/static/project-workspace/editor.js @@ -0,0 +1,489 @@ +'use strict'; +// Editing and branching (spec-w3.md section 3.2, slice T22): the editor in the +// drawer, with two views of one text (Text for every kind, Form for a +// recipe); Keep, Discard edit, Remove and Save through actions.js; Pick on the +// map; Branch from here and the drafts it makes; the steps of a new model; the +// Overview's list of every definition. What is typed is kept in page memory +// per definition until it is kept or discarded, so leaving the drawer or a +// re-render loses nothing, and a reading on the timer or on focus leaves the +// editor and the branch form alone; a draft is kept in sessionStorage under the +// project's id, so it survives a reload of the tab. The form changes only the +// fields touched and writes the JSON back with the indentation the text had. +// Nothing here writes to disk: Keep stores an unsaved edit, Save writes them. +// Text goes in through text nodes only. +window.LabEditor = (() => { + const P = window.TuldokProject, A = window.LabActions, S = window.LabScreens, Tree = window.LabTree, h = window.LabCharts.h; + const { sect, modelState } = S.parts; + const isObject = v => !!v && typeof v === 'object' && !Array.isArray(v); + const button = (cls, text, key, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (key) b.dataset.key = key; if (onClick) b.addEventListener('click', onClick); return b; }; + const KIND_ORDER = ['recipe', 'sweep', 'dataset', 'annotation', 'model', 'step', 'benchmark', 'chart', 'environment']; + const KIND_TITLE = { recipe: 'Recipes', dataset: 'Datasets', model: 'Models', step: 'Steps', benchmark: 'Benchmarks', chart: 'Charts', environment: 'Environments', annotation: 'Annotation tasks', sweep: 'Sweeps' }; + + // ---- Page memory ------------------------------------------------------------ + const buffers = new Map(); // reference -> { text, base }: typed and not kept yet + const views = new Map(); // reference -> 'form' | 'text' + let drafts = []; // { reference, parent, text, inlined, problems }: branches not kept yet + let followed = []; // references branched in this tab and kept: their drawer keeps the steps + let project = null, picking = null, pickNote = '', current = null; + + const storeKey = () => 'tuldok-drafts:' + project; + function persist() { if (!project) return; try { sessionStorage.setItem(storeKey(), JSON.stringify({ drafts, followed })); } catch (e) { /* kept for this page only */ } } + const validDraft = d => isObject(d) && typeof d.reference === 'string' && d.reference.startsWith('recipes/') && typeof d.parent === 'string' && typeof d.text === 'string'; + /** app.js, once the summary is read: the drafts of this project kept in this tab. */ + function load(id) { + if (project === id) return; + project = id; + let saved = null; try { saved = JSON.parse(sessionStorage.getItem(storeKey()) || 'null'); } catch (e) { saved = null; } + drafts = saved && Array.isArray(saved.drafts) ? saved.drafts.filter(validDraft).map(d => ({ reference: d.reference, parent: d.parent, text: d.text, inlined: typeof d.inlined === 'string' ? d.inlined : null, problems: Array.isArray(d.problems) ? d.problems : [] })) : []; + followed = saved && Array.isArray(saved.followed) ? saved.followed.filter(x => typeof x === 'string') : []; + publish(); + } + /** What the map draws of each draft (adapter.js); a draft whose parent is gone, or whose reference is taken, is dropped. */ + function publish() { P.setDrafts(drafts.map(drawn), dropped); } + function dropped(gone) { + const refs = new Set(gone.map(d => d.reference)); + drafts = drafts.filter(d => !refs.has(d.reference)); persist(); P.setDrafts(drafts.map(drawn)); + } + function drawn(d) { + const doc = parse(d.text).doc || {}, str = v => typeof v === 'string' ? v : null; + return { reference: d.reference, parent: d.parent, title: str(doc.title) || d.reference.slice(8), short: isObject(doc.labels) ? str(doc.labels.short) : null, + description: str(doc.description), group: str(doc.group), change: str(doc.change) }; + } + const draftOf = ref => drafts.find(d => d.reference === ref) || null; + const isDraft = ref => A.may('edit') && !!draftOf(ref); + + // ---- One text --------------------------------------------------------------------- + function parse(text) { + try { const doc = JSON.parse(text); return isObject(doc) ? { doc } : { error: 'The text is JSON but not an object.' }; } + catch (e) { return { error: e.message }; } + } + /** The indentation of the text: that of its first indented line, else two spaces. */ + const indentOf = text => { const m = /\n([ \t]+)\S/.exec(text || ''); return m ? m[1] : ' '; }; + const textOf = (ref, def) => { const d = draftOf(ref); return buffers.has(ref) ? buffers.get(ref).text : d ? d.text : def ? def.text : ''; }; + /** Keeps typed text: a draft's own text, else a buffer until it is kept or discarded (none when it is the definition's text again). */ + function setText(ref, text, base) { + const d = draftOf(ref); + if (d) { d.text = text; persist(); return; } + const had = buffers.has(ref); + if (text === base) buffers.delete(ref); else buffers.set(ref, { text, base }); + if (had !== buffers.has(ref)) { decorate(document); if (current && current.ref === ref && current.controls) current.controls(); } + } + /** A change made in the form: the text parsed, fn(document) applied, written back with the text's indentation. */ + function edit(ref, base, fn) { + const text = textOf(ref, base === null ? null : { text: base }), p = parse(text); + if (!p.doc) return; + fn(p.doc); + setText(ref, JSON.stringify(p.doc, null, indentOf(text)) + '\n', base); + } + + /** The marks after names (tree.js's mark): "not kept yet" where something typed is not kept, else "unsaved" or "removed". */ + function decorate(root) { + for (const s of (root || document).querySelectorAll('.umark[data-ref]')) { + const typed = buffers.has(s.dataset.ref) || s.dataset.draft === '1'; + const text = typed ? 'not kept yet' : s.dataset.removed === '1' ? 'removed' : s.dataset.unsaved === '1' ? 'unsaved' : ''; + if (s.textContent !== text) s.textContent = text; // a mark already right is not touched (spec-w4 R2) + s.classList.toggle('typed', typed); + } + } + + // ---- Selections ------------------------------------------------------------------- + const prefixes = ['edit:', 'branch:']; + /** The reference a selection is about: 'edit:' and 'branch:' are about . */ + function subject(sel) { for (const p of prefixes) if (sel && sel.startsWith(p)) return sel.slice(p.length); return sel; } + /** Whether a selection is this module's drawer: the editor, the branch form or a draft, for a session that may edit. */ + function owns(sel, data) { return !!sel && A.may('edit') && (prefixes.some(p => sel.startsWith(p)) || !!draftOf(sel)); } + function drawer(aside, sel, act, data, screen) { + if (sel.startsWith('branch:')) return branchForm(aside, sel, sel.slice(7), act, data); + return editor(aside, sel, subject(sel), act, data, screen); + } + /** Puts node in the drawer in one step (screens.js's putInPlace: the focused control by data-key, its caret and the scroll given back); else the heading takes the focus when the focus was in the drawer or nowhere. */ + function swapInto(aside, node) { + const active = document.activeElement, nowhere = !active || active === document.body; + const r = S.parts.putInPlace(aside, node, aside.parentElement); + if (!r.found && (r.inside || nowhere)) { const t = aside.querySelector('h2'); if (t && aside.closest('.drawer.open')) t.focus({ preventScroll: true }); } + } + + // ---- The editor --------------------------------------------------------------------- + // A reading on the timer or on focus never touches the editor or the branch form (spec-w4 R3): what is typed stays. + // After a control's answer (aside._again 'control') they are drawn again in place. + function editor(aside, sel, ref, act, data, screen) { + const key = 'editor:' + sel; + if (aside._for === key && aside._again !== 'control') return true; + aside._for = key; + const draft = draftOf(ref); + if (!(aside._editing === ref && aside.querySelector('.editor'))) { aside.textContent = ''; aside.appendChild(h('p', 'note', 'Reading…')); } + aside._editing = ref; + Promise.all([draft ? null : P.read('definition', { reference: ref }), P.read('definitions')]).then(([def, all]) => { + if (aside._for !== key) return; + const here = { aside, key, ref, controls: null }; + current = here; + const rebuild = () => { if (aside._for === key) { aside._for = null; editor(aside, sel, ref, act, data, screen); } }; + here.rebuild = rebuild; + swapInto(aside, editorView(here, ref, draft ? null : def, all.definitions, act, data, screen, sel)); + }, error => { if (aside._for !== key) return; aside._editing = null; aside.textContent = ''; aside.appendChild(A.refusal(error)); }); + return true; + } + function editorView(here, ref, def, all, act, data, screen, sel) { + const draft = draftOf(ref), kind = def ? def.kind : 'recipe', base = def ? def.text : null, text = textOf(ref, def), p = parse(text); + const doc = p.doc || (def && def.document) || {}; + const view = kind === 'recipe' && p.doc ? (views.get(ref) || 'form') : 'text'; + const ins = h('div', 'inspector editor'); ins.dataset.ref = ref; + // The header: what it is, its state, and its marks. + const hd = h('header'), parent = draft ? data.map.byId.get(draft.parent) : null; + hd.appendChild(h('span', 'kicker', draft ? 'draft · from ' + (parent ? Tree.nameOf(parent) : draft.parent) : (kind === 'recipe' ? 'editing · ' : 'editing ' + kind + ' · ') + ref)); + const t = h('h2', null, (typeof doc.title === 'string' && doc.title) || (def && def.title) || ref); t.tabIndex = -1; hd.appendChild(t); + const meta = h('div', 'ins-meta'); + if (draft) meta.appendChild(h('span', 'chip ' + (P.KINDS.some(k => k.id === doc.change) ? doc.change : P.UNSET) + ' draft', 'draft')); + else meta.appendChild(modelState(def.state)); + const mark = Tree.mark({ id: ref, unsaved: !!(def && def.unsaved), draft: !!draft }); if (def && def.removed) mark.dataset.removed = '1'; meta.appendChild(mark); + hd.appendChild(meta); ins.appendChild(hd); + // Back to what it was opened from. + const back = h('div', 'actions'); + if (draft) { if (parent) back.appendChild(button('btn sm ghost', 'From ' + Tree.refName(parent), 'editor:' + ref + ':parent', () => act.go('map', parent.id))); } + else if (screen === 'map' && data.map.byId.has(ref)) back.appendChild(button('btn sm ghost', '← The model', 'editor:' + ref + ':back', () => act.select(ref))); + else if (screen === 'data') back.appendChild(button('btn sm ghost', '← The dataset', 'editor:' + ref + ':back', () => act.select(ref))); + if (back.children.length) ins.appendChild(back); + if (picking && picking.reference === ref) ins.appendChild(pickingBanner(act)); + // The two views of the one text. + const seg = h('div', 'seg'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'View'); + if (kind === 'recipe' && p.doc) for (const [v, label] of [['form', 'Form'], ['text', 'Text']]) { + const b = button(null, label, 'editor:' + ref + ':view-' + v, () => { views.set(ref, v); here.rebuild(); }); b.setAttribute('aria-pressed', String(view === v)); seg.appendChild(b); + } + if (seg.children.length) ins.appendChild(seg); + if (kind === 'recipe' && !p.doc) ins.appendChild(h('p', 'failure', 'The text is not valid JSON, so the form cannot show it: ' + p.error + ' Correct it in the text below.')); + // A draft opens with what keeping it would find, and with where its settings came from. + if (draft && draft.problems.length) ins.appendChild(problemList(draft.problems, 'When it was drafted, keeping it would have found:')); + if (draft && draft.inlined) ins.appendChild(h('p', 'note', 'The settings were copied from ' + draft.inlined + ' so that they can be changed here; that file is not changed.')); + ins.appendChild(view === 'form' ? recipeForm(ref, base, p.doc, all, data, act, sel) : textView(ref, base, text)); + if (draft) ins.appendChild(stepsBlock(ref, { kept: false }, act, data, null)); + else { + // Keep, Discard edit, Remove and Save; drawn again when typing starts or ends (Discard edit then applies). + const box = h('div'); + const controls = () => { + const typed = buffers.has(ref), edited = def.unsaved || def.removed; + box.textContent = ''; + box.appendChild(A.group('edit:' + ref, [ + { id: 'keep', label: 'Keep', cls: 'btn primary', who: 'edit', op: 'set_definition', busy: 'Keeping…', args: () => ({ reference: ref, text: textOf(ref, def) }), done: () => buffers.delete(ref) }, + { id: 'discard', label: 'Discard edit', cls: 'btn', who: 'edit', busy: 'Discarding…', hidden: !typed && !edited, + confirm: (typed && edited ? 'Drop what was typed here, and put back' : typed ? 'Drop what was typed here; the definition stays as it is kept.' : 'Put back') + (edited ? ' the file in the project folder in place of the unsaved ' + (def.removed ? 'removal' : 'edit') + '; a definition the folder has no file for is dropped.' : ''), + // Whether there is an unsaved edit to put back is read afresh: this drawer may have been drawn before a Keep. + run: async () => { + buffers.delete(ref); decorate(document); + const now = await P.call('definition', { reference: ref }); + return now.unsaved || now.removed ? P.call('revert_definition', { reference: ref }) : { reference: ref, dropped: false }; + }, + done: r => { if (r && r.dropped) act.select(null); } }, + { id: 'remove', label: 'Remove', cls: 'btn ghost', who: 'edit', op: 'remove_definition', busy: 'Removing…', hidden: !!def.removed, args: () => ({ reference: ref }), + confirm: 'Remove ' + ref + ': it is marked removed and its file is deleted when the project is saved; one never saved is dropped now.', + done: () => { buffers.delete(ref); act.select(null); } }, + { id: 'save', label: 'Save', cls: 'btn', who: 'edit', op: 'save', busy: 'Saving…', hidden: !edited } + ])); + box.appendChild(h('p', 'note', 'Keep checks the text and keeps it in Tuldok as an unsaved edit; Save writes every unsaved edit to the project folder.')); + }; + here.controls = controls; controls(); ins.appendChild(box); + } + decorate(ins); + return ins; + } + function textView(ref, base, text) { + const l = h('label', 'code-field'); + l.appendChild(h('span', 'small muted', 'The definition, as JSON')); + const area = h('textarea', 'code'); area.value = text; area.spellcheck = false; area.autocomplete = 'off'; area.wrap = 'off'; + area.rows = Math.min(28, Math.max(10, text.split('\n').length + 1)); area.dataset.key = 'editor:' + ref + ':text'; + area.addEventListener('input', () => setText(ref, area.value, base)); + l.appendChild(area); + return l; + } + function problemList(problems, lead) { + const errors = problems.some(p => p.severity !== 'warning'); + const box = h('div', errors ? 'failure refusal problems' : 'problems warnings'); + box.appendChild(h('p', null, lead)); + const ul = h('ul'); + for (const p of problems) { const where = [p.path, p.pointer].filter(x => x && x !== '/').join(' '); ul.appendChild(h('li', null, (p.severity === 'warning' ? 'Warning: ' : '') + (where ? where + ': ' : '') + (p.message || ''))); } + box.appendChild(ul); + box.appendChild(h('p', 'small', 'Keep checks the text again and lists what is still wrong.')); + return box; + } + + // ---- The form of a recipe --------------------------------------------------------------- + function recipeForm(ref, base, doc, all, data, act, sel) { + const f = h('div', 'form editor-form'), k = name => 'editor:' + ref + ':' + name, upd = fn => edit(ref, base, fn); + const field = (label, control, hint) => { const l = h('label'); l.appendChild(document.createTextNode(label)); l.appendChild(control); if (hint) l.appendChild(h('span', 'hint', hint)); return l; }; + const input = (type, value, name, max) => { const i = h('input'); i.type = type; if (value !== undefined && value !== null) i.value = String(value); if (max) i.maxLength = max; i.dataset.key = k(name); i.autocomplete = 'off'; if (type === 'text') i.spellcheck = false; return i; }; + const choose = (options, value, name, label) => { const s = h('select'); s.dataset.key = k(name); s.setAttribute('aria-label', label); for (const [v, t] of options) { const o = h('option', null, t); o.value = v; o.selected = v === value; s.appendChild(o); } return s; }; + const asNumber = text => { const t = text.trim(); if (!t) return null; const n = Number(t); return Number.isFinite(n) ? n : undefined; }; + // A draft's node on the map follows its title, group and kind when such a field is left. + const isDraftRef = !!draftOf(ref), redraw = () => { if (isDraftRef) { publish(); act.redraw(); } }; + + const title = input('text', doc.title, 'title', 200); title.required = true; + title.addEventListener('input', () => upd(d => { d.title = title.value; })); title.addEventListener('change', redraw); + f.appendChild(field('Title', title)); + const two = h('div', 'two'); + const short = input('text', isObject(doc.labels) && typeof doc.labels.short === 'string' ? doc.labels.short : '', 'short', 16); + short.addEventListener('input', () => upd(d => { + if (short.value) { if (!isObject(d.labels)) d.labels = {}; d.labels.short = short.value; } + else if (isObject(d.labels)) { delete d.labels.short; if (!Object.keys(d.labels).length) delete d.labels; } + })); + short.addEventListener('change', redraw); + two.appendChild(field('Short label', short, 'at most 16 characters')); + const groups = (data.map.groups || []).map(g => { const up = []; let at = g; const byId = new Map(data.map.groups.map(x => [x.id, x])); while (at.parent && byId.has(at.parent) && up.length < 8) { at = byId.get(at.parent); up.unshift(at.title); } return [g.id, up.concat([g.title]).join(' / ')]; }); + const groupValue = typeof doc.group === 'string' ? doc.group : ''; + if (groupValue && !groups.some(g => g[0] === groupValue)) groups.push([groupValue, groupValue + ' (not a group of the project)']); + const grp = choose([['', 'None']].concat(groups), groupValue, 'group', 'Group'); + grp.addEventListener('change', () => { upd(d => { if (grp.value) d.group = grp.value; else delete d.group; }); redraw(); }); + two.appendChild(field('Group', grp)); + f.appendChild(two); + // Kind of change: the four kinds, or not set. + const kinds = h('div', 'radio'); kinds.setAttribute('role', 'radiogroup'); kinds.setAttribute('aria-label', 'Kind of change'); + const changeValue = P.KINDS.some(x => x.id === doc.change) ? doc.change : ''; + for (const [v, label] of P.KINDS.map(x => [x.id, x.label]).concat([['', 'Not set']])) { + const l = h('label', 'chip-radio'); const r = h('input'); r.type = 'radio'; r.name = 'change-' + ref; r.value = v; r.checked = v === changeValue; r.dataset.key = k('change-' + (v || 'unset')); + r.addEventListener('change', () => { if (!r.checked) return; upd(d => { if (v) d.change = v; else delete d.change; }); redraw(); }); + l.appendChild(r); l.appendChild(h('span', 'chip ' + (v || P.UNSET), label)); kinds.appendChild(l); + } + f.appendChild(field('Kind of change', kinds)); + // Derived from: the parent's name, Pick on the map, Clear. + const parentRef = typeof doc.derived_from === 'string' ? doc.derived_from : null, pm = parentRef && data.map.byId.get(parentRef); + const der = h('div', 'derived'); + der.appendChild(h('span', 'derived-name' + (parentRef ? '' : ' faint'), parentRef ? (pm ? Tree.nameOf(pm) : parentRef) : 'none')); + der.appendChild(button('btn sm', 'Pick on the map', k('pick'), () => startPicking(ref, base, sel, act))); + if (parentRef) der.appendChild(button('btn sm ghost', 'Clear', k('clear'), () => { upd(d => { delete d.derived_from; }); if (current && current.rebuild) current.rebuild(); })); + f.appendChild(field('Derived from', der)); + const desc = h('textarea'); desc.rows = 3; desc.maxLength = 4000; desc.value = typeof doc.description === 'string' ? doc.description : ''; desc.dataset.key = k('description'); + desc.addEventListener('input', () => upd(d => { if (desc.value) d.description = desc.value; else delete d.description; })); + f.appendChild(field('What this model changes', desc, 'at most 4,000 characters')); + if (!isObject(doc.run)) { f.appendChild(h('p', 'note', 'This recipe has no run: it is earlier work, or its run is to be written in the text view.')); return f; } + const run = doc.run, defs = all.filter(d => !d.removed), env = typeof run.environment === 'string' ? run.environment : ''; + const envs = defs.filter(d => d.kind === 'environment').map(d => [d.reference, d.title + ' · ' + d.reference]); + if (env && !envs.some(e => e[0] === env)) envs.push([env, env + ' (not an environment of the project)']); + const es = choose((env ? [] : [['', 'None']]).concat(envs), env, 'environment', 'Environment'); + es.addEventListener('change', () => upd(d => { if (!isObject(d.run)) return; if (es.value) d.run.environment = es.value; else delete d.run.environment; })); + f.appendChild(field('Environment', es)); + // Each input from a dataset: another dataset, keeping a named output. + const sets = defs.filter(d => d.kind === 'dataset').map(d => [d.reference, d.title + ' · ' + d.reference]); + for (const [name, v] of Object.entries(isObject(run.inputs) ? run.inputs : {})) { + const from = typeof v === 'string' ? v : isObject(v) && typeof v.from === 'string' ? v.from : null; + if (!from || !from.startsWith('datasets/')) continue; + const options = sets.some(s => s[0] === from) ? sets : sets.concat([[from, from]]); + const s = choose(options, from, 'input-' + name, 'Input ' + name); + s.addEventListener('change', () => upd(d => { const ins = isObject(d.run) && isObject(d.run.inputs) ? d.run.inputs : null; if (!ins) return; if (isObject(ins[name])) ins[name].from = s.value; else ins[name] = s.value; })); + f.appendChild(field('Input ' + name + (isObject(v) && v.output ? ' · its output ' + v.output : ''), s)); + } + // Settings: one field per top-level number, text or true/false; the rest are edited in the text. + const st = h('div', 'settings'); st.appendChild(h('h3', null, 'Settings')); + if (isObject(run.settings)) { + const grid = h('div', 'two'), rest = []; + for (const [name, v] of Object.entries(run.settings)) { + const set = fn => upd(d => { if (isObject(d.run) && isObject(d.run.settings)) fn(d.run.settings); }); + if (typeof v === 'number') { + const i = input('number', v, 'setting-' + name); i.step = 'any'; + i.addEventListener('input', () => { const n = asNumber(i.value); if (n === null || n === undefined) return; set(s => { s[name] = n; }); }); + grid.appendChild(field(name, i)); + } else if (typeof v === 'string') { + const i = input('text', v, 'setting-' + name); i.addEventListener('input', () => set(s => { s[name] = i.value; })); grid.appendChild(field(name, i)); + } else if (typeof v === 'boolean') { + const l = h('label', 'check'); const c = h('input'); c.type = 'checkbox'; c.checked = v; c.dataset.key = k('setting-' + name); + c.addEventListener('change', () => set(s => { s[name] = c.checked; })); l.appendChild(c); l.appendChild(document.createTextNode(' ' + name)); grid.appendChild(l); + } else rest.push(name); + } + if (grid.children.length) st.appendChild(grid); else st.appendChild(h('p', 'note', 'No setting is a number, a text or true/false.')); + if (rest.length) st.appendChild(h('p', 'note', 'Edited in the text: ' + rest.join(', ') + '.')); + } else if (typeof run.settings_file === 'string') { + st.appendChild(h('p', 'note mono', run.settings_file)); + st.appendChild(h('p', 'note', 'Its settings are in this file, which is changed in the code. A branch of this model carries its own copy of them, which its form can change.')); + } else st.appendChild(h('p', 'note', 'It has no settings.')); + f.appendChild(st); + // Resources. + const rs = h('div', 'settings'); rs.appendChild(h('h3', null, 'Resources')); + const res = isObject(run.resources) ? run.resources : {}; + const setRes = fn => upd(d => { if (!isObject(d.run)) return; if (!isObject(d.run.resources)) d.run.resources = {}; fn(d.run.resources); if (!Object.keys(d.run.resources).length) delete d.run.resources; }); + const gl = h('label', 'check'); const gpu = h('input'); gpu.type = 'checkbox'; gpu.checked = res.gpu === true; gpu.dataset.key = k('gpu'); + gpu.addEventListener('change', () => setRes(r => { if (gpu.checked) r.gpu = true; else if ('gpu' in r) r.gpu = false; })); + gl.appendChild(gpu); gl.appendChild(document.createTextNode(' Uses the GPU')); rs.appendChild(gl); + const rg = h('div', 'two'); + for (const [name, label] of [['wall_hours', 'Wall hours'], ['disk_gb', 'Disk (GB)']]) { + const i = input('number', typeof res[name] === 'number' ? res[name] : '', name); i.step = 'any'; i.min = '0'; + i.addEventListener('input', () => { const n = asNumber(i.value); if (n === undefined) return; setRes(r => { if (n === null) delete r[name]; else r[name] = n; }); }); + rg.appendChild(field(label, i, 'empty: not stated')); + } + rs.appendChild(rg); f.appendChild(rs); + return f; + } + + // ---- Pick on the map -------------------------------------------------------------------- + function startPicking(ref, base, sel, act) { + picking = { reference: ref, base, sel: draftOf(ref) ? ref : 'edit:' + ref }; pickNote = ''; + if (location.hash.startsWith('#map/')) act.reopen(); else act.go('map', picking.sel); + setTimeout(() => A.focusKey('editor:' + ref + ':stop-picking'), 0); + } + function pickingBanner(act) { + const box = h('div', 'picking'); box.setAttribute('role', 'status'); + box.appendChild(h('p', null, 'Press a model on the map to make it the parent · Escape to stop')); + if (pickNote) box.appendChild(h('p', 'small warnline', pickNote)); + box.appendChild(button('btn sm', 'Stop', 'editor:' + picking.reference + ':stop-picking', () => { stopPicking(act); })); + return box; + } + function stopPicking(act) { const was = picking; picking = null; pickNote = ''; act.reopen(); if (was) setTimeout(() => A.focusKey('editor:' + was.reference + ':pick'), 0); } + /** screens.js, when a model is pressed on the map: true when it was taken as the parent being picked. */ + function picked(id, act) { + if (!picking) return false; + if (id === picking.reference) { pickNote = 'A model is not derived from itself: press another one.'; act.reopen(); return true; } + if (draftOf(id)) { pickNote = 'A draft cannot be a parent before it is kept: press another one.'; act.reopen(); return true; } + const { reference, base, sel } = picking; picking = null; pickNote = ''; + edit(reference, base, d => { d.derived_from = id; }); + if (draftOf(reference)) publish(); + act.go('map', sel); act.reopen(); + setTimeout(() => A.focusKey('editor:' + reference + ':pick'), 0); + return true; + } + /** app.js, on Escape: leaves the picking state; true when it was in it. */ + function escape() { if (!picking) return false; const was = picking; picking = null; pickNote = ''; if (current) current.rebuild && current.rebuild(); setTimeout(() => A.focusKey('editor:' + was.reference + ':pick'), 0); return true; } + + // ---- Branch from here ------------------------------------------------------------------- + function branchForm(aside, sel, parentRef, act, data) { + if (aside._for === sel && aside._again !== 'control') return true; + aside._for = sel; aside._editing = null; + const m = data.map.byId.get(parentRef); + const ins = h('div', 'inspector branch'); + if (!m || m.draft) { ins.appendChild(h('p', 'empty', 'There is no model ' + parentRef + ' on the map to branch from.')); swapInto(aside, ins); return true; } + const key = 'branch:' + parentRef, s = A.slot(key); + if (s.text.change === undefined) s.text.change = m.kind === P.UNSET ? '' : m.kind; + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'branch · from ' + Tree.nameOf(m))); + const t = h('h2', null, 'A new model from ' + Tree.refName(m)); t.tabIndex = -1; hd.appendChild(t); ins.appendChild(hd); + ins.appendChild(h('p', 'note', 'A new recipe written from this one, with its model, group, run and settings, derived from it. It is a draft in this page until it is kept.')); + const f = h('div', 'form'); + const field = (label, control, hint) => { const l = h('label'); l.appendChild(document.createTextNode(label)); l.appendChild(control); if (hint) l.appendChild(h('span', 'hint', hint)); return l; }; + const title = h('input'); title.type = 'text'; title.maxLength = 200; title.required = true; title.value = s.text.title || ''; title.dataset.key = key + ':field:title'; title.autocomplete = 'off'; + title.placeholder = 'What the new model is'; + title.addEventListener('input', () => { + s.text.title = title.value; + // A refusal for a missing title is over once one is typed. + if (s.refusal && title.value.trim() && /title/.test(s.refusal.error.message)) { s.refusal = null; const r = ins.querySelector('.act-group .refusal'); if (r) r.remove(); } + }); + title.addEventListener('keydown', e => { if (e.key === 'Enter') { e.preventDefault(); A.focusKey(key + ':create'); const b = ins.querySelector('[data-key="' + CSS.escape(key + ':create') + '"]'); if (b && !b.disabled) b.click(); } }); + f.appendChild(field('Title (required)', title, 'at most 200 characters')); + const kinds = h('div', 'radio'); kinds.setAttribute('role', 'radiogroup'); kinds.setAttribute('aria-label', 'Kind of change'); + for (const [v, label] of P.KINDS.map(x => [x.id, x.label]).concat([['', 'Not set']])) { + const l = h('label', 'chip-radio'); const r = h('input'); r.type = 'radio'; r.name = key + ':change'; r.value = v; r.checked = (s.text.change || '') === v; r.dataset.key = key + ':change-' + (v || 'unset'); + r.addEventListener('change', () => { if (r.checked) s.text.change = v; }); + l.appendChild(r); l.appendChild(h('span', 'chip ' + (v || P.UNSET), label)); kinds.appendChild(l); + } + f.appendChild(field('Kind of change', kinds, m.kind === P.UNSET ? 'its parent names none' : 'its parent\'s to begin with')); + const desc = h('textarea'); desc.rows = 3; desc.maxLength = 4000; desc.value = s.text.description || ''; desc.dataset.key = key + ':field:description'; + desc.addEventListener('input', () => { s.text.description = desc.value; }); + f.appendChild(field('What the model changes', desc, 'at most 4,000 characters')); + ins.appendChild(f); + ins.appendChild(A.group(key, [ + { id: 'create', label: 'Create', cls: 'btn primary', who: 'edit', busy: 'Creating…', local: true, + run: async st => { + const wanted = (st.text.title || '').trim(); + if (!wanted) throw Error('A branch needs a title.'); + const args = { recipe: parentRef, title: wanted.slice(0, 200) }; + if (st.text.change) args.change = st.text.change; + const about = (st.text.description || '').trim(); if (about) args.description = about.slice(0, 4000); + let answer = await P.call('draft_branch', args); + // A draft of this page holds the name already: the next free one. + const stem = answer.reference.slice('recipes/'.length).replace(/-\d+$/, '').slice(0, 110); + for (let n = 2; draftOf(answer.reference) && n < 200; n++) { + if (draftOf('recipes/' + stem + '-' + n)) continue; + try { answer = await P.call('draft_branch', Object.assign({}, args, { name: stem + '-' + n })); } catch (error) { if (error.code !== 'invalid') throw error; } + } + if (draftOf(answer.reference)) throw Error('No free name was found for the branch; give it another title.'); + return answer; + }, + done: answer => { + drafts.push({ reference: answer.reference, parent: parentRef, text: answer.text, inlined: answer.inlined || null, problems: answer.problems || [] }); + views.set(answer.reference, 'form'); persist(); publish(); + act.redraw().then(() => act.go('map', answer.reference)); + } }, + { id: 'back', label: 'Back', cls: 'btn ghost', onClick: () => act.go('map', parentRef) } + ])); + swapInto(aside, ins); + return true; + } + + // ---- The steps of a new model ---------------------------------------------------------------- + /** Whether a model's drawer shows the steps of a new model: a draft, one branched here, or one never built (unless it is earlier work). */ + const followsSteps = m => !!m && (m.draft || m.planned || followed.includes(m.id)); + /** + * The steps of a model of the map (screens.js): read its definition, runs and + * the build requests; earlier work gets planGroup() instead (as phase 2). + * Fills box, on the page or not yet (a drawer read again is built off the + * page), and resolves once it is filled. + */ + function steps(box, m, act, data, planGroup) { + return Promise.all([P.read('definition', { reference: m.id }), P.read('runs', { reference: m.id }), P.read('build_requests')]).then(([def, runs, requests]) => { + box.textContent = ''; + if (isObject(def.document) && def.document.earlier_work && !followed.includes(m.id)) { if (['not-built', 'out-of-date', 'failed'].includes(m.state)) box.appendChild(planGroup()); return; } + const asked = requests.requests.filter(r => (r.targets || []).includes(m.id) && !['declined', 'cancelled', 'expired'].includes(r.state)); + const runList = runs.runs, started = runList.filter(r => r.started_at || ['running', 'succeeded', 'failed'].includes(r.state)); + box.appendChild(stepsBlock(m.id, { + kept: true, saved: !def.unsaved, requested: asked.length > 0 || runList.length > 0, running: started.length > 0, + runningNow: runList.some(r => r.state === 'running'), built: ['built', 'out-of-date'].includes(m.state), unsaved: def.unsaved, + request: asked[0] || null, run: runList[0] || null, state: m.state + }, act, data, planGroup)); + }, error => { box.textContent = ''; box.appendChild(A.refusal(error)); }); + } + function stepsBlock(ref, f, act, data) { + const wrap = h('div', 'new-steps'); + wrap.appendChild(h('h3', null, 'From draft to built')); + const ol = h('ol', 'steps'); + const rows = [ + ['Kept', f.kept, f.kept ? 'Kept in Tuldok as a definition.' : 'Not kept yet: it is in this page only, and Keep checks it.'], + ['Saved', f.saved, f.saved ? 'Written to the project folder.' : f.kept ? 'An unsaved edit: Save writes it to the project folder.' : 'Not in the project folder.'], + ['Build requested', f.requested, f.request ? 'Asked for' + (f.request.state === 'awaiting-approval' ? ', awaiting approval.' : ' · ' + f.request.state + '.') : f.requested ? 'Asked for.' : 'Not asked for: plan its build first.'], + ['Running', f.running, f.runningNow ? 'Running now.' : f.running ? 'Its build has run' + (f.run ? ' · ' + f.run.state : '') + '.' : 'Not started.'], + ['Built', f.built, f.built ? 'Built: its product is on this machine.' : 'No product yet.'] + ]; + let next = false; + for (const [title, done, sub] of rows) { + const now = !done && !next; if (now) next = true; + const li = h('li', done ? 'done' : 'todo'); li.dataset.step = title; + li.appendChild(S.stateEl(done ? 'good' : now && (title === 'Running' && f.runningNow) ? 'pending' : 'neutral', '')); + const t = h('span'); t.appendChild(h('span', 'title', title)); t.appendChild(h('span', 'sub', sub)); li.appendChild(t); ol.appendChild(li); + } + wrap.appendChild(ol); + // One primary button, the next step; and Discard. + const items = []; + if (!f.kept) { + items.push({ id: 'keep', label: 'Keep', cls: 'btn primary', who: 'edit', op: 'set_definition', busy: 'Keeping…', args: () => ({ reference: ref, text: textOf(ref, null) }), + done: () => { drafts = drafts.filter(d => d.reference !== ref); if (!followed.includes(ref)) followed.push(ref); persist(); publish(); } }); + items.push({ id: 'discard', label: 'Discard', cls: 'btn ghost', who: 'edit', local: true, confirm: 'Drop this draft; nothing of it was kept.', run: async () => true, + done: () => { const d = draftOf(ref); drafts = drafts.filter(x => x.reference !== ref); persist(); publish(); act.redraw().then(() => act.go('map', d ? d.parent : undefined)); } }); + } else { + if (!f.saved) items.push({ id: 'save', label: 'Save', cls: 'btn primary', who: 'edit', op: 'save', busy: 'Saving…' }); + else if (!f.requested) items.push({ id: 'plan', label: 'Plan build', cls: 'btn primary', who: 'request', onClick: () => act.select('plan:' + ref) }); + else items.push({ id: 'train', label: 'Open in Train', cls: 'btn primary', onClick: () => act.go('train', f.request && !f.run ? 'request:' + f.request.id : f.run ? 'run:' + f.run.id : undefined) }); + if (f.unsaved) items.push({ id: 'discard', label: 'Discard', cls: 'btn ghost', who: 'edit', op: 'revert_definition', busy: 'Discarding…', args: () => ({ reference: ref }), + confirm: 'Put back the file in the project folder in place of the unsaved edit; a model never saved is dropped.', done: r => { if (r && r.dropped) act.select(null); } }); + } + wrap.appendChild(A.group(f.kept ? 'model:' + ref : 'draft:' + ref, items)); + return wrap; + } + + // ---- The Overview's list of every definition ---------------------------------------------------- + function definitionList(box, defs, sel, act) { + const can = A.may('edit'); + box.appendChild(sect('Every definition', defs.length + (can ? ' · each opens the editor' : ''))); + if (!defs.length) { box.appendChild(h('p', 'empty', 'The project has no definitions.')); return; } + const kinds = KIND_ORDER.concat([...new Set(defs.map(d => d.kind))].filter(k => !KIND_ORDER.includes(k))); + for (const kind of kinds) { + const mine = defs.filter(d => d.kind === kind); if (!mine.length) continue; + box.appendChild(h('h3', 'ov-kind', (KIND_TITLE[kind] || kind) + ' · ' + mine.length)); + const ul = h('ul', 'rows defs'); + for (const d of mine) { + const li = h('li'), row = can ? button('row def', null, 'open-edit:' + d.reference, () => act.select(sel === 'edit:' + d.reference ? null : 'edit:' + d.reference)) : h('div', 'row def'); + if (can) row.setAttribute('aria-pressed', String(sel === 'edit:' + d.reference)); + row.dataset.ref = d.reference; + row.appendChild(h('span', 'mono name', d.reference.slice(d.reference.indexOf('/') + 1))); + row.appendChild(h('span', 'title', d.title)); + row.appendChild(modelState(d.state)); + const mark = Tree.mark({ id: d.reference, unsaved: d.unsaved }); if (d.removed) mark.dataset.removed = '1'; row.appendChild(mark); + li.appendChild(row); ul.appendChild(li); + } + box.appendChild(ul); + } + } + + return { load, subject, owns, drawer, isDraft, picked, escape, decorate, followsSteps, steps, definitionList }; +})(); diff --git a/static/project-workspace/fields.js b/static/project-workspace/fields.js new file mode 100644 index 00000000..86d5f7a8 --- /dev/null +++ b/static/project-workspace/fields.js @@ -0,0 +1,283 @@ +'use strict'; +// The fields a run kept (docs/plans/definition-first-projects/spec-d6.md section 3): the section "Kept fields" of +// the Analysis tab, below "Per case", for the chosen subject. When the subject's model has a current product whose +// run kept fields (fields), the section shows one case (a select when there are several): its name, split and +// shape and the run it comes from; three panels side by side, Native, Predicted and Signed error (predicted − +// native), slices through the field read by field_compare; Native and Predicted on one sequential scale over the +// slice's range, the error on a diverging scale symmetric about zero; a colour bar under each with its ends; the +// axis cut across, a slider for the position on it (and one more for a fourth axis) and "Remove the mean"; a tip +// with the cell's position and its three values under the pointer; the error's figures over the whole field, and +// what the run reported beside them. The section is absent when the model has no product, one line when its +// products keep no fields. +// +// Updating in place (spec-w4): tabs.js reads what the section shows with the rest of the tab (read()) and builds +// it with the tab off the page (section()), so a reading that changed something draws it again in place, where the +// person left it (the case, axis, position and mean option are the tab's settings, kept in the address). The +// person's own moves are drawn here and nothing else of the tab: a slider, the axis or "Remove the mean" asks +// field_compare again (at most one request in flight, the last position wins) and the panels are drawn again in +// their canvases; another case builds the section again and puts it in place. A timer or a return to the window +// with nothing changed never reaches here. +window.LabFields = (() => { + const P = window.TuldokProject, S = window.LabScreens, C = window.LabCharts, h = C.h, R = P.product; + const { sect, putInPlace } = S.parts; + const D = window.LabInside.drawing; + const fmt = v => v == null ? '—' : C.fmtSci(v); + const num = n => n == null ? '—' : n.toLocaleString('en-GB'); + const shapeText = s => s && s.length ? s.join(' × ') : '—'; + const button = (cls, text, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (onClick) b.addEventListener('click', onClick); return b; }; + const refusal = error => window.LabActions.refusal(error); + const PANEL_MOST = 420; // the CSS height of a panel at most + const axisNames = nd => nd === 3 ? ['x', 'y', 'z'] : Array.from({ length: nd }, (_, i) => 'axis ' + i); + + // ---- Colours: the tokens of base.css ------------------------------------------------------------------------------- + const mix = (a, b, t) => [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t]; + /** Sequential from --ramp-lo to --ramp-hi over [min, max], whatever the signs. */ + function sequential(min, max) { const lo = D.token('--ramp-lo'), hi = D.token('--ramp-hi'), span = max - min || 1; return v => mix(lo, hi, Math.min(1, Math.max(0, (v - min) / span))); } + /** Diverging about zero over [−m, m]: --ramp-mid at zero, --ramp-neg and --ramp-pos at the ends. */ + function diverging(m) { const neg = D.token('--ramp-neg'), mid = D.token('--ramp-mid'), pos = D.token('--ramp-pos'), s = m || 1; return v => v < 0 ? mix(mid, neg, Math.min(1, -v / s)) : mix(mid, pos, Math.min(1, v / s)); } + + // ---- Reading -------------------------------------------------------------------------------------------------------- + const keyOf = (run, entry, one) => run + ':' + entry + ':' + one; + const intOr = (v, d) => { const n = Number(v); return v !== null && v !== undefined && v !== '' && Number.isInteger(n) && n >= 0 ? n : d; }; + /** The person's place in a case from the tab's settings (strings when they came from the address), checked against its shape. */ + function viewFor(c, ui) { + const nd = c.shape ? c.shape.length : 0, mid = (c.shape || []).map(n => Math.floor(n / 2)); + let axis = intOr(ui.anaAxis, nd - 1); if (axis >= nd) axis = nd - 1; + let index = typeof ui.anaIndex === 'string' ? ui.anaIndex.split(',').map(x => intOr(x, -1)) : Array.isArray(ui.anaIndex) ? ui.anaIndex.slice() : null; + if (!index || index.length !== nd || index.some((p, i) => p < 0 || p >= c.shape[i])) index = mid; + const meanFree = ui.anaMeanFree === true || ui.anaMeanFree === '1' || ui.anaMeanFree === 'true'; + return { axis, index, meanFree }; + } + const argsOf = (c, v) => ({ entry: c.entry, case: c.case, axis: c.shape.length >= 3 ? v.axis : null, index: v.index.slice(), mean_free: v.meanFree }); + + /** + * What the section shows for a subject's recipe, read in full: null when the model has no current product (the + * section is absent), { none } when none of its products keeps fields, else the cases, the one chosen, the + * person's place in it and its first slice. + */ + async function read(recipe, ui) { + const found = await keptCases(recipe); + if (!found || found.none) return found; + const cases = found.cases; + const chosen = cases.find(c => c.key === ui.anaCase) || cases.find(c => c.comparable) || cases[0]; + const out = { cases, chosen, view: null, cmp: null, error: null }; + if (chosen.comparable) { + out.view = viewFor(chosen, !ui.anaCase || chosen.key === ui.anaCase ? ui : {}); + try { out.cmp = await R.fieldCompare(chosen.run, argsOf(chosen, out.view)); } catch (error) { out.error = error; } + } + return out; + } + + /** + * The cases kept by every current product of a recipe's model: null when it has no current product, { none } when + * none of them keeps fields (with the first refusal, or empty when an index lists no case), else { cases }, each + * case as fields lists it with its run, subject, key, the run's reported figures and a label. Shared with the + * Compare tab (compare.js, spec-t3 section 5.1). + */ + async function keptCases(recipe) { + if (!recipe) return null; + const products = (await P.read('products', { reference: recipe })).products.filter(p => p.run_id); + if (!products.length) return null; + const got = await Promise.all(products.map(p => R.fields(p.run_id).then(f => ({ p, f }), error => ({ p, error })))); + const kept = got.filter(g => g.f && g.f.kept); + if (!kept.length) { const failed = got.find(g => g.error); return { none: true, error: failed ? failed.error : null }; } + const several = kept.length > 1; + const cases = kept.flatMap(g => g.f.cases.map(c => Object.assign({}, c, { run: g.p.run_id, subject: g.p.subject || '', key: keyOf(g.p.run_id, c.entry, c.case), reported: g.f.reported, label: c.name + (several && g.p.subject ? ' · ' + g.p.subject : '') }))); + if (!cases.length) return { none: true, empty: true, run: kept[0].p.run_id }; + return { cases }; + } + + // ---- The section ---------------------------------------------------------------------------------------------------- + /** The section for what read() gave (built off the page); null when it is absent. act: the tab's actions (keepUI). */ + function section(model, act) { + if (!model) return null; + const box = h('section', 'fld'); box.dataset.key = 'fld'; + fill(box, model, act); + return box; + } + function fill(box, model, act) { + box.appendChild(sect('Kept fields', model.none ? undefined : model.chosen.label)); + if (model.none) { + box.appendChild(h('p', 'empty fld-none', model.empty ? 'The run kept a field index that lists no case.' : model.error ? 'Whether this model\'s run kept fields could not be read: ' + (model.error.message || model.error) : 'The run of this model kept no fields: its product holds no field-index.json.')); + return; + } + const c = model.chosen, ctx = { box, model, c, act, v: model.view, cmp: model.cmp, busy: false, again: false }; + box.appendChild(head(ctx)); + if (!c.comparable) { box.appendChild(h('p', 'empty fld-reason', 'This case cannot be compared: ' + c.reason)); return; } + if (c.bytes_mismatch && c.bytes_mismatch.length) box.appendChild(h('p', 'small warnline fld-bytes', 'The index gives another size for ' + c.bytes_mismatch.join(' and ') + ' than the file has.')); + if (model.error) { box.appendChild(refusal(model.error)); return; } + ctx.ctl = h('div', 'iv-ctl fld-ctl'); controls(ctx); box.appendChild(ctx.ctl); + ctx.panels = panels(ctx); box.appendChild(ctx.panels); + ctx.figs = h('div', 'fld-figs'); figures(ctx); box.appendChild(ctx.figs); + } + /** The head: the case (a select when there are several), its split and shape, and the run, linked to Train. */ + function head(ctx) { + const { model, c, act } = ctx, hd = h('div', 'fld-head'); + if (model.cases.length > 1) { + const s = h('select'); s.setAttribute('aria-label', 'Case'); s.dataset.key = 'fld-case'; + for (const x of model.cases) { const o = h('option', null, x.label + (x.split ? ' · ' + x.split : '') + (x.comparable ? '' : ' (cannot be compared)')); o.value = x.key; o.selected = x === c; s.appendChild(o); } + s.addEventListener('change', () => chooseCase(ctx, s.value)); + const l = h('label', 'iv-field fld-pick'); l.appendChild(document.createTextNode('Case')); l.appendChild(s); hd.appendChild(l); + } else hd.appendChild(h('p', 'mono fld-name', c.name)); + const facts = [c.split ? 'split ' + c.split : null, c.shape ? shapeText(c.shape) : null, c.dtype, c.predicted + ' against ' + c.reference + (c.mask ? ', inside ' + c.mask : '')].filter(Boolean); + const line = h('p', 'small muted fld-facts', facts.join(' · ') + ' · from run '); + const run = h('a', 'mono fld-run', c.run.slice(0, 8)); run.href = '#train/' + encodeURIComponent('run:' + c.run); run.dataset.key = 'fld-run'; run.title = 'The run ' + c.run + ' on the Train tab'; + run.addEventListener('click', ev => { ev.preventDefault(); act.go('train', 'run:' + c.run); }); + line.appendChild(run); hd.appendChild(line); + return hd; + } + /** Another case: its first slice read, then the section built again and put in place. */ + async function chooseCase(ctx, key) { + const { model, act, box } = ctx, c = model.cases.find(x => x.key === key); if (!c) return; + act.keepUI('anaCase', key); act.keepUI('anaAxis', null); act.keepUI('anaIndex', null); + const next = { cases: model.cases, chosen: c, view: null, cmp: null, error: null }; + box.setAttribute('aria-busy', 'true'); + if (c.comparable) { next.view = viewFor(c, { anaMeanFree: ctx.v ? ctx.v.meanFree : false }); try { next.cmp = await R.fieldCompare(c.run, argsOf(c, next.view), true); } catch (error) { next.error = error; } } + if (!box.isConnected) return; + const fresh = h('div'); fill(fresh, next, act); + const frag = document.createDocumentFragment(); frag.append(...fresh.childNodes); + box.removeAttribute('aria-busy'); + putInPlace(box, frag, document.getElementById('main')); + } + + /** The axis cut across, the sliders of the axes not drawn and "Remove the mean"; drawn again when the axis changes. */ + function controls(ctx) { + const { c, v } = ctx, nd = c.shape.length, names = axisNames(nd), items = []; + if (nd >= 3) { + const s = h('select'); s.setAttribute('aria-label', 'The axis cut across'); s.dataset.key = 'fld-axis'; + for (let a = 0; a < nd; a++) { const o = h('option', null, names[a] + ' (' + c.shape[a] + ')'); o.value = String(a); o.selected = a === v.axis; s.appendChild(o); } + s.addEventListener('change', () => { v.axis = Number(s.value); ctx.act.keepUI('anaAxis', String(v.axis)); controls(ctx); ask(ctx); }); + const l = h('label', 'iv-field'); l.appendChild(document.createTextNode('Cut across')); l.appendChild(s); items.push(l); + for (const a of fixedAxes(nd, v.axis)) { + const l2 = h('label', 'iv-field iv-slider'); l2.appendChild(h('span', null, a === v.axis ? 'Position on ' + names[a] : names[a])); + const r = h('input'); r.type = 'range'; r.min = '0'; r.max = String(c.shape[a] - 1); r.value = String(v.index[a]); r.dataset.key = 'fld-slider-' + a; r.dataset.axis = String(a); + r.setAttribute('aria-label', 'Position on ' + names[a] + ', 0 to ' + (c.shape[a] - 1)); + const at = h('span', 'num small fld-at', v.index[a] + ' of ' + c.shape[a]); + r.addEventListener('input', () => { v.index[a] = Number(r.value); at.textContent = v.index[a] + ' of ' + c.shape[a]; ctx.act.keepUI('anaIndex', v.index.join(',')); ask(ctx); }); + l2.appendChild(r); l2.appendChild(at); items.push(l2); + } + } + const mean = button('btn sm fld-mean', 'Remove the mean', () => { v.meanFree = !v.meanFree; mean.setAttribute('aria-pressed', String(v.meanFree)); ctx.act.keepUI('anaMeanFree', v.meanFree ? '1' : null); ask(ctx); }); + mean.dataset.key = 'fld-mean'; mean.setAttribute('aria-pressed', String(v.meanFree)); mean.title = 'Take each field\'s mean from it before the error is formed (a pressure is defined only up to a constant)'; + items.push(mean); + if (ctx.ctl.isConnected) { const frag = document.createDocumentFragment(); frag.append(...items); putInPlace(ctx.ctl, frag, document.getElementById('main')); } + else ctx.ctl.replaceChildren(...items); + } + /** The axes not drawn of a field of nd axes cut across axis: axis first, then the other. */ + const fixedAxes = (nd, axis) => { const others = []; for (let a = 0; a < nd; a++) if (a !== axis) others.push(a); return [axis].concat(others.slice(0, -2)); }; + + /** Asks field_compare for the person's place: at most one request in flight; one asked meanwhile waits, the latest wins. */ + function ask(ctx) { + if (ctx.busy) { ctx.again = true; return; } + ctx.busy = true; ctx.again = false; ctx.panels.setAttribute('aria-busy', 'true'); + R.fieldCompare(ctx.c.run, argsOf(ctx.c, ctx.v), true).then(cmp => { + if (ctx.again || !ctx.panels.isConnected) return; + ctx.cmp = cmp; ctx.err.replaceChildren(); redraw(ctx); figures(ctx); + }, error => { if (!ctx.again) ctx.err.replaceChildren(refusal(error)); }).finally(() => { + ctx.busy = false; + if (ctx.again && ctx.panels.isConnected) ask(ctx); else ctx.panels.removeAttribute('aria-busy'); + }); + } + + // ---- The three panels ----------------------------------------------------------------------------------------------- + const PANELS = [['reference', 'Native', 'reference_f32'], ['predicted', 'Predicted', 'predicted_f32'], ['error', 'Signed error (predicted − native)', 'error_f32']]; + function panels(ctx) { + const wrap = h('div', 'fld-panels'); ctx.parts = {}; + for (const [id, title, field] of PANELS) { + const fig = h('figure', 'fld-panel'); fig.dataset.panel = id; + fig.appendChild(h('figcaption', 'fld-title', title)); + const cv = h('canvas', 'fld-cv'); cv.setAttribute('role', 'img'); cv.dataset.panel = id; fig.appendChild(cv); + const bar = h('canvas', 'fld-bar'); bar.setAttribute('aria-hidden', 'true'); fig.appendChild(bar); + const ends = h('div', 'iv-ends small faint fld-ends'); fig.appendChild(ends); + ctx.parts[id] = { cv, bar, ends, field }; + D.paints(cv, () => drawPanel(ctx, id)); D.paints(bar, () => drawBar(ctx, id)); + cv.addEventListener('mousemove', ev => hover(ctx, id, ev)); cv.addEventListener('mouseleave', () => { ctx.tip.hidden = true; }); + wrap.appendChild(fig); + } + ctx.tip = h('div', 'fld-tip'); ctx.tip.hidden = true; ctx.tip.setAttribute('role', 'status'); wrap.appendChild(ctx.tip); + ctx.say = h('p', 'small muted fld-where'); ctx.err = h('div', 'fld-err'); + const outer = h('div', 'fld-view'); outer.appendChild(wrap); outer.appendChild(ctx.say); outer.appendChild(ctx.err); + ctx.wrap = wrap; decode(ctx); label(ctx); + return outer; + } + /** The answer's three slices as floats, and the scales they are drawn on. */ + function decode(ctx) { + const a = ctx.cmp, range = a.range || [0, 0]; + ctx.values = { reference: D.floats(a.reference_f32), predicted: D.floats(a.predicted_f32), error: D.floats(a.error_f32) }; + ctx.scale = { reference: sequential(range[0], range[1]), predicted: sequential(range[0], range[1]), error: diverging(a.error_limit || 0) }; + } + function redraw(ctx) { + decode(ctx); label(ctx); + for (const id of Object.keys(ctx.parts)) { drawPanel(ctx, id); drawBar(ctx, id); } + ctx.wrap.dataset.index = JSON.stringify(ctx.cmp.index); ctx.wrap.dataset.axis = String(ctx.cmp.axis); ctx.wrap.dataset.meanFree = String(ctx.cmp.mean_free); + } + /** What the panels show, in words: their labels, the colour bars' ends and the line under them. */ + function label(ctx) { + const a = ctx.cmp, c = ctx.c, nd = c.shape.length, names = axisNames(nd), range = a.range; + const where = nd >= 3 ? 'through ' + a.axes.map(x => names[x]).join(' and ') + ' at ' + fixedAxes(nd, a.axis).map(x => names[x] + ' = ' + a.index[x]).join(', ') : nd === 2 ? 'the whole field' : 'the whole line'; + for (const [id, title] of PANELS) { + const p = ctx.parts[id]; + p.cv.setAttribute('aria-label', title + ', ' + a.rows + ' by ' + a.columns + ' cells ' + where + (id === 'error' ? ', ±' + fmt(a.error_limit) + ' about zero' + (a.mean_free ? ', each mean removed' : '') : range ? ', from ' + fmt(range[0]) + ' to ' + fmt(range[1]) : '')); + p.ends.replaceChildren(...(id === 'error' ? ['−' + fmt(a.error_limit), '0', '+' + fmt(a.error_limit)] : range ? [fmt(range[0]), fmt(range[1])] : ['—', '—']).map(t => h('span', null, t))); + } + ctx.say.textContent = 'A slice ' + where + (a.stride > 1 ? ', every ' + D.ordinal(a.stride) + ' row and column' : '') + '. Native and predicted share one scale' + (range ? ' from ' + fmt(range[0]) + ' to ' + fmt(range[1]) : '') + '; the error is coloured on ±' + fmt(a.error_limit) + ' about zero' + (a.mean_free ? ', with each field\'s mean removed first' : '') + (c.mask ? '; outside the mask is left blank' : '') + '.'; + } + function drawPanel(ctx, id) { + const p = ctx.parts[id], cv = p.cv, a = ctx.cmp, w = cv.clientWidth; if (!w) return; + const cssH = a.rows === 1 ? 28 : Math.max(32, Math.min(PANEL_MOST, Math.round(w * a.rows / a.columns))); + const size = D.fit(cv, cssH); if (!size) return; const g = cv.getContext('2d'); g.clearRect(0, 0, size[0], size[1]); + const rect = a.rows === 1 ? [0, 0, size[0], size[1]] : D.contain(a.rows, a.columns, size[0], size[1]); + p.rect = rect.map(x => x / (window.devicePixelRatio || 1)); + D.heat(g, ctx.values[id], a.rows, a.columns, ctx.scale[id], rect[0], rect[1], rect[2], rect[3]); + cv.dataset.drawn = String((cv._drawn = (cv._drawn || 0) + 1)); + } + function drawBar(ctx, id) { + const p = ctx.parts[id], size = D.fit(p.bar, 10); if (!size) return; + const a = ctx.cmp, n = 64, vals = new Float32Array(n), m = a.error_limit || 0, range = a.range || [0, 0]; + for (let i = 0; i < n; i++) { const t = (i + 0.5) / n; vals[i] = id === 'error' ? -m + 2 * m * t : range[0] + (range[1] - range[0]) * t; } + D.heat(p.bar.getContext('2d'), vals, 1, n, id === 'error' ? diverging(m) : ctx.scale[id], 0, 0, size[0], size[1]); + } + /** The tip under the pointer: the cell's position in the field and its three values. */ + function hover(ctx, id, ev) { + const p = ctx.parts[id], a = ctx.cmp; if (!p.rect) return; + const box = p.cv.getBoundingClientRect(), x = ev.clientX - box.left - p.rect[0], y = ev.clientY - box.top - p.rect[1]; + const col = Math.floor(x / p.rect[2] * a.columns), row = Math.floor(y / p.rect[3] * a.rows); + if (col < 0 || row < 0 || col >= a.columns || row >= a.rows) { ctx.tip.hidden = true; return; } + const i = row * a.columns + col, at = a.index.slice(), nd = ctx.c.shape.length; + if (nd === 1) at[0] = col * a.stride; else { at[a.axes[0]] = row * a.stride; at[a.axes[1]] = col * a.stride; } + const names = axisNames(nd), v = ctx.values; + const outside = !Number.isFinite(v.reference[i]) && !Number.isFinite(v.predicted[i]) && ctx.c.mask; + ctx.tip.replaceChildren(h('div', 'mono fld-tip-at', (nd === 3 ? at.map((q, k) => names[k] + ' ' + q).join(', ') : '(' + at.join(', ') + ')'))); + if (outside) ctx.tip.appendChild(h('div', null, 'outside the mask')); + else for (const [key, word] of [['reference', 'Native'], ['predicted', 'Predicted'], ['error', a.mean_free ? 'Error, means removed' : 'Error']]) { const r = h('div', 'fld-tip-row'); r.dataset.value = key; r.appendChild(h('span', null, word)); r.appendChild(h('span', 'num', fmt(v[key][i]))); ctx.tip.appendChild(r); } + ctx.tip.dataset.cell = JSON.stringify(at); + ctx.tip.hidden = false; + const wrap = ctx.wrap.getBoundingClientRect(), tw = ctx.tip.offsetWidth, th = ctx.tip.offsetHeight; + let left = ev.clientX - wrap.left + 14, top = ev.clientY - wrap.top + 14; + if (left + tw > wrap.width) left = Math.max(0, ev.clientX - wrap.left - tw - 14); + if (top + th > wrap.height + 40) top = Math.max(0, ev.clientY - wrap.top - th - 14); + ctx.tip.style.left = left + 'px'; ctx.tip.style.top = top + 'px'; + } + + // ---- The figures ---------------------------------------------------------------------------------------------------- + const FIGURES = [['count', 'Cells counted'], ['nonfinite', 'Values not finite'], ['max_abs_error', 'Largest |error|'], ['mean_error', 'Mean error'], ['rmse', 'RMSE'], ['relative_l2', 'Relative L2'], ['mean_free_relative_l2', 'Mean-free relative L2']]; + function figures(ctx) { + const f = ctx.cmp.figures, c = ctx.c, items = []; + const t = h('table', 'tv fld-figures'); t.appendChild(h('caption', 'small muted', 'Over the whole field' + (c.mask ? ', inside the mask' : '') + ', whatever the slice')); + const tb = h('tbody'); + for (const [key, word] of FIGURES) { const tr = h('tr'); tr.dataset.fig = key; tr.dataset.value = f[key] == null ? '' : String(f[key]); tr.appendChild(h('th', null, word)); tr.appendChild(h('td', 'n', key === 'count' || key === 'nonfinite' ? num(f[key]) : fmt(f[key]))); tb.appendChild(tr); } + t.appendChild(tb); items.push(t); + if (c.reported && c.reported.length) { + const r = h('table', 'tv fld-reported'); r.appendChild(h('caption', 'small muted', 'The run reported')); + const rb = h('tbody'); + for (const x of c.reported) { const tr = h('tr'); tr.dataset.value = String(x.value); const th = h('th', 'mono', x.field); th.title = 'In ' + x.file; tr.appendChild(th); tr.appendChild(h('td', 'n', fmt(x.value))); rb.appendChild(tr); } + r.appendChild(rb); items.push(r); + } + ctx.figs.replaceChildren(...items); + } + + // The parts the Compare tab draws its outputs with (compare.js): the cases, the person's place in one, the request's + // arguments, the axes' names and the colour scales. + const parts = { keptCases, viewFor, argsOf, axisNames, fixedAxes, sequential, diverging, PANEL_MOST }; + return { read, section, parts }; +})(); diff --git a/static/project-workspace/generate.js b/static/project-workspace/generate.js new file mode 100644 index 00000000..ae0c8639 --- /dev/null +++ b/static/project-workspace/generate.js @@ -0,0 +1,259 @@ +'use strict'; +// The Pumas endpoint on the page (docs/plans/definition-first-projects/spec-d5.md section 5): the endpoint on the Train +// tab, beside the environments (its address, whether it answers, its version and the models it serves with what each +// takes and makes; for the owner, Set, Clear and Find on this machine in its drawer), and the Data tab's Generate form, +// which keeps a new dataset whose run is the step generate as an unsaved edit, as Import and Freeze do (annotate.js). +// Everything offered comes from the answer of `endpoint` at that moment: what to make is only what some served model +// makes, a model only one that makes it, an option only one its capability declares, with the advertised bounds as the +// field's minimum and maximum. Building the dataset is the existing plan and request. Suggest on the Annotate tab is +// annotate.js's. +// +// Updating in place (spec-w4): the endpoint's answer is not watched; it is read with the Train tab and forgotten, as +// every answer not watched, when a reading finds something changed (or after a control's answer), so the section is +// drawn again in place only then. Its drawer is drawn again in place the same way, its typed address kept in its +// group's slot (actions.js). The Generate form is a drawer form like Import and Freeze: never touched by a reading on +// the timer or on focus, its text kept in its group's slot. +window.LabGenerate = (() => { + const P = window.TuldokProject, S = window.LabScreens, A = window.LabActions, h = window.LabCharts.h; + const { sect, putInPlace, block } = S.parts; + const FORMAT = 'tuldok_definition_v1'; + const MAKE_WORD = { text: 'Texts', image: 'Images (PNG)' }; + const REQUIRED = { image: ['width', 'height'] }; // the options a generate of this kind cannot do without (docs/contracts/project-folder.md) + const button = (cls, text, key, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (key) b.dataset.key = key; if (onClick) b.addEventListener('click', onClick); return b; }; + const num = v => Number.isInteger(v) ? v.toLocaleString('en-GB') : String(v); + const bound = b => num(b.minimum) + ' to ' + num(b.maximum); + const listed = xs => xs.length < 2 ? xs.join('') : xs.slice(0, -1).join(', ') + ' and ' + xs[xs.length - 1]; + const entryKey = m => m.model + '\u0000' + m.profile; + // The options of a capability in the order the contract names them (the server's answer holds them in key order). + const ORDER = ['max_tokens', 'temperature', 'top_p', 'width', 'height']; + const rank = n => { const i = ORDER.indexOf(n); return i < 0 ? ORDER.length : i; }; + const optionsOf = made => Object.entries(made.options).sort(([a], [b]) => rank(a) - rank(b) || (a < b ? -1 : a > b ? 1 : 0)); + const forms = () => window.LabAnnotate.forms; + + // ---- What the endpoint answer says, in words ------------------------------------------------------------------- + /** {tone, word, line}: the endpoint's state in one mark and one sentence. */ + function standing(ep) { + if (ep.error) return { tone: 'serious', word: 'not read', line: ep.error.message }; + if (!ep.address) return { tone: 'warn', word: 'not set', line: 'No Pumas endpoint is set on this machine, so nothing can be generated or suggested here.' }; + if (!ep.answered) return { tone: 'serious', word: 'does not answer', line: ep.problem || 'It did not answer.' }; + return { tone: 'good', word: 'answers', line: (ep.version ? 'Pumas ' + ep.version : 'Version not reported') + ' · ' + ep.models.length + (ep.models.length === 1 ? ' model' : ' models') + ' served' }; + } + /** The (model, profile) entries that can make a dataset now, by what they make. */ + function makers(ep) { + const by = {}; + if (ep && !ep.error && ep.answered) for (const m of ep.models) for (const made of m.makes) (by[made.make] = by[made.make] || []).push({ m, made }); + return by; + } + /** Whether the profile has to be written: the endpoint says so, or it serves the model with more than one. */ + const needsProfile = (ep, m) => m.profile_required || ep.models.filter(x => x.model === m.model).length > 1; + + // ---- Train: the endpoint beside the environments ------------------------------------------------------------------ + /** The section of the Train tab: a row opening the endpoint's drawer, and the models it serves. */ + function section(frag, ep, sel, act) { + const st = standing(ep); + const head = sect('Pumas endpoint', 'a setting of this machine'); head.classList.add('gap'); + head.appendChild(h('span', 'spacer')); + head.appendChild(A.group('train:endpoint', [{ id: 'open', label: ep.address ? 'Change' : 'Set the endpoint', cls: 'btn sm', who: 'owner', onClick: () => act.select(sel === 'endpoint' ? null : 'endpoint') }], { inline: true })); + frag.appendChild(head); + const ul = h('ul', 'rows'), li = h('li'), r = button('row env ep-row', null, 'open-endpoint', () => act.select(sel === 'endpoint' ? null : 'endpoint')); + r.setAttribute('aria-pressed', String(sel === 'endpoint')); + r.appendChild(S.stateEl(st.tone, st.word)); + const t = h('span'); t.appendChild(h('span', 'title', ep.address || 'No endpoint set')); t.appendChild(h('span', 'sub' + (st.tone === 'serious' ? ' warnline' : ''), st.line)); r.appendChild(t); + r.appendChild(h('span', 'sub', ep.address ? 'generation and suggestions go here' : A.may('owner') ? 'set it to generate and suggest' : 'the owner sets it')); + li.appendChild(r); ul.appendChild(li); frag.appendChild(ul); + if (ep.error || !ep.answered) return; + if (!ep.models.length) { frag.appendChild(h('p', 'empty', 'The endpoint serves no model.')); return; } + const ml = h('ul', 'rows ep-models'); ml.setAttribute('aria-label', 'The models the endpoint serves'); + for (const m of ep.models) ml.appendChild(modelRow(m)); + frag.appendChild(ml); + } + /** One served (model, profile): what it takes and makes, what a dataset can be made of with it, its capabilities. */ + function modelRow(m) { + const li = h('li', 'ep-model'); li.dataset.model = m.model; li.dataset.profile = m.profile; + const who = h('span', 'ep-who'); who.appendChild(h('span', 'title mono', m.model)); who.appendChild(h('span', 'sub mono', m.profile + (m.profile_required ? ' · profile required' : ''))); li.appendChild(who); + const what = h('span', 'ep-what'); + what.appendChild(h('span', 'title', 'Takes ' + (listed(m.modalities.input) || 'nothing') + ' · makes ' + (listed(m.modalities.output) || 'nothing'))); + const makes = h('span', 'sub ep-makes'); + if (m.makes.length) { + makes.appendChild(h('b', null, 'A dataset of: ')); + makes.appendChild(document.createTextNode(m.makes.map(x => (x.make === 'image' ? 'images' : 'texts') + ' by ' + x.capability + (Object.keys(x.options).length ? ' (' + optionsOf(x).map(([k, b]) => k + ' ' + bound(b)).join(', ') + ')' : '') + (x.seed ? ', with a seed' : '')).join('; '))); + } else makes.textContent = 'No dataset can be made with it.'; + what.appendChild(makes); + const on = m.capabilities.filter(c => c.available), off = m.capabilities.filter(c => !c.available); + if (on.length) what.appendChild(h('span', 'sub ep-caps', 'Available: ' + on.map(c => c.capability + ' (' + c.input_formats.join(', ') + ' → ' + c.output_formats.join(', ') + ')').join(', '))); + if (off.length) what.appendChild(h('span', 'sub ep-off', 'Unavailable: ' + off.map(c => c.capability + (c.reason ? ' (' + c.reason + ')' : '')).join(', '))); + li.appendChild(what); + li.appendChild(h('span', 'sub ep-load', [m.provider, m.load_state, m.context_tokens ? m.context_tokens.toLocaleString('en-GB') + ' tokens' : null].filter(Boolean).join(' · '))); + return li; + } + + // ---- Train: the endpoint's drawer ----------------------------------------------------------------------------------- + let found = null; // the last answer of find_endpoints, shown until the page is left + /** The drawer: the endpoint's state; for the owner Set (an address), Clear and Find on this machine; Ask it again. */ + function drawer(aside, act) { + S.drawer(aside, 'endpoint', () => P.read('endpoint'), (ins, ep) => { + const st = standing(ep); + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'Pumas endpoint · a setting of this machine')); + const t = h('h2', null, ep.address || 'No endpoint set'); t.tabIndex = -1; hd.appendChild(t); + const meta = h('div', 'ins-meta'); meta.appendChild(S.stateEl(st.tone, st.word)); hd.appendChild(meta); ins.appendChild(hd); + const dl = h('dl', 'setup'); const put = (k, v) => { dl.appendChild(h('dt', null, k)); dl.appendChild(h('dd', null, v)); }; + put('Address', ep.address || 'none'); put('Answers', ep.address ? (ep.answered ? 'yes' : 'no') : '—'); + if (ep.answered) { put('Version', ep.version || 'not reported'); put('Models', String(ep.models.length)); } + if (ep.problem && ep.address) put('Problem', ep.problem); + ins.appendChild(dl); + ins.appendChild(h('p', 'note', 'The address is kept in Tuldok\'s settings file on this machine, never in the project folder. Generation and suggestions go to it over HTTP; nothing of Pumas runs inside Tuldok.')); + ins.appendChild(A.group('endpoint:again', [{ id: 'again', label: 'Ask it again', cls: 'btn sm', onClick: () => { P.forget(['endpoint']); act.refresh(); } }])); + if (window.LabCuration) window.LabCuration.embedderPart(ins, ep); // the machine's embedder (spec-t5 section 2) + if (!A.may('owner')) { ins.appendChild(h('p', 'note', 'Only the owner of this Tuldok sets the endpoint.')); return; } + const set = h('div', 'ep-part'); set.appendChild(h('h3', null, ep.address ? 'Set another address' : 'Set the address')); + set.appendChild(h('p', 'note', 'A numeric loopback HTTP address, such as http://127.0.0.1:8765, without a path. Tuldok asks it once what it serves and keeps it only when it answers.')); + const f = h('div', 'form'); + f.appendChild(A.group('endpoint:set', [{ id: 'set', label: 'Set', cls: 'btn', who: 'owner', op: 'set_endpoint', busy: 'Asking it…', + input: { name: 'address', label: 'Address', placeholder: 'http://127.0.0.1:8765', max: 2048, always: true }, + args: s => ({ address: (s.text.address || '').trim() }) }])); + set.appendChild(f); ins.appendChild(set); + const find = h('div', 'ep-part'); find.appendChild(h('h3', null, 'Find on this machine')); + find.appendChild(h('p', 'note', 'Lists the Pumas endpoints that advertise themselves on this machine\'s loopback listeners, to pick from. Nothing is set until you pick one.')); + const list = h('div', 'ep-found'); + find.appendChild(A.group('endpoint:find', [{ id: 'find', label: 'Find on this machine', cls: 'btn', who: 'owner', op: 'find_endpoints', busy: 'Looking…', local: true, + done: r => { found = r; putInPlace(list, foundList()); } }])); + list.appendChild(foundList()); find.appendChild(list); ins.appendChild(find); + if (ep.address) ins.appendChild(A.group('endpoint:clear', [{ id: 'clear', label: 'Clear the endpoint', cls: 'btn ghost', who: 'owner', op: 'set_endpoint', args: () => ({ address: null }), busy: 'Clearing…', + confirm: 'Clear the endpoint of this machine: nothing can be generated or suggested here until one is set again. Nothing else changes.' }])); + }); + } + /** What Find on this machine found, each with Use this. */ + function foundList() { + const box = h('div'); + if (!found) return box; + if (!found.endpoints.length) box.appendChild(h('p', 'empty', 'No Pumas endpoint advertises itself on this machine.')); + else { + const ul = h('ul', 'rows'); + for (const e of found.endpoints) { + const li = h('li', 'with-act'); const t = h('span', 'row ep-hit'); t.style.gridTemplateColumns = 'minmax(0, 1fr)'; + const s = h('span'); s.appendChild(h('span', 'title mono', e.address)); s.appendChild(h('span', 'sub', [e.version ? 'Pumas ' + e.version : 'version not reported', e.component].filter(Boolean).join(' · '))); t.appendChild(s); li.appendChild(t); + li.appendChild(A.group('endpoint:use:' + e.address, [{ id: 'use', label: 'Use this', cls: 'btn sm', who: 'owner', op: 'set_endpoint', args: () => ({ address: e.address }), busy: 'Asking it…' }], { inline: true })); + ul.appendChild(li); + } + box.appendChild(ul); + } + for (const r of found.rejected || []) box.appendChild(h('p', 'small warnline', r.server_url + ' answered, but its advertisement was refused: ' + r.reason)); + return box; + } + + // ---- Data: the Generate form ------------------------------------------------------------------------------------------ + /** Generate: a new datasets/ whose run is the step generate, kept as an unsaved edit. */ + function form(aside, act) { + const F = forms(); + F.formDrawer(aside, 'data:new:generate', () => Promise.all([P.read('endpoint').catch(error => ({ error })), P.read('definitions', { kind: 'dataset' })]), (ins, [ep, defs]) => { + const key = 'newgenerate', s = A.slot(key), by = makers(ep), kinds = Object.keys(MAKE_WORD).filter(k => by[k]); + F.header(ins, 'new dataset · generate', 'Generate a dataset', + 'A dataset made by asking a model the Pumas endpoint serves, one request per item. It is built as any build is, on a machine with an endpoint set; a model need not make the same thing twice.'); + if (!kinds.length) { + const why = h('div', 'gen-none'); why.setAttribute('role', 'status'); + why.appendChild(h('p', 'failure', ep.error ? 'What the endpoint serves could not be read: ' + ep.error.message + : !ep.address ? 'No Pumas endpoint is set on this machine, so there is nothing to generate with.' + : !ep.answered ? 'The Pumas endpoint at ' + ep.address + ' does not answer: ' + (ep.problem || 'no reason was given.') + : 'The Pumas endpoint at ' + ep.address + ' serves ' + ep.models.length + (ep.models.length === 1 ? ' model' : ' models') + ', and none can make a text or an image now.')); + why.appendChild(h('p', 'note', 'Generate offers only what a served model can make at the moment the form is opened.')); + const acts = h('div', 'actions'); acts.appendChild(button('btn', 'The endpoint on the Train tab', 'gen:endpoint', () => act.go('train', 'endpoint'))); acts.appendChild(button('btn ghost', 'Cancel', 'gen:cancel', () => act.select(null))); + why.appendChild(acts); ins.appendChild(why); return; + } + const sets = defs.definitions.filter(d => !d.removed); + const box = h('div', 'form gen-form'); + const fields = () => { + const f = document.createDocumentFragment(); + let nameBox = null; + f.appendChild(F.input(s, key, 'title', 'Title (required)', { max: 200, onInput: () => { if (!s.text.nameTyped && nameBox) { nameBox.value = F.slug(s.text.title || ''); s.text.name = nameBox.value; } } })); + const nl = F.input(s, key, 'name', 'Name', { max: 120, hint: 'the file is datasets/.json', placeholder: 'made from the title', onInput: v => { s.text.nameTyped = !!v; } }); + nameBox = nl.querySelector('input'); if (!s.text.nameTyped) { nameBox.value = F.slug(s.text.title || ''); s.text.name = nameBox.value; } + f.appendChild(nl); + f.appendChild(F.choice(s, key, 'make', 'What to make', kinds.map(k => [k, MAKE_WORD[k]]), kinds[0], redraw)); + const offer = by[s.text.make]; + f.appendChild(F.choice(s, key, 'model', 'Model, of those that make ' + (s.text.make === 'image' ? 'images' : 'texts') + ' now', offer.map(({ m, made }) => [entryKey(m), m.model + ' · ' + m.profile + ' (' + made.capability + ')']), entryKey(offer[0].m), redraw)); + const { m, made } = offer.find(o => entryKey(o.m) === s.text.model); + if (needsProfile(ep, m)) f.appendChild(h('p', 'small faint gen-line', 'The profile ' + m.profile + ' is written with the model: ' + (m.profile_required ? 'the endpoint requires it.' : 'the endpoint serves ' + m.model + ' with more than one.'))); + const prompts = s.text.from === 'prompts'; + f.appendChild(F.input(s, key, 'prompt', 'Prompt (required)', { area: 3, max: 16000, placeholder: s.text.make === 'image' ? 'A picture of {text}, number {n}.' : 'Write line {n} of a short poem.' })); + f.appendChild(h('p', 'small faint gen-line', '{n} is replaced by the item\'s number from 1' + (prompts ? '; {text} by the text of the prompting record' : '; with a dataset of prompts, {text} by the text of the prompting record') + '. {{ and }} are written as braces.')); + const how = h('div', 'ann-field'); how.appendChild(h('h3', null, 'How many')); + const rd = h('div', 'radio'); rd.setAttribute('role', 'radiogroup'); rd.setAttribute('aria-label', 'How many'); + for (const [v, label] of [['count', 'A count'], ['prompts', 'One for each text record of a dataset of prompts']]) { + const l = h('label'), i = h('input'); i.type = 'radio'; i.name = 'gen-from'; i.value = v; i.checked = (s.text.from || 'count') === v; i.dataset.key = key + ':field:from-' + v; + i.addEventListener('change', () => { if (i.checked) { s.text.from = v; redraw(); } }); l.appendChild(i); l.appendChild(h('span', null, label)); rd.appendChild(l); + } + how.appendChild(rd); + if (!prompts) how.appendChild(numberField(s, key, 'count', 'Count (required)', { minimum: 1, maximum: 100000 }, { value: '10', step: 1 })); + else { + const two2 = h('div', 'two'); + if (!sets.length) two2.appendChild(h('p', 'failure', 'The project has no dataset to take prompts from.')); + else two2.appendChild(F.choice(s, key, 'prompts', 'The dataset of prompts', sets.map(d => [d.reference, d.title + ' · ' + d.reference]), sets[0].reference)); + two2.appendChild(numberField(s, key, 'per', 'For each record', { minimum: 1, maximum: 1000 }, { value: '1', step: 1, hint: 'how many for each of its text records' })); + how.appendChild(two2); + } + f.appendChild(how); + // The seed and the options, as the chosen capability declares them. + const opts = h('div', 'ann-field gen-opts'); opts.appendChild(h('h3', null, 'Options of ' + made.capability)); + const names = optionsOf(made).map(([n]) => n), need = REQUIRED[s.text.make] || []; + if (!names.length && !made.seed) opts.appendChild(h('p', 'note', made.capability + ' declares no option Tuldok can set; the endpoint\'s defaults apply.')); + const grid = h('div', 'gen-grid'); + if (made.seed) grid.appendChild(numberField(s, key, 'seed', 'Seed', { minimum: 0, maximum: 4294967295 }, { step: 1 })); + for (const n of names) grid.appendChild(numberField(s, key, 'opt:' + n, n + (need.includes(n) ? ' (required)' : ''), made.options[n], { step: 'any' })); + if (grid.children.length) opts.appendChild(grid); + const said = []; + if (made.seed) said.push('Item n is asked with the seed + n − 1; with no seed the endpoint chooses one.'); + if (names.length) said.push('An option left empty is not sent' + (need.length ? ', except ' + listed(need) + ', which an image needs' : '') + '. The bounds are what the endpoint advertises now' + (s.text.make === 'image' ? '; Tuldok asks for at most 2,048 pixels a side.' : '.')); + if (said.length) opts.appendChild(h('p', 'small faint gen-line', said.join(' '))); + f.appendChild(opts); + f.appendChild(F.input(s, key, 'rights', 'Rights (required)', { area: 2, max: 1000, hint: 'what may be done with what is made', placeholder: 'Made by a model; anything may be done with it.' })); + return f; + }; + const redraw = () => putInPlace(box, fields()); + box.appendChild(fields()); + ins.appendChild(box); + ins.appendChild(A.group(key, [ + { id: 'keep', label: 'Keep', cls: 'btn primary', who: 'edit', op: 'set_definition', busy: 'Keeping…', + args: st => { const o = by[st.text.make].find(x => entryKey(x.m) === st.text.model); return { reference: 'datasets/' + (st.text.name || '').trim(), text: F.textOf(generateDoc(st.text, o.m, o.made, needsProfile(ep, o.m))) }; }, + done: r => { act.select(r.reference); } }, + { id: 'back', label: 'Cancel', cls: 'btn ghost', onClick: () => act.select(null) } + ])); + ins.appendChild(h('p', 'note', 'Keep checks the definition and keeps it in Tuldok as an unsaved edit; Save writes it. Building it is planned and requested as any build is; an agent\'s request waits for the owner.')); + }); + } + /** A number field whose value lives in the slot; the bounds are its minimum and maximum. */ + function numberField(s, key, name, label, b, { value, step, hint } = {}) { + const l = forms().input(s, key, name, label, { type: 'number', value, hint: hint || bound(b) }); + const i = l.querySelector('input'); i.min = String(b.minimum); i.max = String(b.maximum); i.step = String(step || 'any'); i.inputMode = step === 1 ? 'numeric' : 'decimal'; + return l; + } + const refuse = text => { throw Object.assign(Error(text), { code: 'invalid' }); }; + /** A whole number or a number from the field's text, within its bounds; null when empty and not required. */ + function numberOf(text, label, b, whole, required) { + const raw = String(text == null ? '' : text).trim(); + if (!raw) { if (required) refuse(label + ' is required.'); return null; } + const v = Number(raw); + if (!Number.isFinite(v) || (whole && !Number.isInteger(v))) refuse(label + ' is ' + (whole ? 'a whole number' : 'a number') + '; "' + raw + '" is not.'); + if (v < b.minimum || v > b.maximum) refuse(label + ' ' + raw + ' is outside ' + bound(b) + '.'); + return v; + } + /** The definition a generate writes (docs/contracts/project-folder.md, Steps Tuldok provides). */ + function generateDoc(x, m, made, profile) { + const settings = { make: x.make, model: m.model }; + if (profile) settings.profile = m.profile; + settings.prompt = (x.prompt || '').trim(); + const prompts = x.from === 'prompts'; + if (prompts) { const per = numberOf(x.per, 'For each record', { minimum: 1, maximum: 1000 }, true, false); if (per !== null && per !== 1) settings.per = per; } + else settings.count = numberOf(x.count, 'The count', { minimum: 1, maximum: 100000 }, true, true); + if (made.seed) { const seed = numberOf(x.seed, 'The seed', { minimum: 0, maximum: 4294967295 }, true, false); if (seed !== null) settings.seed = seed; } + const options = {}, need = REQUIRED[x.make] || []; + for (const [n, b] of optionsOf(made)) { const v = numberOf(x['opt:' + n], n, b, ['max_tokens', 'width', 'height'].includes(n), need.includes(n)); if (v !== null) options[n] = v; } + if (Object.keys(options).length) settings.options = options; + settings.rights = (x.rights || '').trim(); + const run = { step: 'generate' }; + if (prompts) run.inputs = { prompts: x.prompts }; + run.settings = settings; + return { format: FORMAT, kind: 'dataset', title: (x.title || '').trim(), run }; + } + + return { section, drawer, form }; +})(); diff --git a/static/project-workspace/inside.js b/static/project-workspace/inside.js new file mode 100644 index 00000000..d2e47ec1 --- /dev/null +++ b/static/project-workspace/inside.js @@ -0,0 +1,724 @@ +'use strict'; +// Looking inside a product (docs/plans/definition-first-projects/spec-d1.md section 7): the Data tab's +// second state, one dataset opened (#data/inside:), and its viewers. The records are paged 48 at +// a time as tiles beside the selected record's viewer (below it under 1,100 px); a "Files" switch browses +// the product as a tree (directories, archives and the arrays of an .npy or .npz) from which a file or an +// array opens in the same viewers. Everything is read through window.TuldokProject's five operations +// (records, record, inside, array_slice, array_summary) and the file route for an image's or a sound's +// bytes; nothing is written. Drawing is Canvas 2D with the tokens of base.css, drawn again when the theme +// changes or a canvas's box changes size, and on requestAnimationFrame only while something changes. +// +// Updating in place (spec-w4): the view is built off the page and put in place in one step. A reading that +// changed nothing never reaches here (app.js draws nothing); one that changed something forgets these +// answers, and open() is called with again set: the view is read and drawn again in place, keeping the +// page, the filters, the selection, the viewer's settings and the scroll. A person's selection changes the +// tiles' pressed state and the viewer only; a page, a filter or the Files switch draws the view again. +window.LabInside = (() => { + const P = window.TuldokProject, S = window.LabScreens, C = window.LabCharts, h = C.h; + const { sect, putInPlace, bytes, time, failure } = S.parts; + const PER_PAGE = 48; // tiles on a page (spec-d1 section 7) + const TILE_MOST = 48; // rows and columns of an array tile's slice + const VIEW_MOST = 256; // rows and columns of the viewer's slice + const STACK_MOST = 24, STACK_SLICES = 10; // the stacked-slice drawing: slices read, and their size + const MESH_AS_POINTS = 60000; // above this many triangles a mesh is drawn as its vertices + const TREE_LIMIT = 200; // entries of the Files tree read at a time + const KINDS = ['text', 'image', 'audio', 'pointcloud', 'mesh', 'array', 'file']; + const WORD = { text: 'Text', image: 'Image', audio: 'Sound', pointcloud: 'Point cloud', mesh: 'Mesh', array: 'Array', file: 'File' }; + const count = (n, one, many) => n.toLocaleString('en-GB') + ' ' + (n === 1 ? one : many || one + 's'); + const button = (cls, text, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (onClick) b.addEventListener('click', onClick); return b; }; + const fmt = v => v == null ? '—' : C.fmtSci(v); + + // ---- Decoding ------------------------------------------------------------------------------------------- + function raw(b64) { const s = atob(b64 || ''), u = new Uint8Array(s.length); for (let i = 0; i < s.length; i++) u[i] = s.charCodeAt(i); return u; } + /** Little-endian 32-bit floats from base64, whatever the machine's byte order. */ + function floats(b64) { const u = raw(b64), d = new DataView(u.buffer), out = new Float32Array(u.length >> 2); for (let i = 0; i < out.length; i++) out[i] = d.getFloat32(i * 4, true); return out; } + function uints(b64) { const u = raw(b64), d = new DataView(u.buffer), out = new Uint32Array(u.length >> 2); for (let i = 0; i < out.length; i++) out[i] = d.getUint32(i * 4, true); return out; } + + // ---- Colours, from the tokens of base.css ----------------------------------------------------------------- + function token(name) { + const v = getComputedStyle(document.documentElement).getPropertyValue(name).trim(); + const m = /^#([0-9a-f]{6})$/i.exec(v); if (m) { const n = parseInt(m[1], 16); return [n >> 16, (n >> 8) & 255, n & 255]; } + return [128, 128, 128]; + } + const mix = (a, b, t) => [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t]; + /** value -> [r, g, b]: diverging about zero when the range has both signs, else sequential from low to high. */ + function ramp(min, max) { + const lo = token('--ramp-lo'), hi = token('--ramp-hi'), neg = token('--ramp-neg'), mid = token('--ramp-mid'), pos = token('--ramp-pos'); + if (min < 0 && max > 0) { const m = Math.max(-min, max); return v => v < 0 ? mix(mid, neg, Math.min(1, -v / m)) : mix(mid, pos, Math.min(1, v / m)); } + const span = max - min || 1; return v => mix(lo, hi, Math.min(1, Math.max(0, (v - min) / span))); + } + + // ---- Canvases: sized to their box, drawn again on a new size or theme ------------------------------------- + const painters = new Map(); // canvas -> draw() + const sizes = new ResizeObserver(entries => { for (const e of entries) { const draw = painters.get(e.target); if (draw && e.target.isConnected) { e.target._shown = true; draw(); } } }); + /** Registers draw(canvas) for a canvas: it is called when the canvas's box gets a size, changes size, or the theme changes. */ + function paints(canvas, draw) { prune(); painters.set(canvas, draw); sizes.observe(canvas); } + /** Forgets the canvases that were on the page and are no more (a viewer replaced, the view left), and what they drew from. */ + function prune() { for (const canvas of painters.keys()) if (canvas._shown && !canvas.isConnected) { painters.delete(canvas); sizes.unobserve(canvas); } } + function retheme() { prune(); for (const [canvas, draw] of painters) if (canvas.isConnected) draw(); } + new MutationObserver(retheme).observe(document.documentElement, { attributes: true, attributeFilter: ['data-theme'] }); + matchMedia('(prefers-color-scheme: dark)').addEventListener('change', retheme); + /** Sets a canvas's pixel size for its CSS width and a height (CSS pixels); returns [width, height] in device pixels or null when it has no width yet. */ + function fit(canvas, cssHeight) { + const w = canvas.clientWidth; if (!w) return null; + const dpr = window.devicePixelRatio || 1, W = Math.max(1, Math.round(w * dpr)), H = Math.max(1, Math.round(cssHeight * dpr)); + if (canvas.style.height !== cssHeight + 'px') canvas.style.height = cssHeight + 'px'; + if (canvas.width !== W) canvas.width = W; if (canvas.height !== H) canvas.height = H; + return [W, H]; + } + /** A heat map of rows x columns values into ctx's rectangle (x, y, w, h), unsmoothed; non-finite values left clear. */ + function heat(ctx, values, rows, cols, colour, x, y, w, hgt) { + const off = document.createElement('canvas'); off.width = cols; off.height = rows; + const o = off.getContext('2d'), img = o.createImageData(cols, rows); + for (let i = 0; i < rows * cols; i++) { const v = values[i]; if (!Number.isFinite(v)) continue; const c = colour(v); img.data[i * 4] = c[0]; img.data[i * 4 + 1] = c[1]; img.data[i * 4 + 2] = c[2]; img.data[i * 4 + 3] = 255; } + o.putImageData(img, 0, 0); ctx.imageSmoothingEnabled = false; ctx.drawImage(off, x, y, w, hgt); + } + /** The rectangle a rows x columns grid takes in W x H, keeping square cells. */ + function contain(rows, cols, W, H) { const s = Math.min(W / cols, H / rows), w = cols * s, hh = rows * s; return [(W - w) / 2, (H - hh) / 2, w, hh]; } + + // ---- Glyphs --------------------------------------------------------------------------------------------- + const GLYPHS = { + text: 'M5 6h14M5 10h14M5 14h10M5 18h12', + image: 'M4 5h16v14H4zM4 16l5-5 4 4 3-3 4 4M15 9.5a1 1 0 1 0 0 .1', + audio: 'M3 12h2M7 8v8M11 4v16M15 7v10M19 10v4', + pointcloud: 'M6 7h.01M10 5h.01M15 6h.01M18 10h.01M8 11h.01M13 10h.01M5 15h.01M11 15h.01M16 14h.01M9 19h.01M14 18h.01M19 17h.01', + mesh: 'M12 3 3 19h18zM12 3l-3 9 12 7M9 12 3 19', + array: 'M4 4h16v16H4zM4 9.3h16M4 14.6h16M9.3 4v16M14.6 4v16', + file: 'M6 3h8l4 4v14H6zM14 3v4h4', + directory: 'M3 6h6l2 2h10v11H3z', + archive: 'M5 4h14v16H5zM11 4v2h2v2h-2v2h2v2h-2v2', + refused: 'M12 3 2 21h20zM12 10v5M12 18h.01' + }; + function glyph(name) { + const ns = 'http://www.w3.org/2000/svg', s = document.createElementNS(ns, 'svg'), p = document.createElementNS(ns, 'path'); + s.setAttribute('viewBox', '0 0 24 24'); s.setAttribute('aria-hidden', 'true'); s.setAttribute('class', 'glyph ' + name); + p.setAttribute('d', GLYPHS[name] || GLYPHS.file); p.setAttribute('fill', 'none'); p.setAttribute('stroke', 'currentColor'); p.setAttribute('stroke-width', name === 'pointcloud' ? '2.6' : '1.6'); p.setAttribute('stroke-linecap', 'round'); p.setAttribute('stroke-linejoin', 'round'); + s.appendChild(p); return s; + } + + // ---- What is kept per dataset, across drawings --------------------------------------------------------------- + const views = new Map(); // reference -> the person's place in it + const viewOf = ref => { if (!views.has(ref)) views.set(ref, { run: null, page: 0, kind: null, split: null, sel: null, mode: 'records', path: '', treeOffset: 0, treeTarget: null, viewers: new Map() }); return views.get(ref); }; + const settingsOf = (v, key, make) => { if (!v.viewers.has(key)) v.viewers.set(key, make()); return v.viewers.get(key); }; + let current = null; // { ref, main, act, root, io, vturn } + const R = P.product; // the five reads and the file route (adapter.js) + + /** + * Shows the dataset ref opened in main. What is on screen for it stays unless again (a reading changed + * something): then it is read and drawn again in place. "Reading…" only when it is not on screen yet. + */ + function open(main, ref, act, again) { + const shown = !!(current && current.ref === ref && current.root && main.contains(current.root)); + if (shown && !again) { current.act = act; return; } + if (!current || current.ref !== ref) { leave(); current = { ref, main, act, root: null, io: null, vturn: 0, focusSel: false }; } + current.act = act; + draw(current, !shown); + } + /** The dataset view is no longer shown: its tile observer stops. */ + function leave() { if (current && current.io) current.io.disconnect(); current = null; } + /** Whether the page still shows this dataset opened (another tab may have been drawn meanwhile). */ + const here = c => current === c && decodeURIComponent(location.hash.slice(1)) === 'data/inside:' + c.ref; + + /** Reads what the view shows: the dataset, its products, a page of records or a level of the tree, and the viewer's subject. */ + async function load(c, v) { + const [defs, prods] = await Promise.all([P.read('definitions', { kind: 'dataset' }), P.read('products', { reference: c.ref })]); + const def = defs.definitions.find(d => d.reference === c.ref) || { reference: c.ref, title: c.ref }; + const products = prods.products.filter(p => p.run_id); + if (!products.length) return { def, products }; + const product = products.find(p => p.run_id === v.run) || products[0]; v.run = product.run_id; + // The dataset's embeddings (curation.js, spec-t5): with any that can be read, Map joins Records and Files. + const em = window.LabCuration ? await window.LabCuration.sets(c.ref) : null; + const mapped = !!em && window.LabCuration.usable(em).length > 0; + if (v.mode === 'map' && !mapped) v.mode = 'records'; + const args = { offset: v.page * PER_PAGE, limit: PER_PAGE, kind: v.kind, split: v.split }; + let page = await R.records(v.run, args); + if (!page.records.length && v.page > 0 && page.total) { v.page = Math.floor((page.total - 1) / PER_PAGE); args.offset = v.page * PER_PAGE; page = await R.records(v.run, args); } + let tree = null; + if (v.mode === 'files') tree = await R.inside(v.run, { path: v.path, offset: v.treeOffset, limit: TREE_LIMIT }).catch(error => ({ error })); + if (v.mode === 'records' && !page.records.some(r => r.id === v.sel)) v.sel = page.records.length ? page.records[v.selAt === -1 ? page.records.length - 1 : 0].id : null; + v.selAt = 0; + const target = v.mode === 'files' ? v.treeTarget : v.mode === 'map' ? v.mapTarget || null : v.sel ? { record: v.sel } : null; + const viewer = target ? await viewerFor(c, v, target) : null; + return { def, products, product, page, tree, viewer, em, mapped }; + } + + /** Draws the whole view (first: not on screen yet), built off the page and put in place in one step. */ + async function draw(c, first) { + const main = c.main, mine = main._paint = (main._paint || 0) + 1, v = viewOf(c.ref); + if (first) { main.textContent = ''; main.appendChild(h('p', 'note', 'Reading…')); main.scrollTop = 0; main.scrollLeft = 0; } + main._painted = 'data/inside:' + c.ref; // so that the cards, drawn again later, start from "Reading…" (tabs.js's paint) + main.setAttribute('aria-busy', 'true'); + let got; try { got = await load(c, v); } catch (error) { got = { error }; } + if (main._paint === mine) main.removeAttribute('aria-busy'); + if (main._paint !== mine || !here(c)) return; + const root = h('div', 'inside'); + try { build(c, v, got, root); } catch (error) { root.textContent = ''; failure(root, error); console.error(error); } + if (c.io) c.io.disconnect(); + putInPlace(main, root); + c.root = root; + if (c.focusSel) { c.focusSel = false; const t = root.querySelector('.rec[aria-pressed="true"]'); if (t) t.focus(); } + watchTiles(c, root); + } + + function build(c, v, got, root) { + const back = button('btn sm ghost ds-back', '← All datasets', () => c.act.go('data')); back.dataset.key = 'inside-back'; root.appendChild(back); + if (got.error) { root.appendChild(window.LabActions.refusal(got.error)); return; } + const { def, products, product, page, tree } = got; + const head = h('div', 'ds-head'); head.appendChild(h('h2', null, def.title)); + const meta = h('div', 'meta'); + if (product) { + meta.appendChild(h('span', 'mono', 'run ' + product.run_id.slice(0, 8))); + meta.appendChild(h('span', null, (product.adopted ? 'adopted ' : 'built ') + time(product.finished_at))); + meta.appendChild(h('span', null, bytes(product.bytes))); + if (page) meta.appendChild(h('span', null, count(Object.values(page.kinds).reduce((a, b) => a + b, 0), 'record') + ' · ' + ({ 'records.jsonl': 'from records.jsonl', view: 'from the dataset\'s view', files: 'one per file' }[page.source] || page.source))); + } + head.appendChild(meta); + if (products.length > 1) { + const s = h('select'); s.setAttribute('aria-label', 'Product'); s.dataset.key = 'inside-product'; + for (const p of products) { const o = h('option', null, (p.subject ? p.subject + ' · ' : '') + 'run ' + p.run_id.slice(0, 8) + ' · ' + time(p.finished_at)); o.value = p.run_id; o.selected = p.run_id === v.run; s.appendChild(o); } + s.addEventListener('change', () => { Object.assign(v, { run: s.value, page: 0, sel: null, path: '', treeOffset: 0, treeTarget: null }); draw(c, false); }); + head.appendChild(s); + } + root.appendChild(head); + if (!product) { root.appendChild(h('p', 'empty', 'This dataset has no product on this machine: there is nothing to look inside.')); return; } + if (page.problems && page.problems.length) { + const p = h('p', 'small warnline inside-problems', page.problems.slice(0, 3).map(x => (x.line != null ? 'Line ' + x.line + ': ' : '') + x.message).join(' ') + (page.problems.length > 3 ? ' (' + page.problems.length + ' problems in all)' : '')); + root.appendChild(p); + } + // The Files switch, the kinds and the splits as filters. + const bar = h('div', 'filters'); + const seg = h('div', 'seg'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'Show'); + for (const [mode, label] of [['records', 'Records'], ['files', 'Files']].concat(got.mapped ? [['map', 'Map']] : [])) { const b = button(null, label, () => { if (v.mode !== mode) { v.mode = mode; draw(c, false); } }); b.setAttribute('aria-pressed', String(v.mode === mode)); b.dataset.key = 'inside-mode-' + mode; seg.appendChild(b); } + bar.appendChild(seg); + if (v.mode === 'records') { + const all = Object.values(page.kinds).reduce((a, b) => a + b, 0); + bar.appendChild(chips('Kind', [[null, 'All', all]].concat(KINDS.filter(k => page.kinds[k]).map(k => [k, WORD[k], page.kinds[k]])), v.kind, k => { v.kind = k; v.page = 0; v.sel = null; draw(c, false); }, 'kind')); + const splits = Object.entries(page.splits || {}); + if (splits.length) bar.appendChild(chips('Split', [[null, 'All', null]].concat(splits.map(([s, n]) => [s, s, n])), v.split, s => { v.split = s; v.page = 0; v.sel = null; draw(c, false); }, 'split')); + } + root.appendChild(bar); + const body = h('div', 'ds-body' + (v.mode === 'map' ? ' cur-body' : '')); + const left = h('div', 'inside-list'); + if (v.mode === 'records') records(c, v, page, left); + else if (v.mode === 'map') left.appendChild(window.LabCuration.panel({ ref: c.ref, em: got.em, act: c.act, live: () => here(c), + show: (id, run) => { v.mapTarget = { record: id, run }; showIn(c, v, v.mapTarget); } })); + else files(c, v, tree, left); + body.appendChild(left); + const panel = h('section', 'viewer'); panel.setAttribute('aria-label', 'Viewer'); + panel.appendChild(got.viewer || h('p', 'empty', v.mode === 'files' ? 'Choose a file or an array in the tree.' : v.mode === 'map' ? 'Rest on a point, or open one, to see its record.' : 'Choose a record.')); + body.appendChild(panel); root.appendChild(body); + } + function chips(label, items, value, pick, key) { + const g = h('div', 'grp'); g.setAttribute('role', 'group'); g.setAttribute('aria-label', label); + for (const [id, text, n] of items) { const b = button('btn sm', text + (n != null ? ' · ' + n.toLocaleString('en-GB') : ''), () => { if (value !== id) pick(id); }); b.setAttribute('aria-pressed', String(value === id)); b.dataset.key = 'inside-' + key + '-' + (id || 'all'); g.appendChild(b); } + return g; + } + + // ---- The records: a page of tiles ------------------------------------------------------------------------ + function records(c, v, page, left) { + const first = v.page * PER_PAGE + 1, last = v.page * PER_PAGE + page.records.length; + left.appendChild(sect('Records', page.total ? first.toLocaleString('en-GB') + '–' + last.toLocaleString('en-GB') + ' of ' + page.total.toLocaleString('en-GB') : '0')); + if (!page.total) { left.appendChild(h('p', 'empty', v.kind || v.split ? 'No record matches the filters.' : 'This product holds no records.')); return; } + const grid = h('div', 'rec-grid'); grid.setAttribute('role', 'group'); grid.setAttribute('aria-label', 'Records ' + first + ' to ' + last + ' of ' + page.total + '; the left and right arrows move the selection'); + for (const r of page.records) grid.appendChild(tile(c, v, r)); + left.appendChild(grid); + const pages = Math.ceil(page.total / PER_PAGE); + if (pages > 1) { + const pg = h('div', 'pager'); + const pb = button('btn sm', 'Previous', () => turnPage(c, v, v.page - 1, 0)); pb.disabled = v.page === 0; pb.dataset.key = 'inside-prev'; + const nb = button('btn sm', 'Next', () => turnPage(c, v, v.page + 1, 0)); nb.disabled = v.page + 1 >= pages; nb.dataset.key = 'inside-next'; + pg.appendChild(pb); pg.appendChild(nb); pg.appendChild(h('span', null, 'page ' + (v.page + 1) + ' of ' + pages)); left.appendChild(pg); + } + left._records = page.records; left._pages = pages; + } + function turnPage(c, v, n, at) { v.page = n; v.sel = null; v.selAt = at; draw(c, false); } + /** One record as a tile: its id always; text's first characters; an image or a small array slice once the tile is seen; else the kind's glyph. */ + function tile(c, v, r) { + const b = h('button', 'rec ' + r.kind); b.type = 'button'; b.dataset.id = r.id; b.dataset.key = 'rec-' + r.id; b.dataset.kind = r.kind; + b.setAttribute('aria-pressed', String(r.id === v.sel)); b.setAttribute('aria-label', WORD[r.kind] + ' ' + r.id); b.title = r.id; + const th = h('span', 'th'); + if (r.kind === 'text' && r.preview != null) th.appendChild(h('span', 'pre', r.preview.slice(0, 160))); + else th.appendChild(glyph(r.kind)); + if (r.kind === 'image' || r.kind === 'array') b.dataset.lazy = '1'; + const label = h('span', 'id'); label.appendChild(h('bdi', null, r.id)); b.appendChild(th); b.appendChild(label); // cut at its start when long: the name shows + b.addEventListener('click', () => select(c, v, r.id)); + return b; + } + /** Fills a tile's picture when it comes into view: the image by its url, an array by a small slice of the array shown first. */ + function watchTiles(c, root) { + const lazy = root.querySelectorAll('.rec[data-lazy]'); if (!lazy.length) return; + c.io = new IntersectionObserver(entries => { for (const e of entries) if (e.isIntersecting) { c.io.unobserve(e.target); fillTile(c, e.target); } }, { root: c.main, rootMargin: '120px 0px' }); + for (const t of lazy) c.io.observe(t); + } + async function fillTile(c, t) { + const v = viewOf(c.ref), run = v.run, id = t.dataset.id, th = t.querySelector('.th'), kind = t.dataset.kind; + const live = () => t.isConnected && current === c && viewOf(c.ref).run === run; + try { + const rec = await R.record(run, id); if (!live()) return; + if (kind === 'image') { + const img = h('img'); img.alt = ''; img.decoding = 'async'; img.src = rec.record.url; + await new Promise(resolve => { img.onload = img.onerror = resolve; }); if (!live()) return; + if (!img.naturalWidth) throw Error('The image could not be decoded.'); + th.replaceChildren(img); + } else { + const arrays = rec.content.arrays || [], name = rec.content.show, a = arrays.find(x => x.array === name) || arrays[0]; + if (!a) throw Error('No array.'); + const s = await R.slice(run, { path: rec.record.path, array: a.array, most: TILE_MOST }); if (!live()) return; + const cv = h('canvas'); cv.setAttribute('aria-hidden', 'true'); + const values = floats(s.values_f32); + paints(cv, () => { const size = fit(cv, cv.clientWidth); if (!size) return; const ctx = cv.getContext('2d'); ctx.clearRect(0, 0, size[0], size[1]); if (s.rows === 1 && s.columns > 1) line(ctx, values, s.min, s.max, 0, 0, size[0], size[1], 2); else { const [x, y, w, hh] = contain(s.rows, s.columns, size[0], size[1]); heat(ctx, values, s.rows, s.columns, ramp(s.min, s.max), x, y, w, hh); } }); + th.replaceChildren(cv); + } + } catch (error) { + if (!live()) return; + th.replaceChildren(glyph('refused')); t.title = id + ': ' + (error.message || error); t.classList.add('refused'); + } + } + /** Selects a record: the tiles' pressed state is set in place and the viewer replaced once its new content is read. */ + async function select(c, v, id) { + if (!c.root) return; + v.sel = id; + for (const t of c.root.querySelectorAll('.rec')) { const on = String(t.dataset.id === id); if (t.getAttribute('aria-pressed') !== on) t.setAttribute('aria-pressed', on); } + await showIn(c, v, { record: id }); + } + async function showIn(c, v, target) { + const panel = c.root && c.root.querySelector('.viewer'); if (!panel) return; + const mine = ++c.vturn; + panel.setAttribute('aria-busy', 'true'); + const slow = setTimeout(() => { if (c.vturn === mine) panel.replaceChildren(h('p', 'note', 'Reading…')); }, 250); + const el = await viewerFor(c, v, target); + clearTimeout(slow); + if (c.vturn !== mine || !panel.isConnected) return; + panel.removeAttribute('aria-busy'); + putInPlace(panel, el); + } + /** Left and right arrows move the selection in the grid, across pages, when the focus is not in a field or a viewer that turns. */ + function onKey(e) { + if (e.defaultPrevented || e.altKey || e.ctrlKey || e.metaKey || (e.key !== 'ArrowLeft' && e.key !== 'ArrowRight')) return; + const c = current; if (!c || !c.root || !c.root.isConnected) return; + const v = viewOf(c.ref); if (v.mode !== 'records') return; + const t = e.target; if (t && t.closest && (t.closest('input, select, textarea, [contenteditable], audio, .turns') )) return; + const list = c.root.querySelector('.inside-list'), recs = list && list._records; if (!recs || !recs.length) return; + e.preventDefault(); + const i = recs.findIndex(r => r.id === v.sel), d = e.key === 'ArrowRight' ? 1 : -1, n = i + d; + const onTile = !!(t && t.classList && t.classList.contains('rec')); + if (n >= 0 && n < recs.length) { select(c, v, recs[n].id); if (onTile) { const next = c.root.querySelector('.rec[data-id="' + CSS.escape(recs[n].id) + '"]'); if (next) next.focus(); } return; } + c.focusSel = onTile; + if (n >= recs.length && v.page + 1 < list._pages) turnPage(c, v, v.page + 1, 0); + else if (n < 0 && v.page > 0) turnPage(c, v, v.page - 1, -1); + else c.focusSel = false; + } + window.addEventListener('keydown', onKey); + + // ---- Files: the product as a tree ------------------------------------------------------------------------ + function files(c, v, tree, left) { + // The way down, each step a link back up. + const crumbs = h('nav', 'crumbs'); crumbs.setAttribute('aria-label', 'Where in the product'); + const parts = v.path ? v.path.split(/(?<=!)|\//).filter(Boolean) : []; + const steps = [['', 'Product']]; let at = ''; + for (const p of parts) { at = at ? (at.endsWith('!') ? at + p : at + '/' + p) : p; steps.push([at.replace(/!$/, ''), p.replace(/!$/, '')]); } + steps.forEach(([path, name], i) => { + if (i) crumbs.appendChild(h('span', 'sep', '/')); + if (i === steps.length - 1) { const s = h('span', 'here', name); s.setAttribute('aria-current', 'location'); crumbs.appendChild(s); } + else { const b = button('linkish', name, () => go(c, v, path)); b.dataset.key = 'crumb-' + path; crumbs.appendChild(b); } + }); + left.appendChild(sect('Files', tree && !tree.error ? count(tree.total, 'entry', 'entries') : '')); + left.appendChild(crumbs); + if (!tree) return; + if (tree.error) { left.appendChild(window.LabActions.refusal(tree.error)); return; } + if (!tree.entries.length) { left.appendChild(h('p', 'empty', 'Nothing here.')); return; } + const ul = h('ul', 'rows tree-rows'); + for (const e of tree.entries) { + const li = h('li'), into = e.type === 'directory' || e.type === 'archive'; + const key = e.type === 'array' ? e.path + '#' + e.array : e.path; + const b = button('row tree-entry'); b.dataset.key = 'entry-' + key; b.dataset.path = e.path; b.dataset.type = e.type; if (e.array) b.dataset.array = e.array; + const t = v.treeTarget; if (!into) b.setAttribute('aria-pressed', String(!!t && t.path === e.path && (t.array || null) === (e.array || null))); + b.appendChild(glyph(e.type === 'directory' ? 'directory' : e.type === 'archive' ? 'archive' : e.kind || 'file')); + const name = h('span'); name.appendChild(h('span', 'title', e.type === 'array' ? e.array || '(the array)' : e.name)); + const what = e.type === 'array' ? e.dtype + ' · ' + e.shape.join(' × ') : e.type === 'directory' ? 'directory' : e.type === 'archive' ? 'archive' + (e.kind ? ' of ' + e.kind + 's' : '') : (WORD[e.kind] || 'File').toLowerCase(); + name.appendChild(h('span', 'sub', what)); b.appendChild(name); + b.appendChild(h('span', 'sub num', e.bytes == null ? '' : bytes(e.bytes))); + b.addEventListener('click', () => { + if (into || (e.type === 'file' && e.kind === 'array')) { go(c, v, e.path); return; } + const siblings = e.type === 'array' ? tree.entries.filter(x => x.type === 'array') : null; + v.treeTarget = { path: e.path, array: e.array || null, kind: e.type === 'array' ? 'array' : e.kind || 'file', name: e.name, arrays: siblings }; + for (const x of left.querySelectorAll('.tree-entry[aria-pressed]')) { const on = String(x === b); if (x.getAttribute('aria-pressed') !== on) x.setAttribute('aria-pressed', on); } + showIn(c, v, v.treeTarget); + }); + li.appendChild(b); ul.appendChild(li); + } + left.appendChild(ul); + if (tree.total > tree.entries.length) { + const pg = h('div', 'pager'); + const pb = button('btn sm', 'Previous', () => { v.treeOffset = Math.max(0, v.treeOffset - TREE_LIMIT); draw(c, false); }); pb.disabled = v.treeOffset === 0; pb.dataset.key = 'tree-prev'; + const nb = button('btn sm', 'Next', () => { v.treeOffset += TREE_LIMIT; draw(c, false); }); nb.disabled = v.treeOffset + tree.entries.length >= tree.total; nb.dataset.key = 'tree-next'; + pg.appendChild(pb); pg.appendChild(nb); pg.appendChild(h('span', null, (v.treeOffset + 1) + '–' + (v.treeOffset + tree.entries.length) + ' of ' + tree.total)); left.appendChild(pg); + } + } + function go(c, v, path) { v.path = path; v.treeOffset = 0; draw(c, false); } + + // ---- The viewers --------------------------------------------------------------------------------------------- + /** + * The viewer of a target, read in full before it is returned: {record: id}, or a place in the tree + * {path, array, kind, arrays}. A refusal is shown as the server's sentence. + */ + async function viewerFor(c, v, target) { + const run = target.run || v.run, el = h('div', 'iv'); + try { + if (target.record != null) await recordView(c, target.run ? { ...v, run } : v, el, await R.record(run, target.record)); + else await placeView(c, v, el, target); + } catch (error) { + el.replaceChildren(heading(target.record != null ? target.record : target.path + (target.array ? ' · ' + target.array : ''), null)); + el.appendChild(window.LabActions.refusal(error)); + } + return el; + } + function heading(id, line) { const hd = h('header', 'iv-head'); hd.appendChild(h('h3', 'mono', id)); if (line) hd.appendChild(h('p', 'small muted', line)); return hd; } + async function recordView(c, v, el, got) { + const r = got.record, k = got.content || {}; + el.appendChild(heading(r.id, [WORD[r.kind], bytes(r.bytes), r.path && r.path !== r.id ? r.path : null].filter(Boolean).join(' · '))); + const key = 'rec:' + r.id; + if (r.kind === 'text') textView(el, k); + else if (r.kind === 'image') imageView(el, r.url, k.width, k.height, r.id, settingsOf(v, key, () => ({ actual: false }))); + else if (r.kind === 'audio') audioView(el, r.url, k, r.id); + else if (r.kind === 'pointcloud') cloudView(el, k, settingsOf(v, key, turned)); + else if (r.kind === 'mesh') meshView(el, k, settingsOf(v, key, turned)); + else if (r.kind === 'array') await arrayView(el, v.run, r.path, k.arrays || [], k.show, settingsOf(v, key, () => ({}))); + else el.appendChild(h('p', 'note', 'A file of no kind Tuldok shows: it is listed, never opened.')); + const facts = h('dl', 'setup iv-facts'); + const put = (name, value) => { if (value == null || value === '') return; facts.appendChild(h('dt', null, name)); facts.appendChild(h('dd', typeof value === 'string' ? null : 'mono pre', typeof value === 'string' ? value : JSON.stringify(value, null, 1))); }; + put('Split', r.split); put('Rights', r.rights); put('Origin', r.origin); put('Meta', r.meta); + if (facts.children.length) el.appendChild(facts); + } + /** A place in the tree: an array by its path and name, any other file read as a record by its address (record's path). */ + async function placeView(c, v, el, t) { + if (t.kind === 'array') { el.appendChild(heading(t.path, 'Array')); await arrayView(el, v.run, t.path, t.arrays || [], t.array, settingsOf(v, 'path:' + t.path + '#' + t.array, () => ({}))); return; } + await recordView(c, v, el, await R.recordAt(v.run, t.path)); // any file, inside an archive too, by its address + } + function textView(el, k) { + const line = [count(k.characters != null ? k.characters : k.text.length, 'character'), k.truncated ? 'truncated: the first ' + k.text.length.toLocaleString('en-GB') + ' are shown' : null, k.replaced ? count(k.replaced, 'byte') + ' not UTF-8, shown as �' : null].filter(Boolean).join(' · '); + el.appendChild(h('p', 'small muted', line)); + const pre = h('pre', 'iv-text'); pre.textContent = k.text; pre.tabIndex = 0; pre.setAttribute('aria-label', 'The text'); el.appendChild(pre); + } + function imageView(el, url, width, height, id, st) { + const b = h('button', 'iv-img'); b.type = 'button'; b.dataset.key = 'iv-img'; b.setAttribute('aria-pressed', String(!!st.actual)); + const img = h('img'); img.src = url; img.alt = 'Image ' + id + (width && height ? ', ' + width + ' by ' + height + ' pixels' : ''); + b.appendChild(img); + const size = h('p', 'small muted'); + const say = () => { const w = width || img.naturalWidth, hh = height || img.naturalHeight; size.textContent = (w && hh ? w + ' × ' + hh + ' pixels' : img.complete ? 'size not found' : 'size known once it loads') + ' · ' + (st.actual ? 'actual size; press for fitted' : 'fitted; press for actual size'); b.classList.toggle('actual', !!st.actual); b.setAttribute('aria-pressed', String(!!st.actual)); b.setAttribute('aria-label', 'Image ' + id + ', ' + (st.actual ? 'at actual size' : 'fitted to the panel') + '; press to toggle'); }; + img.addEventListener('load', () => { if (!width) say(); }, { once: true }); + img.addEventListener('error', () => { size.textContent = 'The image could not be shown.'; }, { once: true }); + b.addEventListener('click', () => { st.actual = !st.actual; say(); }); + say(); el.appendChild(size); const box = h('div', 'iv-imgbox'); box.appendChild(b); el.appendChild(box); + } + function audioView(el, url, k, id) { + const has = k && k.seconds != null; + el.appendChild(h('p', 'small muted', has ? [k.seconds.toFixed(k.seconds < 10 ? 2 : 1) + ' s', k.rate.toLocaleString('en-GB') + ' Hz', count(k.channels, 'channel'), count(k.frames, 'frame')].join(' · ') : 'Length, rate and channels are read from a WAV only; the player reads the rest.')); + if (has && k.peaks && k.peaks.length) { + const cv = h('canvas', 'iv-wave'); cv.setAttribute('role', 'img'); cv.setAttribute('aria-label', 'Waveform of ' + id + ', ' + k.seconds.toFixed(2) + ' seconds'); + paints(cv, () => { const size = fit(cv, 96); if (!size) return; const ctx = cv.getContext('2d'); ctx.clearRect(0, 0, size[0], size[1]); wave(ctx, size[0], size[1], k.peaks); }); + el.appendChild(cv); + } + const a = h('audio'); a.controls = true; a.preload = 'metadata'; a.src = url; a.dataset.key = 'iv-audio'; a.setAttribute('aria-label', 'Play ' + id); el.appendChild(a); + } + + /** A sound's waveform from its peaks ([low, high] per bucket, -1 to 1) across a W by H canvas: the axis, then a bar per column. */ + function wave(ctx, W, H, peaks) { + const ink = token('--mark'), axis = token('--line-strong'), n = peaks.length; + ctx.fillStyle = 'rgb(' + axis.join(',') + ')'; ctx.fillRect(0, Math.floor(H / 2), W, 1); + ctx.fillStyle = 'rgb(' + ink.join(',') + ')'; + for (let x = 0; x < W; x++) { + const a = Math.floor(x / W * n), b = Math.max(a + 1, Math.floor((x + 1) / W * n)); let lo = 1, hi = -1; + for (let i = a; i < b && i < n; i++) { lo = Math.min(lo, peaks[i][0]); hi = Math.max(hi, peaks[i][1]); } + if (hi < lo) continue; const y0 = (1 - hi) / 2 * (H - 2) + 1, y1 = (1 - lo) / 2 * (H - 2) + 1; ctx.fillRect(x, y0, 1, Math.max(1, y1 - y0)); + } + } + + // ---- Point clouds and meshes: turned by dragging, zoomed by the wheel ---------------------------------------------- + const turned = () => ({ yaw: -0.6, pitch: 0.45, zoom: 1 }); + function cloudView(el, k, st) { + const pts = floats(k.points_f32), colours = k.colors_u8 ? raw(k.colors_u8) : null; + el.appendChild(h('p', 'small muted', count(k.count, 'point') + (k.stride > 1 ? ' · every ' + ordinal(k.stride) + ' shown (' + count(pts.length / 3, 'point') + ')' : '') + ' · ' + (colours ? 'their own colours' : 'coloured by height'))); + space(el, { points: pts, colours, bounds: k.bounds }, st, 'Point cloud, ' + count(pts.length / 3, 'point') + (k.stride > 1 ? ' of ' + k.count.toLocaleString('en-GB') : '')); + } + function meshView(el, k, st) { + const pos = floats(k.positions_f32), asPoints = k.triangles > MESH_AS_POINTS; + el.appendChild(h('p', 'small muted', count(k.vertices, 'vertex', 'vertices') + ' · ' + count(k.triangles, 'triangle') + (asPoints ? ' · above ' + MESH_AS_POINTS.toLocaleString('en-GB') + ' triangles the vertices are drawn as points' : ' · flat shaded'))); + if (asPoints) space(el, { points: pos, colours: null, bounds: k.bounds }, st, 'Mesh of ' + count(k.triangles, 'triangle') + ', drawn as its ' + count(k.vertices, 'vertex', 'vertices') + ' as points'); + else space(el, { points: pos, triangles: uints(k.triangles_u32), bounds: k.bounds }, st, 'Mesh, ' + count(k.vertices, 'vertex', 'vertices') + ' and ' + count(k.triangles, 'triangle') + ', flat shaded'); + } + const ordinal = n => n + (n % 100 >= 11 && n % 100 <= 13 ? 'th' : ['th', 'st', 'nd', 'rd'][n % 10] || 'th'); + /** A canvas showing points or triangles in 3-D; z is up. Drawn on requestAnimationFrame only after a change. */ + function space(el, m, st, label) { + const cv = h('canvas', 'iv-3d turns'); cv.tabIndex = 0; cv.setAttribute('role', 'img'); cv.dataset.key = 'iv-3d'; + cv.setAttribute('aria-label', label + '. Drag or use the arrow keys to turn it, the wheel or + and − to zoom.'); + const [lo, hi] = m.bounds || [[-1, -1, -1], [1, 1, 1]], centre = [0, 1, 2].map(i => (lo[i] + hi[i]) / 2); + const radius = Math.max(1e-9, Math.hypot(hi[0] - lo[0], hi[1] - lo[1], hi[2] - lo[2]) / 2); + let frame = 0; + const draw = () => { + frame = 0; + const size = fit(cv, Math.min(420, Math.round(cv.clientWidth * 0.75))); if (!size) return; + const [W, H] = size, ctx = cv.getContext('2d'), img = ctx.createImageData(W, H), px = img.data, depth = new Float32Array(W * H).fill(Infinity); + const cy = Math.cos(st.yaw), sy = Math.sin(st.yaw), cp = Math.cos(st.pitch), sp = Math.sin(st.pitch), s = st.zoom * 0.46 * Math.min(W, H) / radius; + const n = m.points.length / 3, X = new Float32Array(n), Y = new Float32Array(n), Z = new Float32Array(n); + for (let i = 0; i < n; i++) { + const x = m.points[i * 3] - centre[0], y = m.points[i * 3 + 1] - centre[1], z = m.points[i * 3 + 2] - centre[2]; + const x1 = x * cy - y * sy, y1 = x * sy + y * cy, d = y1 * cp + z * sp, up = -y1 * sp + z * cp; + X[i] = W / 2 + x1 * s; Y[i] = H / 2 - up * s; Z[i] = d; + } + if (m.triangles) { + const base = token('--mark'), L = [-0.35, 0.55, -0.76], t = m.triangles; + for (let f = 0; f + 2 < t.length; f += 3) { + const a = t[f], b = t[f + 1], c = t[f + 2]; + // The normal in screen space (x right, y up, z away), for flat shading lit from the front. + const ux = X[b] - X[a], uy = Y[a] - Y[b], uz = (Z[b] - Z[a]) * s, vx = X[c] - X[a], vy = Y[a] - Y[c], vz = (Z[c] - Z[a]) * s; + const nx = uy * vz - uz * vy, ny = uz * vx - ux * vz, nz = ux * vy - uy * vx, len = Math.hypot(nx, ny, nz) || 1; + const shade = 0.32 + 0.68 * Math.abs((nx * L[0] + ny * L[1] + nz * L[2]) / len); + triangle(px, depth, W, H, X[a], Y[a], Z[a], X[b], Y[b], Z[b], X[c], Y[c], Z[c], base[0] * shade, base[1] * shade, base[2] * shade); + } + } else { + const r = Math.max(1, Math.round((window.devicePixelRatio || 1) * 1.6)); + const neg = token('--ramp-neg'), pos = token('--ramp-pos'), span = (hi[2] - lo[2]) || 1; + for (let i = 0; i < n; i++) { + const x0 = Math.round(X[i]), y0 = Math.round(Y[i]); if (x0 < 0 || y0 < 0 || x0 >= W || y0 >= H) continue; + let R, G, B; if (m.colours) { R = m.colours[i * 3]; G = m.colours[i * 3 + 1]; B = m.colours[i * 3 + 2]; } else { const c = mix(neg, pos, (m.points[i * 3 + 2] - lo[2]) / span); R = c[0]; G = c[1]; B = c[2]; } + for (let dy = 0; dy < r; dy++) for (let dx = 0; dx < r; dx++) { + const xx = x0 + dx, yy = y0 + dy; if (xx >= W || yy >= H) continue; const o = yy * W + xx; + if (Z[i] < depth[o]) { depth[o] = Z[i]; px[o * 4] = R; px[o * 4 + 1] = G; px[o * 4 + 2] = B; px[o * 4 + 3] = 255; } + } + } + } + ctx.putImageData(img, 0, 0); + // Wireframes (spec-t4: 3-D boxes): m.lines is [[x, y, z, x, y, z, [r, g, b]], ...], drawn over the points. + const project = (x, y, z) => { x -= centre[0]; y -= centre[1]; z -= centre[2]; const x1 = x * cy - y * sy, y1 = x * sy + y * cy; return [W / 2 + x1 * s, H / 2 - (-y1 * sp + z * cp) * s]; }; + for (const l of m.lines || []) { const a = project(l[0], l[1], l[2]), b = project(l[3], l[4], l[5]); ctx.strokeStyle = 'rgb(' + l[6].join(',') + ')'; ctx.lineWidth = Math.max(1, (window.devicePixelRatio || 1) * 1.5); ctx.beginPath(); ctx.moveTo(a[0], a[1]); ctx.lineTo(b[0], b[1]); ctx.stroke(); } + if (pick) { const k = W / cv.clientWidth; ctx.strokeStyle = 'rgb(' + token('--serious').join(',') + ')'; ctx.setLineDash([4, 3]); ctx.strokeRect(pick.x0 * k, pick.y0 * k, (pick.x1 - pick.x0) * k, (pick.y1 - pick.y0) * k); ctx.setLineDash([]); } + shown = { X, Y, W }; + }; + const later = () => { if (!frame) frame = requestAnimationFrame(draw); }; + paints(cv, draw); + let drag = null, pick = null, shown = null; + // With m.select and st.selecting, a drag draws a rectangle and m.select gets the indices of the points shown in it. + cv.addEventListener('pointerdown', e => { if (e.button !== 0) return; const q = cv.getBoundingClientRect(); if (m.select && st.selecting) { pick = { x0: e.clientX - q.left, y0: e.clientY - q.top, x1: e.clientX - q.left, y1: e.clientY - q.top, id: e.pointerId, left: q.left, top: q.top }; cv.setPointerCapture(e.pointerId); later(); return; } drag = { x: e.clientX, y: e.clientY, id: e.pointerId }; cv.setPointerCapture(e.pointerId); cv.classList.add('dragging'); }); + cv.addEventListener('pointermove', e => { if (pick && e.pointerId === pick.id) { pick.x1 = e.clientX - pick.left; pick.y1 = e.clientY - pick.top; later(); return; } if (!drag || e.pointerId !== drag.id) return; st.yaw += (e.clientX - drag.x) * 0.01; st.pitch = Math.max(-1.55, Math.min(1.55, st.pitch + (e.clientY - drag.y) * 0.01)); drag.x = e.clientX; drag.y = e.clientY; later(); }); + const end = e => { + if (pick && e.pointerId === pick.id) { + const k = shown ? shown.W / cv.clientWidth : 1, [x0, x1] = [Math.min(pick.x0, pick.x1) * k, Math.max(pick.x0, pick.x1) * k], [y0, y1] = [Math.min(pick.y0, pick.y1) * k, Math.max(pick.y0, pick.y1) * k], found = []; + if (shown) for (let i = 0; i < shown.X.length; i++) if (shown.X[i] >= x0 && shown.X[i] <= x1 && shown.Y[i] >= y0 && shown.Y[i] <= y1) found.push(i); + pick = null; later(); m.select(found); return; + } + if (drag && e.pointerId === drag.id) { drag = null; cv.classList.remove('dragging'); } + }; + cv.addEventListener('pointerup', end); cv.addEventListener('pointercancel', end); + cv.addEventListener('wheel', e => { e.preventDefault(); st.zoom = Math.max(0.2, Math.min(25, st.zoom * Math.exp(-e.deltaY * 0.0015))); later(); }, { passive: false }); + cv.addEventListener('keydown', e => { + const k = e.key; let used = true; + if (k === 'ArrowLeft') st.yaw -= 0.15; else if (k === 'ArrowRight') st.yaw += 0.15; + else if (k === 'ArrowUp') st.pitch = Math.max(-1.55, st.pitch - 0.15); else if (k === 'ArrowDown') st.pitch = Math.min(1.55, st.pitch + 0.15); + else if (k === '+' || k === '=') st.zoom = Math.min(25, st.zoom * 1.2); else if (k === '-' || k === '−') st.zoom = Math.max(0.2, st.zoom / 1.2); + else used = false; + if (used) { e.preventDefault(); later(); } + }); + el.appendChild(cv); + const bar = h('div', 'iv-ctl'); + const reset = button('btn sm', 'Reset view', () => { Object.assign(st, turned()); later(); }); reset.dataset.key = 'iv-reset'; bar.appendChild(reset); + bar.appendChild(h('span', 'small faint', 'Drag to turn, wheel to zoom')); + el.appendChild(bar); + } + /** Fills one triangle into the pixel and depth buffers (nearest wins). */ + function triangle(px, depth, W, H, ax, ay, az, bx, by, bz, cx, cy, cz, R, G, B) { + const area = (bx - ax) * (cy - ay) - (by - ay) * (cx - ax); if (Math.abs(area) < 1e-9) return; + const x0 = Math.max(0, Math.floor(Math.min(ax, bx, cx))), x1 = Math.min(W - 1, Math.ceil(Math.max(ax, bx, cx))); + const y0 = Math.max(0, Math.floor(Math.min(ay, by, cy))), y1 = Math.min(H - 1, Math.ceil(Math.max(ay, by, cy))); + for (let y = y0; y <= y1; y++) { + const py = y + 0.5; + for (let x = x0; x <= x1; x++) { + const qx = x + 0.5; + const w0 = ((cx - bx) * (py - by) - (cy - by) * (qx - bx)) / area, w1 = ((ax - cx) * (py - cy) - (ay - cy) * (qx - cx)) / area, w2 = 1 - w0 - w1; + if (w0 < 0 || w1 < 0 || w2 < 0) continue; + const z = w0 * az + w1 * bz + w2 * cz, o = y * W + x; + if (z < depth[o]) { depth[o] = z; px[o * 4] = R; px[o * 4 + 1] = G; px[o * 4 + 2] = B; px[o * 4 + 3] = 255; } + } + } + } + + // ---- Arrays: a slice along two axes, sliders for the others, the stacked slices and the histogram --------------------- + /** A line of a one-row slice across (x, y, w, hh). */ + function line(ctx, values, min, max, x, y, w, hh, width) { + const n = values.length, span = (max - min) || 1, ink = token('--mark'); + ctx.strokeStyle = 'rgb(' + ink.join(',') + ')'; ctx.lineWidth = width; ctx.beginPath(); + let on = false; + for (let i = 0; i < n; i++) { const v = values[i]; if (!Number.isFinite(v)) { on = false; continue; } const px = x + (n === 1 ? w / 2 : i / (n - 1) * w), py = y + hh - 4 - (v - min) / span * (hh - 8); if (on) ctx.lineTo(px, py); else ctx.moveTo(px, py); on = true; } + ctx.stroke(); + } + async function arrayView(el, run, path, arrays, show, st) { + if (!arrays.length) { el.appendChild(h('p', 'note', 'This file holds no array.')); return; } + const pick = arrays.find(a => a.array === st.array) || arrays.find(a => a.array === show) || arrays.find(a => a.shape.length >= 2) || arrays[0]; + st.array = pick.array; + const box = h('div', 'iv-array'); + /** Draws this array viewer again (another array or other axes), replacing it in one step with the focus kept. */ + const again = async () => { + const host = h('div'); await arrayView(host, run, path, arrays, show, st); + const fresh = host.firstChild; if (!fresh || !box.isConnected) return; + const key = box.contains(document.activeElement) && document.activeElement.dataset ? document.activeElement.dataset.key : null; + box.replaceWith(fresh); + if (key) { const e = [...fresh.querySelectorAll('[data-key]')].find(x => x.dataset.key === key); if (e) e.focus({ preventScroll: true }); } + }; + // The arrays, with shape and type; the one chosen is drawn. + if (arrays.length > 1) { + const s = h('select'); s.setAttribute('aria-label', 'Array'); s.dataset.key = 'iv-array'; + for (const a of arrays) { const o = h('option', null, (a.array == null ? '(the array)' : a.array) + ' · ' + a.dtype + ' · ' + (a.shape.length ? a.shape.join(' × ') : 'one value')); o.value = a.array; o.selected = a === pick; s.appendChild(o); } + s.addEventListener('change', () => { st.array = s.value; st.axes = null; st.index = null; again(); }); + const l = h('label', 'iv-field'); l.appendChild(document.createTextNode('Array')); l.appendChild(s); box.appendChild(l); + } else box.appendChild(h('p', 'small muted', (pick.array ? pick.array + ' · ' : '') + pick.dtype + ' · ' + (pick.shape.length ? pick.shape.join(' × ') : 'one value'))); + const shape = pick.shape, nd = shape.length; + if (!st.axes || st.axes.some(a => a >= nd)) st.axes = nd >= 2 ? [nd - 2, nd - 1] : nd === 1 ? [0] : []; + if (!st.index || st.index.length !== nd) st.index = shape.map(n => Math.floor(n / 2)); + const base = { path, array: pick.array, most: VIEW_MOST }; + const args = () => { const a = Object.assign({}, base); if (nd >= 2) { a.axes = st.axes.slice(); a.index = st.index.slice(); } return a; }; + // Two axes drawn (for three or more dimensions they can be chosen), a slider per other axis. + const ctl = h('div', 'iv-ctl'); + if (nd > 2) for (const [i, label] of [[0, 'Rows'], [1, 'Columns']]) { + const s = h('select'); s.setAttribute('aria-label', label + ' axis'); s.dataset.key = 'iv-axis-' + i; + for (let a = 0; a < nd; a++) { const o = h('option', null, 'axis ' + a + ' (' + shape[a] + ')'); o.value = String(a); o.selected = st.axes[i] === a; o.disabled = st.axes[1 - i] === a; s.appendChild(o); } + s.addEventListener('change', () => { st.axes[i] = Number(s.value); again(); }); + const l = h('label', 'iv-field'); l.appendChild(document.createTextNode(label)); l.appendChild(s); ctl.appendChild(l); + } + const others = nd >= 2 ? shape.map((_, a) => a).filter(a => !st.axes.includes(a)) : []; + const sliders = []; + for (const a of others) { + const l = h('label', 'iv-field iv-slider'); const name = h('span', null, 'Axis ' + a); l.appendChild(name); + const r = h('input'); r.type = 'range'; r.min = '0'; r.max = String(shape[a] - 1); r.value = String(st.index[a]); r.dataset.key = 'iv-axis-slider-' + a; r.dataset.axis = String(a); + const at = h('span', 'num small', st.index[a] + ' of ' + shape[a]); + r.setAttribute('aria-label', 'Position on axis ' + a + ', 0 to ' + (shape[a] - 1)); + l.appendChild(r); l.appendChild(at); ctl.appendChild(l); sliders.push({ a, r, at }); + } + if (ctl.children.length) box.appendChild(ctl); + // The slice, read now. + let s; + try { s = await R.slice(run, args(), true); } catch (error) { box.appendChild(window.LabActions.refusal(error)); el.appendChild(box); return; } + const sliceBox = h('div', 'iv-slice'), cv = h('canvas', 'iv-heat'); cv.setAttribute('role', 'img'); sliceBox.appendChild(cv); + const range = h('p', 'small muted iv-range'), probe = h('p', 'small iv-probe'); probe.textContent = 'Point at the slice to read a value.'; + let values = floats(s.values_f32), rect = null; + const say = () => { + cv.dataset.index = JSON.stringify(s.index); cv.dataset.axes = JSON.stringify(s.axes || []); + const what = s.rows === 1 && nd <= 1 ? (nd === 0 ? 'The value' : 'The ' + s.columns + ' values') : 'Slice of ' + s.rows + ' by ' + s.columns + ' values along axes ' + (s.axes || []).join(' and ') + (others.length ? ' at ' + others.map(a => 'axis ' + a + ' = ' + s.index[a]).join(', ') : ''); + cv.setAttribute('aria-label', what + ' of ' + (pick.array || path) + ', from ' + fmt(s.min) + ' to ' + fmt(s.max)); + range.textContent = 'From ' + fmt(s.min) + ' to ' + fmt(s.max) + (s.stride > 1 ? ' · every ' + ordinal(s.stride) + ' row and column' : '') + (s.nonfinite ? ' · ' + count(s.nonfinite, 'value') + ' not finite, left clear' : ''); + }; + const paintSlice = () => { + const wide = s.rows === 1, w = cv.clientWidth; if (!w) return; + const cssH = wide ? 120 : Math.max(80, Math.min(360, Math.round(w * s.rows / s.columns))); + const size = fit(cv, cssH); if (!size) return; const [W, H] = size, ctx = cv.getContext('2d'); ctx.clearRect(0, 0, W, H); + if (wide) { rect = [0, 0, W, H]; line(ctx, values, s.min == null ? 0 : s.min, s.max == null ? 1 : s.max, 0, 0, W, H, Math.max(1.5, W / 400)); } + else { rect = contain(s.rows, s.columns, W, H); heat(ctx, values, s.rows, s.columns, ramp(s.min == null ? 0 : s.min, s.max == null ? 1 : s.max), ...rect); } + const where = rect.map(n => Math.round(n * 100) / 100).join(',') + ';' + s.rows + 'x' + s.columns; // where the cells are, for a reader of the page + if (cv.dataset.cells !== where) cv.dataset.cells = where; + }; + paints(cv, paintSlice); + cv.addEventListener('pointermove', e => { + if (!rect) return; const dpr = cv.width / cv.clientWidth, x = e.offsetX * dpr - rect[0], y = e.offsetY * (cv.height / cv.clientHeight) - rect[1]; + if (x < 0 || y < 0 || x >= rect[2] || y >= rect[3]) { probe.textContent = 'Point at the slice to read a value.'; delete probe.dataset.index; return; } + const col = Math.min(s.columns - 1, Math.floor(x / rect[2] * s.columns)), row = s.rows === 1 ? 0 : Math.min(s.rows - 1, Math.floor(y / rect[3] * s.rows)); + const index = (s.index || []).slice(); if (nd >= 2) { index[s.axes[0]] = row * s.stride; index[s.axes[1]] = col * s.stride; } else if (nd === 1) index[0] = col * s.stride; + const v = values[row * s.columns + col]; + probe.textContent = '[' + index.join(', ') + '] = ' + (Number.isFinite(v) ? String(v) : String(v) + ' (not finite)'); + probe.dataset.index = JSON.stringify(index); probe.dataset.value = String(v); + }); + cv.addEventListener('pointerleave', () => { probe.textContent = 'Point at the slice to read a value.'; delete probe.dataset.index; }); + box.appendChild(sliceBox); box.appendChild(range); box.appendChild(probe); say(); + // Three or more dimensions: the stacked slices along the first other axis, the slider's slice outlined (the mockup's drawVolume). + let stack = null; + if (nd >= 3) { + const axis = others[0], n = shape[axis], at = Array.from(new Set(Array.from({ length: Math.min(n, STACK_SLICES) }, (_, i) => Math.round(n <= STACK_SLICES ? i : i * (n - 1) / (STACK_SLICES - 1))))); + const vol = h('canvas', 'iv-vol'); vol.setAttribute('role', 'img'); + stack = { axis, at, vol, slices: null, key: null }; + const loadStack = async () => { + const key = JSON.stringify([st.axes, st.index.filter((_, i) => i !== axis)]); if (stack.key === key) return; stack.key = key; + const got = await Promise.all(at.map(p => { const a = args(); a.most = STACK_MOST; a.index[axis] = p; return R.slice(run, a); })); + if (stack.key !== key) return; + stack.slices = got.map(g => ({ values: floats(g.values_f32), rows: g.rows, cols: g.columns, min: g.min, max: g.max })); + }; + const paintStack = () => { + if (!stack.slices) return; const w = vol.clientWidth; if (!w) return; + const size = fit(vol, Math.round(Math.min(360, w * 0.62))); if (!size) return; + volume(vol, stack.slices, at.indexOf(nearest(at, st.index[axis]))); + vol.setAttribute('aria-label', 'Stacked slices of ' + (pick.array || path) + ' along axis ' + axis + ': ' + at.length + ' of its ' + n + ' positions, the one at ' + nearest(at, st.index[axis]) + ' outlined'); + }; + try { await loadStack(); } catch (error) { stack = null; box.appendChild(window.LabActions.refusal(error)); } + if (stack) { paints(vol, paintStack); stack.paint = paintStack; stack.load = loadStack; const vb = h('div', 'iv-volbox'); vb.appendChild(vol); box.insertBefore(vb, sliceBox); } + } + // A slider moves the slice: read again (the latest wins), redraw in place. + let wanted = 0, timer = 0; + for (const { a, r, at } of sliders) r.addEventListener('input', () => { + st.index[a] = Number(r.value); at.textContent = st.index[a] + ' of ' + shape[a]; + if (stack && a === stack.axis) stack.paint && stack.paint(); + clearTimeout(timer); const mine = ++wanted; + timer = setTimeout(async () => { + try { + const next = await R.slice(run, args(), true); if (mine !== wanted) return; + s = next; values = floats(s.values_f32); paintSlice(); say(); + probe.textContent = 'Point at the slice to read a value.'; delete probe.dataset.index; + if (stack && a !== stack.axis) { await stack.load(); stack.paint(); } + } catch (error) { if (mine === wanted) { probe.textContent = error.message; } } + }, 90); + }); + // The summary's histogram, on request. + const hb = button('btn sm', st.hist ? 'Hide the histogram' : 'Show the histogram'); hb.dataset.key = 'iv-hist'; hb.setAttribute('aria-expanded', String(!!st.hist)); + const hist = h('div', 'iv-hist'); hist.hidden = !st.hist; + const fillHist = async () => { + if (hist.firstChild) return; + try { + const sum = await R.summary(run, { path, array: pick.array, bins: 64 }); + const hc = h('canvas', 'iv-histc'); hc.setAttribute('role', 'img'); hc.setAttribute('aria-label', 'Histogram of ' + (pick.array || path) + ' in ' + sum.histogram.counts.length + ' bins from ' + fmt(sum.histogram.low) + ' to ' + fmt(sum.histogram.high)); + paints(hc, () => { + const size = fit(hc, 110); if (!size) return; const [W, H] = size, ctx = hc.getContext('2d'); ctx.clearRect(0, 0, W, H); + const counts = sum.histogram.counts, most = Math.max(1, ...counts), bw = W / counts.length, ink = token('--mark'), axis = token('--line-strong'); + ctx.fillStyle = 'rgb(' + ink.join(',') + ')'; + counts.forEach((n, i) => { if (!n) return; const bh = Math.max(1, n / most * (H - 6)); ctx.fillRect(i * bw + 0.5, H - bh, Math.max(1, bw - 1), bh); }); + ctx.fillStyle = 'rgb(' + axis.join(',') + ')'; ctx.fillRect(0, H - 1, W, 1); + }); + hist.appendChild(hc); + const ends = h('div', 'iv-ends small faint'); ends.appendChild(h('span', null, fmt(sum.histogram.low))); ends.appendChild(h('span', null, fmt(sum.histogram.high))); hist.appendChild(ends); + const dl = h('dl', 'setup iv-facts'); + for (const [k, val] of [['Values', sum.count.toLocaleString('en-GB') + (sum.stride > 1 ? ', every ' + ordinal(sum.stride) + ' read' : '')], ['Mean', fmt(sum.mean)], ['Deviation', fmt(sum.std)], ['Zeros', sum.zeros.toLocaleString('en-GB')], ['Not finite', sum.nonfinite.toLocaleString('en-GB')]]) { dl.appendChild(h('dt', null, k)); dl.appendChild(h('dd', 'num', val)); } + hist.appendChild(dl); + } catch (error) { hist.appendChild(window.LabActions.refusal(error)); } + }; + hb.addEventListener('click', async () => { st.hist = !st.hist; hb.textContent = st.hist ? 'Hide the histogram' : 'Show the histogram'; hb.setAttribute('aria-expanded', String(st.hist)); if (st.hist) await fillHist(); hist.hidden = !st.hist; }); + if (st.hist) await fillHist(); + const hrow = h('div', 'iv-ctl'); hrow.appendChild(hb); box.appendChild(hrow); box.appendChild(hist); + el.appendChild(box); + } + const nearest = (list, x) => list.reduce((a, b) => Math.abs(b - x) < Math.abs(a - x) ? b : a, list[0]); + /** The stacked-slice drawing of the mockup (drawVolume): each slice a sheared sheet of cells, the chosen one outlined. */ + function volume(canvas, slices, hot) { + const W = canvas.width, H = canvas.height, ctx = canvas.getContext('2d'); ctx.clearRect(0, 0, W, H); + let min = Infinity, max = -Infinity; for (const sl of slices) { if (sl.min != null) min = Math.min(min, sl.min); if (sl.max != null) max = Math.max(max, sl.max); } + if (!Number.isFinite(min)) { min = 0; max = 1; } + const colour = ramp(min, max), first = slices[0], sx = Math.min(16, first.cols), sy = Math.max(2, Math.min(16, Math.round(16 * first.rows / Math.max(1, first.cols)), first.rows)), nsl = slices.length; + const cw = Math.min(W * 0.62 / (sx + sy / 2), 30 * W / 480), ch = cw * 0.5, dz = Math.min(H * 0.07, (H - 30 - sy * ch) / Math.max(1, nsl - 1)); + const ox = W / 2 - (sx * cw) / 2 + (sy * cw) / 4, oy = H - 16 - (nsl - 1) * dz - sy * ch; + const accent = token('--accent'); + for (let k = 0; k < nsl; k++) { + const sl = slices[k], isHot = k === hot; ctx.globalAlpha = isHot ? 1 : 0.55; + for (let j = 0; j < sy; j++) for (let i = 0; i < sx; i++) { + const v = sl.values[Math.min(sl.rows - 1, Math.floor(j * sl.rows / sy)) * sl.cols + Math.min(sl.cols - 1, Math.floor(i * sl.cols / sx))]; + if (!Number.isFinite(v)) continue; const c = colour(v); ctx.fillStyle = 'rgb(' + c.map(Math.round).join(',') + ')'; + const px = ox + i * cw - j * (cw / 2), py = oy + (nsl - 1 - k) * dz + j * ch; + ctx.beginPath(); ctx.moveTo(px, py); ctx.lineTo(px + cw, py); ctx.lineTo(px + cw / 2, py + ch); ctx.lineTo(px - cw / 2, py + ch); ctx.closePath(); ctx.fill(); + } + if (isHot) { ctx.globalAlpha = 1; ctx.strokeStyle = 'rgb(' + accent.join(',') + ')'; ctx.lineWidth = Math.max(2, W / 300); const py = oy + (nsl - 1 - k) * dz; ctx.beginPath(); ctx.moveTo(ox, py); ctx.lineTo(ox + sx * cw, py); ctx.lineTo(ox + sx * cw - sy * cw / 2, py + sy * ch); ctx.lineTo(ox - sy * cw / 2, py + sy * ch); ctx.closePath(); ctx.stroke(); } + } + ctx.globalAlpha = 1; + } + + // The drawing parts other viewers reuse (weights.js): decoding, the tokens' colours, the canvases drawn again on a new size or theme. + /** + * What the Annotate tab (annotate.js) draws with: a record's viewer as this module draws it, from the answer of + * `record` (run: the product's run; store: a Map that keeps each viewer's settings, as a dataset opened here does), + * and the parts its own drawing surfaces share with these viewers. + */ + const viewers = { record: (el, got, run, store) => recordView(null, { run, viewers: store }, el, got), wave, paints, fit, token, glyph, WORD, space, turned }; + return { open, leave, PER_PAGE, viewers, drawing: { floats, token, ramp, paints, fit, heat, contain, ordinal } }; +})(); diff --git a/static/project-workspace/markdown.js b/static/project-workspace/markdown.js new file mode 100644 index 00000000..1ac1273d --- /dev/null +++ b/static/project-workspace/markdown.js @@ -0,0 +1,222 @@ +'use strict'; +// A small Markdown renderer for a model's card. It builds elements and text +// nodes only: no project text is ever assigned as markup. It covers what the +// cards use: ATX and setext headings, paragraphs, bold and italic, inline +// code, fenced code blocks, nested lists, block quotes, rules and tables. +// Links and images are shown as their text (a link's address is its title); +// nothing on the page is fetched or opened from a card. +window.TuldokMarkdown = (() => { + const h = (tag, cls) => { const n = document.createElement(tag); if (cls) n.className = cls; return n; }; + const BLANK = /^\s*$/; + const FENCE = /^( {0,3})(`{3,}|~{3,})(.*)$/; + const HEADING = /^ {0,3}(#{1,6})(?:[ \t]+(.*?))?(?:[ \t]+#+)?[ \t]*$/; + const RULE = /^ {0,3}([-*_])(?:[ \t]*\1){2,}[ \t]*$/; + const QUOTE = /^ {0,3}> ?/; + const ITEM = /^( {0,3})([-*+]|\d{1,9}[.)])([ \t]+|$)(.*)$/; + const SETEXT = /^ {0,3}(=+|-+)[ \t]*$/; + const ESCAPABLE = /[\\`*_{}[\]()#+\-.!|>~<]/; + const indentOf = line => line.match(/^ */)[0].length; + const detab = line => line.replace(/\t/g, ' '); + + // ---- Inline ------------------------------------------------------------- + function closer(text, delimiter, from) { + for (let j = text.indexOf(delimiter, from); j >= 0; j = text.indexOf(delimiter, j + 1)) { + if (j === from || /\s/.test(text[j - 1])) continue; + const after = text[j + delimiter.length]; + if (delimiter.length === 1 && after === delimiter) { j++; continue; } // part of a double + if (delimiter[0] === '_' && after && /[A-Za-z0-9]/.test(after)) continue; // intraword underscore + return j; + } + return -1; + } + function bracket(text, from) { // index of the ] that closes the [ at from, or -1 + let depth = 0; + for (let j = from; j < text.length; j++) { + if (text[j] === '\\') { j++; continue; } + if (text[j] === '[') depth++; + else if (text[j] === ']' && --depth === 0) return j; + } + return -1; + } + function inline(text, parent) { + let buffer = '', i = 0; + const flush = () => { if (buffer) { parent.appendChild(document.createTextNode(buffer)); buffer = ''; } }; + while (i < text.length) { + const c = text[i]; + if (c === '\\' && i + 1 < text.length && ESCAPABLE.test(text[i + 1])) { buffer += text[i + 1]; i += 2; continue; } + if (c === '`') { + let n = 1; while (text[i + n] === '`') n++; + const fence = '`'.repeat(n); + let end = text.indexOf(fence, i + n); + while (end >= 0 && text[end + n] === '`') { let k = end; while (text[k] === '`') k++; end = text.indexOf(fence, k); } + if (end < 0) { buffer += fence; i += n; continue; } + let body = text.slice(i + n, end).replace(/\n/g, ' '); + if (body.length > 2 && body[0] === ' ' && body[body.length - 1] === ' ' && body.trim()) body = body.slice(1, -1); + flush(); const code = h('code'); code.textContent = body; parent.appendChild(code); i = end + n; continue; + } + if (c === '*' || c === '_') { + let n = 1; while (text[i + n] === c && n < 3) n++; + const before = text[i - 1], next = text[i + n]; + const opens = next && !/\s/.test(next) && !(c === '_' && before && /[A-Za-z0-9]/.test(before)); + let done = false; + for (let size = n; opens && size >= 1 && !done; size--) { + const delimiter = c.repeat(size), end = closer(text, delimiter, i + size); + if (end < 0 || end === i + size) continue; + flush(); + const inner = text.slice(i + size, end); + let outer, target; + if (size === 3) { outer = h('strong'); target = h('em'); outer.appendChild(target); } + else { outer = target = h(size === 2 ? 'strong' : 'em'); } + inline(inner, target); parent.appendChild(outer); i = end + size; done = true; + } + if (!done) { buffer += c.repeat(n); i += n; } + continue; + } + if (c === '[' || (c === '!' && text[i + 1] === '[')) { + const open = c === '!' ? i + 1 : i, close = bracket(text, open); + if (close > 0 && text[close + 1] === '(') { + const end = text.indexOf(')', close + 2); + if (end > 0) { + const label = text.slice(open + 1, close), target = text.slice(close + 2, end).trim().split(/\s+/)[0] || ''; + flush(); + const span = h('span', 'link'); span.title = target; + if (c === '!') span.appendChild(document.createTextNode('image: ' + (label || target))); + else inline(label, span); + parent.appendChild(span); i = end + 1; continue; + } + } + buffer += c; i++; continue; + } + if (c === '<') { + const auto = /^<((?:https?|mailto|file):[^\s<>]+)>/.exec(text.slice(i)); + if (auto) { flush(); const span = h('span', 'link'); span.title = auto[1]; span.textContent = auto[1]; parent.appendChild(span); i += auto[0].length; continue; } + } + if (c === '\n') { + if (/ {2,}$/.test(buffer)) { buffer = buffer.replace(/ +$/, ''); flush(); parent.appendChild(h('br')); } + else buffer += ' '; + i++; continue; + } + buffer += c; i++; + } + flush(); + } + + // ---- Tables ------------------------------------------------------------- + function cells(line) { + let row = line.trim(); + if (row.startsWith('|')) row = row.slice(1); + if (row.endsWith('|') && !row.endsWith('\\|')) row = row.slice(0, -1); + const out = []; let cell = '', code = 0; + for (let i = 0; i < row.length; i++) { + const c = row[i]; + if (c === '\\' && row[i + 1] === '|') { cell += '|'; i++; continue; } + if (c === '`') code = code ? 0 : 1; + if (c === '|' && !code) { out.push(cell.trim()); cell = ''; continue; } + cell += c; + } + out.push(cell.trim()); + return out; + } + const isDelimiterRow = line => line.includes('-') && /^[\s|:-]+$/.test(line) && cells(line).every(cell => /^:?-+:?$/.test(cell)); + function startsTable(lines, i) { return lines[i].includes('|') && i + 1 < lines.length && isDelimiterRow(lines[i + 1]) && cells(lines[i]).length === cells(lines[i + 1]).length; } + function table(lines, i, parent) { + const head = cells(lines[i]), align = cells(lines[i + 1]).map(cell => cell.startsWith(':') && cell.endsWith(':') ? 'center' : cell.endsWith(':') ? 'right' : ''); + const wrap = h('div', 'tbl'), t = h('table'), thead = h('thead'), tr = h('tr'); + head.forEach((cell, k) => { const th = h('th'); if (align[k]) th.style.textAlign = align[k]; inline(cell, th); tr.appendChild(th); }); + thead.appendChild(tr); t.appendChild(thead); + const tbody = h('tbody'); i += 2; + for (; i < lines.length && !BLANK.test(lines[i]) && lines[i].includes('|'); i++) { + const row = cells(lines[i]), r = h('tr'); + for (let k = 0; k < head.length; k++) { const td = h('td'); if (align[k]) td.style.textAlign = align[k]; inline(row[k] || '', td); r.appendChild(td); } + tbody.appendChild(r); + } + t.appendChild(tbody); wrap.appendChild(t); parent.appendChild(wrap); + return i; + } + + // ---- Lists -------------------------------------------------------------- + function list(lines, i, parent) { + const first = ITEM.exec(lines[i]), base = first[1].length, ordered = /\d/.test(first[2]); + const node = h(ordered ? 'ol' : 'ul'); + if (ordered && parseInt(first[2], 10) !== 1) node.start = parseInt(first[2], 10); + let loose = false; + while (i < lines.length) { + const m = ITEM.exec(lines[i]); + if (!m || m[1].length !== base || /\d/.test(m[2]) !== ordered) break; + const offset = m[1].length + m[2].length + Math.min(Math.max(m[3].length, 1), 4); + const body = [m[4]]; i++; + let blankRun = 0; + while (i < lines.length) { + const line = lines[i]; + if (BLANK.test(line)) { blankRun++; body.push(''); i++; continue; } + const indent = indentOf(line); + if (indent >= offset || (indent > base && !ITEM.test(line))) { body.push(line.slice(Math.min(indent, offset))); blankRun = 0; i++; continue; } + if (indent > base && ITEM.test(line)) { body.push(line.slice(Math.min(indent, offset))); blankRun = 0; i++; continue; } + if (!blankRun && !ITEM.test(line) && !FENCE.test(line) && !HEADING.test(line) && !QUOTE.test(line) && !RULE.test(line)) { body.push(line.trim()); i++; continue; } // lazy continuation + break; + } + while (body.length && body[body.length - 1] === '') body.pop(); + if (blankRun && i < lines.length && ITEM.test(lines[i])) loose = true; + const li = h('li'); blocks(body, li); node.appendChild(li); + } + if (!loose) for (const li of node.children) { // a tight item shows its first paragraph without the paragraph box + const p = li.firstElementChild; + if (p && p.tagName === 'P') { while (p.firstChild) li.insertBefore(p.firstChild, p); p.remove(); } + } + parent.appendChild(node); + return i; + } + + // ---- Blocks ------------------------------------------------------------- + function interrupts(lines, i) { + const line = lines[i]; + return FENCE.test(line) || HEADING.test(line) || RULE.test(line) || QUOTE.test(line) || ITEM.test(line) || startsTable(lines, i); + } + function blocks(lines, parent) { + let i = 0; + while (i < lines.length) { + const line = lines[i]; + if (BLANK.test(line)) { i++; continue; } + let m = FENCE.exec(line); + if (m) { + const indent = m[1].length, mark = m[2], body = []; + i++; + while (i < lines.length) { + const close = FENCE.exec(lines[i]); + if (close && close[2][0] === mark[0] && close[2].length >= mark.length && !close[3].trim()) { i++; break; } + body.push(lines[i].slice(Math.min(indentOf(lines[i]), indent))); i++; + } + const pre = h('pre'), code = h('code'); code.textContent = body.join('\n'); + const language = m[3].trim().split(/\s+/)[0]; if (language) code.dataset.language = language; + pre.appendChild(code); parent.appendChild(pre); continue; + } + m = HEADING.exec(line); + if (m) { const n = h('h' + m[1].length); inline(m[2] || '', n); parent.appendChild(n); i++; continue; } + if (RULE.test(line)) { parent.appendChild(h('hr')); i++; continue; } + if (QUOTE.test(line)) { + const body = []; + while (i < lines.length && !BLANK.test(lines[i]) && (QUOTE.test(lines[i]) || !interrupts(lines, i))) { body.push(lines[i].replace(QUOTE, '')); i++; } + const quote = h('blockquote'); blocks(body, quote); parent.appendChild(quote); continue; + } + if (startsTable(lines, i)) { i = table(lines, i, parent); continue; } + if (ITEM.test(line)) { i = list(lines, i, parent); continue; } + const body = [line.trim()]; i++; + let level = 0; + while (i < lines.length && !BLANK.test(lines[i])) { + const s = SETEXT.exec(lines[i]); + if (s) { level = s[1][0] === '=' ? 1 : 2; i++; break; } + if (interrupts(lines, i)) break; + body.push(lines[i].replace(/^\s+/, '')); i++; + } + const n = h(level ? 'h' + level : 'p'); inline(body.join('\n'), n); parent.appendChild(n); + } + } + + /** A div.card-md holding text rendered as elements. */ + function render(text) { + const root = h('div', 'card-md'); + blocks(String(text).replace(/\r\n?/g, '\n').split('\n').map(detab), root); + return root; + } + return { render }; +})(); diff --git a/static/project-workspace/parallel.js b/static/project-workspace/parallel.js new file mode 100644 index 00000000..50e6a433 --- /dev/null +++ b/static/project-workspace/parallel.js @@ -0,0 +1,218 @@ +'use strict'; +// Parallel coordinates on the Benchmarks tab (docs/plans/training-and-curation/spec-t3.md section 5.2): one vertical +// axis per metric of the benchmark and per recipe setting that varies between its subjects (benchmark_axes), one +// line per subject in its kind of change's colour. The person chooses the axes (at most twelve; the metrics and the +// six settings that go most with the first metric to begin with), moves them left and right, puts a metric whose +// values are all positive on a logarithmic scale, and drags a range on any axis: the lines outside every range are +// faint, the subjects table hides their rows and the charts above draw their points faint. A click on an axis +// without a drag clears its range; "Clear ranges" clears all. +// +// Updating in place (spec-w4): the section is built with the tab (tabs.js's paint) from the tab's settings +// (pcAxes, pcLog, pcBrush, kept per benchmark in state.ui), so a reading that changed something draws it again where +// the person left it. The person's own moves touch only this section (drawn again in place) and, for a range, the +// table's rows and the charts' marks (apply()); nothing is read again. +window.LabParallel = (() => { + const C = window.LabCharts, h = C.h, el = C.el, S = window.LabScreens; + const MOST = 12, SETTINGS_FIRST = 6, H = 300, T = 46, B = 30; + const fmt = C.fmtSci; + let last = null; // the benchmark drawn last: { benchmark, data, axes } (apply() reads it) + const button = (cls, text, onClick, key, label) => { const b = h('button', cls, text); b.type = 'button'; if (key) b.dataset.key = key; if (label) b.setAttribute('aria-label', label); if (onClick) b.addEventListener('click', onClick); return b; }; + const tail = path => { const m = path.match(/([^.\[\]]+|\[[^\]]*\])$/); return m ? m[1].replace(/^\["?|"?\]$/g, '') : path; }; + + /** Every axis the answer offers, metrics first: id, kind (metric, number, category), titles and range. */ + function axesOf(data) { + const out = data.metrics.map(m => ({ id: 'm:' + m.id, kind: 'metric', title: m.title, short: m.title, low: m.low, high: m.high, positive: m.positive, better: m.better, units: m.units, key: m.id })); + for (const s of data.settings) out.push({ id: 's:' + s.path, kind: s.kind, title: s.path, short: tail(s.path), low: s.low, high: s.high, categories: s.categories, association: s.association, positive: s.kind === 'number' && s.low > 0, key: s.path, count: s.count }); + return out; + } + const defaults = all => all.filter(a => a.kind === 'metric').concat(all.filter(a => a.kind !== 'metric').slice(0, SETTINGS_FIRST)).slice(0, MOST).map(a => a.id); + /** The axes shown, in the person's order: those of the tab's settings that this benchmark has, else the defaults. */ + function shownAxes(all, ui) { + const byId = new Map(all.map(a => [a.id, a])); + const kept = Array.isArray(ui.pcAxes) ? ui.pcAxes.filter(id => byId.has(id)).slice(0, MOST) : []; + return (kept.length ? kept : defaults(all)).map(id => byId.get(id)); + } + const valueOf = (row, a) => { const v = a.kind === 'metric' ? row.metrics[a.key] : row.settings[a.key]; return v == null ? null : v; }; + const isLog = (a, ui) => a.positive && (ui.pcLog || []).includes(a.id) && a.low > 0 && a.high > a.low; + /** The axis's scale: t(v) in [0, 1] from its low to its high end, and back. */ + function scaleOf(a, ui) { + const lo = a.low, hi = a.high; + if (lo == null || hi == null) return { t: () => 0.5, v: () => null }; + if (isLog(a, ui)) { const l0 = Math.log10(lo), l1 = Math.log10(hi); return { t: v => (Math.log10(Math.max(v, lo)) - l0) / (l1 - l0), v: t => Math.pow(10, l0 + t * (l1 - l0)) }; } + if (hi === lo) return { t: () => 0.5, v: () => lo }; + return { t: v => (v - lo) / (hi - lo), v: t => lo + t * (hi - lo) }; + } + /** The ranges drawn, of axes that exist: {id: [low, high]}. */ + function rangesOf(all, ui) { const ids = new Set(all.map(a => a.id)), out = {}; for (const [id, r] of Object.entries(ui.pcBrush || {})) if (ids.has(id) && Array.isArray(r) && r.length === 2) out[id] = r; return out; } + /** The names of the subjects inside every range, or null when no range is drawn. */ + function keptOf(data, all, ui) { + const ranges = rangesOf(all, ui), ids = Object.keys(ranges); if (!ids.length) return null; + const byId = new Map(all.map(a => [a.id, a])), keep = new Set(); + for (const row of data.subjects) if (ids.every(id => { const v = valueOf(row, byId.get(id)); return v != null && v >= ranges[id][0] && v <= ranges[id][1]; })) keep.add(row.subject); + return keep; + } + + // ---- The section ------------------------------------------------------------------------------------------------ + function section(data, view, state, act, width) { + const ui = state.ui; + if (ui.pcFor !== data.benchmark) { ui.pcFor = data.benchmark; ui.pcAxes = null; ui.pcLog = null; ui.pcBrush = null; } // the settings are per benchmark + const all = axesOf(data); + last = { benchmark: data.benchmark, data, all }; + const sec = h('section', 'pc'); sec.dataset.key = 'pc'; + const ctx = { data, view, state, act, width, sec, all }; + fill(ctx); + return sec; + } + /** Builds the section's content (put in place when it is on the page). */ + function fill(ctx) { + const { data, state, sec, all } = ctx, ui = state.ui, axes = shownAxes(all, ui), keep = keptOf(data, all, ui); + const items = []; + const head = S.parts.sect('Settings and metrics', data.subjects.length + ' ' + (ctx.view.noun || 'subjects') + ' · ' + axes.length + ' of ' + all.length + ' axes'); + const count = h('span', 'pc-count small'); count.setAttribute('aria-live', 'polite'); head.appendChild(count); ctx.count = count; + items.push(head); + items.push(h('p', 'small faint bench-desc', ctx.view.lead ? ctx.view.lead + (data.left_out ? ' ' + data.left_out + ' more settings are not offered.' : '') : 'One line per subject through its metrics (the ' + data.statistic + ' over its cases) and the recipe settings that vary between subjects. Drag along an axis to keep the subjects inside a range; the table and the charts above follow. ' + (data.left_out ? data.left_out + ' more settings are not offered.' : ''))); + const host = h('div', 'pc-draw'); ctx.host = host; items.push(host); + items.push(axisControls(ctx, axes)); + items.push(chooser(ctx, axes)); + draw(ctx, axes, keep); + setCount(ctx, keep); + if (sec.isConnected) { const frag = document.createDocumentFragment(); frag.append(...items); S.parts.putInPlace(sec, frag, document.getElementById('main')); } + else sec.replaceChildren(...items); + } + function setCount(ctx, keep) { + const n = ctx.data.subjects.length, text = keep ? keep.size + ' of ' + n + ' inside the ranges' : 'no range drawn'; + if (ctx.count.textContent !== text) ctx.count.textContent = text; + ctx.count.dataset.kept = keep ? String(keep.size) : ''; + } + + /** The SVG: the axes, their names and ends, the lines, the ranges, and the axes' hit areas for dragging. */ + function draw(ctx, axes, keep) { + const { data, view, state, host } = ctx, ui = state.ui, W = Math.max(320, ctx.width), k = axes.length; + const L = 30, R = 30, X = i => k > 1 ? L + i * (W - L - R) / (k - 1) : W / 2, Y = t => T + (1 - t) * (H - T - B); + const gap = k > 1 ? (W - L - R) / (k - 1) : W, chars = Math.max(4, Math.floor(gap / 6.4) - 1); + const cut = (t, n) => t.length > n ? t.slice(0, Math.max(1, n - 1)) + '…' : t; + const svg = el('svg', { viewBox: `0 0 ${W} ${H}`, role: 'group', 'aria-label': 'Parallel coordinates: ' + data.subjects.length + ' subjects through ' + k + ' axes: ' + axes.map(a => a.short).join(', '), class: 'pc-svg' }); + const scales = axes.map(a => scaleOf(a, ui)); + // The lines. + const lines = el('g', { class: 'pc-lines' }); + const sel = view.selected !== undefined ? view.selected : state.sel.benchmarks || null; // a sweep's view gives its own + for (const row of data.subjects) { + const s = view.byName.get(row.subject), kind = s ? s.kind : 'other'; + let d = '', open = false; + axes.forEach((a, i) => { const v = valueOf(row, a); if (v == null || !Number.isFinite(v)) { open = false; return; } d += (open ? 'L' : 'M') + X(i).toFixed(1) + ',' + Y(scales[i].t(v)).toFixed(1); open = true; }); + if (!d) continue; + const on = s && s.key === sel; + const p = el('path', { d, class: 'pcl ' + kind + (keep && !keep.has(row.subject) ? ' out' : '') + (on ? ' on' : ''), 'data-subject': row.subject }); + p.appendChild(el('title', {}, (s ? s.full : row.title) + ': ' + axes.map(a => a.short + ' ' + shownValue(a, valueOf(row, a))).join(', '))); + lines.appendChild(p); + } + svg.appendChild(lines); + // The axes. + const ranges = rangesOf(ctx.all, ui); + axes.forEach((a, i) => { + const x = X(i), g = el('g', { class: 'pc-axis', 'data-axis': a.id }), right = i < k - 1 || k === 1, anchor = right ? 'start' : 'end', dx = right ? 5 : -5; + g.appendChild(el('line', { x1: x, y1: Y(0), x2: x, y2: Y(1), class: 'pc-line' })); + const title = el('text', { x, y: 14, 'text-anchor': i === 0 && k > 1 ? 'start' : i === k - 1 && k > 1 ? 'end' : 'middle', class: 'pc-title' }, cut(a.short, chars)); + title.appendChild(el('title', {}, a.title + (a.kind === 'metric' ? '' : ' (' + a.kind + ')'))); + g.appendChild(title); + const sub = a.kind === 'metric' ? (a.better === 'lower' ? 'lower better' : a.better === 'higher' ? 'higher better' : '') + (isLog(a, ui) ? ' · log' : '') : a.association == null ? '' : 'ρ ' + a.association.toFixed(2); + g.appendChild(el('text', { x, y: 28, 'text-anchor': i === 0 && k > 1 ? 'start' : i === k - 1 && k > 1 ? 'end' : 'middle', class: 'pc-sub' }, cut(sub, chars))); + if (a.kind === 'category') { + const n = a.categories.length, every = Math.max(1, Math.ceil(n / Math.max(1, Math.floor((H - T - B) / 13)))); + a.categories.forEach((c, j) => { if (j % every && j !== n - 1) return; g.appendChild(el('text', { x: x + dx, y: Y(n > 1 ? j / (n - 1) : 0.5) + 4, 'text-anchor': anchor, class: 'pc-tick' }, cut(typeof c === 'string' ? c : JSON.stringify(c), Math.min(12, chars)))); }); + } else if (a.low != null) { + g.appendChild(el('text', { x: x + dx, y: Y(1) + 4, 'text-anchor': anchor, class: 'pc-tick' }, fmt(a.high))); + g.appendChild(el('text', { x: x + dx, y: Y(0) + 4, 'text-anchor': anchor, class: 'pc-tick' }, fmt(a.low))); + } + const r = ranges[a.id]; + if (r) { const y0 = Y(scales[i].t(r[1])), y1 = Y(scales[i].t(r[0])); g.appendChild(el('rect', { x: x - 7, y: Math.min(y0, y1), width: 14, height: Math.max(2, Math.abs(y1 - y0)), class: 'pc-brush', 'data-low': r[0], 'data-high': r[1] })); } + const hit = el('rect', { x: x - 12, y: Y(1) - 6, width: 24, height: Y(0) - Y(1) + 12, class: 'pc-hit', 'data-axis': a.id }); + hit.appendChild(el('title', {}, 'Drag to keep a range of ' + a.title + '; a click clears it')); + g.appendChild(hit); + brushing(ctx, hit, g, a, scales[i], Y, x); + svg.appendChild(g); + }); + host.replaceChildren(svg); + ctx.svg = svg; + } + const shownValue = (a, v) => v == null ? '—' : a.kind === 'category' ? (c => typeof c === 'string' ? c : JSON.stringify(c))(a.categories[v]) : fmt(v); + + /** A range dragged on an axis: kept in the tab's settings, the lines, the table and the charts set in place. */ + function brushing(ctx, hit, g, a, scale, Y, x) { + let start = null, box = null; + const at = ev => { const pt = ctx.svg.createSVGPoint(); pt.x = ev.clientX; pt.y = ev.clientY; return pt.matrixTransform(ctx.svg.getScreenCTM().inverse()).y; }; + const clamp = y => Math.min(Y(0), Math.max(Y(1), y)); + hit.addEventListener('pointerdown', ev => { + ev.preventDefault(); start = clamp(at(ev)); try { hit.setPointerCapture(ev.pointerId); } catch (e) { /* a synthetic pointer */ } + box = el('rect', { x: x - 7, y: start, width: 14, height: 0, class: 'pc-brush pc-dragging' }); g.insertBefore(box, hit); + }); + hit.addEventListener('pointermove', ev => { if (start === null) return; const y = clamp(at(ev)); box.setAttribute('y', String(Math.min(start, y))); box.setAttribute('height', String(Math.abs(y - start))); }); + const end = ev => { + if (start === null) return; + const y = clamp(at(ev)), moved = Math.abs(y - start) >= 3, ui = ctx.state.ui, ranges = Object.assign({}, ui.pcBrush || {}); + const y0 = start; start = null; + if (moved) { const t0 = (Y(0) - Math.max(y, y0)) / (Y(0) - Y(1)), t1 = (Y(0) - Math.min(y, y0)) / (Y(0) - Y(1)); ranges[a.id] = [scale.v(t0), scale.v(t1)]; } + else delete ranges[a.id]; + ctx.act.keepUI('pcBrush', Object.keys(ranges).length ? ranges : null); + fill(ctx); apply(document.getElementById('main'), ctx.view, ctx.state); + }; + hit.addEventListener('pointerup', end); + hit.addEventListener('pointercancel', () => { start = null; if (box) box.remove(); }); + } + + /** The chosen axes in order, each moved left or right, put on a log scale or taken away; and Clear ranges. */ + function axisControls(ctx, axes) { + const ui = ctx.state.ui, box = h('div', 'pc-axes'), ids = axes.map(a => a.id); + const set = (k, v) => { ctx.act.keepUI(k, v); fill(ctx); apply(document.getElementById('main'), ctx.view, ctx.state); }; + axes.forEach((a, i) => { + const chip = h('span', 'pc-ax' + (a.kind === 'metric' ? ' metric' : '')); chip.dataset.axis = a.id; + const move = (d) => { const next = ids.slice(); next.splice(i, 1); next.splice(i + d, 0, a.id); set('pcAxes', next); }; + const left = button('pc-mv', '◀', () => move(-1), 'pc-left-' + a.id, 'Move ' + a.short + ' left'); left.disabled = i === 0; chip.appendChild(left); + const name = h('span', 'pc-name', a.short); name.title = a.title; chip.appendChild(name); + const right = button('pc-mv', '▶', () => move(1), 'pc-right-' + a.id, 'Move ' + a.short + ' right'); right.disabled = i === axes.length - 1; chip.appendChild(right); + if (a.positive && a.kind !== 'category') { const on = isLog(a, ui), lg = button('pc-log', 'log', () => { const list = (ui.pcLog || []).filter(x => x !== a.id); if (!on) list.push(a.id); set('pcLog', list); }, 'pc-log-' + a.id, (on ? 'Linear scale for ' : 'Logarithmic scale for ') + a.short); lg.setAttribute('aria-pressed', String(on)); chip.appendChild(lg); } + const x = button('pc-x', '×', () => set('pcAxes', ids.filter(id => id !== a.id)), 'pc-drop-' + a.id, 'Take ' + a.short + ' away'); x.disabled = axes.length <= 2; chip.appendChild(x); + box.appendChild(chip); + }); + const clear = button('btn sm ghost', 'Clear ranges', () => set('pcBrush', null), 'pc-clear'); clear.disabled = !Object.keys(rangesOf(ctx.all, ui)).length; box.appendChild(clear); + return box; + } + /** Every axis on offer, the settings with how strongly they go with the first metric; at most twelve on. */ + function chooser(ctx, axes) { + const ui = ctx.state.ui, on = new Set(axes.map(a => a.id)), d = h('details', 'pc-choose'); d.dataset.key = 'pc-choose'; + if (ui.pcChooser) d.open = true; + d.addEventListener('toggle', () => { ctx.state.ui.pcChooser = d.open; }); + d.appendChild(h('summary', null, 'Choose axes · ' + on.size + ' of at most ' + MOST)); + const ul = h('ul', 'pc-list'); + for (const a of ctx.all) { + const li = h('li'), l = h('label'), cb = h('input'); cb.type = 'checkbox'; cb.checked = on.has(a.id); cb.disabled = !cb.checked && on.size >= MOST || cb.checked && on.size <= 2; cb.dataset.key = 'pc-pick-' + a.id; + cb.addEventListener('change', () => { const ids = axes.map(x => x.id); ctx.act.keepUI('pcAxes', cb.checked ? ids.concat([a.id]) : ids.filter(id => id !== a.id)); fill(ctx); apply(document.getElementById('main'), ctx.view, ctx.state); }); + l.appendChild(cb); l.appendChild(h('span', a.kind === 'metric' ? null : 'mono', a.title)); + l.appendChild(h('span', 'small faint', a.kind === 'metric' ? ' metric' : ' ' + a.kind + (a.association == null ? '' : ' · ρ ' + a.association.toFixed(2)) + ' · ' + a.count + ' subjects')); + li.appendChild(l); ul.appendChild(li); + } + d.appendChild(ul); + return d; + } + + /** + * Sets the ranges drawn on the axes into the rest of the tab, under root (the main area, or the tab's content + * while it is built): the subjects table hides the rows outside them and says how many it shows, and the + * charts' marks of those subjects are faint. Nothing is read; nothing else is drawn. + */ + function apply(root, view, state) { + const keep = last && last.benchmark === view.name ? keptOf(last.data, last.all, state.ui) : null; + for (const tr of root.querySelectorAll('.bench-all tr[data-subject]')) { const hide = !!keep && !keep.has(tr.dataset.subject); if (tr.hidden !== hide) tr.hidden = hide; } + for (const mark of root.querySelectorAll('.bench-chart [data-id]')) { const s = view.byKey.get(mark.dataset.id), out = !!keep && !!s && !keep.has(s.name); mark.parentNode.classList.toggle('pc-out', out); } + const table = root.querySelector('.bench-all'); + if (table) { + let note = table.querySelector('.pc-filtered'); + const shown = keep ? [...table.querySelectorAll('tr[data-subject]')].filter(tr => !tr.hidden).length : null; + if (keep && !note) { note = h('p', 'small muted pc-filtered'); const sect = table.querySelector('.sect'); if (sect) sect.after(note); else table.prepend(note); } + if (keep) { const text = shown + ' rows shown: the subjects inside the ranges drawn on the axes of Settings and metrics.'; if (note.textContent !== text) note.textContent = text; note.dataset.shown = String(shown); } + else if (note) note.remove(); + } + } + + return { section, apply }; +})(); diff --git a/static/project-workspace/review.css b/static/project-workspace/review.css new file mode 100644 index 00000000..c97c036c --- /dev/null +++ b/static/project-workspace/review.css @@ -0,0 +1,35 @@ +/* Review mode of the Annotate tab (docs/plans/training-and-curation/spec-t4.md section 7). The three columns are + the Annotate tab's own (.ann-body, .ann-list, .ann-view, .ann-form in base.css); only what review adds is here. */ +.ann-mode { flex: 0 0 auto; } +.rv-picks { display: grid; grid-template-columns: minmax(0, 1fr); gap: 6px; margin-bottom: 8px; } +.rv-pick, .rv-min { display: grid; grid-template-columns: 2.2em minmax(0, 1fr); gap: 6px; align-items: center; font-size: 12.5px; color: var(--ink-2); min-width: 0; } +.rv-min { grid-template-columns: auto 5.5em; margin: 8px 0 12px; } +.rv-pick select { min-width: 0; width: 100%; } +.rv-mode { margin: 4px 0; } +.rv-agree { border: 1px solid var(--line); border-radius: 8px; padding: 10px 12px; margin-bottom: 14px; background: var(--surface); min-width: 0; } +.rv-agree .rv-score { display: block; font-size: 28px; line-height: 1.1; margin: 2px 0 4px; color: var(--ink); } +.rv-figures { margin: 8px 0 0; } +.rv-disputed { list-style: none; margin: 0; padding: 0; display: grid; gap: 4px; max-height: 60vh; overflow-y: auto; } +.rv-rec { display: grid; grid-template-columns: minmax(0, 1fr) 64px 3.2em; grid-template-areas: "id bar n" "miss miss miss"; gap: 2px 8px; align-items: center; width: 100%; + text-align: left; padding: 6px 8px; border: 1px solid var(--line); border-radius: 6px; background: transparent; color: var(--ink); font: inherit; cursor: pointer; min-width: 0; } +.rv-rec[aria-pressed="true"] { border-color: var(--accent); background: var(--accent-wash); } +.rv-rec .rv-id { grid-area: id; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: 12.5px; } +.rv-rec .rv-bar { grid-area: bar; display: block; height: 6px; border-radius: 3px; background: var(--line); overflow: hidden; } +.rv-rec .rv-bar b { display: block; height: 100%; background: var(--serious); } +.rv-rec .rv-dis { grid-area: n; text-align: right; } +.rv-rec .faint { grid-area: miss; } +.rv-legend { display: flex; flex-wrap: wrap; align-items: center; gap: 4px; margin: 0 0 8px; } +.rv-sw { display: inline-block; width: 12px; height: 12px; border-radius: 2px; margin-right: 2px; } +.rv-panels { display: grid; grid-template-columns: minmax(0, 1fr); gap: 12px; } +.rv-panels.two { grid-template-columns: repeat(2, minmax(0, 1fr)); } +@container (max-width: 700px) { .rv-panels.two { grid-template-columns: minmax(0, 1fr); } } +.rv-panel { margin: 0; min-width: 0; } +.rv-panel figcaption { margin-bottom: 4px; color: var(--ink-2); overflow-wrap: anywhere; } +.rv-canvas { display: block; width: 100%; border: 1px solid var(--line); border-radius: 4px; background: var(--surface); } +.rv-canvas:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; } +.rv-frame { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin-top: 10px; font-size: 12.5px; color: var(--ink-2); } +.rv-frame input { flex: 1 1 160px; min-width: 0; } +.rv-values dd { overflow-wrap: anywhere; white-space: pre-wrap; } +.rv-decide .seg, .rv-decide .actions { flex-wrap: wrap; } +.rv-kept { margin: 8px 0 12px; overflow-wrap: anywhere; } +.rv-correct { border-top: 1px solid var(--line); padding-top: 10px; } diff --git a/static/project-workspace/review.js b/static/project-workspace/review.js new file mode 100644 index 00000000..ae0c6f14 --- /dev/null +++ b/static/project-workspace/review.js @@ -0,0 +1,467 @@ +'use strict'; +// Review mode of the Annotate tab (docs/plans/training-and-curation/spec-t4.md section 7): two of a task's annotation +// sets (the kept set and the annotators' products) compared. Left: the set choosers, side by side or overlaid, the +// confidence filter, the agreement of the two and the records they disagree on most. Middle: the record with each +// set drawn over it (masks, polygons, boxes, keypoints and depth on an image; point objects and 3-D boxes on a point +// cloud; timeseries and segments as bands under a sound or a series; tracks on an array's frames with a frame +// slider). Right: each set's value in words, this record's agreement, and the decision (Keep A, Keep B, Keep both, +// Reject both), kept with keep_annotation as an unsaved edit that Save writes. A light correction of the kept value +// (a polygon's vertices moved or deleted, points removed from a point object) is kept with set_annotation. +// +// Updating in place (spec-w4): the three parts are each built off the page and put in place only when what they show +// differs; the selection is set in place; the frame slider repaints the canvases without drawing anything again. +// The choices (sets, mode, filter, record, frame, a correction under way) are the page's and survive readings. +window.LabReview = (() => { + const P = window.TuldokProject, S = window.LabScreens, A = window.LabActions, C = window.LabCharts, h = C.h, V = window.LabInside.viewers, D = window.LabInside.drawing; + const { sect, putInPlace } = S.parts; + const NEW = ['polygons', 'mask', 'depth', 'keypoints', 'ranking', 'timeseries', 'track', 'points', 'box3d']; + const LISTS = ['boxes', 'spans', 'segments', 'polygons', 'keypoints', 'timeseries', 'track', 'box3d']; + const KEPT = 'kept'; + const count = (n, one, many) => n.toLocaleString('en-GB') + ' ' + (n === 1 ? one : many || one + 's'); + const button = (cls, text, key, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (key) b.dataset.key = key; if (onClick) b.addEventListener('click', onClick); return b; }; + const rgb = (c, a) => a == null ? 'rgb(' + c.join(',') + ')' : 'rgba(' + c.join(',') + ',' + a + ')'; + const fixed = (v, n = 2) => v == null || !Number.isFinite(v) ? '—' : v.toFixed(n); + const shortSet = name => name === KEPT ? 'kept' : name.slice(name.indexOf('/') + 1); + const copy = v => v == null ? v : JSON.parse(JSON.stringify(v)); + + // ---- What is kept in this page ------------------------------------------------------------------------------- + const ui = { pick: new Map(), mode: 'side', min: new Map(), sel: new Map(), frame: 0, editing: null, removing: false, notes: new Map() }; + const store = new Map(); // the 3-D views' settings, per record and panel + let here = null; // { box, t, keys, parts, repaint: [] } + + const keepPaint = paint => { if (here && here.repaint) here.repaint.push(paint); }; + /** Is list value: the types whose value may be two sets' together. */ + const isList = (t, v) => Array.isArray(v) && (LISTS.includes(t.type) || t.type === 'points'); + function choose(t, sets) { + const usable = sets.filter(s => !s.problem && (s.annotated || s.set === KEPT)).map(s => s.set); + let [a, b] = ui.pick.get(t.task) || []; + const made = usable.filter(n => n !== KEPT); + if (!usable.includes(a)) a = made[0] || usable[0]; + if (!usable.includes(b) || b === a) b = made.find(n => n !== a) || usable.find(n => n !== a); + ui.pick.set(t.task, [a, b]); + return { a, b, usable }; + } + + async function load(t, sets) { + const { a, b, usable } = choose(t, sets), min = ui.min.get(t.task); + if (!a || !b) return { a, b, usable, agreement: null }; + const args = { task: t.task, sets: [a, b], limit: 200 }; if (min != null) args.min_confidence = min; + const agreement = await P.read('annotation_agreement', args); + let fleiss = null; + if (t.type === 'class' && usable.length >= 3) fleiss = await P.read('annotation_agreement', { task: t.task, sets: usable.slice(0, 8), limit: 1 }).then(r => r.fleiss, () => null); + let id = ui.sel.get(t.task); + if (!id || !agreement.disputed.some(r => r.record === id)) id = agreement.disputed.length ? agreement.disputed[0].record : null; + if (id) ui.sel.set(t.task, id); + let compare = null, record = null, kept = null; + if (id) { + const names = [a, b].concat(a !== KEPT && b !== KEPT ? [KEPT] : []), cargs = { task: t.task, record: id, sets: names }; if (min != null) cargs.min_confidence = min; + [compare, kept] = await Promise.all([P.read('annotation_compare', cargs), P.annotations.one(t.task, id).catch(() => null)]); + if (compare.in_product) record = await P.product.record(t.run, id).catch(error => ({ error })); + } + return { a, b, usable, agreement, fleiss, id, compare, record, kept }; + } + + /** Draws review mode into box: each part put in place only when its content differs. */ + async function draw(box, t, sets, act) { + const mine = (box._review = (box._review || 0) + 1); + if (!here || here.box !== box) here = { box, keys: {}, parts: null, repaint: [] }; + here.act = act; here.t = t; + let got; try { got = await load(t, sets); } catch (error) { got = { error }; } + if (box._review !== mine) return; + const may = A.may('edit'); + const keys = { + left: JSON.stringify([t.task, got.error ? got.error.message : null, got.a, got.b, got.usable, ui.mode, ui.min.get(t.task), got.agreement, got.fleiss]), + middle: JSON.stringify([t.task, got.id, got.compare, got.record && (got.record.error ? 'error' : got.record.record), ui.mode, ui.editing && [ui.editing.key, ui.editing.version], ui.removing]), + right: JSON.stringify([t.task, got.id, got.compare && got.compare.sets.map(s => [s.set, s.value, s.confidence]), got.compare && got.compare.pairs, got.kept && [got.kept.annotation, got.kept.unsaved], may, ui.editing && [ui.editing.key, ui.editing.dirty, ui.editing.removed], ui.removing, got.a, got.b]) + }; + if (!here.parts) { + const body = h('div', 'ann-body rv-body'); + here.parts = { left: h('div', 'ann-list rv-left'), middle: h('div', 'ann-view viewer rv-middle'), right: h('div', 'ann-form rv-right') }; + here.parts.middle.setAttribute('aria-label', 'The record and its annotations'); + for (const p of ['left', 'middle', 'right']) body.appendChild(here.parts[p]); + box.replaceChildren(body); + } + here.got = got; + const put = (name, make) => { if (here.keys[name] === keys[name]) return; here.keys[name] = keys[name]; putInPlace(here.parts[name], make(), box.closest('main') || box); }; + put('left', () => buildLeft(t, sets, got)); + put('middle', () => { here.repaint = []; return buildMiddle(t, got); }); + put('right', () => buildRight(t, got, may)); + for (const b of here.parts.left.querySelectorAll('.rv-rec')) { const on = String(b.dataset.id === got.id); if (b.getAttribute('aria-pressed') !== on) b.setAttribute('aria-pressed', on); } + } + const again = () => { if (here && here.box && here.box.isConnected) window.LabAnnotate.redraw(); }; + + // ---- Left: choosers, agreement, the disputed list ------------------------------------------------------------- + function buildLeft(t, sets, got) { + const box = h('div', 'rv-leftbox'); + if (got.error) { box.appendChild(A.refusal(got.error)); return box; } + const pick = (label, which, value) => { + const l = h('label', 'rv-pick'); l.appendChild(document.createTextNode(label)); + const s = h('select'); s.dataset.key = 'rv:set-' + which; + for (const name of got.usable) { const found = sets.find(x => x.set === name) || {}; const o = h('option', null, shortSet(name) + (found.by && found.by.name ? ' · ' + found.by.name : '') + ' (' + count(found.annotated || 0, 'record') + ')'); o.value = name; o.selected = name === value; s.appendChild(o); } + s.addEventListener('change', () => { const pair = ui.pick.get(t.task).slice(); pair[which === 'a' ? 0 : 1] = s.value; if (pair[0] === pair[1]) pair[which === 'a' ? 1 : 0] = got.usable.find(n => n !== s.value); ui.pick.set(t.task, pair); ui.editing = null; again(); }); + l.appendChild(s); return l; + }; + box.appendChild(sect('Compare', count(got.usable.length, 'set'))); + const picks = h('div', 'rv-picks'); picks.appendChild(pick('A', 'a', got.a)); picks.appendChild(pick('B', 'b', got.b)); box.appendChild(picks); + const seg = h('div', 'seg rv-mode'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'How the two sets are drawn'); + for (const [id, label] of [['side', 'Side by side'], ['over', 'Overlaid']]) { const b = button(null, label, 'rv:mode-' + id, () => { if (ui.mode !== id) { ui.mode = id; again(); } }); b.setAttribute('aria-pressed', String(ui.mode === id)); seg.appendChild(b); } + box.appendChild(seg); + const fl = h('label', 'rv-min'); fl.appendChild(document.createTextNode('Confidence at least')); + const fi = h('input'); fi.type = 'number'; fi.min = '0'; fi.max = '1'; fi.step = '0.05'; fi.placeholder = 'any'; fi.dataset.key = 'rv:min'; fi.className = 'ann-num'; + const min = ui.min.get(t.task); fi.value = min == null ? '' : String(min); + fi.addEventListener('change', () => { const v = fi.value === '' ? null : Number(fi.value); if (v != null && !(v >= 0 && v <= 1)) { fi.value = min == null ? '' : String(min); return; } ui.min.set(t.task, v); again(); }); + fl.appendChild(fi); box.appendChild(fl); + if (!got.agreement) { box.appendChild(h('p', 'empty', 'Two sets are needed to compare; build an annotator of this task.')); return box; } + const ag = got.agreement, pair = ag.pairs[0] || {}; + const card = h('div', 'rv-agree'); card.setAttribute('data-key', 'rv:agreement'); + card.appendChild(h('span', 'kicker', 'Agreement of A and B')); + card.appendChild(h('b', 'rv-score mono', fixed(pair.agreement))); + card.appendChild(h('p', 'small muted', 'over ' + count(pair.records || 0, 'record') + '. ' + ag.measure)); + const figures = []; + if (pair.kappa !== undefined) figures.push(['Cohen\'s κ', fixed(pair.kappa)]); + if (got.fleiss != null) figures.push(['Fleiss\' κ, all sets', fixed(got.fleiss)]); + if (pair.mean_iou !== undefined) figures.push(['Mean IoU of matches', fixed(pair.mean_iou)]); + if (pair.mean_oks !== undefined) figures.push(['Mean OKS of matches', fixed(pair.mean_oks)]); + if (pair.f1 !== undefined) figures.push([pair.mean_oks !== undefined ? 'F1 at OKS 0.5' : 'F1 at IoU 0.5', fixed(pair.f1)]); + if (pair.matches !== undefined) figures.push(['Matches', String(pair.matches)]); + if (pair.tau !== undefined) figures.push(['Mean Kendall τ', fixed(pair.tau)]); + if (pair.exact !== undefined) figures.push(['Exactly equal', String(pair.exact)]); + if (pair.mean_abs !== undefined) figures.push(['Mean |A − B|' + (t.units ? ' (' + t.units + ')' : ''), fixed(pair.mean_abs, 3)]); + if (pair.delta !== undefined) figures.push(['Within δ 1.25', fixed(pair.delta)]); + if (pair.switches !== undefined) figures.push(['Identity switches', String(pair.switches)], ['Fragments A · B', pair.fragments.join(' · ')]); + if (figures.length) { const dl = h('dl', 'setup rv-figures'); for (const [k, v] of figures) { dl.appendChild(h('dt', null, k)); dl.appendChild(h('dd', 'mono', v)); } card.appendChild(dl); } + if (ag.truncated) card.appendChild(h('p', 'small warn', 'A set holds more records than are compared here; the first 20,000 of each are.')); + box.appendChild(card); + box.appendChild(sect('Most disputed', ag.total ? count(ag.total, 'record') : '0')); + if (!ag.disputed.length) { box.appendChild(h('p', 'empty', 'Neither set annotates a record.')); return box; } + const ol = h('ol', 'rv-disputed'); ol.setAttribute('aria-label', 'The records A and B disagree on most, most first'); + for (const row of ag.disputed) { + const li = h('li'), b = button('rv-rec', null, 'rv-rec-' + row.record, () => { if (ui.sel.get(t.task) !== row.record) { ui.sel.set(t.task, row.record); ui.editing = null; ui.removing = false; again(); } }); + b.dataset.id = row.record; b.setAttribute('aria-pressed', 'false'); + b.appendChild(h('bdi', 'mono rv-id', row.record)); + const bar = h('i', 'rv-bar'), fill = h('b'); fill.style.width = Math.round(100 * row.disagreement) + '%'; bar.appendChild(fill); b.appendChild(bar); + b.appendChild(h('span', 'mono small rv-dis', fixed(row.disagreement))); + if (row.missing.length) b.appendChild(h('span', 'small faint', 'only in ' + ['A', 'B'].filter((x, i) => !row.missing.includes([got.a, got.b][i])).join(' '))); + b.setAttribute('aria-label', row.record + ', disagreement ' + fixed(row.disagreement) + (row.missing.length ? ', missing in ' + row.missing.map(shortSet).join(' and ') : '')); + li.appendChild(b); ol.appendChild(li); + } + box.appendChild(ol); + if (ag.total > ag.disputed.length) box.appendChild(h('p', 'small faint', 'The ' + ag.disputed.length + ' most disputed of ' + ag.total.toLocaleString('en-GB') + ' are listed.')); + return box; + } + + // ---- Colours ------------------------------------------------------------------------------------------------------ + const colourOf = which => V.token(which === 'a' ? '--ramp-neg' : which === 'b' ? '--ramp-pos' : '--good'); + const PALETTE = ['--ramp-neg', '--ramp-pos', '--good', '--warn', '--serious', '--accent']; + const termColour = (t, term) => { const i = Math.max(0, (t.vocabulary || []).indexOf(term)); return V.token(PALETTE[i % PALETTE.length]); }; + + // ---- Middle: the record with the sets drawn over it ----------------------------------------------------------- + function buildMiddle(t, got) { + const el = h('div', 'iv rv-iv'); + if (!got.id) { el.appendChild(h('p', 'empty', 'Choose a record.')); return el; } + const cmp = got.compare, byName = Object.fromEntries(cmp.sets.map(s => [s.set, s])); + el.appendChild(h('header', 'iv-head')).appendChild(h('h3', 'mono', got.id)); + if (!cmp.in_product) { el.appendChild(h('p', 'empty', 'This record is not in the current product of ' + t.dataset + '.')); return el; } + if (got.record && got.record.error) { el.appendChild(A.refusal(got.record.error)); return el; } + const legend = h('p', 'small rv-legend'); + const swatch = (which, text) => { const s = h('span', 'rv-sw'); s.style.background = rgb(colourOf(which)); legend.appendChild(s); legend.appendChild(document.createTextNode(text + ' ')); }; + if (ui.mode === 'over' && !ui.editing) { swatch('a', 'A ' + shortSet(got.a)); swatch('b', 'B ' + shortSet(got.b)); swatch('both', 'both'); legend.appendChild(h('span', 'faint', ' · dashed: only in one')); el.appendChild(legend); } + for (const s of cmp.sets) if (s.problem) el.appendChild(h('p', 'small warn', shortSet(s.set) + ': ' + s.problem)); + if (cmp.problem) el.appendChild(h('p', 'small warn', cmp.problem)); + const panels = ui.editing ? [['kept', [ui.editing.set]]] : ui.mode === 'over' ? [['over', [got.a, got.b]]] : [['a', [got.a]], ['b', [got.b]]]; + const grid = h('div', 'rv-panels' + (panels.length > 1 ? ' two' : '')); el.appendChild(grid); + const r = got.record ? got.record.record : null, k = got.record ? got.record.content || {} : {}; + for (const [which, names] of panels) { + const panel = h('figure', 'rv-panel'); grid.appendChild(panel); + const values = ui.editing ? [{ set: KEPT, value: ui.editing.value, drawn: null }] : names.map(n => byName[n]); + const cap = h('figcaption', 'small'); cap.textContent = ui.editing ? 'Correcting the kept annotation' : which === 'over' ? 'A and B' : which.toUpperCase() + ' · ' + shortSet(names[0]); panel.appendChild(cap); + const roles = ui.editing ? ['kept'] : which === 'over' ? ['a', 'b'] : [which]; + if (r && r.kind === 'image') imagePanel(panel, t, r, k, values, roles, cmp); + else if (r && r.kind === 'pointcloud') cloudPanel(panel, t, got.id, k, values, roles, which); + else if (cmp.frames) framesPanel(panel, t, cmp.frames, values, roles); + else if (cmp.series || (r && r.kind === 'audio')) bandsPanel(panel, t, cmp.series, k, values, roles); + else textPanel(panel, t, values, roles, r, k); + } + if (cmp.frames) el.appendChild(frameSlider(cmp.frames)); + return el; + } + + /** Matching for marking: an object of one side with no object of the same term at IoU 0.5 or more on the other. */ + function unmatched(objs, others) { + const iou = (a, b) => { const ix = Math.max(0, Math.min(a[0] + a[2], b[0] + b[2]) - Math.max(a[0], b[0])), iy = Math.max(0, Math.min(a[1] + a[3], b[1] + b[3]) - Math.max(a[1], b[1])), i = ix * iy, u = a[2] * a[3] + b[2] * b[3] - i; return u > 0 ? i / u : 0; }; + return objs.map(o => !others.some(p => p.term === o.term && iou(o.box, p.box) >= 0.5)); + } + const bboxOf = pts => { const xs = pts.filter(Boolean).map(p => p[0]), ys = pts.filter(Boolean).map(p => p[1]); if (!xs.length) return [0, 0, 0, 0]; const x0 = Math.min(...xs), y0 = Math.min(...ys); return [x0, y0, Math.max(...xs) - x0 || 1, Math.max(...ys) - y0 || 1]; }; + function objectsOf(t, value) { + if (!Array.isArray(value)) return []; + if (t.type === 'boxes') return value.map(o => ({ term: o.term, box: [o.x, o.y, o.w, o.h], o })); + if (t.type === 'polygons') return value.map(o => ({ term: o.term, box: bboxOf(o.points), o })); + if (t.type === 'keypoints') return value.map(o => ({ term: o.term, box: bboxOf(o.points), o })); + return []; + } + + function imagePanel(panel, t, r, k, values, roles, cmp) { + const cv = h('canvas', 'rv-canvas'); cv.tabIndex = 0; cv.dataset.key = 'rv:canvas-' + roles.join(''); cv.setAttribute('role', 'img'); + cv.setAttribute('aria-label', 'Image ' + r.id + ' with ' + roles.map(x => x === 'kept' ? 'the kept annotation' : 'set ' + x.toUpperCase()).join(' and ') + ' drawn over it'); + panel.appendChild(cv); + const img = new Image(); img.src = r.url; + const nw = k.width || (cmp.size || [])[0] || 1, nh = k.height || (cmp.size || [])[1] || 1; + const layers = values.map((s, i) => ({ s, role: roles[i], mask: s && s.drawn && s.drawn.runs ? maskPixels(s.drawn) : null })); + const objs = layers.map(l => objectsOf(t, l.s && l.s.value)); + const lone = layers.length === 2 ? [unmatched(objs[0], objs[1]), unmatched(objs[1], objs[0])] : [objs[0].map(() => false)]; + const edit = ui.editing && t.type === 'polygons' ? ui.editing : null; + // What is drawn, per set, as data-drawn (objects, or pixels of a mask or a depth map): read by the browser check. + cv.dataset.drawn = layers.map((l, i) => roles[i] + ':' + (l.mask ? l.mask.px.reduce((n, v) => n + (v ? 1 : 0), 0) : l.s && l.s.drawn && l.s.drawn.values_u8 ? atob(l.s.drawn.values_u8).split('').filter(c => c !== '\0').length : objs[i].length)).join(' '); + const paint = () => { + const size = V.fit(cv, Math.max(120, Math.round(cv.clientWidth * nh / nw))); if (!size) return; + const [W, H] = size, ctx = cv.getContext('2d'), sx = W / nw, sy = H / nh; + ctx.clearRect(0, 0, W, H); ctx.imageSmoothingEnabled = false; + if (img.complete && img.naturalWidth) ctx.drawImage(img, 0, 0, W, H); + if (t.type === 'mask') paintMasks(ctx, t, layers, W, H); + if (t.type === 'depth') paintDepth(ctx, layers, W, H); + const dpr = window.devicePixelRatio || 1; + layers.forEach((l, li) => { + if (!l.s || !Array.isArray(l.s.value)) return; + const col = roles.length > 1 ? colourOf(l.role) : null; + objs[li].forEach((ob, oi) => { + const c = col || termColour(t, ob.term), dashed = lone[li][oi] && roles.length > 1; + ctx.strokeStyle = rgb(c); ctx.fillStyle = rgb(c, 0.16); ctx.lineWidth = (dashed ? 2.5 : 1.6) * dpr; ctx.setLineDash(dashed ? [5 * dpr, 3 * dpr] : []); + const o = ob.o; + if (t.type === 'boxes') { ctx.strokeRect(o.x * sx, o.y * sy, o.w * sx, o.h * sy); } + else if (t.type === 'polygons') { ctx.beginPath(); o.points.forEach((p, i) => i ? ctx.lineTo(p[0] * sx, p[1] * sy) : ctx.moveTo(p[0] * sx, p[1] * sy)); ctx.closePath(); ctx.fill(); ctx.stroke(); } + else if (t.type === 'keypoints') { + const names = t.keypoints || [], at = n => o.points[names.indexOf(n)]; + ctx.setLineDash([]); for (const [m, n] of t.skeleton || []) { const p = at(m), q = at(n); if (p && q) { ctx.beginPath(); ctx.moveTo(p[0] * sx, p[1] * sy); ctx.lineTo(q[0] * sx, q[1] * sy); ctx.stroke(); } } + for (const p of o.points) if (p && (p.length < 3 || p[2] > 0)) { ctx.beginPath(); ctx.arc(p[0] * sx, p[1] * sy, 3 * dpr, 0, 7); ctx.fillStyle = rgb(c); ctx.fill(); } + } + ctx.setLineDash([]); ctx.fillStyle = rgb(c); ctx.font = (11 * dpr) + 'px ' + getComputedStyle(document.body).fontFamily; + ctx.fillText(ob.term + (o.confidence != null ? ' ' + o.confidence.toFixed(2) : ''), ob.box[0] * sx + 2 * dpr, Math.max(11 * dpr, ob.box[1] * sy - 3 * dpr)); + }); + }); + if (edit) for (const [pi, p] of edit.value.entries()) p.points.forEach((q, qi) => { const on = edit.vertex && edit.vertex[0] === pi && edit.vertex[1] === qi; ctx.fillStyle = rgb(on ? V.token('--serious') : V.token('--accent')); ctx.fillRect(q[0] * sx - 4 * dpr, q[1] * sy - 4 * dpr, 8 * dpr, 8 * dpr); }); + }; + img.addEventListener('load', paint, { once: true }); + V.paints(cv, paint); keepPaint(paint); + if (t.type === 'depth') { const d = values.map(s => s && s.drawn).filter(Boolean)[0]; if (d) panel.appendChild(h('p', 'small faint', roles.length > 1 ? 'Overlaid: the difference A − B, blue where A is nearer.' : 'Depth ' + fixed(d.low, 2) + ' to ' + fixed(d.high, 2) + (t.units ? ' ' + t.units : '') + ', light to dark.')); } + if (edit) correctPolygons(cv, edit, nw, nh, paint); + } + + /** A drawn mask's runs decoded to its pixel values. */ + function maskPixels(d) { const out = new Uint16Array(d.width * d.height); let at = 0; for (const [v, n] of d.runs) { out.fill(v, at, at + n); at += n; } return { d, px: out }; } + function paintMasks(ctx, t, layers, W, H) { + const ms = layers.map(l => l.mask); if (!ms.some(Boolean)) return; + const d = ms.find(Boolean).d, off = document.createElement('canvas'); off.width = d.width; off.height = d.height; + const o = off.getContext('2d'), img = o.createImageData(d.width, d.height), both = colourOf('both'); + const instance = t.mask === 'instance'; + for (let i = 0; i < d.width * d.height; i++) { + let c = null; + if (ms.length === 2) { const a = ms[0] ? ms[0].px[i] : 0, b = ms[1] ? ms[1].px[i] : 0; c = a && b ? both : a ? colourOf('a') : b ? colourOf('b') : null; } + else if (ms[0] && ms[0].px[i]) { const v = ms[0].px[i]; c = instance ? V.token(PALETTE[v % PALETTE.length]) : termColour(t, (ms[0].d.classes || {})[String(v)]); } + if (c) { img.data[i * 4] = c[0]; img.data[i * 4 + 1] = c[1]; img.data[i * 4 + 2] = c[2]; img.data[i * 4 + 3] = 170; } + } + o.putImageData(img, 0, 0); ctx.imageSmoothingEnabled = false; ctx.drawImage(off, 0, 0, W, H); + } + function paintDepth(ctx, layers, W, H) { + const ds = layers.map(l => l.s && l.s.drawn).filter(Boolean); if (!ds.length) return; + const raw = d => { const s = atob(d.values_u8), out = new Float32Array(s.length); for (let i = 0; i < s.length; i++) { const u = s.charCodeAt(i); out[i] = u ? d.low + (u - 1) / 254 * (d.high - d.low) : NaN; } return out; }; + let values = raw(ds[0]), colour = D.ramp(ds[0].low, ds[0].high); + if (ds.length === 2 && ds[1].width === ds[0].width) { const b = raw(ds[1]); values = values.map((v, i) => v - b[i]); const m = Math.max(1e-9, ...Array.from(values).filter(Number.isFinite).map(Math.abs)); colour = D.ramp(-m, m); } + ctx.globalAlpha = 0.85; D.heat(ctx, values, ds[0].height, ds[0].width, colour, 0, 0, W, H); ctx.globalAlpha = 1; + } + + /** The light correction of polygons: a vertex dragged, or chosen and deleted (a polygon keeps three). */ + function correctPolygons(cv, edit, nw, nh, paint) { + cv.setAttribute('aria-label', cv.getAttribute('aria-label') + '. Drag a vertex to move it; click one to choose it and Delete removes it.'); + const at = e => { const q = cv.getBoundingClientRect(); return [(e.clientX - q.left) / q.width * nw, (e.clientY - q.top) / q.height * nh]; }; + const near = p => { let best = null, dist = 8 * nw / cv.getBoundingClientRect().width; edit.value.forEach((poly, pi) => poly.points.forEach((q, qi) => { const d = Math.hypot(q[0] - p[0], q[1] - p[1]); if (d <= dist) { dist = d; best = [pi, qi]; } })); return best; }; + let dragging = null; + cv.addEventListener('pointerdown', e => { const v = near(at(e)); edit.vertex = v; if (v) { dragging = e.pointerId; cv.setPointerCapture(e.pointerId); } paint(); }); + cv.addEventListener('pointermove', e => { if (dragging !== e.pointerId || !edit.vertex) return; const [x, y] = at(e), q = edit.value[edit.vertex[0]].points[edit.vertex[1]]; q[0] = Math.round(Math.min(nw, Math.max(0, x)) * 10) / 10; q[1] = Math.round(Math.min(nh, Math.max(0, y)) * 10) / 10; edit.dirty = true; paint(); }); + cv.addEventListener('pointerup', e => { if (dragging === e.pointerId) { dragging = null; again(); } }); + cv.addEventListener('keydown', e => { + if ((e.key === 'Delete' || e.key === 'Backspace') && edit.vertex) { const poly = edit.value[edit.vertex[0]]; if (poly.points.length > 3) { poly.points.splice(edit.vertex[1], 1); edit.vertex = null; edit.dirty = true; paint(); again(); } e.preventDefault(); } + }); + } + + function cloudPanel(panel, t, id, k, values, roles, which) { + if (!k.points_f32) { panel.appendChild(h('p', 'note', 'The point cloud could not be read.')); return; } + const pts = D.floats(k.points_f32), n = pts.length / 3, stride = k.stride || 1, colours = new Uint8Array(n * 3), base = V.token('--ink-2'); + for (let i = 0; i < n; i++) colours.set(base, i * 3); + const objects = values.map(s => !s ? [] : s.drawn && s.drawn.points ? s.drawn.points : Array.isArray(s.value) && t.type === 'points' ? s.value : []); + const member = objects.map(list => { const m = new Map(); list.forEach(o => { for (const idx of o.indices) if (idx % stride === 0) m.set(idx / stride, o.term); }); return m; }); + for (let i = 0; i < n; i++) { + const inA = member[0] && member[0].has(i), inB = member[1] && member[1].has(i); + if (roles.length === 2) { if (inA || inB) colours.set(inA && inB ? colourOf('both') : inA ? colourOf('a') : colourOf('b'), i * 3); } + else if (inA) colours.set(termColour(t, member[0].get(i)), i * 3); + } + const lines = []; + values.forEach((s, li) => { + if (!s || t.type !== 'box3d' || !Array.isArray(s.value)) return; + const col = roles.length > 1 ? colourOf(roles[li]) : null; + for (const b of s.value) { + const [cx, cy, cz] = b.center, [l, w, hh] = b.size, c = Math.cos(b.yaw), sn = Math.sin(b.yaw); + const corner = (dx, dy, dz) => [cx + c * dx - sn * dy, cy + sn * dx + c * dy, cz + dz]; + const pts8 = []; for (const dz of [-hh / 2, hh / 2]) for (const [dx, dy] of [[l / 2, w / 2], [-l / 2, w / 2], [-l / 2, -w / 2], [l / 2, -w / 2]]) pts8.push(corner(dx, dy, dz)); + const edges = [[0, 1], [1, 2], [2, 3], [3, 0], [4, 5], [5, 6], [6, 7], [7, 4], [0, 4], [1, 5], [2, 6], [3, 7]]; + for (const [p, q] of edges) lines.push(pts8[p].concat(pts8[q], [col || termColour(t, b.term)])); + } + }); + const st = store.get(id + '\n' + which) || V.turned(); store.set(id + '\n' + which, st); + panel.dataset.drawn = roles.map((role, i) => role + ':' + member[i].size + '/' + values.filter((s, j) => j === i && s && Array.isArray(s.value) && t.type === 'box3d').reduce((n, s) => n + s.value.length, 0)).join(' '); + const editing = ui.editing && t.type === 'points' ? ui.editing : null; + st.selecting = !!(editing && ui.removing); + const m = { points: pts, colours, bounds: k.bounds, lines }; + if (editing) m.select = shown => { const gone = new Set(shown.map(i => i * stride)); let removed = 0; for (const o of editing.value) { const before = o.indices.length; o.indices = o.indices.filter(x => !gone.has(x)); removed += before - o.indices.length; } editing.value = editing.value.filter(o => o.indices.length); if (removed) { editing.dirty = true; editing.version = (editing.version || 0) + 1; editing.removed = (editing.removed || 0) + removed; again(); } }; + V.space(panel, m, st, 'Point cloud ' + id + ' with ' + roles.map(x => x === 'kept' ? 'the kept point objects' : 'set ' + x.toUpperCase()).join(' and ') + (st.selecting ? '. Drag a rectangle to remove the points in it from the kept objects.' : '')); + const said = t.type === 'box3d' ? values.map((s, i) => (roles[i] === 'kept' ? 'Kept' : roles[i].toUpperCase()) + ': ' + count(Array.isArray(s && s.value) ? s.value.length : 0, 'box', 'boxes')).join(' · ') + : objects.map((list, i) => (roles[i] === 'kept' ? 'Kept' : roles[i].toUpperCase()) + ': ' + (list.length ? list.map(o => o.term + ' ' + count(o.indices.length, 'point')).join(', ') : 'no object')).join(' · '); + panel.appendChild(h('p', 'small faint', said)); + } + + function framesPanel(panel, t, frames, values, roles) { + const cv = h('canvas', 'rv-canvas'); cv.tabIndex = 0; cv.dataset.key = 'rv:frames-' + roles.join(''); cv.setAttribute('role', 'img'); + cv.setAttribute('aria-label', 'Frames of the array with the tracks of ' + roles.map(x => 'set ' + x.toUpperCase()).join(' and ') + '; the left and right arrows change the frame'); + panel.appendChild(cv); + const paint = () => { + const f = frames.frames[Math.min(ui.frame, frames.frames.length - 1)]; if (!f) return; + const size = V.fit(cv, Math.max(120, Math.round(cv.clientWidth * frames.height / frames.width))); if (!size) return; + const [W, H] = size, ctx = cv.getContext('2d'), raw = atob(f.u8), off = document.createElement('canvas'); off.width = frames.width; off.height = frames.height; + const o = off.getContext('2d'), img = o.createImageData(frames.width, frames.height); + for (let i = 0; i < raw.length; i++) { const v = raw.charCodeAt(i); img.data[i * 4] = img.data[i * 4 + 1] = img.data[i * 4 + 2] = v; img.data[i * 4 + 3] = 255; } + o.putImageData(img, 0, 0); ctx.imageSmoothingEnabled = false; ctx.drawImage(off, 0, 0, W, H); + const sx = W / (frames.width * frames.scale), sy = H / (frames.height * frames.scale), dpr = window.devicePixelRatio || 1; + const drawn = values.map(() => 0); + values.forEach((s, li) => { + if (!s || !Array.isArray(s.value)) return; + for (const tr of s.value) { const b = tr.frames.find(x => x.frame === f.frame); if (!b) continue; drawn[li]++; const c = roles.length > 1 ? colourOf(roles[li]) : termColour(t, tr.term); ctx.strokeStyle = rgb(c); ctx.lineWidth = 1.8 * dpr; ctx.strokeRect(b.x * sx, b.y * sy, b.w * sx, b.h * sy); ctx.fillStyle = rgb(c); ctx.font = (11 * dpr) + 'px ' + getComputedStyle(document.body).fontFamily; ctx.fillText(tr.track, b.x * sx + 2 * dpr, Math.max(11 * dpr, b.y * sy - 3 * dpr)); } + }); + const said = 'frame ' + f.frame + ' ' + roles.map((role, i) => role + ':' + drawn[i] + (values[i] && Array.isArray(values[i].value) ? '=' + values[i].value.filter(tr => tr.frames.some(x => x.frame === f.frame)).map(tr => tr.track).join(',') : '')).join(' '); + if (cv.dataset.drawn !== said) cv.dataset.drawn = said; + }; + cv.addEventListener('keydown', e => { if (e.key === 'ArrowLeft' || e.key === 'ArrowRight') { e.preventDefault(); setFrame(ui.frame + (e.key === 'ArrowRight' ? 1 : -1), frames); } }); + V.paints(cv, paint); keepPaint(paint); + } + function frameSlider(frames) { + const box = h('label', 'rv-frame'); const n = frames.frames.length; + ui.frame = Math.min(ui.frame, n - 1); + const out = h('span', 'mono small'), say = () => { out.textContent = 'frame ' + frames.frames[ui.frame].frame + ' of ' + frames.count; }; + const r = h('input'); r.type = 'range'; r.min = '0'; r.max = String(n - 1); r.value = String(ui.frame); r.dataset.key = 'rv:frame'; r.setAttribute('aria-label', 'Frame'); + r.addEventListener('input', () => setFrame(Number(r.value), frames)); + box.appendChild(document.createTextNode('Frame')); box.appendChild(r); box.appendChild(out); say(); box._say = say; + return box; + } + /** A new frame: the canvases repainted and the slider set, in place. */ + function setFrame(i, frames) { + ui.frame = Math.max(0, Math.min(frames.frames.length - 1, i)); + const slider = here.parts.middle.querySelector('.rv-frame'); if (slider) { slider.querySelector('input').value = String(ui.frame); slider._say(); } + for (const p of here.repaint) p(); + } + + /** Timeseries or segments: bands of each set under a series (an array) or a waveform (a sound). */ + function bandsPanel(panel, t, series, k, values, roles) { + const cv = h('canvas', 'rv-canvas rv-bands'); cv.setAttribute('role', 'img'); cv.dataset.key = 'rv:bands-' + roles.join(''); + const length = series ? series.positions : k.seconds || 1; + cv.setAttribute('aria-label', (series ? 'The series' : 'The waveform') + ' with the stretches of ' + roles.map(x => 'set ' + x.toUpperCase()).join(' and ')); + panel.appendChild(cv); + const rows = values.length, terms = t.vocabulary || []; + cv.dataset.drawn = roles.map((role, i) => role + ':' + (values[i] && Array.isArray(values[i].value) ? values[i].value.length : 0)).join(' '); + const paint = () => { + const size = V.fit(cv, 96 + rows * 22); if (!size) return; + const [W, H] = size, ctx = cv.getContext('2d'), dpr = window.devicePixelRatio || 1, top = 96 * dpr; + ctx.clearRect(0, 0, W, H); + if (series) { + const all = series.channels.flat().filter(Number.isFinite), lo = Math.min(...all), hi = Math.max(...all), span = hi - lo || 1; + series.channels.forEach((line, ci) => { ctx.strokeStyle = rgb(V.token(PALETTE[(ci + 2) % PALETTE.length])); ctx.lineWidth = dpr; ctx.beginPath(); line.forEach((v, i) => { const x = i / Math.max(1, line.length - 1) * W, y = top - 4 * dpr - (v - lo) / span * (top - 8 * dpr); i ? ctx.lineTo(x, y) : ctx.moveTo(x, y); }); ctx.stroke(); }); + } else if (k.peaks) V.wave(ctx, W, top, k.peaks); + values.forEach((s, li) => { + const y = top + li * 22 * dpr + 3 * dpr, c0 = roles.length > 1 ? colourOf(roles[li]) : null; + ctx.fillStyle = rgb(V.token('--line-strong'), 0.25); ctx.fillRect(0, y, W, 16 * dpr); + for (const it of (s && Array.isArray(s.value) ? s.value : [])) { const c = c0 || termColour(t, it.term); ctx.fillStyle = rgb(c, 0.75); const x0 = it.start / length * W, x1 = it.end / length * W; ctx.fillRect(x0, y, Math.max(1, x1 - x0), 16 * dpr); ctx.fillStyle = rgb(V.token('--ink') || c); ctx.font = (10 * dpr) + 'px ' + getComputedStyle(document.body).fontFamily; ctx.fillText(it.term, x0 + 2 * dpr, y + 12 * dpr); } + ctx.fillStyle = rgb(c0 || V.token('--line-strong')); ctx.fillText(roles[li].toUpperCase(), W - 14 * dpr, y + 12 * dpr); + }); + void terms; + }; + V.paints(cv, paint); keepPaint(paint); + panel.appendChild(h('p', 'small faint', series ? count(series.positions, 'position') + (series.step > 1 ? ', every ' + series.step + 'th drawn' : '') : (k.seconds ? k.seconds.toFixed(2) + ' s' : 'The sound\'s length is read from WAV only.'))); + } + + /** What has no drawing surface here: each set's value in words (ranking, class, tags, text, spans). */ + function textPanel(panel, t, values, roles) { + const dl = h('dl', 'setup rv-values'); + values.forEach((s, i) => { dl.appendChild(h('dt', null, roles[i].toUpperCase())); dl.appendChild(h('dd', 'mono pre', s && s.value != null ? (Array.isArray(s.value) && t.type === 'ranking' ? s.value.map((x, j) => (j + 1) + '. ' + x).join('\n') : JSON.stringify(s.value, null, 1)) : 'not annotated')); }); + panel.appendChild(dl); + } + + // ---- Right: the values in words, this record's agreement, the decision ------------------------------------- + function summary(t, value) { + if (value == null) return 'not annotated'; + if (t.type === 'mask') return 'a mask of ' + count(Object.keys(value.classes || {}).length, 'class', 'classes') + (value.file ? ' (PNG)' : ' (runs)'); + if (t.type === 'depth') return 'a depth map'; + if (t.type === 'ranking') return value.join(' › '); + if (typeof value === 'string') return value.length > 80 ? value.slice(0, 80) + '…' : value; + if (!Array.isArray(value)) return value.file ? 'point ids in a file' : JSON.stringify(value).slice(0, 80); + if (t.type === 'tags') return value.join(', ') || 'no tag'; + if (t.type === 'points') return value.map(o => o.term + ' (' + count(o.indices.length, 'point') + ')').join(', ') || 'no object'; + const terms = {}; for (const o of value) terms[o.term] = (terms[o.term] || 0) + 1; + const conf = value.map(o => o.confidence).filter(x => x != null); + return (count(value.length, t.type === 'track' ? 'track' : 'object') + (value.length ? ': ' + Object.entries(terms).map(([k, n]) => n + ' ' + k).join(', ') : '')) + (conf.length ? ' · confidence ' + Math.min(...conf).toFixed(2) + '–' + Math.max(...conf).toFixed(2) : ''); + } + function buildRight(t, got, may) { + const box = h('div', 'inspector rv-decide'); + if (!got.id || !got.compare) { box.appendChild(h('p', 'empty', 'Choose a record from the list.')); return box; } + const cmp = got.compare, byName = Object.fromEntries(cmp.sets.map(s => [s.set, s])), line = got.kept ? got.kept.annotation : null; + const hd = h('header'); hd.appendChild(h('span', 'kicker', t.title + ' · ' + t.type)); const t2 = h('h2', 'mono', got.id); t2.tabIndex = -1; hd.appendChild(t2); box.appendChild(hd); + const dl = h('dl', 'setup'); + for (const [label, name] of [['A', got.a], ['B', got.b]]) { const s = byName[name]; dl.appendChild(h('dt', null, label + ' · ' + shortSet(name))); dl.appendChild(h('dd', null, summary(t, s && s.value) + (s && s.confidence != null ? ' · line confidence ' + s.confidence.toFixed(2) : ''))); } + const ab = cmp.pairs.find(p => (p.a === got.a && p.b === got.b) || (p.a === got.b && p.b === got.a)); + dl.appendChild(h('dt', null, 'Agreement here')); dl.appendChild(h('dd', 'mono', ab ? (ab.problem || fixed(ab.agreement)) : '—')); + box.appendChild(dl); + // What is kept now. + const kept = h('p', 'small rv-kept'); + if (line) kept.textContent = 'Kept: ' + line.state + (line.from && line.from.set ? ' · from ' + shortSet(line.from.set) : line.from && line.from.both ? ' · from both' : line.from && line.from.rejected ? ' · both rejected' : '') + (got.kept.unsaved ? ' · unsaved' : '') + ' · ' + summary(t, line.value); + else kept.textContent = 'Nothing is kept for this record yet.'; + box.appendChild(kept); + if (ui.editing) { box.appendChild(correctionBox(t, got)); return box; } + // The decision. + const note = h('textarea'); note.rows = 2; note.maxLength = 4000; note.placeholder = 'a note (optional)'; note.dataset.key = 'rv:note'; note.disabled = !may; + const nk = t.task + '\n' + got.id; note.value = ui.notes.get(nk) || ''; note.addEventListener('input', () => ui.notes.set(nk, note.value)); + const nl = h('div', 'form rv-note'), lab = h('label'); lab.appendChild(document.createTextNode('Note')); lab.appendChild(note); nl.appendChild(lab); + const va = byName[got.a] && byName[got.a].value, vb = byName[got.b] && byName[got.b].value; + const args = (keep, sets) => () => { const a = { task: t.task, record: got.id, keep, sets }; const n = (ui.notes.get(nk) || '').trim(); if (n) a.note = n; return a; }; + const done = () => { ui.notes.delete(nk); }; + box.appendChild(A.group('review:decide', [ + { id: 'keep-a', label: 'Keep A', cls: 'btn primary', who: 'edit', op: 'keep_annotation', busy: 'Keeping…', hidden: got.a === KEPT || va == null, args: args('one', [got.a]), done }, + { id: 'keep-b', label: 'Keep B', cls: 'btn primary', who: 'edit', op: 'keep_annotation', busy: 'Keeping…', hidden: got.b === KEPT || vb == null, args: args('one', [got.b]), done }, + { id: 'keep-both', label: 'Keep both', cls: 'btn', who: 'edit', op: 'keep_annotation', busy: 'Keeping…', hidden: got.a === KEPT || got.b === KEPT || !isList(t, va) || !isList(t, vb), args: args('both', [got.a, got.b]), done }, + { id: 'reject', label: 'Reject both', cls: 'btn', who: 'edit', op: 'keep_annotation', busy: 'Rejecting…', args: args('none', [got.a, got.b]), done } + ])); + box.appendChild(nl); + const correctable = line && line.state !== 'rejected' && Array.isArray(line.value) && ((t.type === 'polygons' && line.value.length) || (t.type === 'points' && line.value.length)); + if (may && correctable) box.appendChild(A.group('review:correct', [{ id: 'correct', label: t.type === 'polygons' ? 'Correct the kept polygons' : 'Correct the kept point objects', cls: 'btn ghost', onClick: () => { ui.editing = { key: nk, set: KEPT, value: copy(line.value), from: line.from || null, state: line.state, note: line.note || null, dirty: false }; ui.removing = t.type === 'points'; again(); } }])); + box.appendChild(h('p', 'note', may ? 'A decision is kept in Tuldok as an unsaved edit of ' + t.task + '.jsonl, naming the set it came from; Save writes it.' : 'This session may look but not decide: that needs the right to edit.')); + return box; + } + function correctionBox(t, got) { + const e = ui.editing, box = h('div', 'rv-correct'); + box.appendChild(h('p', 'small', t.type === 'polygons' ? 'Drag a vertex to move it. Click a vertex to choose it, then Delete removes it (a polygon keeps three).' : 'Drag a rectangle on the cloud to remove the points in it from the kept objects.' + (e.removed ? ' ' + count(e.removed, 'point') + ' removed so far.' : ''))); + const item = () => { const it = { record: got.id, state: 'accepted', value: e.value }; if (e.note) it.note = e.note; return it; }; + box.appendChild(A.group('review:correction', [ + { id: 'keep-correction', label: 'Keep the correction', cls: 'btn primary', who: 'edit', op: 'set_annotation', busy: 'Keeping…', hidden: !e.dirty, + args: () => Object.assign({ task: t.task, items: [item()] }, e.from ? { from: e.from } : {}), done: () => { ui.editing = null; ui.removing = false; } }, + { id: 'drop-correction', label: 'Drop the correction', cls: 'btn ghost', onClick: () => { ui.editing = null; ui.removing = false; again(); } } + ])); + return box; + } + + /** For annotate mode: a kept value of a new type drawn read-only over its record (no drawing tools). */ + function overlay(el, got, t, value) { + const panel = h('figure', 'rv-panel'), r = got.record, k = got.content || {}, s = { set: KEPT, value, drawn: null }; + const cmp = { size: k.width ? [k.width, k.height] : null }; + if (r.kind === 'image' && !['mask', 'depth'].includes(t.type)) imagePanel(panel, t, r, k, [s], ['kept'], cmp); + else if (r.kind === 'pointcloud' && ['points', 'box3d'].includes(t.type) && Array.isArray(value)) cloudPanel(panel, t, r.id, k, [s], ['kept'], 'annotate'); + else return false; + el.appendChild(panel); return true; + } + + return { draw, NEW, summary, overlay }; +})(); diff --git a/static/project-workspace/screens.js b/static/project-workspace/screens.js new file mode 100644 index 00000000..3355683f --- /dev/null +++ b/static/project-workspace/screens.js @@ -0,0 +1,335 @@ +'use strict'; +// The screens, from the mockup's screens.js. A screen is one function +// (main, aside, state, act, data) that fills the main area, fills the drawer when it +// has something to show there, and returns whether the drawer is open. It +// reads the project only through window.TuldokProject. Rule: the chrome holds +// no project words; names, groups, metrics and text come from the project. +// What is drawn again is built off the page and put in place whole, in one +// step, keeping the scroll and the focus (putInPlace, spec-w4.md); the map +// keeps its tree while its answer is unchanged, and keeps its place when +// rebuilt. The editor, the branch form, a draft's drawer, the steps of a new model and +// the Overview's list of every definition are editor.js's (window.LabEditor). +window.LabScreens = (() => { + const P = window.TuldokProject, Tree = window.LabTree, C = window.LabCharts, h = C.h; + const svgNS = 'http://www.w3.org/2000/svg'; + const ICONS = { + check: 'M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20zm-1.3 14.2-3.6-3.6 1.4-1.4 2.2 2.2 5.1-5.1 1.4 1.4z', + alert: 'M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20zm-1 5h2v6h-2zm0 8h2v2h-2z', + triangle: 'M12 3 1 21h22zm-1 7h2v5h-2zm0 7h2v2h-2z', + minus: 'M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20zm-5 9h10v2H7z', + branch: 'M7 3a3 3 0 0 0-1 5.8v6.4A3 3 0 1 0 8 15.2V9.9c1.2 1.3 2.9 2.1 5 2.1 2.8 0 4.4-1.5 5-4.1A3 3 0 1 0 16 7.6c-.5 1.6-1.4 2.4-3 2.4-2.6 0-4.3-1.6-5-3.2A3 3 0 0 0 7 3z', + clock: 'M12 2a10 10 0 1 0 0 20 10 10 0 0 0 0-20zm1 5h-2v6l5 3 1-1.7-4-2.3z', + x: 'M18.3 5.7 12 12l6.3 6.3-1.4 1.4L10.6 13.4 4.3 19.7 2.9 18.3 9.2 12 2.9 5.7l1.4-1.4 6.3 6.3 6.3-6.3z', + theme: 'M12 3a9 9 0 1 0 0 18 9 9 0 0 0 0-18zm0 2v14a7 7 0 0 1 0-14z' + }; + const icon = name => { const s = document.createElementNS(svgNS, 'svg'); s.setAttribute('viewBox', '0 0 24 24'); s.setAttribute('aria-hidden', 'true'); const p = document.createElementNS(svgNS, 'path'); p.setAttribute('d', ICONS[name] || ICONS.minus); p.setAttribute('fill', 'currentColor'); p.setAttribute('fill-rule', 'evenodd'); s.appendChild(p); return s; }; + const stateEl = (kind, text) => { const map = { good: 'check', warn: 'alert', serious: 'triangle', neutral: 'minus', pending: 'clock' }; const s = h('span', 'state ' + (kind === 'pending' ? 'warn' : kind)); s.appendChild(icon(map[kind] || 'minus')); s.appendChild(document.createTextNode(text)); return s; }; + const modelState = s => { const [word, tone] = P.stateWord(s); return stateEl(tone, word); }; + const sect = (title, sum) => { const d = h('div', 'sect'); d.appendChild(h('h2', null, title)); if (sum !== undefined) d.appendChild(h('span', 'sum', sum)); return d; }; + const cell = (k, v, u) => { const d = h('div'); d.appendChild(h('dt', null, k)); const dd = h('dd', null, v); if (u) dd.appendChild(h('small', null, u)); d.appendChild(dd); return d; }; + const block = (title, ...children) => { const d = h('div'); d.appendChild(h('h3', null, title)); for (const c of children) if (c) d.appendChild(c); return d; }; + const time = iso => iso ? new Date(iso).toLocaleString('en-GB', { day: 'numeric', month: 'short', year: 'numeric', hour: '2-digit', minute: '2-digit' }) : '—'; + const bytes = b => b == null ? '—' : b >= 1e9 ? (b / 1e9).toFixed(1) + ' GB' : b >= 1e6 ? (b / 1e6).toFixed(1) + ' MB' : b >= 1e3 ? Math.round(b / 1e3) + ' KB' : b + ' B'; + const failure = (parent, error) => { parent.appendChild(h('p', 'failure', error.message || String(error))); }; + // ---- Putting in place (spec-w4 R4) ------------------------------------------ + /** Every element under root that is scrolled, by its path of child indexes and its tag. */ + function scrolls(root) { + const out = []; + const walk = (el, path) => { for (let i = 0; i < el.children.length; i++) { const c = el.children[i], p = path.concat([i]); if (c.scrollTop || c.scrollLeft) out.push({ path: p, tag: c.tagName, top: c.scrollTop, left: c.scrollLeft }); walk(c, p); } }; + walk(root, []); return out; + } + /** + * Puts content, built off the page, in box in one step in place of what it + * holds, keeping the scroll of scroller (box unless named) in both + * directions, the scroll of every scrolled element inside box (the same + * path and tag), and the focused control (by data-key) with its caret. + * Returns whether the focus was inside and whether its control was found. + */ + function putInPlace(box, content, scroller = box) { + const active = document.activeElement, inside = !!active && box.contains(active), key = inside && active.dataset ? active.dataset.key : null; + let caret = null; try { caret = key && typeof active.selectionStart === 'number' ? [active.selectionStart, active.selectionEnd, active.scrollTop] : null; } catch (e) { caret = null; } + const top = scroller.scrollTop, left = scroller.scrollLeft, inner = scrolls(box); + box.replaceChildren(content); + scroller.scrollTop = top; scroller.scrollLeft = left; + for (const s of inner) { let e = box; for (const i of s.path) { e = e.children[i]; if (!e) break; } if (e && e.tagName === s.tag) { e.scrollTop = s.top; e.scrollLeft = s.left; } } + const again = key && [...box.querySelectorAll('[data-key]')].find(e => e.dataset.key === key); + if (again) { again.focus({ preventScroll: true }); if (caret && again.setSelectionRange) { try { again.setSelectionRange(caret[0], caret[1]); } catch (e) { /* not a text field */ } again.scrollTop = caret[2]; } } + return { inside, found: !!again }; + } + /** Puts a drawer's content in place (the drawer's own scroll kept); when the focused control is gone, the heading takes the focus. */ + function putDrawer(aside, content) { + const r = putInPlace(aside, content, aside.parentElement); + if (r.inside && !r.found) { const t = aside.querySelector('h2'); if (t) t.focus({ preventScroll: true }); } + } + /** A new turn of the drawer: a read started for an older turn puts nothing in it. */ + const turn = aside => (aside._turn = (aside._turn || 0) + 1); + + const NEEDS_ICON = { approval: 'clock', changed: 'alert', missing: 'triangle' }; + /** What needs the person, one sentence each; controls(item) gives the controls drawn under a sentence. */ + function needsList(items, controls) { + const ul = h('ul', 'needs'); + if (!items.length) { ul.appendChild(h('li', 'empty', 'Nothing needs you.')); return ul; } + for (const n of items) { + const li = h('li', n.kind); li.appendChild(icon(NEEDS_ICON[n.kind])); + const body = h('div', 'need'); body.appendChild(h('span', null, n.text)); + const c = controls ? controls(n) : null; if (c) body.appendChild(c); + li.appendChild(body); ul.appendChild(li); + } + return ul; + } + + // ---- Map ----------------------------------------------------------------- + let graph = null, graphOf = null; + function graphFor(shaped) { if (graphOf !== shaped) { graph = Tree.build(shaped); graphOf = shaped; } return graph; } + function map(main, aside, state, act, data) { + // A selection 'plan:' is the plan view in the drawer, with its first target selected on the map; + // 'edit:' and 'branch:' are the editor and the branch form (editor.js), with that model selected. + const shaped = data.map, g = graphFor(shaped), raw = state.sel.map || '', planSel = raw.startsWith('plan:') ? raw : null, E = window.LabEditor; + const focus = planSel ? planSel.slice(5).split(',')[0] : E ? E.subject(raw) : raw, sel = focus && g.byId.has(focus) ? focus : null; + const kinds = P.KINDS.map(k => k.id).concat(g.nodes.some(m => m.kind === P.UNSET) ? [P.UNSET] : []); + const withValue = g.nodes.filter(m => m.value != null).length; + const buildBar = () => { + const bar = h('div', 'filters'); + const q = h('input'); q.type = 'search'; q.placeholder = 'Find a model'; q.value = state.query || ''; q.setAttribute('aria-label', 'Find a model'); q.addEventListener('input', () => act.setQuery(q.value)); bar.appendChild(q); + const grp = h('div', 'grp'); grp.setAttribute('role', 'group'); grp.setAttribute('aria-label', 'Kind of change'); + for (const k of kinds) { const b = h('button', 'btn sm chip ' + k, P.kindLabel(k)); b.type = 'button'; b.dataset.kind = k; b.setAttribute('aria-pressed', String(!state.kinds || state.kinds.has(k))); b.addEventListener('click', () => act.toggleKind(k, kinds)); grp.appendChild(b); } + bar.appendChild(grp); + if (g.track) { const tb = h('button', 'btn sm', 'Accuracy track'); tb.type = 'button'; tb.title = P.trackName(g.track) + ', ' + P.better(g.track); tb.setAttribute('aria-pressed', String(!!state.track)); tb.addEventListener('click', () => act.toggleTrack()); bar.appendChild(tb); } + const z = h('div', 'zoomctl'); for (const [t, d, l] of [['−', -0.2, 'Zoom out'], ['+', 0.2, 'Zoom in']]) { const b = h('button', 'btn sm', t); b.type = 'button'; b.setAttribute('aria-label', l); b.addEventListener('click', () => act.zoom(d)); z.appendChild(b); } bar.appendChild(z); + bar.appendChild(h('span', 'pop', g.nodes.length + ' models · ' + (g.track ? withValue + ' on the track' : 'no track'))); + return bar; + }; + if (!g.nodes.length) { main.textContent = ''; main._tree = null; main.appendChild(h('p', 'empty', 'This project has no recipes, so there are no models to map.')); aside.textContent = ''; return false; } + const sig = JSON.stringify([state.kinds ? [...state.kinds] : null, state.query || '', state.zoom || 1, !!state.track]); + const prev = main._tree; + // While a parent is being picked (editor.js), the next model pressed is the parent instead. + const common = { onSelect: id => { if (!(E && E.picked(id, act))) act.select(id); }, kinds: state.kinds, query: state.query }; + if (prev && prev.sig === sig && prev.shaped === shaped && main.contains(prev.bar)) { + // The same tree: only a new selection is drawn (spec-w4 R2, R6). + if (sel !== prev.sel) { if (sel) { prev.map.scrollTo(sel, true); prev.list.scrollTo(sel, true); } prev.map.select(sel); prev.list.select(sel); prev.sel = sel; } + } else { + const focused = prev && prev.bar && prev.bar.contains(document.activeElement) && document.activeElement.type === 'search' ? document.activeElement.selectionStart : null; + // A changed map (the same filters) is rebuilt where it was: the scroll of the map and of the list kept (R6). + const place = prev && prev.sig === sig && main.contains(prev.bar) ? [main.querySelector('.tree-map'), main.querySelector('.tree-host')].map(e => e ? [e.scrollLeft, e.scrollTop] : null) : null; + main.textContent = ''; const bar = buildBar(); main.appendChild(bar); + const mh = h('div', 'tree-map'); main.appendChild(mh); + const inst = { sig, bar, sel, shaped, map: null, list: null }; + inst.map = Tree.renderMap(mh, g, Object.assign({ selected: sel, zoom: state.zoom || 1, track: !!state.track }, common)); + const lh = h('div', 'tree-host'); main.appendChild(lh); inst.list = Tree.renderList(lh, g, Object.assign({ selected: sel }, common)); + if (place) { for (const [e, at] of [[mh, place[0]], [lh, place[1]]]) if (at) { e.scrollLeft = at[0]; e.scrollTop = at[1]; } } + else if (sel) { inst.map.scrollTo(sel, false); inst.list.scrollTo(sel, false); } + if (focused !== null) { const q = bar.querySelector('input[type="search"]'); q.focus(); q.setSelectionRange(focused, focused); } + main._tree = inst; + } + if (E) E.decorate(main); + if (planSel) { window.LabActions.planView(aside, planSel, act.go, sel ? () => act.select(sel) : null); return true; } + if (E && E.owns(raw, data)) { E.drawer(aside, raw, act, data, 'map'); return true; } + if (sel) { + // Compare (spec-t3 section 5.1): the model into the page's compare set, kept current on every render (tabs.js). + const more = window.LabScreens.compareActs ? ins => window.LabScreens.compareActs(ins, sel, state, act, 'map') : undefined; + inspectorModel(aside, sel, act, data, more); + const ins = aside.querySelector('.inspector'); if (ins && more) more(ins); + return true; + } + aside.textContent = ''; aside._for = null; return false; + } + + // ---- Drawer: one model --------------------------------------------------- + const TO_BUILD = ['not-built', 'out-of-date', 'failed']; + /** + * The drawer of one model: its header, controls and lineage at once, and + * four blocks filled as their reads arrive (Results, Runs and product, Uses, + * Card; and the steps of a new model). Read again for the model on screen + * (aside._again), the new drawer is built off the page and put in place in + * one step once every block is filled. more(ins): added by the caller to + * each drawing (the Benchmarks tab's Compare). + */ + function inspectorModel(aside, id, act, data, more) { + const E = window.LabEditor, A = window.LabActions; + if (E && E.isDraft(id)) return E.drawer(aside, id, act, data, 'map'); // a draft's drawer is the editor + if (aside._for === id && !aside._again) return; + const same = aside._for === id && !!aside.firstChild, mine = turn(aside); + aside._for = id; if (!same) aside.textContent = ''; + const live = () => aside._turn === mine && aside._for === id, reads = []; + const g = graphFor(data.map), m = g.byId.get(id); + const ins = h('div', 'inspector'); + const hd = h('header'); + hd.appendChild(h('span', 'kicker', (m.short ? m.short + ' · ' : '') + (m.groupTitle || ''))); + const t = h('h2', null, m.title); t.tabIndex = -1; hd.appendChild(t); + const meta = h('div', 'ins-meta'); meta.appendChild(h('span', 'chip ' + m.kind, P.kindLabel(m.kind))); meta.appendChild(modelState(m.state)); + meta.appendChild(h('span', 'small faint', m.trainedAt ? 'trained ' + P.day(m.trainedAt) : m.planned ? 'never built' : 'no training date')); + meta.appendChild(Tree.mark(m)); + hd.appendChild(meta); ins.appendChild(hd); + if (E) E.decorate(hd); + // Plan build, for a model to build (spec-w3 section 3.1): the plan view, never a request from here. A model + // never built that is not earlier work has the steps of a new model instead (editor.js), whose next step it is. + const planGroup = () => A.group('model:' + m.id, [{ id: 'plan', label: 'Plan build', cls: 'btn primary', who: 'request', onClick: () => act.select('plan:' + m.id) }]); + if (E && E.followsSteps(m)) { const box = h('div', 'steps-box'); box.appendChild(h('p', 'note', 'Reading…')); ins.appendChild(box); reads.push(E.steps(box, m, act, data, planGroup)); } + else if (TO_BUILD.includes(m.state)) ins.appendChild(planGroup()); + // Edit and Branch from here (spec-w3 section 3.2), for a session that may edit. + ins.appendChild(A.group('modelact:' + m.id, [ + { id: 'edit', label: 'Edit', cls: 'btn sm', who: 'edit', onClick: () => act.go('map', 'edit:' + m.id) }, + { id: 'branch', label: 'Branch from here', cls: 'btn sm', who: 'edit', onClick: () => act.go('map', 'branch:' + m.id) } + ], { inline: true })); + // Lineage: what it was derived from, what derives from it, what it takes as input. + const lin = h('div', 'actions'); + const jump = (target, text) => { const b = h('button', 'btn sm ghost'); b.type = 'button'; b.appendChild(icon('branch')); b.appendChild(document.createTextNode(' ' + text)); b.addEventListener('click', () => act.select(target.id)); return b; }; + if (m.parent) lin.appendChild(jump(g.byId.get(m.parent), 'From ' + Tree.refName(g.byId.get(m.parent)))); + else lin.appendChild(h('span', 'small faint', 'No recorded ancestry')); + const kids = g.nodes.filter(n => n.parent === m.id); + if (kids.length === 1) lin.appendChild(jump(kids[0], Tree.refName(kids[0]) + ' derives from it')); + else if (kids.length) { const b = h('button', 'btn sm ghost', kids.length + ' models derive from it'); b.type = 'button'; b.addEventListener('click', () => act.select(kids[0].id)); lin.appendChild(b); } + ins.appendChild(lin); + const uses = g.edges.filter(e => e.kind === 'input' && e.to === m.id).map(e => g.byId.get(e.from)); + const usedBy = g.edges.filter(e => e.kind === 'input' && e.from === m.id).map(e => g.byId.get(e.to)); + for (const [title, list] of [['Takes as input', uses], ['Is an input of', usedBy]]) if (list.length) { + const ul = h('ul', 'links'); for (const o of list) { const li = h('li'); const b = h('button'); b.type = 'button'; b.textContent = Tree.nameOf(o); b.addEventListener('click', () => act.select(o.id)); li.appendChild(b); ul.appendChild(li); } + ins.appendChild(block(title, ul)); + } + // The track value. + const dl = h('dl', 'metrics'); + if (g.track) dl.appendChild(cell(g.track.metric.title, m.value == null ? '—' : C.fmtSci(m.value), m.value == null ? '' : P.units(g.track).trim())); + dl.appendChild(cell('Cases', m.count == null ? '—' : m.count.toLocaleString('en-GB'))); + dl.appendChild(cell('Benchmarks with results', String(m.benchmarks))); + dl.appendChild(cell('State', P.stateWord(m.state)[0] + (m.adopted ? ', adopted' : ''))); + ins.appendChild(dl); + ins.appendChild(h('p', 'note', g.track ? (m.value == null ? 'No result on ' + g.track.benchmark + ' for this model.' : 'The ' + g.track.statistic + ' over ' + m.count + ' cases of ' + g.track.benchmark + ', ' + P.better(g.track) + '.') : 'The project names no track for the map.')); + if (m.description) ins.appendChild(block('Description', h('p', null, m.description))); + const cmp = Tree.comparisons(g, m.id); + if (cmp.length) { + const ul = h('ul', 'links'); + for (const c of cmp) { const o = g.byId.get(c.other); const li = h('li'); const b = h('button'); b.type = 'button'; b.appendChild(document.createTextNode(Tree.nameOf(o))); const said = c.excerpt || c.explanation || ''; b.appendChild(h('span', 'why', said.slice(0, 140) + (said.length > 140 ? '…' : ''))); b.title = c.explanation || ''; b.addEventListener('click', () => act.select(o.id)); li.appendChild(b); ul.appendChild(li); } + ins.appendChild(block('Compared with', ul)); + } + // What is read when the model is opened. + const results = block('Results'), runs = block('Runs and product'), uses2 = block('Uses'), card = block('Card'); + for (const b of [results, runs, uses2, card]) { b.appendChild(h('p', 'note', 'Reading…')); ins.appendChild(b); } + // Weights (weights.js, spec-d2 section 4), after Runs and product: shown only when the model has a current product with a weights file. + if (window.LabWeights) { + const wts = h('div', 'wts-box'); wts.hidden = true; ins.insertBefore(wts, uses2); + reads.push(window.LabWeights.block(wts, m, g, aside.parentElement, live).catch(error => { if (!live()) return; wts.hidden = false; wts.replaceChildren(h('h3', null, 'Weights'), A.refusal(error)); })); + } + // Structure (structure.js, spec-t2 section 4), after Weights: shown only when a current product holds model-graph.json. + if (window.LabStructure) { + const stb = h('div', 'st-box'); stb.hidden = true; ins.insertBefore(stb, uses2); + reads.push(window.LabStructure.block(stb, m, g, aside.parentElement, live).catch(error => { if (!live()) return; stb.hidden = false; stb.replaceChildren(h('h3', null, 'Structure'), A.refusal(error)); })); + } + // Serving (serving.js, spec-t6 section 7), after Structure: shown only when its recipe declares serve. + if (window.LabServing) { + const sv = h('div', 'srv-box'); sv.hidden = true; ins.insertBefore(sv, uses2); + reads.push(window.LabServing.block(sv, m, live).catch(error => { if (!live()) return; sv.hidden = false; sv.replaceChildren(h('h3', null, 'Serving'), A.refusal(error)); })); + } + if (more) more(ins); + if (!same) aside.appendChild(ins); // first shown: at once, its blocks filled as they come + const fill = (b, build) => p => { if (!live()) return; b.lastChild.remove(); try { build(b, p); } catch (error) { failure(b, error); } }; + const fail = b => error => { if (!live()) return; b.lastChild.remove(); failure(b, error); }; + reads.push(Promise.all([P.read('recipe_results', { recipe: m.id }), P.read('definitions', { kind: 'benchmark' }).catch(() => ({ definitions: [] }))]) + .then(fill(results, (b, [found, defs]) => { + if (!found.rows.length) { b.appendChild(h('p', 'empty', found.benchmarks.length ? 'Named in ' + found.benchmarks.length + ' benchmarks, with no results yet.' : 'No benchmark measures this model.')); return; } + const titles = new Map(defs.definitions.map(d => [d.reference.split('/').slice(1).join('/'), d.title])); + const byBench = new Map(); for (const r of found.rows) { if (!byBench.has(r.benchmark)) byBench.set(r.benchmark, []); byBench.get(r.benchmark).push(r); } + for (const [bench, rows] of byBench) { + b.appendChild(h('p', 'small muted', titles.get(bench) || bench)); + const t = h('table', 'tv'); const hr = h('tr'); for (const [c, k] of [['Metric', ''], ['Median', 'n'], ['Cases', 'n']]) hr.appendChild(h('th', k, c)); const th = h('thead'); th.appendChild(hr); t.appendChild(th); + const tb = h('tbody'); for (const r of rows) { const tr = h('tr'); tr.appendChild(h('td', 'ck', r.metric)); tr.appendChild(h('td', 'n', C.fmtSci(r.median))); tr.appendChild(h('td', 'n', String(r.count))); tb.appendChild(tr); } t.appendChild(tb); b.appendChild(t); + } + }), fail(results))); + reads.push(Promise.all([P.read('runs', { reference: m.id }), P.read('products', { reference: m.id })]) + .then(fill(runs, (b, [r, p]) => { + if (!r.runs.length) b.appendChild(h('p', 'empty', 'No run of this model in this project.')); + else { const ul = h('ul', 'rows'); for (const run of r.runs.slice(0, 8)) { const li = h('li'); const row = h('div', 'row'); row.style.gridTemplateColumns = 'minmax(0, 1fr) auto'; const t = h('span'); t.appendChild(h('span', 'title', (run.subject ? run.subject + ' · ' : '') + time(run.finished_at || run.started_at || run.queued_at))); t.appendChild(h('span', 'sub', [run.seconds != null ? Math.round(run.seconds) + ' s' : null, run.bytes != null ? bytes(run.bytes) : null, run.scratch ? 'scratch' : null].filter(Boolean).join(' · ') || run.id)); row.appendChild(t); row.appendChild(modelState(run.state)); li.appendChild(row); ul.appendChild(li); } b.appendChild(ul); if (r.runs.length > 8) b.appendChild(h('p', 'note', (r.runs.length - 8) + ' earlier runs are not listed.')); } + if (!p.products.length) b.appendChild(h('p', 'note', 'No product on this machine.')); + else { const dl = h('dl', 'facts'); for (const pr of p.products) { dl.appendChild(h('dt', null, (pr.subject || 'Product') + (pr.adopted ? ' (adopted)' : ''))); dl.appendChild(h('dd', 'mono', pr.directory + ' · ' + bytes(pr.bytes))); } b.appendChild(dl); } + }), fail(runs))); + reads.push(P.read('definition', { reference: m.id }) + .then(fill(uses2, (b, d) => { + const other = d.uses.filter(u => !u.reference.startsWith('recipes/')); + if (!other.length) { b.appendChild(h('p', 'empty', 'It uses no dataset, model or environment.')); return; } + const dl = h('dl', 'facts'); for (const u of other) { dl.appendChild(h('dt', null, u.role)); dl.appendChild(h('dd', 'mono', u.reference)); } b.appendChild(dl); + }), fail(uses2))); + if (!m.card) fill(card, b => b.appendChild(h('p', 'empty', 'This model has no card.')))(); + else if (!m.card.startsWith('notes/')) fill(card, b => b.appendChild(h('p', 'empty', 'Its card ' + m.card + ' is not under notes/ and is not read.')))(); + else reads.push(P.read('note', { path: m.card.slice('notes/'.length) }).then(fill(card, (b, note) => { b.appendChild(h('p', 'note mono', m.card)); b.appendChild(window.TuldokMarkdown.render(note.text)); }), fail(card))); + // Read again: the drawer on screen stays until the new one is complete. + if (same) Promise.all(reads).then(() => { if (live()) putDrawer(aside, ins); }); + } + + // ---- Overview -------------------------------------------------------------- + // Read once and built off the page (tabs.js's paint), so the refresh while a build runs keeps the place. + function overview(main, aside, state, act, data) { + const E = window.LabEditor, sel = state.sel.overview || ''; + if (sel === 'report') { aside.textContent = ''; aside._for = null; return report(main, data); } + const editing = !!(E && E.owns(sel, data)); + if (editing) E.drawer(aside, sel, act, data, 'overview'); else { aside.textContent = ''; aside._for = null; } + S.paint(main, () => Promise.all([P.read('definitions'), P.read('environments'), P.read('activity', { limit: 12 })].map(p => p.then(v => ({ v }), error => ({ error })))), (frag, [d, e, a]) => { + const standing = data.standing; + const top = h('section', 'ov-status'); + const line = h('div', 'ov-line'); line.appendChild(stateEl(standing.tone, standing.word)); line.appendChild(h('p', 'ov-text', standing.text)); top.appendChild(line); + const at = h('p', 'small faint', 'Read at ' + data.readAt); at.dataset.readAt = '1'; top.appendChild(at); // set in place on every reading (app.js) + frag.appendChild(top); + const row = h('div', 'ov-row'); + const needs = h('section'); needs.appendChild(sect('Needs you', data.needs ? String(data.needs.length) : '…')); + if (data.needs) needs.appendChild(needsList(data.needs, n => window.LabActions.needControls(n, act.go))); else needs.appendChild(h('p', 'note', 'Reading…')); + row.appendChild(needs); + const facts = h('section'); facts.appendChild(sect('Project')); + const dl = h('dl', 'facts'), s = data.summary; + for (const [k, v, c] of [['Folder', s.folder, 'mono'], ['Code root', s.code_root, 'mono'], ['Project', s.project_id, 'mono'], ['Build worker', s.worker]]) { dl.appendChild(h('dt', null, k)); dl.appendChild(h('dd', c || '', v)); } + dl.appendChild(h('dt', null, 'Report')); const dd = h('dd'); const link = h('a', null, 'Open the report'); link.href = '#overview/report'; dd.appendChild(link); dl.appendChild(dd); + facts.appendChild(dl); row.appendChild(facts); frag.appendChild(row); + const cols = h('div', 'ov-cols'); + const defs = h('section'); defs.appendChild(sect('Definitions', String(Object.values(s.kinds).reduce((x, y) => x + y, 0)))); cols.appendChild(defs); + const envs = h('section'); envs.appendChild(sect('Environments', String(s.kinds.environment || 0))); cols.appendChild(envs); + frag.appendChild(cols); + // Every definition by kind, each opening the editor (editor.js) for a session that may edit. + const every = h('section', 'ov-defs'); frag.appendChild(every); + const act2 = h('section', 'ov-history'); act2.appendChild(sect('Latest activity')); frag.appendChild(act2); + const fill = (b, got, build) => { if (got.error) failure(b, got.error); else build(b, got.v); }; + fill(defs, d, (b, d) => { + const states = [...new Set(d.definitions.map(x => x.state))].sort(); + const kinds = [...new Set(d.definitions.map(x => x.kind))].sort(); + if (!kinds.length) { b.appendChild(h('p', 'empty', 'The project has no definitions.')); return; } + const t = h('table', 'tv kinds'), th = h('thead'), hr = h('tr'); hr.appendChild(h('th', null, 'Kind')); for (const st of states) hr.appendChild(h('th', 'n', P.stateWord(st)[0])); hr.appendChild(h('th', 'n', 'All')); th.appendChild(hr); t.appendChild(th); + const tb = h('tbody'); for (const k of kinds) { const tr = h('tr'); tr.appendChild(h('td', null, k)); const mine = d.definitions.filter(x => x.kind === k); for (const st of states) { const n = mine.filter(x => x.state === st).length; tr.appendChild(h('td', 'n' + (n ? '' : ' faint'), n ? String(n) : '·')); } tr.appendChild(h('td', 'n', String(mine.length))); tb.appendChild(tr); } t.appendChild(tb); + const wrap = h('div'); wrap.style.overflowX = 'auto'; wrap.appendChild(t); b.appendChild(wrap); + }); + fill(every, d, (b, d) => { if (E) E.definitionList(b, d.definitions, sel, act); }); + fill(envs, e, (b, e) => { + if (!e.environments.length) { b.appendChild(h('p', 'empty', 'The project defines no environment.')); return; } + const ul = h('ul', 'rows'); for (const env of e.environments) { const li = h('li'); const r = h('div', 'row ov-env'); const t = h('span'); t.appendChild(h('span', 'title', env.title)); t.appendChild(h('span', 'sub', ['Python ' + env.python, env.device, Object.keys(env.packages || {}).length + ' packages', env.checked_at ? 'checked ' + P.day(env.checked_at) : 'never checked'].filter(Boolean).join(' · '))); r.appendChild(t); r.appendChild(modelState(env.state)); li.appendChild(r); ul.appendChild(li); } b.appendChild(ul); + }); + fill(act2, a, (b, a) => { + if (!a.items.length) { b.appendChild(h('p', 'empty', 'Nothing has been asked of this project through Tuldok yet.')); return; } + const ul = h('ul', 'rows'); for (const it of a.items) { const li = h('li'); const r = h('div', 'row ov-act'); r.appendChild(h('span', 'sub num', time(it.at))); const t = h('span'); t.appendChild(h('span', 'title', it.operation + (it.target ? ' · ' + it.target : ''))); t.appendChild(h('span', 'sub', [(it.actor && (it.actor.label || it.actor.kind)) || '', it.detail].filter(Boolean).join(' · '))); r.appendChild(t); r.appendChild(stateEl(it.outcome === 'done' ? 'good' : 'serious', it.outcome)); li.appendChild(r); ul.appendChild(li); } + b.appendChild(ul); if (a.total > a.items.length) b.appendChild(h('p', 'note', (a.total - a.items.length) + ' earlier entries are not shown.')); + }); + if (E) E.decorate(frag); + }); + return editing; + } + /** The report in a frame: "Reading…" the first time; read again, the frame on screen stays until the new one is put in its place. */ + function report(main, data) { + const view = main._view, mine = main._paint = (main._paint || 0) + 1; + const head = () => { const bar = h('div', 'report-bar'); const back = h('a', 'btn sm ghost', '← Overview'); back.href = '#overview'; bar.appendChild(back); bar.appendChild(h('h2', null, 'Report')); return bar; }; + if (main._painted !== view) { main.textContent = ''; main.appendChild(head()); main.appendChild(h('p', 'note', 'Reading…')); } + const theme = getComputedStyle(document.documentElement).colorScheme === 'dark' ? 'dark' : 'light'; + const put = add => { if (main._paint !== mine || main._view !== view) return; const frag = document.createDocumentFragment(); frag.appendChild(head()); add(frag); putInPlace(main, frag); main._painted = view; }; + P.call('report', { theme }).then(r => put(frag => { const frame = h('iframe', 'report-frame'); frame.title = 'Project report'; frame.setAttribute('sandbox', ''); frame.srcdoc = r.html; frag.appendChild(frame); }), error => put(frag => failure(frag, error))); + return false; + } + + // ---- Theme toggle (system, light, dark) ----------------------------------- + const theme = { + init(btn) { + let cur = 'auto'; try { cur = localStorage.getItem('tuldok-theme') || 'auto'; } catch (e) { /* storage refused: the system theme */ } + const q = new URLSearchParams(location.search).get('theme'); if (q === 'light' || q === 'dark') cur = q; + const apply = () => { if (cur === 'auto') document.documentElement.removeAttribute('data-theme'); else document.documentElement.setAttribute('data-theme', cur); btn.textContent = ''; btn.appendChild(icon('theme')); btn.appendChild(document.createTextNode(' ' + { auto: 'System', light: 'Light', dark: 'Dark' }[cur])); btn.setAttribute('aria-label', 'Theme: ' + { auto: 'system', light: 'light', dark: 'dark' }[cur]); }; + btn.addEventListener('click', () => { cur = { auto: 'light', light: 'dark', dark: 'auto' }[cur]; try { localStorage.setItem('tuldok-theme', cur); } catch (e) { /* kept for this page only */ } apply(); }); + apply(); + } + }; + + // The Data, Train, Benchmarks and Analysis screens are added by tabs.js, which builds with these parts. + const parts = { sect, cell, block, time, bytes, failure, modelState, graphFor, putInPlace, putDrawer, turn }; + const S = { map, overview, inspectorModel, needsList, icon, stateEl, theme, parts }; + return S; +})(); diff --git a/static/project-workspace/serving.css b/static/project-workspace/serving.css new file mode 100644 index 00000000..f2cca336 --- /dev/null +++ b/static/project-workspace/serving.css @@ -0,0 +1,19 @@ +/* Serving a trained model (docs/plans/training-and-curation/spec-t6.md section 7), on the tokens of base.css, so light + and dark follow it. */ +.srv { display: grid; gap: 8px; min-width: 0; } +.srv-head { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 10px; min-width: 0; } +.srv-line { color: var(--ink-2); overflow-wrap: anywhere; min-width: 0; } +.srv-address { margin: 0; color: var(--ink-2); overflow-wrap: anywhere; } +.srv-acts:empty { display: none; } +.srv-fail p { margin: 0 0 6px; color: var(--serious); } +.srv-tail { max-height: 180px; } +.srv h4, .srv-times h4 { margin: 4px 0 2px; font-size: 13px; } +.srv-summary { font-size: 12.5px; } +.srv-try { display: grid; gap: 6px; min-width: 0; } +.srv-inputs { width: 100%; min-width: 0; font: 12px/1.5 var(--mono); padding: 8px; white-space: pre; overflow: auto; box-sizing: border-box; } +.srv-try .actions { margin: 0; } +.srv-answer { min-width: 0; } +.srv-timing { margin: 0 0 4px; color: var(--ink-2); } +.srv-outputs { max-height: 220px; } +.srv-rows .srv-row .sub.mono { overflow-wrap: anywhere; } +.srv-box { min-width: 0; } diff --git a/static/project-workspace/serving.js b/static/project-workspace/serving.js new file mode 100644 index 00000000..284d1691 --- /dev/null +++ b/static/project-workspace/serving.js @@ -0,0 +1,212 @@ +'use strict'; +// Serving a trained model on the page (docs/plans/training-and-curation/spec-t6.md section 7): the Train tab's +// Served models (a row per recipe that declares serve, opening its serving drawer, serve:), and a Serving +// block in a model's drawer on the Map. Each panel shows the model's state, the endpoint's address, the controls +// (Serve, or Ask the owner to serve it; Approve and Decline; Stop, in two presses), the output's tail of a failure, +// the session's timing summary and Try it: the inputs as JSON, Send, and the answer of POST /api/serve//predict. +// +// Updating in place (spec-w4): a control's answer reads the project again as every control does (actions.js). While a +// panel or a row on the page shows a session starting or awaiting approval, `served` is asked every second and each +// one's state mark, line and controls are set in place, its Try it shown once it is ready; nothing else is read again +// and nothing is rebuilt. Try it reads nothing of the project; after an answer the panel's timing summary is read +// again (serve_session) and set in place. +window.LabServing = (() => { + const P = window.TuldokProject, S = window.LabScreens, A = window.LabActions, h = window.LabCharts.h; + const POLL_MS = 1000, SHOWN = 4000; + const TONE = { 'awaiting-approval': ['warn', 'awaiting approval'], starting: ['pending', 'starting'], ready: ['good', 'ready'], + failed: ['serious', 'failed'], stopped: ['neutral', 'stopped'], declined: ['neutral', 'declined'], expired: ['neutral', 'expired'] }; + const OPEN = ['awaiting-approval', 'starting', 'ready'], WAITING = ['awaiting-approval', 'starting']; + const nameOf = ref => ref.slice(ref.indexOf('/') + 1); + const address = ref => 'POST /api/serve/' + nameOf(ref) + '/predict'; + const ms = v => v == null ? '—' : v < 10 ? v.toFixed(2) + ' ms' : v < 1000 ? v.toFixed(1) + ' ms' : (v / 1000).toFixed(2) + ' s'; + const button = (cls, text, key, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (key) b.dataset.key = key; if (onClick) b.addEventListener('click', onClick); return b; }; + const current = item => item.session && OPEN.includes(item.session.state) ? item.session : null; + + /** {tone, word, line} of a servable model: its open session, else its last one, else whether it can be served. */ + function standing(item) { + const s = item.session; + if (s && OPEN.includes(s.state)) { + const [tone, word] = TONE[s.state]; + const line = s.state === 'ready' ? s.requests + (s.requests === 1 ? ' request' : ' requests') + (s.failed_requests ? ', ' + s.failed_requests + ' failed' : '') + ' · stops after ' + s.idle_minutes + ' min idle' + : s.state === 'starting' ? 'loading its product in ' + (s.environment || 'its environment') + : 'asked by ' + ((s.requested_by && (s.requested_by.label || s.requested_by.kind)) || 'someone') + '; the owner approves it'; + return { tone, word, line }; + } + if (s) { const [tone, word] = TONE[s.state] || ['neutral', s.state]; return { tone, word, line: (s.reason || s.problem || '') + (item.built ? '' : ' · not built') }; } + return item.built ? { tone: 'neutral', word: 'not served', line: 'built; it can be served' } : { tone: 'neutral', word: 'not served', line: 'not built: build it before serving it' }; + } + function controls(item) { + const s = current(item), ref = item.model; + const items = []; + if (!s && item.built) items.push({ id: 'serve', label: A.may('owner') ? 'Serve' : 'Ask the owner to serve it', cls: 'btn sm primary', who: 'request', op: 'serve_start', args: () => ({ model: ref }), busy: 'Starting…' }); + if (s && s.state === 'awaiting-approval') { + items.push({ id: 'approve', label: 'Approve', cls: 'btn sm primary', who: 'owner', op: 'serve_approve', args: () => ({ session: s.id }), busy: 'Approving…' }); + items.push({ id: 'decline', label: 'Decline', cls: 'btn sm', who: 'cancel', op: 'serve_stop', args: () => ({ session: s.id }), busy: 'Declining…', confirm: 'Decline serving this model; nothing is started.' }); + } + if (s && (s.state === 'starting' || s.state === 'ready')) + items.push({ id: 'stop', label: 'Stop', cls: 'btn sm', who: 'cancel', op: 'serve_stop', args: () => ({ session: s.id }), busy: 'Stopping…', confirm: 'Stop serving this model; requests to it are refused from then on.' }); + return A.group('serve:' + ref + ':' + (s ? s.id + ':' + s.state : 'none'), items, { inline: true }); + } + + // ---- In place -------------------------------------------------------------------------------------------------- + /** Sets the state of every row and panel of item.model on the page from item, in place. */ + function update(item) { + for (const el of document.querySelectorAll('[data-serve-model]')) { + if (el.dataset.serveModel !== item.model || !el._serve) continue; + el._serve(item); + } + } + let timer = null; + function watch() { + if (timer) return; + timer = setTimeout(async () => { + timer = null; + const waiting = [...document.querySelectorAll('[data-serve-model]')].filter(el => el.isConnected && WAITING.includes(el.dataset.serveState)); + if (!waiting.length) return; + try { const served = await P.call('served'); for (const item of served.servable) update(item); } catch (error) { /* asked again on the next tick */ } + watch(); + }, POLL_MS); + } + /** The state mark and line of an element: replaced only when they differ. */ + function paintState(el, mark, line, item) { + const st = standing(item), stateKey = st.tone + ' ' + st.word; + if (mark._key !== stateKey) { const fresh = S.stateEl(st.tone, st.word); fresh._key = stateKey; fresh.classList.add('srv-mark'); mark.replaceWith(fresh); mark = fresh; } + if (line.textContent !== st.line) line.textContent = st.line; + el.dataset.serveState = item.session ? item.session.state : 'none'; + if (WAITING.includes(el.dataset.serveState)) watch(); + return mark; + } + + // ---- Train: Served models ------------------------------------------------------------------------------------------ + function section(frag, served, sel, act) { + const head = S.parts.sect('Served models', served.error ? '' : served.servable.length + ' declare serve · at most ' + served.most + ' at once'); head.classList.add('gap'); + frag.appendChild(head); + if (served.error) { frag.appendChild(A.refusal(served.error)); return; } + if (!served.servable.length) { frag.appendChild(h('p', 'empty', 'No recipe declares serve. A recipe serves its model through its own serve.py (see the project format).')); return; } + const ul = h('ul', 'rows srv-rows'); + for (const item of served.servable) { + const key = 'serve:' + item.model, li = h('li'), b = button('row env srv-row', null, 'open-' + key, () => act.select(sel === key ? null : key)); + b.setAttribute('aria-pressed', String(sel === key)); b.dataset.serveModel = item.model; + let mark = h('span'); b.appendChild(mark); + const t = h('span'), line = h('span', 'sub'); t.appendChild(h('span', 'title', item.title)); t.appendChild(line); b.appendChild(t); + b.appendChild(h('span', 'sub mono', nameOf(item.model) + ' · ' + item.module_file)); + b._serve = it => { mark = paintState(b, mark, line, it); }; + b._serve(item); + li.appendChild(b); ul.appendChild(li); + } + frag.appendChild(ul); + } + + // ---- A panel: the drawer and the block ------------------------------------------------------------------------------ + /** The panel of one servable item, updated in place by _serve(item). */ + function panel(item, heading) { + const box = h('section', 'srv'); box.dataset.serveModel = item.model; + if (heading) box.appendChild(h('h3', null, heading)); + const head = h('div', 'srv-head'); let mark = h('span'); const line = h('span', 'small srv-line'); head.appendChild(mark); head.appendChild(line); box.appendChild(head); + const where = h('p', 'small mono srv-address', address(item.model)); box.appendChild(where); + const acts = h('div', 'srv-acts'); box.appendChild(acts); + const fail = h('div', 'srv-fail'); fail.hidden = true; box.appendChild(fail); + const times = h('div', 'srv-times'); times.hidden = true; box.appendChild(times); + const tryBox = tryIt(item, () => box._times()); tryBox.hidden = true; box.appendChild(tryBox); + let cur = item, shown = null; + box._times = async () => { + const s = cur.session; + if (!s) { times.hidden = true; return; } + let found; + try { found = await P.call('serve_session', { session: s.id }); } catch (error) { times.hidden = true; return; } + if (cur.session && cur.session.id === found.id) { // the counts of the line, as the summary has them + cur = Object.assign({}, cur, { session: Object.assign({}, cur.session, { requests: found.requests, failed_requests: found.failed_requests }) }); + mark = paintState(box, mark, line, cur); + } + paintTimes(times, found); + }; + box._serve = it => { + cur = it; + mark = paintState(box, mark, line, it); + const s = it.session, now = (s ? s.id + ':' + s.state : 'none') + ':' + it.built; + if (now === shown) return; // the controls, the failure and Try it change with the state only + shown = now; + acts.replaceChildren(controls(it)); + const failed = s && s.state === 'failed'; + fail.hidden = !failed; fail.replaceChildren(); + if (failed) { fail.appendChild(h('p', 'small', s.problem || 'It failed.')); if (s.output_tail) { const pre = h('pre', 'evtext srv-tail'); pre.textContent = s.output_tail; fail.appendChild(pre); } } + tryBox.hidden = !(s && s.state === 'ready'); + box._times(); + }; + box._serve(item); + return box; + } + function paintTimes(times, found) { + const sum = found.summary; + times.hidden = !sum.count; + if (!sum.count) return; + const dl = h('dl', 'facts srv-summary'); + const put = (k, v) => { dl.appendChild(h('dt', null, k)); dl.appendChild(h('dd', 'num', v)); }; + put('Requests', sum.count + (sum.failed ? ' (' + sum.failed + ' failed)' : '') + (sum.throughput_per_second != null ? ' · ' + sum.throughput_per_second.toFixed(1) + ' a second' : '')); + for (const [k, title] of [['total_ms', 'Total'], ['model_ms', 'Model'], ['queue_ms', 'Queue']]) if (sum[k]) put(title, 'p50 ' + ms(sum[k].p50) + ' · p95 ' + ms(sum[k].p95) + ' · p99 ' + ms(sum[k].p99)); + times.replaceChildren(h('h4', null, 'This session'), dl); + } + /** Try it: the inputs as JSON, Send, and the answer; nothing of the project is read again. */ + function tryIt(item, after) { + const box = h('div', 'srv-try'), ref = item.model; + box.appendChild(h('h4', null, 'Try it')); + const area = h('textarea', 'srv-inputs'); area.rows = 4; area.spellcheck = false; area.dataset.key = 'serve-inputs-' + ref; area.setAttribute('aria-label', 'Inputs as JSON'); + area.value = JSON.stringify(item.example || {}); + const out = h('div', 'srv-answer'); out.setAttribute('aria-live', 'polite'); + const send = button('btn sm primary', 'Send', 'serve-send-' + ref, async () => { + let inputs; + try { inputs = JSON.parse(area.value); if (!inputs || typeof inputs !== 'object' || Array.isArray(inputs)) throw Error('The inputs must be a JSON object.'); } + catch (error) { out.replaceChildren(A.refusal(Error(error.message))); return; } + send.disabled = true; send.textContent = 'Sending…'; send.setAttribute('aria-busy', 'true'); + try { + const response = await window.TuldokAuth.fetch('/api/serve/' + nameOf(ref) + '/predict', { method: 'POST', cache: 'no-store', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ inputs }) }); + let value; try { value = await response.json(); } catch (e) { value = { error: 'The server answered ' + response.status + ' without JSON.' }; } + if (!response.ok) throw Object.assign(Error(value.error || 'Refused.'), { code: value.code }); + const text = JSON.stringify(value.outputs, null, 1), pre = h('pre', 'evtext srv-outputs'); + pre.textContent = text.length > SHOWN ? text.slice(0, SHOWN) + '\n…' : text; + out.replaceChildren(h('p', 'small srv-timing', 'total ' + ms(value.timing.total_ms) + ' · model ' + ms(value.timing.model_ms) + ' · queue ' + ms(value.timing.queue_ms)), pre); + after(); + } catch (error) { out.replaceChildren(A.refusal(error)); } + finally { send.disabled = false; send.textContent = 'Send'; send.removeAttribute('aria-busy'); } + }); + const row = h('div', 'actions'); row.appendChild(send); + box.appendChild(area); box.appendChild(row); box.appendChild(out); + box.appendChild(h('p', 'note', 'Arrays may be written as {"array": "float32", "shape": […], "base64": "…"}; the answer is shown as it came.')); + return box; + } + + /** The Train tab's drawer of one servable model (serve:). */ + function drawer(aside, key, act) { + const ref = key.slice(6); + if (aside._for === key && !aside._again) return; + aside._for = key; + P.call('served').then(served => { + if (aside._for !== key) return; + const item = served.servable.find(x => x.model === ref); + const ins = h('div', 'inspector'), hd = h('header'); + hd.appendChild(h('span', 'kicker', 'serving · ' + nameOf(ref))); + const t = h('h2', null, item ? item.title : ref); t.tabIndex = -1; hd.appendChild(t); ins.appendChild(hd); + if (!item) ins.appendChild(h('p', 'empty', ref + ' declares no serve.')); + else { + const go = h('div', 'actions'); go.appendChild(button('btn sm ghost', 'On the map', null, () => act.go('map', ref))); ins.appendChild(go); + ins.appendChild(panel(item, null)); + const dl = h('dl', 'facts'); + for (const [k, v] of [['Entry point', item.module_file], ['Environment', item.environment || '—'], ['Product', item.product_run ? 'run ' + item.product_run.slice(0, 8) : 'none'], ['Idle stop', item.idle_minutes + ' min']]) { dl.appendChild(h('dt', null, k)); dl.appendChild(h('dd', 'mono', v)); } + ins.appendChild(dl); + } + S.parts.putDrawer(aside, ins); + }, error => { if (aside._for !== key) return; const frag = document.createDocumentFragment(); frag.appendChild(A.refusal(error)); S.parts.putDrawer(aside, frag); }); + } + + /** The Serving block of a model's drawer (screens.js): shown only for a model whose recipe declares serve. */ + async function block(box, m, live) { + const served = await P.call('served'); + if (!live()) return; + const item = served.servable.find(x => x.model === m.id); + if (!item) { box.hidden = true; return; } + box.replaceChildren(panel(item, 'Serving')); + box.hidden = false; + } + + return { section, drawer, block }; +})(); diff --git a/static/project-workspace/structure.js b/static/project-workspace/structure.js new file mode 100644 index 00000000..8ebcfb60 --- /dev/null +++ b/static/project-workspace/structure.js @@ -0,0 +1,366 @@ +'use strict'; +// A model's structure (docs/plans/training-and-curation/spec-t2.md section 4): the block "Structure" of a model's +// drawer, right after Weights, when one of the model's current products holds model-graph.json. The graph the +// project's script recorded is drawn left to right in SVG (ranks by the longest path from the inputs, each rank +// ordered by the barycentre of its neighbours), its modules folded into groups that open when pressed and close +// from the module outline under the drawing. An operation pressed shows its details, whose tensors open in the +// Weights block (LabWeights.reveal). "Colour by" fills the nodes from model_graph_overlay; "Compare with" draws +// the graph of the model it derives from, or of one derived from it, under this one, folded the same way, and +// marks the modules model_graph_compare finds added, missing or changed. +// +// Updating in place (spec-w4): the block is built off the page and put in place whole, like weights.js's. Read +// again for the model on screen (screens.js), it is built from the person's place in it (folding, selection, +// colour, comparison) and resolves only once complete; the person's own choices draw it again the same way, the +// focus kept by data-key and the drawing's scroll by putInPlace. +window.LabStructure = (() => { + const P = window.TuldokProject, S = window.LabScreens, C = window.LabCharts, h = C.h, R = P.product; + const { putInPlace } = S.parts; + const SVG = 'http://www.w3.org/2000/svg'; + const W = 132, H = 42, GAP = 40, VGAP = 14, PAD = 10, SWEEPS = 4, LABEL = 18; + const num = n => n == null ? '—' : n.toLocaleString('en-GB'); + const plural = (n, one, many) => num(n) + ' ' + (n === 1 ? one : many || one + 's'); + const shapeText = s => !s ? '' : s.length ? s.join('×') : 'scalar'; + const valuesText = list => (list || []).map(v => shapeText(v.shape) + ' ' + v.dtype).join(', ') || '—'; + const short = text => text.length > LABEL ? text.slice(0, LABEL - 1) + '…' : text; + const classShort = c => c ? c.split('.').pop() : ''; + const button = (cls, text, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (onClick) b.addEventListener('click', onClick); return b; }; + const refusal = error => window.LabActions.refusal(error); + const svg = (tag, attrs) => { const e = document.createElementNS(SVG, tag); for (const [k, v] of Object.entries(attrs || {})) e.setAttribute(k, v); return e; }; + const COLOURS = [['', 'Nothing'], ['parameters', 'Parameters'], ['outputs', 'Output size'], ['operations', 'Operations'], ['change:start', 'Change from the weights it started from'], ['change:parent', 'Change from the model it derives from']]; + let drawings = 0; // a number per drawing, for the ids of its arrow markers + + // ---- The person's place in each model's block ------------------------------------------------------------------- + const places = new Map(); + const placeOf = id => { if (!places.has(id)) places.set(id, { depth: null, open: [], close: [], selected: null, colour: '', compare: '', fit: false }); return places.get(id); }; + const foldArgs = (st, depth) => st.depth == null && !st.open.length && !st.close.length && depth == null ? {} : { depth: st.depth == null ? depth : st.depth, open: st.open.slice().sort(), close: st.close.slice().sort() }; + + // ---- Reading ---------------------------------------------------------------------------------------------------------- + /** The first current product of a model that holds a graph: { run, subject, graph } (its graph folded the default way), or null. */ + async function graphRun(reference) { + const products = (await P.read('products', { reference })).products.filter(p => p.run_id); + for (const p of products) { + const graph = await R.graph(p.run_id, {}); + if (graph.path) return { run: p.run_id, subject: p.subject || '', graph }; // path null: no graph in this product + } + return null; + } + + /** + * Fills box (a block of the drawer of model m, on the page or not yet) with the Structure block, or hides it + * when no product of the model holds a graph. Resolves once the block is complete. + */ + async function block(box, m, g, scroller, live) { + const st = placeOf(m.id), ctx = { box, m, g, st, scroller, live, turn: 0 }; + let root; + try { root = await build(ctx); } catch (error) { root = h('div', 'st'); root.appendChild(h('h3', null, 'Structure')); root.appendChild(refusal(error)); } + if (!live()) return; + st.ctx = ctx; + if (!root) { box.hidden = true; box.replaceChildren(); return; } + box.hidden = false; box.replaceChildren(root); + } + async function again(ctx) { + const mine = ++ctx.turn; + ctx.box.setAttribute('aria-busy', 'true'); + let root; try { root = await build(ctx); } catch (error) { root = h('div', 'st'); root.appendChild(h('h3', null, 'Structure')); root.appendChild(refusal(error)); } + if (mine !== ctx.turn || !ctx.box.isConnected) return; + ctx.box.removeAttribute('aria-busy'); + putInPlace(ctx.box, root || h('div'), ctx.scroller); + } + + /** The Structure block drawn again over the page, full width; Escape or "Back to the drawer" closes it. */ + function openWide(ctx) { + if (ctx.st.wideCtx) return; + const box = h('div', 'st-overlay'); box.setAttribute('role', 'dialog'); box.setAttribute('aria-label', 'Structure, full width'); + document.body.appendChild(box); + const w = { ...ctx, box, scroller: box, turn: 0, wide: true, base: ctx }; + w.esc = ev => { if (ev.key === 'Escape') closeWide(w); }; + document.addEventListener('keydown', w.esc); + ctx.st.wideCtx = w; + again(w).then(() => { const b = box.querySelector('[data-key="st-wide"]'); if (b) b.focus(); }); + } + function closeWide(w) { + document.removeEventListener('keydown', w.esc); + w.box.remove(); w.st.wideCtx = null; + again(w.base).then(() => { const b = w.base.box.querySelector('[data-key="st-wide"]'); if (b) b.focus(); }); + } + + /** The other models it can be compared with: the model it derives from and those derived from it, with a graph. */ + async function choicesFor(m, g) { + const refs = [m.parent].filter(p => p && g.byId.has(p)).map(p => ['parent', p]).concat(g.nodes.filter(n => n.parent === m.id).map(n => ['child', n.id])); + const out = []; + for (const [rel, ref] of refs) { + const got = await graphRun(ref).catch(() => null), o = g.byId.get(ref); + if (got) out.push({ key: ref, run: got.run, label: (rel === 'parent' ? 'The model it derives from: ' : 'Derived from it: ') + (o.short ? o.short + ' · ' : '') + o.title }); + } + return out; + } + + async function build(ctx) { + const { m, g, st } = ctx; + const found = await graphRun(m.id); + if (!found) return null; + const { run } = found; + const args = foldArgs(st, null); + const graph = Object.keys(args).length ? await R.graph(run, args) : found.graph; + const root = h('div', 'st'); + root.appendChild(h('h3', null, 'Structure')); + const c = graph.counts; + root.appendChild(h('p', 'small muted st-size', [graph.path + (found.subject ? ' · ' + found.subject : ''), plural(c.operations, 'operation'), plural(c.modules, 'module'), plural(c.tensors, 'tensor')].join(' · '))); + root.appendChild(h('p', 'small faint st-how', 'Recorded by ' + (graph.method || 'the project') + (graph.torch ? ' with PyTorch ' + graph.torch : '') + (graph.inputs.length ? '; example input ' + graph.inputs.map(v => v.name + ' ' + shapeText(v.shape)).join(', ') : '') + '.')); + for (const note of graph.notes) root.appendChild(h('p', 'small warnline', note)); + if (!graph.complete) { + root.appendChild(h('p', 'empty st-reason', 'Only the module tree was recorded: ' + (graph.reason || 'no reason given') + '.')); + root.appendChild(outline(ctx, graph, null)); + return root; + } + // The controls: colour, comparison, folding. + const colourChoices = COLOURS.filter(([k]) => k !== 'change:parent' || m.parent); + if (!colourChoices.some(([k]) => k === st.colour)) st.colour = ''; + const compareChoices = await choicesFor(m, g); + if (!compareChoices.some(x => x.key === st.compare)) st.compare = ''; + const bar = h('div', 'st-bar'); + const sel = (label, key, options, value, onChange) => { + const s = h('select'); s.setAttribute('aria-label', label); s.dataset.key = key; + for (const [v, t] of options) { const o = h('option', null, t); o.value = v; o.selected = v === value; s.appendChild(o); } + s.addEventListener('change', () => onChange(s.value)); + const l = h('label', 'iv-field st-pick'); l.appendChild(document.createTextNode(label)); l.appendChild(s); return l; + }; + bar.appendChild(sel('Colour by', 'st-colour', colourChoices, st.colour, v => { st.colour = v; again(ctx); })); + if (compareChoices.length) bar.appendChild(sel('Compare with', 'st-compare', [['', 'Nothing']].concat(compareChoices.map(x => [x.key, x.label])), st.compare, v => { st.compare = v; again(ctx); })); + const folds = h('div', 'st-folds'); + const all = button('btn sm', 'Open all', () => { Object.assign(st, { depth: 64, open: [], close: [] }); again(ctx); }); all.dataset.key = 'st-open-all'; + const one = button('btn sm', 'Fold to depth 1', () => { Object.assign(st, { depth: 1, open: [], close: [] }); again(ctx); }); one.dataset.key = 'st-fold-1'; + const fit = button('btn sm', 'Fit to width', () => { st.fit = !st.fit; again(ctx); }); fit.dataset.key = 'st-fit'; fit.setAttribute('aria-pressed', String(st.fit)); + fit.title = 'Shrink the drawings to the drawer\'s width, for an overview; press again for their actual size'; + // The drawer is narrow: the same view, with the same state, opens over the page at its full width. + const wide = button('btn sm', ctx.wide ? 'Back to the drawer' : 'Open wide', () => (ctx.wide ? closeWide(ctx) : openWide(ctx))); + wide.dataset.key = 'st-wide'; + folds.appendChild(all); folds.appendChild(one); folds.appendChild(fit); folds.appendChild(wide); bar.appendChild(folds); + root.appendChild(bar); + // The overlay and the comparison, read in full before anything is drawn. + let overlay = null, overlayError = null; + if (st.colour) { + const [kind, against] = st.colour.split(':'); + const oargs = { kind }; + if (against === 'start') oargs.against = 'start'; + if (against === 'parent') { const p = (await P.read('products', { reference: m.parent })).products.find(x => x.run_id); oargs.against = 'run'; oargs.other_run = p ? p.run_id : null; } + try { overlay = await R.graphOverlay(run, oargs); } catch (error) { overlayError = error; } + } + const choice = compareChoices.find(x => x.key === st.compare) || null; + let other = null, cmp = null, cmpError = null; + if (choice) { + try { [other, cmp] = await Promise.all([R.graph(choice.run, { depth: graph.depth, open: st.open.slice().sort(), close: st.close.slice().sort() }), R.graphCompare(run, choice.run)]); } + catch (error) { cmpError = error; } + } + if (overlayError) root.appendChild(refusal(overlayError)); + if (cmpError) root.appendChild(refusal(cmpError)); + const status = cmp ? statuses(cmp, false) : null; + root.appendChild(drawing(ctx, graph, overlay, status, 'This model', true)); + if (overlay) root.appendChild(legend(overlay)); + const chosen = graph.nodes.find(n => n.id === st.selected); + if (chosen) root.appendChild(details(ctx, graph, chosen, overlay, status)); + if (other) { + const head = h('h4', 'st-sub', (g.byId.get(choice.key).short ? g.byId.get(choice.key).short + ' · ' : '') + g.byId.get(choice.key).title); + root.appendChild(head); + root.appendChild(drawing(ctx, other, null, statuses(cmp, true), 'The other model', false)); + root.appendChild(differences(cmp)); + } + root.appendChild(outline(ctx, graph, status)); + return root; + } + + // ---- The comparison -------------------------------------------------------------------------------------------------- + /** {path: status} from a model_graph_compare answer, seen from this graph or (mirrored) from the other's. */ + function statuses(cmp, mirrored) { + const out = new Map(); + for (const r of cmp.modules) out.set(r.path, mirrored ? ({ added: 'missing', missing: 'added' }[r.status] || r.status) : r.status); + return out; + } + /** A node's mark: its module's status; a group's, or "inside" when a module within it differs. */ + function markOf(node, status) { + if (!status || node.kind === 'input' || node.kind === 'output') return null; + const own = status.get(node.module); + if (own && own !== 'same' && own !== 'missing') return own; + if (node.kind === 'group') for (const [path, s] of status) if (s !== 'same' && s !== 'missing' && path.startsWith(node.module + '.')) return 'inside'; + return null; + } + const MARK_WORD = { added: 'only in this model', missing: 'only in the other model', changed: 'changed', inside: 'something inside it differs' }; + function differences(cmp) { + const box = h('div', 'st-diff'); const rows = cmp.modules.filter(r => r.status !== 'same'); + box.appendChild(h('p', 'small muted', rows.length ? 'Modules matched by path: ' + ['changed', 'added', 'missing'].map(k => cmp.counts[k] + ' ' + { changed: 'changed', added: 'only here', missing: 'only in the other' }[k]).join(', ') + '.' : 'Every module matches by path, class and tensor shapes.')); + if (!rows.length) return box; + const ul = h('ul', 'st-difflist'); + for (const r of rows) { + const li = h('li', 'st-d-' + r.status); li.dataset.path = r.path; li.dataset.status = r.status; + li.appendChild(h('span', 'mono', r.path || '(the model)')); + const why = r.status === 'added' ? 'only in this model' : r.status === 'missing' ? 'only in the other' : [r.class !== r.other_class ? classShort(r.other_class) + ' → ' + classShort(r.class) : null].concat(r.reshaped.map(x => x.name.split('.').pop() + ' ' + shapeText(x.other_shape) + ' → ' + shapeText(x.shape)), r.only_here.map(n => n.split('.').pop() + ' added'), r.only_there.map(n => n.split('.').pop() + ' removed')).filter(Boolean).join('; '); + li.appendChild(h('span', 'small muted', ' ' + why)); ul.appendChild(li); + } + box.appendChild(ul); return box; + } + + // ---- The layout ---------------------------------------------------------------------------------------------------------- + /** Positions of the shown nodes: ranks by the longest path from the inputs, each rank ordered by barycentre. */ + function layout(nodes, edges) { + const at = new Map(nodes.map((n, i) => [n.id, i])), rank = new Array(nodes.length).fill(0); + const into = nodes.map(() => []), out = nodes.map(() => []), forward = []; + for (const e of edges) { const a = at.get(e.from), b = at.get(e.to); if (a == null || b == null) continue; if (a < b) { into[b].push(a); out[a].push(b); forward.push(e); } } + for (let i = 0; i < nodes.length; i++) for (const a of into[i]) rank[i] = Math.max(rank[i], rank[a] + 1); + const ranks = []; for (let i = 0; i < nodes.length; i++) (ranks[rank[i]] = ranks[rank[i]] || []).push(i); + const pos = new Array(nodes.length).fill(0); + const place = () => { for (const r of ranks) r.forEach((i, k) => { pos[i] = k; }); }; + place(); + const bary = (i, side) => { const n = side[i]; return n.length ? n.reduce((s, j) => s + pos[j], 0) / n.length : pos[i]; }; + for (let sweep = 0; sweep < SWEEPS; sweep++) { + const down = sweep % 2 === 0, order = down ? ranks.slice(1) : ranks.slice(0, -1).reverse(); + for (const r of order) { const b = new Map(r.map(i => [i, bary(i, down ? into : out)])); r.sort((x, y) => b.get(x) - b.get(y) || x - y); r.forEach((i, k) => { pos[i] = k; }); } + } + const most = Math.max(1, ...ranks.map(r => r.length)); + const xy = nodes.map((_, i) => { const r = ranks[rank[i]]; return { x: PAD + rank[i] * (W + GAP), y: PAD + (pos[i] + (most - r.length) / 2) * (H + VGAP) }; }); + return { xy, at, width: PAD * 2 + ranks.length * W + (ranks.length - 1) * GAP, height: PAD * 2 + most * H + (most - 1) * VGAP + 18, rank }; + } + + // ---- The drawing ------------------------------------------------------------------------------------------------------ + /** The value of a node in an overlay: a group's module's, an operation's own. */ + const valueOf = (node, overlay) => !overlay ? null : node.kind === 'group' ? overlay.modules[node.module] : node.kind === 'operation' ? overlay.nodes[node.id] : null; + const LOG = { parameters: true, outputs: true, operations: true }; + /** 0..1 of a value on the overlay's scale (logarithmic for counts). */ + function scaled(v, overlay) { + if (v == null || !Number.isFinite(v)) return null; + const f = LOG[overlay.kind] ? x => Math.log10(1 + Math.max(0, x)) : x => x; + const lo = Math.min(f(0), f(overlay.low == null ? 0 : overlay.low)), hi = f(overlay.high == null ? 0 : overlay.high); + return hi > lo ? Math.min(1, Math.max(0, (f(v) - lo) / (hi - lo))) : 0; + } + const fmtValue = (v, overlay) => v == null ? 'no value' : overlay.kind === 'change' ? C.fmtSci(v) : num(v) + ' ' + overlay.unit; + + function drawing(ctx, graph, overlay, status, title, mine) { + const { st } = ctx, id = ++drawings, lay = layout(graph.nodes, graph.edges); + const box = h('div', 'st-draw'); box.dataset.which = mine ? 'this' : 'other'; + const s = svg('svg', { width: lay.width, height: lay.height, viewBox: '0 0 ' + lay.width + ' ' + lay.height, role: 'group', 'aria-label': title + ': ' + plural(graph.nodes.length, 'node') + ', ' + plural(graph.edges.length, 'edge') + ', drawn left to right' }); + const defs = svg('defs'), marker = svg('marker', { id: 'st-arrow-' + id, viewBox: '0 0 8 8', refX: 7, refY: 4, markerWidth: 7, markerHeight: 7, orient: 'auto-start-reverse' }); + marker.appendChild(svg('path', { d: 'M0 0 L8 4 L0 8 z', class: 'st-arrowhead' })); defs.appendChild(marker); s.appendChild(defs); + const edges = svg('g', { class: 'st-edges' }); + for (const e of graph.edges) { + const a = lay.at.get(e.from), b = lay.at.get(e.to); if (a == null || b == null) continue; + const p = lay.xy[a], q = lay.xy[b]; + let d; + if (lay.rank[b] > lay.rank[a]) { const x1 = p.x + W, y1 = p.y + H / 2, x2 = q.x - 2, y2 = q.y + H / 2, mx = (x1 + x2) / 2; d = `M${x1} ${y1} C${mx} ${y1} ${mx} ${y2} ${x2} ${y2}`; } + else { const x1 = p.x + W / 2, y1 = p.y + H, x2 = q.x + W / 2, y2 = q.y + H + 2, low = Math.max(y1, y2) + 16; d = `M${x1} ${y1} C${x1} ${low} ${x2} ${low} ${x2} ${y2}`; } + const path = svg('path', { d, class: 'st-edge' + (lay.rank[b] > lay.rank[a] ? '' : ' back'), 'marker-end': 'url(#st-arrow-' + id + ')' }); + const t = svg('title'); t.textContent = e.from + ' → ' + e.to + (e.shape ? ': ' + shapeText(e.shape) + ' ' + e.dtype : '') + (e.count > 1 ? ' (' + e.count + ' values)' : ''); path.appendChild(t); + edges.appendChild(path); + } + s.appendChild(edges); + const layer = svg('g', { class: 'st-nodes' }); + graph.nodes.forEach((n, i) => { + const { x, y } = lay.xy[i], v = valueOf(n, overlay), t = overlay ? scaled(v, overlay) : null, mark = markOf(n, status); + const gEl = svg('g', { class: 'st-node ' + n.kind + (mark ? ' mark-' + mark : '') + (mine && st.selected === n.id ? ' selected' : '') + (t != null && t > 0.55 ? ' deep' : ''), transform: `translate(${x} ${y})`, tabindex: '0', role: 'button' }); + gEl.dataset.key = (mine ? 'st-node-' : 'st-other-') + n.id; gEl.dataset.id = n.id; gEl.dataset.module = n.module; if (mark) gEl.dataset.mark = mark; + if (t != null) gEl.dataset.value = String(v); + const sub = n.kind === 'group' ? classShort(n.class) + ' · ' + plural(n.operations, 'op') : n.outputs.length ? shapeText(n.outputs[0].shape) : ''; + const what = n.kind === 'group' ? 'Module ' + n.module + ' (' + classShort(n.class) + '), ' + plural(n.operations, 'operation') + ' folded; press to open it' : n.kind === 'operation' ? 'Operation ' + n.label + ' in ' + (n.module || 'the model') + ', gives ' + valuesText(n.outputs) + (mine ? '; press for its details' : '') : (n.kind === 'input' ? 'Input ' : 'Output ') + n.id + ', ' + valuesText(n.outputs); + gEl.setAttribute('aria-label', what + (overlay ? '; ' + fmtValue(v, overlay) : '') + (mark ? '; ' + MARK_WORD[mark] : '')); + const rect = svg('rect', { width: W, height: H, rx: n.kind === 'operation' ? 6 : n.kind === 'group' ? 3 : 14, class: 'st-box' }); + if (t != null) rect.style.fill = `color-mix(in srgb, var(--ramp-hi) ${Math.round(t * 100)}%, var(--ramp-lo))`; + gEl.appendChild(rect); + if (n.kind === 'group') gEl.appendChild(svg('rect', { x: 3, y: 3, width: W - 6, height: H - 6, rx: 2, class: 'st-inner' })); + const l1 = svg('text', { x: 8, y: 17, class: 'st-label' }); l1.textContent = short(n.kind === 'group' ? n.module.split('.').slice(-2).join('.') : n.label); gEl.appendChild(l1); + const l2 = svg('text', { x: 8, y: 32, class: 'st-subl' }); l2.textContent = short(sub); gEl.appendChild(l2); + if (mark) { const b = svg('circle', { cx: W - 8, cy: 8, r: 4, class: 'st-badge' }); gEl.appendChild(b); } + const tip = svg('title'); tip.textContent = what; gEl.appendChild(tip); + const act = () => { + if (n.kind === 'group') { openModule(st, n.module, graph.depth); st.selected = null; again(ctx); return; } + if (!mine) return; + st.selected = st.selected === n.id ? null : n.id; again(ctx); + }; + gEl.addEventListener('click', act); + gEl.addEventListener('keydown', ev => { if (ev.key === 'Enter' || ev.key === ' ') { ev.preventDefault(); act(); } }); + layer.appendChild(gEl); + }); + s.appendChild(layer); + if (st.fit) { s.setAttribute('width', '100%'); s.removeAttribute('height'); s.style.maxWidth = lay.width + 'px'; } // shrunk to the box, never grown + const scroll = h('div', 'st-scroll' + (st.fit ? ' fit' : '')); scroll.appendChild(s); box.appendChild(scroll); + return box; + } + function openModule(st, path, depth) { + if (st.depth == null) st.depth = depth; + st.close = st.close.filter(p => p !== path); + // Its parents must be open too for it to show. + const parts = path.split('.'); + for (let k = 1; k <= parts.length; k++) { const p = parts.slice(0, k).join('.'); st.close = st.close.filter(x => x !== p); if (k > st.depth && !st.open.includes(p)) st.open.push(p); } + } + function closeModule(st, path, depth) { + if (st.depth == null) st.depth = depth; + st.open = st.open.filter(p => p !== path && !p.startsWith(path + '.')); + if (path.split('.').length <= st.depth && !st.close.includes(path)) st.close.push(path); + } + + function legend(overlay) { + const box = h('div', 'st-legend'); + const ramp = h('span', 'st-ramp'); ramp.setAttribute('aria-hidden', 'true'); box.appendChild(ramp); + const lo = LOG[overlay.kind] ? 0 : overlay.low; + box.appendChild(h('span', 'small faint', fmtValue(lo, overlay) + ' to ' + fmtValue(overlay.high, overlay) + (LOG[overlay.kind] ? ', on a logarithmic scale' : '') + (overlay.against ? '; against ' + overlay.against.other_path + (overlay.against.other_run !== overlay.against.run ? ' of run ' + overlay.against.other_run.slice(0, 8) : '') : '') + '. Plain: no value.')); + box.dataset.kind = overlay.kind; + return box; + } + + // ---- A node's details -------------------------------------------------------------------------------------------------- + function details(ctx, graph, n, overlay, status) { + const { st, m } = ctx, box = h('div', 'st-detail'); box.dataset.id = n.id; + const head = h('p', 'st-dtitle'); head.appendChild(h('strong', 'mono', n.kind === 'operation' ? n.label : n.id)); if (n.kind === 'operation') head.appendChild(h('span', 'small muted mono', ' ' + n.operation)); box.appendChild(head); + const dl = h('dl', 'setup iv-facts st-facts'); + const put = (k, v) => { dl.appendChild(h('dt', null, k)); dl.appendChild(h('dd', null, v)); }; + put('Module', (n.module || 'the model itself') + (n.class ? ' · ' + classShort(n.class) : '')); + put('Takes', valuesText(n.inputs)); put('Gives', valuesText(n.outputs)); + const attrs = Object.entries(n.attributes || {}); + if (attrs.length) put('Attributes', attrs.map(([k, v]) => k + ' = ' + JSON.stringify(v)).join(', ')); + if (overlay) put(COLOURS.find(([k]) => k.startsWith(overlay.kind))[1], fmtValue(valueOf(n, overlay), overlay)); + const mark = markOf(n, status); if (mark) put('Compared', MARK_WORD[mark]); + box.appendChild(dl); + if (n.tensors.length) { + box.appendChild(h('p', 'small muted', 'Its tensors, in the Weights block above:')); + const ul = h('ul', 'st-tensors'); + for (const name of n.tensors) { + const t = graph.tensors[name], li = h('li'); + const b = button('linkish mono', name, () => { if (window.LabWeights && window.LabWeights.reveal) window.LabWeights.reveal(m.id, name); }); + b.dataset.key = 'st-tensor-' + name; b.dataset.tensor = name; li.appendChild(b); + if (t) li.appendChild(h('span', 'small muted', ' ' + shapeText(t.shape) + ' ' + t.dtype + (t.kind !== 'parameter' ? ' · ' + t.kind : ''))); + ul.appendChild(li); + } + box.appendChild(ul); + } + if (n.module) { const c = button('btn sm', 'Fold ' + n.module, () => { closeModule(st, n.module, graph.depth); st.selected = null; again(ctx); }); c.dataset.key = 'st-fold-here'; box.appendChild(c); } + return box; + } + + // ---- The module outline ------------------------------------------------------------------------------------------------ + /** The module tree as a list; a module's row opens or closes it; the children of a closed module are not listed. */ + function outline(ctx, graph, status) { + const { st } = ctx, open = new Set(graph.open), box = h('div', 'st-outline'); + box.appendChild(h('p', 'small muted', 'Modules')); + const walk = (node, depth) => { + const ul = h('ul', 'st-tree'); + for (const child of node.children) { + const li = h('li'); li.dataset.path = child.path; + const isOpen = open.has(child.path), mark = status ? status.get(child.path) : null; + const row = h('div', 'st-row'); + const b = button('st-toggle', null, () => { if (isOpen) closeModule(st, child.path, graph.depth); else openModule(st, child.path, graph.depth); again(ctx); }); + b.dataset.key = 'st-mod-' + child.path; b.setAttribute('aria-expanded', String(isOpen)); + b.setAttribute('aria-label', (isOpen ? 'Fold ' : 'Open ') + child.path); + b.appendChild(h('span', 'st-caret', isOpen ? '▾' : '▸')); + b.appendChild(h('span', 'mono', child.path.split('.').pop())); + row.appendChild(b); + row.appendChild(h('span', 'small muted', classShort(child.class) + ' · ' + num(child.parameters))); + if (mark && mark !== 'same') { const s = h('span', 'wts-mark st-m-' + mark, { added: 'only here', missing: 'only there', changed: 'changed' }[mark]); row.appendChild(s); } + li.appendChild(row); + if (isOpen && child.children.length && depth < 64) li.appendChild(walk(child, depth + 1)); + ul.appendChild(li); + } + return ul; + }; + box.appendChild(graph.modules.children.length ? walk(graph.modules, 1) : h('p', 'small faint', 'The model has no modules inside it.')); + return box; + } + + return { block }; +})(); diff --git a/static/project-workspace/sweeps.css b/static/project-workspace/sweeps.css new file mode 100644 index 00000000..83991722 --- /dev/null +++ b/static/project-workspace/sweeps.css @@ -0,0 +1,46 @@ +/* The Sweeps tab and the kind of change "sweep trial" (docs/plans/training-and-curation/spec-t7.md section 7), on + the tokens of base.css. The trial's colour is the purple of the training runs (--run-3), already checked beside + the other kinds' colours for colour-blind separation and contrast in both themes. */ +:root { --k-sweep-trial: #8a4fbf; } +@media (prefers-color-scheme: dark) { :root:not([data-theme="light"]) { --k-sweep-trial: #9a72d8; } } +:root[data-theme="dark"] { --k-sweep-trial: #9a72d8; } + +/* The kind of change wherever the others are drawn. */ +.chip.sweep-trial::before { background: var(--k-sweep-trial); } +.chart .pt.sweep-trial { fill: var(--k-sweep-trial); } +.tree-list .dot.sweep-trial { fill: var(--k-sweep-trial); } +.lk-sweep-trial { stroke: var(--k-sweep-trial); } +.tree-map .tpt.sweep-trial { fill: var(--k-sweep-trial); } +.tree-map .node.sweep-trial circle { fill: var(--k-sweep-trial); } +.tree-map .node.planned.sweep-trial circle { fill: var(--surface); stroke: var(--k-sweep-trial); } +.tree-list .dot.planned.sweep-trial { fill: var(--surface); stroke: var(--k-sweep-trial); } +.pc .pcl.sweep-trial { stroke: var(--k-sweep-trial); } + +/* The tab. */ +.sw-head { display: grid; gap: 8px; min-width: 0; } +.sw-head .sect { margin-top: 6px; } +.sw-what { margin: 0; max-width: 980px; overflow-wrap: anywhere; } +.linkish { background: none; border: 0; padding: 0; color: var(--accent); cursor: pointer; font: inherit; text-decoration: underline; text-underline-offset: 2px; } +.linkish:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; } +.sw-controls { margin-top: 12px; display: grid; gap: 6px; } +.sw-controls .sw-ask { margin: 0; } +.sw-state { margin-top: 14px; } +.sw-dl { grid-template-columns: minmax(110px, max-content) minmax(0, 1fr); } +.sw-dl dd { min-width: 0; overflow-wrap: anywhere; } +.sw-gap { margin-left: 6px; } +.sw-meter { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; } +.sw-meter .hair { position: relative; width: 160px; max-width: 40vw; height: 4px; background: var(--grid); border-radius: 2px; flex: none; overflow: hidden; } +.sw-meter .hair i { position: absolute; left: 0; top: 0; bottom: 0; background: var(--accent); transition: width .3s; } +.sw-pc, .sw-chart, .sw-trials { margin-top: 18px; } +.sw-table tr.sw-row { cursor: pointer; } +.sw-table tr.sw-row:hover td, .sw-table tr.sw-row.on td { background: var(--accent-wash); } +.sw-table tr.sw-row:focus-visible { outline: 2px solid var(--focus); outline-offset: -2px; } +.sw-new .sw-text { width: 100%; min-height: 280px; font-size: 12.5px; } + +/* The trials' lines in parallel coordinates, by how they ended. */ +.pc .pcl.tr-completed { stroke: var(--k-sweep-trial); } +.pc .pcl.tr-running { stroke: var(--accent); stroke-width: 2.2; opacity: 1; } +.pc .pcl.tr-pruned { stroke: var(--ink-3); stroke-dasharray: 4 3; } +.pc .pcl.tr-stopped { stroke: var(--ink-3); stroke-dasharray: 1 3; } +.pc .pcl.tr-failed { stroke: var(--serious); stroke-dasharray: 4 3; } +.pc .pcl.tr-pruned.out, .pc .pcl.tr-stopped.out, .pc .pcl.tr-failed.out, .pc .pcl.tr-completed.out { opacity: .08; } diff --git a/static/project-workspace/sweeps.js b/static/project-workspace/sweeps.js new file mode 100644 index 00000000..d214fee2 --- /dev/null +++ b/static/project-workspace/sweeps.js @@ -0,0 +1,335 @@ +'use strict'; +// The Sweeps tab (docs/plans/training-and-curation/spec-t7.md section 7), added to window.LabScreens: one sweep at a +// time (the chooser, kept in the address as ?sweep=), its state live (the run, the trials by how they ended, the +// budget used, the running trial, the best so far), its controls (Plan sweep, Approve and Decline when a request +// awaits approval, Cancel), its trials through parallel coordinates (parallel.js, the trials as a benchmark's subjects, +// the main view), its frontier with two objectives (charts.js's tradeoff chart) or the progression of its one +// objective, and every trial in a table that opens it in the drawer or on the Map. The drawer holds a trial (the +// model's drawer), the plan view of the sweep, or New sweep: a recipe chosen, a sweep's text written from its +// settings, kept with Keep. +// +// Updating in place (spec-w4): the tab is built once for a sweep and the person's choices (tabs.js's paint). A +// reading of the page (app.js marks the drawer _again, and S.tick while a sweep runs) asks `sweep` again and sets what +// changed in place: the texts and bars of the state block (no node of it added or removed), the cells of the trials' +// rows (a row appended for a new trial), the charts drawn again in their hosts and parallel coordinates in their +// section, the controls when what they offer differs. The tab is never emptied by a reading. +(() => { + const S = window.LabScreens, P = window.TuldokProject, Tree = window.LabTree, C = window.LabCharts, h = C.h; + const A = window.LabActions; + const { sect, failure, putInPlace } = S.parts; + const fmt = C.fmtSci; + const TRIAL_TONE = { completed: 'good', running: 'pending', pruned: 'warn', stopped: 'neutral', failed: 'serious' }; + const TRIAL_STATES = ['completed', 'running', 'pruned', 'stopped', 'failed']; + const RUN_TONE = { running: 'pending', finished: 'good', cancelled: 'neutral', failed: 'serious' }; + const ENDED = { schedule: 'its schedule was done', trials: 'its trials were spent', minutes: 'its minutes were spent', cancelled: 'it was cancelled', failed: 'it failed' }; + const STATE_WORD = { 'not-built': 'not run yet', running: 'running', built: 'finished', failed: 'failed' }; + const REFRESH_MS = 500; // S.tick asks nothing when the drawing of this reading has just asked + const button = (cls, text, onClick, key) => { const b = h('button', cls, text); b.type = 'button'; if (key) b.dataset.key = key; if (onClick) b.addEventListener('click', onClick); return b; }; + const shown = v => v === null || v === undefined ? '—' : typeof v === 'number' ? fmt(v) : typeof v === 'string' ? v : JSON.stringify(v); + const plural = (n, one, many) => n + ' ' + (n === 1 ? one : many || one + 's'); + const minutes = s => s == null ? '—' : s < 90 ? Math.round(s) + ' s' : (s / 60).toFixed(1) + ' min'; + const title = o => o.title || o.metric; + let live = null; // the sweep on screen: { reference, main, answer, parts, at, busy } + + // ---- Reading ------------------------------------------------------------------------------------------------------ + const sweepOf = reference => P.call('sweep', { sweep: reference }); + + // ---- The screen --------------------------------------------------------------------------------------------------- + function sweeps(main, aside, state, act, data) { + const ui = state.ui, sel = state.sel.sweeps || null; + const again = document.getElementById('aside')._again; + if (again && live && live.main === main && main._painted === main._view && live.wanted === (ui.sweep || null)) refresh(act); + else S.paint(main, async () => { + const list = (await P.read('sweeps')).sweeps; + const chosen = list.find(s => s.reference === ui.sweep) || list[0] || null; + return { list, answer: chosen ? await sweepOf(chosen.reference) : null }; + }, (frag, { list, answer }, width) => { + live = null; + frag.appendChild(head(list, answer, act, ui)); + if (!answer) { frag.appendChild(h('p', 'empty', 'This project has no sweep yet. A sweep tries settings of a recipe, one trial after another, within a budget: New sweep writes one from a recipe.')); return; } + live = { reference: answer.reference, wanted: ui.sweep || null, main, answer, width, state, act, data, at: Date.now(), parts: {} }; + build(frag, live); + }); + return drawer(aside, sel, state, act, data); + } + + /** The drawer: New sweep, the plan view of the sweep, or a trial (the model's drawer). */ + function drawer(aside, sel, state, act, data) { + if (!sel) { aside.textContent = ''; aside._for = null; return false; } + if (sel === 'new') { newSweep(aside, act, data); return true; } + if (sel.startsWith('plan:')) { A.planView(aside, sel, act.go, () => act.select(null)); return true; } + if (data.map.byId.has(sel)) { S.inspectorModel(aside, sel, act, data); return true; } + aside.textContent = ''; aside._for = null; return false; + } + + /** The chooser, what the sweep is, and New sweep. */ + function head(list, answer, act, ui) { + const box = h('div', 'sw-head'); + const bar = h('div', 'filters'); + if (list.length) { + const s = h('select'); s.setAttribute('aria-label', 'Sweep'); s.dataset.key = 'sweep'; + for (const item of list) { const o = h('option', null, item.title + ' · ' + (STATE_WORD[item.state] || item.state)); o.value = item.reference; o.selected = answer && item.reference === answer.reference; s.appendChild(o); } + s.addEventListener('change', () => act.setUI('sweep', s.value)); + bar.appendChild(s); + } + if (A.may('edit')) bar.appendChild(button('btn sm', 'New sweep', () => act.select('new'), 'sweep-new')); + box.appendChild(bar); + if (!answer) return box; + box.appendChild(sect(answer.title, answer.reference)); + const what = h('p', 'small muted sw-what'); + what.appendChild(document.createTextNode('Varies ')); + const link = button('linkish mono', answer.recipe, () => act.go('map', answer.recipe), 'sweep-recipe'); link.title = 'Open the recipe on the Map'; what.appendChild(link); + what.appendChild(document.createTextNode(': ' + answer.parameters.map(p => p.path + ' ' + rangeWords(p)).join('; ') + '. ' + strategyWords(answer.strategy) + ' Objectives: ' + answer.objectives.map(o => title(o) + ' (' + (o.take === 'best' ? 'best' : 'last') + ', ' + o.better + ' is better)').join(' and ') + '. Budget: ' + plural(answer.budget.trials, 'trial') + (answer.budget.minutes ? ', ' + answer.budget.minutes + ' min' : '') + ', one at a time.')); + box.appendChild(what); + if (answer.description) box.appendChild(h('p', 'small faint sw-what', answer.description)); + return box; + } + const rangeWords = p => p.values ? 'in ' + p.values.map(shown).join(', ') : 'from ' + fmt(p.low) + ' to ' + fmt(p.high) + (p.scale === 'log' ? ' (log)' : '') + (p.integer ? ', whole numbers' : '') + (p.count ? ', ' + p.count + ' points' : ''); + function strategyWords(s) { + if (s.kind === 'grid') return 'Every point of the grid, in order.'; + if (s.kind === 'random') return 'Random draws (seed ' + s.seed + ').'; + return 'Random draws (seed ' + s.seed + '), stopping a trial at a rung (' + (s.unit === 'time' ? 'seconds' : 'steps') + ' ' + s.min + ', ×' + (s.factor || 3) + ' after) when its ' + s.metric + ' is not among the best 1/' + (s.factor || 3) + ' there.'; + } + + /** The sweep's sections, into frag; the parts that a reading sets in place are kept in v.parts. */ + function build(frag, v) { + const a = v.answer; + const controls = h('div', 'sw-controls'); controls.dataset.key = 'sw-controls'; v.parts.controls = controls; fillControls(v); + frag.appendChild(controls); + frag.appendChild(stateBlock(v)); + const pc = h('div', 'sw-pc'); v.parts.pc = pc; frag.appendChild(pc); drawAxes(v); + const chart = h('section', 'bench-chart sw-chart'); v.parts.chart = chart; frag.appendChild(chart); drawChart(v); + frag.appendChild(trialTable(v)); + if (window.LabParallel) window.LabParallel.apply(frag, viewOf(a), v.state); + setState(v); + } + + // ---- The state, set in place -------------------------------------------------------------------------------------- + function stateBlock(v) { + const sec = h('section', 'sw-state'); sec.dataset.key = 'sw-state'; sec.setAttribute('aria-live', 'polite'); + sec.appendChild(sect('Where it stands')); + const dl = h('dl', 'setup sw-dl'), p = v.parts; + const text = (cls) => { const t = document.createTextNode(''); const dd = h('dd', cls || null); dd.appendChild(t); return [dd, t]; }; + const row = (label, dd) => { dl.appendChild(h('dt', null, label)); dl.appendChild(dd); }; + { const dd = h('dd'); const chip = h('span', 'state neutral sw-chip'); p.chipText = document.createTextNode(''); chip.appendChild(p.chipText); p.chip = chip; dd.appendChild(chip); const t = document.createTextNode(''); p.runText = t; dd.appendChild(h('span', 'small faint sw-gap')).appendChild(t); row('Run', dd); } + { const [dd, t] = text(); p.counts = t; row('Trials', dd); } + { const dd = h('dd', 'sw-meter'); const hair = h('span', 'hair'); const i = h('i'); hair.appendChild(i); p.trialsBar = i; dd.appendChild(hair); const t = document.createTextNode(''); p.trialsText = t; dd.appendChild(h('span', 'small')).appendChild(t); row('Budget, trials', dd); } + { const dd = h('dd', 'sw-meter'); const hair = h('span', 'hair'); const i = h('i'); hair.appendChild(i); p.minutesBar = i; dd.appendChild(hair); const t = document.createTextNode(''); p.minutesText = t; dd.appendChild(h('span', 'small')).appendChild(t); row('Budget, time', dd); } + { const [dd, t] = text('mono'); p.running = t; row('Running', dd); } + { const [dd, t] = text(); p.best = t; row('Best so far', dd); } + sec.appendChild(dl); + p.state = sec; + return sec; + } + const setText = (node, text) => { if (node.data !== text) node.data = text; }; + function setState(v) { + const a = v.answer, p = v.parts, run = a.run, counts = run ? run.counts : {}; + const tone = run ? RUN_TONE[run.state] || 'neutral' : 'neutral'; + const cls = 'state sw-chip ' + (tone === 'pending' ? 'warn' : tone); + if (p.chip.className !== cls) p.chip.className = cls; + setText(p.chipText, run ? run.state : (a.request ? a.request.state.replace('-', ' ') : 'not run yet')); + setText(p.runText, run ? ' · started ' + S.parts.time(run.started_at) + (run.finished_at ? ', ended: ' + (ENDED[run.ended] || run.ended) : ' · ' + minutes(run.seconds) + ' so far') + (run.problem ? ' · ' + run.problem : '') + (a.changed ? ' · the sweep has changed since' : '') : a.request ? ' · a build request ' + a.request.state.replace('-', ' ') : ' · ask for a build of the sweep: Plan sweep'); + setText(p.counts, TRIAL_STATES.map(s => (counts[s] || 0) + ' ' + s).join(' · ')); + const used = run ? run.trials : 0, most = a.budget.trials; + const width = Math.round(100 * Math.min(1, used / Math.max(1, most))) + '%'; if (p.trialsBar.style.width !== width) p.trialsBar.style.width = width; + setText(p.trialsText, used + ' of ' + most); + const limit = a.budget.minutes ? a.budget.minutes * 60 : null, spent = run ? run.seconds : 0; + const mw = limit ? Math.round(100 * Math.min(1, (spent || 0) / limit)) + '%' : '0%'; if (p.minutesBar.style.width !== mw) p.minutesBar.style.width = mw; + setText(p.minutesText, minutes(spent || 0) + (limit ? ' of ' + minutes(limit) : ', no limit')); + const now = a.trials.find(t => t.state === 'running'); + setText(p.running, now ? 'Trial ' + now.number + ' · step ' + shown(now.latest && now.latest.step) + a.objectives.map(o => ' · ' + title(o) + ' ' + shown(now.objectives && now.objectives[o.metric])).join('') : '—'); + setText(p.best, a.best ? 'Trial ' + a.best.number + ' · ' + title(a.objectives[0]) + ' ' + fmt(a.best.value) : '—'); + } + + // ---- Controls ----------------------------------------------------------------------------------------------------- + function fillControls(v) { + const a = v.answer, r = a.request, waiting = r && r.state === 'awaiting-approval', active = r && ['awaiting-approval', 'queued', 'running'].includes(r.state); + const key = JSON.stringify([a.reference, a.state, r && r.id, r && r.state, A.may('owner'), A.may('request'), A.may('cancel')]); + const box = v.parts.controls; + if (box._key === key) return; + box._key = key; + const items = [{ id: 'plan', label: 'Plan sweep', cls: 'btn' + (active ? '' : ' primary'), who: 'request', onClick: () => v.act.select('plan:' + a.reference), hidden: !!active }]; + if (r) { + items.push({ id: 'approve', label: 'Approve', cls: 'btn primary', who: 'owner', op: 'approve_build', args: () => ({ request: r.id }), busy: 'Approving…', hidden: !waiting }); + items.push(Object.assign(A.declineItem(r.id), { cls: 'btn', hidden: !waiting })); + items.push({ id: 'cancel', label: 'Cancel', cls: 'btn ghost', who: 'cancel', op: 'cancel_build', args: () => ({ request: r.id }), busy: 'Cancelling…', hidden: !active, + confirm: r.state === 'running' ? 'Cancel the sweep: the running trial is stopped and no other starts.' : 'Cancel this request; no trial runs.' }); + } + const frag = document.createDocumentFragment(); + if (waiting) frag.appendChild(h('p', 'note sw-ask', (r.requested_by && (r.requested_by.label || r.requested_by.kind) || 'Someone') + ' asks for this sweep. One approval covers its whole budget: up to ' + plural(a.budget.trials, 'trial') + (a.budget.minutes ? ' and ' + a.budget.minutes + ' min' : '') + ', one after another.')); + frag.appendChild(A.group('sweep:' + a.reference + ':' + (r ? r.id : ''), items, { inline: true })); + if (box.isConnected) putInPlace(box, frag, document.getElementById('main')); else box.replaceChildren(frag); + } + + // ---- Parallel coordinates and the chart --------------------------------------------------------------------------- + /** The trials as parallel.js and its apply() see a benchmark's subjects: by reference, coloured by their state. */ + function viewOf(a) { + const byName = new Map(); + for (const t of a.trials) byName.set(t.reference, { name: t.reference, key: t.reference, full: 'Trial ' + t.number, kind: 'tr-' + t.state }); + return { name: 'sweep:' + a.reference, byName, byKey: byName, selected: null, noun: 'trials', lead: 'One line per trial through the settings it was given and its objectives (the colour is how it ended). Drag along an axis to keep the trials inside a range; the table below follows.' }; + } + function drawAxes(v) { + const a = v.answer, host = v.parts.pc; + host._key = JSON.stringify(a.axes); + if (!window.LabParallel || !a.axes || a.trials.length < 2) { host.replaceChildren(h('p', 'empty', a.trials.length ? 'Parallel coordinates are drawn from two trials on.' : 'No trial yet: its trials are drawn here as lines through their settings and objectives.')); return; } + const data = Object.assign({}, a.axes, { benchmark: 'sweep:' + a.reference, statistic: 'value' }); + const view = viewOf(a); + host.replaceChildren(window.LabParallel.section(data, view, v.state, v.act, v.width)); + } + function drawChart(v) { + const a = v.answer, host = v.parts.chart, done = a.trials.filter(t => t.state === 'completed'); + host._key = JSON.stringify([a.objectives, done.map(t => [t.number, t.objectives]), a.frontier]); + host.dataset.chart = 'sweep'; + const [y, x] = a.objectives; + const parts = []; + if (x) parts.push(sect(title(y) + ' against ' + title(x), done.length + ' completed trials · ' + a.frontier.length + ' on the frontier')); + else parts.push(sect(title(y) + ' by trial', done.length + ' completed trials, the best so far as a step line')); + const draw = h('div'); + const valued = done.filter(t => t.objectives && t.objectives[y.metric] != null && (!x || t.objectives[x.metric] != null)); + if (!valued.length) parts.push(h('p', 'empty', 'No completed trial has its objectives yet.')); + else { + parts.push(draw); + const front = new Set(a.frontier); + const rows = t => () => [{ b: 'Trial ' + t.number }].concat(a.objectives.map(o => ({ v: fmt(t.objectives[o.metric]), l: title(o) }))).concat(Object.entries(t.values).map(([k, val]) => ({ v: shown(val), l: k }))); + const select = id => v.act.select(v.state.sel.sweeps === id ? null : id); + if (x) { + C.scatter(draw, valued.map(t => ({ id: t.reference, name: 'Trial ' + t.number, short: 'T' + t.number, kind: 'sweep-trial', x: t.objectives[x.metric], y: t.objectives[y.metric], frontier: front.has(t.number), rows: rows(t) })), + { width: v.width, height: 320, xTitle: title(x), yTitle: title(y), xName: title(x), yName: title(y), xBetter: x.better, yBetter: y.better, selected: v.state.sel.sweeps, onSelect: select }); + parts.push(h('p', 'small faint', 'Line: the frontier of the completed trials, where no other is at least as good on both and better on one. Pruned and stopped trials are left out: they did not finish.')); + } else { + let best = null; + C.progression(draw, valued.map(t => { const val = t.objectives[y.metric]; if (best === null || (y.better === 'higher' ? val > best : val < best)) best = val; return { id: t.reference, name: 'Trial ' + t.number, short: 'T' + t.number, kind: 'sweep-trial', y: val, best, rows: rows(t) }; }), + { width: v.width, height: 240, yTitle: title(y), selected: v.state.sel.sweeps, onSelect: select }); + } + } + if (host.isConnected) { const frag = document.createDocumentFragment(); frag.append(...parts); putInPlace(host, frag, document.getElementById('main')); } + else host.replaceChildren(...parts); + } + + // ---- The trials --------------------------------------------------------------------------------------------------- + function trialTable(v) { + const a = v.answer, sec = h('section', 'bench-all sw-trials'); + sec.appendChild(sect('Trials', plural(a.trials.length, 'trial') + (a.run ? ' of this run' : ''))); + const t = h('table', 'tv sw-table'); t.setAttribute('aria-label', 'The trials of ' + a.title); + const thead = h('thead'), tr = h('tr'); + for (const [label, cls] of [['#', 'n']].concat(a.parameters.map(p => [p.path, 'n mono']), [['State'], ['Rung']], a.objectives.map(o => [title(o), 'n']), [['Time', 'n'], ['']])) tr.appendChild(h('th', cls || null, label)); + thead.appendChild(tr); t.appendChild(thead); + const tb = h('tbody'); v.parts.rows = new Map(); v.parts.tbody = tb; + for (const trial of a.trials) tb.appendChild(trialRow(v, trial)); + t.appendChild(tb); + const wrap = h('div', 'tv-wrap'); wrap.appendChild(t); sec.appendChild(wrap); + if (!a.trials.length) sec.appendChild(h('p', 'note', a.run ? 'No trial has started yet.' : 'No trial yet.')); + return sec; + } + function trialRow(v, trial) { + const a = v.answer, tr = h('tr'); tr.dataset.subject = trial.reference; tr.dataset.trial = String(trial.number); + const cells = { values: [], objectives: [] }; + const td = (cls, text) => { const c = h('td', cls || null); const t = document.createTextNode(text); c.appendChild(t); tr.appendChild(c); return t; }; + td('n', String(trial.number)); + for (const p of a.parameters) cells.values.push(td('n mono', shown(trial.values[p.path]))); + const st = h('td'); const chip = h('span', 'state'); const chipText = document.createTextNode(''); chip.appendChild(chipText); st.appendChild(chip); tr.appendChild(st); + cells.chip = chip; cells.chipText = chipText; + cells.rung = td('small'); + for (const o of a.objectives) cells.objectives.push(td('n')); + cells.seconds = td('n small'); + const act = h('td', 'nowrap'); + const open = button('btn sm ghost', 'Open on the Map', ev => { ev.stopPropagation(); v.act.go('map', trial.reference); }, 'sweep-map-' + trial.number); open.disabled = !trial.defined; act.appendChild(open); tr.appendChild(act); + tr.tabIndex = 0; tr.classList.add('sw-row'); + tr.addEventListener('click', () => v.act.select(v.state.sel.sweeps === trial.reference ? null : trial.reference)); + tr.addEventListener('keydown', ev => { if (ev.key === 'Enter' && ev.target === tr) v.act.select(trial.reference); }); + v.parts.rows.set(trial.number, { tr, cells, open }); + setRow(v, trial); + return tr; + } + function setRow(v, trial) { + const { tr, cells, open } = v.parts.rows.get(trial.number), a = v.answer; + const tone = TRIAL_TONE[trial.state] || 'neutral', cls = 'state ' + (tone === 'pending' ? 'warn' : tone); + if (cells.chip.className !== cls) cells.chip.className = cls; + setText(cells.chipText, trial.state + (trial.reused ? ' (built before)' : '')); + const last = (trial.rungs || []).slice(-1)[0]; + setText(cells.rung, last ? 'rung ' + (last.rung + 1) + (last.goes_on ? '' : ', stopped there') : ''); + a.objectives.forEach((o, i) => setText(cells.objectives[i], shown(trial.objectives && trial.objectives[o.metric]))); + setText(cells.seconds, trial.seconds == null ? (trial.state === 'running' && trial.latest ? 'step ' + trial.latest.step : '') : minutes(trial.seconds)); + if (open.disabled !== !trial.defined) open.disabled = !trial.defined; + if (trial.problem && tr.title !== trial.problem) tr.title = trial.problem; + const on = v.state.sel.sweeps === trial.reference; if (tr.classList.contains('on') !== on) tr.classList.toggle('on', on); + } + + // ---- A reading: everything set in place --------------------------------------------------------------------------- + async function refresh(act) { + const v = live; + if (!v || !v.main.isConnected || v.busy) return; + v.busy = true; + try { + const answer = await sweepOf(v.reference); + if (live !== v) return; + v.at = Date.now(); v.answer = answer; + if (act) v.act = act; + setState(v); fillControls(v); + const option = v.main.querySelector('select[data-key="sweep"] option:checked'), word = option && option.textContent.replace(/ · [^·]*$/, ' · ' + (STATE_WORD[answer.state] || answer.state)); + if (option && option.textContent !== word) option.textContent = word; + for (const trial of answer.trials) { if (v.parts.rows.has(trial.number)) setRow(v, trial); else v.parts.tbody.appendChild(trialRow(v, trial)); } + if (v.parts.pc._key !== JSON.stringify(answer.axes)) drawAxes(v); + const done = answer.trials.filter(t => t.state === 'completed'); + if (v.parts.chart._key !== JSON.stringify([answer.objectives, done.map(t => [t.number, t.objectives]), answer.frontier])) drawChart(v); + if (window.LabParallel) window.LabParallel.apply(v.main, viewOf(answer), v.state); + } catch (error) { /* the strip says when the project could not be read; what is on screen stays */ } + finally { v.busy = false; } + } + // Every reading of the page (app.js calls S.tick after drawing): the sweep on screen, while it runs or is asked for. + const before = S.tick; + S.tick = async () => { + if (before) await before(); + const v = live; + if (v && v.main.isConnected && Date.now() - v.at >= REFRESH_MS && document.location.hash.startsWith('#sweeps') && (v.answer.state === 'running' || (v.answer.request && ['awaiting-approval', 'queued', 'running'].includes(v.answer.request.state)))) await refresh(); + }; + + // ---- New sweep ---------------------------------------------------------------------------------------------------- + const drafting = { recipe: null, name: '', text: '', touched: false }; + /** A sweep's text from a recipe's settings: its numeric ones as ranges about their values, random draws, eight trials. */ + function template(reference, doc, name) { + const settings = doc.run && doc.run.settings && typeof doc.run.settings === 'object' ? doc.run.settings : {}; + const parameters = []; + for (const [k, val] of Object.entries(settings)) { + if (parameters.length >= 4) break; + if (typeof val === 'number' && Number.isInteger(val) && val >= 1) parameters.push({ path: k, low: Math.max(1, Math.floor(val / 2)), high: val * 2, integer: true }); + else if (typeof val === 'number' && val > 0) parameters.push({ path: k, low: +(val / 10).toPrecision(6), high: +(val * 10).toPrecision(6), scale: 'log' }); + else if (typeof val === 'boolean') parameters.push({ path: k, values: [true, false] }); + } + return JSON.stringify({ format: 'tuldok_definition_v1', kind: 'sweep', title: 'A search over ' + (doc.title || reference), + recipe: reference, parameters: parameters.length ? parameters : [{ path: 'learning_rate', low: 0.0001, high: 0.01, scale: 'log' }], + strategy: { kind: 'random', seed: 1 }, objectives: [{ metric: 'loss', better: 'lower' }], budget: { trials: 8 } }, null, 2) + '\n'; + } + function newSweep(aside, act, data) { + if (aside._for === 'new' && !aside._again) return; + const recipes = data.map.models.filter(m => m.kind !== 'sweep-trial' && !m.draft); + if (!drafting.recipe || !recipes.some(m => m.id === drafting.recipe)) { drafting.recipe = recipes.length ? recipes[0].id : null; drafting.touched = false; } + aside._for = 'new'; + const ins = h('div', 'inspector sw-new'); + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'new sweep · kept as an unsaved edit')); const t = h('h2', null, 'A sweep of a recipe\'s settings'); t.tabIndex = -1; hd.appendChild(t); ins.appendChild(hd); + ins.appendChild(h('p', 'note', 'Choose the recipe; the text is written from its settings. Change the parameters, the objectives (the series its script logs) and the budget, then Keep. A recipe whose settings are in a file is branched first: Branch from here gives the branch its own copy.')); + const f = h('div', 'form'); + const lab = h('label', null, 'Recipe'); const s = h('select'); s.dataset.key = 'sweep-new-recipe'; + for (const m of recipes) { const o = h('option', null, Tree.nameOf(m) + ' · ' + m.id); o.value = m.id; o.selected = m.id === drafting.recipe; s.appendChild(o); } + lab.appendChild(s); f.appendChild(lab); + const nl = h('label', null, 'Name'); const name = h('input'); name.dataset.key = 'sweep-new-name'; name.value = drafting.name; name.placeholder = 'lowercase letters, digits, . _ -'; nl.appendChild(name); f.appendChild(nl); + const tl = h('label', null, 'Text'); const area = h('textarea', 'mono sw-text'); area.dataset.key = 'sweep-new-text'; area.rows = 18; area.spellcheck = false; area.value = drafting.text; tl.appendChild(area); f.appendChild(tl); + ins.appendChild(f); + const fill = async () => { + if (!drafting.recipe) return; + const def = await P.read('definition', { reference: drafting.recipe }); + if (!drafting.name || !drafting.touchedName) { drafting.name = drafting.recipe.slice(8, 100) + '-sweep'; name.value = drafting.name; } + drafting.text = template(drafting.recipe, def.document || {}, drafting.name); area.value = drafting.text; drafting.touched = false; + }; + s.addEventListener('change', () => { drafting.recipe = s.value; fill(); }); + name.addEventListener('input', () => { drafting.name = name.value; drafting.touchedName = true; }); + area.addEventListener('input', () => { drafting.text = area.value; drafting.touched = true; }); + ins.appendChild(A.group('sweep:new', [{ id: 'keep', label: 'Keep', cls: 'btn primary', who: 'edit', op: 'set_definition', busy: 'Keeping…', + args: () => ({ reference: 'sweeps/' + drafting.name.trim(), text: drafting.text }), + done: r => { const ref = r.reference; drafting.text = ''; drafting.name = ''; drafting.touched = false; drafting.touchedName = false; act.setUI('sweep', ref); act.select(null); } }])); + if (!drafting.text) fill().catch(error => failure(ins, error)); + S.parts.putDrawer(aside, ins); + } + + S.sweeps = sweeps; +})(); diff --git a/static/project-workspace/tabs.js b/static/project-workspace/tabs.js new file mode 100644 index 00000000..8a249987 --- /dev/null +++ b/static/project-workspace/tabs.js @@ -0,0 +1,851 @@ +'use strict'; +// The Data, Train, Benchmarks and Analysis screens, from the mockup's +// screens.js, added to window.LabScreens. Each is a screen like the others +// (main, aside, state, act, data): it shows "Reading…" the first time, reads +// what it needs through window.TuldokProject, then builds its content off the +// page and puts it in place in one step, so a re-render (a sort, a filter, a +// reading that found something changed) keeps the scroll, inside tables too, +// and the focus; a drawer read again for its subject stays on screen until +// its new content replaces it whole (spec-w4.md). Data, Benchmarks and Analysis +// only read; Analysis shows, below the per-case values, the fields the subject's +// run kept (fields.js, spec-d6.md). Train draws the controls of spec-w3.md section 3.1 (actions.js): +// Plan build, a build request's Approve, Decline and Cancel, a run's Cancel +// run and Stop, an environment's Check again, Connect and Disconnect, and it +// shows a running run live, the models served (serving.js), and the machine's Pumas endpoint beside the environments (generate.js); +// it lists first what is planned (the page's drafts +// and the models never built, spec-w3 section 3.2). Data's drawer has Edit; +// the editor itself is editor.js's. Rule: the chrome holds no project words; titles, +// metrics and text come from the project. +(() => { + const S = window.LabScreens, P = window.TuldokProject, Tree = window.LabTree, C = window.LabCharts, h = C.h, A = window.LabActions; + const { sect, cell, block, time, bytes, failure, modelState, graphFor, putInPlace, putDrawer, turn } = S.parts; + const fmt = C.fmtSci; + const STATISTICS = ['median', 'mean', 'p10', 'p90', 'min', 'max']; + const COMPARE_MOST = 4; // models in the compare set (app.js keeps the same bound; spec-t3 section 5.1) + const plural = (n, one, many) => n.toLocaleString('en-GB') + ' ' + (n === 1 ? one : many || one + 's'); + const nameOfRef = ref => ref.slice(ref.indexOf('/') + 1); + const seconds = s => s == null ? '' : s < 1 ? '<1 s' : s < 90 ? Math.round(s) + ' s' : s < 5400 ? Math.round(s / 60) + ' min' : (s / 3600).toFixed(1) + ' h'; + const button = (cls, text, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (onClick) b.addEventListener('click', onClick); return b; }; + const better = m => m.better === 'higher' ? 'higher is better' : m.better === 'lower' ? 'lower is better' : 'no better direction'; + const units = m => m.units && m.units !== '1' ? ' (' + m.units + ')' : ''; + + /** + * The pixel width the main area gives its content, for charts drawn before + * they are on the page: from the drawer's class, not the padding, which may + * be mid-transition. + */ + function contentWidth(main) { const left = parseFloat(getComputedStyle(main).paddingLeft) || 20; return Math.max(360, main.clientWidth - left - (main.classList.contains('with-drawer') ? Math.min(420, 0.92 * innerWidth) + 20 : left)); } + + /** + * Fills main from load() (a promise of what build reads): "Reading…" only + * when this view has not been painted, the content built off the page and + * put in place in one step when it is still the newest render of the same + * view, keeping what spec-w4 R4 lists (screens.js's putInPlace): the scroll + * of main and of every scrolled element in it, and the focused control (by + * data-key) with its caret. + */ + function paint(main, load, build) { + const view = main._view, mine = main._paint = (main._paint || 0) + 1; + if (main._painted !== view) { main.textContent = ''; main.appendChild(h('p', 'note', 'Reading…')); } + main.setAttribute('aria-busy', 'true'); + const done = () => { if (main._paint === mine) main.removeAttribute('aria-busy'); }; + load().then(value => { + done(); if (main._paint !== mine || main._view !== view) return; + const width = contentWidth(main), frag = document.createDocumentFragment(); + try { build(frag, value, width); } catch (error) { frag.textContent = ''; failure(frag, error); console.error(error); } + swap(main, frag); main._painted = view; + }, error => { + done(); if (main._paint !== mine || main._view !== view) return; + const frag = document.createDocumentFragment(); failure(frag, error); swap(main, frag); main._painted = view; + }); + } + const swap = (main, frag) => putInPlace(main, frag); + /** + * Fills the drawer from load() for the subject id; build(ins, value). Once + * per id; read again for the subject on screen (app.js marks the drawer + * with _again after a change), what is on screen stays until the new + * content is built off the page, then is replaced in one step with the + * drawer's scroll kept. "Reading…" only for a subject not on screen yet. + */ + function drawer(aside, id, load, build) { + if (aside._for === id && !aside._again) return; + const same = aside._for === id && !!aside.firstChild, mine = turn(aside); + aside._for = id; + if (!same) { aside.textContent = ''; aside.appendChild(h('p', 'note', 'Reading…')); } + const live = () => aside._turn === mine && aside._for === id; + load().then(value => { if (!live()) return; const ins = h('div', 'inspector'); try { build(ins, value); } catch (error) { failure(ins, error); } putDrawer(aside, ins); }, + error => { if (!live()) return; const frag = document.createDocumentFragment(); failure(frag, error); putDrawer(aside, frag); }); + } + const closeDrawer = aside => { aside.textContent = ''; aside._for = null; return false; }; + const setup = () => { const dl = h('dl', 'setup'); return { dl, put(k, v, cls) { dl.appendChild(h('dt', null, k)); const dd = h('dd', cls || null); if (typeof v === 'string') dd.textContent = v; else if (v) dd.appendChild(v); dl.appendChild(dd); } }; }; + const searchBox = (state, act, placeholder) => { const q = h('input'); q.type = 'search'; q.placeholder = placeholder; q.value = state.query || ''; q.dataset.key = 'search'; q.setAttribute('aria-label', placeholder); q.addEventListener('input', () => act.setQuery(q.value)); return q; }; + function kindChips(state, act, present) { + const kinds = P.KINDS.map(k => k.id).concat(present.includes(P.UNSET) ? [P.UNSET] : []); + const grp = h('div', 'grp'); grp.setAttribute('role', 'group'); grp.setAttribute('aria-label', 'Kind of change'); + for (const k of kinds) { const b = button('btn sm chip ' + k, P.kindLabel(k), () => act.toggleKind(k, kinds)); b.dataset.kind = k; b.dataset.key = 'kind-' + k; b.setAttribute('aria-pressed', String(!state.kinds || state.kinds.has(k))); grp.appendChild(b); } + return grp; + } + const toggle = (label, on, onClick, key) => { const b = button('btn sm', label, onClick); b.dataset.key = key; b.setAttribute('aria-pressed', String(!!on)); return b; }; + const select = (label, options, value, onChange, key) => { + const s = h('select'); s.setAttribute('aria-label', label); s.dataset.key = key; + for (const [v, t] of options) { const o = h('option', null, t); o.value = v; o.selected = v === value; s.appendChild(o); } + s.addEventListener('change', () => onChange(s.value)); return s; + }; + const wrapTable = t => { const d = h('div', 'tv-wrap'); d.appendChild(t); return d; }; + function headRow(cols) { const thead = h('thead'), tr = h('tr'); for (const [t, cls] of cols) tr.appendChild(h('th', cls || null, t)); thead.appendChild(tr); return thead; } + + // ---- Benchmarks: what is read -------------------------------------------- + async function allResults(name) { + const rows = []; let offset = 0; + for (;;) { const page = await P.read('results', { benchmark: name, group: '', offset, limit: 2000 }); rows.push(...page.rows); offset += page.rows.length; if (!page.rows.length || offset >= page.total) return rows; } + } + /** Each chart definition's benchmark and form, read once. */ + async function chartIndex() { + const defs = (await P.read('definitions', { kind: 'chart' })).definitions.filter(d => !d.removed); + return Promise.all(defs.map(async d => { const full = await P.read('definition', { reference: d.reference }); const doc = full.document || {}; return { reference: d.reference, title: d.title, benchmark: typeof doc.benchmark === 'string' ? nameOfRef(doc.benchmark) : null, form: doc.form }; })); + } + async function benchmarkList() { + const [overview, defs, charts] = await Promise.all([P.read('results_overview'), P.read('definitions', { kind: 'benchmark' }), chartIndex()]); + const titles = new Map(defs.definitions.map(d => [nameOfRef(d.reference), d])); + return overview.benchmarks.map(b => ({ ...b, title: titles.has(b.benchmark) ? titles.get(b.benchmark).title : b.benchmark, state: titles.has(b.benchmark) ? titles.get(b.benchmark).state : null, + forms: charts.filter(c => c.benchmark === b.benchmark).map(c => c.form) })); + } + /** + * The benchmark the tab shows: the one asked for; else, among those with + * results, the first with an accuracy-against-time chart, then the first + * with any chart, then the first; else the first of all. + */ + const pickBenchmark = (list, wanted) => { + const withResults = list.filter(b => b.subjects_with_results > 0); + return list.find(b => b.benchmark === wanted) || withResults.find(b => (b.forms || []).includes('tradeoff')) + || withResults.find(b => (b.forms || []).length) || withResults[0] || list[0] || null; + }; + /** + * One benchmark as the tab shows it: its metrics, its subjects (each with + * the model of its recipe from the map, its values per metric for every + * statistic, and whether its summaries come from per-case values held + * here), and its charts' data. + */ + async function benchmarkView(name, mapShaped) { + const [def, rows, charts] = await Promise.all([P.read('definition', { reference: 'benchmarks/' + name }), allResults(name), chartIndex()]); + const doc = def.document || {}, metrics = (doc.metrics || []).map(m => ({ id: m.id, title: m.title || m.id, units: m.units, better: m.better || 'none' })); + const subjects = (doc.subjects || []).map(s => { + const model = s.recipe && mapShaped.byId.get(s.recipe) || null; + return { name: s.name, title: s.title || s.name, recipe: s.recipe || null, model, key: model ? model.id : 'subject:' + s.name, + short: model ? Tree.refName(model) : s.name, full: model ? Tree.nameOf(model) : s.title || s.name, kind: model ? model.kind : 'other', + group: model ? model.groupTitle || '' : '', values: {}, count: null, perCase: false }; + }); + const byName = new Map(subjects.map(s => [s.name, s])); + for (const r of rows) { const s = byName.get(r.subject); if (!s) continue; s.values[r.metric] = r; s.count = Math.max(s.count || 0, r.count); if (r.origin === 'values') s.perCase = true; } + const ORDER = ['tradeoff', 'progression', 'ranking', 'comparison']; // the mockup leads with accuracy against time + const mine = charts.filter(c => c.benchmark === name).sort((a, b) => ORDER.indexOf(a.form) - ORDER.indexOf(b.form) || a.title.localeCompare(b.title)); + const data = await Promise.all(mine.map(c => P.read('chart', { chart: c.reference }).then(r => ({ ...c, data: r.data }), error => ({ ...c, error })))); + return { name, title: def.title, description: doc.description || '', metrics, subjects, byName, byKey: new Map(subjects.map(s => [s.key, s])), charts: data, cases: doc.cases || null }; + } + const valueOf = (s, metric, stat) => { const r = s.values[metric]; return r && r.count > 0 && r[stat] != null ? r[stat] : null; }; + const visibleSubject = (state) => s => (!state.kinds || state.kinds.has(s.kind)) && (!state.query || [s.full, s.name, s.title, s.recipe].some(t => t && t.toLowerCase().includes(state.query.toLowerCase()))); + + // ---- Benchmarks: the screen ------------------------------------------------- + function benchmarks(main, aside, state, act, data) { + const sel = state.sel.benchmarks || null, ui = state.ui; + const stat0 = STATISTICS.includes(ui.stat) ? ui.stat : 'median'; + paint(main, async () => { + const list = await benchmarkList(), b = pickBenchmark(list, state.benchmark); + if (!b) return { list, view: null, picked: b }; + // Parallel coordinates (parallel.js) are read with the tab; the intervals, which may take seconds to make the + // first time (spec-t3 section 2), only when they have already arrived: else after the tab is put in place. + const [view, axes] = await Promise.all([benchmarkView(b.benchmark, data.map), window.LabParallel ? P.read('benchmark_axes', { benchmark: b.benchmark, statistic: stat0 }).catch(error => ({ error })) : null]); + return { list, view, picked: b, axes, iv: P.peek('intervals', { benchmark: b.benchmark }) }; + }, (frag, { list, view, picked, axes, iv }, width) => { + if (!view) { frag.appendChild(h('p', 'empty', 'This project defines no benchmark.')); return; } + const stat = stat0, cmp = state.cmp.filter(k => view.byKey.has(k)); + if (iv === undefined) P.read('intervals', { benchmark: view.name }).then(() => { if (state.screen === 'benchmarks' && main._view === 'benchmarks/') act.setUI('ivFor', view.name); }, () => { /* no intervals: the tab stays as it is */ }); + const ivs = intervalIndex(iv); + const bar = h('div', 'filters'); + bar.appendChild(select('Benchmark', list.map(b => [b.benchmark, b.title + ' · ' + plural(b.subjects_with_results, 'subject') + ' with results']), view.name, v => act.setBenchmark(v), 'benchmark')); + bar.appendChild(kindChips(state, act, view.subjects.map(s => s.kind))); + bar.appendChild(select('Statistic', STATISTICS.map(s => [s, s.charAt(0).toUpperCase() + s.slice(1)]), stat, v => act.setUI('stat', v), 'stat')); + if (view.charts.some(c => c.form === 'tradeoff')) bar.appendChild(toggle('Frontier only', ui.frontierOnly, () => act.setUI('frontierOnly', !ui.frontierOnly), 'frontier')); + bar.appendChild(toggle('With results only', ui.resultsOnly, () => act.setUI('resultsOnly', !ui.resultsOnly), 'results-only')); + bar.appendChild(toggle('Table view', ui.chartTables, () => act.setUI('chartTables', !ui.chartTables), 'tables')); + bar.appendChild(searchBox(state, act, 'Find a subject')); + frag.appendChild(bar); + // Coverage: how many subjects have a result here, so the smaller plotted counts are explained. + const every = view.subjects.filter(s => view.metrics.length && view.metrics.every(m => valueOf(s, m.id, stat) != null)).length; + const some = view.subjects.filter(s => view.metrics.some(m => valueOf(s, m.id, stat) != null)).length - every; + const none = view.subjects.length - every - some, perCase = view.subjects.filter(s => s.perCase).length; + const cov = h('div', 'coverage'); const add = (n, t) => { const sp = h('span'); sp.appendChild(h('b', null, n.toLocaleString('en-GB'))); sp.appendChild(document.createTextNode(' ' + t)); cov.appendChild(sp); }; + add(view.subjects.length, view.subjects.length === 1 ? 'subject' : 'subjects'); if (every) add(every, view.metrics.length === 1 ? 'with a result' : 'with every metric'); if (some) add(some, 'with some metrics'); if (none) add(none, 'with no result'); add(perCase, 'with per-case values on this machine'); + frag.appendChild(cov); + if (picked.differs_from_file) frag.appendChild(h('p', 'small muted bench-note', 'The results held here differ from the results file of this benchmark.')); + const show = visibleSubject(state), select_ = key => act.select(key === sel ? null : key); + // Charts, by their form. + if (!view.charts.length) frag.appendChild(h('p', 'empty bench-note', 'No chart is defined for this benchmark; every value is in the table below.')); + for (const c of view.charts) frag.appendChild(chartSection(c, view, { width, sel, onSelect: select_, show, frontierOnly: !!ui.frontierOnly, tables: !!ui.chartTables, ivs })); + if (ivs.rows) frag.appendChild(h('p', 'small faint bench-desc bench-iv', 'Intervals: 95 % percentile bootstrap, ' + iv.resamples.toLocaleString('en-GB') + ' resamples of each subject\'s per-case values held here (seed ' + iv.seed + '); a dashed ring on the frontier: not told apart from a neighbour on both metrics.')); + // Parallel coordinates over the metrics and the recipe settings that vary (parallel.js, spec-t3 section 5.2). + if (axes && window.LabParallel && view.subjects.length >= 2) frag.appendChild(axes.error ? (() => { const d = h('section', 'pc'); d.appendChild(sect('Settings and metrics')); failure(d, axes.error); return d; })() : window.LabParallel.section(axes, view, state, act, width)); + // The compare set. + if (cmp.length) frag.appendChild(compareSection(view, cmp, stat, act)); + // Every subject. + frag.appendChild(subjectTable(view, state, act, stat, sel, cmp, ivs)); + if (window.LabParallel) window.LabParallel.apply(frag, view, state); // the ranges drawn on the axes + }); + if (!sel) return closeDrawer(aside); + benchmarkDrawer(aside, sel, state, act, data); + return true; + } + /** The drawer of a selected subject: the model's drawer when it is made from a recipe of the map, with Compare. */ + function benchmarkDrawer(aside, sel, state, act, data) { + const isModel = data.map.byId.has(sel); + if (isModel) S.inspectorModel(aside, sel, act, data, ins => compareActs(ins, sel, state, act, 'benchmarks')); + else if (aside._for !== sel || aside._again) { + const name = sel.startsWith('subject:') ? sel.slice('subject:'.length) : sel; + drawer(aside, sel, async () => { const list = await benchmarkList(); const b = pickBenchmark(list, state.benchmark); return b ? benchmarkView(b.benchmark, data.map) : null; }, (ins, view) => { + const s = view && view.byName.get(name); + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'benchmark subject · ' + name)); hd.appendChild(h('h2', null, s ? s.title : name)); ins.appendChild(hd); + if (!s) { ins.appendChild(h('p', 'empty', 'This benchmark has no subject ' + name + '.')); return; } + ins.appendChild(h('p', 'note', s.recipe ? 'Made from ' + s.recipe + ', which is not on the map.' : 'Not made from a recipe of this project.')); + const dl = h('dl', 'metrics'); for (const m of view.metrics) dl.appendChild(cell(m.title, fmt(valueOf(s, m.id, 'median')), m.units && m.units !== '1' ? m.units : '')); ins.appendChild(dl); + ins.appendChild(h('p', 'note', 'Medians over ' + (s.count == null ? 'no cases' : plural(s.count, 'case')) + ' of ' + view.title + '.')); + compareActs(ins, sel, state, act, 'subject'); + }); + } + const ins = aside.querySelector('.inspector'); if (ins) compareActs(ins, sel, state, act, isModel ? 'benchmarks' : 'subject'); + } + /** + * Compare, kept current on every render (a drawer is built once per selection), in a model's drawer on the + * Benchmarks tab, the Map and the Compare tab (where: 'benchmarks', 'subject', 'map', 'compare'): adds the model + * to the compare set or takes it out, opens the Compare tab once the set holds two, and On the map. + */ + function compareActs(ins, sel, state, act, where) { + const old = ins.querySelector('.bench-acts'); if (old) old.remove(); + const acts = h('div', 'actions bench-acts'), on = state.cmp.includes(sel); + const b = button('btn', on ? 'Remove from compare' : 'Compare', () => act.toggleCompare(sel)); b.disabled = !on && state.cmp.length >= COMPARE_MOST; b.dataset.key = 'cmp-toggle-' + sel; acts.appendChild(b); + if (where !== 'compare' && state.cmp.length >= 2) { const o = button('btn ghost', 'Open Compare · ' + state.cmp.length, () => act.go('compare')); o.dataset.key = 'cmp-open'; acts.appendChild(o); } + if (where !== 'map' && where !== 'subject') acts.appendChild(button('btn ghost', 'On the map', () => act.go('map', sel))); + const head = ins.querySelector('header'); head.after(acts); + } + S.compareActs = compareActs; + + /** + * The intervals answer (spec-t3 section 4) as the tab looks it up: of(subject, metric, statistic) gives [low, high] + * or null, frontiers by chart reference; rows is how many there are (0 while they have not arrived). + */ + function intervalIndex(iv) { + const by = new Map(), frontiers = new Map(); + if (iv) { for (const r of iv.intervals) by.set(r.subject + '\u0000' + r.metric, r); for (const f of iv.frontiers) frontiers.set(f.chart, f); } + return { rows: by.size, frontiers, of: (subject, metric, stat) => { const r = by.get(subject + '\u0000' + metric); return r && r[stat] ? r[stat] : null; } }; + } + /** One chart: its heading and population in words, the drawing by form (a panel per group value), and its table when asked. */ + function chartSection(c, view, o) { + const sec = h('section', 'bench-chart'); sec.dataset.chart = c.reference; + if (c.error) { sec.appendChild(sect(c.title)); failure(sec, c.error); return sec; } + const d = c.data, series = d.series || []; + const plotted = d.form === 'comparison' ? new Set(d.rows.map(r => r.subject)).size : new Set(series.flatMap(s => s.points.map(p => p.subject))).size; + const words = [plotted + ' of ' + plural(d.subject_count, 'subject') + ' drawn', d.statistic + ' over each subject\'s cases']; + if (d.form === 'tradeoff') words.push(series.reduce((n, s) => n + s.points.filter(p => p.frontier).length, 0) + ' on the frontier'); + if (d.missing.length) words.push(d.missing.length + ' without a value not drawn'); + sec.appendChild(sect(c.title, words.join(' · '))); + if (d.description) sec.appendChild(h('p', 'small muted bench-desc', d.description)); + const subjectOf = p => view.byName.get(p.subject); + const point = p => { const s = subjectOf(p); return { id: s ? s.key : 'subject:' + p.subject, name: s ? s.full : p.title, short: s ? s.short : p.subject, kind: s ? s.kind : 'other', dim: s ? !o.show(s) : false }; }; + const tipRows = (p, extra) => () => { const s = subjectOf(p); return [{ b: s ? s.full : p.title }, { v: fmt(p.value), l: d.metric.title + units(d.metric) }].concat(extra || []).concat([{ v: s && s.model ? P.kindLabel(s.kind) : 'not a model of the map', l: (p.count != null ? '· ' + plural(p.count, 'case') : '') }]); }; + const panels = series.length > 1 || d.group; + if (d.form === 'comparison') { sec.appendChild(comparisonTable(d, view, o)); return sec; } + if (!series.some(s => s.points.length)) { sec.appendChild(h('p', 'empty', 'No subject has a value for this chart yet.')); return sec; } + for (const s of series) { + const host = h('div'); host.dataset.series = s.group; + if (panels) sec.appendChild(h('h3', 'panel', d.group + ' = ' + s.group)); + if (!s.points.length) { sec.appendChild(h('p', 'empty', 'No subject has a value here.')); continue; } + sec.appendChild(host); + if (d.form === 'tradeoff') { + const front = !d.group && o.ivs.frontiers.get(c.reference), apart = p => front ? front.pairs.filter(q => q.separated === false && q.subjects.includes(p.subject)).map(q => q.subjects.find(x => x !== p.subject)) : []; + const pts = s.points.map(p => { const away = p.frontier ? apart(p) : []; return { ...point(p), x: p.x, y: p.value, frontier: p.frontier, nsig: away.length > 0, rows: tipRows(p, [{ v: fmt(p.x), l: d.metric_x.title + units(d.metric_x) }, { l: p.frontier ? 'on the frontier' + (away.length ? ', not told apart from ' + away.join(', ') : '') : '' }]) }; }); + C.scatter(host, pts, { width: o.width, height: 340, xTitle: d.metric_x.title + units(d.metric_x), yTitle: d.metric.title + units(d.metric), xName: d.metric_x.title, yName: d.metric.title, xBetter: d.metric_x.better, yBetter: d.metric.better, selected: o.sel, onSelect: o.onSelect, dimOff: o.frontierOnly }); + host.appendChild(legend(pts, 'Line: the frontier, where no other subject is at least as good on both and better on one' + (pts.some(p => p.nsig) ? '; dashed ring: not told apart from a neighbour by the per-case values' : ''))); + } else if (d.form === 'progression') { + const pts = s.points.map(p => ({ ...point(p), y: p.value, best: p.best_so_far, rows: tipRows(p, [{ v: fmt(p.best_so_far), l: 'best so far' }, { l: p.trained_at ? 'trained ' + P.day(p.trained_at) : 'no training date' }]) })); + C.progression(host, pts, { width: o.width, height: 240, yTitle: d.metric.title + units(d.metric), selected: o.sel, onSelect: o.onSelect }); + host.appendChild(legend(pts, 'Step line: the best so far, ' + better(d.metric))); + } else { + const ci = p => !d.group && o.ivs.of(p.subject, d.metric.id, d.statistic); + const pts = s.points.map(p => { const e = ci(p); return { ...point(p), y: p.value, lo: e ? e[0] : null, hi: e ? e[1] : null, rows: tipRows(p, e ? [{ v: fmt(e[0]) + ' – ' + fmt(e[1]), l: '95 % interval' }] : []) }; }); + C.ranking(host, pts, { width: o.width, yTitle: d.metric.title + units(d.metric), selected: o.sel, onSelect: o.onSelect }); + host.appendChild(legend(pts, 'Best first, ' + better(d.metric))); + } + if (o.tables) sec.appendChild(pointsTable(d, s)); + } + return sec; + } + function legend(points, line) { + const ul = h('ul', 'legend'), kinds = new Set(points.map(p => p.kind)); + for (const k of P.KINDS.map(x => x.id).concat([P.UNSET, 'other'])) if (kinds.has(k)) ul.appendChild(h('li', 'chip ' + k, k === 'other' ? 'Not a model of the map' : P.kindLabel(k))); + ul.appendChild(h('li', null, line)); return ul; + } + /** The table view of one chart panel: every drawn value as text. */ + function pointsTable(d, s) { + const cols = [['Subject'], [d.metric.title + units(d.metric), 'n']]; + if (d.form === 'tradeoff') cols.push([d.metric_x.title + units(d.metric_x), 'n'], ['Frontier']); + if (d.form === 'progression') cols.push(['Best so far', 'n'], ['Trained']); + cols.push(['Cases', 'n']); + const t = h('table', 'tv pts'); t.setAttribute('aria-label', d.title + (s.group ? ', ' + d.group + ' = ' + s.group : '') + ', as a table'); t.appendChild(headRow(cols)); + const tb = h('tbody'); + for (const p of s.points) { + const tr = h('tr'); tr.appendChild(h('td', null, p.title)); tr.appendChild(h('td', 'n', fmt(p.value))); + if (d.form === 'tradeoff') { tr.appendChild(h('td', 'n', fmt(p.x))); tr.appendChild(h('td', null, p.frontier ? 'yes' : '')); } + if (d.form === 'progression') { tr.appendChild(h('td', 'n', fmt(p.best_so_far))); tr.appendChild(h('td', 'small faint nowrap', p.trained_at ? P.day(p.trained_at) : '—')); } + tr.appendChild(h('td', 'n', String(p.count))); tb.appendChild(tr); + } + t.appendChild(tb); return wrapTable(t); + } + /** A comparison chart: every metric per subject, the best of each column marked. */ + function comparisonTable(d, view, o) { + if (!d.rows.length) return h('p', 'empty', 'No subject has a value for this chart yet.'); + const best = {}; + for (const m of d.metrics) { const xs = d.rows.map(r => r.values[m.id]).filter(v => v != null); if (xs.length && m.better !== 'none') best[m.id] = m.better === 'higher' ? Math.max(...xs) : Math.min(...xs); } + const cols = [['Subject']].concat(d.group ? [[d.group]] : []).concat(d.metrics.map(m => [m.title + units(m), 'n'])); + const t = h('table', 'tv cmpgrid'); t.appendChild(headRow(cols)); const tb = h('tbody'); + for (const r of d.rows) { + const s = view.byName.get(r.subject), key = s ? s.key : 'subject:' + r.subject; + const tr = h('tr', 'rowbtn' + (key === o.sel ? ' is-sel' : '')); tr.tabIndex = 0; tr.dataset.key = 'cmpgrid-' + key + '-' + r.group; + tr.addEventListener('click', () => o.onSelect(key)); tr.addEventListener('keydown', e => { if (e.key === 'Enter') o.onSelect(key); }); + tr.appendChild(h('td', null, s ? s.full : r.title)); if (d.group) tr.appendChild(h('td', 'small faint', r.group)); + for (const m of d.metrics) { const v = r.values[m.id], td = h('td', 'n' + (v != null && v === best[m.id] ? ' best' : ''), fmt(v)); if (v != null && v === best[m.id]) td.title = 'Best of the column, ' + better(m); tr.appendChild(td); } + tb.appendChild(tr); + } + t.appendChild(tb); const w = wrapTable(t); + const foot = h('p', 'small faint bench-desc', 'Marked: the best value of each column.'); const box = h('div'); box.appendChild(w); box.appendChild(foot); return box; + } + /** Up to four subjects side by side: every metric, with the change from the first. */ + function compareSection(view, cmp, stat, act) { + const sec = h('section', 'bench-cmp'); sec.appendChild(sect('Compare', cmp.length + ' of ' + COMPARE_MOST)); + const list = cmp.map(k => view.byKey.get(k)); + const chips = h('div', 'cmp-chips'); + list.forEach((s, i) => { const c = h('span', 'c s' + (i + 1)); c.appendChild(h('i')); c.appendChild(document.createTextNode(s.full)); const x = button(null, '×', () => act.toggleCompare(s.key)); x.setAttribute('aria-label', 'Remove ' + s.full); c.appendChild(x); chips.appendChild(c); }); + chips.appendChild(button('btn sm ghost', 'Clear', () => act.clearCompare())); + if (cmp.length >= 2) { const o = button('btn sm', 'Open Compare', () => act.go('compare')); o.dataset.key = 'bench-open-compare'; chips.appendChild(o); } + sec.appendChild(chips); + const t = h('table', 'tv cmpt'); t.appendChild(headRow([['']].concat(list.map(s => [s.full, 'n'])))); const tb = h('tbody'), base = list[0]; + for (const m of view.metrics) { + const tr = h('tr'); tr.appendChild(h('td', null, m.title + units(m) + ' · ' + stat)); + list.forEach((s, i) => { + const v = valueOf(s, m.id, stat), td = h('td', 'n', fmt(v)), b0 = valueOf(base, m.id, stat); + if (i > 0 && v != null && b0 != null && b0 !== 0) { const d = (v - b0) / Math.abs(b0), good = m.better === 'higher' ? d > 0 : m.better === 'lower' ? d < 0 : null; td.appendChild(h('span', 'd' + (d === 0 || good === null ? '' : good ? ' better' : ' worse'), ' ' + (d > 0 ? '+' : '') + (Math.abs(d) >= 10 ? (d * 100).toFixed(0) : (d * 100).toFixed(1)) + '%')); } + tr.appendChild(td); + }); + tb.appendChild(tr); + } + const cr = h('tr'); cr.appendChild(h('td', null, 'Cases')); list.forEach(s => cr.appendChild(h('td', 'n', s.count == null ? '—' : String(s.count)))); tb.appendChild(cr); + const vr = h('tr'); vr.appendChild(h('td', null, 'Per-case values here')); list.forEach(s => vr.appendChild(h('td', 'n', s.perCase ? 'yes' : 'no'))); tb.appendChild(vr); + t.appendChild(tb); sec.appendChild(wrapTable(t)); + sec.appendChild(h('p', 'small faint bench-desc', 'The change is from ' + base.full + '; it is coloured only for metrics with a better direction.')); + return sec; + } + /** Every subject of the benchmark, sortable by any column; a row opens the subject, its box adds it to the compare set. */ + function subjectTable(view, state, act, stat, sel, cmp, ivs) { + const sec = h('section', 'bench-all'), show = visibleSubject(state); + const rows = view.subjects.filter(s => show(s) && (!state.ui.resultsOnly || view.metrics.some(m => valueOf(s, m.id, stat) != null))); + const sort = state.ui.sort || { key: view.metrics.length ? 'm:' + view.metrics[0].id : 'full', dir: 1 }; + const keyOf = (s, k) => k === 'short' ? (s.model && s.model.short) || null : k === 'full' ? (s.model ? s.model.title : s.title) : k === 'kind' ? (s.model ? P.kindLabel(s.kind) : null) : k === 'group' ? s.group || null : k === 'count' ? s.count : k === 'perCase' ? (s.perCase ? 1 : null) : k.startsWith('m:') ? valueOf(s, k.slice(2), stat) : null; + const metricOf = k => k.startsWith('m:') ? view.metrics.find(m => m.id === k.slice(2)) : null; + const flip = (metricOf(sort.key) || {}).better === 'higher' ? -1 : 1; // a metric column sorts best first + rows.sort((a, b) => { const ka = keyOf(a, sort.key), kb = keyOf(b, sort.key); if (ka == null && kb == null) return a.full.localeCompare(b.full); if (ka == null) return 1; if (kb == null) return -1; return (typeof ka === 'number' ? (ka - kb) * flip : String(ka).localeCompare(String(kb))) * sort.dir; }); + sec.appendChild(sect('All subjects', rows.length + ' of ' + view.subjects.length + ' · ' + stat + ' of each metric')); + const cols = [['', 'cmp'], ['', 'short', 'ck'], ['Subject', 'full'], ['Kind', 'kind'], ['Group', 'group']].concat(view.metrics.map(m => [m.title + units(m), 'm:' + m.id, 'n'])).concat([['Cases', 'count', 'n'], ['Per case', 'perCase']]); + const t = h('table', 'tv all'), thead = h('thead'), tr = h('tr'); + for (const [label, key, cls] of cols) { + const th = h('th', cls === 'n' ? 'n' : null); + if (key === 'cmp') { th.appendChild(h('span', 'sr-only', 'Compare')); tr.appendChild(th); continue; } + const b = button(null, label || 'Short name', () => act.setUI('sort', sort.key === key ? { key, dir: -sort.dir } : { key, dir: 1 })); b.dataset.key = 'sort-' + key; + if (!label) { b.textContent = ''; b.appendChild(h('span', 'sr-only', 'Short name')); b.appendChild(document.createTextNode('#')); } + if (sort.key === key) { b.setAttribute('aria-sort', sort.dir > 0 ? 'ascending' : 'descending'); th.setAttribute('aria-sort', sort.dir > 0 ? 'ascending' : 'descending'); } + th.appendChild(b); tr.appendChild(th); + } + thead.appendChild(tr); t.appendChild(thead); + const tb = h('tbody'); + for (const s of rows) { + const row = h('tr', 'rowbtn' + (s.key === sel ? ' is-sel' : '')); row.dataset.subject = s.name; row.addEventListener('click', () => act.select(s.key === sel ? null : s.key)); + const tdc = h('td'), cb = h('input'); cb.type = 'checkbox'; cb.checked = cmp.includes(s.key); cb.disabled = !cb.checked && state.cmp.length >= COMPARE_MOST; cb.dataset.key = 'cmp-' + s.key; cb.setAttribute('aria-label', 'Compare ' + s.full); + cb.addEventListener('click', ev => { ev.stopPropagation(); act.toggleCompare(s.key); }); tdc.appendChild(cb); row.appendChild(tdc); + row.appendChild(h('td', 'ck', s.model && s.model.short ? s.model.short : '')); + const nt = h('td'); const nb = button('linkish', s.model ? s.model.title : s.title, ev => { ev.stopPropagation(); act.select(s.key === sel ? null : s.key); }); nb.dataset.key = 'open-' + s.key; nt.appendChild(nb); row.appendChild(nt); + const kt = h('td'); kt.appendChild(s.model ? h('span', 'chip ' + s.kind, P.kindLabel(s.kind)) : h('span', 'small faint', 'not on the map')); row.appendChild(kt); + row.appendChild(h('td', 'small faint nowrap grp', s.group)); + for (const m of view.metrics) { + const v = valueOf(s, m.id, stat), td = h('td', 'n', v == null ? '' : fmt(v)), e = v == null ? null : ivs.of(s.name, m.id, stat); + if (e) { const ci = h('span', 'ci', fmt(e[0]) + ' – ' + fmt(e[1])); ci.title = 'The 95 % interval of the ' + stat + ' from the per-case values'; td.appendChild(ci); td.dataset.low = String(e[0]); td.dataset.high = String(e[1]); } + row.appendChild(td); + } + row.appendChild(h('td', 'n', s.count == null ? '' : String(s.count))); row.appendChild(h('td', 'small faint', s.perCase ? 'held here' : '')); + tb.appendChild(row); + } + t.appendChild(tb); sec.appendChild(wrapTable(t)); + if (!rows.length) sec.appendChild(h('p', 'empty', 'No subject matches the filters.')); + return sec; + } + + // ---- Train -------------------------------------------------------------------- + const TO_BUILD = ['not-built', 'out-of-date', 'failed']; + const RUN_TONE = { succeeded: 'good', adopted: 'good', running: 'pending', queued: 'pending', failed: 'serious', 'timed-out': 'serious', 'over-disk': 'serious', unknown: 'warn', cancelled: 'neutral' }; + const REQUEST_TONE = { 'awaiting-approval': 'pending', queued: 'pending', running: 'pending', succeeded: 'good', failed: 'serious', cancelled: 'neutral', declined: 'neutral', expired: 'neutral' }; + const runState = st => S.stateEl(RUN_TONE[st] || 'neutral', st === 'timed-out' ? 'ran out of time' : st === 'over-disk' ? 'over its disk budget' : st); + const requestState = st => S.stateEl(REQUEST_TONE[st] || 'neutral', st === 'awaiting-approval' ? 'awaiting approval' : st); + const who = r => r && (r.label || r.kind) || 'someone'; + const covering = r => r.every ? 'everything buildable' : (r.targets || []).join(', ') || 'nothing named'; + function train(main, aside, state, act, data) { + const sel = state.sel.train || null; + paint(main, async () => { + const [recipes, runs, requests, envs, endpoint, served] = await Promise.all([P.read('definitions', { kind: 'recipe' }), P.read('runs'), P.read('build_requests'), P.read('environments'), + window.LabGenerate ? P.read('endpoint').catch(error => ({ error })) : null, window.LabServing ? P.call('served').catch(error => ({ error })) : null]); + const running = runs.runs.filter(r => r.state === 'running'); + const progress = new Map(await Promise.all(running.map(r => P.call('progress', { run: r.id }).then(p => [r.id, p.event], () => [r.id, null])))); + return { recipes: recipes.definitions, runs: runs.runs, requests: requests.requests, envs: envs.environments, progress, endpoint, served }; + }, (frag, { recipes, runs, requests, envs, progress, endpoint, served }) => { + const g = graphFor(data.map), deviceOf = new Map(envs.map(e => [e.reference, e.device])); + // Planned (spec-w3 section 3.2): the drafts of this page, then the models never built. + const E = window.LabEditor, drafts = g.nodes.filter(m => m.draft); + const never = recipes.filter(d => !d.removed && d.state === 'not-built' && (!g.byId.has(d.reference) || g.byId.get(d.reference).planned)); + const neverRefs = new Set(never.map(d => d.reference)); + frag.appendChild(sect('Planned', String(drafts.length + never.length))); + if (!drafts.length && !never.length) frag.appendChild(h('p', 'empty', 'Nothing is planned. Branch from a model on the map to plan a new one.')); + else { + const ul = h('ul', 'rows planned'); + for (const m of drafts) { + const li = h('li'), b = button('row todo'); b.dataset.key = 'open-' + m.id; b.dataset.draft = m.id; + b.appendChild(S.stateEl('neutral', 'not kept yet')); const t = h('span'); t.appendChild(h('span', 'title', m.title)); t.appendChild(h('span', 'sub', 'from ' + (g.byId.has(m.parent) ? Tree.nameOf(g.byId.get(m.parent)) : m.parent) + ' · on the map until it is kept')); b.appendChild(t); + b.appendChild(h('span', 'chip ' + m.kind + ' draft', 'draft')); + b.addEventListener('click', () => act.go('map', m.id)); li.appendChild(b); ul.appendChild(li); + } + for (const d of never) ul.appendChild(todoRow(d, g, sel, act)); + frag.appendChild(ul); + } + // Models to build again: out of date or failed. + const todo = recipes.filter(d => !d.removed && TO_BUILD.includes(d.state) && !neverRefs.has(d.reference)); + const th = sect('To build', todo.length + ' of ' + plural(recipes.length, 'model')); th.classList.add('gap'); th.appendChild(h('span', 'spacer')); + th.appendChild(A.group('train:every', [{ id: 'plan', label: 'Plan everything buildable', cls: 'btn sm', who: 'request', onClick: () => act.select('plan:*') }], { inline: true })); + frag.appendChild(th); + if (!todo.length) frag.appendChild(h('p', 'empty', 'No model is out of date or failed.')); + else { + const ul = h('ul', 'rows'); + for (const d of todo) ul.appendChild(todoRow(d, g, sel, act)); + frag.appendChild(ul); + } + // Build requests. + const rh = sect('Build requests', String(requests.length)); rh.classList.add('gap'); frag.appendChild(rh); + if (!requests.length) frag.appendChild(h('p', 'empty', 'Nobody has asked for a build of this project through Tuldok.')); + else { + const ul = h('ul', 'rows'); + for (const r of requests) { + const li = h('li'), key = 'request:' + r.id, b = button('row request'); b.setAttribute('aria-pressed', String(key === sel)); b.dataset.key = 'open-' + key; + b.appendChild(requestState(r.state)); const t = h('span'); t.appendChild(h('span', 'title', 'A build of ' + covering(r))); t.appendChild(h('span', 'sub', 'asked by ' + who(r.requested_by) + (r.scratch ? ' · from the code as it is' : '') + (r.problem ? ' · ' + r.problem : ''))); b.appendChild(t); + b.appendChild(h('span', 'sub num', time(r.requested_at))); + b.addEventListener('click', () => act.select(key === sel ? null : key)); li.appendChild(b); ul.appendChild(li); + } + frag.appendChild(ul); + } + // Served models (serving.js, spec-t6 section 7). + if (served) window.LabServing.section(frag, served, sel, act); + // Runs, newest first. + const bar = h('div', 'filters gap'); bar.appendChild(searchBox(state, act, 'Find a run')); bar.appendChild(kindChips(state, act, g.nodes.map(m => m.kind))); + bar.appendChild(h('span', 'pop', plural(runs.length, 'run') + ' on this machine')); + frag.appendChild(bar); + const q = (state.query || '').toLowerCase(); + const shown = runs.filter(r => { const m = g.byId.get(r.reference); return (!state.kinds || (m && state.kinds.has(m.kind))) && (!q || [r.reference, r.subject, r.id, m && Tree.nameOf(m)].some(t => t && t.toLowerCase().includes(q))); }); + frag.appendChild(sect('Runs', shown.length + ' of ' + runs.length + ' · newest first')); + if (!runs.length) frag.appendChild(h('p', 'empty', 'Nothing has been built or adopted in this project on this machine.')); + else { + const ul = h('ul', 'rows'); + for (const r of shown) { + const m = g.byId.get(r.reference), li = h('li'), key = 'run:' + r.id, b = button('row run'); b.setAttribute('aria-pressed', String(key === sel)); b.dataset.key = 'open-' + key; b.dataset.run = r.id; + b.appendChild(h('span', 'ck', m ? Tree.refName(m) : nameOfRef(r.reference))); + const t = h('span'); t.appendChild(h('span', 'title', (m ? m.title : r.reference) + (r.subject ? ' · ' + r.subject : ''))); + const when = r.finished_at || r.started_at || r.queued_at, ev = progress.get(r.id); + if (r.state === 'running' && ev) t.appendChild(rowProgress(ev)); + else t.appendChild(h('span', 'sub', [r.state === 'running' ? 'started ' + time(r.started_at) + ' · no progress reported yet' : time(when), seconds(r.seconds), r.bytes != null ? bytes(r.bytes) : null, r.scratch ? 'from the code as it is' : null].filter(Boolean).join(' · '))); + b.appendChild(t); + b.appendChild(m ? h('span', 'chip ' + m.kind, P.kindLabel(m.kind)) : h('span', 'small faint', nameOfRef(r.reference).length ? r.reference.split('/')[0].replace(/s$/, '') : '')); + b.appendChild(h('span', 'sub', (r.environment && deviceOf.get(r.environment.reference)) || '—')); + b.appendChild(runState(r.state)); + b.addEventListener('click', () => act.select(key === sel ? null : key)); li.appendChild(b); ul.appendChild(li); + } + frag.appendChild(ul); + if (runs.length >= 100) frag.appendChild(h('p', 'note', 'The hundred newest runs are listed.')); + } + // Environments. + const eh = sect('Environments', String(envs.length)); eh.classList.add('gap'); frag.appendChild(eh); + if (!envs.length) frag.appendChild(h('p', 'empty', 'The project defines no environment.')); + else { + const ul = h('ul', 'rows'); + for (const e of envs) { + const li = h('li'), key = 'env:' + e.reference, r = button('row env'); r.appendChild(modelState(e.state)); r.setAttribute('aria-pressed', String(key === sel)); r.dataset.key = 'open-' + key; r.dataset.env = e.reference; + r.addEventListener('click', () => act.select(key === sel ? null : key)); + const t = h('span'); t.appendChild(h('span', 'title', e.title)); + const asks = ['asks Python ' + (e.python || '—'), e.device, plural(Object.keys(e.packages || {}).length, 'package')].filter(Boolean).join(', '); + const found = e.found ? 'found Python ' + (e.found.python || '?') + ', ' + plural(Object.keys(e.found.packages || {}).length, 'package') : e.interpreter ? 'not checked' : 'no interpreter recorded here'; + t.appendChild(h('span', 'sub', asks + ' · ' + found + (e.checked_at ? ' · checked ' + P.day(e.checked_at) : ''))); + if (e.problems && e.problems.length) t.appendChild(h('span', 'sub warnline', e.problems.join(' '))); + r.appendChild(t); r.appendChild(h('span', 'sub mono', e.interpreter || '')); li.appendChild(r); ul.appendChild(li); + } + frag.appendChild(ul); + } + // The Pumas endpoint of this machine, beside the environments (spec-d5 section 5, generate.js). + if (endpoint) window.LabGenerate.section(frag, endpoint, sel, act); + }); + if (!sel) return closeDrawer(aside); + if (sel === 'endpoint' && window.LabGenerate) window.LabGenerate.drawer(aside, act); + else if (sel.startsWith('run:')) runDrawer(aside, sel, act, data, state); + else if (sel.startsWith('request:')) requestDrawer(aside, sel, act, data); + else if (sel.startsWith('plan:')) A.planView(aside, sel, act.go, () => { const one = sel.slice(5); act.select(data.map.byId.has(one) ? one : null); }); + else if (sel.startsWith('env:')) envDrawer(aside, sel, act, data); + else if (sel.startsWith('serve:') && window.LabServing) window.LabServing.drawer(aside, sel, act); + else if (window.LabEditor && window.LabEditor.owns(sel, data)) window.LabEditor.drawer(aside, sel, act, data, 'train'); + else if (data.map.byId.has(sel)) S.inspectorModel(aside, sel, act, data); + else if (aside._for !== sel) { aside.textContent = ''; aside._for = sel; aside.appendChild(h('p', 'empty', 'There is no ' + sel + ' here.')); } + return true; + } + /** A row of a model to build, opening its drawer, with Plan build. */ + function todoRow(d, g, sel, act) { + const m = g.byId.get(d.reference), li = h('li'), b = button('row todo'); b.setAttribute('aria-pressed', String(d.reference === sel)); b.dataset.key = 'open-' + d.reference; + b.appendChild(modelState(d.state)); const t = h('span'); const title = h('span', 'title', m ? Tree.nameOf(m) : d.title); if (m) title.appendChild(Tree.mark(m)); t.appendChild(title); t.appendChild(h('span', 'sub', m ? m.groupTitle + (m.description ? ' · ' + m.description : '') : d.reference)); b.appendChild(t); + b.appendChild(m ? h('span', 'chip ' + m.kind, P.kindLabel(m.kind)) : h('span')); + b.addEventListener('click', () => act.select(d.reference === sel ? null : d.reference)); li.appendChild(b); + const plan = A.group('todo:' + d.reference, [{ id: 'plan', label: 'Plan build', cls: 'btn sm', who: 'request', onClick: () => act.select('plan:' + d.reference) }], { inline: true }); + if (!plan.hidden) { li.classList.add('with-act'); li.appendChild(plan); } + if (window.LabEditor) window.LabEditor.decorate(li); + return li; + } + const tail = (text, lines) => { const all = text.replace(/\n$/, '').split('\n'); return (all.length > lines ? '…\n' : '') + all.slice(-lines).join('\n'); }; + /** A definition's value in one line: a list of words as a command line, an object as key: value pairs. */ + const brief = v => v == null ? '—' : typeof v !== 'object' ? String(v) : Array.isArray(v) ? (v.every(x => typeof x === 'string') ? v.join(' ') : v.map(brief).join(', ')) : Object.entries(v).map(([k, x]) => k + ': ' + (x && typeof x === 'object' && !Array.isArray(x) ? '{' + brief(x) + '}' : brief(x))).join(', '); + const json = v => { if (typeof v === 'string') { try { v = JSON.parse(v); } catch (e) { return v; } } return brief(v); }; + const ACTIVE = ['queued', 'running']; + const LOG_LINES = 200; // lines of each log kept on the page while a run is read live + const hairOf = ev => { const hair = h('span', 'hair'); const i = h('i'); i.style.width = Math.round(100 * ev.completed / Math.max(1, ev.total)) + '%'; hair.appendChild(i); hair.setAttribute('role', 'progressbar'); hair.setAttribute('aria-valuemin', '0'); hair.setAttribute('aria-valuemax', String(ev.total)); hair.setAttribute('aria-valuenow', String(ev.completed)); return hair; }; + const eventText = ev => ev.stage + ': ' + ev.completed + ' of ' + ev.total + Object.entries(ev.metrics).map(([k, v]) => ' · ' + k + ' ' + fmt(v)).join(''); + // A running run's progress in its row of Train's list: a hairline and a line. + const rowWidth = ev => Math.round(100 * ev.completed / Math.max(1, ev.total)) + '%'; + const rowText = ev => ev.stage + ' ' + ev.completed + ' of ' + ev.total + Object.entries(ev.metrics).slice(0, 2).map(([k, v]) => ' · ' + k + ' ' + fmt(v)).join(''); + function rowProgress(ev) { const line = h('span', 'sub live'); const hair = h('span', 'hair'); const i = h('i'); i.style.width = rowWidth(ev); hair.appendChild(i); line.appendChild(hair); line.appendChild(document.createTextNode(rowText(ev))); return line; } + /** + * Sets the latest progress of the running runs (a Map of run id to event, + * app.js on every reading) into their rows of Train's list, in place: the + * hairline's width and the line's text, when they differ (spec-w4 R7). + */ + S.progress = events => { + for (const b of document.querySelectorAll('#main button.row.run[data-run]')) { + const ev = events.get(b.dataset.run); if (!ev) continue; + const line = b.querySelector('.sub.live'); + if (!line) { const sub = b.children[1] && b.children[1].querySelector('.sub'); if (sub) sub.replaceWith(rowProgress(ev)); continue; } // its first progress + const i = line.querySelector('i'), width = rowWidth(ev), text = rowText(ev); + if (i.style.width !== width) i.style.width = width; + if (line.lastChild.data !== text) line.lastChild.data = text; + } + }; + let live = null; // the open run read live: {key, run, offsets, update(progress), ended()} + /** + * One step of the live run (app.js calls it on every reading, every two + * seconds while a build runs): its progress and its logs from the offsets, + * updated in place (spec-w4 R7). When it has ended, the project is read + * again and its drawer is drawn again in place. + */ + S.tick = async () => { + if (window.LabTraining) await window.LabTraining.tick(); // the open run's Training block, in place (training.js) + const aside = document.getElementById('aside'); + if (!live || aside._for !== live.key) { live = null; return; } + const mine = live; + try { + const pr = await P.call('progress', { run: mine.run, stdout_offset: mine.offsets.stdout, stderr_offset: mine.offsets.stderr }); + if (live !== mine || aside._for !== mine.key) return; + mine.update(pr); + if (pr.ended) { live = null; mine.ended(); } // not awaited: the reading it starts waits for the one calling this + } catch (error) { /* the next tick asks again; the strip says what fails */ } + }; + /** A run's name in the Training block: its model's short name (or its definition) and the start of its id. */ + const runLabel = (runs, data) => id => { const r = runs.find(x => x.id === id), m = r && graphFor(data.map).byId.get(r.reference); return (m ? Tree.refName(m) : r ? nameOfRef(r.reference) : 'run') + (r && r.subject ? ' · ' + r.subject : '') + ' · ' + id.slice(0, 8); }; + function runDrawer(aside, key, act, data, state) { + const id = key.slice(4), T = window.LabTraining; + drawer(aside, key, () => Promise.all([P.read('run', { run: id }), P.call('progress', { run: id }), + T ? T.read(id, state.ui).catch(error => ({ error })) : null, T ? P.read('runs').then(a => a.runs, () => []) : []]), (ins, [r, pr, tr, runs]) => { + const m = graphFor(data.map).byId.get(r.reference); + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'run ' + r.id)); const t2 = h('h2', null, (m ? Tree.nameOf(m) : r.reference) + (r.subject ? ' · ' + r.subject : '')); t2.tabIndex = -1; hd.appendChild(t2); + const meta = h('div', 'ins-meta'); if (m) meta.appendChild(h('span', 'chip ' + m.kind, P.kindLabel(m.kind))); meta.appendChild(runState(r.state)); if (r.scratch) meta.appendChild(h('span', 'small faint', 'from the code as it is')); hd.appendChild(meta); ins.appendChild(hd); + if (m) { const acts = h('div', 'actions'); acts.appendChild(button('btn sm ghost', 'On the map', () => act.go('map', m.id))); acts.appendChild(button('btn sm ghost', 'Its model', () => act.select(m.id))); ins.appendChild(acts); } + // Cancel run while it is queued or running; Stop what it left running when its build is gone. + const active = ACTIVE.includes(r.state), orphan = active && r.builder_alive === false; + ins.appendChild(A.group('run:' + r.id, [ + { id: 'cancel', label: 'Cancel run', cls: 'btn', who: 'cancel', op: 'cancel_run', args: () => ({ run: r.id }), busy: 'Cancelling…', hidden: !active || orphan, + confirm: 'Ask this run to stop. It ends cancelled and leaves no product.' }, + { id: 'stop', label: 'Stop what it left running', cls: 'btn', who: 'cancel', op: 'stop_run', args: () => ({ run: r.id }), busy: 'Stopping…', hidden: !orphan, + confirm: 'Kill every process this run left behind and mark it unknown; its build is gone.' } + ])); + const su = setup(); + su.put('Definition', r.reference, 'mono'); + if (r.subject) su.put('Subject', r.subject); + su.put('Queued', time(r.queued_at)); su.put('Started', time(r.started_at)); su.put('Finished', time(r.finished_at)); + su.put('Took', r.seconds == null ? '—' : seconds(r.seconds)); if (r.exit_status != null) su.put('Exit status', String(r.exit_status)); + if (r.environment) su.put('Environment', r.environment.reference + (r.environment.found ? ' · Python ' + r.environment.found.python : '') + (r.environment.interpreter ? ' · ' + r.environment.interpreter : ''), 'mono'); + su.put('Commit', r.commit || '—', 'mono'); + su.put('Command', (r.argv || []).join(' ') || '—', 'mono cmdline'); + const inputs = Object.entries(r.inputs || {}); + su.put('Inputs', inputs.length ? inputs.map(([k, v]) => k + ' ← ' + (v.reference || '?') + (v.output ? ' (' + v.output + ')' : '') + (v.run_id ? ' · run ' + v.run_id.slice(0, 8) : '')).join('\n') : 'none', 'mono pre'); + if (r.settings != null) su.put('Settings', json(r.settings), 'mono'); + if (r.env_names && r.env_names.length) su.put('Variables', r.env_names.join(', ') + ' (values not shown)', 'mono'); + if (r.product) su.put('Product', r.product.directory + '\n' + Object.entries(r.product.outputs || {}).map(([k, v]) => k + ': ' + v.path + ' · ' + bytes(v.bytes)).join('\n') + (r.product.current ? '' : '\n(no longer current)') + (r.product.adopted ? '\n(adopted)' : ''), 'mono pre'); + else su.put('Product', 'none'); + if (r.bytes != null) su.put('Size', bytes(r.bytes)); + if (r.problem) su.put('Problem', r.problem); + // The latest progress: a hairline and its line, kept current while the run is read live. + const prog = h('div', 'progress-line'); + const showEvent = ev => { prog.textContent = ''; if (ev) { prog.appendChild(hairOf(ev)); prog.appendChild(h('span', 'small', eventText(ev))); } else prog.appendChild(h('p', 'note', active ? 'No progress reported yet.' : 'No progress was reported.')); }; + showEvent(pr.event); + const progBlock = block('Latest progress', prog); if (!pr.event && !active) progBlock.hidden = true; + // A run read live shows its progress and logs first; an ended one its record first. + ins.appendChild(progBlock); + // What its training script logged, drawn live while it runs (training.js, spec-t1.md section 6). + if (tr && tr.error) ins.appendChild(block('Training', h('p', 'note', 'Its training could not be read: ' + tr.error.message))); + else if (tr) ins.appendChild(T.block(tr, act, runLabel(runs, data), runs)); + if (!active) ins.appendChild(su.dl); + const logs = {}; + for (const [name, title] of [['stdout', 'Standard output'], ['stderr', 'Standard error']]) { + const log = pr[name] || { text: '' }, b = h('div'), head = h('h3', null, title + (log.size != null ? ' · ' + bytes(log.size) : '')); + b.appendChild(head); + const pre = h('pre', 'evtext'); pre.dataset.log = name; let lines = log.text ? log.text.replace(/\n$/, '').split('\n').slice(-LOG_LINES) : []; + const note = h('p', 'note', log.size == null ? 'No log on this machine.' : 'Empty.'); + const paintLog = () => { pre.textContent = (lines.length >= LOG_LINES ? '…\n' : '') + lines.join('\n'); pre.hidden = !lines.length; note.hidden = !!lines.length; pre.scrollTop = pre.scrollHeight; }; + b.appendChild(pre); b.appendChild(note); paintLog(); ins.appendChild(b); + logs[name] = { head, title, add: text => { if (!text) return; lines = lines.concat(text.replace(/\n$/, '').split('\n')).slice(-LOG_LINES); paintLog(); } }; + } + if (active) ins.appendChild(su.dl); + if (!active && live && live.key === key) live = null; // drawn again once it ended + if (active) { + live = { key, run: r.id, offsets: { stdout: (pr.stdout || {}).offset || 0, stderr: (pr.stderr || {}).offset || 0 }, + update: p => { + showEvent(p.event); progBlock.hidden = false; + for (const name of ['stdout', 'stderr']) { const log = p[name]; if (!log) continue; logs[name].add(log.text); live.offsets[name] = log.offset; if (log.size != null) logs[name].head.textContent = logs[name].title + ' · ' + bytes(log.size); } + }, + ended: () => act.refresh() }; + ins.appendChild(h('p', 'note live-note', 'Read live every two seconds while it runs.')); + } + }); + } + function requestDrawer(aside, key, act, data) { + const id = key.slice(8); + drawer(aside, key, () => P.read('build_request', { request: id }), (ins, r) => { + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'build request ' + r.id)); hd.appendChild(h('h2', null, 'A build of ' + covering(r))); + const meta = h('div', 'ins-meta'); meta.appendChild(requestState(r.state)); hd.appendChild(meta); ins.appendChild(hd); + if (r.state === 'declined' && r.problem) ins.appendChild(h('p', 'reason', 'Declined: ' + r.problem)); + // Approve and Decline while it awaits approval; Cancel while it awaits approval, is queued or runs. + const waiting = r.state === 'awaiting-approval', cancellable = ['awaiting-approval', 'queued', 'running'].includes(r.state); + ins.appendChild(A.group('request:' + r.id, [ + { id: 'approve', label: 'Approve', cls: 'btn primary', who: 'owner', op: 'approve_build', args: () => ({ request: r.id }), busy: 'Approving…', hidden: !waiting }, + Object.assign(A.declineItem(r.id), { cls: 'btn', hidden: !waiting }), + { id: 'cancel', label: 'Cancel', cls: 'btn ghost', who: 'cancel', op: 'cancel_build', args: () => ({ request: r.id }), busy: 'Cancelling…', hidden: !cancellable, + confirm: r.state === 'running' ? 'Cancel this build: its running step is asked to stop and nothing after it runs.' : 'Cancel this build request; nothing of it runs.' } + ])); + const su = setup(); + su.put('Asked by', who(r.requested_by) + ' · ' + time(r.requested_at)); + su.put('Targets', r.every ? 'everything buildable' + ((r.targets || []).length ? ', and ' + r.targets.join(', ') : '') : (r.targets || []).join(', ') || '—', 'mono'); + if (r.decided_at) su.put('Decided by', who(r.decided_by) + ' · ' + time(r.decided_at)); + if (r.started_at) su.put('Started', time(r.started_at)); if (r.finished_at) su.put('Finished', time(r.finished_at)); + if (r.expires_at) su.put('Expires', time(r.expires_at)); + su.put('Code', r.scratch ? 'as it is on disk, uncommitted changes included; the product is marked scratch' : (r.covers && r.covers.commit ? 'commit ' + r.covers.commit : 'committed'), 'mono'); + if (r.covers && r.covers.definitions) su.put('Definitions', plural(Object.keys(r.covers.definitions).length, 'definition') + ', as they stand when it is asked'); + if (r.problem) su.put(r.state === 'declined' ? 'Reason' : 'Problem', r.problem); + if (r.result) su.put('Result', json(r.result), 'mono'); + ins.appendChild(su.dl); + const steps = (r.covers && r.covers.steps) || []; + const ul = h('ul', 'links'); + for (const s of steps) { const m = data.map.byId.get(s.reference), li = h('li'); const b = button(null, m ? Tree.nameOf(m) : s.reference, m ? () => act.select(m.id) : null); if (!m) b.disabled = true; if (s.subjects && s.subjects.length) b.appendChild(h('span', 'why', 'subjects: ' + s.subjects.join(', '))); li.appendChild(b); ul.appendChild(li); } + ins.appendChild(block('What it covers · ' + plural(steps.length, 'step'), steps.length ? ul : h('p', 'note', 'Nothing.'))); + if (r.covers && r.covers.definitions) ins.appendChild(h('p', 'note', 'Approved as these ' + Object.keys(r.covers.definitions).length + ' definitions stand now: ' + Object.keys(r.covers.definitions).join(', ') + '.')); + const rl = h('ul', 'rows'); + for (const run of r.runs || []) { const li = h('li'), b = button('row'); b.style.gridTemplateColumns = 'minmax(0, 1fr) auto'; const t = h('span'); t.appendChild(h('span', 'title', run.reference + (run.subject ? ' · ' + run.subject : ''))); t.appendChild(h('span', 'sub', [time(run.finished_at || run.started_at || run.queued_at), seconds(run.seconds)].filter(Boolean).join(' · '))); b.appendChild(t); b.appendChild(runState(run.state)); b.addEventListener('click', () => act.select('run:' + run.id)); li.appendChild(b); rl.appendChild(li); } + ins.appendChild(block('Its runs', (r.runs || []).length ? rl : h('p', 'note', r.state === 'awaiting-approval' ? 'None: it has not been approved.' : 'None.'))); + }); + } + + // ---- Data ----------------------------------------------------------------------- + // The tab has two states (spec-d1 section 7): the cards, and one dataset opened, '#data/inside:' + // (inside.js); a selection '' is that dataset's drawer beside the cards, as before. + const INSIDE = 'inside:'; + const lookInside = (ref, act) => { const b = button('btn sm', 'Look inside', ev => { ev.stopPropagation(); act.go('data', INSIDE + ref); }); b.dataset.key = 'inside-' + ref; return b; }; + /** Data: the dataset cards, a dataset's drawer, or one dataset opened (inside.js). */ + function data_(main, aside, state, act, data) { + const E = window.LabEditor, raw = state.sel.data || null; + if (raw && raw.startsWith(INSIDE)) { closeDrawer(aside); window.LabInside.open(main, raw.slice(INSIDE.length), act, !!aside._again); return false; } + if (window.LabInside) window.LabInside.leave(); + if (main._painted && String(main._painted).startsWith('data/' + INSIDE)) { main.scrollTop = 0; main.scrollLeft = 0; } // back to the cards: from their top + const sel = raw && E ? E.subject(raw) : raw; + paint(main, () => Promise.all([P.read('definitions', { kind: 'dataset' }), P.read('products'), P.read('graph')]), (frag, [defs, products, graph]) => { + const sets = defs.definitions.filter(d => !d.removed), byRef = new Map(); + for (const p of products.products) { if (!byRef.has(p.reference)) byRef.set(p.reference, []); byRef.get(p.reference).push(p); } + const users = new Map(); for (const e of graph.edges) { if (!users.has(e.to)) users.set(e.to, new Set()); users.get(e.to).add(e.from); } + const built = sets.filter(d => byRef.has(d.reference)).length, adopted = sets.filter(d => (byRef.get(d.reference) || []).some(p => p.adopted)).length; + const total = sets.reduce((n, d) => n + (byRef.get(d.reference) || []).reduce((a, p) => a + (p.bytes || 0), 0), 0); + const head = sect('Datasets', sets.length + ' · ' + built + ' with files on this machine' + (adopted ? ', ' + adopted + ' adopted' : '') + (total ? ' · ' + bytes(total) : '')); + // Three forms that keep a new dataset as an unsaved edit (spec-d4 section 7, annotate.js; spec-d5 section 5, generate.js): files brought in, a dataset frozen with splits, and one generated through the Pumas endpoint. + if (window.LabAnnotate) { head.appendChild(h('span', 'spacer')); head.appendChild(A.group('data:new', [ + { id: 'import', label: 'Import files', cls: 'btn sm', who: 'edit', onClick: () => act.select(sel === 'new:import' ? null : 'new:import') }, + { id: 'freeze', label: 'Freeze a dataset', cls: 'btn sm', who: 'edit', onClick: () => act.select(sel === 'new:freeze' ? null : 'new:freeze') }, + { id: 'generate', label: 'Generate', cls: 'btn sm', who: 'edit', hidden: !window.LabGenerate, onClick: () => act.select(sel === 'new:generate' ? null : 'new:generate') }, + { id: 'embed', label: 'Embed', cls: 'btn sm', who: 'edit', hidden: !window.LabCuration, onClick: () => act.select(sel === 'new:embed' ? null : 'new:embed') }], { inline: true })); } + frag.appendChild(head); + if (!sets.length) { frag.appendChild(h('p', 'empty', 'The project defines no dataset.')); return; } + const grid = h('div', 'ds-grid'); + for (const d of sets) { + const ps = byRef.get(d.reference) || [], outputs = ps.flatMap(p => Object.entries(p.outputs || {}).map(([k, v]) => ({ name: k, ...v, directory: p.directory }))), size = ps.reduce((a, p) => a + (p.bytes || 0), 0); + const card = button('ds-card'); card.setAttribute('aria-pressed', String(d.reference === sel)); card.dataset.key = 'open-' + d.reference; card.dataset.ref = d.reference; + const pv = h('div', 'pv text'); pv.setAttribute('aria-hidden', 'true'); + // A product with no declared outputs is its directory as a whole. + const whole = !outputs.length && ps.length > 0; + pv.textContent = outputs.length ? outputs.slice(0, 4).map(o => o.path + ' ' + bytes(o.bytes)).join('\n') + (outputs.length > 4 ? '\n…' : '') : whole ? 'the whole directory ' + bytes(size) : 'no files on this machine'; + card.appendChild(pv); + const t = h('div', 't'); t.appendChild(h('div', 'title', d.title)); + const meta = h('div', 'meta'); meta.appendChild(h('span', null, outputs.length ? plural(outputs.length, 'output') + ' · ' + bytes(size) : whole ? 'a directory · ' + bytes(size) : 'no files here')); + const st = ps.some(p => p.adopted) ? S.stateEl('good', 'adopted') : modelState(d.state); st.classList.add('glyphs'); meta.appendChild(st); t.appendChild(meta); + const used = [...(users.get(d.reference) || [])].sort(); + t.appendChild(h('div', 'uses', used.length ? 'used by ' + used.length + ': ' + used.map(nameOfRef).join(', ') : 'used by nothing')); + t.appendChild(h('div', 'path', ps.length ? ps[0].directory : d.reference)); card.appendChild(t); + card.title = d.title; + card.addEventListener('click', () => act.select(d.reference === sel ? null : d.reference)); + // "Look inside" beside the card (a button cannot hold a button), when the dataset has a product here. + const item = h('div', 'ds-item'); item.appendChild(card); + if (ps.some(p => p.run_id)) { const foot = h('div', 'ds-foot'); foot.appendChild(lookInside(d.reference, act)); item.appendChild(foot); } + grid.appendChild(item); + } + frag.appendChild(grid); + }); + if (!sel) return closeDrawer(aside); + if (raw === 'new:generate' && window.LabGenerate && window.LabAnnotate) { window.LabGenerate.form(aside, act); return true; } + if (raw === 'new:embed' && window.LabCuration && window.LabAnnotate) { window.LabCuration.form(aside, act); return true; } // spec-t5, curation.js + if (raw.startsWith('new:') && window.LabAnnotate) { window.LabAnnotate.datasetForm(aside, raw.slice(4), act); return true; } + if (E && E.owns(raw, data)) { E.drawer(aside, raw, act, data, 'data'); return true; } + drawer(aside, sel, () => Promise.all([P.read('definition', { reference: sel }), P.read('products', { reference: sel })]), (ins, [d, p]) => { + const hd = h('header'); hd.appendChild(h('span', 'kicker', d.reference)); const t2 = h('h2', null, d.title); t2.tabIndex = -1; hd.appendChild(t2); + const meta = h('div', 'ins-meta'); meta.appendChild(p.products.some(x => x.adopted) ? S.stateEl('good', 'adopted') : modelState(d.state)); meta.appendChild(Tree.mark({ id: d.reference, unsaved: d.unsaved })); hd.appendChild(meta); ins.appendChild(hd); + // Edit (spec-w3 section 3.2): the editor, in the text view. + if (p.products.some(x => x.run_id)) { const look = h('div', 'actions'); look.appendChild(lookInside(d.reference, act)); ins.appendChild(look); } + const doc = d.document || {}; + ins.appendChild(A.group('dataedit:' + d.reference, [{ id: 'edit', label: 'Edit', cls: 'btn sm', who: 'edit', onClick: () => act.select('edit:' + d.reference) }, + { id: 'plan', label: 'Plan build', cls: 'btn sm', who: 'request', hidden: !(doc.run && typeof doc.run === 'object'), onClick: () => act.go('train', 'plan:' + d.reference) }])); + if (E) E.decorate(hd); + ins.appendChild(block('Description', doc.description ? h('p', null, doc.description) : h('p', 'note', 'The definition has no description.'))); + const su = setup(); const skip = new Set(['format', 'kind', 'title', 'description']); + for (const [k, v] of Object.entries(doc)) { + if (skip.has(k)) continue; + if (v && typeof v === 'object' && !Array.isArray(v)) for (const [k2, v2] of Object.entries(v)) su.put(k + ' · ' + k2, brief(v2), 'mono'); + else su.put(k, brief(v), 'mono'); + } + ins.appendChild(block('Definition', su.dl.children.length ? su.dl : h('p', 'note', 'No other fields.'))); + const files = h('div'); + if (!p.products.length) files.appendChild(h('p', 'note', 'No files of this dataset on this machine.')); + for (const pr of p.products) { + const dl = setup(); dl.put('Folder', pr.directory, 'mono'); + for (const [k, o] of Object.entries(pr.outputs || {})) dl.put(k, pr.directory.replace(/\/$/, '') + '/' + o.path + ' · ' + bytes(o.bytes), 'mono'); + dl.put('Made', (pr.adopted ? 'adopted' : 'built') + ' · ' + time(pr.finished_at) + (pr.commit ? ' · ' + pr.commit.slice(0, 12) : '') + (pr.scratch ? ' · from the code as it is' : '')); + if (pr.subject) dl.put('Subject', pr.subject); + files.appendChild(dl.dl); + } + ins.appendChild(block('Files', files)); + const ul = h('ul', 'links'); + for (const u of d.used_by) { const m = data.map.byId.get(u.reference), li = h('li'); const b = button(null, m ? Tree.nameOf(m) : u.reference, m ? () => act.go('map', m.id) : null); b.appendChild(h('span', 'why', u.role)); if (!m) b.disabled = true; li.appendChild(b); ul.appendChild(li); } + ins.appendChild(block('Used by', d.used_by.length ? ul : h('p', 'note', 'Nothing uses it.'))); + }); + return true; + } + + // ---- Analysis: per-case values --------------------------------------------------- + const PAGE = 200; + function analysis(main, aside, state, act, data) { + const ui = state.ui, sel = state.sel.analysis || null; + paint(main, async () => { + const list = await benchmarkList(); + const b = list.find(x => x.benchmark === (ui.anaBench || state.benchmark)) || list.find(x => x.subjects_with_values > 0) || pickBenchmark(list, null); + if (!b) return { list }; + const view = await benchmarkView(b.benchmark, data.map); + const subject = view.byName.get(ui.anaSubject) || view.subjects.find(s => s.perCase) || view.subjects[0] || null; + const metric = view.metrics.find(m => m.id === ui.anaMetric) || view.metrics[0] || null; + const page = Math.max(0, ui.anaPage || 0); + const [values, kept] = await Promise.all([subject && metric ? P.read('values', { benchmark: view.name, subject: subject.name, metric: metric.id, offset: page * PAGE, limit: PAGE }) : null, + // The fields the subject's run kept (spec-d6, fields.js): read with the rest, so the tab is put in place in one step. + subject && window.LabFields ? window.LabFields.read(subject.recipe, ui).catch(error => ({ none: true, error })) : null]); + return { list, b, view, subject, metric, page, values, kept }; + }, (frag, { list, view, subject, metric, page, values, kept }, width) => { + if (!view) { frag.appendChild(h('p', 'empty', 'This project defines no benchmark.')); return; } + const bar = h('div', 'viewer-bar'); + const fieldsLeft = () => { state.ui.anaCase = null; state.ui.anaAxis = null; state.ui.anaIndex = null; }; // a case belongs to its subject + bar.appendChild(select('Benchmark', list.map(x => [x.benchmark, x.title + (x.subjects_with_values ? ' · ' + x.subjects_with_values + ' with per-case values' : '')]), view.name, v => { state.ui.anaSubject = null; state.ui.anaPage = 0; fieldsLeft(); act.setUI('anaBench', v); }, 'ana-bench')); + if (view.subjects.length) bar.appendChild(select('Subject', view.subjects.map(s => [s.name, s.full + (s.perCase ? ' · per-case values' : '')]), subject.name, v => { state.ui.anaPage = 0; fieldsLeft(); act.setUI('anaSubject', v); }, 'ana-subject')); + if (view.metrics.length) bar.appendChild(select('Metric', view.metrics.map(m => [m.id, m.title + units(m)]), metric.id, v => { state.ui.anaPage = 0; act.setUI('anaMetric', v); }, 'ana-metric')); + frag.appendChild(bar); + // Kept fields (fields.js), below Per case: absent when the subject's model has no product. + const fields = () => { const f = window.LabFields && window.LabFields.section(kept, act); if (f) frag.appendChild(f); }; + if (!subject || !metric) { frag.appendChild(h('p', 'empty', 'This benchmark has no ' + (subject ? 'metric' : 'subject') + '.')); if (subject) fields(); return; } + if (!values.total) { + frag.appendChild(sect('Per case', metric.title + units(metric) + ' · ' + subject.full)); + const p = h('p', 'empty ana-none'); p.appendChild(document.createTextNode('This machine holds no per-case values of ' + subject.full + ' for ' + metric.title + ' on ' + view.title + '; its summaries are on the ')); + const a = h('a', null, 'Benchmarks tab'); a.href = '#benchmarks'; a.addEventListener('click', ev => { ev.preventDefault(); act.setBenchmark(view.name); act.go('benchmarks'); }); p.appendChild(a); p.appendChild(document.createTextNode('.')); frag.appendChild(p); fields(); return; + } + const rows = values.rows.map(r => ({ ...r, id: r.case + (r.group ? ' · ' + r.group : ''), label: r.case + (r.group ? ' · ' + r.group.replace(/;/g, ', ') : '') })); + const pages = Math.ceil(values.total / PAGE), first = page * PAGE + 1, last = page * PAGE + rows.length; + const sec = h('section'); sec.appendChild(sect('Per case', metric.title + units(metric) + ' · ' + subject.full + ' · ' + (pages > 1 ? first + '–' + last + ' of ' : '') + plural(values.total, 'value'))); + const xs = rows.map(r => r.value).filter(v => v != null), sorted = xs.slice().sort((a, b) => a - b); + const dl = h('dl', 'metrics ana-sum'); + const worst = !xs.length ? null : metric.better === 'higher' ? sorted[0] : sorted[sorted.length - 1]; + dl.appendChild(cell(pages > 1 ? 'Values on this page' : 'Values', String(xs.length))); + dl.appendChild(cell('Median', fmt(sorted.length ? sorted[Math.floor(sorted.length / 2)] : null))); + dl.appendChild(cell(metric.better === 'none' ? 'Largest' : 'Worst', fmt(metric.better === 'none' ? (sorted.length ? sorted[sorted.length - 1] : null) : worst))); + dl.appendChild(cell('Without a value', String(rows.length - xs.length))); + sec.appendChild(dl); + const host = h('div'); sec.appendChild(host); + C.strip(host, rows, { width, height: 110, label: metric.title + ' per case of ' + subject.full, valueTitle: metric.title + units(metric), selected: sel, onSelect: id => act.select(id === sel ? null : id) }); + sec.appendChild(h('p', 'small faint bench-desc', 'One mark per case' + (rows.some(r => r.group) ? ' and group' : '') + ', in the order of the cases; ' + better(metric) + '.' + (rows.length > xs.length ? ' A short hollow mark has no value.' : ''))); + if (pages > 1) { + const pg = h('div', 'pager'); + const pb = button('btn sm', 'Previous', () => act.setUI('anaPage', page - 1)); pb.disabled = page === 0; pb.dataset.key = 'prev'; + const nb = button('btn sm', 'Next', () => act.setUI('anaPage', page + 1)); nb.disabled = page + 1 >= pages; nb.dataset.key = 'next'; + pg.appendChild(pb); pg.appendChild(nb); pg.appendChild(h('span', null, 'page ' + (page + 1) + ' of ' + pages)); sec.appendChild(pg); + } + const t = h('table', 'tv cases-t'); t.appendChild(headRow([['Case'], ['Group'], [metric.title + units(metric), 'n'], ['Why no value']])); const tb = h('tbody'); + for (const r of rows) { const tr = h('tr', 'rowbtn' + (r.id === sel ? ' is-sel' : '')); tr.addEventListener('click', () => act.select(r.id === sel ? null : r.id)); tr.appendChild(h('td', 'ck', r.case)); tr.appendChild(h('td', 'small faint', r.group || '')); tr.appendChild(h('td', 'n', r.value == null ? '—' : fmt(r.value))); tr.appendChild(h('td', 'small faint', r.reason || '')); tb.appendChild(tr); } + t.appendChild(tb); sec.appendChild(wrapTable(t)); frag.appendChild(sec); + fields(); + }); + return closeDrawer(aside); + } + + /** An environment: what it asks, what was found on this machine, and Check again, Connect and Disconnect. */ + function envDrawer(aside, key, act, data) { + const ref = key.slice(4); + drawer(aside, key, () => P.read('environment', { reference: ref }), (ins, e) => { + const hd = h('header'); hd.appendChild(h('span', 'kicker', 'environment · ' + e.reference)); const t = h('h2', null, e.title); t.tabIndex = -1; hd.appendChild(t); + const meta = h('div', 'ins-meta'); meta.appendChild(modelState(e.state)); if (e.origin) meta.appendChild(h('span', 'small faint', e.origin)); hd.appendChild(meta); ins.appendChild(hd); + const su = setup(); + su.put('Asks', ['Python ' + (e.python || '—'), e.device, plural(Object.keys(e.packages || {}).length, 'package')].filter(Boolean).join(', ')); + if (Object.keys(e.packages || {}).length) su.put('Packages', Object.entries(e.packages).map(([k, v]) => k + ' ' + v).join(', '), 'mono'); + su.put('Interpreter', e.interpreter || 'none recorded on this machine', e.interpreter ? 'mono' : null); + if (e.found) su.put('Found', 'Python ' + (e.found.python || '?') + ', ' + plural(Object.keys(e.found.packages || {}).length, 'package')); + su.put('Checked', e.checked_at ? time(e.checked_at) + (e.ok === false ? ', failed' : e.ok ? ', passed' : '') : 'never'); + if (e.recorded_at) su.put('Recorded', time(e.recorded_at)); + ins.appendChild(su.dl); + if (e.problems && e.problems.length) { const ul = h('ul', 'plain'); for (const p of e.problems) ul.appendChild(h('li', 'small warnline', typeof p === 'string' ? p : p.message || JSON.stringify(p))); ins.appendChild(block('What the check found', ul)); } + const connected = !!e.interpreter; + ins.appendChild(A.group('envcheck:' + ref, [Object.assign(A.checkItem(ref), { cls: 'btn', hidden: !connected })])); + if (A.may('owner')) { + const d = h('div'); d.appendChild(h('h3', null, connected ? 'Connect another interpreter' : 'Connect an interpreter')); + d.appendChild(h('p', 'note', 'An existing Python on this machine, by its absolute path; Tuldok records it for this project and checks it. Nothing is installed.')); + const f = h('div', 'form'); + f.appendChild(A.group('envconnect:' + ref, [{ id: 'connect', label: 'Connect', cls: 'btn', who: 'owner', op: 'connect_environment', busy: 'Connecting…', + input: { name: 'interpreter', label: 'Interpreter (absolute path)', placeholder: '/path/to/python', max: 4096, always: true }, + args: s => ({ reference: ref, interpreter: (s.text.interpreter || '').trim() }) }])); + d.appendChild(f); ins.appendChild(d); + if (connected) ins.appendChild(A.group('envdisconnect:' + ref, [{ id: 'disconnect', label: 'Disconnect', cls: 'btn ghost', who: 'owner', op: 'disconnect_environment', args: () => ({ reference: ref }), busy: 'Disconnecting…', + confirm: 'Forget the interpreter recorded for ' + e.title + ' on this machine; nothing is deleted.' }])); + } + }); + } + + S.paint = paint; S.drawer = drawer; + Object.assign(S, { benchmarks, train, data: data_, analysis }); +})(); diff --git a/static/project-workspace/training.js b/static/project-workspace/training.js new file mode 100644 index 00000000..24296f88 --- /dev/null +++ b/static/project-workspace/training.js @@ -0,0 +1,476 @@ +'use strict'; +// What a run's training script logged (docs/plans/training-and-curation/spec-t1.md section 6): the block "Training" +// of a run's drawer on the Train tab. Curves of the chosen series, one small chart a series (never two scales on +// one axis), each run in its own colour (the run opened and up to three overlaid from "Compare with"), smoothing, +// a log scale, step or time across, the band of each bucket's least and greatest value when a long series is +// reduced, and the checkpoints as ticks; under them, each tensor's histograms over the steps as a heat strip +// (weights or gradients), read when the strip comes on screen; and the last samples of a name as a strip. +// +// Updating in place (spec-w4): tabs.js reads the block's data with the drawer (read()) and builds it with the +// drawer off the page (block()); the answers are kept here by run for as long as the page lives, so a drawer drawn +// again (the run ended) builds the block from what is held. While a run in view is active, every reading of the +// page (tick(), every two seconds while something builds) asks `training` once a run; when lines were added it asks +// for the new buckets only (training_series from where the page is), draws the canvases again, reads again the +// strips on screen when histograms were added, and appends the new samples. The block is never built again by a +// reading, and a reading that found nothing new touches no element. The person's own choices (a series, a run to +// compare, weights or gradients, all the tensors, a sample name) build the block's content again in the same +// element; smoothing, log scale and step or time only draw the canvases again. The settings that are not of one +// run are kept in the address (trWith, trSeries, trSmooth, trLog, trX). +window.LabTraining = (() => { + const P = window.TuldokProject, C = window.LabCharts, h = C.h, D = window.LabInside.drawing; + const fmt = v => v == null || !Number.isFinite(v) ? '—' : C.fmtSci(v); + const button = (cls, text, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (onClick) b.addEventListener('click', onClick); return b; }; + const COLOURS = ['--run-1', '--run-2', '--run-3', '--run-4']; // the runs in view, in the order they were added + const RUNS_MOST = 4, SERIES_MOST = 4, POINTS = 800, HIST_ROWS = 120, HIST_BINS = 32, STRIPS_FIRST = 8, SAMPLES_SHOWN = 12; + const CHART_HEIGHT = 150, STRIP_HEIGHT = 36, SAMPLE_SIZE = 72; + const ACTIVE = ['queued', 'running']; + /** Sets an element's text in place: its one text node's data when it has one, so no node is added or removed. */ + const setText = (el, text) => { if (el.childNodes.length === 1 && el.firstChild.nodeType === 3) { if (el.firstChild.data !== text) el.firstChild.data = text; } else if (el.textContent !== text) el.textContent = text; }; + const seconds = s => s == null ? '—' : s < 90 ? s.toFixed(s < 10 ? 1 : 0) + ' s' : s < 5400 ? (s / 60).toFixed(1) + ' min' : (s / 3600).toFixed(2) + ' h'; + + // ---- What is held, by run, for the page's life --------------------------------------------------------------- + const held = new Map(); // run id -> { over, series: Map(name -> {size, points, lines, buckets}), samples: Map(name -> {total, items}), arrays: Map(index -> answer), hist: Map(key -> {answer, at}) } + const runOf = id => { let r = held.get(id); if (!r) { r = { over: null, series: new Map(), samples: new Map(), arrays: new Map(), hist: new Map() }; held.set(id, r); } return r; }; + let view = null; // the block on screen: { el, id, ids, settings, act, labelOf, sampleName, kind, all, charts: Map(name -> canvas), redraw() } + + /** The settings the address keeps, from the tab's state. */ + function settingsOf(ui, id) { + const others = String(ui.trWith || '').split(',').filter(x => x && x !== id).slice(0, RUNS_MOST - 1); + const smooth = Math.min(0.99, Math.max(0, Number(ui.trSmooth) || 0)); + return { with: others, series: String(ui.trSeries || '').split(',').filter(Boolean).slice(0, SERIES_MOST), smooth, log: ui.trLog === '1', x: ui.trX === 'time' ? 'time' : 'step' }; + } + /** The series shown: the chosen ones the runs logged, else loss (or the first). */ + function shownSeries(ids, chosen) { + const names = []; + for (const id of ids) { const o = runOf(id).over; if (o) for (const s of o.series) if (!names.includes(s.name)) names.push(s.name); } + const picked = chosen.filter(n => names.includes(n)); + if (picked.length) return { names, picked }; + const first = names.includes('loss') ? 'loss' : names[0]; + return { names, picked: first ? [first] : [] }; + } + + // ---- Reading --------------------------------------------------------------------------------------------------- + async function overview(id) { const r = runOf(id); r.over = await P.call('training', { run: id }); return r.over; } + /** The buckets of the named series of a run that are new since what is held (all of them when the size changed). */ + async function seriesOf(id, names) { + const r = runOf(id), o = r.over; if (!o) return false; + const want = names.filter(n => o.series.some(s => s.name === n) && (!r.series.has(n) || r.series.get(n).lines !== o.lines)); + if (!want.length) return false; + const answer = await P.call('training_series', { run: id, most: POINTS, series: want.map(n => { const s = r.series.get(n); return s ? { name: n, size: s.size, since: Math.max(0, s.buckets.length - 1) } : { name: n }; }) }); + for (const a of answer.series) { + const s = r.series.get(a.name); + if (s && s.size === a.size && a.first <= s.buckets.length) s.buckets.splice(a.first, Infinity, ...a.buckets); + else r.series.set(a.name, { size: a.size, buckets: a.buckets }); + Object.assign(r.series.get(a.name), { size: a.size, points: a.points, lines: answer.lines }); + } + return true; + } + /** The last samples of one name; only those added since what is held. */ + async function samplesOf(id, name) { + const r = runOf(id), o = r.over; if (!o || !name) return false; + const listed = o.samples.find(s => s.name === name); if (!listed) return false; + const s = r.samples.get(name); + if (s && s.total === listed.count) return false; + const from = s ? s.total : Math.max(0, listed.count - SAMPLES_SHOWN); + const answer = await P.call('training_samples', { run: id, name, offset: Math.max(from, listed.count - SAMPLES_SHOWN), limit: SAMPLES_SHOWN }); + const items = (s ? s.items : []).concat(answer.samples).slice(-SAMPLES_SHOWN); + r.samples.set(name, { total: answer.total, items }); + return answer.samples; + } + const sampleName = (o, chosen) => o && o.samples.length ? (o.samples.some(s => s.name === chosen) ? chosen : o.samples[0].name) : null; + + /** + * What the block of run id needs, for the drawer's load: the run's and the overlaid runs' overviews (asked + * afresh), the new buckets of the series shown and the last samples. Resolves once complete. + */ + async function read(id, ui) { + const settings = settingsOf(ui || {}, id), ids = [id].concat(settings.with); + await Promise.all(ids.map(x => overview(x).catch(error => { if (x === id) throw error; runOf(x).over = null; }))); + const { picked } = shownSeries(ids, settings.series); + await Promise.all(ids.map(x => seriesOf(x, picked))); + const r = runOf(id); r.sampleName = sampleName(r.over, r.sampleName); + await samplesOf(id, r.sampleName); + return { id, settings }; + } + + // ---- The block ------------------------------------------------------------------------------------------------- + /** The block for what read() gave, built off the page; act: the tab's actions (keepUI); labelOf(run id) names a run. */ + function block(value, act, labelOf, runs) { + const el = h('div', 'trn'); el.dataset.run = value.id; + view = { el, id: value.id, settings: value.settings, act, labelOf, runs: runs || [], charts: new Map(), kind: (view && view.id === value.id && view.kind) || 'weights', all: view && view.id === value.id ? view.all : false }; + fill(view); + return el; + } + const idsOf = v => [v.id].concat(v.settings.with.filter(x => runOf(x).over)); + /** Builds the content of the block (in its element, on or off the page). */ + function fill(v) { + const el = v.el, o = runOf(v.id).over; + el.textContent = ''; + el.appendChild(h('h3', null, 'Training')); + if (!o || !o.present || (!o.lines && !o.histogram_lines && !o.sample_lines)) { + const p = h('p', 'note trn-none', o && o.run.active ? 'Nothing logged yet. A script logs its training into training/ of its output (tuldok_log, or by hand); it is drawn here as it arrives.' : 'This run logged no training. A script logs into training/ of its output, with tuldok_log or by hand.'); + el.appendChild(p); v.charts = new Map(); return; + } + const ids = idsOf(v); + // The line under the title: where the run is. + const at = h('p', 'small trn-at'); at.dataset.key = 'trn-at'; el.appendChild(at); v.at = at; setAt(v); + // Series and controls. + const { names, picked } = shownSeries(ids, v.settings.series); + v.picked = picked; + const chooser = h('div', 'trn-series'); chooser.setAttribute('role', 'group'); chooser.setAttribute('aria-label', 'Series shown'); + v.chooser = chooser; chooserButtons(v, names); + el.appendChild(chooser); + el.appendChild(controls(v)); + // The runs in view: a legend, and Compare with. + el.appendChild(legend(v, ids)); + // One chart a series. + v.charts = new Map(); + for (const n of picked) { + const fig = h('figure', 'trn-fig'); const cap = h('figcaption', 'small trn-cap'); cap.appendChild(h('b', null, n)); const sum = h('span', 'faint'); cap.appendChild(sum); fig.appendChild(cap); + const wrap = h('div', 'trn-plot'); const c = h('canvas', 'trn-chart'); c.dataset.series = n; c.setAttribute('role', 'img'); wrap.appendChild(c); + const tip = h('div', 'trn-tip'); tip.hidden = true; wrap.appendChild(tip); fig.appendChild(wrap); + const note = h('p', 'small faint trn-note'); fig.appendChild(note); + el.appendChild(fig); + const chart = { canvas: c, tip, sum, note, name: n }; + v.charts.set(n, chart); + D.paints(c, () => drawChart(v, chart)); + hover(v, chart); + } + if (!picked.length) el.appendChild(h('p', 'note', 'This run logged no scalar series.')); + // Histograms and samples. + v.histBox = h('div', 'trn-hists-slot'); v.sampleBox = h('div', 'trn-samples-slot'); + el.appendChild(v.histBox); el.appendChild(v.sampleBox); + histograms(v); + samples(v); + const problems = o.problems || []; + if (o.bad_lines || o.truncated) { + const p = h('p', 'small warnline trn-problems', (o.bad_lines ? o.bad_lines + ' line' + (o.bad_lines === 1 ? '' : 's') + ' could not be read' + (problems.length ? ' (first: ' + problems[0].file + ' line ' + problems[0].line + ', ' + problems[0].problem.replace(/\.$/, '') + ')' : '') + '. ' : '') + (o.truncated ? 'Only the first part of a long file is read.' : '')); + el.appendChild(p); + } + if (o.other_keys && o.other_keys.length) el.appendChild(h('p', 'small faint', 'Also logged, not drawn: ' + o.other_keys.slice(0, 8).join(', ') + (o.other_keys.length > 8 ? '…' : ''))); + } + /** The buttons of the series chooser, one a name; a name already there keeps its button. */ + function chooserButtons(v, names) { + const have = new Set([...v.chooser.children].map(b => b.dataset.series)); + for (const n of names) { + if (have.has(n)) continue; + const b = button('btn sm', n, () => toggleSeries(v, n)); b.setAttribute('aria-pressed', String(v.picked.includes(n))); b.dataset.key = 'trn-series-' + n; b.dataset.series = n; + if (!v.picked.includes(n) && v.picked.length >= SERIES_MOST) b.disabled = true; + v.chooser.appendChild(b); + } + } + function setAt(v) { + const o = runOf(v.id).over; if (!v.at || !o) return; + const text = (o.last_step == null ? 'No metrics yet' : 'Step ' + o.last_step.toLocaleString('en-GB') + ' · ' + seconds(o.last_time)) + ' · ' + o.lines.toLocaleString('en-GB') + ' lines' + (o.started_at ? ' · started ' + P.day(o.started_at) : '') + (o.run.active ? ' · read live' : ''); + setText(v.at, text); + } + function controls(v) { + const row = h('div', 'trn-controls'); + const sm = h('label', 'trn-smooth'); sm.appendChild(h('span', 'small', 'Smoothing')); + const range = h('input'); range.type = 'range'; range.min = '0'; range.max = '0.99'; range.step = '0.01'; range.value = String(v.settings.smooth); range.dataset.key = 'trn-smooth'; + const shown = h('span', 'small num', v.settings.smooth.toFixed(2)); sm.appendChild(range); sm.appendChild(shown); + range.addEventListener('input', () => { v.settings.smooth = Number(range.value); shown.textContent = v.settings.smooth.toFixed(2); v.act.keepUI('trSmooth', v.settings.smooth ? String(v.settings.smooth) : null); drawAll(v); }); + row.appendChild(sm); + const log = button('btn sm', 'Log scale', () => { v.settings.log = !v.settings.log; log.setAttribute('aria-pressed', String(v.settings.log)); v.act.keepUI('trLog', v.settings.log ? '1' : null); drawAll(v); }); + log.setAttribute('aria-pressed', String(v.settings.log)); log.dataset.key = 'trn-log'; row.appendChild(log); + const x = h('select'); x.dataset.key = 'trn-x'; x.setAttribute('aria-label', 'Across'); for (const [val, text] of [['step', 'By step'], ['time', 'By time']]) { const op = h('option', null, text); op.value = val; x.appendChild(op); } x.value = v.settings.x; + x.addEventListener('change', () => { v.settings.x = x.value; v.act.keepUI('trX', x.value === 'time' ? 'time' : null); drawAll(v); }); + row.appendChild(x); + return row; + } + function legend(v, ids) { + const box = h('div', 'trn-runs'); + const ul = h('ul', 'trn-legend'); + ids.forEach((id, i) => { + const o = runOf(id).over, li = h('li'); li.dataset.run = id; + const sw = h('span', 'trn-swatch'); sw.style.background = 'var(' + COLOURS[i] + ')'; li.appendChild(sw); + li.appendChild(h('span', null, v.labelOf(id))); + li.appendChild(h('span', 'small faint', o ? o.run.state : '')); + if (i > 0) { const x = button('btn sm ghost trn-remove', '×', () => setWith(v, v.settings.with.filter(w => w !== id))); x.setAttribute('aria-label', 'Stop comparing with ' + v.labelOf(id)); x.dataset.key = 'trn-remove-' + id; li.appendChild(x); } + ul.appendChild(li); + }); + box.appendChild(ul); + if (ids.length < RUNS_MOST) { + const s = h('select'); s.dataset.key = 'trn-with'; s.setAttribute('aria-label', 'Compare with another run'); + const first = h('option', null, 'Compare with…'); first.value = ''; s.appendChild(first); + const me = v.runs.find(r => r.id === v.id); + const others = v.runs.filter(r => !ids.includes(r.id) && r.state !== 'queued'); + others.sort((a, b) => (me && b.reference === me.reference) - (me && a.reference === me.reference)); + for (const r of others.slice(0, 50)) { const op = h('option', null, v.labelOf(r.id) + ' · ' + r.state); op.value = r.id; s.appendChild(op); } + s.disabled = !others.length; + s.addEventListener('change', () => { if (s.value) setWith(v, v.settings.with.concat([s.value])); }); + box.appendChild(s); + } + return box; + } + /** The person chose other runs to compare with: read them, then build the content again in place. */ + async function setWith(v, others) { + v.settings.with = others.slice(0, RUNS_MOST - 1); + v.act.keepUI('trWith', v.settings.with.join(',') || null); + await rebuild(v); + } + function toggleSeries(v, n) { + const now = v.picked.includes(n) ? v.picked.filter(x => x !== n) : v.picked.concat([n]).slice(0, SERIES_MOST); + v.settings.series = now; + v.act.keepUI('trSeries', now.join(',') || null); + rebuild(v); + } + /** Reads what the content needs and builds it again in the block's element, the focus kept by data-key. */ + async function rebuild(v) { + v.el.setAttribute('aria-busy', 'true'); + try { + const ids = [v.id].concat(v.settings.with); + await Promise.all(ids.map(x => runOf(x).over ? null : overview(x).catch(() => null))); + const { picked } = shownSeries(ids, v.settings.series); + await Promise.all(ids.map(x => seriesOf(x, picked))); + await samplesOf(v.id, runOf(v.id).sampleName); + } catch (error) { console.warn(error); } + v.el.removeAttribute('aria-busy'); + if (view !== v || !v.el.isConnected) return; + const key = document.activeElement && v.el.contains(document.activeElement) ? document.activeElement.dataset.key : null; + fill(v); + if (key) { const again = v.el.querySelector('[data-key="' + CSS.escape(key) + '"]'); if (again) again.focus({ preventScroll: true }); } + } + const drawAll = v => { for (const chart of v.charts.values()) drawChart(v, chart); }; + + // ---- A chart --------------------------------------------------------------------------------------------------- + /** The lines of a chart: for each run in view, its points [x, y, low, high] (y the bucket's mean). */ + function linesOf(v, name) { + return idsOf(v).map((id, i) => { + const s = runOf(id).series.get(name), colour = D.token(COLOURS[i]); + const pts = s ? s.buckets.filter(b => b[3] != null).map(b => [v.settings.x === 'time' ? b[2] : b[1], b[3], b[4], b[5], b[1], b[2]]) : []; + return { id, colour, pts, size: s ? s.size : 1, points: s ? s.points : 0 }; + }); + } + /** Exponential moving average with the start's bias removed, as TensorBoard draws it. */ + function smooth(values, w) { + if (!w) return values; + let last = 0, n = 0; return values.map(y => { last = last * w + (1 - w) * y; n++; return last / (1 - Math.pow(w, n)); }); + } + function scaleOf(lines, log) { + let x0 = Infinity, x1 = -Infinity, y0 = Infinity, y1 = -Infinity; + for (const l of lines) for (const p of l.pts) { + x0 = Math.min(x0, p[0]); x1 = Math.max(x1, p[0]); + for (const y of [p[1], p[2], p[3]]) if (y != null && (!log || y > 0)) { y0 = Math.min(y0, y); y1 = Math.max(y1, y); } + } + if (!Number.isFinite(x0)) return null; + if (x1 === x0) { x0 -= 0.5; x1 += 0.5; } + if (!Number.isFinite(y0)) return null; + let t = log ? [Math.log10(y0), Math.log10(y1)] : [y0, y1]; + if (t[1] === t[0]) t = [t[0] - (Math.abs(t[0]) || 1) * 0.05, t[1] + (Math.abs(t[1]) || 1) * 0.05]; + const pad = (t[1] - t[0]) * 0.06; + return { x0, x1, y0: t[0] - pad, y1: t[1] + pad, log }; + } + const ticks = (a, b, n) => { const span = b - a, step0 = Math.pow(10, Math.floor(Math.log10(span / n))), err = n / (span / step0), step = step0 * (err <= 0.15 ? 10 : err <= 0.35 ? 5 : err <= 0.75 ? 2 : 1); const out = []; for (let t = Math.ceil(a / step) * step; t <= b + step * 1e-9; t += step) out.push(+t.toPrecision(12)); return out; }; + function drawChart(v, chart) { + const c = chart.canvas, size = D.fit(c, CHART_HEIGHT); if (!size) return; + const [W, H] = size, dpr = window.devicePixelRatio || 1, g = c.getContext('2d'); + g.setTransform(1, 0, 0, 1, 0, 0); g.clearRect(0, 0, W, H); g.scale(dpr, dpr); + const w = W / dpr, hh = H / dpr, L = 48, R = 8, T = 8, B = 20; + const lines = linesOf(v, chart.name), sc = scaleOf(lines, v.settings.log); + chart.lines = lines; chart.scale = sc; chart.box = { L, R, T, B, w, hh }; + const ink = 'rgb(' + D.token('--ink-3').join(',') + ')', grid = 'rgb(' + D.token('--grid').join(',') + ')'; + g.font = '11px ' + getComputedStyle(document.body).fontFamily; g.fillStyle = ink; g.strokeStyle = grid; g.lineWidth = 1; + const total = lines.reduce((n, l) => n + l.pts.length, 0); + c.dataset.points = String(total); c.dataset.drawn = String(Number(c.dataset.drawn || 0) + 1); + c.dataset.last = lines[0] && lines[0].pts.length ? String(lines[0].pts[lines[0].pts.length - 1][4]) : ''; + c.dataset.lines = String(lines.filter(l => l.pts.length).length); + const nonpos = v.settings.log && lines.some(l => l.pts.some(p => p[1] <= 0)); + const reduced = lines.find(l => l.size > 1); + setText(chart.note, [reduced ? 'Each point is the mean of ' + reduced.size + ' logged values; the band their least and greatest.' : '', nonpos ? 'Values of 0 or less are left out on the log scale.' : '', v.settings.smooth ? 'Smoothed (' + v.settings.smooth.toFixed(2) + '); the faint line is as logged.' : ''].filter(Boolean).join(' ')); + const main = runOf(v.id).series.get(chart.name), last = main && main.buckets.length ? main.buckets[main.buckets.length - 1][3] : null; + setText(chart.sum, main ? ' · ' + main.points.toLocaleString('en-GB') + ' points · last ' + fmt(last) : ' · not logged by this run'); + c.setAttribute('aria-label', chart.name + ' against the ' + v.settings.x + ', ' + lines.filter(l => l.pts.length).length + ' run' + (lines.length === 1 ? '' : 's') + (last != null ? '; the last value of this run ' + fmt(last) : '')); + if (!sc) { g.fillText('Nothing to draw yet.', L, T + 14); return; } + const X = x => L + (x - sc.x0) / (sc.x1 - sc.x0) * (w - L - R), Y = y => T + (1 - ((sc.log ? Math.log10(y) : y) - sc.y0) / (sc.y1 - sc.y0)) * (hh - T - B); + chart.X = X; chart.Y = Y; + // Grid and axes. + const yt = sc.log ? ticks(sc.y0, sc.y1, 3).filter(t => Number.isInteger(t)).map(t => Math.pow(10, t)) : ticks(sc.y0, sc.y1, 3); + g.textAlign = 'right'; g.textBaseline = 'middle'; + for (const t of yt) { const y = Y(t); if (y < T - 1 || y > hh - B + 1) continue; g.beginPath(); g.moveTo(L, Math.round(y) + 0.5); g.lineTo(w - R, Math.round(y) + 0.5); g.stroke(); g.fillText(C.fmtSci(t), L - 4, y); } + g.textAlign = 'center'; g.textBaseline = 'top'; + for (const t of ticks(sc.x0, sc.x1, 4)) { const x = X(t); if (x < L - 1 || x > w - R + 1) continue; g.fillText(v.settings.x === 'time' ? seconds(t) : t.toLocaleString('en-GB'), x, hh - B + 4); } + // Checkpoints of the run opened, as ticks on the step axis. + const o = runOf(v.id).over; + if (o && v.settings.x === 'step') { g.strokeStyle = ink; for (const k of o.checkpoints || []) { const x = X(k.step); if (x < L || x > w - R) continue; g.beginPath(); g.moveTo(x, hh - B); g.lineTo(x, hh - B - 5); g.stroke(); } } + // Bands, then raw lines (faint when smoothed), then smoothed lines. + g.save(); g.beginPath(); g.rect(L, T, w - L - R, hh - T - B); g.clip(); + const ok = y => y != null && (!sc.log || y > 0); + for (const l of lines) { + const rgb = l.colour.join(','); + if (l.size > 1) { + const band = l.pts.filter(p => ok(p[2]) && ok(p[3])); + if (band.length > 1) { g.fillStyle = 'rgba(' + rgb + ',0.16)'; g.beginPath(); band.forEach((p, i) => (i ? g.lineTo : g.moveTo).call(g, X(p[0]), Y(p[3]))); for (let i = band.length - 1; i >= 0; i--) g.lineTo(X(band[i][0]), Y(band[i][2])); g.closePath(); g.fill(); } + } + const path = (ys, alpha, width) => { g.strokeStyle = 'rgba(' + rgb + ',' + alpha + ')'; g.lineWidth = width; g.lineJoin = 'round'; g.beginPath(); let on = false; l.pts.forEach((p, i) => { const y = ys[i]; if (!ok(y)) { on = false; return; } if (on) g.lineTo(X(p[0]), Y(y)); else { g.moveTo(X(p[0]), Y(y)); on = true; } }); g.stroke(); if (l.pts.length === 1 && ok(ys[0])) { g.fillStyle = g.strokeStyle; g.beginPath(); g.arc(X(l.pts[0][0]), Y(ys[0]), 3, 0, 7); g.fill(); } }; + const raw = l.pts.map(p => p[1]); + if (v.settings.smooth) { path(raw, 0.25, 1); path(smoothLog(raw, v.settings.smooth, sc.log), 1, 2); } + else path(raw, 1, 2); + } + g.restore(); + } + /** Smoothing in the drawn space: on a log scale the logarithms are smoothed, so a falling loss is not pulled up. */ + function smoothLog(raw, w, log) { + if (!log) return smooth(raw, w); + const out = new Array(raw.length).fill(null), idx = [], logs = []; + raw.forEach((y, i) => { if (y > 0) { idx.push(i); logs.push(Math.log10(y)); } }); + smooth(logs, w).forEach((y, k) => { out[idx[k]] = Math.pow(10, y); }); + return out; + } + function hover(v, chart) { + const c = chart.canvas, tip = chart.tip; + c.addEventListener('mousemove', e => { + const sc = chart.scale; if (!sc || !chart.X) return; + const r = c.getBoundingClientRect(), mx = e.clientX - r.left, { L, R, w } = chart.box; + if (mx < L || mx > w - R) { tip.hidden = true; return; } + const x = sc.x0 + (mx - L) / (w - L - R) * (sc.x1 - sc.x0); + tip.textContent = ''; + const head = h('div', 'trn-tip-head', (v.settings.x === 'time' ? 'at ' + seconds(x) : 'step ' + Math.round(x).toLocaleString('en-GB'))); tip.appendChild(head); + chart.lines.forEach(l => { + if (!l.pts.length) return; + let best = l.pts[0]; for (const p of l.pts) if (Math.abs(p[0] - x) < Math.abs(best[0] - x)) best = p; + const row = h('div', 'trn-tip-row'); const sw = h('span', 'trn-swatch'); sw.style.background = 'rgb(' + l.colour.join(',') + ')'; row.appendChild(sw); + row.appendChild(h('span', null, v.labelOf(l.id) + ': ' + fmt(best[1]) + ' at step ' + best[4].toLocaleString('en-GB') + ' · ' + seconds(best[5]))); + tip.appendChild(row); + }); + tip.hidden = false; + const left = Math.min(Math.max(0, mx + 12), r.width - tip.offsetWidth - 2); tip.style.left = Math.max(0, left) + 'px'; tip.style.top = '4px'; + }); + c.addEventListener('mouseleave', () => { tip.hidden = true; }); + } + + // ---- Histograms ------------------------------------------------------------------------------------------------ + function histograms(v) { + v.histBox.textContent = ''; + const o = runOf(v.id).over; if (!o || !o.histograms.length) return; + const box = h('div', 'trn-hists'); box.appendChild(h('h4', null, 'Histograms over the steps')); + const kinds = [...new Set(o.histograms.map(x => x.kind))]; + if (!kinds.includes(v.kind)) v.kind = kinds[0]; + const head = h('div', 'trn-controls'); + if (kinds.length > 1) { + const sw = h('div', 'seg'); sw.setAttribute('role', 'group'); sw.setAttribute('aria-label', 'Which values'); + for (const k of kinds) { const b = button('btn sm', k === 'weights' ? 'Weights' : 'Gradients', () => { v.kind = k; rebuild(v); }); b.setAttribute('aria-pressed', String(k === v.kind)); b.dataset.key = 'trn-kind-' + k; sw.appendChild(b); } + head.appendChild(sw); + } + head.appendChild(h('span', 'small faint', 'Steps across, values up; each column shaded by its share of the values.')); + box.appendChild(head); + const list = o.histograms.filter(x => x.kind === v.kind), shown = v.all ? list : list.slice(0, STRIPS_FIRST); + for (const item of shown) { + const row = h('div', 'trn-strip-row'); + const label = h('div', 'small trn-strip-label'); label.appendChild(h('span', 'mono', item.name)); const range = h('span', 'faint'); label.appendChild(range); row.appendChild(label); + const c = h('canvas', 'trn-strip'); c.dataset.tensor = item.name; c.dataset.kind = item.kind; c.setAttribute('role', 'img'); c.setAttribute('aria-label', 'The ' + item.kind + ' of ' + item.name + ' over the steps'); row.appendChild(c); + box.appendChild(row); + const strip = { canvas: c, range, name: item.name, kind: item.kind }; + D.paints(c, () => drawStrip(v, strip)); + watch(v, strip); + } + if (list.length > shown.length) { const more = button('btn sm ghost', 'Show all ' + list.length, () => { v.all = true; rebuild(v); }); more.dataset.key = 'trn-all'; box.appendChild(more); } + v.histBox.appendChild(box); + } + const histKey = (name, kind) => name + '\u0000' + kind; + const seen = new IntersectionObserver(entries => { for (const e of entries) if (e.isIntersecting && e.target._strip) { seen.unobserve(e.target); readStrip(e.target._strip.v, e.target._strip.s); } }); + /** A strip is read when it comes on screen, and again then when histograms were added since it was read. */ + function watch(v, strip) { strip.canvas._strip = { v, s: strip }; const r = runOf(v.id), kept = r.hist.get(histKey(strip.name, strip.kind)); if (kept && r.over && kept.at === r.over.histogram_lines) return; seen.observe(strip.canvas); } + async function readStrip(v, strip) { + const r = runOf(v.id), key = histKey(strip.name, strip.kind), at = r.over ? r.over.histogram_lines : 0; + try { r.hist.set(key, { answer: await P.call('training_histograms', { run: v.id, name: strip.name, kind: strip.kind, most: HIST_ROWS, bins: HIST_BINS }), at }); } + catch (error) { strip.range.textContent = ' · ' + error.message; return; } + if (strip.canvas.isConnected) drawStrip(v, strip); + } + function drawStrip(v, strip) { + const kept = runOf(v.id).hist.get(histKey(strip.name, strip.kind)), c = strip.canvas, size = D.fit(c, STRIP_HEIGHT); + if (!size || !kept) return; + const a = kept.answer, cols = a.rows.length, rows = a.bins, values = new Float32Array(rows * cols); + for (let j = 0; j < cols; j++) { const total = a.totals[j] || 1, col = a.counts[j]; for (let b = 0; b < rows; b++) values[(rows - 1 - b) * cols + j] = Math.sqrt(col[b] / total); } + const lo = D.token('--ramp-lo'), hi = D.token('--ramp-hi'), colour = t => [lo[0] + (hi[0] - lo[0]) * t, lo[1] + (hi[1] - lo[1]) * t, lo[2] + (hi[2] - lo[2]) * t]; + const g = c.getContext('2d'); g.setTransform(1, 0, 0, 1, 0, 0); g.clearRect(0, 0, size[0], size[1]); + D.heat(g, values, rows, cols, colour, 0, 0, size[0], size[1]); + c.dataset.drawn = String(Number(c.dataset.drawn || 0) + 1); c.dataset.rows = String(cols); + const text = ' · ' + fmt(a.low) + ' to ' + fmt(a.high) + ' · steps ' + a.rows[0].step.toLocaleString('en-GB') + '–' + a.rows[cols - 1].step.toLocaleString('en-GB'); + setText(strip.range, text); + } + + // ---- Samples --------------------------------------------------------------------------------------------------- + function samples(v) { + v.sampleBox.textContent = ''; v.samplesStrip = null; + const r = runOf(v.id), o = r.over; if (!o || !o.samples.length) return; + const box = h('div', 'trn-samples-box'); const head = h('div', 'trn-controls'); head.appendChild(h('h4', null, 'Samples')); + if (o.samples.length > 1) { + const s = h('select'); s.dataset.key = 'trn-sample'; s.setAttribute('aria-label', 'Which samples'); + for (const x of o.samples) { const op = h('option', null, x.name + ' · ' + x.count); op.value = x.name; s.appendChild(op); } + s.value = r.sampleName; s.addEventListener('change', () => { r.sampleName = s.value; rebuild(v); }); head.appendChild(s); + } else head.appendChild(h('span', 'small faint', o.samples[0].name)); + box.appendChild(head); + const strip = h('div', 'trn-samples'); strip.setAttribute('role', 'list'); strip.tabIndex = 0; strip.setAttribute('aria-label', 'The last samples, newest last'); + const held_ = r.samples.get(r.sampleName); + for (const item of held_ ? held_.items : []) strip.appendChild(sampleFigure(v, item)); + v.samplesStrip = strip; + box.appendChild(strip); v.sampleBox.appendChild(box); + requestAnimationFrame(() => { strip.scrollLeft = strip.scrollWidth; }); + } + function sampleFigure(v, item) { + const fig = h('figure', 'trn-sample'); fig.setAttribute('role', 'listitem'); fig.dataset.step = String(item.step); fig.dataset.index = String(item.index); + if (item.problem) fig.appendChild(h('div', 'trn-thumb small faint', item.problem)); + else if (item.kind === 'image') { const img = h('img', 'trn-thumb'); img.alt = item.name + ' at step ' + item.step; img.src = item.url; img.width = SAMPLE_SIZE; img.height = SAMPLE_SIZE; img.loading = 'lazy'; fig.appendChild(img); } + else { const c = h('canvas', 'trn-thumb'); c.setAttribute('role', 'img'); c.setAttribute('aria-label', item.name + ' at step ' + item.step); fig.appendChild(c); D.paints(c, () => drawArray(v, item, c)); } + fig.appendChild(h('figcaption', 'small num', 'step ' + item.step.toLocaleString('en-GB'))); + return fig; + } + async function drawArray(v, item, c) { + const r = runOf(v.id); let a = r.arrays.get(item.index); + if (!a) { try { a = (await P.call('training_sample', { run: v.id, index: item.index })).content; r.arrays.set(item.index, a); } catch (error) { c.title = error.message; return; } } + const size = D.fit(c, SAMPLE_SIZE); if (!size) return; + const values = D.floats(a.values_f32), g = c.getContext('2d'); g.setTransform(1, 0, 0, 1, 0, 0); g.clearRect(0, 0, size[0], size[1]); + const [x, y, w, hh] = D.contain(a.rows, a.columns, size[0], size[1]); + D.heat(g, values, a.rows, a.columns, D.ramp(a.min == null ? 0 : a.min, a.max == null ? 1 : a.max), x, y, w, hh); + c.dataset.drawn = String(Number(c.dataset.drawn || 0) + 1); + } + + // ---- Live ------------------------------------------------------------------------------------------------------ + let ticking = false; + /** + * One step of the live block (tabs.js calls it on every reading): for each run in view whose last answer said it + * was active, `training` once; then only what grew is read and drawn, in place. + */ + async function tick() { + const v = view; if (!v || !v.el.isConnected || ticking) return; + const ids = [v.id].concat(v.settings.with).filter(id => { const o = runOf(id).over; return o && o.run.active; }); + if (!ids.length) return; + ticking = true; + try { + for (const id of ids) { + const r = runOf(id), before = r.over; + const o = await overview(id); + if (view !== v || !v.el.isConnected) return; + const grewLines = o.lines !== before.lines, grewHist = o.histogram_lines !== before.histogram_lines, grewSamples = o.sample_lines !== before.sample_lines; + const wasEmpty = !before.lines && !before.histogram_lines && !before.sample_lines; + if (id === v.id && wasEmpty && (grewLines || grewHist || grewSamples)) { await rebuild(v); continue; } // its first lines: the block's content is drawn + let samplesDrawn = false; + if (id === v.id) { // a name not there before: its control or its part is added, the rest left as it is + const names = shownSeries(idsOf(v), v.settings.series).names; + if (names.some(n => !before.series.some(b => b.name === n))) chooserButtons(v, names); + if (o.histograms.length !== before.histograms.length) histograms(v); + if (o.samples.length !== before.samples.length) { r.sampleName = sampleName(o, r.sampleName); await samplesOf(id, r.sampleName); if (view !== v) return; samples(v); samplesDrawn = true; } + } + if (grewLines) { await seriesOf(id, v.picked || []); if (view !== v) return; drawAll(v); } + if (id === v.id) { + setAt(v); + if (grewHist) for (const c of v.el.querySelectorAll('canvas.trn-strip')) { const s = c._strip && c._strip.s; if (s && r.hist.has(histKey(s.name, s.kind))) await readStrip(v, s); } + if (grewSamples && v.samplesStrip && !samplesDrawn) { + const added = await samplesOf(id, r.sampleName); + if (added && added.length && view === v) { + for (const item of added) v.samplesStrip.appendChild(sampleFigure(v, item)); + while (v.samplesStrip.children.length > SAMPLES_SHOWN) v.samplesStrip.firstChild.remove(); + v.samplesStrip.scrollLeft = v.samplesStrip.scrollWidth; + } + } + } + } + } catch (error) { /* the next reading asks again */ } + finally { ticking = false; } + } + /** Whether a run in the block on screen is read live (app.js keeps reading every two seconds while one is). */ + const live = () => !!(view && view.el.isConnected && [view.id].concat(view.settings.with).some(id => { const o = runOf(id).over; return o && o.run.active; })); + + return { read, block, tick, live, ACTIVE }; +})(); diff --git a/static/project-workspace/tree.js b/static/project-workspace/tree.js new file mode 100644 index 00000000..a15444e8 --- /dev/null +++ b/static/project-workspace/tree.js @@ -0,0 +1,284 @@ +'use strict'; +// Model lineage, from the mockup's tree.js: one graph built from the project's +// map, rendered as a horizontal lane map (with the track beneath it when +// asked) and as a vertical list (git-graph style) under it. A model's lane +// comes from what it was derived from; solid lines are derivations (always) +// and inputs (of the selected model); the faint line is training order only +// and makes no ancestry claim; dashed arcs are the authored comparisons of +// the selected model. +// Both renderers return select() and scrollTo() so a selection change never +// rebuilds the view. +window.LabTree = (() => { + const P = window.TuldokProject, C = window.LabCharts, el = C.el, h = C.h; + const reduced = () => window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + const SHORT = 16; // characters of a short label shown on the map (spec-w1 section 1) + const shortOf = m => m.draft ? 'new' : m.short ? m.short.slice(0, SHORT) : null; + const nameOf = m => (m.short ? m.short + ' · ' : '') + m.title; + const refName = m => m.short || (m.title.length > 24 ? m.title.slice(0, 23) + '…' : m.title); + // A short run of highlights from the previous selection to the new one, so + // the eye can follow the change. Never longer than ~450 ms; skipped when the + // viewer prefers reduced motion. + function walker() { + let timers = []; + return (ordered, fromId, toId, mark) => { + for (const t of timers) clearTimeout(t); timers = []; + if (reduced() || !fromId || !toId || fromId === toId) return; + const i0 = ordered.indexOf(fromId), i1 = ordered.indexOf(toId); if (i0 < 0 || i1 < 0) return; + let steps = i1 > i0 ? ordered.slice(i0 + 1, i1) : ordered.slice(i1 + 1, i0).reverse(); + if (steps.length > 22) { const out = []; for (let k = 0; k < 22; k++) out.push(steps[Math.round(k * (steps.length - 1) / 21)]); steps = out; } + const delay = Math.min(34, 420 / Math.max(1, steps.length)); + steps.forEach((id, k) => { timers.push(setTimeout(() => { mark(id, true); timers.push(setTimeout(() => mark(id, false), 170)); }, k * delay)); }); + }; + } + + /** + * The graph of a shaped map: nodes in section order with their index t and + * lane, families (the models derived from one model share a lane, lanes + * reused once free), the order line (consecutive models of the main lane + * whose training dates go forward), derivations, inputs and comparisons. + */ + function build(shaped) { + const nodes = shaped.models.map((m, t) => Object.assign({}, m, { t, lane: 0 })); + const byId = new Map(nodes.map(m => [m.id, m])); + const sections = []; let t = 0; + for (const s of shaped.sections) { sections.push({ id: s.id, label: s.title, up: s.up, loose: !!s.loose, start: t, end: t + s.models.length - 1, count: s.models.length }); for (const m of s.models) byId.get(m.id).section = sections.length - 1; t += s.models.length; } + const byParent = new Map(); + for (const m of nodes) if (m.parent && byId.has(m.parent)) { if (!byParent.has(m.parent)) byParent.set(m.parent, []); byParent.get(m.parent).push(m.id); } + const families = [...byParent].map(([parent, children]) => { + const ts = children.map(id => byId.get(id).t).concat([byId.get(parent).t]); + return { parent, children, start: Math.min(...ts), end: Math.max(...ts), lane: 0 }; + }).sort((a, b) => a.start - b.start || a.end - b.end); + const laneLast = [Infinity]; + for (const f of families) { + let lane = 1; while (laneLast[lane] !== undefined && laneLast[lane] >= f.start) lane++; + f.lane = lane; laneLast[lane] = f.end; + for (const id of f.children) byId.get(id).lane = lane; + } + // The order line joins consecutive models of the main lane whose training dates go forward, within a + // section or across neighbouring sections of the same parent group (as the mockup's checkpoint line did). + const sameParent = (a, b) => a === b || (Math.abs(a - b) === 1 && sections[a].up.length > 0 && sections[a].up.join('/') === sections[b].up.join('/')); + const edges = []; let prev = null; + for (const m of nodes) { + if (m.lane !== 0) continue; + if (prev && prev.trainedAt && m.trainedAt && prev.trainedAt <= m.trainedAt && sameParent(prev.section, m.section)) edges.push({ from: prev.id, to: m.id, kind: 'sequence' }); + prev = m; + } + for (const m of nodes) if (m.parent && byId.has(m.parent)) edges.push({ from: m.parent, to: m.id, kind: 'derivation' }); + for (const e of shaped.inputs) edges.push({ from: e.from, to: e.to, kind: 'input' }); + for (const r of shaped.comparisons) edges.push({ from: r.from, to: r.to, kind: 'comparison', explanation: r.explanation, excerpt: r.excerpt }); + const seqSpan = new Array(nodes.length).fill(false); + for (const e of edges) if (e.kind === 'sequence') for (let k = byId.get(e.from).t + 1; k < byId.get(e.to).t; k++) seqSpan[k] = true; + const seqIn = new Set(edges.filter(e => e.kind === 'sequence').map(e => e.to)); + const seqOut = new Set(edges.filter(e => e.kind === 'sequence').map(e => e.from)); + return { nodes, byId, edges, sections, families, laneCount: laneLast.length, seqIn, seqOut, seqSpan, track: shaped.track, scale: shaped.trackScale }; + } + /** The mark after a model's name: "unsaved" for an unsaved edit; editor.js writes "not kept yet" into it (data-ref). */ + function mark(m) { const s = h('span', 'umark', m.unsaved ? 'unsaved' : ''); s.dataset.ref = m.id; if (m.unsaved) s.dataset.unsaved = '1'; if (m.draft) s.dataset.draft = '1'; return s; } + function related(graph, id) { const out = new Set(); for (const e of graph.edges) if (e.kind !== 'sequence' && (e.from === id || e.to === id)) out.add(e.from === id ? e.to : e.from); return out; } + function comparisons(graph, id) { return graph.edges.filter(e => e.kind === 'comparison' && (e.from === id || e.to === id)).map(e => ({ other: e.from === id ? e.to : e.from, explanation: e.explanation, excerpt: e.excerpt })); } + function matches(m, q) { + if (!q) return true; q = q.toLowerCase(); + return [m.title, m.short, m.description, m.groupTitle, m.id].some(s => s && s.toLowerCase().includes(q)); + } + const visibleFor = (kinds, query) => m => (!kinds || kinds.has(m.kind)) && matches(m, query); + const trackText = (graph, m) => m.value == null ? 'no value' : C.fmtSci(m.value) + P.units(graph.track); + + // ---- Vertical list ------------------------------------------------------- + function renderList(container, graph, { selected = null, onSelect = () => {}, kinds = null, query = '' } = {}) { + container.textContent = ''; + const laneW = 16, pad = 10, gw = pad * 2 + laneW * Math.max(1, graph.laneCount - 1) + 8, rowH = 30; + const shortest = Math.max(3, ...graph.nodes.map(m => (shortOf(m) || '').length)); + container.style.setProperty('--graphw', gw + 'px'); + container.style.setProperty('--ckw', Math.ceil(shortest * 7 + 4) + 'px'); + const ul = h('ul', 'tree-list'); ul.setAttribute('role', 'listbox'); ul.setAttribute('aria-label', 'Models by section, in training order'); + const active = t => graph.families.filter(f => f.start < t && t <= f.end); + const laneX = l => pad + 4 + l * laneW; + const visible = visibleFor(kinds, query); + const rows = new Map(); let shown = 0; + for (const s of graph.sections) { + const list = graph.nodes.slice(s.start, s.end + 1).filter(visible); + if (!list.length) continue; + const li = h('li'); const head = h('div', 'tsect' + (s.loose ? ' loose' : '')); + const hs = el('svg', { width: gw, height: 20, 'aria-hidden': 'true' }); + for (const f of active(s.start)) hs.appendChild(el('line', { x1: laneX(f.lane), y1: 0, x2: laneX(f.lane), y2: 20, class: 'lanek lk-' + graph.byId.get(f.children[0]).kind })); + if (graph.seqIn.has(graph.nodes[s.start].id) || graph.seqSpan[s.start]) hs.appendChild(el('line', { x1: laneX(0), y1: 0, x2: laneX(0), y2: 20, class: 'lane0' })); + head.appendChild(hs); + const title = h('h3'); if (s.up.length) title.appendChild(h('span', 'up', s.up.join(' / ') + ' / ')); title.appendChild(document.createTextNode(s.label)); title.title = s.up.concat([s.label]).join(' / '); head.appendChild(title); + const fam = graph.families.filter(f => graph.byId.get(f.parent).section === graph.sections.indexOf(s)); + head.appendChild(h('span', 'sum', list.length + (list.length === s.count ? '' : ' of ' + s.count) + (fam.length ? ' · ' + fam.map(f => f.children.length + ' from ' + refName(graph.byId.get(f.parent))).join(', ') : ''))); + li.appendChild(head); ul.appendChild(li); + for (const m of list) { + shown++; + const row = h('li'); const b = h('button', 'trow' + (m.planned ? ' planned' : '') + (m.draft ? ' draft' : '')); b.type = 'button'; b.setAttribute('role', 'option'); b.setAttribute('aria-selected', 'false'); b.dataset.id = m.id; + const svg = el('svg', { width: gw, height: rowH, 'aria-hidden': 'true' }); + const x0 = laneX(0), mid = rowH / 2; + if (graph.seqSpan[m.t]) svg.appendChild(el('line', { x1: x0, y1: 0, x2: x0, y2: rowH, class: 'lane0' })); + if (graph.seqIn.has(m.id)) svg.appendChild(el('line', { x1: x0, y1: 0, x2: x0, y2: mid, class: 'lane0' })); + if (graph.seqOut.has(m.id)) svg.appendChild(el('line', { x1: x0, y1: mid, x2: x0, y2: rowH, class: 'lane0' })); + for (const f of active(m.t)) { + const k = 'lanek lk-' + graph.byId.get(f.children[0]).kind, x = laneX(f.lane), isChild = f.children.includes(m.id); + svg.appendChild(el('line', { x1: x, y1: 0, x2: x, y2: isChild && m.t === f.end ? mid : rowH, class: k })); + } + for (const f of graph.families.filter(f => f.parent === m.id)) { const x = laneX(f.lane), k = 'lanek lk-' + graph.byId.get(f.children[0]).kind; svg.appendChild(el('path', { d: `M${laneX(m.lane)},${mid} C${laneX(m.lane)},${rowH} ${x},${mid} ${x},${rowH}`, class: k })); } + const dot = el('circle', { cx: laneX(m.lane), cy: mid, r: 5, class: 'dot ' + m.kind + (m.planned ? ' planned' : '') }); + svg.appendChild(dot); b.appendChild(svg); + b.appendChild(h('span', 'ck', shortOf(m) || '—')); + // The name, with the mark of an unsaved edit (or, from editor.js, of typing not kept yet). + const nm = h('span', 'nm'); nm.appendChild(document.createTextNode(m.title)); nm.appendChild(mark(m)); b.appendChild(nm); + b.appendChild(h('span', 'pu', m.description)); + b.appendChild(h('span', 'chip ' + m.kind + (m.draft ? ' draft' : ''), m.draft ? 'draft' : P.kindLabel(m.kind))); + const bar = h('span', 'mbar' + (m.value == null ? ' none' : '')); + if (m.value != null && graph.scale) { const fill = h('i'); fill.style.width = Math.round(graph.scale.good(m.value) * 100) + '%'; bar.appendChild(fill); } + bar.title = graph.track ? P.trackName(graph.track) + ': ' + trackText(graph, m) : 'The project names no track'; + b.appendChild(bar); + b.appendChild(h('span', 'ms' + (m.value == null ? ' none' : ''), graph.track ? (m.value == null ? '—' : C.fmtSci(m.value)) : '')); + b.title = nameOf(m); + b.addEventListener('click', () => onSelect(m.id)); + row.appendChild(b); ul.appendChild(row); rows.set(m.id, { b, dot }); + } + } + ul.addEventListener('keydown', e => { if (e.key !== 'ArrowDown' && e.key !== 'ArrowUp') return; const opts = [...ul.querySelectorAll('.trow')]; const i = opts.indexOf(document.activeElement); if (i < 0) return; const n = opts[i + (e.key === 'ArrowDown' ? 1 : -1)]; if (n) { n.focus(); e.preventDefault(); } }); + container.appendChild(ul); + if (!shown) container.appendChild(h('p', 'empty', 'No models match.')); + let current = null; const walk = walker(); const order = [...rows.keys()]; + const api = { + shown, + select(id) { const rel = id ? related(graph, id) : new Set(); for (const [mid, r] of rows) { r.b.setAttribute('aria-selected', String(mid === id)); r.b.classList.toggle('related', rel.has(mid)); r.dot.classList.toggle('sel', mid === id); } walk(order, current, id, (mid, on) => { const r = rows.get(mid); if (r) r.b.classList.toggle('walk', on); }); current = id; }, + scrollTo(id, smooth) { const r = rows.get(id); if (!r) return; const box = r.b.getBoundingClientRect(), frame = container.getBoundingClientRect(); const top = container.scrollTop + box.top - frame.top - container.clientHeight / 2 + box.height / 2; container.scrollTo({ top: Math.max(0, top), behavior: smooth && !reduced() ? 'smooth' : 'auto' }); } + }; + api.select(selected); return api; + } + + // ---- Horizontal lane map, with the track beneath when asked ---------------- + const labelWidth = text => text.length * 7 + 24; // 12px semibold section titles, with room + const nodeLabelWidth = text => text.length * 6.4; // 10.5px mono + function renderMap(container, graph, { selected = null, onSelect = () => {}, kinds = null, query = '', zoom = 1, track = false } = {}) { + container.textContent = ''; + const dx = 30, left = 70, top = 44, laneH = 52, plotH = 120; + // Each section is as wide as its models or its title, whichever is wider. + const sx = []; let cursor = left - dx / 2; + for (const s of graph.sections) { sx.push(cursor); cursor += Math.max(s.count * dx, labelWidth(s.label)); } + const xOf = new Map(graph.nodes.map(m => [m.id, sx[m.section] + dx / 2 + (m.t - graph.sections[m.section].start) * dx])); + const W = cursor + 60; + const lanesH = top + 20 + graph.laneCount * laneH; + const showTrack = track && graph.track && graph.scale; + const trackTop = lanesH + 40, H = showTrack ? trackTop + plotH + 34 : lanesH + 30; + const trackLabel = graph.track ? P.trackName(graph.track) + ', ' + P.better(graph.track) : ''; + const ty = v => trackTop + (1 - graph.scale.at(v)) * plotH; + const x = m => xOf.get(m.id), y = m => top + 20 + m.lane * laneH; + const inner = h('div', 'inner'); inner.style.width = W + 'px'; inner.style.height = H + 'px'; inner.style.transform = 'scale(' + zoom + ')'; + const svg = el('svg', { width: W, height: H, viewBox: `0 0 ${W} ${H}`, role: 'img', 'aria-label': 'Model lineage: sections left to right in training order, branches in lanes' + (showTrack ? ', with ' + trackLabel + ' beneath' : '') }); + const visible = visibleFor(kinds, query); + graph.sections.forEach((s, i) => { + const x1 = sx[i], width = (sx[i + 1] || cursor) - x1; + svg.appendChild(el('line', { x1, y1: 8, x2: x1, y2: H - 8, class: 'band' })); + const t = el('text', { x: x1 + 8, y: 22, class: 'bandlabel' + (s.loose ? ' loose' : '') }, s.label); t.appendChild(el('title', {}, s.up.concat([s.label]).join(' / '))); svg.appendChild(t); + if (showTrack) { + if (i === 0 || width >= labelWidth(trackLabel) * 0.85) svg.appendChild(el('text', { x: x1 + 8, y: trackTop - 14, class: 'ttitle' }, trackLabel)); + if (i === 0 || width >= 60) for (const v of graph.scale.ticks) svg.appendChild(el('text', { x: x1 + 8, y: ty(v) - 3, class: 'tlabel' }, graph.scale.label(v))); + } + }); + for (const e of graph.edges) { if (e.kind !== 'sequence') continue; const a = graph.byId.get(e.from), b = graph.byId.get(e.to); svg.appendChild(el('line', { x1: x(a), y1: y(a), x2: x(b), y2: y(b), class: 'seq' })); } + for (const f of graph.families) { + const p = graph.byId.get(f.parent), kids = f.children.map(id => graph.byId.get(id)), k = kids[0].kind, ly = y(kids[0]); + const x0 = x(p), forward = kids.some(c => x(c) > x0), xb = x0 + (forward ? 1 : -1) * dx * 1.6; + const xs = kids.map(x).concat([xb]), a = Math.min(...xs), b = Math.max(...xs); + svg.appendChild(el('path', { d: `M${x0},${y(p)} C${x0 + (xb - x0) * 0.4},${y(p)} ${xb - (xb - x0) * 0.4},${ly} ${xb},${ly} M${a},${ly} L${b},${ly}`, class: 'der lk-' + k, 'data-family': f.parent })); + svg.appendChild(el('text', { x: Math.max(xb, a) + 4, y: ly - 7, class: 'lanelabel' }, 'from ' + refName(p))); + } + const gInp = el('g', { class: 'inp-layer' }); svg.appendChild(gInp); + const gCmp = el('g', { class: 'cmp-layer' }); svg.appendChild(gCmp); + const trackPts = new Map(); + if (showTrack) { + for (const v of graph.scale.ticks) svg.appendChild(el('line', { x1: left - 16, y1: ty(v), x2: W - 20, y2: ty(v), class: 'tgrid' })); + let d = '', prev = null; + for (const m of graph.nodes) { if (m.lane !== 0 || m.value == null) continue; d += (prev && prev.section === m.section && m.t - prev.t <= 3 ? 'L' : 'M') + x(m) + ',' + ty(m.value) + ' '; prev = m; } + if (d) svg.appendChild(el('path', { d: d.trim(), class: 'tline' })); + for (const m of graph.nodes) { + if (m.value == null) continue; + const c = el('circle', { cx: x(m), cy: ty(m.value), r: 3.5, class: 'tpt ' + m.kind + (visible(m) ? '' : ' dim'), tabindex: 0, role: 'button', 'aria-label': refName(m) + ', ' + trackText(graph, m) }); + c.appendChild(el('title', {}, nameOf(m) + ' · ' + trackText(graph, m))); + c.addEventListener('click', () => onSelect(m.id)); c.addEventListener('keydown', ev => { if (ev.key === 'Enter' || ev.key === ' ') { ev.preventDefault(); onSelect(m.id); } }); + svg.appendChild(c); trackPts.set(m.id, c); + } + } + const host = container.parentElement; host.style.position = 'relative'; + host.querySelectorAll(':scope > .minimap, :scope > .tip').forEach(n => n.remove()); + const tip = C.tooltip(host); + const at = ev => { const r = host.getBoundingClientRect(); return [ev.clientX - r.left, ev.clientY - r.top]; }; + const showTip = (m, ev) => { + const rows = [{ b: nameOf(m) }]; + rows.push(graph.track ? { v: m.value == null ? '—' : C.fmtSci(m.value) + P.units(graph.track), l: P.trackName(graph.track) } : { l: 'The project names no track' }); + rows.push({ l: m.draft ? 'a draft, not kept yet' : m.trainedAt ? 'trained ' + P.day(m.trainedAt) : m.planned ? 'never built' : 'no training date' }); + rows.push({ v: P.kindLabel(m.kind), l: m.parent ? 'from ' + refName(graph.byId.get(m.parent)) : '' }); + const [px, py] = at(ev); tip.show(px, py, rows); + }; + const nodeEls = new Map(); + // A node's short label is shown when it does not crowd the last label shown in its lane. + const laneLabelEnd = new Map(), labelled = new Set(); + for (const m of graph.nodes) { + const text = shortOf(m); if (!text) continue; + const w = nodeLabelWidth(text), last = laneLabelEnd.get(m.lane); + if (last === undefined || x(m) - w / 2 >= last + 14) { labelled.add(m.id); laneLabelEnd.set(m.lane, x(m) + w / 2); } + } + for (const m of graph.nodes) { + const g = el('g', { class: 'node ' + m.kind + (m.planned ? ' planned' : '') + (m.draft ? ' draft' : '') + (visible(m) ? '' : ' dim'), tabindex: 0, role: 'button', 'aria-label': nameOf(m) + (m.draft ? ', a draft' : m.planned ? ', never built' : '') + (m.unsaved ? ', unsaved' : ''), 'data-id': m.id }); + g.appendChild(el('rect', { x: x(m) - 14, y: y(m) - 14, width: 28, height: 28, class: 'hitr' })); + g.appendChild(el('circle', { cx: x(m), cy: y(m), r: 5.5 })); + if (m.unsaved) { const u = el('circle', { cx: x(m) + 7, cy: y(m) - 7, r: 2.5, class: 'unsaved-dot' }); u.appendChild(el('title', {}, 'unsaved')); g.appendChild(u); } + const text = shortOf(m); + const lbl = text ? el('text', { x: x(m), y: y(m) + 18, class: 'nl' + (labelled.has(m.id) || m.draft ? '' : ' off') }, text) : null; + if (lbl) g.appendChild(lbl); + g.addEventListener('click', () => onSelect(m.id)); + g.addEventListener('keydown', ev => { if (ev.key === 'Enter' || ev.key === ' ') { ev.preventDefault(); onSelect(m.id); } }); + g.addEventListener('pointerenter', ev => showTip(m, ev)); g.addEventListener('pointermove', ev => showTip(m, ev)); g.addEventListener('pointerleave', () => tip.hide()); + g.addEventListener('focus', () => { const r = g.getBoundingClientRect(); showTip(m, { clientX: r.left + 14, clientY: r.top + 14 }); }); g.addEventListener('blur', () => tip.hide()); + svg.appendChild(g); nodeEls.set(m.id, { g, lbl }); + } + inner.appendChild(svg); container.appendChild(inner); + const mini = h('div', 'minimap'); mini.setAttribute('role', 'img'); mini.setAttribute('aria-label', 'Whole map; click to move'); + const ms = el('svg', { viewBox: `0 0 ${W} ${H}`, preserveAspectRatio: 'none' }); + for (const m of graph.nodes) ms.appendChild(el('circle', { cx: x(m), cy: y(m), r: 7, class: 'd' })); + const vp = el('rect', { class: 'vp', x: 0, y: 0, width: 10, height: 10 }); ms.appendChild(vp); mini.appendChild(ms); host.appendChild(mini); + const sync = () => { vp.setAttribute('x', container.scrollLeft / zoom); vp.setAttribute('y', container.scrollTop / zoom); vp.setAttribute('width', container.clientWidth / zoom); vp.setAttribute('height', container.clientHeight / zoom); }; + container.addEventListener('scroll', sync); sync(); + new ResizeObserver(sync).observe(container); + mini.addEventListener('click', ev => { const r = mini.getBoundingClientRect(); container.scrollTo({ left: ((ev.clientX - r.left) / r.width) * W * zoom - container.clientWidth / 2, top: ((ev.clientY - r.top) / r.height) * H * zoom - container.clientHeight / 2, behavior: reduced() ? 'auto' : 'smooth' }); }); + let current = null; const walk = walker(); const order = graph.nodes.map(m => m.id); + const api = { + select(id) { + const sel = graph.byId.get(id); + gCmp.textContent = ''; + walk(order, current, id, (mid, on) => { const n = nodeEls.get(mid); if (n) n.g.classList.toggle('walk', on); }); current = id; + for (const [mid, n] of nodeEls) { + const m = graph.byId.get(mid), on = mid === id; + n.g.classList.toggle('sel', on); n.g.querySelector('circle').setAttribute('r', on ? 7 : 5.5); + const fam = !!sel && (m.parent === id || mid === sel.parent || (!!sel.parent && m.parent === sel.parent)); + if (n.lbl) n.lbl.classList.toggle('off', !(on || fam || labelled.has(mid) || m.draft)); + n.g.classList.toggle('kin', fam && !on); + } + for (const p of svg.querySelectorAll('.der')) p.classList.toggle('hot', !!sel && (p.dataset.family === id || p.dataset.family === sel.parent)); + // Inputs are drawn for the selected model only, so that the map stays quiet (63 in the lab's project). + gInp.textContent = ''; + if (sel) for (const e of graph.edges) if (e.kind === 'input' && (e.from === id || e.to === id)) { + const a = graph.byId.get(e.from), b = graph.byId.get(e.to), lift = Math.min(40, 10 + Math.abs(x(b) - x(a)) / 20); + gInp.appendChild(el('path', { d: `M${x(a)},${y(a)} C${x(a)},${y(a) + lift} ${x(b)},${y(b) + lift} ${x(b)},${y(b)}`, class: 'inp' })); + } + for (const [mid, c] of trackPts) c.classList.toggle('sel', mid === id); + if (sel) for (const e of graph.edges) if (e.kind === 'comparison' && (e.from === id || e.to === id)) { + const a = graph.byId.get(e.from), b = graph.byId.get(e.to), other = e.from === id ? b : a; + const path = el('path', { d: `M${x(a)},${y(a)} C${x(a)},${Math.min(y(a), y(b)) - 34} ${x(b)},${Math.min(y(a), y(b)) - 34} ${x(b)},${y(b)}`, class: 'cmp', tabindex: 0, role: 'link', 'aria-label': 'Compared with ' + refName(other) }); + const said = e.excerpt || e.explanation || ''; + path.appendChild(el('title', {}, 'Compared with ' + nameOf(other) + ': ' + said)); + const show = ev => { const [px, py] = at(ev); tip.show(px, py, [{ b: 'Compared with ' + nameOf(other) }, { l: said.slice(0, 160) + (said.length > 160 ? '…' : '') }]); }; + path.addEventListener('pointerenter', show); path.addEventListener('pointermove', show); path.addEventListener('pointerleave', () => tip.hide()); + path.addEventListener('click', () => onSelect(other.id)); path.addEventListener('keydown', ev => { if (ev.key === 'Enter') onSelect(other.id); }); + gCmp.appendChild(path); + } + }, + scrollTo(id, smooth) { const m = graph.byId.get(id); if (!m) return; container.scrollTo({ left: x(m) * zoom - container.clientWidth / 2, top: Math.max(0, y(m) * zoom - container.clientHeight / 2), behavior: smooth && !reduced() ? 'smooth' : 'auto' }); } + }; + api.select(selected); return api; + } + return { build, renderList, renderMap, related, comparisons, matches, nameOf, refName, mark }; +})(); diff --git a/static/project-workspace/weights.js b/static/project-workspace/weights.js new file mode 100644 index 00000000..591a1465 --- /dev/null +++ b/static/project-workspace/weights.js @@ -0,0 +1,428 @@ +'use strict'; +// A model's weights (docs/plans/definition-first-projects/spec-d2.md section 4): the block "Weights" of a model's +// drawer, after "Runs and product", when the model has a current product with a weights file. Its head names the +// file (a select when there are several), its size and parameters; a file Tuldok does not read shows its reason +// and nothing else. "Compare with" offers the weights the run started from and the models the map draws a +// derivation line from; choosing one asks weights_compare once. The layers are a table (name, shape of the first +// tensor, parameters, a small distribution asked for when its row comes on screen, and when comparing a bar of +// how much it changed); a layer opened shows a tensor's statistics, its larger distribution and its values as +// images, one image for two axes (array_slice), tiles for three or more (array_tiles), coloured diverging about +// zero; when comparing, this, the other, or their difference (computed here, tile by tile). +// +// Updating in place (spec-w4): the block is built off the page and put in place whole. Read again for the model +// on screen (screens.js), it is built from the person's place in it (file, comparison, sort, layer and tensor +// opened, axes, page, mode) and resolves only once complete, so the drawer puts it in place in one step. The +// person's own choices draw the block again the same way (putInPlace, the focus kept by data-key); a slider moves +// only the tiles. A product does not change once its run finished: the distributions and tiles read are kept here +// for the page's life (bounded), so a block drawn again draws them at once and asks for none again. +window.LabWeights = (() => { + const P = window.TuldokProject, S = window.LabScreens, C = window.LabCharts, h = C.h, R = P.product; + const { bytes, putInPlace } = S.parts; + const D = window.LabInside.drawing; + const SMALL_BINS = 32, BIG_BINS = 64; // a row's distribution, the opened tensor's + const TILES = 16; // tiles on a page + const SLICE_MOST = 256; // rows and columns of a two-axis image + const KEEP = 64; // tiles and slices kept + const MARGIN = '40px 0px'; // how near the drawer's view a row's distribution is asked for + const fmt = v => v == null ? '—' : C.fmtSci(v); + const num = n => n == null ? '—' : n.toLocaleString('en-GB'); + const shapeText = s => s.length ? s.join(' × ') : 'one value'; + const plural = (n, one, many) => num(n) + ' ' + (n === 1 ? one : many || one + 's'); + const button = (cls, text, onClick) => { const b = h('button', cls, text); b.type = 'button'; if (onClick) b.addEventListener('click', onClick); return b; }; + const refusal = error => window.LabActions.refusal(error); + + // ---- What is kept ------------------------------------------------------------------------------------------------ + const places = new Map(); // model id -> the person's place in its block + const placeOf = id => { if (!places.has(id)) places.set(id, { file: null, compare: '', sort: 'file', layer: null, tensor: null, mode: 'this', views: new Map() }); return places.get(id); }; + const summaries = new Map(); // run \n path \n array \n bins -> an answer of array_summary + const sumKey = (run, path, array, bins) => [run, path, array, bins].join('\n'); + const answers = new Map(); // the tiles and slices read, the latest last + async function kept(key, ask) { + if (answers.has(key)) { const v = answers.get(key); answers.delete(key); answers.set(key, v); return v; } + const v = await ask(); answers.set(key, v); while (answers.size > KEEP) answers.delete(answers.keys().next().value); return v; + } + async function summary(run, path, array, bins) { + const key = sumKey(run, path, array, bins); + if (!summaries.has(key)) summaries.set(key, await R.summary(run, { path, array, bins })); + return summaries.get(key); + } + + // ---- Reading --------------------------------------------------------------------------------------------------------- + /** The weights files of a model's current products, each with its run and a key. */ + async function filesOf(reference) { + const products = (await P.read('products', { reference })).products.filter(p => p.run_id); + const lists = await Promise.all(products.map(p => R.weights(p.run_id).then(w => w.files.map(f => Object.assign({ run: p.run_id, subject: p.subject || '', key: p.run_id + '\n' + f.path }, f))))); + return lists.flat(); + } + /** What a file can be compared with: the weights it started from, and the readable weights of each model it derives from. */ + async function choicesFor(m, g, f) { + const out = []; + if (f.starts_from) out.push({ key: 'start', label: 'The weights it started from', detail: f.starts_from, run: f.run, path: f.starts_from }); + const parents = [m.parent].filter(p => p && g.byId.has(p)); + for (const ref of parents) { + const files = await filesOf(ref).catch(() => []), readable = files.filter(x => x.readable), o = g.byId.get(ref); + for (const x of readable) out.push({ key: 'model\n' + ref + '\n' + x.key, label: 'The model it derives from', detail: (o.short ? o.short + ' · ' : '') + o.title + (readable.length > 1 ? ' · ' + x.path : ''), run: x.run, path: x.path, model: ref }); + } + return out; + } + + /** + * Fills box (a block of the drawer of model m, on the page or not yet) with the Weights block, or hides it + * when the model has no weights file. Resolves once the block is complete. scroller: the drawer's own scroll + * (the root of the rows' observer); live(): whether the drawer still shows this model. + */ + async function block(box, m, g, scroller, live) { + const st = placeOf(m.id), ctx = { box, m, g, st, scroller, live, io: null, turn: 0 }; + const root = await build(ctx); + if (!live()) return; + if (st.ctx && st.ctx !== ctx && st.ctx.io) st.ctx.io.disconnect(); // the block it replaces asks for nothing more + st.ctx = ctx; + if (!root) { box.hidden = true; box.replaceChildren(); return; } + box.hidden = false; box.replaceChildren(root); watch(ctx, root); + } + /** Draws the block again for the person's own choice: built off the page, then put in place with the focus and the scroll kept. */ + async function again(ctx) { + const mine = ++ctx.turn; + ctx.box.setAttribute('aria-busy', 'true'); + let root; try { root = await build(ctx); } catch (error) { root = h('div'); root.appendChild(h('h3', null, 'Weights')); root.appendChild(refusal(error)); } + if (mine !== ctx.turn || !ctx.box.isConnected) return; + ctx.box.removeAttribute('aria-busy'); + if (ctx.io) ctx.io.disconnect(); + putInPlace(ctx.box, root || h('div'), ctx.scroller); + watch(ctx, root); + } + + /** The block's content, read in full; null when the model has no weights file. */ + async function build(ctx) { + const { m, g, st } = ctx; + const files = await filesOf(m.id); + if (!files.length) return null; + const f = files.find(x => x.key === st.file) || files.find(x => x.readable) || files[0]; st.file = f.key; + const root = h('div', 'wts'); + root.appendChild(h('h3', null, 'Weights')); + // The head: the file, its size and parameters. + const head = h('div', 'wts-head'); + if (files.length > 1) { + const s = h('select'); s.setAttribute('aria-label', 'Weights file'); s.dataset.key = 'wts-file'; + for (const x of files) { const o = h('option', null, x.path + (x.subject ? ' · ' + x.subject : '') + (x.readable ? '' : ' (not read)')); o.value = x.key; o.selected = x === f; s.appendChild(o); } + s.addEventListener('change', () => { Object.assign(st, { file: s.value, compare: '', layer: null, tensor: null, mode: 'this' }); again(ctx); }); + const l = h('label', 'iv-field wts-pick'); l.appendChild(document.createTextNode('File')); l.appendChild(s); head.appendChild(l); + } else head.appendChild(h('p', 'mono wts-path', f.path)); + head.appendChild(h('p', 'small muted wts-size', [bytes(f.bytes), f.readable ? plural(f.parameters, 'parameter') : null, f.readable ? plural(f.layers.length, 'layer') + ', ' + plural(f.tensors, 'tensor') : null].filter(Boolean).join(' · '))); + root.appendChild(head); + if (!f.readable) { root.appendChild(h('p', 'empty wts-reason', f.reason)); return root; } + if (f.layers.length && f.tensors > f.layers.reduce((n, l) => n + l.tensors.length, 0)) root.appendChild(h('p', 'note', 'The first ' + num(f.layers.reduce((n, l) => n + l.tensors.length, 0)) + ' tensors are listed.')); + // Compare with. + const choices = await choicesFor(m, g, f); + if (!choices.some(c => c.key === st.compare)) st.compare = ''; + const choice = choices.find(c => c.key === st.compare) || null; + let cmp = null, cmpError = null; + if (choice) try { cmp = await R.compare(f.run, f.path, choice.run, choice.path); } catch (error) { cmpError = error; } + if (choices.length) { + const s = h('select'); s.setAttribute('aria-label', 'Compare with'); s.dataset.key = 'wts-compare'; + const none = h('option', null, 'Nothing'); none.value = ''; s.appendChild(none); + for (const c of choices) { const o = h('option', null, c.label + ': ' + c.detail); o.value = c.key; o.selected = c === choice; s.appendChild(o); } + s.addEventListener('change', () => { st.compare = s.value; if (!s.value && st.sort === 'change') st.sort = 'file'; if (!s.value) st.mode = 'this'; again(ctx); }); + const l = h('label', 'iv-field wts-pick'); l.appendChild(document.createTextNode('Compare with')); l.appendChild(s); root.appendChild(l); + } + if (cmpError) root.appendChild(refusal(cmpError)); + const cmpd = cmp ? compared(cmp) : null; + if (cmp) { + const t = cmp.total; + root.appendChild(h('p', 'small muted wts-total', 'All compared: ' + plural(t.parameters, 'parameter') + ', relative change ' + (t.relative_change == null ? (t.other_norm === 0 ? 'from zero' : '—') : fmt(t.relative_change)) + ', cosine ' + fmt(t.cosine) + ', largest change ' + fmt(t.largest_change) + '.')); + if (cmp.only_there.length) root.appendChild(h('p', 'small muted wts-there', 'Only in the other: ' + cmp.only_there.join(', ') + '.')); + } + // Sorting. + if (st.sort === 'change' && !cmp) st.sort = 'file'; + const seg = h('div', 'seg wts-sort'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'Sort the layers by'); + for (const [key, label] of [['file', 'File order'], ['size', 'Size']].concat(cmp ? [['change', 'Change']] : [])) { + const b = button(null, label, () => { if (st.sort !== key) { st.sort = key; again(ctx); } }); b.setAttribute('aria-pressed', String(st.sort === key)); b.dataset.key = 'wts-sort-' + key; seg.appendChild(b); + } + const bar = h('div', 'wts-bar'); bar.appendChild(h('span', 'small muted', 'Sort by')); bar.appendChild(seg); root.appendChild(bar); + // The layers. + const rows = f.layers.map((layer, i) => ({ layer, i, change: cmpd ? layerChange(layer, cmpd) : null })); + const rel = r => r.change && r.change.rel != null ? r.change.rel : r.change && r.change.zero ? Infinity : -1; + if (st.sort === 'size') rows.sort((a, b) => b.layer.parameters - a.layer.parameters || a.i - b.i); + if (st.sort === 'change') rows.sort((a, b) => rel(b) - rel(a) || a.i - b.i); + const most = Math.max(0, ...rows.map(r => r.change && r.change.rel != null ? r.change.rel : 0)); + const table = h('table', 'tv wts-table' + (cmp ? ' comparing' : '')); + const cols = h('colgroup'); for (const c of ['name', 'n', 'dist'].concat(cmp ? ['chg'] : [])) cols.appendChild(h('col', 'wc-' + c)); table.appendChild(cols); + const th = h('thead'), hr = h('tr'); + for (const [label, cls] of [['Layer', ''], ['Parameters', 'n'], ['Values', '']].concat(cmp ? [['Change', '']] : [])) hr.appendChild(h('th', cls, label)); + th.appendChild(hr); table.appendChild(th); + const tb = h('tbody'); + for (const r of rows) { + const { layer } = r, open = st.layer === layer.name, first = layer.tensors[0]; + const tr = h('tr', 'wts-row'); tr.dataset.layer = layer.name; + const name = h('td', 'wts-name'); + const nb = button('linkish', null, () => { if (st.layer === layer.name) { st.layer = null; st.tensor = null; } else { st.layer = layer.name; st.tensor = layer.tensors[0].array; } again(ctx); }); + nb.dataset.key = 'wts-layer-' + layer.name; nb.setAttribute('aria-expanded', String(open)); + nb.appendChild(h('span', 'mono wts-lname', layer.name)); + nb.setAttribute('aria-label', 'Layer ' + layer.name + ', ' + plural(layer.parameters, 'parameter') + (open ? '; press to close' : '; press to open')); + name.appendChild(nb); + name.appendChild(h('span', 'wts-shape', shapeText(first.shape) + (layer.tensors.length > 1 ? ' · ' + plural(layer.tensors.length, 'tensor') : ''))); + if (r.change && r.change.marks.length) for (const mk of r.change.marks) { const s = h('span', 'wts-mark ' + mk.kind, mk.text); s.title = mk.title; name.appendChild(s); } + tr.appendChild(name); + tr.appendChild(h('td', 'n', num(layer.parameters))); + const dist = h('td', 'wts-dist'), cv = h('canvas', 'wts-spark'); cv.setAttribute('role', 'img'); cv.setAttribute('aria-label', 'Distribution of ' + first.array + ': not read yet'); + cv.dataset.run = f.run; cv.dataset.path = f.path; cv.dataset.array = first.array; + dist.appendChild(cv); tr.appendChild(dist); + if (cmp) tr.appendChild(changeCell(r.change, most)); + tb.appendChild(tr); + if (open) { + const or = h('tr', 'wts-open'), td = h('td'); td.colSpan = cmp ? 4 : 3; + td.appendChild(await tensorPanel(ctx, f, layer, cmpd, choice)); + or.appendChild(td); tb.appendChild(or); + } + } + table.appendChild(tb); + const wrap = h('div', 'wts-tablebox'); wrap.appendChild(table); root.appendChild(wrap); + return root; + } + + /** The comparison by array: rows of tensors compared, and the marks of those that are not. */ + function compared(cmp) { + const rows = new Map(cmp.tensors.map(t => [t.array, t])), marks = new Map(); + for (const a of cmp.only_here) marks.set(a, { kind: 'only', text: 'only here', title: a + ' is not in the other file' }); + for (const r of cmp.reshaped) marks.set(r.array, { kind: 'reshaped', text: 'reshaped', title: r.array + ' is ' + shapeText(r.shape) + ' here and ' + shapeText(r.other_shape) + ' in the other' }); + for (const u of cmp.unread || []) marks.set(u.array, { kind: 'unread', text: 'not read', title: u.array + ' is ' + u.dtype + ' here and ' + u.other_dtype + ' in the other; one of them is a type Tuldok does not read' }); + return { rows, marks }; + } + /** A layer's change over its tensors compared: the norm of their differences over the norm of the other's; null when that is zero (started at zero) or not finite. */ + function layerChange(layer, cmpd) { + const got = layer.tensors.map(t => cmpd.rows.get(t.array)).filter(Boolean); + const marks = layer.tensors.map(t => cmpd.marks.has(t.array) ? Object.assign({ array: t.array, name: t.name }, cmpd.marks.get(t.array)) : null).filter(Boolean) + .map(mk => Object.assign({}, mk, { text: (layer.tensors.length > 1 ? mk.name + ' ' : '') + mk.text })); + if (!got.length) return { rel: null, none: true, marks }; + if (got.some(r => r.difference_norm == null || r.other_norm == null)) return { rel: null, nonfinite: true, marks }; + const d = Math.sqrt(got.reduce((a, r) => a + r.difference_norm * r.difference_norm, 0)), o = Math.sqrt(got.reduce((a, r) => a + r.other_norm * r.other_norm, 0)); + return { rel: o ? d / o : null, zero: !o, marks, partial: got.length < layer.tensors.length }; + } + function changeCell(c, most) { + const td = h('td', 'wts-chg'); + if (c.none) { td.appendChild(h('span', 'small faint', 'not compared')); td.dataset.change = 'none'; return td; } + if (c.zero) { const s = h('span', 'wts-zero', 'from zero'); s.title = 'It started at zero, so its change has no relative size'; td.appendChild(s); td.dataset.change = 'zero'; return td; } + if (c.rel == null) { td.appendChild(h('span', 'small faint', 'not finite')); td.dataset.change = 'nonfinite'; return td; } + const bar = h('span', 'wts-cbar'); bar.setAttribute('aria-hidden', 'true'); const i = h('i'); i.style.width = (most > 0 ? Math.max(1, 100 * c.rel / most) : 0).toFixed(1) + '%'; bar.appendChild(i); + td.appendChild(bar); td.appendChild(h('span', 'wts-fig', fmt(c.rel))); td.dataset.change = String(c.rel); + td.title = 'Relative change ' + fmt(c.rel) + (c.partial ? ', over the tensors compared' : ''); + return td; + } + + // ---- A row's distribution, when it comes on screen --------------------------------------------------------------------- + function watch(ctx, root) { + if (ctx.io) ctx.io.disconnect(); ctx.io = null; + if (!root) return; + const lazy = []; + for (const cv of root.querySelectorAll('canvas.wts-spark')) { + const got = summaries.get(sumKey(cv.dataset.run, cv.dataset.path, cv.dataset.array, SMALL_BINS)); + if (got) spark(cv, got); else lazy.push(cv); + } + if (!lazy.length) return; + const io = ctx.io = new IntersectionObserver(entries => { + for (const e of entries) if (e.isIntersecting && e.target.isConnected) { + io.unobserve(e.target); + const cv = e.target; + summary(cv.dataset.run, cv.dataset.path, cv.dataset.array, SMALL_BINS).then(s => spark(cv, s), error => { cv.setAttribute('aria-label', 'Distribution of ' + cv.dataset.array + ': ' + (error.message || error)); cv.title = error.message || String(error); cv.classList.add('refused'); }); + } + }, { root: ctx.scroller, rootMargin: MARGIN }); + for (const cv of lazy) io.observe(cv); + } + function spark(cv, s) { + cv.dataset.drawn = '1'; + cv.setAttribute('aria-label', 'Distribution of ' + cv.dataset.array + ' in ' + s.histogram.counts.length + ' bins from ' + fmt(s.histogram.low) + ' to ' + fmt(s.histogram.high) + ', mean ' + fmt(s.mean)); + D.paints(cv, () => histogram(cv, s, 22, false)); + } + /** Bars of a histogram in a canvas of a CSS height; a mark at zero when zero is inside the range. */ + function histogram(cv, s, cssH, axis) { + const size = D.fit(cv, cssH); if (!size) return; const [W, H] = size, ctx = cv.getContext('2d'); ctx.clearRect(0, 0, W, H); + const counts = s.histogram.counts, most = Math.max(1, ...counts), bw = W / Math.max(1, counts.length), ink = D.token('--mark'), line = D.token('--line-strong'), dpr = window.devicePixelRatio || 1; + const { low, high } = s.histogram; + if (low != null && high != null && low < 0 && high > 0) { ctx.fillStyle = 'rgb(' + D.token('--ink-3').join(',') + ')'; ctx.fillRect(Math.round((0 - low) / (high - low) * W), 0, Math.max(1, Math.round(dpr)), H); } + ctx.fillStyle = 'rgb(' + ink.join(',') + ')'; + counts.forEach((n, i) => { if (!n) return; const bh = Math.max(Math.round(dpr), n / most * (H - 2 * dpr)); ctx.fillRect(i * bw + (bw > 3 ? 0.5 : 0), H - bh, Math.max(1, bw - (bw > 3 ? 1 : 0)), bh); }); + if (axis) { ctx.fillStyle = 'rgb(' + line.join(',') + ')'; ctx.fillRect(0, H - Math.max(1, Math.round(dpr)), W, Math.max(1, Math.round(dpr))); } + } + + // ---- A tensor opened --------------------------------------------------------------------------------------------------- + const viewOf = (st, f, t) => { const key = f.key + '\n' + t.array; if (!st.views.has(key)) st.views.set(key, { across: null, index: t.shape.map(n => Math.floor(n / 2)), first: 0 }); return st.views.get(key); }; + async function tensorPanel(ctx, f, layer, cmpd, choice) { + const { st } = ctx; + const t = layer.tensors.find(x => x.array === st.tensor) || layer.tensors[0]; st.tensor = t.array; + const panel = h('div', 'wts-tensor'); panel.setAttribute('aria-label', 'Tensor ' + t.array); + if (layer.tensors.length > 1) { + const seg = h('div', 'seg wts-tensors'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'Tensor of ' + layer.name); + for (const x of layer.tensors) { const b = button(null, x.name, () => { if (st.tensor !== x.array) { st.tensor = x.array; again(ctx); } }); b.setAttribute('aria-pressed', String(x === t)); b.dataset.key = 'wts-tensor-' + x.array; b.title = x.array + ' · ' + shapeText(x.shape); seg.appendChild(b); } + panel.appendChild(seg); + } + panel.appendChild(h('p', 'small muted wts-what', [t.array, t.dtype, shapeText(t.shape), plural(t.parameters, 'value')].join(' · '))); + // When comparing: how it changed, or why it is not compared; this, the other or their difference. + const row = cmpd ? cmpd.rows.get(t.array) : null, mark = cmpd ? cmpd.marks.get(t.array) : null; + if (!row) st.mode = 'this'; + if (cmpd) { + if (row) panel.appendChild(h('p', 'small wts-row-cmp', 'Against ' + choice.detail + ': relative change ' + (row.relative_change == null ? (row.other_norm === 0 ? 'from zero' : '—') : fmt(row.relative_change)) + ', cosine ' + fmt(row.cosine) + ', largest change ' + fmt(row.largest_change) + '.')); + else panel.appendChild(h('p', 'small warnline', mark ? mark.title + ': it is not compared.' : 'Not compared.')); + if (row) { + const seg = h('div', 'seg wts-mode'); seg.setAttribute('role', 'group'); seg.setAttribute('aria-label', 'Show'); + for (const [key, label] of [['this', 'This'], ['other', 'The other'], ['difference', 'Difference']]) { const b = button(null, label, () => { if (st.mode !== key) { st.mode = key; again(ctx); } }); b.setAttribute('aria-pressed', String(st.mode === key)); b.dataset.key = 'wts-mode-' + key; seg.appendChild(b); } + panel.appendChild(seg); + } + } + // Its statistics and the larger distribution (of this file's tensor). + try { + const s = await summary(f.run, f.path, t.array, BIG_BINS); + if (row) panel.appendChild(h('p', 'small faint', 'Statistics of this file\'s tensor')); + const dl = h('dl', 'setup iv-facts wts-stats'); + const put = (k, text, stat, value) => { dl.appendChild(h('dt', null, k)); const dd = h('dd', 'num', text); dd.dataset.stat = stat; if (value != null) dd.dataset.value = String(value); dl.appendChild(dd); }; + put('Range', fmt(s.min) + ' to ' + fmt(s.max), 'range', JSON.stringify([s.min, s.max])); + put('Mean', fmt(s.mean), 'mean', s.mean); put('Spread', fmt(s.std), 'std', s.std); + put('Zeros', num(s.zeros), 'zeros', s.zeros); + put('Values', num(s.count) + (s.stride > 1 ? ', every ' + D.ordinal(s.stride) + ' read' : ''), 'count', s.count); + if (s.nonfinite) put('Not finite', num(s.nonfinite), 'nonfinite', s.nonfinite); + panel.appendChild(dl); + if (s.histogram && s.histogram.counts.length) { + const hc = h('canvas', 'wts-hist'); hc.setAttribute('role', 'img'); + hc.setAttribute('aria-label', 'Distribution of ' + t.array + ' in ' + s.histogram.counts.length + ' bins from ' + fmt(s.histogram.low) + ' to ' + fmt(s.histogram.high)); + D.paints(hc, () => histogram(hc, s, 72, true)); panel.appendChild(hc); + const ends = h('div', 'iv-ends small faint'); ends.appendChild(h('span', null, fmt(s.histogram.low))); ends.appendChild(h('span', null, fmt(s.histogram.high))); panel.appendChild(ends); + } + } catch (error) { panel.appendChild(refusal(error)); } + // Its values as images. + const images = h('div', 'wts-images'); + await fillImages(ctx, images, images, f, t, row ? choice : null); + panel.appendChild(images); + return panel; + } + + /** The drawn axes for a tensor of three or more axes counted across one: the last two of the others. */ + const drawnFor = (nd, across) => { const others = []; for (let a = 0; a < nd; a++) if (a !== across) others.push(a); return others.slice(-2); }; + /** Reads what the images show for the mode: this, the other, or their difference (each tile by tile). */ + async function readImages(f, t, v, mode, other) { + const nd = t.shape.length; + const ask = (run, path) => { + if (nd >= 3) { const args = { path, array: t.array, across: v.across, index: v.index.slice(), first: v.first, most: TILES }; return kept(JSON.stringify(['tiles', run, args]), () => R.tiles(run, args, true)); } + const args = { path, array: t.array, most: SLICE_MOST }; return kept(JSON.stringify(['slice', run, args]), () => R.slice(run, args, true)); + }; + const here = await ask(f.run, f.path); + if (mode === 'this') return { a: here, values: D.floats(here.values_f32) }; + const there = await ask(other.run, other.path), tv = D.floats(there.values_f32); + if (mode === 'other') return { a: there, values: tv }; + const hv = D.floats(here.values_f32), d = new Float32Array(hv.length); + for (let i = 0; i < d.length; i++) d[i] = hv[i] - tv[i]; + let min = Infinity, max = -Infinity, nonfinite = 0; for (const x of d) { if (!Number.isFinite(x)) { nonfinite++; continue; } if (x < min) min = x; if (x > max) max = x; } + return { a: Object.assign({}, here, { min: Number.isFinite(min) ? min : null, max: Number.isFinite(max) ? max : null, nonfinite }), values: d }; + } + const MODE_WORD = { this: 'this file', other: 'the other file', difference: 'this minus the other' }; + /** + * Fills target with the controls and the images of tensor t (built in full before it returns); host is the + * element they are shown in (target itself, or the box they will be put in place in). + */ + async function fillImages(ctx, target, host, f, t, other) { + const { st } = ctx, nd = t.shape.length, v = viewOf(st, f, t), mode = other ? st.mode : 'this'; + if (nd === 0) return; + if (nd >= 3) { if (v.across == null || v.across >= nd) v.across = 0; if (v.first >= t.shape[v.across]) v.first = 0; } + const ctl = h('div', 'iv-ctl wts-ctl'); + let sliders = []; + if (nd >= 3) { + const s = h('select'); s.setAttribute('aria-label', 'Axis counted across the tiles'); s.dataset.key = 'wts-across'; + for (let a = 0; a < nd; a++) { const o = h('option', null, 'axis ' + a + ' (' + t.shape[a] + ')'); o.value = String(a); o.selected = a === v.across; s.appendChild(o); } + s.addEventListener('change', () => { v.across = Number(s.value); v.first = 0; redrawImages(ctx, host, f, t, other); }); + const l = h('label', 'iv-field'); l.appendChild(document.createTextNode('Tiles across')); l.appendChild(s); ctl.appendChild(l); + const drawn = drawnFor(nd, v.across); + for (let a = 0; a < nd; a++) if (a !== v.across && !drawn.includes(a)) { + const l2 = h('label', 'iv-field iv-slider'); l2.appendChild(h('span', null, 'Axis ' + a)); + const r = h('input'); r.type = 'range'; r.min = '0'; r.max = String(t.shape[a] - 1); r.value = String(v.index[a]); r.dataset.key = 'wts-axis-slider-' + a; r.dataset.axis = String(a); + r.setAttribute('aria-label', 'Position on axis ' + a + ', 0 to ' + (t.shape[a] - 1)); + const at = h('span', 'num small', v.index[a] + ' of ' + t.shape[a]); + l2.appendChild(r); l2.appendChild(at); ctl.appendChild(l2); sliders.push({ a, r, at }); + } + } + let got; + try { got = await readImages(f, t, v, mode, other); } catch (error) { target.replaceChildren(ctl, refusal(error)); return; } + const pics = h('div', 'wts-pics'), legend = h('div', 'wts-legend'); + const paint = g => { drawPics(pics, legend, g, t, v, mode); }; + paint(got); + if (nd >= 3) { + const n = t.shape[v.across], last = v.first + got.a.tiles; + const pg = h('div', 'pager wts-pager'); + const pb = button('btn sm', 'Previous', () => { v.first = Math.max(0, v.first - TILES); redrawImages(ctx, host, f, t, other); }); pb.disabled = v.first === 0; pb.dataset.key = 'wts-prev'; + const nb = button('btn sm', 'Next', () => { v.first += TILES; redrawImages(ctx, host, f, t, other); }); nb.disabled = last >= n; nb.dataset.key = 'wts-next'; + pg.appendChild(pb); pg.appendChild(nb); pg.appendChild(h('span', 'small muted', 'Tiles ' + (v.first + 1) + '–' + last + ' of ' + n + ' along axis ' + v.across)); + target.replaceChildren(ctl, pics, legend, pg); + // A slider moves the tiles: read again (the latest wins), drawn again in place; the controls stay. + let wanted = 0, timer = 0; + for (const { a, r, at } of sliders) r.addEventListener('input', () => { + v.index[a] = Number(r.value); at.textContent = v.index[a] + ' of ' + t.shape[a]; + clearTimeout(timer); const mine = ++wanted; host.setAttribute('aria-busy', 'true'); + timer = setTimeout(async () => { + try { const next = await readImages(f, t, v, mode, other); if (mine !== wanted || !pics.isConnected) return; paint(next); } + catch (error) { if (mine === wanted) legend.replaceChildren(refusal(error)); } + if (mine === wanted) host.removeAttribute('aria-busy'); + }, 90); + }); + } else target.replaceChildren(pics, legend); + host.dataset.mode = mode; + } + /** Draws the images again for a control (an axis, a page): built off the page, put in place with the focus kept. */ + async function redrawImages(ctx, box, f, t, other) { + const mine = box._turn = (box._turn || 0) + 1, fresh = h('div'); box.setAttribute('aria-busy', 'true'); + await fillImages(ctx, fresh, box, f, t, other); + if (mine !== box._turn || !box.isConnected) return; + box.removeAttribute('aria-busy'); + const frag = document.createDocumentFragment(); frag.append(...fresh.childNodes); + putInPlace(box, frag, ctx.scroller); + } + + /** The tiles (three or more axes) or the one image (one or two axes), on a range symmetric about zero, and the legend. */ + function drawPics(pics, legend, g, t, v, mode) { + const a = g.a, nd = t.shape.length, m = Math.max(Math.abs(a.min || 0), Math.abs(a.max || 0)) || 1, colour = D.ramp(-m, m); + const tiles = nd >= 3 ? a.tiles : 1, rows = a.rows, cols = a.columns, per = rows * cols; + const where = nd >= 3 ? ' along axes ' + a.axes.join(' and ') + (a.index.some((_, i) => i !== a.across && !a.axes.includes(i)) ? ' at ' + a.index.map((p, i) => i !== a.across && !a.axes.includes(i) ? 'axis ' + i + ' = ' + p : null).filter(Boolean).join(', ') : '') : ''; + const els = []; + for (let k = 0; k < tiles; k++) { + const fig = h('figure', 'wts-tile'), cv = h('canvas'); cv.setAttribute('role', 'img'); + const values = g.values.subarray(k * per, (k + 1) * per); + if (nd >= 3) cv.setAttribute('aria-label', 'Tile at axis ' + a.across + ' = ' + (a.first + k) + ' of ' + t.array + ' (' + MODE_WORD[mode] + '): ' + rows + ' by ' + cols + ' values' + where); + else cv.setAttribute('aria-label', (nd === 1 ? 'The ' + cols + ' values' : rows + ' by ' + cols + ' values') + ' of ' + t.array + ' (' + MODE_WORD[mode] + '), from ' + fmt(a.min) + ' to ' + fmt(a.max) + (a.stride > 1 ? ', every ' + D.ordinal(a.stride) + ' row and column' : '')); + D.paints(cv, () => { + const w = cv.clientWidth; if (!w) return; + const cssH = nd === 1 ? 28 : Math.max(nd >= 3 ? 24 : 32, Math.min(nd >= 3 ? w * 1.5 : 320, Math.round(w * rows / cols))); + const size = D.fit(cv, cssH); if (!size) return; const ctx = cv.getContext('2d'); ctx.clearRect(0, 0, size[0], size[1]); + const [x, y, ww, hh] = nd === 1 ? [0, 0, size[0], size[1]] : D.contain(rows, cols, size[0], size[1]); + D.heat(ctx, values, rows, cols, colour, x, y, ww, hh); + }); + fig.appendChild(cv); + if (nd >= 3) fig.appendChild(h('figcaption', 'num', String(a.first + k))); + els.push(fig); + } + pics.className = 'wts-pics' + (nd >= 3 ? ' tiles' : ' one'); + pics.dataset.mode = mode; pics.dataset.first = String(a.first || 0); pics.dataset.index = JSON.stringify(a.index || []); pics.dataset.across = String(nd >= 3 ? a.across : ''); + pics.replaceChildren(...els); + // The legend: the ramp from -m to +m. + const ramp = h('canvas', 'wts-ramp'); ramp.setAttribute('aria-hidden', 'true'); + D.paints(ramp, () => { const size = D.fit(ramp, 10); if (!size) return; const n = 64, vals = new Float32Array(n); for (let i = 0; i < n; i++) vals[i] = -m + 2 * m * (i + 0.5) / n; D.heat(ramp.getContext('2d'), vals, 1, n, colour, 0, 0, size[0], size[1]); }); + const ends = h('div', 'iv-ends small faint'); ends.appendChild(h('span', null, '−' + fmt(m))); ends.appendChild(h('span', null, '0')); ends.appendChild(h('span', null, '+' + fmt(m))); + const say = h('p', 'small muted wts-range', (mode === 'difference' ? 'This minus the other' : mode === 'other' ? 'The other file' : 'This file') + ': from ' + fmt(a.min) + ' to ' + fmt(a.max) + ', coloured on ±' + fmt(m) + ' about zero' + (a.stride > 1 ? ' · every ' + D.ordinal(a.stride) + ' row and column' : '') + (a.nonfinite ? ' · ' + plural(a.nonfinite, 'value') + ' not finite, left clear' : '') + '.'); + legend.replaceChildren(ramp, ends, say); + } + + /** + * Opens the tensor called array in the Weights block of model id (structure.js's link from a node of the graph): + * its layer opened with that tensor, drawn in place, then scrolled to and focused. Resolves to whether it was + * found among the layers of the file shown. + */ + async function reveal(id, array) { + const st = places.get(id), ctx = st && st.ctx; + if (!ctx || !ctx.box.isConnected) return false; + const dot = array.lastIndexOf('.'), layer = dot > 0 ? array.slice(0, dot) : array; + Object.assign(st, { layer, tensor: array }); + await again(ctx); + const button = [...ctx.box.querySelectorAll('[data-key]')].find(e => e.dataset.key === 'wts-layer-' + layer); + if (!button) return false; + button.closest('tr').scrollIntoView({ block: 'start', behavior: 'auto' }); + button.focus({ preventScroll: true }); + return true; + } + + return { block, reveal }; +})(); diff --git a/static/pumas-gateways.js b/static/pumas-gateways.js index f6bc193c..6214e738 100644 --- a/static/pumas-gateways.js +++ b/static/pumas-gateways.js @@ -31,7 +31,9 @@ $('pumas-gateway-scan').addEventListener('click',async()=>{ pumasGateways=result.gateways; $('pumas-gateway-choice').replaceChildren(new Option('Choose an advertised gateway',''),...pumasGateways.map(row=> new Option(row.server_url+' · '+row.advertisement.build_info.package_version+' · '+row.advertisement.instance.registry_library_id,row.server_url))); - pumasGatewayStatus(pumasGateways.length?'Choose a gateway and annotation workflow. Model support is checked when you request a proposal.':'No compatible HTTP advertisements found. Start Pumas with HTTP discovery, or enter a legacy gateway URL in the proposal form.'); + // Advertisements Pumas served but Tuldok refused keep their reasons. + const refused=(Array.isArray(result.rejected)?result.rejected:[]).map(row=>' Refused '+row.server_url+': '+row.reason).join(''); + pumasGatewayStatus((pumasGateways.length?'Choose a gateway and annotation workflow. Model support is checked when you request a proposal.':'No compatible HTTP advertisements found. Start Pumas with HTTP discovery, or enter a legacy gateway URL in the proposal form.')+refused); }catch(error){if(token.epoch===pumasGatewayEpoch&&!pumasGatewayPaused)pumasGatewayStatus(error.message);} finally{if(pumasGatewayOperation===token){pumasGatewayOperation=null;renderPumasGateway();}} }); diff --git a/static/retained-measurement-maxima.vl.json b/static/retained-measurement-maxima.vl.json deleted file mode 100644 index 42035e08..00000000 --- a/static/retained-measurement-maxima.vl.json +++ /dev/null @@ -1 +0,0 @@ -{"$schema":"https://vega.github.io/schema/vega-lite/v5.20.1.json","description":"Maximum retained scalar for each selected subject over all matching cases. Exact values, missing counts and keyboard inspection follow the chart.","width":"container","height":{"step":30},"data":{"name":"maxima"},"mark":{"type":"point","filled":true,"size":100,"color":"#245d8a"},"encoding":{"x":{"field":"value","type":"quantitative","title":"Maximum value (selected metric units)","scale":{"zero":false},"axis":{"format":".3~e"}},"y":{"field":"subject_id","type":"nominal","title":"Subject","sort":null}},"params":[{"name":"measurement_select","select":{"type":"point","fields":["observation_id"],"on":"click","clear":false,"toggle":false}}]} diff --git a/static/vendor/ubuntu-sans/LICENCE.txt b/static/vendor/ubuntu-sans/LICENCE.txt new file mode 100644 index 00000000..2d881d1d --- /dev/null +++ b/static/vendor/ubuntu-sans/LICENCE.txt @@ -0,0 +1,108 @@ +Ubuntu Sans and Ubuntu Sans Mono (the four .ttf files in this directory) +Copyright 2010-2022 Canonical Ltd + +These files are copied unmodified, with their original names, from the +Ubuntu package fonts-ubuntu 0.869+git20240321-0ubuntu1 (/usr/share/fonts/truetype/ubuntu/). They are +distributed under the Ubuntu Font Licence 1.0 below, and under no other licence. + +Ubuntu and Canonical are registered trademarks of Canonical Ltd. + +The licence accompanying these works does not grant any rights +under trademark law and all such rights are reserved. + +------------------------------- +UBUNTU FONT LICENCE Version 1.0 +------------------------------- + +PREAMBLE +This licence allows the licensed fonts to be used, studied, modified and +redistributed freely. The fonts, including any derivative works, can be +bundled, embedded, and redistributed provided the terms of this licence +are met. The fonts and derivatives, however, cannot be released under +any other licence. The requirement for fonts to remain under this +licence does not require any document created using the fonts or their +derivatives to be published under this licence, as long as the primary +purpose of the document is not to be a vehicle for the distribution of +the fonts. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this licence and clearly marked as such. This may +include source files, build scripts and documentation. + +"Original Version" refers to the collection of Font Software components +as received under this licence. + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to +a new environment. + +"Copyright Holder(s)" refers to all individuals and companies who have a +copyright ownership of the Font Software. + +"Substantially Changed" refers to Modified Versions which can be easily +identified as dissimilar to the Font Software by users of the Font +Software comparing the Original Version with the Modified Version. + +To "Propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification and with or without charging +a redistribution fee), making available to the public, and in some +countries other activities as well. + +PERMISSION & CONDITIONS +This licence does not grant any rights under trademark law and all such +rights are reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of the Font Software, to propagate the Font Software, subject to +the below conditions: + +1) Each copy of the Font Software must contain the above copyright +notice and this licence. These can be included either as stand-alone +text files, human-readable headers or in the appropriate machine- +readable metadata fields within text or binary files as long as those +fields can be easily viewed by the user. + +2) The font name complies with the following: +(a) The Original Version must retain its name, unmodified. +(b) Modified Versions which are Substantially Changed must be renamed to +avoid use of the name of the Original Version or similar names entirely. +(c) Modified Versions which are not Substantially Changed must be +renamed to both (i) retain the name of the Original Version and (ii) add +additional naming elements to distinguish the Modified Version from the +Original Version. The name of such Modified Versions must be the name of +the Original Version, with "derivative X" where X represents the name of +the new work, appended to that name. + +3) The name(s) of the Copyright Holder(s) and any contributor to the +Font Software shall not be used to promote, endorse or advertise any +Modified Version, except (i) as required by this licence, (ii) to +acknowledge the contribution(s) of the Copyright Holder(s) or (iii) with +their explicit written permission. + +4) The Font Software, modified or unmodified, in part or in whole, must +be distributed entirely under this licence, and must not be distributed +under any other licence. The requirement for fonts to remain under this +licence does not affect any document created using the Font Software, +except any version of the Font Software extracted from a document +created using the Font Software may only be distributed under this +licence. + +TERMINATION +This licence becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER +DEALINGS IN THE FONT SOFTWARE. diff --git a/static/vendor/ubuntu-sans/UbuntuSans-Italic[wdth,wght].ttf b/static/vendor/ubuntu-sans/UbuntuSans-Italic[wdth,wght].ttf new file mode 100644 index 00000000..5391d7d9 Binary files /dev/null and b/static/vendor/ubuntu-sans/UbuntuSans-Italic[wdth,wght].ttf differ diff --git a/static/vendor/ubuntu-sans/UbuntuSansMono-Italic[wght].ttf b/static/vendor/ubuntu-sans/UbuntuSansMono-Italic[wght].ttf new file mode 100644 index 00000000..f4ff323c Binary files /dev/null and b/static/vendor/ubuntu-sans/UbuntuSansMono-Italic[wght].ttf differ diff --git a/static/vendor/ubuntu-sans/UbuntuSansMono[wght].ttf b/static/vendor/ubuntu-sans/UbuntuSansMono[wght].ttf new file mode 100644 index 00000000..2c8895a6 Binary files /dev/null and b/static/vendor/ubuntu-sans/UbuntuSansMono[wght].ttf differ diff --git a/static/vendor/ubuntu-sans/UbuntuSans[wdth,wght].ttf b/static/vendor/ubuntu-sans/UbuntuSans[wdth,wght].ttf new file mode 100644 index 00000000..d1ebb552 Binary files /dev/null and b/static/vendor/ubuntu-sans/UbuntuSans[wdth,wght].ttf differ diff --git a/tests/acceptance_project_workspace.cjs b/tests/acceptance_project_workspace.cjs new file mode 100644 index 00000000..134ffa7e --- /dev/null +++ b/tests/acceptance_project_workspace.cjs @@ -0,0 +1,281 @@ +'use strict'; +// The acceptance run of docs/plans/definition-first-projects/spec-w3.md ("Done when"): the project workspace on a +// copy of examples/tiny-project with real interpreters and a GPU, driven in a headless browser. After Tuldok +// starts, everything is done on the page: the owner connects the two interpreters, plans and requests a build of +// everything and watches the training's progress, branches the trained model, changes a setting in the form, +// keeps, saves, plans and requests the branch and sees it built on the map without a reload, adds it to the +// benchmark in the text view, and approves from Needs you a build an agent asked for (noticed by the idle page). +// Only the agent's request and the read-backs that check each step are made beside the page. +// It trains two small networks (seconds each on a GPU) and WRITES INTO THE PROJECT FOLDER it is given: give it a +// copy of examples/tiny-project committed to a Git repository of its own, with its own id in tuldok.json, and +// set TULDOK_DATABASE_URL (a disposable cluster), TULDOK_PROJECT_HOME and TULDOK_SETTINGS (scratch places). +// node tests/acceptance_project_workspace.cjs [output directory] +const assert = require('node:assert/strict'), fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { spawn } = require('node:child_process'), { once } = require('node:events'); +const root = path.resolve(__dirname, '..'), tests = __dirname; +const { authenticatedFixture } = require('./auth_helpers.cjs'), { waitForDebugger } = require('./browser_startup.cjs'), { cdpSession } = require('./browser_cdp_session.cjs'), { qaDirectory } = require('./qa_artifacts.cjs'); +assert.ok(process.argv.length >= 5, 'usage: node tests/acceptance_project_workspace.cjs [output directory]'); +const [folder, CPU, GPU] = process.argv.slice(2, 5).map(p => path.resolve(p)); +const out = process.argv[5] ? path.resolve(process.argv[5]) : qaDirectory(root, 'project-workspace-acceptance'); +assert.ok(process.env.TULDOK_DATABASE_URL && process.env.TULDOK_PROJECT_HOME && process.env.TULDOK_SETTINGS, 'Set TULDOK_DATABASE_URL to a disposable cluster and TULDOK_PROJECT_HOME and TULDOK_SETTINGS to scratch places.'); +assert.ok(fs.existsSync(path.join(folder, 'tuldok.json')), 'The first argument is a project folder (it holds tuldok.json).'); +const venv = path.join(root, '.venv', 'bin'); +if (fs.existsSync(path.join(venv, 'python'))) process.env.PATH = venv + path.delimiter + process.env.PATH; // auth_helpers runs python3 +const python = fs.existsSync(path.join(venv, 'python')) ? path.join(venv, 'python') : 'python3'; +const WIDTH = 1440, HEIGHT = 900; +const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'tuldok-accept-')); +fs.mkdirSync(out, { recursive: true }); +const children = [], errors = [], failures = [], log = []; +let server, serverErrors = '', origin, auth, socket, send, evaluate; +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); +async function until(fn, label, tries = 300, every = 100) { for (let i = 0; i < tries; i++) { const value = await fn(); if (value) return value; await pause(every); } throw Error('Timed out: ' + label); } +function done(text) { log.push(text); console.log('DONE ' + text); } +function launch(command, args, options) { const child = spawn(command, args, options); children.push(child); return child; } +async function stop(child) { if (!child || child.exitCode !== null || child.signalCode !== null) return; const exited = once(child, 'exit'); child.kill('SIGTERM'); await Promise.race([exited, pause(40000)]); if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); } +function findBrowser() { + const candidates = [process.env.BROWSER, '/usr/bin/chromium', '/usr/bin/chromium-browser', '/usr/bin/google-chrome']; + const cache = path.join(os.homedir(), '.cache', 'ms-playwright'); + if (fs.existsSync(cache)) for (const name of fs.readdirSync(cache).filter(n => n.startsWith('chromium_headless_shell-')).sort().reverse()) candidates.push(path.join(cache, name, 'chrome-headless-shell-linux64', 'chrome-headless-shell')); + const found = candidates.find(candidate => candidate && fs.existsSync(candidate)); assert.ok(found, 'No headless browser'); return found; +} +async function contract(name) { + const capabilities = await (await auth.fetch(origin + '/api/agent/capabilities')).json(); + const operation = capabilities.operations.find(item => item.id === 'project.' + name), c = operation.contracts[operation.contracts.length - 1]; + return 'project.' + name + '/' + c.version + '/' + c.sha256; +} +async function api(name, body = {}, bearer = null) { + const headers = { 'Content-Type': 'application/json', 'Tuldok-Operation-Contract': await contract(name) }; + const response = bearer ? await globalThis.fetch(origin + '/api/project/' + name, { method: 'POST', headers: { ...headers, Authorization: 'Bearer ' + bearer }, body: JSON.stringify(body) }) + : await auth.fetch(origin + '/api/project/' + name, { method: 'POST', headers, body: JSON.stringify(body) }); + const value = await response.json(); assert.equal(response.status, 200, name + ': ' + JSON.stringify(value)); return value; +} +async function credential(label, scopes) { + const response = await auth.fetch(origin + '/api/access/credentials', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ label, scopes, ttl_seconds: 3600 }) }); + const value = await response.json(); assert.equal(response.status, 201, JSON.stringify(value)); return value.token; +} +let shotNumber = 0; +async function shot(name) { + await evaluate('new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))'); + const file = path.join(out, String(++shotNumber).padStart(2, '0') + '-' + name + '.jpg'); + fs.writeFileSync(file, Buffer.from((await send('Page.captureScreenshot', { format: 'jpeg', quality: 88, captureBeyondViewport: false })).data, 'base64')); + return file; +} +async function theme(name) { await evaluate(`(() => { document.documentElement.setAttribute('data-theme', '${name}'); document.getElementById('theme').lastChild.textContent = ' ${name === 'dark' ? 'Dark' : 'Light'}'; })()`); } +async function settled(label) { + await until(() => evaluate(`(() => { const reading = e => [...e.querySelectorAll('.note')].some(n => n.textContent === 'Reading…'); return !reading(document.getElementById('main')) && !reading(document.getElementById('aside')) && !document.getElementById('main').hasAttribute('aria-busy') && !document.querySelector('.act-group [aria-busy]'); })()`), label, 1200); + await pause(200); +} +async function open(hash) { + await send('Page.navigate', { url: origin + '/?theme=light#' + hash }); + await until(() => evaluate(`document.readyState === 'complete' && !!document.getElementById('needs-btn') && !document.getElementById('needs-btn').disabled`), 'the project read (' + hash + ')', 600); + await settled('opened ' + hash); +} +async function hash(value) { await evaluate(`location.hash = ${JSON.stringify('#' + value)}`); await settled('#' + value); } +async function mouse(x, y, click = false) { + await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }); + if (click) { await send('Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 }); await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 }); } +} +const byKey = key => `[data-key=${JSON.stringify(key)}]`; +const enc = encodeURIComponent; +async function centre(selector) { + return until(() => evaluate(`(() => { const e = [...document.querySelectorAll(${JSON.stringify(selector)})].find(e => e.offsetParent !== null); if (!e) return null; e.scrollIntoView({ block: 'nearest', inline: 'nearest' }); const r = e.getBoundingClientRect(); return { x: r.left + r.width / 2, y: r.top + r.height / 2 }; })()`), 'shown: ' + selector, 200); +} +async function press(selector) { const { x, y } = await centre(selector); await mouse(x, y, true); } +async function key(name, code = name, keyCode = 0) { await send('Input.dispatchKeyEvent', { type: 'keyDown', key: name, code, windowsVirtualKeyCode: keyCode }); await send('Input.dispatchKeyEvent', { type: 'keyUp', key: name, code, windowsVirtualKeyCode: keyCode }); } +async function replace(selector, text) { + await press(selector); + await send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'a', code: 'KeyA', windowsVirtualKeyCode: 65, modifiers: 2, commands: ['selectAll'] }); + await send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'a', code: 'KeyA', windowsVirtualKeyCode: 65, modifiers: 2 }); + await key('Delete', 'Delete', 46); + if (text) await send('Input.insertText', { text }); +} +const drawerText = () => evaluate(`document.getElementById('aside').innerText`); +const stripText = () => evaluate(`document.getElementById('strip').innerText`); +const drawerState = () => evaluate(`[...document.querySelectorAll('#aside header .state')].map(e => e.textContent.trim()).join(' | ')`); +const isBuilt = async () => { const t = await drawerState(); return /(^|\| )built($| \|)/.test(t) ? t : null; }; +/** The width of the strip's hairline as a number of percent, or null when it is not drawn. */ +const stripHair = () => evaluate(`(() => { const i = document.querySelector('#strip .hair i'); return i ? parseFloat(i.style.width) : null; })()`); +const drawerHair = () => evaluate(`(() => { const i = document.querySelector('#aside .hair i'); return i ? parseFloat(i.style.width) : null; })()`); +const requestState = async id => (await api('build_request', { request: id })).state; +const newestRequest = async () => (await api('build_requests', { limit: 1 })).requests[0]; + +(async () => { + let output = ''; + const catalog = path.join(temporary, 'catalog'); + server = launch(python, ['-u', 'app.py', '--port', '0', '--data', catalog, '--project-folder', folder], { cwd: root, env: process.env, stdio: ['ignore', 'pipe', 'pipe'] }); + server.stdout.on('data', chunk => { output += chunk; }); server.stderr.on('data', chunk => { serverErrors += chunk; }); + const port = await until(() => { if (server.exitCode !== null) throw Error('Tuldok exited: ' + serverErrors); return output.match(/127\.0\.0\.1:(\d+)/)?.[1]; }, 'Tuldok start-up', 1800); + origin = 'http://127.0.0.1:' + port; + auth = authenticatedFixture(); await auth.register(origin, catalog, root); + const browser = launch(findBrowser(), ['--headless', '--no-sandbox', '--disable-gpu', '--no-first-run', '--hide-scrollbars', '--remote-debugging-port=0', `--window-size=${WIDTH},${HEIGHT}`, '--user-data-dir=' + path.join(temporary, 'browser'), 'about:blank'], { stdio: 'ignore', env: { ...process.env, XDG_CONFIG_HOME: temporary, XDG_CACHE_HOME: temporary } }); + const debugPort = await waitForDebugger(browser, path.join(temporary, 'browser', 'DevToolsActivePort')); + const target = (await (await fetch('http://127.0.0.1:' + debugPort + '/json')).json()).find(item => item.type === 'page'); + socket = new WebSocket(target.webSocketDebuggerUrl); await once(socket, 'open'); + const session = cdpSession(socket, { timeoutMs: 120000, onEvent: event => { + if (event.method === 'Runtime.exceptionThrown') errors.push(event.params.exceptionDetails.exception?.description || event.params.exceptionDetails.text); + if (event.method === 'Runtime.consoleAPICalled' && event.params.type === 'error') errors.push(event.params.args.map(a => a.value ?? a.description).join(' ')); + if (event.method === 'Network.loadingFailed' && !event.params.canceled) failures.push({ error: event.params.errorText }); + if (event.method === 'Network.responseReceived' && event.params.response.status >= 400) failures.push({ url: event.params.response.url, status: event.params.response.status }); + } }); + send = session.send; + evaluate = async expression => { const result = await send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true }); if (result.exceptionDetails) throw Error(JSON.stringify(result.exceptionDetails)); return result.result.value; }; + for (const domain of ['Page', 'Runtime', 'Network', 'Log']) await send(domain + '.enable'); + await send('Emulation.setDeviceMetricsOverride', { width: WIDTH, height: HEIGHT, deviceScaleFactor: 1, mobile: false }); + await auth.browser(send, origin); + + // ---- 1. The page opens on a project with nothing built; the owner connects the two interpreters ---------- + await open('map'); + done('Opened: ' + (await stripText()).replace(/\s+/g, ' ').slice(0, 160)); + await shot('map-nothing-built'); + for (const [name, interpreter] of [['cpu', CPU], ['gpu', GPU]]) { + const ref = 'environments/' + name; + await hash('train/env:' + enc(ref)); + await replace(byKey('envconnect:' + ref + ':field:interpreter'), interpreter); + await press(byKey('envconnect:' + ref + ':connect')); + await until(async () => (await api('environment', { reference: ref })).state === 'ready', ref + ' ready', 900); + await settled(ref + ' connected'); + done('Connected ' + ref + ' from the page: ' + (await api('environment', { reference: ref })).found.python); + } + await shot('environments-connected'); + + // ---- 2. Plan and request a build of everything; watch the training on the page ---------------------------- + await hash('train'); + await press(byKey('train:every:plan')); await settled('plan of everything'); + await shot('plan-everything'); + await press(byKey('request:plan:*:request')); + const first = await until(async () => { const r = await newestRequest(); return r && r.every ? r : null; }, 'the request of everything', 100); + let seen = null; + await until(async () => { + const runs = (await api('runs')).runs, run = runs.find(r => r.state === 'running' && r.reference === 'recipes/small-net-a'); + if (run && !seen) { + await hash('train/run:' + run.id); + const width = await until(async () => { const w = await drawerHair(); return (w > 0 && w < 100) ? w : ((await api('run', { run: run.id })).state !== 'running' ? -1 : null); }, 'a hairline partly filled', 600); + if (width > 0) { seen = { width, strip: await stripHair(), text: (await stripText()).replace(/\s+/g, ' ') }; await shot('training-small-net-a-running'); } + } + return ['succeeded', 'failed', 'cancelled'].includes(await requestState(first.id)); + }, 'the build of everything to end', 3600, 250); + const firstEnd = await api('build_request', { request: first.id }); + assert.equal(firstEnd.state, 'succeeded', JSON.stringify(firstEnd)); + assert.ok(seen, 'the training was never seen running on the page'); + done('Built everything from the page (request ' + first.id.slice(0, 8) + ', ' + firstEnd.runs.length + ' runs). While small-net-a trained the run drawer hairline stood at ' + seen.width + '% and the strip at ' + seen.strip + '%: "' + seen.text.slice(0, 200) + '"'); + await hash('map/' + enc('recipes/small-net-a')); await until(isBuilt, 'small-net-a built in its drawer', 200); + await shot('map-small-net-a-built'); + await hash('benchmarks'); await shot('benchmarks-two-subjects'); + + // ---- 3. Branch from here: a draft, a changed setting, kept, saved --------------------------------------- + const parent = 'recipes/small-net-a', ref = 'recipes/small-net-3-000-updates'; + await hash('map/' + enc(parent)); + await press(byKey('modelact:' + parent + ':branch')); await settled('the branch form'); + await replace(byKey('branch:' + parent + ':field:title'), 'Small net, 3,000 updates'); + await press(byKey('branch:' + parent + ':change-training-policy')); + await replace(byKey('branch:' + parent + ':field:description'), 'Half the updates, to see how much accuracy that costs.'); + await shot('branch-form'); + await press(byKey('branch:' + parent + ':create')); + await centre(byKey('draft:' + ref + ':keep')); await settled('the draft'); + const updates = await evaluate(`document.querySelector(${JSON.stringify(byKey('editor:' + ref + ':setting-updates'))}).value`); + assert.equal(updates, '6000', 'the parent settings were copied into the draft'); + await evaluate(`document.getElementById('aside').scrollTop = 0`); + await shot('draft-on-map'); + await replace(byKey('editor:' + ref + ':setting-updates'), '3000'); await key('Tab', 'Tab', 9); + await shot('draft-updates-changed'); + assert.equal(fs.existsSync(path.join(folder, 'recipes', 'small-net-3-000-updates.json')), false); + await press(byKey('draft:' + ref + ':keep')); await settled('kept'); + assert.equal((await api('definition', { reference: ref })).unsaved, true); + done('Branched ' + parent + ' on the page, changed updates 6000 → 3000 in the form, kept: ' + ref + ' is an unsaved definition; strip: "' + (await stripText()).replace(/\s+/g, ' ').slice(0, 140) + '"'); + await shot('draft-kept-unsaved'); + await press(byKey('model:' + ref + ':save')); await settled('saved'); + const saved = JSON.parse(fs.readFileSync(path.join(folder, 'recipes', 'small-net-3-000-updates.json'), 'utf8')); + assert.equal(saved.derived_from, parent); assert.equal(saved.run.settings.updates, 3000); assert.equal(saved.run.settings.learning_rate, 0.001); assert.equal(saved.change, 'training-policy'); + assert.ok(!('settings_file' in saved.run) && !('labels' in saved) && !('card' in saved)); + done('Saved from the page: recipes/small-net-3-000-updates.json is in the folder with derived_from, change and settings.updates 3000'); + + // ---- 4. Plan, request, watch it train, see it built on the map --------------------------------------------- + await press(byKey('model:' + ref + ':plan')); await settled('the plan of the branch'); + await shot('plan-branch'); + await press(byKey('request:plan:' + ref + ':request')); + const second = await until(async () => { const r = await newestRequest(); return r && r.id !== first.id ? r : null; }, 'the request of the branch', 100); + seen = null; + await until(async () => { + const run = (await api('runs', { reference: ref })).runs.find(r => r.state === 'running'); + if (run && !seen) { + await hash('train/run:' + run.id); + const width = await until(async () => { const w = await drawerHair(); return (w > 0 && w < 100) ? w : ((await api('run', { run: run.id })).state !== 'running' ? -1 : null); }, 'a hairline partly filled', 600); + if (width > 0) { seen = { width, strip: await stripHair(), text: (await drawerText()).replace(/\s+/g, ' ') }; await shot('training-branch-running'); await theme('dark'); await shot('training-branch-running-dark'); await theme('light'); } + } + return ['succeeded', 'failed', 'cancelled'].includes(await requestState(second.id)); + }, 'the build of the branch to end', 3600, 250); + assert.equal(await requestState(second.id), 'succeeded'); + assert.ok(seen, 'the branch was never seen training on the page'); + const marker = await evaluate(`(window.__acceptance = window.__acceptance || Math.random())`); + await hash('map/' + enc(ref)); + const turned = Date.now(); await until(isBuilt, 'the branch built in its drawer', 300); + const turnedIn = Date.now() - turned; + await until(async () => /^Idle/.test((await stripText()).trim()), 'the strip idle after the branch', 100); + assert.equal(await evaluate('window.__acceptance'), marker, 'the page was not reloaded'); + const node = await evaluate(`(() => { const g = document.querySelector('#main svg g.node[data-id=${JSON.stringify(ref)}]'); return g ? g.getAttribute('class') : null; })()`); + const model = (await api('map')).models.find(m => m.reference === ref); + assert.equal(model.state, 'built'); assert.equal(model.derived_from, parent); assert.equal(model.unsaved, false); assert.ok(model.trained_at); + done('Requested from the plan view; the branch trained on the GPU (hairline at ' + seen.width + '% in the run drawer); request ' + second.id.slice(0, 8) + ' succeeded; ' + turnedIn + ' ms later, with no reload, the drawer said built; the map shows it built, dated by its run ' + model.trained_at + ', node class "' + node + '"'); + await evaluate(`document.getElementById('aside').scrollTop = 0`); + await shot('map-branch-built'); await theme('dark'); await shot('map-branch-built-dark'); await theme('light'); + + // ---- 5. Edit another definition in the text view: the benchmark gains the new model as a subject ------------ + await hash('overview'); await press(byKey('open-edit:benchmarks/held-out')); await settled('the benchmark editor'); + const text = await evaluate(`document.querySelector(${JSON.stringify(byKey('editor:benchmarks/held-out:text'))}).value`); + const subject = ' {"name": "small-net-b", "title": "Small net, 3,000 updates", "recipe": "' + ref + '", "argument": "model:{recipe}/model.pt"},\n'; + const changed = text.replace(' {"name": "predict-zero"', subject + ' {"name": "predict-zero"'); + assert.notEqual(changed, text); + await replace(byKey('editor:benchmarks/held-out:text'), changed); + await press(byKey('edit:benchmarks/held-out:keep')); await settled('the benchmark kept'); + await shot('benchmark-edited-unsaved'); + await press(byKey('save:save')); await settled('the benchmark saved'); + assert.equal(fs.readFileSync(path.join(folder, 'benchmarks', 'held-out.json'), 'utf8'), changed); + done('Edited benchmarks/held-out in the text view (a third subject), kept and saved from the strip: the file is the text typed, byte for byte'); + + // ---- 6. An agent asks for the benchmark; the owner approves it from Needs you ------------------------------ + const agent = await credential('Acceptance agent', ['inspect', 'execute']); + const asked = await api('request_build', { targets: ['benchmarks/held-out'] }, agent); + assert.equal(asked.state, 'awaiting-approval'); + // The page is open and idle; nothing tells it but its own fifteen-second look. + const before = await evaluate(`document.getElementById('needs-btn').textContent`), mark = await evaluate('window.__acceptance'); + const noticed = Date.now(); + await until(() => evaluate(`document.getElementById('needs-btn').textContent !== ${JSON.stringify(before)}`), 'Needs you to count the agent request without a reload', 400); + assert.equal(await evaluate('window.__acceptance'), mark, 'the page was not reloaded'); + done('The idle page noticed the agent request by itself after ' + Math.round((Date.now() - noticed) / 1000) + ' s: "' + before + '" became "' + await evaluate(`document.getElementById('needs-btn').textContent`) + '"'); + await press('#needs-btn'); await centre(byKey('need:request:' + asked.id + ':approve')); + await shot('needs-you-agent-request'); + await press(byKey('need:request:' + asked.id + ':approve')); + const t0 = Date.now(), seenStrip = []; + const sample = async () => { const t = (await stripText()).replace(/\s+/g, ' ').replace(/ Read at.*$/, '').slice(0, 110); if (!seenStrip.length || seenStrip[seenStrip.length - 1][1] !== t) seenStrip.push([Date.now() - t0, t]); }; + await until(async () => { await sample(); return ['succeeded', 'failed', 'cancelled'].includes(await requestState(asked.id)); }, 'the agent build to end', 3600, 250); + const ended = Date.now() - t0; + await until(async () => { await sample(); return /^Idle/.test(seenStrip[seenStrip.length - 1][1]); }, 'the strip idle after the agent build: ' + JSON.stringify(seenStrip), 60, 250); + done('The strip while the agent build ran (ms after Approve; the request ended at ' + ended + '): ' + seenStrip.map(([at, t]) => at + ' "' + t + '"').join(' → ')); + const agentEnd = await api('build_request', { request: asked.id }); + assert.equal(agentEnd.state, 'succeeded', JSON.stringify(agentEnd)); + await key('Escape', 'Escape', 27); + await hash('benchmarks'); await settled('benchmarks'); + const rows = (await api('results', { benchmark: 'held-out', metric: 'error' })).rows; + const errorOf = name => (rows.find(r => r.subject === name) || {}).median; + assert.ok(errorOf('small-net-b') != null, 'the new model has a result: ' + JSON.stringify(rows.map(r => r.subject))); + done('An agent asked for benchmarks/held-out (awaiting approval); approved from Needs you on the page; it ran ' + agentEnd.runs.length + ' run(s) and succeeded. Median error: small-net-a ' + errorOf('small-net-a') + ', small-net-b (3,000 updates) ' + errorOf('small-net-b') + ', predict-zero ' + errorOf('predict-zero')); + await shot('benchmarks-three-subjects'); await theme('dark'); await shot('benchmarks-three-subjects-dark'); await theme('light'); + await hash('train'); await shot('train-after'); + await hash('map'); await shot('map-after'); + + const summary = await api('summary'), activity = await api('activity', { limit: 100 }); + done('In the end: states ' + JSON.stringify(summary.states) + '; activity: ' + activity.items.map(i => i.operation || i.name).reverse().join(', ')); + assert.deepEqual(errors, [], 'script errors: ' + JSON.stringify(errors)); + assert.deepEqual(failures, [], 'failed requests: ' + JSON.stringify(failures)); + done('No script error and no failed request in the whole run'); + fs.writeFileSync(path.join(out, 'receipt.json'), JSON.stringify({ at: new Date().toISOString(), log }, null, 2) + '\n'); +})().catch(async error => { + console.error('FAILED', error.stack || error); console.error('script errors', JSON.stringify(errors).slice(0, 2000)); console.error('failed requests', JSON.stringify(failures).slice(0, 2000)); + console.error('server stderr tail', serverErrors.slice(-3000)); + try { await shot('failure'); } catch (e) { /* no page */ } + process.exitCode = 1; +}).finally(async () => { + try { socket && socket.close(); } catch (e) { /* closed */ } + for (const child of children.slice().reverse()) await stop(child); + fs.rmSync(temporary, { recursive: true, force: true }); +}); diff --git a/tests/browser_annotation_kinds_fixture.py b/tests/browser_annotation_kinds_fixture.py new file mode 100644 index 00000000..d7dbfdca --- /dev/null +++ b/tests/browser_annotation_kinds_fixture.py @@ -0,0 +1,81 @@ +"""The project of tests/browser_project_workspace_annotation_kinds.cjs: a Git repository with code/ and project/. + +code/ holds examples/tiny-project's make_scenes.py (run here: three images, two point clouds, an array of frames +and a series) and its stand-in annotator, annotators/standin.py. project/ imports the scenes (datasets/scenes) and +has a task of each type the stand-in makes, each with two annotators: the stand-in at two thresholds, in +environments/test-python (the interpreter running the check, connected by the check). Prints the task names, the +annotators and the project's id as JSON. + + python tests/browser_annotation_kinds_fixture.py +""" +import json +import pathlib +import shutil +import subprocess +import sys +import uuid + +EXAMPLE = pathlib.Path(__file__).resolve().parents[1] / 'examples' / 'tiny-project' / 'code' +GIT = ['git', '-c', 'user.name=Tuldok Test', '-c', 'user.email=test@example.invalid', '-c', 'commit.gpgsign=false', + '-c', 'init.defaultBranch=main'] +DEFINITION = 'tuldok_definition_v1' +# task -> (type, vocabulary, fields, thresholds of A and B) +TASKS = { + 'masks': ('mask', ['blob'], {'mask': 'instance'}, (160, 200)), + 'shapes': ('polygons', ['red', 'blue'], {}, (160, 200)), + 'depths': ('depth', None, {'units': 'm'}, (160, 200)), + 'corners': ('keypoints', ['red', 'blue'], {'keypoints': ['head', 'tail'], 'skeleton': [['head', 'tail']]}, + (160, 200)), + 'tops': ('points', ['top'], {}, (128, 140)), + 'boxes3d': ('box3d', ['top'], {}, (128, 140)), + 'stretches': ('timeseries', ['high'], {}, (160, 200)), + 'tracks': ('track', ['square'], {}, (160, 200)), +} + + +def write(path, document): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(document, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') + + +def main(repository): + repository = pathlib.Path(repository) + code, folder = repository / 'code', repository / 'project' + (code / 'annotators').mkdir(parents=True) + shutil.copyfile(EXAMPLE / 'make_scenes.py', code / 'make_scenes.py') + shutil.copyfile(EXAMPLE / 'annotators' / 'standin.py', code / 'annotators' / 'standin.py') + subprocess.run([sys.executable, 'make_scenes.py'], cwd=code, check=True) + project_id = uuid.uuid4().hex + write(folder / 'tuldok.json', {'format': 'tuldok_project_v1', 'id': project_id, 'name': 'Annotation kinds check', + 'code_root': '../code'}) + write(folder / 'environments' / 'test-python.json', { + 'format': DEFINITION, 'kind': 'environment', 'title': 'The interpreter running the check', 'python': '3.12', + 'device': 'cpu', 'packages': {'jsonschema': '4.26.0'}}) + write(folder / 'datasets' / 'scenes.json', { + 'format': DEFINITION, 'kind': 'dataset', 'title': 'Scenes', + 'run': {'step': 'import', 'settings': {'source': 'scenes', 'rights': 'Made for this check.'}}}) + annotators = [] + for name, (kind, vocabulary, fields, thresholds) in TASKS.items(): + task = {'format': DEFINITION, 'kind': 'annotation', 'title': f'The {name}', 'dataset': 'datasets/scenes', + 'task': kind, **fields} + if vocabulary: + task['vocabulary'] = vocabulary + write(folder / 'annotations' / f'{name}.json', task) + for letter, threshold in zip('ab', thresholds): + reference = f'datasets/{name}-{letter}' + annotators.append(reference) + write(folder / f'{reference}.json', { + 'format': DEFINITION, 'kind': 'dataset', 'title': f'Stand-in {letter.upper()} on {name}', + 'run': {'step': 'annotate', 'inputs': {'records': 'datasets/scenes'}, + 'annotations': [f'annotations/{name}'], + 'command': ['python', 'annotators/standin.py', str(threshold)], + 'environment': 'environments/test-python', + 'settings': {'by': {'kind': 'script', 'name': f'stand-in {letter.upper()}', 'version': '1'}}}}) + for arguments in (['init', '-q'], ['add', '-A'], ['commit', '-q', '-m', 'The fixture.']): + subprocess.run([*GIT, '-C', str(repository), *arguments], check=True) + print(json.dumps({'project_id': project_id, 'tasks': {name: spec[0] for name, spec in TASKS.items()}, + 'annotators': annotators})) + + +if __name__ == '__main__': + main(sys.argv[1]) diff --git a/tests/browser_fake_pumas.py b/tests/browser_fake_pumas.py new file mode 100644 index 00000000..c577d4c2 --- /dev/null +++ b/tests/browser_fake_pumas.py @@ -0,0 +1,84 @@ +"""A fake Pumas endpoint for a browser check (tests/browser_project_workspace_endpoint.cjs), driven by lines on stdin. + +It starts tests/fake_pumas_typed.py on a loopback port of its own, serving the +listing tests/test_tuldok_project_pumas.py writes (the vision fixture's models, +writer, painter, decider and listener) and advertising itself as 0.8.0, and +prints its address as the first line on stdout. Each line read on stdin is a +JSON object and is answered with one JSON line: + + {"answer": "..."} the text a suggestion by chat or image_to_text answers + {"answers": {...}} the answers a decision gives + {"modes": [...]} modes taken one per POST (fake_pumas_typed's) + {"posts": true} answers {"posts": [{capability, model, profile, input kind}, ...]} + +A text dataset asked by text (generate) is answered "made: ", an +image a PNG of the asked size. Nothing else is contacted; it ends with stdin. + +With --embedders (tests/browser_project_workspace_curation.cjs) it also serves the embedders of +tests/test_tuldok_project_curation.py (embeddinggemma-2, the default embedder, and nomic-embed) and answers an +embeddings request with a vector made from each item. +""" +import io +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import fake_pumas_typed # noqa: E402 +from test_tuldok_project_pumas import listing # noqa: E402 + + +def main(): + server, base, state = fake_pumas_typed.start() + state['served_models'] = listing() + embedders = '--embedders' in sys.argv[1:] + if embedders: + from test_tuldok_project_curation import embed_respond, embedding_listing + state['served_models'] = embedding_listing() + state['advertise'] = '0.8.0' + said = {'answer': 'A short text.', 'answers': {}} + + def respond(body): + if embedders and body['output'] == 'embeddings_float32': + return embed_respond(body) + if body['capability'] == 'decision': + return {'kind': 'decisions', 'answers': said['answers'], 'input_tokens': 7} + if body['capability'] == 'image_generation': + from PIL import Image + out = io.BytesIO() + Image.new('RGB', (body['options']['width'], body['options']['height']), (35, 95, 160)).save(out, 'PNG') + import base64 + return {'kind': 'image', 'png_base64': base64.b64encode(out.getvalue()).decode(), + 'seed': body['options'].get('seed', 7)} + if body['input'].get('kind') == 'text': + return {'kind': 'text', 'text': 'made: ' + body['input']['text'], 'finish_reason': 'stop'} + return {'kind': 'text', 'text': said['answer'], 'finish_reason': 'stop'} + + state['respond'] = respond + print(base, flush=True) + try: + for line in sys.stdin: + if not line.strip(): + continue + command = json.loads(line) + if 'answer' in command: + said['answer'] = command['answer'] + if 'answers' in command: + said['answers'] = command['answers'] + if 'modes' in command: + state['modes'] = list(command['modes']) + if command.get('posts'): + posts = [{'capability': body.get('capability'), 'model': body.get('model'), 'profile': body.get('profile'), + 'input': (body.get('input') or {}).get('kind')} for body in state['requests']] + print(json.dumps({'posts': posts}), flush=True) + else: + print(json.dumps({'ok': True}), flush=True) + finally: + server.shutdown() + server.server_close() + + +if __name__ == '__main__': + main() diff --git a/tests/browser_model_portfolio.cjs b/tests/browser_model_portfolio.cjs deleted file mode 100644 index 851e650f..00000000 --- a/tests/browser_model_portfolio.cjs +++ /dev/null @@ -1,129 +0,0 @@ -'use strict'; -// Real Chromium interaction/layout and controlled read-lifecycle evidence. -// This isolated static fixture does not claim native project persistence. -const fs=require('node:fs'),path=require('node:path'),http=require('node:http'),crypto=require('node:crypto'); -const {spawn}=require('node:child_process'); -const {waitForDebugger}=require('./browser_startup.cjs'),{cdpSession}=require('./browser_cdp_session.cjs'); -const output=path.resolve(process.argv[2]||''),engine=process.env.TULDOK_BROWSER_EXECUTABLE; -if(!process.argv[2]||!engine||fs.existsSync(output))throw Error('Explicit fresh output directory and browser executable required'); -fs.mkdirSync(output,{recursive:true,mode:0o700}); -const fixture=structuredClone(require('./model_portfolio_fixture.cjs')); -const hash=value=>crypto.createHash('sha256').update(value).digest('hex'); -const first=fixture.document.models[0]; -const oldMethod=first.summary.method;first.summary.method+=' '+('A physical pressure field with explicit case and work scope. '.repeat(60)); -first.label='First model '; -fixture.cards[0].version.text=fixture.cards[0].version.text.replace(oldMethod,first.summary.method); -fixture.cards[0].version.sha256=first.card.sha256=hash(fixture.cards[0].version.text); -fixture.cards[0].version.bytes=Buffer.byteLength(fixture.cards[0].version.text); -for(let index=0;index<20;index++){ - const model=structuredClone(fixture.document.models[1]);model.id='extra-'+index;model.label='Extra model '+index;model.aliases=[]; - model.card.path='notes/extra-'+index+'.md';model.card.file_id=hash(fixture.project+'\0note\0'+model.card.path); - fixture.document.models.push(model);fixture.cards.push({...structuredClone(fixture.cards[1]),id:model.card.file_id,path:model.card.path}); -} -fixture.guide.version.text=JSON.stringify(fixture.document); -fixture.guide.version.sha256=fixture.inspection.expected_sha256=fixture.inspection.observed_sha256=hash(fixture.guide.version.text); -fixture.guide.version.bytes=fixture.inspection.expected_bytes=fixture.inspection.observed_bytes=Buffer.byteLength(fixture.guide.version.text); -const inline=JSON.stringify(fixture).replaceAll('<','\\u003c'); -const html=`Authored organization fixture
                          `; -const staticRoot=path.join(__dirname,'../static'),allowed=new Map([ - ['/project-files.css',['project-files.css','text/css']],['/experiment-lineage.css',['experiment-lineage.css','text/css']], - ['/experiment-graph.js',['experiment-graph.js','text/javascript']],['/project-model-portfolio.js',['project-model-portfolio.js','text/javascript']]]); -const server=http.createServer((request,response)=>{ - if(request.url==='/'){response.writeHead(200,{'Content-Type':'text/html; charset=utf-8'});response.end(html);return;} - const asset=allowed.get(request.url);if(!asset){response.writeHead(404);response.end();return;} - response.writeHead(200,{'Content-Type':asset[1]});response.end(fs.readFileSync(path.join(staticRoot,asset[0]))); -}); -let child,session; -const report={status:'starting',engine,verification_only:true,native_persistence_claim:false,graphics:'Headless Chromium with GPU disabled; no model or rendering speed claim.',checks:[],errors:[],browser_stopped:false}; -const sourceFiles=['app.py','static/project-files.css','static/project-files.html','static/project-files.js','static/project-model-portfolio.js', - 'docs/contracts/project-model-portfolio.md','tests/model_portfolio_fixture.cjs','tests/test_project_model_portfolio.cjs', - 'tests/browser_model_portfolio.cjs','tests/test_project_model_portfolio_http.py','tests/test_workspace_project_files.cjs','tests/test_workspace_project_files_m4.cjs']; -report.source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); -const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); -async function main(){ - await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); - const origin='http://127.0.0.1:'+server.address().port; - try{ - const flags=['--headless','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,1000','--user-data-dir='+path.join(output,'profile'),'about:blank']; - report.flags=flags;report.origin=origin; - const log=fs.openSync(path.join(output,'browser.log'),'w');child=spawn(engine,flags,{stdio:['ignore',log,log]});fs.closeSync(log); - const port=await waitForDebugger(child,path.join(output,'profile/DevToolsActivePort')); - const targets=await(await fetch('http://127.0.0.1:'+port+'/json/list')).json(),target=targets.find(target=>target.type==='page'); - const socket=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{socket.addEventListener('open',resolve,{once:true});socket.addEventListener('error',reject,{once:true});}); - session=cdpSession(socket,{onEvent:event=>{if(event.method==='Runtime.exceptionThrown')report.errors.push(event.params.exceptionDetails.text);}}); - const send=session.send; - await send('Page.enable');await send('Runtime.enable'); - async function evaluate(expression){const result=await send('Runtime.evaluate',{expression,returnByValue:true,awaitPromise:true});if(result.exceptionDetails)throw Error(result.exceptionDetails.exception?.description||result.exceptionDetails.text);return result.result.value;} - async function until(expression){const end=Date.now()+15000;while(Date.now(){const element=document.querySelector(${JSON.stringify(selector)});if(!element||element.disabled)throw Error('Unavailable fixture control');element.scrollIntoView({block:'center'});element.focus();})()`); - await send('Input.dispatchKeyEvent',{type:'keyDown',key:'Enter',code:'Enter',windowsVirtualKeyCode:13,text:'\r'});await send('Input.dispatchKeyEvent',{type:'keyUp',key:'Enter',code:'Enter',windowsVirtualKeyCode:13});} - async function fill(selector,value){await evaluate(`(()=>{const element=document.querySelector(${JSON.stringify(selector)});element.value=${JSON.stringify(value)};element.dispatchEvent(new Event('input',{bubbles:true}));element.dispatchEvent(new Event('change',{bubbles:true}));})()`);} - async function screenshot(name){const shot=await send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,name),Buffer.from(shot.data,'base64'));} - async function load(){await keyClick('#project-portfolio-notes');await until("document.querySelector('#project-portfolio-note').options.length===1");await keyClick('#project-portfolio-load');await until("document.querySelector('#project-portfolio-status').textContent==='Loaded retained authored model organization.'");} - async function selectFirst(){await keyClick('[data-lineage-node="model/fit-first"]');await until("document.querySelector('#project-portfolio-status').textContent==='Exact retained model summary and captured evidence.'");} - await send('Page.navigate',{url:origin+'/'});await until("!!window.fixtureUI"); - await load();await check('readCalls.length===2','Guide loading reads no model card or payload'); - await check("document.querySelector('#project-portfolio-count').textContent.includes('22 models in this guide; 1 source reviewed')",'Exact complete and reviewed guide counts'); - const point=await evaluate("(()=>{const element=document.querySelector('[data-lineage-node=\"group/pressure\"]');element.scrollIntoView({block:'center'});const box=element.getBoundingClientRect();return {x:box.x+box.width/2,y:box.y+box.height/2};})()"); - await send('Input.dispatchMouseEvent',{type:'mousePressed',button:'left',clickCount:1,...point});await send('Input.dispatchMouseEvent',{type:'mouseReleased',button:'left',clickCount:1,...point}); - await check("document.querySelector('#project-portfolio-group').value==='pressure'&&document.activeElement.dataset.lineageNode==='group/pressure'",'Pointer group activation applies the group and retains visible keyboard focus'); - await check("!document.querySelector('#root img')&&!window.injected",'Authored graph labels remain inert text'); - await keyClick('#project-portfolio-next');await check("document.querySelector('#project-portfolio-count').textContent.includes('Showing 21–22.')",'Bounded model paging uses the full population'); - await keyClick('#project-portfolio-prev');await selectFirst(); - await check("document.activeElement.id==='project-portfolio-model-heading'",'Keyboard selection moves focus to selected summary'); - await check("document.querySelector('#project-portfolio-detail').innerText.includes(fixtureData.document.models[0].summary.method)",'Complete long method paragraph is readable retained text'); - await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Desktop detail wraps without horizontal overflow'); - await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('desktop-summary.png'); - await send('Emulation.setDeviceMetricsOverride',{width:390,height:844,deviceScaleFactor:1,mobile:true}); - await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Mobile detail wraps without horizontal overflow'); - await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('mobile-summary.png'); - await send('Emulation.clearDeviceMetricsOverride'); - await fill('#project-portfolio-search','C107');await keyClick('#project-portfolio-filter-form button'); - await check("document.querySelector('#project-portfolio-count').textContent.includes('1 match the applied filters')&&document.querySelector('#project-portfolio-next').disabled",'Alias filtering has exact counts and paging'); - await selectFirst();await keyClick('#project-portfolio-detail button');await until("document.querySelector('#project-portfolio-model-heading')?.textContent==='Second model'"); - await check("document.querySelector('#project-portfolio-detail').innerText.includes('A source-reviewed interpretation has not been recorded')",'Pending review does not invent a method summary'); - await check("document.querySelector('[data-lineage-node=\"model/fit-second\"]').getAttribute('aria-pressed')==='true'",'Off-filter relationship navigation brings selected model into the graph window'); - await selectFirst();await evaluate('window.failNext=true');await keyClick('#project-portfolio-detail button'); - await until("document.querySelector('#project-portfolio-status').textContent==='unavailable: Retained card unavailable.'"); - await check("document.querySelector('#project-portfolio-detail').children.length===0",'Unavailable card clears the previous model detail'); - await evaluate('window.corruptNext=true');await keyClick('[data-lineage-node="model/fit-first"]'); - await until("document.querySelector('#project-portfolio-status').textContent==='invalid: Model card differs from its exact retained organization reference.'"); - await check("document.querySelector('#project-portfolio-detail').children.length===0",'Foreign card revision is refused without stale detail'); - await evaluate('window.holdNext=true');await keyClick('[data-lineage-node="model/fit-first"]');await until('!!window.releaseHeld'); - await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:true});window.releaseHeld()"); - await check("window.heldSignal.aborted&&document.querySelector('#project-portfolio-content').hidden",'Context replacement aborts and clears pending view'); - await load();await check("document.querySelector('#project-portfolio-detail').children.length===0",'Reload after supersession starts without old detail'); - await selectFirst();await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:false})"); - await check("document.querySelector('#root').hidden&&document.querySelector('#project-portfolio-detail').children.length===0&&document.activeElement.id==='outside'",'Permission loss clears private view and restores host focus'); - await evaluate("fixtureUI.setContext({key:'three',project_id:fixtureData.project,enabled:true})");await load(); - await fill('#project-portfolio-search','No match');await keyClick('#project-portfolio-filter-form button'); - await check("document.querySelector('#project-portfolio-count').textContent.includes('0 match the applied filters. Showing 0–0.')",'Empty filter result retains exact zero denominator'); - await check('readCalls.every(call=>[\'files_inspect\',\'file_get\'].includes(call.action))','All interactions use read-only project-file actions'); - if(report.errors.length)throw Error('Browser runtime exception');report.status='passed_authored_portfolio_real_browser_component'; - }catch(error){report.status='failed_authored_portfolio_real_browser_component';report.error=String(error); - if(session){try{const state=await session.send('Runtime.evaluate',{expression:'document.body.innerText',returnByValue:true});report.failure_page=state.result?.value; - const shot=await session.send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,'failed-page.png'),Buffer.from(shot.data,'base64'));}catch(captureError){report.failure_capture_error=String(captureError);}} - throw error;} - finally{session?.close();if(child&&child.exitCode===null&&child.signalCode===null){child.kill('SIGTERM');for(let i=0;i<100&&child.exitCode===null&&child.signalCode===null;i++)await pause(50);if(child.exitCode===null&&child.signalCode===null)child.kill('SIGKILL');} - report.browser_stopped=!child||child.exitCode!==null||child.signalCode!==null;await new Promise(resolve=>server.close(resolve)); - report.end_source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); - report.source_stable_through_tests=JSON.stringify(report.source_sha256)===JSON.stringify(report.end_source_sha256); - if(!report.source_stable_through_tests){report.status='failed_authored_portfolio_real_browser_component';report.errors.push('Source changed during browser qualification');process.exitCode=1;} - report.producer_sha256=hash(fs.readFileSync(__filename));fs.writeFileSync(path.join(output,'report.json'),JSON.stringify(report,null,2)+'\n');} - console.log(report.status,report.checks.length,'checks'); -} -main().catch(error=>{console.error(String(error));process.exitCode=1;}); diff --git a/tests/browser_project_file_deletion.cjs b/tests/browser_project_file_deletion.cjs deleted file mode 100644 index ab811e53..00000000 --- a/tests/browser_project_file_deletion.cjs +++ /dev/null @@ -1,127 +0,0 @@ -'use strict'; -// Actual explicit note deletion, restore, independent fork and cold keyboard review. -const assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); -const {spawn,execFileSync}=require('node:child_process'),{once}=require('node:events'); -const {authenticatedFixture}=require('./auth_helpers.cjs'),{waitForDebugger}=require('./browser_startup.cjs'),{cdpSession}=require('./browser_cdp_session.cjs'),{qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); -const root=path.resolve(__dirname,'..'),hash=raw=>crypto.createHash('sha256').update(raw).digest('hex'); -assert.equal(process.env.TULDOK_NOTE_DELETION_GO,'1','Explicit note-deletion qualification GO required'); -assert.ok(process.env.TULDOK_DATABASE_URL&&process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL,'Disposable source and cold PostgreSQL configurations required'); -assert.ok(process.env.TULDOK_M4_FREEZE_MANIFEST,'Lead must freeze and enable M4 before actual browser acceptance'); -const freeze=JSON.parse(fs.readFileSync(process.env.TULDOK_M4_FREEZE_MANIFEST,'utf8')); -const material={manifest_sha256:hash(fs.readFileSync(process.env.TULDOK_M4_FREEZE_MANIFEST)),files:freeze.files}; -function verifySource(){for(const [file,expected] of Object.entries(material.files))assert.equal(hash(fs.readFileSync(path.join(root,file))),expected,'Frozen source changed: '+file);} -verifySource(); -const testSha=hash(fs.readFileSync(__filename)),temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-note-deletion-browser-')),report=qaDirectory(root,'project-note-deletion'); -// HTML textarea value normalizes CR/CRLF; portable version bytes stay exact. -const displayed=raw=>raw.toString().replace(/\r\n?/g,'\n'); -const sourceDsn=process.env.TULDOK_DATABASE_URL,coldDsn=process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL; -assert.notEqual(sourceDsn,coldDsn); -const external=path.join(temporary,'external');fs.mkdirSync(external); -const children=[],catalogs=[],checks=[],errors=[],httpErrors=[],bounds=[]; -const gitEnv={...process.env,GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_TERMINAL_PROMPT:'0'}; -const routes=JSON.parse(execFileSync('python3',['-c',"import json,agent_api;names=agent_api.ProjectFilesOwnerClient.OPERATIONS;print(json.dumps({n.split('.')[1]:{**{k:agent_api.operation(n)[k] for k in ('path','method','query','side_effects')},'header':agent_api.selected_header(n)} for n in names}))"],{cwd:root,encoding:'utf8'})); -let server,origin,auth,socket,session,send,evaluate,serverErrors='',isolation; -const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); -async function until(fn,label){for(let i=0;i<200;i++){const result=await fn();if(result)return result;await pause(100);}throw Error('Timed out: '+label);} -function launch(command,args,options={}){const child=spawn(command,args,options);children.push(child);return child;} -async function stop(child){if(!child||child.exitCode!==null||child.signalCode!==null)return;const exited=once(child,'exit');child.kill('SIGTERM');await exited;} -async function start(control,directory,dsn){ - process.env.TULDOK_DATABASE_URL=dsn;serverErrors='';let output=''; - server=launch('python3',['-u','app.py','--port','0','--data',control,'--project-files-root','fixture='+directory],{cwd:root,env:{...process.env,TULDOK_DATABASE_URL:dsn},stdio:['ignore','pipe','pipe']}); - server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); - const port=await until(()=>{if(server.exitCode!==null)throw Error('Server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];},'actual server startup'); - origin='http://127.0.0.1:'+port;catalogs.push({control,dsn});auth=authenticatedFixture();await auth.register(origin,control,root); -} -async function state(){const response=await auth.fetch(origin+routes.status.path,{headers:{'Tuldok-Operation-Contract':routes.status.header}});const value=await response.json();assert.equal(response.status,200,JSON.stringify(value));return value;} -async function call(action,fields={}){ - const current=await state(),definition=routes[action],arguments_={binding:current.binding,[definition.side_effects?'expected_checkpoint':'checkpoint']:current.project.checkpoint,...(definition.side_effects?{request_key:crypto.randomUUID().replaceAll('-','')} :{}),...fields}; - let target=definition.path;for(const [key,value] of Object.entries(arguments_))target=target.replace('{'+key+'}',encodeURIComponent(value)); - const query=new URLSearchParams(Object.entries(arguments_).filter(([key])=>definition.query.includes(key)));if(query.size)target+='?'+query; - const body=Object.fromEntries(Object.entries(arguments_).filter(([key])=>!definition.query.includes(key)&&!definition.path.includes('{'+key+'}'))); - const response=await auth.fetch(origin+target,{method:definition.method,headers:{'Content-Type':'application/json','Tuldok-Operation-Contract':definition.header},...(definition.method==='POST'?{body:JSON.stringify(body)}:{})}); - const value=await response.json();assert.equal(response.status,200,JSON.stringify(value));return value.value; -} -async function key(key,code=key,modifiers=0){for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Tab:9,a:65}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});} -async function focus(selector){await until(()=>evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!e||e.disabled||!e.getClientRects().length)return false;e.scrollIntoView({block:'center'});e.focus();return document.activeElement===e;})()`),'focus '+selector);} -async function fill(selector,value){await focus(selector);await key('a','KeyA',2);await send('Input.insertText',{text:value});} -async function activate(selector){await focus(selector);await key('Enter');} -async function choose(selector,value){await focus(selector);await evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!Array.from(e.options).some(o=>o.value===${JSON.stringify(value)}))throw Error('Missing option');e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));e.dispatchEvent(new Event('input',{bubbles:true}));})()`);} -async function settled(){await until(async()=>{const failure=await evaluate('document.getElementById("project-files-error").textContent');if(failure)throw Error('UI error: '+failure);return evaluate('!document.getElementById("project-files-save").disabled');},'acknowledged UI state');} -async function navigate(){await auth.browser(send,origin);await send('Page.navigate',{url:origin+'/project-files'});await until(()=>evaluate('!!document.querySelector("#project-files-root option")&&!document.getElementById("project-files-new").disabled'),'negotiated UI startup');} -async function working(kind,file,raw,revision=0){ - await choose('#project-working-kind',kind);await fill('#project-working-path',file);await fill('#project-working-revision',String(revision));await fill('#project-working-sha',hash(raw)); - await activate('#project-working-adopt-form button[type="submit"]');await settled(); - assert.equal(await evaluate('document.getElementById("project-working-text").value'),displayed(raw)); - const observed=(await call('files_inspect')).items.find(item=>item.path===file);assert.ok(observed);const captured=await call('file_get',{file_id:observed.id,revision:observed.revision});assert.deepEqual(Buffer.from(captured.version.text),raw,'Captured public revision retains exact bytes despite native textarea newline rendering'); -} -async function inspectFile(file){ - await activate('#project-working-load');await settled(); - await evaluate(`(()=>{const b=Array.from(document.querySelectorAll('#project-working-list button')).find(e=>e.textContent.startsWith(${JSON.stringify(file+' ·')}));if(!b)throw Error('Missing working file');b.dataset.qaSelected='yes';})()`); - await activate('#project-working-list button[data-qa-selected="yes"]');await settled(); -} -async function capture(width){ - await send('Emulation.setDeviceMetricsOverride',{width,height:950,deviceScaleFactor:1,mobile:false});await evaluate('document.getElementById("project-working-heading").scrollIntoView({block:"start"})'); - await evaluate('new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))'); - const value=await evaluate('({viewport:innerWidth,width:document.documentElement.scrollWidth,client:document.documentElement.clientWidth,offenders:Array.from(document.querySelectorAll("body *")).filter(e=>e.getClientRects().length&&e.getBoundingClientRect().right>innerWidth+1).map(e=>({tag:e.tagName,id:e.id,width:e.getBoundingClientRect().width,right:e.getBoundingClientRect().right,text:e.textContent.slice(0,100)})).slice(-20)})'); - bounds.push(value);fs.writeFileSync(path.join(report,'bounds-'+width+'.json'),JSON.stringify(value,null,2)+'\n');fs.writeFileSync(path.join(report,'note-cold-'+width+'.jpg'),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64')); - assert.ok(value.width<=value.viewport+1,'Responsive overflow: '+JSON.stringify(value)); -} -function snapshot(directory){const files={};function walk(at){for(const item of fs.readdirSync(at,{withFileTypes:true})){if(['.git','.tuldok-local'].includes(item.name))continue;const next=path.join(at,item.name);if(item.isDirectory())walk(next);else if(item.isFile())files[path.relative(directory,next)]=hash(fs.readFileSync(next));else throw Error('Unexpected special fixture file');}}walk(directory);return files;} -function check(value){checks.push(value);console.log('PASS '+value);} -async function inspectIntent(){ - await activate('#project-intents-load');await settled();await until(()=>evaluate('document.querySelectorAll("#project-intents-list button").length===1'),'one unresolved recipe'); - await activate('#project-intents-list button');await settled(); - return JSON.parse(await evaluate('document.getElementById("project-intent-detail").textContent')); -} -(async()=>{ - isolation=JSON.parse(execFileSync('python3',['-c',"import json,os,psycopg;from psycopg.conninfo import conninfo_to_dict;s=conninfo_to_dict(os.environ['TULDOK_DATABASE_URL']);c=psycopg.connect(os.environ['TULDOK_PROJECT_FILES_COLD_DATABASE_URL']);row=c.execute('SELECT current_database(),current_user,has_database_privilege(current_user,%s,%s)',(s['dbname'],'CONNECT')).fetchone();assert row[0]!=s['dbname'] and not row[2];print(json.dumps(dict(database=row[0],role=row[1],source_connect=row[2])));c.close()"],{cwd:root,encoding:'utf8'})); - await start(path.join(temporary,'control'),external,sourceDsn); - const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--remote-debugging-port=0','--window-size=1440,950','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:'ignore',env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); - const port=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')),tabs=await(await fetch('http://127.0.0.1:'+port+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); - socket=new WebSocket(target.webSocketDebuggerUrl);await once(socket,'open');session=cdpSession(socket,{onEvent:event=>{if(event.method==='Runtime.exceptionThrown')errors.push(event.params.exceptionDetails);if(event.method==='Network.responseReceived'&&event.params.response.status>=400)httpErrors.push({url:event.params.response.url,status:event.params.response.status});}});send=session.send; - evaluate=async expression=>{const result=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; - await send('Page.enable');await send('Runtime.enable');await send('Network.enable');await navigate(); - await fill('#project-files-directory','alpha');await fill('#project-files-name','Note deletion keyboard Ω');await activate('#project-files-new');await settled(); - const originalState=await state(),originalProject=originalState.project.project_id,source=path.join(external,'alpha'),note=path.join(source,'notes/scientist.md'); - fs.mkdirSync(path.join(source,'notes')); - const first=Buffer.from('Exact scientist rationale Ω 山\r\n\n'),restored=Buffer.from('Explicit restored decision 🌸\n'); - fs.writeFileSync(note,first);fs.writeFileSync(path.join(source,'notes/empty.md'),''); - await working('note','notes/scientist.md',first);await working('note','notes/empty.md',Buffer.alloc(0)); - await inspectFile('notes/empty.md');assert.match(await evaluate('document.getElementById("project-working-summary").textContent'),/empty file/);assert.doesNotMatch(await evaluate('document.getElementById("project-working-summary").textContent'),/deletion/); - fs.unlinkSync(note);await inspectFile('notes/scientist.md'); - assert.equal(await evaluate('document.getElementById("project-working-text").value'),displayed(first));assert.equal(await evaluate('document.getElementById("project-working-delete-fields").disabled'),false); - const before=(await state()).project.checkpoint;await activate('#project-working-delete');await pause(100);assert.equal((await state()).project.checkpoint,before,'Unchecked deletion must have no effect'); - await focus('#project-working-delete-confirm');await key(' ','Space');assert.equal(await evaluate('document.getElementById("project-working-delete-confirm").checked'),true); - await activate('#project-working-delete');await settled();assert.ok(!fs.existsSync(note));assert.match(await evaluate('document.getElementById("project-working-summary").textContent'),/note deletion retained/);assert.equal(await evaluate('document.getElementById("project-working-text").value'),''); - await activate('#project-working-history');await settled();assert.equal(await evaluate('document.querySelectorAll("#project-working-history-list button").length'),2);assert.match(await evaluate('document.querySelectorAll("#project-working-history-list button")[1].textContent'),/note deletion/); - await activate('#project-working-history-list button');await settled();assert.equal(await evaluate('document.getElementById("project-working-text").value'),displayed(first));assert.equal(await evaluate('typeof globalThis.noteDeletionSentinel'),'undefined'); - check('Actual keyboard adoption distinguishes empty note and explicit missing-note tombstone; no unchecked effect; original scientific text remains selectable and inert'); - fs.writeFileSync(note,restored);await inspectFile('notes/scientist.md');assert.equal(await evaluate('document.getElementById("project-working-revision").value'),'2');assert.equal(await evaluate('document.getElementById("project-working-sha").value'),hash(restored)); - assert.equal(await evaluate('document.getElementById("project-working-delete-fields").disabled'),true); - await activate('#project-working-adopt-form button[type="submit"]');await settled();assert.equal(await evaluate('document.getElementById("project-working-text").value'),displayed(restored)); - fs.unlinkSync(note);await inspectFile('notes/scientist.md');await focus('#project-working-delete-confirm');await key(' ','Space');await activate('#project-working-delete');await settled(); - assert.match(await evaluate('document.getElementById("project-working-summary").textContent'),/revision 4.*deletion/); - await capture(390);await capture(1440); - const sourceBefore=snapshot(source);await fill('#project-save-as-directory','independent');await fill('#project-save-as-name','Independent deleted note Ω');await activate('#project-files-save-as');await settled(); - const forkState=await state();assert.notEqual(forkState.project.project_id,originalProject);assert.deepEqual(snapshot(source),sourceBefore); - const fork=path.join(external,'independent');assert.ok(!fs.existsSync(path.join(fork,'notes/scientist.md')));await inspectFile('notes/scientist.md');assert.match(await evaluate('document.getElementById("project-working-summary").textContent'),/revision 4.*deletion/); - check('Actual named restore appends positive revision3; second explicit deletion appends revision4; independent Save As retains absence without rewriting source'); - await activate('#project-files-close');await until(async()=>(await state()).state==='no_project_open','close before cold copy');await stop(server); - const coldRoot=path.join(temporary,'cold-external');fs.mkdirSync(coldRoot);const clone=path.join(coldRoot,'clone'),canonical=snapshot(fork); - const git=(...args)=>execFileSync('git',args,{env:gitEnv,stdio:['ignore','pipe','pipe']});git('init','-q','--template=',fork);git('-C',fork,'add','.');git('-C',fork,'-c','user.name=Disposable note QA','-c','user.email=qa@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','Retained explicit note deletion');git('-c','core.hooksPath=/dev/null','clone','-q','--no-hardlinks','--template=',fork,clone);if(fs.existsSync(path.join(fork,'artifacts')))fs.cpSync(path.join(fork,'artifacts'),path.join(clone,'artifacts'),{recursive:true});assert.deepEqual(snapshot(clone),canonical);fs.rmSync(external,{recursive:true}); - await start(path.join(temporary,'cold-control'),coldRoot,coldDsn);await navigate();await fill('#project-files-directory','clone');await activate('#project-files-open');await settled();assert.deepEqual((await state()).project,forkState.project); - await inspectFile('notes/scientist.md');assert.match(await evaluate('document.getElementById("project-working-summary").textContent'),/revision 4.*deletion/);assert.ok(!fs.existsSync(path.join(clone,'notes/scientist.md'))); - await activate('#project-working-history');await settled();assert.equal(await evaluate('document.querySelectorAll("#project-working-history-list button").length'),4); - await activate('#project-working-history-list button');await settled();assert.equal(await evaluate('document.getElementById("project-working-text").value'),displayed(first)); - await activate('#project-working-history-list li:nth-child(3) button');await settled();assert.equal(await evaluate('document.getElementById("project-working-text").value'),displayed(restored)); - await inspectFile('notes/empty.md');assert.match(await evaluate('document.getElementById("project-working-summary").textContent'),/empty file/); - await capture(390);await capture(1440);assert.deepEqual(snapshot(clone),canonical);assert.equal(await evaluate('typeof globalThis.noteDeletionSentinel'),'undefined');assert.deepEqual(errors,[]);assert.deepEqual(httpErrors,[]);verifySource();assert.equal(hash(fs.readFileSync(__filename)),testSha); - check('Cold Git clone on separate database role denied source CONNECT retains exact fork/tombstone/four revisions after source removal; 390/1440 keyboard UI stays responsive and inert'); - fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({status:'pass',checks,material,test_sha256:testSha,isolation,bounds,project:forkState.project,source_project:originalProject,browser_runtime_errors:errors,http_errors:httpErrors,canonical_sha256:canonical,fixture:'Actual named keyboard note adoption/deletion/restore; local Git clone; cold distinct DB denied source CONNECT; no runtime/preflight/Run'},null,2)+'\n');console.log('Report '+path.join(report,'receipt.json')); -})().catch(async error=>{ - fs.writeFileSync(path.join(report,'failure-receipt.json'),JSON.stringify({status:'failed',error:error.message,checks,material,test_sha256:testSha,isolation,bounds,browser_runtime_errors:errors,http_errors:httpErrors},null,2)+'\n');console.error(error);console.error('Server:',serverErrors);if(evaluate)try{console.error('UI:',await evaluate('document.body.innerText'));}catch(_){}process.exitCode=1; -}).finally(async()=>{ - session?.close();socket?.close();for(const child of children.reverse())await stop(child); - for(const {control,dsn} of catalogs)try{execFileSync('python3',['-c',"import json,pathlib,os,psycopg;from psycopg import sql;root=pathlib.Path(os.environ['QA_CONTROL']);c=psycopg.connect(os.environ['TULDOK_DATABASE_URL'],autocommit=True);[(c.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier('projection_'+v['projection_id'] if v['version']==2 else 'catalog_'+v['dataset_id'])))) for p in root.rglob('.tuldok.json') for v in [json.loads(p.read_text())]];c.close()"],{cwd:root,env:{...process.env,TULDOK_DATABASE_URL:dsn,QA_CONTROL:control},stdio:['ignore','pipe','pipe']});}catch(error){console.error('Owned cleanup failed:',error.message);process.exitCode=1;} - process.env.TULDOK_DATABASE_URL=sourceDsn;fs.rmSync(temporary,{recursive:true,force:true}); -}); diff --git a/tests/browser_project_files.cjs b/tests/browser_project_files.cjs deleted file mode 100644 index 19e854a1..00000000 --- a/tests/browser_project_files.cjs +++ /dev/null @@ -1,187 +0,0 @@ -'use strict'; -// Real browser authoring, keyboard actions and a cold process on another database. -// Inputs and runtime roots are disposable; project scripts are inert sentinels. -const assert=require('node:assert/strict'); -const fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); -const {spawn,execFileSync}=require('node:child_process'); -const {once}=require('node:events'); -const {authenticatedFixture}=require('./auth_helpers.cjs'); -const {waitForDebugger}=require('./browser_startup.cjs'); -const {cdpSession}=require('./browser_cdp_session.cjs'); -const {qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); -const root=path.resolve(__dirname,'..'); -assert.ok(process.env.TULDOK_DATABASE_URL,'Use a disposable PostgreSQL TULDOK_DATABASE_URL'); -assert.ok(process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL,'Cold proof requires a separate disposable database URL'); -const sourceDsn=process.env.TULDOK_DATABASE_URL,coldDsn=process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL; -const gitOptions={env:{...process.env,GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_TERMINAL_PROMPT:'0'}}; -assert.notEqual(sourceDsn,coldDsn,'Source and cold connection configuration must differ'); -const temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-project-files-browser-')); -const report=qaDirectory(root,'project-files'); -const external=path.join(temporary,'external');fs.mkdirSync(external); -const sourceControl=path.join(temporary,'control'),coldControl=path.join(temporary,'cold-control'); -const TEXT='Café 山 🧪 — browser-authored source\nThe scientist’s unique rationale stays exact.\n'; -const name='Browser source 山',label='positive 山',corpusNote='Earlier corpus note café 山 — preserve this scientific rationale.'; -const checks=[],errors=[],httpErrors=[],children=[],catalogs=[]; -const observationHeaders=JSON.parse(execFileSync('python3',['-c',"import json,agent_api;print(json.dumps({name:agent_api.selected_header('project_files.'+name) for name in ('status','query','get')}))"],{cwd:root,encoding:'utf8'})); -const material={head:execFileSync('git',['rev-parse','HEAD'],{cwd:root,encoding:'utf8',...gitOptions}).trim(),files:Object.fromEntries(['app.py','metadata.py','access.py','workbench.py','agent_api.py','project_store.py','project_text.py','project_workspace.py','static/project-files.html','static/project-files.js','static/project-files.css','tests/browser_project_files.cjs'].map(file=>[file,fs.existsSync(path.join(root,file))?crypto.createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex'):null]))}; -let socket,session,server,serverErrors='',origin,fixtureAuth,send,evaluate; -const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); -async function until(fn,message='Timed out'){ - for(let i=0;i<200;i++){const value=await fn();if(value)return value;await pause(100);} - throw Error(message); -} -function launch(command,args,options={}){const child=spawn(command,args,options);children.push(child);return child;} -function snapshot(directory){ - const values={}; - function walk(current){for(const entry of fs.readdirSync(current,{withFileTypes:true})){ - if(['.git','.tuldok-local'].includes(entry.name))continue; - const next=path.join(current,entry.name); - if(entry.isDirectory())walk(next); - else if(entry.isFile())values[path.relative(directory,next)]=crypto.createHash('sha256').update(fs.readFileSync(next)).digest('hex'); - else throw Error('Unexpected project special file '+entry.name); - }}walk(directory);return values; -} -async function stop(child){ - if(!child||child.exitCode!==null||child.signalCode!==null)return; - const exited=once(child,'exit');child.kill('SIGTERM');await exited; -} -async function start(control,configuredRoot,dsn){ - process.env.TULDOK_DATABASE_URL=dsn; - serverErrors=''; - server=launch('python3',['-u','app.py','--port','0','--data',control,'--project-files-root','fixture='+configuredRoot],{cwd:root,stdio:['ignore','pipe','pipe'],env:{...process.env,TULDOK_DATABASE_URL:dsn}}); - let output='';server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); - const port=await until(()=>{if(server.exitCode!==null)throw Error('Server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];},'Project-files server readiness: '+serverErrors); - origin='http://127.0.0.1:'+port; - catalogs.push({control,dsn}); - fixtureAuth=authenticatedFixture(); - await fixtureAuth.register(origin,control,root); - return origin; -} -async function jsonApi(relative){ - const route=relative.split('?')[0],operation=route.endsWith('/records')?'query':route.includes('/records/')?'get':'status'; - const response=await fixtureAuth.fetch(origin+relative,{headers:{'Tuldok-Operation-Contract':observationHeaders[operation]}}); - const value=await response.json();assert.equal(response.status,200,JSON.stringify(value));return value; -} -async function state(){return jsonApi('/api/workspace/project-files');} -async function detail(recordId){ - const current=await state(); - return jsonApi('/api/workspace/project-files/records/'+recordId+'?'+new URLSearchParams({binding:current.binding,checkpoint:current.project.checkpoint,history_limit:'4'})); -} -async function key(key,code=key,modifiers=0){ - for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Tab:9,Escape:27,a:65}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})}); -} -async function focus(selector){ - await until(()=>evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!e||e.disabled||!e.getClientRects().length)return false;e.scrollIntoView({block:'center'});e.focus();return document.activeElement===e;})()`),'Focusable control '+selector); -} -async function fill(selector,value){await focus(selector);await key('a','KeyA',2);await send('Input.insertText',{text:value});} -async function activate(selector){await focus(selector);await key('Enter');} -async function choose(selector,value){ - await focus(selector); - await evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!Array.from(e.options).some(o=>o.value===${JSON.stringify(value)}))throw Error('Missing rendered option');e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));})()`); -} -async function ready(){await until(()=>evaluate('!!document.querySelector("#project-files-root option")&&!document.getElementById("project-files-new").disabled'),'Project files UI ready');} -async function navigate(){await fixtureAuth.browser(send,origin);await send('Page.navigate',{url:origin+'/project-files'});await ready();} -async function capture(filename,width,selector='#record-heading'){ - await send('Emulation.setDeviceMetricsOverride',{width,height:950,deviceScaleFactor:1,mobile:false}); - await evaluate(`document.querySelector(${JSON.stringify(selector)}).scrollIntoView({block:'start'})`); - await pause(80); - assert.ok(await evaluate('document.documentElement.scrollWidth<=innerWidth+1'),'No horizontal overflow at '+width); - fs.writeFileSync(path.join(report,filename),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64')); -} -function check(message){checks.push(message);console.log('PASS '+message);} -(async()=>{ - // Cold runtime role must genuinely lack CONNECT to the source database. - const isolation=JSON.parse(execFileSync('python3',['-c', - "import json,os,psycopg;from psycopg.conninfo import conninfo_to_dict;source=conninfo_to_dict(os.environ['TULDOK_DATABASE_URL']);cold=conninfo_to_dict(os.environ['TULDOK_PROJECT_FILES_COLD_DATABASE_URL']);assert source['dbname']!=cold['dbname'];c=psycopg.connect(os.environ['TULDOK_PROJECT_FILES_COLD_DATABASE_URL']);row=c.execute('SELECT current_database(), current_user, has_database_privilege(current_user,%s,%s)',(source['dbname'],'CONNECT')).fetchone();assert not row[2], 'Cold role must not connect to source database';print(json.dumps({'database':row[0],'role':row[1],'source_connect':row[2]}));c.close()" - ],{cwd:root,encoding:'utf8'})); - await start(sourceControl,external,sourceDsn); - const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,950','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:'ignore',env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); - const port=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')); - const tabs=await(await fetch('http://127.0.0.1:'+port+'/json')).json(); - const target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); - socket=new WebSocket(target.webSocketDebuggerUrl);await once(socket,'open'); - session=cdpSession(socket,{onEvent:event=>{ - if(event.method==='Runtime.exceptionThrown')errors.push(event.params.exceptionDetails); - if(event.method==='Network.responseReceived'&&event.params.response.status>=400)httpErrors.push({url:event.params.response.url,status:event.params.response.status}); - }});send=session.send; - evaluate=async expression=>{const value=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(value.exceptionDetails)throw Error(JSON.stringify(value.exceptionDetails));return value.result.value;}; - await send('Page.enable');await send('Runtime.enable');await send('Network.enable'); - await navigate(); - const initial=await state();assert.equal(initial.state,'no_project_open'); - await choose('#project-files-root','fixture');await fill('#project-files-directory','alpha');await fill('#project-files-name','Browser café 🧪'); - await activate('#project-files-new'); - await until(async()=>((await state()).state==='open'),'New via keyboard opens project'); - await until(()=>evaluate('!document.getElementById("import-text").disabled')); - await fill('#text-name',name);await fill('#text-content',TEXT);await fill('#text-groups','source-fixture');await fill('#text-rights','Owned synthetic browser input'); - await activate('#import-text'); - await until(()=>evaluate('document.querySelectorAll("#records-list [data-record-id]").length===1'),'Import appears in actual UI'); - const recordId=await evaluate('document.querySelector("#records-list [data-record-id]").dataset.recordId'); - await activate('#records-list [data-record-id]'); - await until(()=>evaluate('document.getElementById("record-text").value==='+JSON.stringify(TEXT)),'Imported source shown exact'); - await choose('#annotation-task','text_corpus');await fill('#annotation-value',corpusNote);await choose('#record-review','human_reviewed'); - await activate('#annotation-save'); - await until(()=>evaluate('document.getElementById("record-summary").textContent.includes("Revision 2")&&!document.getElementById("annotation-save").disabled'),'Corpus note acknowledged in actual editor'); - await choose('#annotation-task','text_classification'); - await fill('#annotation-value',label);await fill('#annotation-groups','source-fixture,review-fixture');await choose('#record-review','human_reviewed'); - await activate('#annotation-save'); - await until(()=>evaluate('document.getElementById("record-summary").textContent.includes("Revision 3")&&!document.getElementById("annotation-save").disabled'),'Annotation acknowledged in actual editor'); - const expected=await detail(recordId); - assert.equal(expected.record.text,TEXT);assert.equal(expected.record.name,name);assert.equal(expected.record.review,'human_reviewed');assert.equal(expected.history.length,3);assert.ok(expected.history.some(item=>item.annotation?.note===corpusNote)); - await activate('#record-history-more'); - await until(()=>evaluate('document.getElementById("record-history").textContent.includes('+JSON.stringify(corpusNote)+')'),'Prior corpus note is inspectable through older-history UI'); - check('Keyboard New, Unicode import, historical corpus note and human-reviewed classification commit through real UI controls'); - await activate('#project-files-save'); - await until(()=>evaluate('!document.getElementById("project-files-save").disabled')); - const beforeClose=await state(); - await capture('authored-1440.jpg',1440);await capture('authored-390.jpg',390); - await activate('#project-files-close');await until(async()=>((await state()).state==='no_project_open')); - await until(()=>evaluate('document.querySelectorAll("#records-list [data-record-id]").length===0'),'Close clears record panel'); - await stop(server); - const source=path.join(external,'alpha');fs.mkdirSync(path.join(source,'scripts'),{recursive:true}); - fs.writeFileSync(path.join(source,'scripts','must-not-run.py'),"from pathlib import Path\nPath(__file__).with_name('EXECUTED').write_text('must never run')\n"); - const before=snapshot(source); - const coldExternal=path.join(temporary,'cold-external');fs.mkdirSync(coldExternal); - const clone=path.join(coldExternal,'clone'); - // Use an actual Git commit/clone; canonical source files are plain text. - execFileSync('git',['init','-q','--template=',source],gitOptions); - execFileSync('git',['-C',source,'add','.'],gitOptions); - execFileSync('git',['-C',source,'-c','user.name=Disposable Project QA','-c','user.email=project-qa@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','Synthetic external project fixture'],gitOptions); - execFileSync('git',['-c','core.hooksPath=/dev/null','clone','-q','--no-hardlinks','--template=',source,clone],gitOptions); - assert.deepEqual(snapshot(clone),before,'Git clone retained exact canonical authored bytes'); - await start(coldControl,coldExternal,coldDsn); - await navigate(); - assert.equal((await state()).state,'no_project_open'); - await choose('#project-files-root','fixture');await fill('#project-files-directory','clone');await activate('#project-files-open'); - await until(async()=>((await state()).state==='open'),'Cold process Open through keyboard'); - await until(()=>evaluate('document.querySelectorAll("#records-list [data-record-id]").length===1')); - await activate('#records-list [data-record-id]'); - await until(()=>evaluate('document.getElementById("record-text").value==='+JSON.stringify(TEXT))); - await until(()=>evaluate('document.getElementById("annotation-value").value==='+JSON.stringify(label))); - const cold=await state(),actual=await detail(recordId); - assert.deepEqual(cold.project,beforeClose.project); - assert.deepEqual(actual.record,expected.record);assert.deepEqual(actual.history,expected.history); - assert.equal(await evaluate('document.getElementById("record-review").value'),'human_reviewed'); - await activate('#project-files-save');await until(()=>evaluate('!document.getElementById("project-files-save").disabled')); - assert.deepEqual(snapshot(clone),before,'Cold no-op Save has no semantic or byte changes'); - assert.ok(!fs.existsSync(path.join(clone,'scripts','EXECUTED'))); - check('Fresh application process and distinct PostgreSQL role/database reconstruct exact IDs, text, review and history from Git clone; source DB access denied'); - await activate('#record-history-more'); - await until(()=>evaluate('document.getElementById("record-history").textContent.includes('+JSON.stringify(corpusNote)+')'),'Cold Open retained corpus note is inspectable through older-history UI'); - await capture('cold-open-1440.jpg',1440);await capture('cold-open-390.jpg',390);await capture('cold-history-390.jpg',390,'#record-history'); - await activate('#project-files-close');await until(async()=>((await state()).state==='no_project_open')); - assert.deepEqual(errors,[],'No browser runtime errors');assert.deepEqual(httpErrors,[],'No failed browser requests'); - for(const [file,digest] of Object.entries(material.files))assert.equal(crypto.createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex'),digest,'Material source changed during campaign: '+file); - fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({status:'pass',checks,isolation,material,project:beforeClose.project,record_id:recordId,history_count:expected.history.length,canonical_sha256:before,screenshots:['authored-1440.jpg','authored-390.jpg','cold-open-1440.jpg','cold-open-390.jpg','cold-history-390.jpg'],browser_runtime_errors:errors,http_errors:httpErrors,fixture:'Synthetic project only; no lab/scientific execution claim'},null,2)+'\n'); - console.log('Report '+path.join(report,'receipt.json')); -})().catch(async error=>{console.error(error);console.error('Server diagnostics:',serverErrors);console.error('Browser runtime errors:',JSON.stringify(errors));console.error('HTTP errors:',JSON.stringify(httpErrors));if(evaluate)try{console.error('UI diagnostics:',await evaluate('document.body.innerText'));}catch(diagnostic){console.error('UI diagnostics unavailable:',diagnostic.message);}process.exitCode=1;}).finally(async()=>{ - session?.close();socket?.close(); - for(const child of children.reverse())await stop(child); - // Drop only UUID catalog schemas whose manifests live under this owned root. - for(const {control,dsn} of catalogs){ - try{execFileSync('python3',['-c', - "import json,pathlib,os,psycopg;from psycopg import sql;root=pathlib.Path(os.environ['QA_CONTROL']);c=psycopg.connect(os.environ['TULDOK_DATABASE_URL'],autocommit=True);[(c.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(('projection_'+v['projection_id']) if v['version']==2 else ('catalog_'+v['dataset_id']))))) for p in root.rglob('.tuldok.json') for v in [json.loads(p.read_text())]];c.close()" - ],{cwd:root,env:{...process.env,TULDOK_DATABASE_URL:dsn,QA_CONTROL:control},stdio:['ignore','pipe','pipe']});}catch(error){console.error('Owned schema cleanup failed:',error.message);process.exitCode=1;} - } - process.env.TULDOK_DATABASE_URL=sourceDsn; - fs.rmSync(temporary,{recursive:true,force:true}); -}); diff --git a/tests/browser_project_files_m2.cjs b/tests/browser_project_files_m2.cjs deleted file mode 100644 index 39b9b3d7..00000000 --- a/tests/browser_project_files_m2.cjs +++ /dev/null @@ -1,273 +0,0 @@ -'use strict'; -// Real browser authoring, keyboard actions and a cold process on another database. -// Inputs and runtime roots are disposable; project scripts are inert sentinels. -const assert=require('node:assert/strict'); -const fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); -const {spawn,execFileSync}=require('node:child_process'); -const {once}=require('node:events'); -const {authenticatedFixture}=require('./auth_helpers.cjs'); -const {waitForDebugger}=require('./browser_startup.cjs'); -const {cdpSession}=require('./browser_cdp_session.cjs'); -const {qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); -const root=path.resolve(__dirname,'..'); -assert.ok(process.env.TULDOK_DATABASE_URL,'Use a disposable PostgreSQL TULDOK_DATABASE_URL'); -assert.ok(process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL,'Cold proof requires a separate disposable database URL'); -const sourceDsn=process.env.TULDOK_DATABASE_URL,coldDsn=process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL; -const gitOptions={env:{...process.env,GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_TERMINAL_PROMPT:'0'}}; -assert.notEqual(sourceDsn,coldDsn,'Source and cold connection configuration must differ'); -const temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-project-files-browser-')); -const report=qaDirectory(root,'project-files-m2'); -const external=path.join(temporary,'external');fs.mkdirSync(external); -const sourceControl=path.join(temporary,'control'),coldControl=path.join(temporary,'cold-control'); -const TEXT='Café 山 🧪 — browser-authored source\nThe scientist’s unique rationale stays exact.\n'; -const name='Browser source 山',label='positive 山',corpusNote='Earlier corpus note café 山 — preserve this scientific rationale.'; -const checks=[],errors=[],httpErrors=[],children=[],catalogs=[],responsiveFailures=[]; -const observationHeaders=JSON.parse(execFileSync('python3',['-c',"import json,agent_api;print(json.dumps({name:agent_api.selected_header('project_files.'+name) for name in ('status','query','get')}))"],{cwd:root,encoding:'utf8'})); -const material={head:execFileSync('git',['rev-parse','HEAD'],{cwd:root,encoding:'utf8',...gitOptions}).trim(),files:Object.fromEntries(['app.py','metadata.py','access.py','workbench.py','agent_api.py','project_store.py','project_text.py','project_workspace.py','static/project-files.html','static/project-files.js','static/project-files.css','tests/browser_project_files_m2.cjs'].map(file=>[file,fs.existsSync(path.join(root,file))?crypto.createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex'):null]))}; -const freeze=JSON.parse(fs.readFileSync(process.env.TULDOK_M2_FREEZE_MANIFEST||'/workspace/scratch/tuldok-save-load-qa/m2-product-freeze.json','utf8')); -material.files=freeze.files; -for(const [file,sha] of Object.entries(material.files))assert.equal(crypto.createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex'),sha,'Freeze mismatch '+file); -const testMaterial=Object.fromEntries(['tests/browser_project_files_m2.cjs','tests/test_project_files_m2_http.py'].map(file=>[file,crypto.createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex')])); -let socket,session,server,serverErrors='',origin,fixtureAuth,send,evaluate; -const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); -async function until(fn,message='Timed out'){ - for(let i=0;i<200;i++){const value=await fn();if(value)return value;await pause(100);} - throw Error(message); -} -function launch(command,args,options={}){const child=spawn(command,args,options);children.push(child);return child;} -function snapshot(directory){ - const values={}; - function walk(current){for(const entry of fs.readdirSync(current,{withFileTypes:true})){ - if(['.git','.tuldok-local'].includes(entry.name))continue; - const next=path.join(current,entry.name); - if(entry.isDirectory())walk(next); - else if(entry.isFile())values[path.relative(directory,next)]=crypto.createHash('sha256').update(fs.readFileSync(next)).digest('hex'); - else throw Error('Unexpected project special file '+entry.name); - }}walk(directory);return values; -} -async function stop(child){ - if(!child||child.exitCode!==null||child.signalCode!==null)return; - const exited=once(child,'exit');child.kill('SIGTERM');await exited; -} -async function start(control,configuredRoot,dsn){ - process.env.TULDOK_DATABASE_URL=dsn; - serverErrors=''; - server=launch('python3',['-u','app.py','--port','0','--data',control,'--project-files-root','fixture='+configuredRoot],{cwd:root,stdio:['ignore','pipe','pipe'],env:{...process.env,TULDOK_DATABASE_URL:dsn}}); - let output='';server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); - const port=await until(()=>{if(server.exitCode!==null)throw Error('Server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];},'Project-files server readiness: '+serverErrors); - origin='http://127.0.0.1:'+port; - catalogs.push({control,dsn}); - fixtureAuth=authenticatedFixture(); - await fixtureAuth.register(origin,control,root); - return origin; -} -async function jsonApi(relative){ - const route=relative.split('?')[0],operation=route.endsWith('/records')?'query':route.includes('/records/')?'get':'status'; - const response=await fixtureAuth.fetch(origin+relative,{headers:{'Tuldok-Operation-Contract':observationHeaders[operation]}}); - const value=await response.json();assert.equal(response.status,200,JSON.stringify(value));return value; -} -async function state(){return jsonApi('/api/workspace/project-files');} -async function detail(recordId){ - const current=await state(); - return jsonApi('/api/workspace/project-files/records/'+recordId+'?'+new URLSearchParams({binding:current.binding,checkpoint:current.project.checkpoint,history_limit:'4'})); -} -async function key(key,code=key,modifiers=0){ - for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Tab:9,Escape:27,a:65}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})}); -} -async function focus(selector){ - await until(()=>evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!e||e.disabled||!e.getClientRects().length)return false;e.scrollIntoView({block:'center'});e.focus();return document.activeElement===e;})()`),'Focusable control '+selector); -} -async function fill(selector,value){await focus(selector);await key('a','KeyA',2);await send('Input.insertText',{text:value});} -async function activate(selector){await focus(selector);await key('Enter');} -async function choose(selector,value){ - await focus(selector); - await evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!Array.from(e.options).some(o=>o.value===${JSON.stringify(value)}))throw Error('Missing rendered option');e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));})()`); -} -async function ready(){await until(()=>evaluate('!!document.querySelector("#project-files-root option")&&!document.getElementById("project-files-new").disabled'),'Project files UI ready');} -async function navigate(){await fixtureAuth.browser(send,origin);await send('Page.navigate',{url:origin+'/project-files'});await ready();} -async function capture(filename,width,selector='#record-heading'){ - await send('Emulation.setDeviceMetricsOverride',{width,height:950,deviceScaleFactor:1,mobile:false}); - await evaluate(`document.querySelector(${JSON.stringify(selector)}).scrollIntoView({block:'start'})`); - await evaluate('new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))'); - const bounds=await evaluate(`(()=>({viewport:innerWidth,width:document.documentElement.scrollWidth,clientWidth:document.documentElement.clientWidth,selectedValues:['project-scientific-reference','project-explore-kind','project-data-kind','project-files-root'].map(id=>({id,value:document.getElementById(id).value,text:document.getElementById(id).selectedOptions?.[0]?.textContent})),fieldsets:Array.from(document.querySelectorAll('fieldset')).filter(e=>e.getClientRects().length).map(e=>({id:e.id,width:Math.round(e.getBoundingClientRect().width),right:Math.round(e.getBoundingClientRect().right),minWidth:getComputedStyle(e).minWidth})),offenders:Array.from(document.querySelectorAll('body *')).filter(e=>e.getClientRects().length&&e.getBoundingClientRect().right>innerWidth+1).map(e=>({tag:e.tagName,id:e.id,className:e.className,width:Math.round(e.getBoundingClientRect().width),right:Math.round(e.getBoundingClientRect().right),scrollWidth:e.scrollWidth,clientWidth:e.clientWidth,minWidth:getComputedStyle(e).minWidth,text:e.textContent.slice(0,100)})).slice(-25)}))()`); - fs.writeFileSync(path.join(report,'bounds-'+width+'.json'),JSON.stringify(bounds,null,2)+'\n'); - fs.writeFileSync(path.join(report,filename),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64')); - if(bounds.width>bounds.viewport+1){responsiveFailures.push({width,filename,bounds});console.error('RESPONSIVE_FAILURE '+JSON.stringify({width,viewport:bounds.viewport,scrollWidth:bounds.width,clientWidth:bounds.clientWidth,report:path.join(report,'bounds-'+width+'.json')}));} -} -function check(message){checks.push(message);console.log('PASS '+message);} - -const routes=JSON.parse(execFileSync('python3',['-c',"import json,agent_api;names=tuple(n.split('.')[1] for n in agent_api.PROJECT_FILES_M2);print(json.dumps({n:{**{k:agent_api.operation('project_files.'+n)[k] for k in ('path','method','query','side_effects')},'header':agent_api.selected_header('project_files.'+n)} for n in names}))"],{cwd:root,encoding:'utf8'})); -async function observe(action,fields={}){ - const current=await state(),route=routes[action],args={binding:current.binding,...(route.side_effects?{expected_checkpoint:current.project.checkpoint,request_key:crypto.randomUUID().replaceAll('-','')}:{checkpoint:current.project.checkpoint}),...fields}; - let target=route.path;for(const [key,value] of Object.entries(args))target=target.replace('{'+key+'}',encodeURIComponent(value)); - const query=new URLSearchParams(Object.entries(args).filter(([key])=>route.query.includes(key))); - if(query.size)target+='?'+query; - const body=Object.fromEntries(Object.entries(args).filter(([key])=>!route.query.includes(key)&&!route.path.includes('{'+key+'}'))); - const response=await fixtureAuth.fetch(origin+target,{method:route.method,headers:{'Content-Type':'application/json','Tuldok-Operation-Contract':route.header},...(route.method==='POST'?{body:JSON.stringify(body)}:{})}); - const envelope=await response.json();assert.equal(response.status,200,JSON.stringify(envelope));return envelope.value; -} -async function settled(){await until(()=>evaluate('!document.getElementById("project-files-save").disabled&&!document.getElementById("project-files-dirty").textContent.includes("unsent")'),'UI acknowledges saved operation');} -async function explore(kind,contains){ - await choose('#project-explore-kind',kind);await activate('#project-explore-load');await settled(); - await until(()=>evaluate(`Array.from(document.querySelectorAll('#project-explore-list button')).some(e=>e.textContent.includes(${JSON.stringify(contains)}))`),'Rendered saved record '+contains); - await evaluate(`(()=>{for(const e of document.querySelectorAll('#project-explore-list button'))if(e.textContent.includes(${JSON.stringify(contains)})){e.dataset.qaSelected='yes';break;}})()`); - await activate('#project-explore-list button[data-qa-selected="yes"]');await settled(); -} -async function fileInput(selector,file){ - const document=await send('DOM.getDocument',{depth:1});const {nodeId}=await send('DOM.querySelector',{nodeId:document.root.nodeId,selector}); - assert.ok(nodeId,'Actual native file input exists');await send('DOM.setFileInputFiles',{nodeId,files:[file]}); -} -(async()=>{ - // Cold runtime role must genuinely lack CONNECT to the source database. - const isolation=JSON.parse(execFileSync('python3',['-c', - "import json,os,psycopg;from psycopg.conninfo import conninfo_to_dict;source=conninfo_to_dict(os.environ['TULDOK_DATABASE_URL']);cold=conninfo_to_dict(os.environ['TULDOK_PROJECT_FILES_COLD_DATABASE_URL']);assert source['dbname']!=cold['dbname'];c=psycopg.connect(os.environ['TULDOK_PROJECT_FILES_COLD_DATABASE_URL']);row=c.execute('SELECT current_database(), current_user, has_database_privilege(current_user,%s,%s)',(source['dbname'],'CONNECT')).fetchone();assert not row[2], 'Cold role must not connect to source database';print(json.dumps({'database':row[0],'role':row[1],'source_connect':row[2]}));c.close()" - ],{cwd:root,encoding:'utf8'})); - await start(sourceControl,external,sourceDsn); - const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,950','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:'ignore',env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); - const port=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')); - const tabs=await(await fetch('http://127.0.0.1:'+port+'/json')).json(); - const target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); - socket=new WebSocket(target.webSocketDebuggerUrl);await once(socket,'open'); - session=cdpSession(socket,{onEvent:event=>{ - if(event.method==='Runtime.exceptionThrown')errors.push(event.params.exceptionDetails); - if(event.method==='Network.responseReceived'&&event.params.response.status>=400)httpErrors.push({url:event.params.response.url,status:event.params.response.status}); - }});send=session.send; - evaluate=async expression=>{const value=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(value.exceptionDetails)throw Error(JSON.stringify(value.exceptionDetails));return value.result.value;}; - await send('Page.enable');await send('Runtime.enable');await send('Network.enable'); - - execFileSync('python3',['-c',"import pathlib,sys;sys.path.insert(0,'tests');from test_project_files_m2_http import authored_source;from PIL import Image;root=pathlib.Path(sys.argv[1]);authored_source(root);Image.new('RGB',(20,12),'#3355aa').save(root/'image.png','PNG')",external],{cwd:root}); - await navigate(); - await choose('#project-files-root','fixture');await fill('#project-files-directory','alpha');await fill('#project-files-name','Browser M2 authored project 山'); - await activate('#project-files-new');await until(async()=>((await state()).state==='open'));await settled(); - - - if(process.env.TULDOK_M2_RESPONSIVE_ONLY==='1'){ - const imported=await observe('data_import_image',{image:fs.readFileSync(path.join(external,'image.png')).toString('base64'),name:'Cold responsive image café 山',groups:['responsive-image'],rights:'Owned authored pixels'}); - const corrected=await observe('data_rights',{record_id:imported.id,revision:imported.revision,source_revision:imported.source_revision,note:'Exact cold responsive rights café 山'}); - const expected=await observe('data_get',{record_id:imported.id}),expectedHistory=await observe('data_history',{record_id:imported.id,limit:4}); - await activate('#project-files-refresh');await settled();await activate('#project-files-close');await until(async()=>((await state()).state==='no_project_open'));await stop(server); - const source=path.join(external,'alpha'),before=snapshot(source),coldExternal=path.join(temporary,'cold-external');fs.mkdirSync(coldExternal);const clone=path.join(coldExternal,'clone'); - execFileSync('git',['init','-q','--template=',source],gitOptions);execFileSync('git',['-C',source,'add','.'],gitOptions); - execFileSync('git',['-C',source,'-c','user.name=Disposable Responsive QA','-c','user.email=qa@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','Cold history responsive fixture'],gitOptions); - execFileSync('git',['-c','core.hooksPath=/dev/null','clone','-q','--no-hardlinks','--template=',source,clone],gitOptions);fs.cpSync(path.join(source,'artifacts'),path.join(clone,'artifacts'),{recursive:true}); - await start(coldControl,coldExternal,coldDsn);await navigate();await choose('#project-files-root','fixture');await fill('#project-files-directory','clone');await activate('#project-files-open');await until(async()=>((await state()).state==='open'));await settled(); - assert.deepEqual(await observe('data_get',{record_id:imported.id}),expected);assert.deepEqual(await observe('data_history',{record_id:imported.id,limit:4}),expectedHistory); - await choose('#project-data-kind','image');await explore('data_query','Cold responsive image café 山');await activate('#project-detail-history'); - await until(()=>evaluate('document.getElementById("project-detail-history-list").textContent.includes("Exact cold responsive rights café 山")')); - await capture('focused-cold-history-390.jpg',390,'#project-detail-history-list');await capture('focused-cold-history-1440.jpg',1440,'#project-detail-history-list'); - assert.deepEqual(snapshot(clone),before);assert.deepEqual(errors,[]);assert.deepEqual(httpErrors,[]); - for(const [file,digest] of Object.entries(material.files))assert.equal(crypto.createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex'),digest,'Frozen product source changed '+file); - for(const [file,digest] of Object.entries(testMaterial))assert.equal(crypto.createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex'),digest,'Acceptance source changed '+file); - fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({status:responsiveFailures.length?'failed':'pass',gate:'focused_cold_history_responsive',prior_functional_and_cold_receipt:'/workspace/scratch/tuldok-save-load-qa/artifacts/project-files-m2/run-DDH5l1/receipt.json',responsive_failures:responsiveFailures,isolation,material,test_material:testMaterial,image_id:imported.id,history_count:expectedHistory.items.length,browser_runtime_errors:errors,http_errors:httpErrors,canonical_sha256:before,fixture:'Real HTTP-authored image/rights history; real Git metadata clone and explicit untracked payload transfer; actual cold UI history'},null,2)+'\n'); - console.log('Focused responsive report '+path.join(report,'receipt.json'));if(responsiveFailures.length)throw Error('Focused cold history responsive overflow remains');return; - } - let imageId,fieldId,studyId,projectId,expectedImage,expectedField,expectedStudy,rights,scienceNote; - if(process.env.TULDOK_M2_RESUME_HTTP==='1'){ - const seedImage=await observe('data_import_image',{image:fs.readFileSync(path.join(external,'image.png')).toString('base64'),name:'Browser image café 山',groups:['browser-image'],rights:'Owned authored synthetic pixels'});imageId=seedImage.id; - rights='Browser corrected exact rights café 山';await observe('data_rights',{record_id:imageId,revision:seedImage.revision,source_revision:seedImage.source_revision,note:rights});expectedImage=await observe('data_get',{record_id:imageId}); - await observe('selection_create',{document:{name:'Browser fixed image revision 山',items:[{id:imageId,revision:expectedImage.revision,source_revision:expectedImage.source_revision}]}}); - await observe('search_create',{document:{name:'Browser entered search café 山',criteria:{q:'café',kind:'image',review:'',sort:'',task:'',label:'',group:'',rights:''}}}); - const unpinned=JSON.parse(fs.readFileSync(path.join(external,'source-project.json'),'utf8'));delete unpinned.data[0].sha256;fs.writeFileSync(path.join(external,'source-project.json'),JSON.stringify(unpinned)); - const registered=await observe('source_register',{root:'fixture',manifest_path:'source-project.json'}),snapshot=Object.fromEntries(['origin_catalog_id','project_id','manifest_sha256'].map(key=>[key,registered[key]])); - const scientificRaw=fs.readFileSync(path.join(external,'observed-fields.zip')); - const field=await observe('data_import_scientific',{snapshot,reference_id:'fields',expected_sha256:crypto.createHash('sha256').update(scientificRaw).digest('hex'),document:{name:'Browser exact field sequence 山',groups:['browser-science'],rights:'Owned authored scientific arrays'}});fieldId=field.id; - scienceNote='Retained browser observation note café 山';await observe('data_annotate',{record_id:fieldId,revision:field.revision,source_revision:field.source_revision,task:field.task,annotation:{note:scienceNote},groups:field.groups,review:'draft'});expectedField=await observe('data_get',{record_id:fieldId}); - const mismatch={...unpinned,id:crypto.randomUUID().replaceAll('-',''),data:[{...unpinned.data[0],sha256:'0'.repeat(64)}]};fs.writeFileSync(path.join(external,'mismatch-project.json'),JSON.stringify(mismatch)); - const study=await observe('study_create',{name:'Browser Study hypothesis 山',limits:{attempt_slots:1,admitted_wall_seconds:60,retained_bytes:1048576,peak_staging_bytes:1048576},policy:{format:'selection_policy_v1',metric_definition_sha256:'b'.repeat(64),checkpoint_rule:'selected',replicate_reduce:'individual',required_capabilities:[],numerical_acceptance:'diagnostic',tie_break:'trial_id'}}); - const revised=await observe('study_save',{study_id:study.id,revision:study.revision,name:'Browser revised Study hypothesis 山',limits:study.limits,policy:study.policy});studyId=revised.id;projectId=(await state()).project.project_id; - expectedStudy=await observe('study',{study_id:studyId,origin_catalog_id:projectId,limit:1}); - await activate('#project-files-refresh');await settled(); - await fill('#project-register-path','mismatch-project.json');await activate('#project-register-form button[type="submit"]');await settled(); - await explore('studies','Browser revised Study hypothesis 山'); - check('Equivalent authored fixture seeded through actual HTTP; prior4UIauthoringgroups retained, CSS and cold remainder are this targeted campaign'); - }else{ - await fill('#project-source-name','Browser image café 山');await fill('#project-source-groups','browser-image');await fill('#project-source-rights','Owned authored synthetic pixels'); - await fileInput('#project-source-file',path.join(external,'image.png'));await activate('#project-source-form button[type="submit"]');await settled(); - const rows=await observe('data_query',{kind:'image',limit:40});assert.equal(rows.items.length,1);imageId=rows.items[0].id; - await choose('#project-data-kind','image');await explore('data_query','Browser image café 山'); - rights='Browser corrected exact rights café 山';await fill('#project-rights-note',rights);await activate('#project-rights-form button[type="submit"]');await settled(); - await explore('data_query','Browser image café 山'); - await fill('#project-selection-name','Browser fixed image revision 山');await activate('#project-selection-form button[type="submit"]');await settled(); - await fill('#project-data-search','café');await fill('#project-search-name','Browser entered search café 山');await activate('#project-search-form button[type="submit"]');await settled(); - await explore('data_query','Browser image café 山');await activate('#project-detail-history'); - await until(()=>evaluate('document.getElementById("project-detail-history-list").textContent.includes('+JSON.stringify(rights)+')'),'Exact corrected rights shown in actual history controls'); - await activate('#project-detail-availability');await settled(); - expectedImage=await observe('data_get',{record_id:imageId});assert.equal(expectedImage.task,'image_detection');assert.equal(expectedImage.provenance.rights_note_correction.note,rights); - check('Actual native image upload, keyboard rights correction, fixed selection, saved search and visible revision history'); - // Explicit owner registration is a separate form from imported historical paths. - const unpinned=JSON.parse(fs.readFileSync(path.join(external,'source-project.json'),'utf8'));delete unpinned.data[0].sha256;fs.writeFileSync(path.join(external,'source-project.json'),JSON.stringify(unpinned)); - await fill('#project-register-path','source-project.json');await activate('#project-register-form button[type="submit"]');await settled(); - await until(()=>evaluate('document.querySelectorAll("#project-scientific-reference option").length===1'),'Real source reference appears'); - await fill('#project-scientific-name','Browser exact field sequence 山');await fill('#project-scientific-groups','browser-science');await fill('#project-scientific-rights','Owned authored scientific arrays'); - await activate('#project-scientific-form button[type="submit"]');await settled(); - const mismatch={...unpinned,id:crypto.randomUUID().replaceAll('-',''),data:[{...unpinned.data[0],sha256:'0'.repeat(64)}]};fs.writeFileSync(path.join(external,'mismatch-project.json'),JSON.stringify(mismatch)); - await fill('#project-register-path','mismatch-project.json');await activate('#project-register-form button[type="submit"]');await settled(); - assert.equal(await evaluate('Array.from(document.querySelectorAll("#project-scientific-reference option")).filter(option=>!option.disabled).length'),0,'Declared hash mismatch must not be selectable for import'); - await fill('#project-data-search','');await choose('#project-data-kind','field_sequence');await explore('data_query','Browser exact field sequence 山'); - scienceNote='Retained browser observation note café 山';await fill('#project-data-note',scienceNote);await activate('#project-data-note-form button[type="submit"]');await settled(); - const fields=await observe('data_query',{kind:'field_sequence',limit:40});assert.equal(fields.items.length,1);fieldId=fields.items[0].id; - expectedField=await observe('data_get',{record_id:fieldId});assert.equal(expectedField.annotation.note,scienceNote); - await explore('data_query','Browser exact field sequence 山');await activate('#project-detail-availability');await settled(); - check('Actual unpinned source registration uses current observed hash; declared mismatch is unavailable; exact scientific import, note and availability controls'); - // A missing source remains metadata and is resolved only through an explicit - // current-root file selection. No automatic search or source grant occurs. - const artifactSha=expectedField.source_sha256,artifactPath=path.join(external,'alpha','artifacts','sha256',artifactSha); - assert.ok(fs.existsSync(artifactPath),'Imported exact scientific artifact exists'); - const originalArtifact=fs.readFileSync(artifactPath);fs.unlinkSync(artifactPath); - await explore('artifacts',artifactSha);await activate('#project-artifact-status'); - await until(()=>evaluate('!document.getElementById("project-artifact-resolve-form").hidden'),'Missing artifact resolution offered explicitly'); - await choose('#project-artifact-root','fixture');await fill('#project-artifact-path','observed-fields.zip');await activate('#project-artifact-resolve-form button[type="submit"]');await settled(); - await activate('#project-artifact-status');await settled(); - assert.equal((await observe('artifact_status',{sha256:artifactSha})).state,'verified'); - assert.deepEqual(fs.readFileSync(artifactPath),originalArtifact); - check('Missing artifact survives as metadata and explicit current-root resolution restores verified exact bytes'); - await fill('#project-study-create-name','Browser Study hypothesis 山');await fill('#project-study-metric','b'.repeat(64)); - await activate('#project-study-create-form button[type="submit"]');await settled(); - await explore('studies','Browser Study hypothesis 山'); - await fill('#project-metadata-name','Browser revised Study hypothesis 山');await activate('#project-metadata-form button[type="submit"]');await settled(); - await explore('studies','Browser revised Study hypothesis 山'); - const studies=await observe('studies',{limit:40});assert.equal(studies.items.length,1);studyId=studies.items[0].study.id; - projectId=(await state()).project.project_id; - expectedStudy=await observe('study',{study_id:studyId,origin_catalog_id:projectId,limit:1});assert.equal(expectedStudy.study.revision,2); - check('Study creation and metadata revision use actual named controls without execution'); - } - await activate('#project-files-save');await settled();const beforeClose=await state(); - await capture('m2-authored-1440.jpg',1440,'#project-detail-title');await capture('m2-authored-390.jpg',390,'#project-detail-title'); - if(process.env.TULDOK_M2_LAYOUT_ONLY==='1'){console.log('Layout-only probe complete '+report);return;} - await activate('#project-files-close');await until(async()=>((await state()).state==='no_project_open'));await stop(server); - const source=path.join(external,'alpha');fs.mkdirSync(path.join(source,'scripts'),{recursive:true}); - fs.writeFileSync(path.join(source,'scripts','sentinel.py'),"from pathlib import Path\nPath(__file__).with_name('EXECUTED').write_text('bad')\n"); - const before=snapshot(source),coldExternal=path.join(temporary,'cold-external');fs.mkdirSync(coldExternal);const clone=path.join(coldExternal,'clone'); - execFileSync('git',['init','-q','--template=',source],gitOptions);execFileSync('git',['-C',source,'add','.'],gitOptions); - execFileSync('git',['-C',source,'-c','user.name=Disposable M2 QA','-c','user.email=qa@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','Authored M2 browser fixture'],gitOptions); - execFileSync('git',['-c','core.hooksPath=/dev/null','clone','-q','--no-hardlinks','--template=',source,clone],gitOptions);fs.cpSync(path.join(source,'artifacts'),path.join(clone,'artifacts'),{recursive:true});assert.deepEqual(snapshot(clone),before); - await start(coldControl,coldExternal,coldDsn);await navigate();await choose('#project-files-root','fixture');await fill('#project-files-directory','clone');await activate('#project-files-open'); - await until(async()=>((await state()).state==='open'));await settled();assert.deepEqual((await state()).project,beforeClose.project); - assert.deepEqual(await observe('data_get',{record_id:imageId}),expectedImage);assert.deepEqual(await observe('data_get',{record_id:fieldId}),expectedField); - assert.deepEqual(await observe('study',{study_id:studyId,origin_catalog_id:projectId,limit:1}),expectedStudy); - await fill('#project-data-search','');await choose('#project-data-kind','image');await explore('data_query','Browser image café 山'); - assert.equal(await evaluate('document.getElementById("project-rights-note").value'),rights); - await activate('#project-detail-history');await until(()=>evaluate('document.getElementById("project-detail-history-list").textContent.includes('+JSON.stringify(rights)+')')); - await capture('m2-cold-image-390.jpg',390,'#project-detail-history-list'); - await choose('#project-data-kind','field_sequence');await explore('data_query','Browser exact field sequence 山'); - assert.equal(await evaluate('document.getElementById("project-data-note").value'),scienceNote);await activate('#project-detail-availability');await settled(); - await capture('m2-cold-science-1440.jpg',1440,'#project-detail-title'); - await activate('#project-files-save');await settled();assert.deepEqual(snapshot(clone),before);assert.ok(!fs.existsSync(path.join(clone,'scripts','EXECUTED'))); - check('Fresh process, fresh control, distinct PostgreSQL role denied source CONNECT and real Git metadata clone plus explicit untracked artifact transfer reconstruct exact data/history/Study; no-op Save changes no bytes; sentinel stays inert'); - assert.deepEqual(errors,[]);assert.deepEqual(httpErrors,[]); - for(const [file,digest] of Object.entries(material.files))assert.equal(crypto.createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex'),digest,'Frozen product source changed '+file); - for(const [file,digest] of Object.entries(testMaterial))assert.equal(crypto.createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex'),digest,'Acceptance test changed during campaign '+file); - fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({status:responsiveFailures.length?'failed':'pass',functional_gate_passed:true,cold_gate_passed:true,responsive_failures:responsiveFailures,checks,isolation,material,test_material:testMaterial,authoring:process.env.TULDOK_M2_RESUME_HTTP==='1'?'real HTTP equivalent fixture; prior UI evidence retained':'actual UI native authoring',project:beforeClose.project,image_id:imageId,field_id:fieldId,study_id:studyId,canonical_sha256:before,browser_runtime_errors:errors,http_errors:httpErrors,fixture:'Authored disposable synthetic fixtures; no training or scientific validity claim'},null,2)+'\n'); - console.log('Report '+path.join(report,'receipt.json'));if(responsiveFailures.length)throw Error('Responsive overflow remains after independently completed cold gate'); -})().catch(async error=>{fs.writeFileSync(path.join(report,'failure-receipt.json'),JSON.stringify({status:'failed',error:error.message,checks,material,test_material:testMaterial,responsive_failures:responsiveFailures,browser_runtime_errors:errors,http_errors:httpErrors},null,2)+'\n');console.error('Failure receipt '+path.join(report,'failure-receipt.json'));console.error(error);console.error('Server diagnostics:',serverErrors);console.error('Browser runtime errors:',JSON.stringify(errors));console.error('HTTP errors:',JSON.stringify(httpErrors));if(evaluate)try{console.error('UI diagnostics:',await evaluate('document.body.innerText'));}catch(diagnostic){console.error('UI diagnostics unavailable:',diagnostic.message);}process.exitCode=1;}).finally(async()=>{ - session?.close();socket?.close(); - for(const child of children.reverse())await stop(child); - // Drop only UUID catalog schemas whose manifests live under this owned root. - for(const {control,dsn} of catalogs){ - try{execFileSync('python3',['-c', - "import json,pathlib,os,psycopg;from psycopg import sql;root=pathlib.Path(os.environ['QA_CONTROL']);c=psycopg.connect(os.environ['TULDOK_DATABASE_URL'],autocommit=True);[(c.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(('projection_'+v['projection_id']) if v['version']==2 else ('catalog_'+v['dataset_id']))))) for p in root.rglob('.tuldok.json') for v in [json.loads(p.read_text())]];c.close()" - ],{cwd:root,env:{...process.env,TULDOK_DATABASE_URL:dsn,QA_CONTROL:control},stdio:['ignore','pipe','pipe']});}catch(error){console.error('Owned schema cleanup failed:',error.message);process.exitCode=1;} - } - process.env.TULDOK_DATABASE_URL=sourceDsn; - fs.rmSync(temporary,{recursive:true,force:true}); -}); diff --git a/tests/browser_project_files_m3.cjs b/tests/browser_project_files_m3.cjs deleted file mode 100644 index 5fc1a735..00000000 --- a/tests/browser_project_files_m3.cjs +++ /dev/null @@ -1,128 +0,0 @@ -'use strict'; -// Real named runtime controls; offline byte capture, explicit recovery, inert cold Open. -const assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); -const {spawn,execFileSync}=require('node:child_process'),{once}=require('node:events'); -const {authenticatedFixture}=require('./auth_helpers.cjs'),{waitForDebugger}=require('./browser_startup.cjs'),{cdpSession}=require('./browser_cdp_session.cjs'),{qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); -const root=path.resolve(__dirname,'..'),hash=raw=>crypto.createHash('sha256').update(raw).digest('hex'); -assert.ok(process.env.TULDOK_DATABASE_URL&&process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL,'Disposable source and cold PostgreSQL configurations required'); -assert.ok(process.env.TULDOK_M3_FREEZE_MANIFEST,'Lead must freeze and qualify isolated M3 before actual browser acceptance'); -const freeze=JSON.parse(fs.readFileSync(process.env.TULDOK_M3_FREEZE_MANIFEST,'utf8')); -const material={manifest_sha256:hash(fs.readFileSync(process.env.TULDOK_M3_FREEZE_MANIFEST)),files:freeze.files}; -function verifySource(){for(const [file,expected] of Object.entries(material.files))assert.equal(hash(fs.readFileSync(path.join(root,file))),expected,'Frozen source changed: '+file);} -verifySource(); -const testSha=hash(fs.readFileSync(__filename)),temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-m3-browser-')),report=qaDirectory(root,'project-files-m3'); -const sourceDsn=process.env.TULDOK_DATABASE_URL,coldDsn=process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL; -assert.notEqual(sourceDsn,coldDsn); -const external=path.join(temporary,'external');fs.mkdirSync(external); -const children=[],catalogs=[],checks=[],errors=[],httpErrors=[],bounds=[],requests=[]; -const sentinel=path.join(temporary,'EXECUTED'),audit=path.join(temporary,'process-audit.jsonl'),auditDir=path.join(temporary,'audit');fs.mkdirSync(auditDir); -fs.writeFileSync(path.join(auditDir,'sitecustomize.py'),`import sys,os,json -with open(os.environ['QA_AUDIT_INSTALLED'],'w') as f:f.write('installed') -def guard(event,args): - if event in ('subprocess.Popen','os.system','os.exec','os.posix_spawn'): - with open(os.environ['QA_PROCESS_AUDIT'],'a') as f:f.write(json.dumps({'event':event})+'\\n') - raise AssertionError('No process execution is allowed during this capture fixture') -sys.addaudithook(guard) -`); -const configFile=path.join(temporary,'runtime.json'); -const gitEnv={...process.env,GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_TERMINAL_PROMPT:'0'}; -const routes=JSON.parse(execFileSync('python3',['-c',"import json,agent_api;names=agent_api.ProjectFilesOwnerClient.OPERATIONS;print(json.dumps({n.split('.')[1]:{**{k:agent_api.operation(n)[k] for k in ('path','method','query','side_effects')},'header':agent_api.selected_header(n)} for n in names}))"],{cwd:root,encoding:'utf8'})); -let server,origin,auth,socket,session,send,evaluate,serverErrors='',isolation; -const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); -async function until(fn,label){for(let i=0;i<200;i++){const result=await fn();if(result)return result;await pause(100);}throw Error('Timed out: '+label);} -function launch(command,args,options={}){const child=spawn(command,args,options);children.push(child);return child;} -async function stop(child){if(!child||child.exitCode!==null||child.signalCode!==null)return;const exited=once(child,'exit');child.kill('SIGTERM');await exited;} -async function start(control,directory,dsn,configured=false){ - process.env.TULDOK_DATABASE_URL=dsn;serverErrors='';let output=''; - server=launch('python3',['-u','app.py','--port','0','--data',control,'--project-files-root','fixture='+directory,...(configured?['--project-runtime-source','inert-fixture='+configFile]:[])],{cwd:root,env:{...process.env,TULDOK_DATABASE_URL:dsn,QA_PROCESS_AUDIT:audit,QA_AUDIT_INSTALLED:path.join(temporary,'audit-installed-'+catalogs.length),PYTHONPATH:auditDir+path.delimiter+(process.env.PYTHONPATH||'')},stdio:['ignore','pipe','pipe']}); - server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); - const port=await until(()=>{if(server.exitCode!==null)throw Error('Server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];},'actual server startup'); - assert.ok(fs.existsSync(path.join(temporary,'audit-installed-'+catalogs.length)),'App audit assertion installed');origin='http://127.0.0.1:'+port;catalogs.push({control,dsn});auth=authenticatedFixture();await auth.register(origin,control,root); -} -async function state(){const response=await auth.fetch(origin+routes.status.path,{headers:{'Tuldok-Operation-Contract':routes.status.header}});const value=await response.json();assert.equal(response.status,200,JSON.stringify(value));return value;} -async function key(key,code=key,modifiers=0){for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Tab:9,a:65,' ':32}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});} -async function focus(selector){await until(()=>evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!e||e.disabled||!e.getClientRects().length)return false;e.scrollIntoView({block:'center'});e.focus();return document.activeElement===e;})()`),'focus '+selector);} -async function fill(selector,value){await focus(selector);await key('a','KeyA',2);await send('Input.insertText',{text:value});} -async function activate(selector){await focus(selector);await key('Enter');} -async function choose(selector,value){await focus(selector);await evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!Array.from(e.options).some(o=>o.value===${JSON.stringify(value)}))throw Error('Missing option');e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));e.dispatchEvent(new Event('input',{bubbles:true}));})()`);} -async function settled(){await until(async()=>{const failure=await evaluate('document.getElementById("project-files-error").textContent');if(failure)throw Error('UI error: '+failure);return evaluate('!document.getElementById("project-files-save").disabled');},'acknowledged UI state');} -async function navigate(){await auth.browser(send,origin);await send('Page.navigate',{url:origin+'/project-files'});await until(()=>evaluate('!!document.querySelector("#project-files-root option")&&!document.getElementById("project-files-new").disabled'),'negotiated UI startup');} -async function capture(width){ - await send('Emulation.setDeviceMetricsOverride',{width,height:950,deviceScaleFactor:1,mobile:false});await evaluate('document.getElementById("project-runtime-title").scrollIntoView({block:"start"})'); - await evaluate('new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))'); - const value=await evaluate('({viewport:innerWidth,width:document.documentElement.scrollWidth,client:document.documentElement.clientWidth,offenders:Array.from(document.querySelectorAll("body *")).filter(e=>e.getClientRects().length&&e.getBoundingClientRect().right>innerWidth+1).map(e=>({tag:e.tagName,id:e.id,width:e.getBoundingClientRect().width,right:e.getBoundingClientRect().right,text:e.textContent.slice(0,100)})).slice(-20)})'); - bounds.push(value);fs.writeFileSync(path.join(report,'bounds-'+width+'.json'),JSON.stringify(value,null,2)+'\n');fs.writeFileSync(path.join(report,'m3-runtime-'+width+'.jpg'),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64')); - assert.ok(value.width<=value.viewport+1,'Responsive overflow: '+JSON.stringify(value)); -} -function snapshot(directory){const files={};function walk(at){for(const item of fs.readdirSync(at,{withFileTypes:true})){if(['.git','.tuldok-local'].includes(item.name))continue;const next=path.join(at,item.name);if(item.isDirectory())walk(next);else if(item.isFile())files[path.relative(directory,next)]=hash(fs.readFileSync(next));else throw Error('Unexpected special fixture file');}}walk(directory);return files;} -function check(value){checks.push(value);console.log('PASS '+value);} -(async()=>{ - isolation=JSON.parse(execFileSync('python3',['-c',"import json,os,psycopg;from psycopg.conninfo import conninfo_to_dict;s=conninfo_to_dict(os.environ['TULDOK_DATABASE_URL']);c=psycopg.connect(os.environ['TULDOK_PROJECT_FILES_COLD_DATABASE_URL']);row=c.execute('SELECT current_database(),current_user,has_database_privilege(current_user,%s,%s)',(s['dbname'],'CONNECT')).fetchone();assert row[0]!=s['dbname'] and not row[2];print(json.dumps(dict(database=row[0],role=row[1],source_connect=row[2])));c.close()"],{cwd:root,encoding:'utf8'})); - execFileSync('python3',['-c',`import json,pathlib,shutil,sys -p=pathlib.Path(sys.argv[1]);python=p/'explicit-python';site=p/'explicit-packages' -(python/'bin').mkdir(parents=True);(python/'lib/python3.12/lib-dynload').mkdir(parents=True) -shutil.copyfile('/usr/bin/true',python/'bin/python3.12') -for i in range(45):(python/'lib/python3.12/lib-dynload'/('fixture-%03d.txt'%i)).write_text('Owned inert runtime bytes Ω '+str(i)) -packages=[] -for name in ('torch','numpy'): - (site/name).mkdir(parents=True);(site/name/'__init__.py').write_text('from pathlib import Path'+chr(10)+'Path('+repr(sys.argv[2])+').write_text("executed")'+chr(10)+'raise AssertionError("Never import fixture")'+chr(10)) - info=site/(name+'-0.0.dist-info');info.mkdir();(info/'METADATA').write_text('Name: '+name+chr(10)+'Version: 0.0'+chr(10));(info/'LICENSE').write_text('Owned fixture');packages.extend([name,info.name]) -(p/'runtime.json').write_text(json.dumps(dict(schema_version=1,python_root=str(python),python_version='3.12',site_packages=str(site),packages=packages)))`,temporary,sentinel],{cwd:root}); - await start(path.join(temporary,'control'),external,sourceDsn,true); - const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--remote-debugging-port=0','--window-size=1440,950','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:'ignore',env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); - const port=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')),tabs=await(await fetch('http://127.0.0.1:'+port+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); - socket=new WebSocket(target.webSocketDebuggerUrl);await once(socket,'open');session=cdpSession(socket,{onEvent:event=>{if(event.method==='Runtime.exceptionThrown')errors.push(event.params.exceptionDetails);if(event.method==='Network.requestWillBeSent')requests.push({url:event.params.request.url,method:event.params.request.method,body:event.params.request.postData});if(event.method==='Network.responseReceived'&&event.params.response.status>=400)httpErrors.push({url:event.params.response.url,status:event.params.response.status});}});send=session.send; - evaluate=async expression=>{const result=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; - await send('Page.enable');await send('Runtime.enable');await send('Network.enable');await navigate(); - await fill('#project-files-directory','alpha');await fill('#project-files-name','M3 offline runtime browser Ω');await activate('#project-files-new');await settled(); - const initial=await state(),project=initial.project.project_id; - const automatic=()=>requests.filter(r=>r.url.includes('/runtime/')); - assert.equal(automatic().length,0,'New does not automatically inspect or capture runtime'); - await activate('#project-runtime-sources');await settled();assert.equal(await evaluate('document.querySelectorAll("#project-runtime-source option").length'),1); - await activate('#project-runtime-inspect');await settled();assert.equal(await evaluate('document.querySelectorAll("#project-runtime-files li").length'),40); - await activate('#project-runtime-files-more');await settled();assert.ok(await evaluate('document.querySelectorAll("#project-runtime-files li").length>40')); - const summary=await evaluate('document.getElementById("project-runtime-summary").textContent'),inventory=summary.match(/[a-f0-9]{64}/)[0]; - assert.match(await evaluate('document.getElementById("project-runtime-layout").textContent'),/torch/);assert.equal(automatic().filter(r=>r.method==='POST').length,0); - check('Keyboard loads configured alias and bounded paged inventory without capture or executing package imports'); - await fill('#project-runtime-reserve','9007199254740991');await focus('#project-runtime-confirm');await key(' ','Space'); - await activate('#project-runtime-capture'); - await until(()=>evaluate('document.getElementById("project-files-error").textContent'),'expected real insufficient-reserve error'); - const failedCapture=automatic().find(r=>r.method==='POST'&&r.url.endsWith('/runtime/capture'));assert.ok(failedCapture);const failedBody=JSON.parse(failedCapture.body); - assert.match(await evaluate('document.getElementById("project-files-error").textContent'),/disk|reserve/i); - await activate('#project-files-refresh'); - await until(()=>evaluate('!document.getElementById("project-runtime-abandon-fields").disabled'),'current original capture recovery control'); - const pending=await state();assert.ok(['open','recovery_required'].includes(pending.state)); - await fill('#project-runtime-abandon-key',failedBody.request_key);await fill('#project-runtime-abandon-reason','Explicitly abandon impossible reserve; preserve original inventory and intent Ω');await activate('#project-runtime-abandon'); - await until(()=>evaluate('document.getElementById("project-files-status").textContent.includes("partial bytes and scientific intent remain")'),'canonical abandonment ACK'); - await activate('#project-files-refresh');await settled(); - await fill('#project-runtime-status-key',failedBody.request_key);await activate('#project-runtime-status-form button');await settled(); - const abandoned=JSON.parse(await evaluate('document.getElementById("project-runtime-status").textContent'));assert.equal(abandoned.state,'abandoned');assert.equal(abandoned.intent.manifest_sha256,inventory);assert.equal(abandoned.abandonment.capture_key,failedBody.request_key);assert.equal(abandoned.result,null); - check('Real reserve failure retains durable intent; named current-owner abandonment records exact key/reason and historical abandoned outcome'); - await activate('#project-runtime-sources');await settled();await activate('#project-runtime-inspect');await settled();await fill('#project-runtime-reserve','0');await focus('#project-runtime-confirm');await key(' ','Space');await activate('#project-runtime-capture');await settled(); - const captured=JSON.parse(await evaluate('document.getElementById("project-runtime-status").textContent'));assert.equal(captured.format,'project_runtime_capture_v1');assert.equal(captured.project_id,project);assert.equal(captured.manifest_sha256,inventory);assert.notEqual(captured.request_key,failedBody.request_key); - await activate('#project-runtime-status-form button');await settled();const historical=JSON.parse(await evaluate('document.getElementById("project-runtime-status").textContent'));assert.equal(historical.state,'captured');assert.deepEqual(historical.result,captured); - assert.ok(!fs.existsSync(sentinel));assert.ok(!fs.existsSync(audit),'Application never attempted subprocess/exec/install'); - check('Fresh explicit confirmed capture copies offline bytes and retains intent/result; no child process or package import executes'); - // Edits reach the project folder on Save; Close keeps them only in the working copy. - await activate('#project-files-save');await settled(); - await activate('#project-files-close');await until(async()=>(await state()).state==='no_project_open','close captured project');await stop(server); - const source=path.join(external,'alpha'),coldRoot=path.join(temporary,'cold-external');fs.mkdirSync(coldRoot);const clone=path.join(coldRoot,'clone'),canonical=snapshot(source); - const git=(...args)=>execFileSync('git',args,{env:gitEnv,stdio:['ignore','pipe','pipe']});git('init','-q','--template=',source);git('-C',source,'add','.');git('-C',source,'-c','user.name=Disposable M3 QA','-c','user.email=qa@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','Inert canonical runtime facts');git('-c','core.hooksPath=/dev/null','clone','-q','--no-hardlinks','--template=',source,clone);if(fs.existsSync(path.join(source,'artifacts')))fs.cpSync(path.join(source,'artifacts'),path.join(clone,'artifacts'),{recursive:true});assert.deepEqual(snapshot(clone),canonical); - // Remove the original configured source before cold Open. Canonical metadata cannot restore it. - fs.rmSync(path.join(temporary,'explicit-python'),{recursive:true});fs.rmSync(path.join(temporary,'explicit-packages'),{recursive:true});fs.unlinkSync(configFile); - const beforeCold=automatic().length;await start(path.join(temporary,'cold-control'),coldRoot,coldDsn);await navigate();await fill('#project-files-directory','clone');await activate('#project-files-open');await settled();assert.equal((await state()).project.project_id,project);assert.equal(automatic().length,beforeCold,'Cold Open does not auto inspect/capture/status'); - await activate('#project-runtime-sources');await settled();assert.equal(await evaluate('document.querySelectorAll("#project-runtime-source option").length'),0); - await fill('#project-runtime-status-key',captured.request_key);await activate('#project-runtime-status-form button');await settled();assert.deepEqual(JSON.parse(await evaluate('document.getElementById("project-runtime-status").textContent')),historical); - await fill('#project-runtime-status-key',failedBody.request_key);await activate('#project-runtime-status-form button');await settled();assert.deepEqual(JSON.parse(await evaluate('document.getElementById("project-runtime-status").textContent')),abandoned); - const cached=execFileSync('python3',['-c',"from pathlib import Path;import sys;print(len(list(Path(sys.argv[1]).rglob('runtime-snapshots'))))",path.join(temporary,'cold-control')],{encoding:'utf8'}).trim();assert.equal(cached,'0'); - await capture(390);await capture(1440);assert.deepEqual(snapshot(clone),canonical);assert.ok(!fs.existsSync(sentinel));assert.ok(!fs.existsSync(audit));assert.deepEqual(errors,[]); - assert.equal(httpErrors.length,1,JSON.stringify(httpErrors));assert.ok(httpErrors[0].url.endsWith('/runtime/capture'));verifySource();assert.equal(hash(fs.readFileSync(__filename)),testSha); - check('Cold keyboard UI on distinct role denied source CONNECT reconstructs exact captured/abandoned facts with no source alias/cache or automatic regeneration; 390/1440 responsive'); - fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({status:'pass',source_stable:true,checks,material,test_sha256:testSha,isolation,bounds,project_id:project,captured,abandoned,browser_runtime_errors:errors,expected_http_errors:httpErrors,runtime_requests:automatic(),fixture:'M3 enabled only in derived immutable source. Actual app CLI/PG/Chromium; offline true ELF plus inert never-import packages; app-only audit hook rejects child execution; real impossible-reserve failure and explicit current-marker abandonment; Git canonical clone with explicit artifact transfer; source removed before cold Open. No scientific execution or runtime preflight.'},null,2)+'\n');console.log('Report '+path.join(report,'receipt.json')); -})().catch(async error=>{ - fs.writeFileSync(path.join(report,'failure-receipt.json'),JSON.stringify({status:'failed',error:error.message,checks,material,test_sha256:testSha,isolation,bounds,browser_runtime_errors:errors,http_errors:httpErrors,runtime_requests:requests.filter(r=>r.url.includes("/runtime/"))},null,2)+'\n');console.error(error);console.error('Server:',serverErrors);if(evaluate)try{console.error('UI:',await evaluate('document.body.innerText'));}catch(_){}process.exitCode=1; -}).finally(async()=>{ - session?.close();socket?.close();for(const child of children.reverse())await stop(child); - for(const {control,dsn} of catalogs)try{execFileSync('python3',['-c',"import json,pathlib,os,psycopg;from psycopg import sql;root=pathlib.Path(os.environ['QA_CONTROL']);c=psycopg.connect(os.environ['TULDOK_DATABASE_URL'],autocommit=True);[(c.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier('projection_'+v['projection_id'] if v['version']==2 else 'catalog_'+v['dataset_id'])))) for p in root.rglob('.tuldok.json') for v in [json.loads(p.read_text())]];c.close()"],{cwd:root,env:{...process.env,TULDOK_DATABASE_URL:dsn,QA_CONTROL:control},stdio:['ignore','pipe','pipe']});}catch(error){console.error('Owned cleanup failed:',error.message);process.exitCode=1;} - process.env.TULDOK_DATABASE_URL=sourceDsn;fs.rmSync(temporary,{recursive:true,force:true}); -}); diff --git a/tests/browser_project_files_m4.cjs b/tests/browser_project_files_m4.cjs deleted file mode 100644 index 3cc06c20..00000000 --- a/tests/browser_project_files_m4.cjs +++ /dev/null @@ -1,115 +0,0 @@ -'use strict'; -// Real keyboard-driven named M4 controls, followed by a cold metadata clone. -const assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); -const {spawn,execFileSync}=require('node:child_process'),{once}=require('node:events'); -const {authenticatedFixture}=require('./auth_helpers.cjs'),{waitForDebugger}=require('./browser_startup.cjs'),{cdpSession}=require('./browser_cdp_session.cjs'),{qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); -const root=path.resolve(__dirname,'..'),hash=raw=>crypto.createHash('sha256').update(raw).digest('hex'); -assert.ok(process.env.TULDOK_DATABASE_URL&&process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL,'Disposable source and cold PostgreSQL configurations required'); -assert.ok(process.env.TULDOK_M4_FREEZE_MANIFEST,'Lead must freeze and enable M4 before actual browser acceptance'); -const freeze=JSON.parse(fs.readFileSync(process.env.TULDOK_M4_FREEZE_MANIFEST,'utf8')); -const material={manifest_sha256:hash(fs.readFileSync(process.env.TULDOK_M4_FREEZE_MANIFEST)),files:freeze.files}; -function verifySource(){for(const [file,expected] of Object.entries(material.files))assert.equal(hash(fs.readFileSync(path.join(root,file))),expected,'Frozen source changed: '+file);} -verifySource(); -const testSha=hash(fs.readFileSync(__filename)),temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-m4-browser-')),report=qaDirectory(root,'project-files-m4'); -const sourceDsn=process.env.TULDOK_DATABASE_URL,coldDsn=process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL; -assert.notEqual(sourceDsn,coldDsn); -const external=path.join(temporary,'external');fs.mkdirSync(external); -const children=[],catalogs=[],checks=[],errors=[],httpErrors=[],bounds=[]; -const gitEnv={...process.env,GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_TERMINAL_PROMPT:'0'}; -const routes=JSON.parse(execFileSync('python3',['-c',"import json,agent_api;names=agent_api.ProjectFilesOwnerClient.OPERATIONS;print(json.dumps({n.split('.')[1]:{**{k:agent_api.operation(n)[k] for k in ('path','method','query','side_effects')},'header':agent_api.selected_header(n)} for n in names}))"],{cwd:root,encoding:'utf8'})); -let server,origin,auth,socket,session,send,evaluate,serverErrors='',isolation; -const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); -async function until(fn,label){for(let i=0;i<200;i++){const result=await fn();if(result)return result;await pause(100);}throw Error('Timed out: '+label);} -function launch(command,args,options={}){const child=spawn(command,args,options);children.push(child);return child;} -async function stop(child){if(!child||child.exitCode!==null||child.signalCode!==null)return;const exited=once(child,'exit');child.kill('SIGTERM');await exited;} -async function start(control,directory,dsn){ - process.env.TULDOK_DATABASE_URL=dsn;serverErrors='';let output=''; - server=launch('python3',['-u','app.py','--port','0','--data',control,'--project-files-root','fixture='+directory],{cwd:root,env:{...process.env,TULDOK_DATABASE_URL:dsn},stdio:['ignore','pipe','pipe']}); - server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); - const port=await until(()=>{if(server.exitCode!==null)throw Error('Server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];},'actual server startup'); - origin='http://127.0.0.1:'+port;catalogs.push({control,dsn});auth=authenticatedFixture();await auth.register(origin,control,root); -} -async function state(){const response=await auth.fetch(origin+routes.status.path,{headers:{'Tuldok-Operation-Contract':routes.status.header}});const value=await response.json();assert.equal(response.status,200,JSON.stringify(value));return value;} -async function call(action,fields={}){ - const current=await state(),definition=routes[action],arguments_={binding:current.binding,[definition.side_effects?'expected_checkpoint':'checkpoint']:current.project.checkpoint,...(definition.side_effects?{request_key:crypto.randomUUID().replaceAll('-','')} :{}),...fields}; - let target=definition.path;for(const [key,value] of Object.entries(arguments_))target=target.replace('{'+key+'}',encodeURIComponent(value)); - const query=new URLSearchParams(Object.entries(arguments_).filter(([key])=>definition.query.includes(key)));if(query.size)target+='?'+query; - const body=Object.fromEntries(Object.entries(arguments_).filter(([key])=>!definition.query.includes(key)&&!definition.path.includes('{'+key+'}'))); - const response=await auth.fetch(origin+target,{method:definition.method,headers:{'Content-Type':'application/json','Tuldok-Operation-Contract':definition.header},...(definition.method==='POST'?{body:JSON.stringify(body)}:{})}); - const value=await response.json();assert.equal(response.status,200,JSON.stringify(value));return value.value; -} -async function key(key,code=key,modifiers=0){for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Tab:9,a:65}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});} -async function focus(selector){await until(()=>evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!e||e.disabled||!e.getClientRects().length)return false;e.scrollIntoView({block:'center'});e.focus();return document.activeElement===e;})()`),'focus '+selector);} -async function fill(selector,value){await focus(selector);await key('a','KeyA',2);await send('Input.insertText',{text:value});} -async function activate(selector){await focus(selector);await key('Enter');} -async function choose(selector,value){await focus(selector);await evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!Array.from(e.options).some(o=>o.value===${JSON.stringify(value)}))throw Error('Missing option');e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));e.dispatchEvent(new Event('input',{bubbles:true}));})()`);} -async function settled(){await until(async()=>{const failure=await evaluate('document.getElementById("project-files-error").textContent');if(failure)throw Error('UI error: '+failure);return evaluate('!document.getElementById("project-files-save").disabled');},'acknowledged UI state');} -async function navigate(){await auth.browser(send,origin);await send('Page.navigate',{url:origin+'/project-files'});await until(()=>evaluate('!!document.querySelector("#project-files-root option")&&!document.getElementById("project-files-new").disabled'),'negotiated UI startup');} -async function working(kind,file,raw,revision=0){ - await choose('#project-working-kind',kind);await fill('#project-working-path',file);await fill('#project-working-revision',String(revision));await fill('#project-working-sha',hash(raw)); - await activate('#project-working-adopt-form button[type="submit"]');await settled(); - assert.equal(await evaluate('document.getElementById("project-working-text").value'),raw.toString()); -} -async function inspectFile(file){ - await activate('#project-working-load');await settled(); - await evaluate(`(()=>{const b=Array.from(document.querySelectorAll('#project-working-list button')).find(e=>e.textContent.startsWith(${JSON.stringify(file+' ·')}));if(!b)throw Error('Missing working file');b.dataset.qaSelected='yes';})()`); - await activate('#project-working-list button[data-qa-selected="yes"]');await settled(); -} -async function capture(width){ - await send('Emulation.setDeviceMetricsOverride',{width,height:950,deviceScaleFactor:1,mobile:false});await evaluate('document.getElementById("project-intent-title").scrollIntoView({block:"start"})'); - await evaluate('new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))'); - const value=await evaluate('({viewport:innerWidth,width:document.documentElement.scrollWidth,client:document.documentElement.clientWidth,offenders:Array.from(document.querySelectorAll("body *")).filter(e=>e.getClientRects().length&&e.getBoundingClientRect().right>innerWidth+1).map(e=>({tag:e.tagName,id:e.id,width:e.getBoundingClientRect().width,right:e.getBoundingClientRect().right,text:e.textContent.slice(0,100)})).slice(-20)})'); - bounds.push(value);fs.writeFileSync(path.join(report,'bounds-'+width+'.json'),JSON.stringify(value,null,2)+'\n');fs.writeFileSync(path.join(report,'m4-cold-'+width+'.jpg'),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64')); - assert.ok(value.width<=value.viewport+1,'Responsive overflow: '+JSON.stringify(value)); -} -function snapshot(directory){const files={};function walk(at){for(const item of fs.readdirSync(at,{withFileTypes:true})){if(['.git','.tuldok-local'].includes(item.name))continue;const next=path.join(at,item.name);if(item.isDirectory())walk(next);else if(item.isFile())files[path.relative(directory,next)]=hash(fs.readFileSync(next));else throw Error('Unexpected special fixture file');}}walk(directory);return files;} -function check(value){checks.push(value);console.log('PASS '+value);} -async function inspectIntent(){ - await activate('#project-intents-load');await settled();await until(()=>evaluate('document.querySelectorAll("#project-intents-list button").length===1'),'one unresolved recipe'); - await activate('#project-intents-list button');await settled(); - return JSON.parse(await evaluate('document.getElementById("project-intent-detail").textContent')); -} -(async()=>{ - isolation=JSON.parse(execFileSync('python3',['-c',"import json,os,psycopg;from psycopg.conninfo import conninfo_to_dict;s=conninfo_to_dict(os.environ['TULDOK_DATABASE_URL']);c=psycopg.connect(os.environ['TULDOK_PROJECT_FILES_COLD_DATABASE_URL']);row=c.execute('SELECT current_database(),current_user,has_database_privilege(current_user,%s,%s)',(s['dbname'],'CONNECT')).fetchone();assert row[0]!=s['dbname'] and not row[2];print(json.dumps(dict(database=row[0],role=row[1],source_connect=row[2])));c.close()"],{cwd:root,encoding:'utf8'})); - await start(path.join(temporary,'control'),external,sourceDsn); - const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--remote-debugging-port=0','--window-size=1440,950','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:'ignore',env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); - const port=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')),tabs=await(await fetch('http://127.0.0.1:'+port+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); - socket=new WebSocket(target.webSocketDebuggerUrl);await once(socket,'open');session=cdpSession(socket,{onEvent:event=>{if(event.method==='Runtime.exceptionThrown')errors.push(event.params.exceptionDetails);if(event.method==='Network.responseReceived'&&event.params.response.status>=400)httpErrors.push({url:event.params.response.url,status:event.params.response.status});}});send=session.send; - evaluate=async expression=>{const result=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; - await send('Page.enable');await send('Runtime.enable');await send('Network.enable');await navigate(); - await fill('#project-files-directory','alpha');await fill('#project-files-name','M4 original browser Ω');await activate('#project-files-new');await settled(); - const originalState=await state(),originalProject=originalState.project.project_id,source=path.join(external,'alpha'),sentinel=path.join(temporary,'EXECUTED'); - fs.mkdirSync(path.join(source,'scripts'));fs.mkdirSync(path.join(source,'notes')); - const first=Buffer.from('from pathlib import Path\nPath('+JSON.stringify(sentinel)+').write_text("executed")\n# authored Ω 山\n'),second=Buffer.concat([first,Buffer.from('# exact second decision 🌸\n')]); - fs.writeFileSync(path.join(source,'scripts/inert.py'),first);fs.writeFileSync(path.join(source,'notes/empty.md'),''); - await working('script','scripts/inert.py',first);await working('note','notes/empty.md',Buffer.alloc(0)); - fs.writeFileSync(path.join(source,'scripts/inert.py'),second);await inspectFile('scripts/inert.py'); - assert.equal(await evaluate('document.getElementById("project-working-text").value'),first.toString());assert.equal(await evaluate('document.getElementById("project-working-sha").value'),hash(second)); - await activate('#project-working-adopt-form button[type="submit"]');await settled();assert.equal(await evaluate('document.getElementById("project-working-text").value'),second.toString()); - await activate('#project-working-history');await settled();assert.equal(await evaluate('document.querySelectorAll("#project-working-history-list button").length'),2); - await activate('#project-working-history-list button');await settled();assert.equal(await evaluate('document.getElementById("project-working-text").value'),first.toString());assert.ok(!fs.existsSync(sentinel)); - check('Actual keyboard controls adopt script and empty note, review current changed digest and select exact older captured revision without execution'); - const draft=JSON.parse(execFileSync('python3',['-c',"import json,sys;sys.path.insert(0,'tests');from test_project_material_fork import recipe;v=recipe(sys.argv[1]);print(json.dumps({k:x for k,x in v.items() if k not in ('format','id','revision')}))",originalProject],{cwd:root,encoding:'utf8'})); - const retainedRecipe=await call('recipe_create',draft);await activate('#project-files-refresh');await settled(); - const sourceBefore=snapshot(source),sourceCheckpoint=(await state()).project.checkpoint; - await fill('#project-save-as-directory','independent');await fill('#project-save-as-name','M4 independent browser Ω');await activate('#project-files-save-as');await settled(); - const forkState=await state();assert.notEqual(forkState.project.project_id,originalProject);assert.notEqual(forkState.binding,originalState.binding);assert.deepEqual(snapshot(source),sourceBefore); - await activate('#project-fork-load');await settled();assert.match(await evaluate('document.getElementById("project-fork-summary").textContent'),new RegExp(sourceCheckpoint)); - const intent=await inspectIntent();assert.deepEqual(intent.current.definition,retainedRecipe);assert.equal(intent.fork_origin.project_id,originalProject);assert.equal(intent.current.definition.runtime.catalog_id,originalProject);assert.ok(intent.current.required_reselection.includes('runtime')); - await activate('#project-intent-capabilities');await settled();assert.match(await evaluate('document.getElementById("project-intents-list").textContent'),/Reselect/);assert.ok(!fs.existsSync(sentinel)); - check('Actual named Save As selects new identity; fork mapping and selected readonly original binding intent retain source parameters and require reselection'); - const fork=path.join(external,'independent');await activate('#project-files-close');await until(async()=>(await state()).state==='no_project_open','close before cold copy');await stop(server); - const coldRoot=path.join(temporary,'cold-external');fs.mkdirSync(coldRoot);const clone=path.join(coldRoot,'clone'),canonical=snapshot(fork); - const git=(...args)=>execFileSync('git',args,{env:gitEnv,stdio:['ignore','pipe','pipe']});git('init','-q','--template=',fork);git('-C',fork,'add','.');git('-C',fork,'-c','user.name=Disposable M4 QA','-c','user.email=qa@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','Exact independent M4 project');git('-c','core.hooksPath=/dev/null','clone','-q','--no-hardlinks','--template=',fork,clone);if(fs.existsSync(path.join(fork,'artifacts')))fs.cpSync(path.join(fork,'artifacts'),path.join(clone,'artifacts'),{recursive:true});assert.deepEqual(snapshot(clone),canonical); - await start(path.join(temporary,'cold-control'),coldRoot,coldDsn);await navigate();await fill('#project-files-directory','clone');await activate('#project-files-open');await settled();assert.deepEqual((await state()).project,forkState.project); - await inspectFile('scripts/inert.py');assert.equal(await evaluate('document.getElementById("project-working-text").value'),second.toString());await activate('#project-working-history');await settled();await activate('#project-working-history-list button');await settled();assert.equal(await evaluate('document.getElementById("project-working-text").value'),first.toString()); - assert.deepEqual(await inspectIntent(),intent);await capture(390);await capture(1440);assert.deepEqual(snapshot(clone),canonical);assert.ok(!fs.existsSync(sentinel));assert.deepEqual(errors,[]);assert.deepEqual(httpErrors,[]);verifySource();assert.equal(hash(fs.readFileSync(__filename)),testSha); - check('Cold UI on distinct role denied source CONNECT reconstructs exact fork, working-file history and unresolved intent; 390/1440 responsive and sentinel inert'); - fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({status:'pass',checks,material,test_sha256:testSha,isolation,bounds,project:forkState.project,source_project:originalProject,browser_runtime_errors:errors,http_errors:httpErrors,canonical_sha256:canonical,fixture:'Real named keyboard UI; explicitly authored inert recipe references; local Git metadata clone plus explicit untracked payload transfer; no runtime/preflight'},null,2)+'\n');console.log('Report '+path.join(report,'receipt.json')); -})().catch(async error=>{ - fs.writeFileSync(path.join(report,'failure-receipt.json'),JSON.stringify({status:'failed',error:error.message,checks,material,test_sha256:testSha,isolation,bounds,browser_runtime_errors:errors,http_errors:httpErrors},null,2)+'\n');console.error(error);console.error('Server:',serverErrors);if(evaluate)try{console.error('UI:',await evaluate('document.body.innerText'));}catch(_){}process.exitCode=1; -}).finally(async()=>{ - session?.close();socket?.close();for(const child of children.reverse())await stop(child); - for(const {control,dsn} of catalogs)try{execFileSync('python3',['-c',"import json,pathlib,os,psycopg;from psycopg import sql;root=pathlib.Path(os.environ['QA_CONTROL']);c=psycopg.connect(os.environ['TULDOK_DATABASE_URL'],autocommit=True);[(c.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier('projection_'+v['projection_id'] if v['version']==2 else 'catalog_'+v['dataset_id'])))) for p in root.rglob('.tuldok.json') for v in [json.loads(p.read_text())]];c.close()"],{cwd:root,env:{...process.env,TULDOK_DATABASE_URL:dsn,QA_CONTROL:control},stdio:['ignore','pipe','pipe']});}catch(error){console.error('Owned cleanup failed:',error.message);process.exitCode=1;} - process.env.TULDOK_DATABASE_URL=sourceDsn;fs.rmSync(temporary,{recursive:true,force:true}); -}); diff --git a/tests/browser_project_files_native.cjs b/tests/browser_project_files_native.cjs deleted file mode 100644 index b9a978af..00000000 --- a/tests/browser_project_files_native.cjs +++ /dev/null @@ -1,105 +0,0 @@ -'use strict'; -// Actual current native probe and zero-GO acknowledgement recovery; no model fit. -const assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); -const {spawn,execFileSync}=require('node:child_process'),{once}=require('node:events'),readline=require('node:readline'); -const {authenticatedFixture}=require('./auth_helpers.cjs'),{waitForDebugger}=require('./browser_startup.cjs'),{cdpSession}=require('./browser_cdp_session.cjs'),{qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); -const root=path.resolve(__dirname,'..'),hash=raw=>crypto.createHash('sha256').update(raw).digest('hex'); -assert.equal(process.env.TULDOK_NATIVE_PROJECT_GO,'1','Explicit native qualification GO required'); -assert.ok(process.env.TULDOK_DATABASE_URL&&process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL,'Disposable source and cold PostgreSQL configurations required'); -assert.ok(process.env.TULDOK_NATIVE_PUBLIC_FREEZE_MANIFEST,'Lead must freeze and qualify isolated M3 before actual browser acceptance'); -const freeze=JSON.parse(fs.readFileSync(process.env.TULDOK_NATIVE_PUBLIC_FREEZE_MANIFEST,'utf8')); -const material={manifest_sha256:hash(fs.readFileSync(process.env.TULDOK_NATIVE_PUBLIC_FREEZE_MANIFEST)),files:freeze.files}; -function verifySource(){for(const [file,expected] of Object.entries(material.files))assert.equal(hash(fs.readFileSync(path.join(root,file))),expected,'Frozen source changed: '+file);} -verifySource(); -const testSha=hash(fs.readFileSync(__filename)),temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-native-browser-')),report=qaDirectory(root,'project-native'); -const sourceDsn=process.env.TULDOK_DATABASE_URL,coldDsn=process.env.TULDOK_PROJECT_FILES_COLD_DATABASE_URL; -assert.notEqual(sourceDsn,coldDsn); -const children=[],catalogs=[],checks=[],errors=[],httpErrors=[],bounds=[],requests=[]; -const gitEnv={...process.env,GIT_CONFIG_NOSYSTEM:'1',GIT_CONFIG_GLOBAL:'/dev/null',GIT_TERMINAL_PROMPT:'0'}; -const routes=JSON.parse(execFileSync('python3',['-c',"import json,agent_api;names=agent_api.ProjectFilesOwnerClient.OPERATIONS;print(json.dumps({n.split('.')[1]:{**{k:agent_api.operation(n)[k] for k in ('path','method','query','side_effects')},'header':agent_api.selected_header(n)} for n in names}))"],{cwd:root,encoding:'utf8'})); -let server,origin,auth,socket,session,send,evaluate,serverErrors='',isolation,fixture,sourceFacts,coldFacts;const fixtures=[]; -const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); -async function until(fn,label){for(let i=0;i<600;i++){const result=await fn();if(result)return result;await pause(100);}throw Error('Timed out: '+label);} -function launch(command,args,options={}){const child=spawn(command,args,options);children.push(child);return child;} -async function stop(child){if(!child||child.exitCode!==null||child.signalCode!==null)return;const exited=once(child,'exit');child.kill('SIGTERM');await exited;} -async function start(directory,dsn,cold=false){ - process.env.TULDOK_DATABASE_URL=dsn;serverErrors='';let ready; - server=launch('python3',['-u','tests/test_project_native_clients_http.py','--serve','--root',directory,...(cold?['--cold']:[])],{cwd:root,env:{...process.env,TULDOK_DATABASE_URL:dsn},stdio:['pipe','pipe','pipe']}); - const pending=new Map(),lines=readline.createInterface({input:server.stdout}); - lines.on('line',line=>{let value;try{value=JSON.parse(line);}catch(_){serverErrors+='\nUnexpected stdout: '+line;return;}if(value.origin){ready=value;return;}const wait=pending.get(value.id);if(wait){pending.delete(value.id);value.error?wait.reject(Error(value.error)):wait.resolve(value.result);}}); - server.stderr.on('data',chunk=>serverErrors+=chunk); - await until(()=>{if(server.exitCode!==null)throw Error('Fixture app exited: '+serverErrors);return ready;},'actual fixture app startup'); - const process_=server; - const host={process:process_,ready,async command(action){const id=crypto.randomUUID();return new Promise((resolve,reject)=>{const timeout=setTimeout(()=>{pending.delete(id);reject(Error('Fixture command timed out: '+action));},30000);pending.set(id,{resolve:value=>{clearTimeout(timeout);resolve(value);},reject:error=>{clearTimeout(timeout);reject(error);}});process_.stdin.write(JSON.stringify({id,action})+'\n');});},async stop(){if(process_.exitCode!==null)return;const exited=once(process_,'exit');await host.command('stop');const [code]=await exited;assert.equal(code,0,'Fixture cleanup exit: '+serverErrors);}}; - fixtures.push(host);fixture=host;origin=ready.origin;auth=authenticatedFixture();await auth.register(origin,ready.control,root);return host; -} -async function state(){const response=await auth.fetch(origin+routes.status.path,{headers:{'Tuldok-Operation-Contract':routes.status.header}});const value=await response.json();assert.equal(response.status,200,JSON.stringify(value));return value;} -async function key(key,code=key,modifiers=0){for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Tab:9,a:65,' ':32}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});} -async function focus(selector){await until(()=>evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!e||e.disabled||!e.getClientRects().length)return false;e.scrollIntoView({block:'center'});e.focus();return document.activeElement===e;})()`),'focus '+selector);} -async function fill(selector,value){await focus(selector);await key('a','KeyA',2);await send('Input.insertText',{text:value});} -async function activate(selector){await focus(selector);await key('Enter');} -async function choose(selector,value){await focus(selector);await evaluate(`(()=>{const e=document.querySelector(${JSON.stringify(selector)});if(!Array.from(e.options).some(o=>o.value===${JSON.stringify(value)}))throw Error('Missing option');e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));e.dispatchEvent(new Event('input',{bubbles:true}));})()`);} -async function settled(){await until(async()=>{const failure=await evaluate('document.getElementById("project-files-error").textContent');if(failure)throw Error('UI error: '+failure);return evaluate('!document.getElementById("project-files-save").disabled');},'acknowledged UI state');} -async function navigate(){await auth.browser(send,origin);await send('Page.navigate',{url:origin+'/project-files'});await until(()=>evaluate('!!document.querySelector("#project-files-root option")&&!document.getElementById("project-files-new").disabled'),'negotiated UI startup');} -async function capture(width,label){ - await send('Emulation.setDeviceMetricsOverride',{width,height:950,deviceScaleFactor:1,mobile:false});await evaluate('document.getElementById("project-native-title").scrollIntoView({block:"start"})'); - await evaluate('new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))'); - const value=await evaluate('({viewport:innerWidth,width:document.documentElement.scrollWidth,client:document.documentElement.clientWidth,offenders:Array.from(document.querySelectorAll("body *")).filter(e=>e.getClientRects().length&&e.getBoundingClientRect().right>innerWidth+1).map(e=>({tag:e.tagName,id:e.id,width:e.getBoundingClientRect().width,right:e.getBoundingClientRect().right,text:e.textContent.slice(0,100)})).slice(-20)})'); - bounds.push(value);fs.writeFileSync(path.join(report,label+'-bounds-'+width+'.json'),JSON.stringify(value,null,2)+'\n');fs.writeFileSync(path.join(report,label+'-'+width+'.jpg'),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64')); - assert.ok(value.width<=value.viewport+1,'Responsive overflow: '+JSON.stringify(value)); -} -function snapshot(directory){const files={};function walk(at){for(const item of fs.readdirSync(at,{withFileTypes:true})){if(['.git','.tuldok-local'].includes(item.name))continue;const next=path.join(at,item.name);if(item.isDirectory())walk(next);else if(item.isFile())files[path.relative(directory,next)]=hash(fs.readFileSync(next));else throw Error('Unexpected special fixture file');}}walk(directory);return files;} -function check(value){checks.push(value);console.log('PASS '+value);} -(async()=>{ - assert.ok(process.env.TULDOK_TRAINING_PYTHON,'Current explicitly qualified native interpreter required'); - isolation=JSON.parse(execFileSync('python3',['-c',"import json,os,psycopg;from psycopg.conninfo import conninfo_to_dict;s=conninfo_to_dict(os.environ['TULDOK_DATABASE_URL']);c=psycopg.connect(os.environ['TULDOK_PROJECT_FILES_COLD_DATABASE_URL']);row=c.execute('SELECT current_database(),current_user,has_database_privilege(current_user,%s,%s)',(s['dbname'],'CONNECT')).fetchone();assert row[0]!=s['dbname'] and not row[2];print(json.dumps(dict(database=row[0],role=row[1],source_connect=row[2])));c.close()"],{cwd:root,encoding:'utf8'})); - const sourceHost=await start(path.join(temporary,'source-host'),sourceDsn); - const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--remote-debugging-port=0','--window-size=1440,950','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:'ignore',env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); - const port=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')),tabs=await(await fetch('http://127.0.0.1:'+port+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); - socket=new WebSocket(target.webSocketDebuggerUrl);await once(socket,'open');session=cdpSession(socket,{onEvent:event=>{if(event.method==='Runtime.exceptionThrown')errors.push(event.params.exceptionDetails);if(event.method==='Network.requestWillBeSent')requests.push({url:event.params.request.url,method:event.params.request.method,body:event.params.request.postData});if(event.method==='Network.responseReceived'&&event.params.response.status>=400)httpErrors.push({url:event.params.response.url,status:event.params.response.status});}});send=session.send; - evaluate=async expression=>{const result=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; - await send('Page.enable');await send('Runtime.enable');await send('Network.enable');await navigate(); - await fill('#project-files-directory','alpha');await fill('#project-files-name','Native keyboard CPU fixture Ω');await activate('#project-files-new');await settled(); - const project=(await state()).project.project_id,graph=await fixture.command('seed');assert.equal(graph.records,6);await activate('#project-files-refresh');await settled(); - const nativeRequests=()=>requests.filter(value=>value.url.includes('/execution/native/')); - assert.equal(nativeRequests().length,0);assert.equal((await fixture.command('facts')).go_events.length,0); - await activate('#project-native-graphs');await settled();assert.equal(await evaluate('document.querySelectorAll("#project-native-graph option").length'),1);assert.equal(nativeRequests().length,0); - await focus('#project-native-probe-confirm');await key(' ','Space');await activate('#project-native-preflight'); - await until(()=>evaluate('document.getElementById("project-files-status").textContent.includes("Native validation succeeded")'),'actual current CPU probe acknowledgement');await settled(); - let facts=await fixture.command('facts');assert.equal(facts.go_events.length,1);assert.equal(facts.workspace_runs,0);assert.ok(facts.go_events.every(event=>!event.in_transaction)); - assert.equal(await evaluate('document.getElementById("project-native-submit-fields").disabled'),false); - const before=nativeRequests().length;await activate('#project-native-submit');await evaluate('new Promise(resolve=>requestAnimationFrame(resolve))');assert.equal(nativeRequests().length,before,'Unchecked separate Start consent prevents dispatch'); - await capture(390,'validated');await capture(1440,'validated'); - check('Actual keyboard native preflight runs one current CPU probe; saved metadata loading and unchecked separate Start never dispatch a training Run'); - await fill('#project-native-run-id','9'.repeat(32));await focus('#project-native-retry-confirm');await key(' ','Space');await activate('#project-native-retry'); - await until(()=>evaluate('document.getElementById("project-files-error").textContent'),'typed current-only retry refusal');assert.match(await evaluate('document.getElementById("project-files-error").textContent'),/current local Run|current native Run/i); - facts=await fixture.command('facts');assert.equal(facts.go_events.length,1);assert.equal(facts.workspace_runs,0); - await activate('#project-files-discard');await activate('#project-files-refresh');await settled();await activate('#project-native-graphs');await settled(); - await fixture.command('install_fault');await focus('#project-native-probe-confirm');await key(' ','Space');await activate('#project-native-preflight'); - await until(()=>evaluate('document.getElementById("project-files-error").textContent'),'actual canonical/SQL acknowledgement fault');facts=await fixture.command('facts');assert.deepEqual(facts.fired,[true]);assert.equal(facts.go_events.length,1,'Failed GO intent SQL acknowledgement must send no second GO'); - await fixture.command('clear_fault');await activate('#project-files-refresh');await until(()=>evaluate('!document.getElementById("project-execution-health").disabled'),'recovery health current phase control');await activate('#project-execution-health'); - await until(()=>evaluate('document.querySelectorAll("#project-native-phases button").length===1'),'one original unresolved current phase ID'); - await evaluate(`(()=>{const button=Array.from(document.querySelectorAll('#project-native-phases button')).find(value=>/ · (ready|go_intent|unknown)$/.test(value.textContent));if(!button)throw Error('Missing original unresolved phase');button.dataset.qaSelected='yes';})()`); - await activate('#project-native-phases button[data-qa-selected="yes"]');const selectedPhase=await evaluate('document.getElementById("project-native-phase-id").value');assert.match(selectedPhase,/^[a-f0-9]{32}$/); - await activate('#project-native-reconcile');await until(()=>evaluate('document.getElementById("project-files-status").textContent.includes("does not start or resume")'),'original current phase receipt reconciliation'); - facts=await fixture.command('facts');assert.equal(facts.go_events.length,1);assert.equal(facts.pending,false);assert.equal(facts.available,true);assert.equal(facts.workspace_runs,0); - await activate('#project-files-discard');await activate('#project-files-refresh');await settled();sourceFacts=await fixture.command('facts'); - check('Named retry refuses absent current Run; real deferrable SQL fault after canonical GO intent sends zero GO, then keyboard health phase selection and explicit reconciliation restore availability without execution'); - // Edits reach the project folder on Save; Close keeps them only in the working copy. - await activate('#project-files-save');await settled(); - const source=path.join(sourceHost.ready.external,'alpha'),canonical=snapshot(source),saved=(await state()).project; - await activate('#project-files-close');await until(async()=>(await state()).state==='no_project_open','close before cold clone');await sourceHost.stop(); - const coldHostRoot=path.join(temporary,'cold-host'),external=path.join(coldHostRoot,'external');fs.mkdirSync(external,{recursive:true});const clone=path.join(external,'clone'); - const git=(...args)=>execFileSync('git',args,{env:gitEnv,stdio:['ignore','pipe','pipe']});git('init','-q','--template=',source);git('-C',source,'add','.');git('-C',source,'-c','user.name=Disposable native QA','-c','user.email=qa@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','Inert native probe science and exact failed phase');git('-c','core.hooksPath=/dev/null','clone','-q','--no-hardlinks','--template=',source,clone);if(fs.existsSync(path.join(source,'artifacts')))fs.cpSync(path.join(source,'artifacts'),path.join(clone,'artifacts'),{recursive:true});assert.deepEqual(snapshot(clone),canonical); - fs.rmSync(path.join(temporary,'source-host'),{recursive:true,force:true}); - const beforeCold=nativeRequests().length;await start(coldHostRoot,coldDsn,true);await navigate();await fill('#project-files-directory','clone');await activate('#project-files-open');await settled();assert.deepEqual((await state()).project,saved);assert.equal(nativeRequests().length,beforeCold); - await activate('#project-execution-health');await until(()=>evaluate('document.getElementById("project-execution-summary").textContent.includes("0 original current native phases")'),'cold phases remain historical');assert.equal(await evaluate('document.querySelectorAll("#project-native-phases button").length'),0); - await activate('#project-native-graphs');await settled();assert.equal(await evaluate('document.querySelectorAll("#project-native-graph option").length'),1);coldFacts=await fixture.command('facts');assert.equal(coldFacts.portable_phase_count,sourceFacts.portable_phase_count);assert.ok(coldFacts.portable_phase_count>=2);assert.deepEqual(coldFacts.go_events,[]);assert.deepEqual(coldFacts.forbidden,[]);assert.equal(coldFacts.workspace_runs,0);assert.deepEqual(snapshot(clone),canonical); - await capture(390,'cold');await capture(1440,'cold');assert.deepEqual(errors,[]);assert.equal(httpErrors.length,2,JSON.stringify(httpErrors));assert.ok(httpErrors.some(value=>value.status===409&&value.url.endsWith('/retry')));assert.ok(httpErrors.some(value=>value.status===503&&value.url.endsWith('/preflight')));verifySource();assert.equal(hash(fs.readFileSync(__filename)),testSha); - check('Cold keyboard Open on distinct role denied source CONNECT preserves canonical native facts but no local phases/Run owner; removed original host is not followed and 390/1440 layouts fit'); - fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({status:'pass',source_stable:true,checks,material,test_sha256:testSha,isolation,bounds,project_id:project,source_facts:sourceFacts,cold_facts:coldFacts,browser_runtime_errors:errors,expected_http_errors:httpErrors,native_requests:nativeRequests(),fixture:'Actual app handler/PG/Chromium and explicit qualified CPU interpreter; real source/release, one native probe, real deferred SQL ACK fault and no second GO; no model fit or training Run. Test-only stdin fixture commands seed authored data and wrap the actual before_commit hook; native owners unmocked. Cold host forbids process preparation/Runs construction as assertions.'},null,2)+'\n');console.log('Report '+path.join(report,'receipt.json')); -})().catch(async error=>{ - fs.writeFileSync(path.join(report,'failure-receipt.json'),JSON.stringify({status:'failed',error:error.message,checks,material,test_sha256:testSha,isolation,bounds,source_facts:sourceFacts,cold_facts:coldFacts,browser_runtime_errors:errors,http_errors:httpErrors,native_requests:requests.filter(value=>value.url.includes('/execution/native/'))},null,2)+'\n');console.error(error);console.error('Server:',serverErrors);if(evaluate)try{console.error('UI:',await evaluate('document.body.innerText'));}catch(_){}process.exitCode=1; -}).finally(async()=>{ - session?.close();socket?.close();for(const host of fixtures.reverse())try{await host.stop();}catch(error){console.error('Fixture cleanup failed:',error.message);process.exitCode=1;}for(const child of children.reverse())await stop(child);process.env.TULDOK_DATABASE_URL=sourceDsn;fs.rmSync(temporary,{recursive:true,force:true}); -}); diff --git a/tests/browser_project_workspace.cjs b/tests/browser_project_workspace.cjs new file mode 100644 index 00000000..9adf472a --- /dev/null +++ b/tests/browser_project_workspace.cjs @@ -0,0 +1,456 @@ +'use strict'; +// The project workspace (docs/plans/definition-first-projects/spec-w2.md, slices T18 and T19) in a headless +// browser: Tuldok started with --project-folder on a copy of the minimal fixture, or with --lab-root on a real +// project folder (opened in place, read only: opening imports it into the disposable database named by +// TULDOK_DATABASE_URL). The fixture copy also gets the buildable fixture's definitions and stand-in commands +// (which finish in a second and train nothing), and before Tuldok starts the command line builds some of them, +// lets one fail, imports per-case values, and an agent asks for a build that waits for approval: so the Train, +// Data and Analysis tabs have runs, products, a request and values to show. +// Signs in, opens every tab in light and dark at 1440 by 900, opens a model and its card, hovers a map node, uses +// the kind and text filters, the track, the status strip and Needs you; on Benchmarks picks a benchmark, hovers +// and focuses a point, sorts the table and makes a compare set of two that the address keeps; opens a run and a +// build request on Train, a dataset on Data, picks a metric on Analysis; and fails on any script error, failed +// request, sideways overflow or empty main area. The page is signed in with a credential that may only inspect, so +// it draws no control (spec-w3.md section 1); browser_project_workspace_actions.cjs checks the controls. Nothing may +// be asked of the project that changes it: its activity log gains nothing while the page is open and the fixture +// copy is unchanged byte for byte. +// node tests/browser_project_workspace.cjs [--lab-root ] +const assert = require('node:assert/strict'), fs = require('node:fs'), os = require('node:os'), path = require('node:path'), crypto = require('node:crypto'); +const { spawn, execFileSync } = require('node:child_process'), { once } = require('node:events'); +const { authenticatedFixture } = require('./auth_helpers.cjs'), { waitForDebugger } = require('./browser_startup.cjs'), { cdpSession } = require('./browser_cdp_session.cjs'), { qaDirectory, screenshotOptions } = require('./qa_artifacts.cjs'); + +const root = path.resolve(__dirname, '..'); +const argv = process.argv.slice(2), labAt = argv.indexOf('--lab-root'); +const labRoot = labAt >= 0 ? path.resolve(argv[labAt + 1] || '') : null; +if (labAt >= 0) assert.ok(argv[labAt + 1] && fs.existsSync(path.join(labRoot, 'tuldok.json')), '--lab-root names a project folder (it holds tuldok.json)'); +assert.ok(process.env.TULDOK_DATABASE_URL, 'TULDOK_DATABASE_URL must name a disposable PostgreSQL cluster'); +const venv = path.join(root, '.venv', 'bin'); +if (fs.existsSync(path.join(venv, 'python'))) process.env.PATH = venv + path.delimiter + process.env.PATH; // auth_helpers runs python3 +const python = fs.existsSync(path.join(venv, 'python')) ? path.join(venv, 'python') : 'python3'; +const WIDTH = 1440, HEIGHT = 900; +const TABS = ['map', 'data', 'train', 'benchmarks', 'analysis', 'overview']; + +const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'tuldok-workspace-browser-')); +const report = qaDirectory(root, labRoot ? 'project-workspace-lab' : 'project-workspace'); +const children = [], checks = [], errors = [], failedRequests = [], bounds = []; +let server, serverErrors = '', origin, auth, socket, session, send, evaluate; +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); +async function until(fn, label, tries = 300) { for (let i = 0; i < tries; i++) { const value = await fn(); if (value) return value; await pause(100); } throw Error('Timed out: ' + label); } +function check(text) { checks.push(text); console.log('PASS ' + text); } +function launch(command, args, options) { const child = spawn(command, args, options); children.push(child); return child; } +async function stop(child) { if (!child || child.exitCode !== null || child.signalCode !== null) return; const exited = once(child, 'exit'); child.kill('SIGTERM'); await Promise.race([exited, pause(10000)]); if (child.exitCode === null && child.signalCode === null) { child.kill('SIGKILL'); await exited; } } +function snapshot(directory) { + const files = {}; + (function walk(at) { for (const item of fs.readdirSync(at, { withFileTypes: true })) { const next = path.join(at, item.name); if (item.isDirectory()) walk(next); else files[path.relative(directory, next)] = item.isFile() ? crypto.createHash('sha256').update(fs.readFileSync(next)).digest('hex') : 'special'; } })(directory); + return files; +} +function findBrowser() { + const candidates = [process.env.BROWSER, '/usr/bin/chromium', '/usr/bin/chromium-browser', '/usr/bin/google-chrome']; + const cache = path.join(os.homedir(), '.cache', 'ms-playwright'); + if (fs.existsSync(cache)) for (const name of fs.readdirSync(cache).filter(n => n.startsWith('chromium_headless_shell-')).sort().reverse()) candidates.push(path.join(cache, name, 'chrome-headless-shell-linux64', 'chrome-headless-shell')); + const found = candidates.find(candidate => candidate && fs.existsSync(candidate)); + assert.ok(found, 'No headless browser: set BROWSER'); + return found; +} +/** Drops the project's schema and the catalog's from the disposable database, so each run imports afresh. */ +function dropOwned(folder, catalog) { + const id = JSON.parse(fs.readFileSync(path.join(folder, 'tuldok.json'), 'utf8')).id; + execFileSync(python, ['-c', `import json, os, pathlib, sys, psycopg +from psycopg import sql +from tuldok_project import store +store.drop_project(sys.argv[1], os.environ['TULDOK_DATABASE_URL']) +root = pathlib.Path(sys.argv[2]) +if root.exists(): + with psycopg.connect(os.environ['TULDOK_DATABASE_URL'], autocommit=True) as c: + for p in root.rglob('.tuldok.json'): + v = json.loads(p.read_text()) + name = 'projection_' + v['projection_id'] if v.get('version') == 2 else 'catalog_' + v['dataset_id'] + c.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(name))) +`, id, catalog], { cwd: root, stdio: ['ignore', 'pipe', 'pipe'] }); +} +let owned = null; +async function api(name, body = {}) { + const capabilities = await (await auth.fetch(origin + '/api/agent/capabilities')).json(); + const operation = capabilities.operations.find(item => item.id === 'project.' + name), contract = operation.contracts[operation.contracts.length - 1]; + const response = await auth.fetch(origin + '/api/project/' + name, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Tuldok-Operation-Contract': 'project.' + name + '/' + contract.version + '/' + contract.sha256 }, body: JSON.stringify(body) }); + const value = await response.json(); assert.equal(response.status, 200, JSON.stringify(value)); return value; +} + +async function shot(name) { + await evaluate('new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))'); + fs.writeFileSync(path.join(report, name + '.jpg'), Buffer.from((await send('Page.captureScreenshot', screenshotOptions)).data, 'base64')); +} +/** No sideways overflow of the page or of a non-map main area, and a main area that shows something. */ +async function inspect(label) { + const value = await evaluate(`(() => { + const main = document.getElementById('main'), top = document.querySelector('.top'), strip = document.getElementById('strip'); + const text = main.innerText.trim(); + return { label: ${JSON.stringify(label)}, viewport: innerWidth, page: document.documentElement.scrollWidth, body: document.body.scrollWidth, + main: main.classList.contains('map') ? null : [main.scrollWidth, main.clientWidth], top: [top.scrollWidth, top.clientWidth], strip: [strip.scrollWidth, strip.clientWidth], + text: text.length, children: main.children.length, reading: /^Reading/.test(text), + shifted: [document.body, document.querySelector('.stage')].some(e => e.scrollLeft || e.scrollTop) }; + })()`); + bounds.push(value); + assert.ok(value.page <= value.viewport && value.body <= value.viewport, 'Sideways overflow of the page: ' + JSON.stringify(value)); + assert.ok(!value.main || value.main[0] <= value.main[1] + 1, 'Sideways overflow of the main area: ' + JSON.stringify(value)); + assert.ok(value.top[0] <= value.top[1] + 1 && value.strip[0] <= value.strip[1] + 1, 'Sideways overflow of the top bar or strip: ' + JSON.stringify(value)); + assert.ok(value.text > 0 && value.children > 0 && !value.reading, 'Empty main area: ' + JSON.stringify(value)); + assert.ok(!value.shifted, 'The shell is scrolled: ' + JSON.stringify(value)); +} +async function open(theme, hash) { + await send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-color-scheme', value: theme }, { name: 'prefers-reduced-motion', value: 'reduce' }] }); + await send('Page.navigate', { url: origin + '/?theme=' + theme + '#' + hash }); + await until(() => evaluate(`document.readyState === 'complete' && !!document.getElementById('needs-btn') && !document.getElementById('needs-btn').disabled`), 'the project read (' + hash + ')'); +} +async function tab(name) { + await evaluate(`location.hash = '#${name}'`); + await until(() => evaluate(`(() => { const t = document.getElementById('main').innerText; return t.trim().length > 0 && !t.includes('Reading'); })()`), 'tab ' + name); +} +async function mouse(x, y, click = false) { + await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }); + if (click) { await send('Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 }); await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 }); } +} +async function centre(selector) { + const box = await evaluate(`(() => { const e = document.querySelector(${JSON.stringify(selector)}); if (!e) return null; e.scrollIntoView({ block: 'nearest', inline: 'nearest' }); const r = e.getBoundingClientRect(); return { x: r.left + r.width / 2, y: r.top + r.height / 2 }; })()`); + assert.ok(box, 'Missing ' + selector); + return box; +} +async function press(selector) { const { x, y } = await centre(selector); await mouse(x, y, true); } + +/** Waits until neither the main area nor the drawer is still reading. */ +async function settled(label) { + await until(() => evaluate(`(() => { const reading = e => [...e.querySelectorAll('.note')].some(n => n.textContent === 'Reading…'); return !reading(document.getElementById('main')) && !reading(document.getElementById('aside')) && !document.getElementById('main').hasAttribute('aria-busy'); })()`), label); + await pause(150); +} +async function choose(selector, value) { + await evaluate(`(() => { const s = document.querySelector(${JSON.stringify(selector)}); s.value = ${JSON.stringify(value)}; s.dispatchEvent(new Event('change')); })()`); + await settled('the choice ' + value); +} +async function tipShown() { return until(() => evaluate(`(() => { const t = [...document.querySelectorAll('#main .tip')].find(t => !t.hidden); return t && t.innerText; })()`), 'a chart tip'); } +async function drawerOpen() { await until(() => evaluate(`document.getElementById('drawer').classList.contains('open')`), 'the drawer'); await settled('the drawer filled'); assert.deepEqual(await evaluate(`[...document.querySelectorAll('#aside .failure')].map(e => e.textContent)`), [], 'The drawer reads without failure'); } +async function scrollTo(selector) { await evaluate(`(() => { const e = document.querySelector(${JSON.stringify(selector)}); const m = document.getElementById('main'); m.scrollTop += e.getBoundingClientRect().top - m.getBoundingClientRect().top - 8; })()`); } + +/** The Benchmarks, Train, Data and Analysis tabs, with what can be done on them, in both themes. */ +async function otherTabs(map) { + const overview = await api('results_overview'), charts = (await api('definitions', { kind: 'chart' })).definitions; + const chartBench = {}; + for (const c of charts) { const d = await api('definition', { reference: c.reference }); chartBench[c.reference] = { benchmark: String(d.document.benchmark).replace(/^benchmarks\//, ''), form: d.document.form }; } + // The benchmark with the most chart forms (the fixture's rollout has all four). + const forms = name => new Set(Object.values(chartBench).filter(c => c.benchmark === name).map(c => c.form)).size; + const bench = overview.benchmarks.slice().sort((a, b) => forms(b.benchmark) - forms(a.benchmark) || b.subjects_with_results - a.subjects_with_results)[0].benchmark; + const runs = (await api('runs')).runs, requests = (await api('build_requests')).requests, products = (await api('products')).products; + for (const theme of ['light', 'dark']) { + // Benchmarks: pick, hover and focus a point, sort, compare two, open one. + await open(theme, 'benchmarks'); await settled('benchmarks'); + if (theme === 'light') { // opened without a choice: a benchmark with results and an accuracy-against-time chart, if there is one + const has = (name, form) => Object.values(chartBench).some(c => c.benchmark === name && (!form || c.form === form)); + const withResults = overview.benchmarks.filter(b => b.subjects_with_results > 0); + const expected = (withResults.find(b => has(b.benchmark, 'tradeoff')) || withResults.find(b => has(b.benchmark)) || withResults[0] || overview.benchmarks[0]).benchmark; + const shown = await evaluate(`document.querySelector('#main select[data-key="benchmark"]').value`); + assert.equal(shown, expected, 'The tab opens on a benchmark with results and an accuracy-against-time chart'); + check('Benchmarks opens on ' + shown); + } + await choose('#main select[data-key="benchmark"]', bench); + const drawn = await evaluate(`({ charts: document.querySelectorAll('#main .bench-chart').length, svgs: document.querySelectorAll('#main .bench-chart svg').length, rows: document.querySelectorAll('#main .bench-all tbody tr').length, coverage: document.querySelector('#main .coverage').innerText, heads: [...document.querySelectorAll('#main .bench-chart .sect')].map(e => e.innerText.replace(/\\n/g, ' ')) })`); + assert.equal(drawn.charts, Object.values(chartBench).filter(c => c.benchmark === bench).length, 'One section per chart of ' + bench); + assert.ok(drawn.rows > 0, 'The all-subjects table has rows'); + await inspect(theme + ' benchmarks ' + bench); await shot(theme + '-benchmarks-picked'); + for (let i = 1; i < Math.min(drawn.charts, 4); i++) { await scrollTo(`#main .bench-chart:nth-of-type(${i + 1})`); await shot(theme + '-benchmarks-chart-' + (i + 1)); } + await evaluate(`document.getElementById('main').scrollTop = 0`); + if (drawn.svgs) { + const hit = await evaluate(`(() => { const e = document.querySelector('#main .bench-chart .hit'); return e.getAttribute('aria-label'); })()`); + const { x, y } = await centre('#main .bench-chart .hit'); await mouse(x, y); + const tip = await tipShown(); assert.ok(tip.length > 0, 'The hovered point has a tip'); + await shot(theme + '-benchmarks-hover'); + await mouse(5, 5); + await evaluate(`document.querySelectorAll('#main .bench-chart .hit')[1]?.focus() ?? document.querySelector('#main .bench-chart .hit').focus()`); + const focused = await tipShown(); assert.ok(focused.length > 0, 'A focused point has a tip'); + await evaluate(`document.activeElement.blur()`); + check(theme + ' Benchmarks ' + bench + ': ' + drawn.charts + ' charts (' + drawn.heads.join(' | ') + '); "' + drawn.coverage.replace(/\n/g, ' ') + '"; hovered "' + hit + '" and focused a point, each with a tip'); + } else check(theme + ' Benchmarks ' + bench + ': no chart drawn; "' + drawn.coverage.replace(/\n/g, ' ') + '"'); + // Sort by subject, then the other way. + await press('#main .bench-all th button[data-key="sort-full"]'); await settled('sorted'); + const names = async () => evaluate(`[...document.querySelectorAll('#main .bench-all tbody td:nth-child(3)')].map(e => e.textContent)`); + const up = await names(); assert.deepEqual(up, up.slice().sort((a, b) => a.localeCompare(b)), 'Sorted by subject'); + assert.equal(await evaluate(`document.querySelector('#main .bench-all th button[data-key="sort-full"]').getAttribute('aria-sort')`), 'ascending'); + await press('#main .bench-all th button[data-key="sort-full"]'); await settled('sorted back'); + assert.deepEqual(await names(), up.slice().reverse(), 'Sorted by subject, descending'); + await scrollTo('#main .bench-all'); await shot(theme + '-benchmarks-sorted'); + // A compare set of two, kept by the address. + await press('#main .bench-all tbody tr:nth-child(1) input[type="checkbox"]'); await settled('compare one'); + await press('#main .bench-all tbody tr:nth-child(2) input[type="checkbox"]'); await settled('compare two'); + const cmp = await evaluate(`({ chips: document.querySelectorAll('#main .bench-cmp .cmp-chips .c').length, cols: document.querySelectorAll('#main .bench-cmp thead th').length, search: location.search })`); + assert.equal(cmp.chips, 2); assert.equal(cmp.cols, 3); assert.match(cmp.search, /[?&]cmp=[^&,]+,[^&,]+/); assert.match(cmp.search, new RegExp('[?&]benchmark=' + bench)); + await scrollTo('#main .bench-cmp'); await inspect(theme + ' benchmarks compare'); await shot(theme + '-benchmarks-compare'); + // Open a subject in the drawer. + await press('#main .bench-all tbody tr:nth-child(1) .linkish'); await drawerOpen(); + const opened = await evaluate(`({ title: document.querySelector('#aside h2').textContent, compare: !!document.querySelector('#aside .bench-acts'), hash: location.hash })`); + assert.ok(opened.compare, 'The drawer offers Compare'); + await inspect(theme + ' benchmarks drawer'); await shot(theme + '-benchmarks-drawer'); + // The address alone gives the same compare set. + const shared = origin + '/' + cmp.search + '#benchmarks'; + await send('Page.navigate', { url: shared }); await until(() => evaluate(`!!document.querySelector('#main .bench-cmp')`), 'the shared compare set'); + assert.equal(await evaluate(`document.querySelectorAll('#main .bench-cmp .cmp-chips .c').length`), 2); + // The table view of the charts. + await settled('shared'); if (drawn.svgs) { await press('#main .filters button[data-key="tables"]'); await settled('tables'); assert.ok(await evaluate(`document.querySelectorAll('#main table.pts').length > 0`)); await scrollTo('#main table.pts'); await shot(theme + '-benchmarks-tables'); await inspect(theme + ' benchmarks tables'); await press('#main .filters button[data-key="tables"]'); } + check(theme + ' Benchmarks: table sorted both ways, a compare set of two (' + cmp.search + ') that the address restores, ' + opened.title + ' opened in the drawer (' + opened.hash + ')'); + + // Train: open a run and a build request. + await open(theme, 'train'); await settled('train'); + const train = await evaluate(`({ runs: document.querySelectorAll('#main .row.run').length, requests: document.querySelectorAll('#main .row.request').length, todo: document.querySelectorAll('#main .row.todo').length, envs: document.querySelectorAll('#main .row.env').length, text: document.getElementById('main').innerText })`); + assert.equal(train.runs, Math.min(100, runs.length)); assert.equal(train.requests, requests.length); + let trained = theme + ' Train: ' + train.todo + ' to build, ' + train.requests + ' build requests, ' + train.runs + ' runs, ' + train.envs + ' environments'; + if (runs.length) { + const run = runs.find(r => r.state === 'failed') || runs[0]; + await press(`#main .row.run[data-run="${run.id}"]`); await drawerOpen(); + const shown = await evaluate(`({ blocks: [...document.querySelectorAll('#aside h3')].map(e => e.textContent), command: [...document.querySelectorAll('#aside .setup dt')].some(e => e.textContent === 'Command'), logs: [...document.querySelectorAll('#aside pre.evtext')].map(e => e.textContent.length) })`); + assert.ok(shown.command, 'The run shows its command'); assert.ok(shown.blocks.some(b => b.startsWith('Standard error')), 'The run shows its logs'); + await inspect(theme + ' train run'); await shot(theme + '-train-run'); + trained += '; run ' + run.id.slice(0, 8) + ' (' + run.state + ') opened: ' + shown.blocks.join(', '); + } + if (requests.length) { + await press('#main .row.request'); await drawerOpen(); + const req = await evaluate(`({ title: document.querySelector('#aside h2').textContent, buttons: [...document.querySelectorAll('#aside button')].map(b => b.textContent) })`); + assert.ok(!req.buttons.some(t => /approve|decline|cancel/i.test(t)), 'No control that changes the project: ' + req.buttons.join(', ')); + await shot(theme + '-train-request'); + trained += '; request opened: ' + req.title; + } else await shot(theme + '-train-empty'); + check(trained); + + // Data: open a dataset (one with files when there is one). + await open(theme, 'data'); await settled('data'); + const withFiles = products.find(p => p.reference.startsWith('datasets/')); + const pick = withFiles ? withFiles.reference : await evaluate(`document.querySelector('#main .ds-card').dataset.ref`); + await press(`#main .ds-card[data-ref="${pick}"]`); await drawerOpen(); + const ds = await evaluate(`({ cards: document.querySelectorAll('#main .ds-card').length, blocks: [...document.querySelectorAll('#aside h3')].map(e => e.textContent), files: [...document.querySelectorAll('#aside .setup dd.mono')].map(e => e.textContent).filter(t => t.includes('/')).length })`); + assert.deepEqual(ds.blocks.slice(0, 4), ['Description', 'Definition', 'Files', 'Used by']); + if (withFiles) assert.ok(ds.files > 0, 'The dataset shows its files\' paths'); + await inspect(theme + ' data drawer'); await shot(theme + '-data-dataset'); + check(theme + ' Data: ' + ds.cards + ' cards; ' + pick + ' opened with ' + ds.files + ' paths'); + + // Analysis: pick a metric; hover a case when there are values. + await open(theme, 'analysis'); await settled('analysis'); + const metrics = await evaluate(`[...document.querySelectorAll('#main select[data-key="ana-metric"] option')].map(o => o.value)`); + if (metrics.length > 1) await choose('#main select[data-key="ana-metric"]', metrics[1]); + const ana = await evaluate(`({ marks: document.querySelectorAll('#main .cases .b').length, rows: document.querySelectorAll('#main table.cases-t tbody tr').length, none: document.querySelector('#main .ana-none')?.innerText || '' })`); + if (ana.marks) { + assert.equal(ana.marks, ana.rows); + const { x, y } = await centre('#main .cases .hit'); await mouse(x, y); const tip = await tipShown(); + await inspect(theme + ' analysis'); await shot(theme + '-analysis-metric'); + check(theme + ' Analysis: metric ' + (metrics[1] || metrics[0]) + ', ' + ana.marks + ' cases, a hovered case reads "' + tip.replace(/\n/g, ' ') + '"'); + } else { + assert.ok(ana.none.length > 0, 'Without per-case values a sentence says so'); + await inspect(theme + ' analysis'); await shot(theme + '-analysis-metric'); + check(theme + ' Analysis: no per-case values: "' + ana.none + '"'); + } + } +} + +(async () => { + let folder; + if (labRoot) folder = labRoot; + else { + fs.cpSync(path.join(root, 'tests', 'fixtures', 'tuldok_project', 'minimal'), path.join(temporary, 'fixture'), { recursive: true }); + folder = path.join(temporary, 'fixture', 'project'); + // One more model with no group, no kind of change, no training date and no result, to show each absence. + fs.writeFileSync(path.join(folder, 'recipes', 'loose.json'), JSON.stringify({ format: 'tuldok_definition_v1', kind: 'recipe', title: 'A recipe with nothing set', description: 'Neither grouped, nor kinded, nor dated, nor measured.', earlier_work: { rebuild: 'no', reason: 'A browser check fixture.' }, model: 'models/stepper' }, null, 2) + '\n'); + // The buildable fixture's definitions and stand-in commands, beside the minimal fixture's (no name is shared). + const buildable = path.join(root, 'tests', 'fixtures', 'tuldok_project', 'buildable'); + for (const kind of ['environments', 'datasets', 'recipes', 'benchmarks', 'charts']) for (const name of fs.readdirSync(path.join(buildable, 'project', kind))) fs.copyFileSync(path.join(buildable, 'project', kind, name), path.join(folder, kind, name)); + for (const name of fs.readdirSync(path.join(buildable, 'code'))) fs.cpSync(path.join(buildable, 'code', name), path.join(temporary, 'fixture', 'code', name), { recursive: true }); + const git = ['-c', 'user.name=Tuldok Test', '-c', 'user.email=test@example.invalid', '-c', 'commit.gpgsign=false', '-c', 'init.defaultBranch=main', '-C', path.join(temporary, 'fixture')]; + execFileSync('git', [...git, 'init', '-q']); execFileSync('git', [...git, 'add', '-A']); execFileSync('git', [...git, 'commit', '-q', '-m', 'The fixture.']); + } + const home = path.join(temporary, 'home'), catalog = path.join(temporary, 'catalog'); fs.mkdirSync(home); + owned = [folder, catalog]; dropOwned(folder, catalog); + if (!labRoot) { // runs, products, a failed run, per-case values and a request awaiting approval, before Tuldok starts + const cli = (...args) => execFileSync(python, ['-m', 'tuldok_project', ...args], { cwd: root, env: { ...process.env, TULDOK_PROJECT_HOME: home }, stdio: ['ignore', 'pipe', 'pipe'] }).toString(); + cli('connect', folder, 'test-python', python); + cli('build', folder, 'datasets/numbers', 'recipes/fit-a', 'benchmarks/score', '--yes'); + try { cli('build', folder, 'recipes/fit-b', '--yes', '--env', 'BUILDABLE_FAIL=train'); assert.fail('The build of recipes/fit-b was to fail'); } catch (error) { if (error.code === 'ERR_ASSERTION') throw error; } + cli('import-values', folder, ...fs.readdirSync(path.join(temporary, 'fixture', 'values')).map(name => path.join(temporary, 'fixture', 'values', name))); + execFileSync(python, ['-c', `import sys +from tuldok_project import service +agent = {'actor_id': 'agent-browser-check', 'kind': 'agent', 'label': 'A browser check agent', 'scopes': ['inspect', 'execute']} +assert service.Service(sys.argv[1]).call('request_build', {'targets': ['recipes/fit-b']}, agent)['state'] == 'awaiting-approval' +`, folder], { cwd: root, env: { ...process.env, TULDOK_PROJECT_HOME: home }, stdio: ['ignore', 'pipe', 'pipe'] }); + } + const before = labRoot ? null : snapshot(folder); + let output = ''; + server = launch(python, ['-u', 'app.py', '--port', '0', '--data', catalog, '--project-folder', folder], { cwd: root, env: { ...process.env, TULDOK_PROJECT_HOME: home }, stdio: ['ignore', 'pipe', 'pipe'] }); + server.stdout.on('data', chunk => { output += chunk; }); server.stderr.on('data', chunk => { serverErrors += chunk; }); + const port = await until(() => { if (server.exitCode !== null) throw Error('Tuldok exited: ' + serverErrors); return output.match(/127\.0\.0\.1:(\d+)/)?.[1]; }, 'Tuldok start-up', 1800); + origin = 'http://127.0.0.1:' + port; + auth = authenticatedFixture(); await auth.register(origin, catalog, root); + const map = await api('map'), summary = await api('summary'), activityBefore = (await api('activity')).total; + check('Tuldok serves ' + summary.name + ' with ' + map.models.length + ' models, ' + map.groups.length + ' groups, ' + map.edges.filter(e => e.kind === 'derived').length + ' derivations, ' + map.comparisons.length + ' comparisons and ' + map.models.filter(m => m.value != null).length + ' track values'); + + const browser = launch(findBrowser(), ['--headless', '--no-sandbox', '--disable-gpu', '--no-first-run', '--hide-scrollbars', '--remote-debugging-port=0', `--window-size=${WIDTH},${HEIGHT}`, '--user-data-dir=' + path.join(temporary, 'browser'), 'about:blank'], { stdio: 'ignore', env: { ...process.env, XDG_CONFIG_HOME: temporary, XDG_CACHE_HOME: temporary } }); + const debugPort = await waitForDebugger(browser, path.join(temporary, 'browser', 'DevToolsActivePort')); + const target = (await (await fetch('http://127.0.0.1:' + debugPort + '/json')).json()).find(item => item.type === 'page'); + socket = new WebSocket(target.webSocketDebuggerUrl); await once(socket, 'open'); + session = cdpSession(socket, { timeoutMs: 60000, onEvent: event => { + if (event.method === 'Runtime.exceptionThrown') errors.push(event.params.exceptionDetails.exception?.description || event.params.exceptionDetails.text); + if (event.method === 'Runtime.consoleAPICalled' && event.params.type === 'error') errors.push(event.params.args.map(a => a.value ?? a.description).join(' ')); + if (event.method === 'Log.entryAdded' && event.params.entry.level === 'error') errors.push(event.params.entry.text + ' ' + (event.params.entry.url || '')); + if (event.method === 'Network.loadingFailed' && !event.params.canceled) failedRequests.push({ id: event.params.requestId, error: event.params.errorText }); + if (event.method === 'Network.responseReceived' && event.params.response.status >= 400) failedRequests.push({ url: event.params.response.url, status: event.params.response.status }); + } }); + send = session.send; + evaluate = async expression => { const result = await send('Runtime.evaluate', { expression, awaitPromise: true, returnByValue: true }); if (result.exceptionDetails) throw Error(JSON.stringify(result.exceptionDetails)); return result.result.value; }; + await send('Page.enable'); await send('Runtime.enable'); await send('Network.enable'); await send('Log.enable'); + await send('Emulation.setDeviceMetricsOverride', { width: WIDTH, height: HEIGHT, deviceScaleFactor: 1, mobile: false }); + // The page is signed in with a credential that may only inspect: it draws no control that changes the project + // (spec-w3.md section 1), so this check sees the page of phase 2. The check's own calls stay the owner's. + const issued = await auth.fetch(origin + '/api/access/credentials', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ label: 'A read-only browser check', scopes: ['inspect'], ttl_seconds: 3600 }) }); + const inspector = await issued.json(); assert.equal(issued.status, 201, JSON.stringify(inspector)); + assert.notEqual((await send('Network.setCookie', { name: 'tuldok_session', value: inspector.token, url: origin + '/', httpOnly: true, sameSite: 'Strict', path: '/' })).success, false); + + for (const theme of ['light', 'dark']) { + await open(theme, 'map'); + assert.equal(await evaluate(`getComputedStyle(document.documentElement).colorScheme`), theme); + for (const name of TABS) { + await tab(name); + await inspect(theme + ' ' + name); + assert.doesNotMatch(await evaluate(`document.getElementById('main').innerText`), /is not built yet/); + await shot(theme + '-' + name); + if (name === 'map') { // the rest of a wide map, and the end of the list + const wide = await evaluate(`(() => { const m = document.querySelector('.tree-map'); return m.scrollWidth > m.clientWidth + 10; })()`); + if (wide) { + await evaluate(`(() => { const m = document.querySelector('.tree-map'); m.scrollLeft = (m.scrollWidth - m.clientWidth) / 2; })()`); await shot(theme + '-map-middle'); + await evaluate(`(() => { const m = document.querySelector('.tree-map'); m.scrollLeft = m.scrollWidth; document.querySelector('.tree-host').scrollTop = 1e7; })()`); await shot(theme + '-map-end'); + await evaluate(`(() => { document.querySelector('.tree-map').scrollLeft = 0; document.querySelector('.tree-host').scrollTop = 0; })()`); + } + } + } + check('Every tab in ' + theme + ' at ' + WIDTH + ' by ' + HEIGHT + ': no overflow, no empty main area'); + } + const fonts = await evaluate(`document.fonts.ready.then(() => [...document.fonts].filter(f => f.status === 'loaded').map(f => f.family + ' ' + f.style))`); + assert.ok(fonts.some(f => /Ubuntu Sans"? normal/.test(f)) && fonts.some(f => /Ubuntu Sans Mono"? normal/.test(f)), 'The vendored faces load: ' + JSON.stringify(fonts)); + check('The vendored Ubuntu Sans and Ubuntu Sans Mono load: ' + fonts.join(', ')); + + for (const theme of ['light', 'dark']) { + await open(theme, 'map'); + const shown = await evaluate(`({ sections: [...document.querySelectorAll('.tree-map .bandlabel')].map(t => t.firstChild.textContent), nodes: document.querySelectorAll('.tree-map .node').length, rows: document.querySelectorAll('.tree-list .trow').length, lines: document.querySelectorAll('.tree-map .der').length, + unset: document.querySelectorAll('.tree-map .node.unset').length, unsetChips: [...document.querySelectorAll('.tree-list .chip.unset')].map(c => c.textContent), noValue: document.querySelectorAll('.tree-list .mbar.none').length })`); + assert.equal(shown.nodes, map.models.length); assert.equal(shown.rows, map.models.length); + // Each absence is shown: a section for models without a group, a hollow mark named for those without a kind, an empty bar without a value. + assert.equal(shown.sections.includes('Not in a group'), map.models.some(m => !m.group)); + assert.equal(shown.unset, map.models.filter(m => !m.change).length); assert.ok(shown.unsetChips.every(t => t === 'Kind not set')); + assert.equal(shown.noValue, map.models.filter(m => m.value == null).length); + // Hover a node with a track value: its tip names it. + const hovered = map.models.find(m => m.value != null && m.short) || map.models[0]; + const { x, y } = await centre(`.tree-map .node[data-id="${hovered.reference}"]`); + await mouse(x, y); + const tip = await until(() => evaluate(`(() => { const t = document.querySelector('#main > .tip'); return t && !t.hidden && t.innerText; })()`), 'the hover tip'); + assert.ok(tip.includes(hovered.title), 'The tip names the hovered model: ' + tip); + await shot(theme + '-map-hover'); + // Open a model with a card in the drawer. + const opened = map.models.find(m => m.card && map.comparisons.some(c => c.from === m.reference || c.to === m.reference)) || map.models.find(m => m.card) || map.models[0]; + await press(`.tree-map .node[data-id="${opened.reference}"]`); + await until(() => evaluate(`document.getElementById('drawer').classList.contains('open') && ![...document.querySelectorAll('#aside .note')].some(e => e.textContent === 'Reading…')`), 'the drawer filled'); + const drawer = await evaluate(`({ title: document.querySelector('#aside h2').textContent, card: document.querySelector('#aside .card-md') ? document.querySelector('#aside .card-md').children.length : 0, arcs: document.querySelectorAll('.tree-map .cmp').length, failures: [...document.querySelectorAll('#aside .failure')].map(e => e.textContent), hash: location.hash })`); + assert.equal(drawer.title, opened.title); assert.deepEqual(drawer.failures, []); + if (opened.card) assert.ok(drawer.card > 0, 'The card is rendered'); + assert.equal(drawer.hash, '#map/' + encodeURIComponent(opened.reference)); + await evaluate(`document.getElementById('drawer').scrollTop = 0`); + await shot(theme + '-map-drawer'); + await evaluate(`(() => { const c = document.querySelector('#aside .card-md'); if (c) c.scrollIntoView({ block: 'start' }); })()`); + await shot(theme + '-map-card'); + // The track. + if (map.track) { + await press('#main .filters button[title]'); + await until(() => evaluate(`document.querySelectorAll('.tree-map .tpt').length`), 'the track'); + assert.equal(await evaluate(`document.querySelectorAll('.tree-map .tpt').length`), map.models.filter(m => m.value != null).length); + await evaluate(`document.querySelector('.tree-map').scrollTop = 1e6`); + await shot(theme + '-map-track'); + await press('#main .filters button[title]'); + } + await press('#drawer .close'); + await until(() => evaluate(`!document.getElementById('drawer').classList.contains('open')`), 'the drawer closed'); + check(theme + ': ' + shown.sections.length + ' sections, every model on the map and in the list, a hover tip, ' + opened.reference + ' opened with its card (' + drawer.card + ' blocks, ' + drawer.arcs + ' comparison arcs)' + (map.track ? ', the track drawn' : '')); + } + + // The card renderer builds elements only, whatever the text holds. + await open('light', 'map'); + const sample = ['# Title ', '', 'Some **bold**, *italic*, `code` and [a link](https://example.invalid/x) not bold.', '', + '- one', ' - nested *two*', '- three', '', '1. first', '2. second', '', '| A | B |', '| --- | ---: |', '| `x|y` | 2 |', '', + '```js', 'const a = "') + self.write('box.zip', zip_bytes({'in.wav': wav16([1, 2, 3])}, zipfile.ZIP_DEFLATED)) + for address, media, disposition in (('pic.png', 'image/png', 'inline'), + ('x.svg', 'application/octet-stream', 'attachment'), + ('x.html', 'application/octet-stream', 'attachment'), + ('box.zip!in.wav', 'audio/wav', 'inline')): + with self.subTest(address): + answer = inside.file_response(self.product, address) + headers = dict(answer.headers) + self.assertEqual(headers['Content-Type'], media) + self.assertTrue(headers['Content-Disposition'].startswith(disposition)) + self.assertEqual(b''.join(bytes(chunk) for chunk in answer.chunks), inside_archive.locate( + self.root, address).blob().view.tobytes()) + answer = inside.file_response(self.product, 'pic.png', 'bytes=2-5') + self.assertEqual((answer.status, dict(answer.headers)['Content-Range'], b''.join(answer.chunks)), + (206, f'bytes 2-5/{len(png(2, 2))}', png(2, 2)[2:6])) + self.assertEqual(inside.file_response(self.product, 'pic.png', 'bytes=1-2,4-5').status, 200) + + +# ------------------------------------------------------------ through the service + +class ProductCase(builds_base.BuildCase): + """BuildCase with an adopted product for datasets/numbers, whose output directory the tests fill.""" + + def setUp(self): + super().setUp() + patcher = mock.patch.dict(os.environ, {local.SETTINGS_VARIABLE: str(self.temporary / 'settings.json')}) + patcher.start() + self.addCleanup(patcher.stop) + inside_archive.forget_members() + inside.forget_indexes() + source = self.temporary / 'numbers.json' + source.write_text('{"numbers": [1, 2]}', encoding='utf-8') + manifest = {'format': 'tuldok_adoption_v1', + 'products': [{'reference': 'datasets/numbers', 'files': {'numbers.json': str(source)}}]} + with self.opened() as project: + done = adoption.adopt(project, adoption.make_plan(project, manifest, copy=True)) + self.run_id = done['entries'][0]['run_id'] + self.output = Path(done['entries'][0]['directory']) + + def write(self, address, data): + path = self.output / address + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data if isinstance(data, bytes) else data.encode()) + + +@unittest.skipUnless(builds_base.DATABASE, 'TULDOK_DATABASE_URL is not set; database tests need a disposable cluster.') +class Operations(ProductCase): + + def setUp(self): + super().setUp() + self.service = Service(self.folder) + self.addCleanup(self.service.close) + + def call(self, name, **arguments): + result = self.service.call(name, arguments, OWNER) + json.dumps(result, allow_nan=False) + return result + + def test_the_five_operations_on_an_adopted_product(self): + self.write('field.npy', npy((4, 6), '') + self.write('x.html', '') + token = self.agent['token'] + for address, body, media, disposition in (('p.png', picture, 'image/png', 'inline'), + ('s.wav', sound, 'audio/wav', 'inline'), + ('x.svg', None, 'application/octet-stream', 'attachment'), + ('x.html', None, 'application/octet-stream', 'attachment')): + with self.subTest(address): + status, headers, data = self.get(address, token) + self.assertEqual((status, headers['Content-Type']), (200, media)) + self.assertTrue(headers['Content-Disposition'].startswith(disposition)) + self.assertEqual((headers['X-Content-Type-Options'], headers['Cache-Control']), ('nosniff', 'no-store')) + if body is not None: + self.assertEqual(data, body) + status, headers, data = self.get('s.wav', token, {'Range': 'bytes=4-9'}) + self.assertEqual((status, data, headers['Content-Range']), (206, sound[4:10], f'bytes 4-9/{len(sound)}')) + status, _, data = self.get('p.png') + self.assertEqual(status, 401, data) + status, _, data = self.get('p.png', 'not-a-token') + self.assertIn(status, (401, 403)) + status, _, data = self.get('../p.png', token) + self.assertEqual((status, json.loads(data)['code']), (400, 'invalid')) + status, _, data = self.get('missing.png', token) + self.assertEqual((status, json.loads(data)['code']), (404, 'not-found')) + request = Request(f'{self.root}/api/project/file?run={self.run_id}', headers={'Authorization': 'Bearer ' + token}) + with self.assertRaises(HTTPError) as caught: + urlopen(request, timeout=30) + self.assertEqual(caught.exception.code, 400) + request = Request(f'{self.root}/api/samples', headers={'Authorization': 'Bearer ' + token}) + with self.assertRaises(HTTPError) as caught: + urlopen(request, timeout=30) + self.assertEqual(caught.exception.code, 409) # every other /api/ GET is still refused + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_launcher.py b/tests/test_tuldok_project_launcher.py new file mode 100644 index 00000000..7fe6cec6 --- /dev/null +++ b/tests/test_tuldok_project_launcher.py @@ -0,0 +1,588 @@ +"""The launcher (spec-m6.md section 3): run, strays, stop_strays and progress. + +Stand-in commands are small Python scripts run by the test interpreter under +the real custodian process. The module's intervals are shortened (run() hands +them to the custodian) so the file runs in seconds. Every test +kills whatever carries its run id when it ends, pass or fail, using its own +/proc scan rather than the code under test, and the run tests check that the +test process is left with no zombie child. +""" +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import textwrap +import threading +import time +import unittest +from unittest import mock +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from tuldok_project import launcher # noqa: E402 + +FAST = {'WATCH_INTERVAL': 0.02, 'DISK_INTERVAL': 0.1, 'STOP_GRACE': 0.6, + 'SWEEP_LIMIT': 3.0, 'SWEEP_PAUSE': 0.02} +GENEROUS = {'wall_seconds': 30, 'disk_bytes': 10 ** 9} + + +def _carriers(run_id): + """Pids of live processes with exactly TULDOK_RUN_ID=, found independently of launcher.""" + needle = f'TULDOK_RUN_ID={run_id}'.encode() + found = [] + for name in os.listdir('/proc'): + if not name.isdigit() or int(name) == os.getpid(): + continue + try: + with open(f'/proc/{name}/stat', 'rb') as file: + state = file.read().rsplit(b')', 1)[1].split()[0] + with open(f'/proc/{name}/environ', 'rb') as file: + environ = file.read().split(b'\0') + except OSError: + continue + if state != b'Z' and needle in environ: + found.append(int(name)) + return found + + +def _kill_run(run_id): + for _ in range(100): + pids = _carriers(run_id) + if not pids: + return + for pid in pids: + try: + os.kill(pid, signal.SIGKILL) + except OSError: + pass + time.sleep(0.02) + + +def _children(*, zombies=False): + found = [] + for name in os.listdir('/proc'): + if not name.isdigit(): + continue + try: + with open(f'/proc/{name}/stat', 'rb') as file: + fields = file.read().rsplit(b')', 1)[1].split() + except OSError: + continue + if int(fields[1]) == os.getpid() and (fields[0] == b'Z') == zombies: + found.append(int(name)) + return found + + +def _zombie_children(): + return _children(zombies=True) + + +def _alive(pid): + try: + with open(f'/proc/{pid}/stat', 'rb') as file: + return file.read().rsplit(b')', 1)[1].split()[0] not in (b'Z', b'X') + except OSError: + return False + + +class LauncherCase(unittest.TestCase): + def setUp(self): + patcher = mock.patch.multiple(launcher, **FAST) + patcher.start() + self.addCleanup(patcher.stop) + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + self.scratch = Path(scratch.name) + self.directory = self.scratch / 'run' + self.directory.mkdir() + self.run_id = 'test-' + uuid.uuid4().hex + self.addCleanup(_kill_run, self.run_id) + self.handlers = (signal.getsignal(signal.SIGINT), signal.getsignal(signal.SIGTERM)) + + def script(self, body): + path = self.scratch / f'command-{uuid.uuid4().hex[:8]}.py' + path.write_text(textwrap.dedent(body)) + return path + + def env(self): + return {'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'LANG': 'C.UTF-8', + 'PYTHONDONTWRITEBYTECODE': '1'} + + def run_script(self, body, *, limits=GENEROUS, on_tick=None, args=()): + outcome = launcher.run([sys.executable, str(self.script(body)), *args], + cwd=str(self.scratch), env=self.env(), run_id=self.run_id, + directory=str(self.directory), limits=limits, on_tick=on_tick) + self.assertEqual(_carriers(self.run_id), [], 'a process of the run outlived run()') + self.assertEqual(_zombie_children(), [], 'run() left a zombie child') + self.assertEqual((signal.getsignal(signal.SIGINT), signal.getsignal(signal.SIGTERM)), + self.handlers, 'signal handlers were not restored') + self.assertFalse((self.directory / launcher.DESCRIPTION_FILE).exists(), + 'the description file outlived the run') + return outcome + + def log(self, name): + return (self.directory / f'{name}.log').read_text() + + def wait_for(self, path, timeout=10): + deadline = time.monotonic() + timeout + while not path.exists() or not path.read_text().strip(): + if time.monotonic() > deadline: + self.fail(f'{path.name} was never written') + time.sleep(0.01) + return int(path.read_text()) + + +class RunTests(LauncherCase): + def test_success_logs_and_environment(self): + outcome = self.run_script(''' + import json, os, sys + print('hello out') + print('hello err', file=sys.stderr) + print(json.dumps(sorted(os.environ))) + print(os.environ['TULDOK_RUN_ID']) + print(sys.stdin.read() == '') + ''') + self.assertEqual(outcome.state, 'succeeded') + self.assertEqual(outcome.exit_status, 0) + self.assertEqual(outcome.strays, 0) + self.assertGreater(outcome.bytes, 0) + lines = self.log('stdout').splitlines() + self.assertEqual(lines[0], 'hello out') + self.assertEqual(json.loads(lines[1]), sorted([*self.env(), 'TULDOK_RUN_ID'])) + self.assertEqual(lines[2], self.run_id) + self.assertEqual(lines[3], 'True') + self.assertEqual(self.log('stderr'), 'hello err\n') + + def test_logs_are_appended(self): + (self.directory / 'stdout.log').write_text('earlier\n') + self.run_script('print("later")') + self.assertEqual(self.log('stdout'), 'earlier\nlater\n') + + def test_new_session(self): + self.run_script('import os; print(os.getsid(0) == os.getpid())') + self.assertEqual(self.log('stdout'), 'True\n') + + def test_non_zero_exit(self): + outcome = self.run_script('import sys; sys.exit(3)') + self.assertEqual((outcome.state, outcome.exit_status), ('failed', 3)) + + def test_wall_time_limit(self): + started = time.monotonic() + outcome = self.run_script('import time; time.sleep(30)', + limits={'wall_seconds': 0.3, 'disk_bytes': 10 ** 9}) + self.assertEqual(outcome.state, 'timed-out') + self.assertEqual(outcome.exit_status, -signal.SIGTERM) + self.assertLess(time.monotonic() - started, 5) + + def test_disk_limit(self): + outcome = self.run_script(f''' + import os, time + os.makedirs({str(self.directory / 'output' / 'deep')!r}) + with open({str(self.directory / 'output' / 'deep' / 'big')!r}, 'wb') as file: + file.write(b'x' * 100000) + time.sleep(30) + ''', limits={'wall_seconds': 30, 'disk_bytes': 50000}) + self.assertEqual(outcome.state, 'over-disk') + self.assertGreaterEqual(outcome.bytes, 100000) + + def test_disk_over_at_exit_is_not_success(self): + outcome = self.run_script(f''' + with open({str(self.directory / 'big')!r}, 'wb') as file: + file.write(b'x' * 100000) + ''', limits={'wall_seconds': 30, 'disk_bytes': 50000}) + self.assertEqual((outcome.state, outcome.exit_status), ('over-disk', 0)) + + def test_cancel_file(self): + def tick(elapsed): + if elapsed > 0.2: + (self.directory / 'cancel').touch() + outcome = self.run_script('import time; time.sleep(30)', on_tick=tick) + self.assertEqual(outcome.state, 'cancelled') + + def test_on_tick_is_given_elapsed_seconds(self): + seen = [] + self.run_script('import time; time.sleep(0.3)', on_tick=seen.append) + self.assertTrue(seen) + self.assertEqual(seen, sorted(seen)) + + def test_on_tick_exception_stops_the_run(self): + def tick(elapsed): + if elapsed > 0.1: + raise RuntimeError('tick failed') + pid_file = self.scratch / 'pid' + with self.assertRaisesRegex(RuntimeError, 'tick failed'): + launcher.run([sys.executable, str(self.script(f''' + import os, time + open({str(pid_file)!r}, 'w').write(str(os.getpid())) + time.sleep(30) + '''))], cwd=str(self.scratch), env=self.env(), run_id=self.run_id, + directory=str(self.directory), limits=GENEROUS, on_tick=tick) + self.assertEqual(_carriers(self.run_id), []) + self.assertEqual(_zombie_children(), []) + self.assertEqual((signal.getsignal(signal.SIGINT), signal.getsignal(signal.SIGTERM)), self.handlers) + + def test_command_ignoring_sigterm_is_killed_after_grace(self): + started = time.monotonic() + outcome = self.run_script(''' + import signal, time + signal.signal(signal.SIGTERM, signal.SIG_IGN) + time.sleep(30) + ''', limits={'wall_seconds': 0.3, 'disk_bytes': 10 ** 9}) + self.assertEqual(outcome.state, 'timed-out') + self.assertEqual(outcome.exit_status, -signal.SIGKILL) + self.assertGreaterEqual(time.monotonic() - started, 0.3 + FAST['STOP_GRACE']) + + def test_child_outliving_the_command_is_a_stray(self): + pid_file = self.scratch / 'pid' + outcome = self.run_script(f''' + import os, time + if os.fork() == 0: + open({str(pid_file)!r}, 'w').write(str(os.getpid())) + time.sleep(30) + os._exit(0) + while not os.path.exists({str(pid_file)!r}): + time.sleep(0.01) + ''') + self.assertEqual(outcome.state, 'succeeded') + self.assertEqual(outcome.strays, 1) + self.assertFalse(_alive(int(pid_file.read_text()))) + + def test_grandchild_in_own_session_ignoring_sigterm(self): + pid_file = self.scratch / 'pid' + body = f''' + import os, signal, sys, time + if os.fork() == 0: + os.setsid() + if os.fork() == 0: + signal.signal(signal.SIGTERM, signal.SIG_IGN) + open({str(pid_file)!r} + '.tmp', 'w').write(str(os.getpid())) + os.rename({str(pid_file)!r} + '.tmp', {str(pid_file)!r}) + time.sleep(30) + os._exit(0) + os.wait() + while not os.path.exists({str(pid_file)!r}): + time.sleep(0.01) + if sys.argv[1] == 'stay': + time.sleep(30) + ''' + for mode, state in (('exit', 'succeeded'), ('stay', 'timed-out')): + with self.subTest(mode=mode): + pid_file.unlink(missing_ok=True) + outcome = self.run_script(body, args=(mode,), + limits={'wall_seconds': 1.0, 'disk_bytes': 10 ** 9}) + self.assertEqual(outcome.state, state) + grandchild = int(pid_file.read_text()) + self.assertFalse(_alive(grandchild)) + if mode == 'exit': + self.assertEqual(outcome.strays, 1) + + def test_sigterm_to_launcher_cancels_and_caller_continues(self): + for signum in (signal.SIGTERM, signal.SIGINT): + with self.subTest(signal=signum.name): + timer = threading.Timer(0.3, os.kill, (os.getpid(), signum)) + timer.start() + self.addCleanup(timer.cancel) + outcome = self.run_script('import time; time.sleep(30)') + timer.join() + self.assertEqual(outcome.state, 'cancelled') + self.assertEqual(outcome.exit_status, -signal.SIGTERM) + + def test_run_from_another_thread(self): + results = [] + thread = threading.Thread(target=lambda: results.append(launcher.run( + [sys.executable, '-c', 'print(1)'], cwd=str(self.scratch), env=self.env(), + run_id=self.run_id, directory=str(self.directory), limits=GENEROUS))) + thread.start() + thread.join(30) + self.assertEqual(results[0].state, 'succeeded') + self.assertEqual(_zombie_children(), []) + + def test_run_id_prefix_is_not_a_match(self): + neighbour = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'], + env={**self.env(), 'TULDOK_RUN_ID': self.run_id + 'x'}) + self.addCleanup(neighbour.wait) + self.addCleanup(neighbour.kill) + outcome = self.run_script('print(1)') + self.assertEqual((outcome.state, outcome.strays), ('succeeded', 0)) + self.assertIsNone(neighbour.poll()) + + def custodian_with_failing_prctl(self, option, argument): + """Run the custodian in a process whose prctl(option, argument) fails; return run()'s result.""" + description = self.directory / launcher.DESCRIPTION_FILE + launcher._write_description(str(description), [sys.executable, '-c', 'print(1)'], + str(self.scratch), self.env(), self.run_id, str(self.directory), GENEROUS) + subprocess.run([sys.executable, '-c', textwrap.dedent(f''' + import sys + sys.path.insert(0, {str(ROOT)!r}) + from tuldok_project import launcher + real = launcher._prctl + def failing(option, argument): + if (option, argument) == ({option}, {argument}): + raise OSError(1, 'refused') + return real(option, argument) + launcher._prctl = failing + sys.exit(launcher._custodian_main([{str(description)!r}])) + ''')], check=True, timeout=30) + self.assertFalse(description.exists()) + return launcher._read_outcome(str(self.directory / launcher.OUTCOME_FILE), 0, self.run_id, + str(self.directory), time.monotonic()) + + def test_subreaper_failure_refuses_to_run(self): + with self.assertRaisesRegex(OSError, 'refused'): + self.custodian_with_failing_prctl(launcher.PR_SET_CHILD_SUBREAPER, 1) + self.assertFalse((self.directory / 'stdout.log').exists()) + + def test_pdeathsig_failure_in_command_raises(self): + with self.assertRaises(subprocess.SubprocessError): + self.custodian_with_failing_prctl(launcher.PR_SET_PDEATHSIG, int(signal.SIGKILL)) + self.assertEqual(self.log('stdout'), '') + self.assertEqual(_carriers(self.run_id), []) + + def test_missing_program_raises(self): + with self.assertRaises(FileNotFoundError): + launcher.run([str(self.scratch / 'no-such-program')], cwd=str(self.scratch), env=self.env(), + run_id=self.run_id, directory=str(self.directory), limits=GENEROUS) + self.assertEqual(_zombie_children(), []) + self.assertFalse((self.directory / launcher.DESCRIPTION_FILE).exists()) + + def test_description_file_is_private(self): + path = self.scratch / 'description.json' + launcher._write_description(str(path), ['true'], str(self.scratch), {'SECRET': 'x'}, + self.run_id, str(self.directory), GENEROUS) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + + def test_exited_double_fork_intermediate_leaves_no_zombie(self): + daemon_file, background_file = self.scratch / 'daemon', self.scratch / 'background' + outcome = self.run_script(f''' + import os, time + if os.fork() == 0: # the intermediate: leaves the session, forks, exits at once + os.setsid() + if os.fork() == 0: + open({str(daemon_file)!r} + '.tmp', 'w').write(str(os.getpid())) + os.rename({str(daemon_file)!r} + '.tmp', {str(daemon_file)!r}) + time.sleep(30) + os._exit(0) + if os.fork() == 0: # a plain background child + open({str(background_file)!r} + '.tmp', 'w').write(str(os.getpid())) + os.rename({str(background_file)!r} + '.tmp', {str(background_file)!r}) + time.sleep(30) + os._exit(0) + while not (os.path.exists({str(daemon_file)!r}) and os.path.exists({str(background_file)!r})): + time.sleep(0.01) + ''') # the intermediate is never waited for: it is a zombie when the command exits + self.assertEqual((outcome.state, outcome.strays), ('succeeded', 2)) + self.assertFalse(_alive(int(daemon_file.read_text()))) + self.assertFalse(_alive(int(background_file.read_text()))) + + def test_callers_own_child_keeps_its_exit_status(self): + own = subprocess.Popen([sys.executable, '-c', 'import time, sys; time.sleep(0.3); sys.exit(7)']) + self.addCleanup(own.kill) + outcome = launcher.run([sys.executable, '-c', 'import time; time.sleep(1.0)'], + cwd=str(self.scratch), env=self.env(), run_id=self.run_id, + directory=str(self.directory), limits=GENEROUS) + self.assertEqual(outcome.state, 'succeeded') + self.assertEqual(_zombie_children(), [own.pid]) # exited during the run, and still ours to wait on + self.assertEqual(own.wait(5), 7) + self.assertEqual(_zombie_children(), []) + + def test_caller_killed_mid_run_is_stopped_by_custodian(self): + started_file = self.scratch / 'started' + command = self.script(f''' + import time + open({str(started_file)!r}, 'w').write('1') + time.sleep(30) + ''') + caller = subprocess.Popen([sys.executable, '-c', textwrap.dedent(f''' + import sys + sys.path.insert(0, {str(ROOT)!r}) + from tuldok_project import launcher + for name, value in {FAST!r}.items(): + setattr(launcher, name, value) + launcher.run([sys.executable, {str(command)!r}], cwd={str(self.scratch)!r}, + env={self.env()!r}, run_id={self.run_id!r}, directory={str(self.directory)!r}, + limits={GENEROUS!r}) + ''')]) + self.addCleanup(caller.wait) + self.addCleanup(caller.kill) + deadline = time.monotonic() + 20 + while not started_file.exists(): + self.assertLess(time.monotonic(), deadline) + time.sleep(0.01) + caller.kill() + caller.wait() + outcome_file = self.directory / launcher.OUTCOME_FILE + while not outcome_file.exists(): + self.assertLess(time.monotonic(), deadline) + time.sleep(0.02) + self.assertEqual(json.loads(outcome_file.read_text())['state'], 'cancelled') + self.assertEqual(_carriers(self.run_id), []) + + def test_custodian_killed_mid_run(self): + pid_file = self.scratch / 'pid' + + def kill_custodian(elapsed): + if elapsed > 0.3 and pid_file.exists(): + for pid in _children(): + try: + with open(f'/proc/{pid}/cmdline', 'rb') as file: + if b'tuldok_project.launcher' in file.read(): + os.kill(pid, signal.SIGKILL) + except OSError: + pass + outcome = self.run_script(f''' + import os, time + if os.fork() == 0: + os.setsid() # out of the group, and with no parent-death signal of its own + open({str(pid_file)!r} + '.tmp', 'w').write(str(os.getpid())) + os.rename({str(pid_file)!r} + '.tmp', {str(pid_file)!r}) + time.sleep(30) + os._exit(0) + time.sleep(30) + ''', on_tick=kill_custodian) + self.assertEqual(outcome.state, 'unknown') + self.assertIsNone(outcome.exit_status) + self.assertIn('custodian ended without an outcome', outcome.problem) + self.assertGreaterEqual(outcome.strays, 1) + self.assertFalse(_alive(int(pid_file.read_text()))) + + def test_bad_arguments(self): + good = dict(cwd=str(self.scratch), env=self.env(), run_id=self.run_id, + directory=str(self.directory), limits=GENEROUS) + for change in ({'limits': {'wall_seconds': 0, 'disk_bytes': 1}}, + {'limits': {'wall_seconds': 1, 'disk_bytes': 1.5}}, + {'limits': {'wall_seconds': 1}}, + {'run_id': ''}, {'run_id': 'a=b'}, + {'directory': str(self.scratch / 'missing')}): + with self.subTest(change=change), self.assertRaises(ValueError): + launcher.run([sys.executable, '-c', 'print(1)'], **{**good, **change}) + + +class StrayTests(LauncherCase): + def start_outside(self, run_id): + process = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'], + env={**self.env(), 'TULDOK_RUN_ID': run_id}) + self.addCleanup(process.wait) + self.addCleanup(process.kill) + return process + + def test_strays_and_stop_strays(self): + process = self.start_outside(self.run_id) + deadline = time.monotonic() + 10 + while process.pid not in launcher.strays(self.run_id): # environ is set at exec + self.assertLess(time.monotonic(), deadline) + time.sleep(0.01) + self.assertEqual(launcher.strays(self.run_id), [process.pid]) + result = launcher.stop_strays(self.run_id) + self.assertEqual(result, {'killed': 1, 'remaining': []}) + self.assertEqual(process.wait(5), -signal.SIGKILL) + self.assertEqual(launcher.strays(self.run_id), []) + + def test_prefix_and_extension_do_not_match(self): + longer = self.start_outside(self.run_id + 'c') + self.addCleanup(_kill_run, self.run_id + 'c') + deadline = time.monotonic() + 10 + while longer.pid not in launcher.strays(self.run_id + 'c'): + self.assertLess(time.monotonic(), deadline) + time.sleep(0.01) + self.assertEqual(launcher.strays(self.run_id), []) + self.assertEqual(launcher.stop_strays(self.run_id), {'killed': 0, 'remaining': []}) + self.assertIsNone(longer.poll()) + + def test_run_id_is_checked(self): + for bad in ('', 'a\0b', 'a=b', 7): + with self.subTest(run_id=bad), self.assertRaises(ValueError): + launcher.strays(bad) + + +def event(**change): + value = {'format': 'tuldok_progress_v1', 'stage': 'train', 'completed': 5, 'total': 10, + 'metrics': {'loss': 0.5}} + value.update(change) + return json.dumps(value) + + +class ProgressTests(unittest.TestCase): + def setUp(self): + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + self.directory = Path(scratch.name) + + def write(self, name, data): + (self.directory / f'{name}.log').write_bytes(data if type(data) is bytes else data.encode()) + + def test_valid_event(self): + self.write('stdout', 'starting\n' + event() + '\n' + 'plain text\n') + self.write('stderr', 'a warning\n') + result = launcher.progress(str(self.directory)) + self.assertEqual(result['event'], json.loads(event())) + self.assertEqual(result['stderr'], 'a warning\n') + self.assertEqual(result['stderr_bytes'], 10) + self.assertEqual(result['stdout_bytes'], len(result['stdout'])) + + def test_invalid_events_are_skipped(self): + earlier = event(completed=1) + for bad in (event(completed=11), event(stage='-bad'), event(total=1000001), + event(metrics={f'm{i}': 1 for i in range(9)}), event(completed=True), + event(extra=1), event(format='other'), + event()[:-1] + ', "stage": "x"}', + event(metrics={'loss': 1}).replace('1}', 'NaN}'), 'not json', '[]'): + with self.subTest(line=bad): + self.write('stdout', earlier + '\n' + bad + '\n') + self.assertEqual(launcher.progress(str(self.directory))['event'], json.loads(earlier)) + + def test_over_long_line_is_ignored(self): + long_line = event(completed=9)[:-1] + ' ' * 4100 + '}' + self.write('stdout', event() + '\n' + long_line + '\n') + self.assertEqual(launcher.progress(str(self.directory))['event'], json.loads(event())) + + def test_incomplete_last_line_is_ignored(self): + self.write('stdout', event() + '\n' + event(completed=9)) + self.assertEqual(launcher.progress(str(self.directory))['event'], json.loads(event())) + + def test_missing_logs(self): + self.assertEqual(launcher.progress(str(self.directory)), + {'stdout': '', 'stderr': '', 'stdout_bytes': None, 'stderr_bytes': None, + 'event': None}) + self.assertEqual(launcher.progress(str(self.directory / 'absent'))['event'], None) + + def test_symbolic_link_is_not_followed(self): + target = self.directory / 'elsewhere.txt' + target.write_text(event() + '\n') + (self.directory / 'stdout.log').symlink_to(target) + result = launcher.progress(str(self.directory)) + self.assertEqual((result['stdout'], result['stdout_bytes'], result['event']), ('', None, None)) + linked = self.directory.parent / (self.directory.name + '-link') + linked.symlink_to(self.directory) + self.addCleanup(linked.unlink) + self.assertIsNone(launcher.progress(str(linked))['stdout_bytes']) + + def test_large_log_reads_only_the_tail(self): + data = ('x' * 99 + '\n') * 20000 + event() + '\n' + self.write('stdout', data) + with mock.patch('os.pread', wraps=os.pread) as pread: + result = launcher.progress(str(self.directory)) + self.assertTrue(all(call.args[1] <= 8192 for call in pread.call_args_list)) + self.assertEqual(result['stdout_bytes'], len(data)) + self.assertLessEqual(len(result['stdout'].encode()), 8192) + self.assertEqual(result['event'], json.loads(event())) + + def test_cut_first_line_is_not_an_event(self): + # The tail starts inside a long line whose remainder is itself a valid event. + data = 'y' * 300 + event() + '\n' + 'z' * (8192 - len(event()) - 2) + '\n' + self.assertEqual(len(data) - 8192, 300) # the tail begins exactly at the '{' + self.write('stdout', data) + self.assertIsNone(launcher.progress(str(self.directory))['event']) + + def test_text_is_sanitised(self): + self.write('stderr', b'ok\tline\n\x1b[31mred\x00\r\n\xff\xfe end\n') + self.assertEqual(launcher.progress(str(self.directory))['stderr'], + 'ok\tline\n[31mred\n�� end\n') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_local.py b/tests/test_tuldok_project_local.py new file mode 100644 index 00000000..8b79bd04 --- /dev/null +++ b/tests/test_tuldok_project_local.py @@ -0,0 +1,249 @@ +"""The project directory setting, `home` and `new` (docs/plans/definition-first-projects/spec-m8.md, sections 1 and 2). + +None of these tests needs a database. Every one points TULDOK_SETTINGS at a +file in a temporary directory, so the real ~/.local/share/tuldok/settings.json +is never read or written, and every directory made is under that temporary +directory. Commands run as child processes, as a person runs them. +""" +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tuldok_project import local, store # noqa: E402 +from tuldok_project.check import check_folder # noqa: E402 + +REPOSITORY = Path(__file__).resolve().parents[1] +PROJECT_ID = 'a' * 32 + + +class SettingsCase(unittest.TestCase): + """A temporary directory, TULDOK_SETTINGS pointing into it and TULDOK_PROJECT_HOME unset.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.temporary = Path(directory.name) + self.settings = self.temporary / 'tuldok' / 'settings.json' + patcher = mock.patch.dict(os.environ, {local.SETTINGS_VARIABLE: str(self.settings)}) + patcher.start() + self.addCleanup(patcher.stop) + os.environ.pop(local.HOME_VARIABLE, None) + + def write_settings(self, value): + self.settings.parent.mkdir(parents=True, exist_ok=True) + self.settings.write_text(value if isinstance(value, str) else json.dumps(value), encoding='utf-8') + + def run_cli(self, *arguments, home=None): + environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1') + environment[local.SETTINGS_VARIABLE] = str(self.settings) + environment.pop(local.HOME_VARIABLE, None) + environment.pop('TULDOK_DATABASE_URL', None) + if home is not None: + environment[local.HOME_VARIABLE] = str(home) + return subprocess.run([sys.executable, '-m', 'tuldok_project', *map(str, arguments)], cwd=REPOSITORY, + env=environment, capture_output=True, text=True, timeout=120) + + +class Precedence(SettingsCase): + + def test_the_variable_then_the_setting_then_the_default(self): + default = os.path.expanduser('~/.local/share/tuldok/projects') + self.assertEqual(local.projects_home_source(), (default, 'default')) # no settings file + self.write_settings({'format': 'tuldok_settings_v1'}) + self.assertEqual(local.projects_home_source(), (default, 'default')) # a file without project_home + chosen = str(self.temporary / 'chosen') + self.write_settings({'format': 'tuldok_settings_v1', 'project_home': chosen}) + self.assertEqual(local.projects_home_source(), (chosen, 'settings')) + self.assertEqual(local.home(PROJECT_ID), os.path.join(chosen, PROJECT_ID)) + self.assertFalse(os.path.exists(chosen)) # working out a path makes nothing + with mock.patch.dict(os.environ, {local.HOME_VARIABLE: str(self.temporary / 'variable')}): + self.assertEqual(local.projects_home_source(), (str(self.temporary / 'variable'), 'environment')) + with mock.patch.dict(os.environ, {local.HOME_VARIABLE: ''}): # set but empty: not used + self.assertEqual(local.projects_home(), chosen) + + def test_a_broken_file_stops_and_never_falls_back(self): + broken = { + 'not JSON': '{"format": ', + 'a list': '[]', + 'a duplicate key': '{"format": "tuldok_settings_v1", "format": "tuldok_settings_v1"}', + 'NaN': '{"format": "tuldok_settings_v1", "project_home": NaN}', + 'an unknown field': {'format': 'tuldok_settings_v1', 'colour': 'red'}, + 'another format': {'format': 'tuldok_settings_v2'}, + 'no format': {'project_home': '/x'}, + 'a relative project_home': {'format': 'tuldok_settings_v1', 'project_home': 'relative/path'}, + 'a project_home that is not text': {'format': 'tuldok_settings_v1', 'project_home': 3}, + 'too large': '{"format": "tuldok_settings_v1", "project_home": "/' + 'x' * 70000 + '"}', + } + for name, value in broken.items(): + with self.subTest(name): + self.write_settings(value) + with self.assertRaises(local.SettingsError) as caught: + local.projects_home() + self.assertIn(str(self.settings), str(caught.exception)) + with self.assertRaises(local.SettingsError): + local.home(PROJECT_ID, create=True) + self.settings.unlink() + self.settings.mkdir() # a directory where the file should be: cannot be read + with self.assertRaises(local.SettingsError) as caught: + local.projects_home() + self.assertIn('cannot be read', str(caught.exception)) + + def test_the_variable_decides_without_reading_the_file(self): + self.write_settings('{"format": ') + with mock.patch.dict(os.environ, {local.HOME_VARIABLE: str(self.temporary / 'variable')}): + self.assertEqual(local.projects_home(), str(self.temporary / 'variable')) + + def test_commands_stop_with_the_message_and_status_2(self): + self.write_settings({'format': 'tuldok_settings_v1', 'extra': 1}) + shown = self.run_cli('home') + self.assertEqual(shown.returncode, 2) + self.assertEqual(shown.stdout, '') + self.assertIn(f'The settings file {self.settings} is not valid', shown.stderr) + self.assertIn("'extra'", shown.stderr) + setting = self.run_cli('home', '--set', self.temporary / 'new-home') + self.assertEqual(setting.returncode, 2) + self.assertIn('is not valid', setting.stderr) + self.assertFalse((self.temporary / 'new-home').exists()) + self.assertEqual(json.loads(self.settings.read_text())['extra'], 1) # a broken file is not replaced + + def test_the_app_ends_start_up(self): + from tuldok_project.host import ProjectHost + + self.write_settings('not json') + with self.assertRaises(store.ProjectError) as caught: + ProjectHost(self.temporary / 'data', self.temporary / 'folder') + self.assertEqual(caught.exception.code, 'settings') + self.assertIn(str(self.settings), str(caught.exception)) + self.assertFalse((self.temporary / 'data').exists()) # stopped before anything was opened + + +class Home(SettingsCase): + + def test_home_shows_the_directory_its_source_its_disk_and_its_projects(self): + chosen = self.temporary / 'chosen' + (chosen / PROJECT_ID).mkdir(parents=True) + (chosen / ('b' * 32)).mkdir() + (chosen / 'not-a-project').mkdir() + (chosen / 'gpu.lock').write_text('') + self.write_settings({'format': 'tuldok_settings_v1', 'project_home': str(chosen)}) + shown = self.run_cli('home') + self.assertEqual(shown.returncode, 0, shown.stderr) + rows = dict(line.split(None, 1) for line in shown.stdout.splitlines()) + self.assertEqual(rows['directory'], str(chosen)) + self.assertEqual(rows['from'], f'the settings file {self.settings}') + self.assertTrue(rows['free'].endswith(' on its disk'), rows['free']) + self.assertEqual(rows['projects'], '2') + listed = json.loads(self.run_cli('home', '--json').stdout) + self.assertEqual((listed['directory'], listed['source'], listed['projects'], listed['exists']), + (str(chosen), 'settings', 2, True)) + self.assertIsInstance(listed['free_bytes'], int) + variable = json.loads(self.run_cli('home', '--json', home=self.temporary / 'other').stdout) + self.assertEqual((variable['source'], variable['exists'], variable['projects']), ('environment', False, 0)) + self.assertIn('the environment variable TULDOK_PROJECT_HOME', self.run_cli('home', home=chosen).stdout) + self.settings.unlink() + self.assertIn('the default; no setting is saved in', self.run_cli('home').stdout) + + def test_set_makes_the_directory_and_writes_the_file_mode_0600(self): + chosen = self.temporary / 'disk' / 'tuldok' / 'projects' + done = self.run_cli('home', '--set', chosen) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(done.stdout, f'Saved {chosen} as the project directory in {self.settings}.\n') + self.assertTrue(chosen.is_dir()) + self.assertEqual(json.loads(self.settings.read_text()), + {'format': 'tuldok_settings_v1', 'project_home': str(chosen)}) + self.assertEqual(stat.S_IMODE(os.stat(self.settings).st_mode), 0o600) + self.assertEqual(sorted(os.listdir(self.settings.parent)), ['settings.json']) # no temporary file left + self.assertEqual(local.projects_home(), str(chosen)) # what every command now uses + + def test_set_refuses_a_relative_path_and_a_directory_it_cannot_write(self): + relative = self.run_cli('home', '--set', 'relative/path') + self.assertEqual(relative.returncode, 2) + self.assertIn('is not an absolute path', relative.stderr) + self.assertFalse(self.settings.exists()) + locked = self.temporary / 'locked' + locked.mkdir(mode=0o500) + self.addCleanup(os.chmod, locked, 0o700) + if os.access(locked, os.W_OK): + self.skipTest('this user can write any directory') + refused = self.run_cli('home', '--set', locked) + self.assertEqual(refused.returncode, 2) + self.assertIn('is not a directory this user can write', refused.stderr) + self.assertFalse(self.settings.exists()) + file = self.temporary / 'a-file' + file.write_text('') + self.assertEqual(self.run_cli('home', '--set', file).returncode, 2) + + def test_set_says_when_projects_stay_behind_and_when_the_variable_overrides(self): + before = self.temporary / 'before' + (before / PROJECT_ID).mkdir(parents=True) + self.write_settings({'format': 'tuldok_settings_v1', 'project_home': str(before)}) + after = self.temporary / 'after' + done = self.run_cli('home', '--set', after) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertIn(f'The directory used before, {before}, holds 1 project directory; it was not moved.', + done.stdout) + self.assertNotIn('overrides', done.stdout) + self.assertTrue((before / PROJECT_ID).is_dir()) + again = self.run_cli('home', '--set', self.temporary / 'third', home=self.temporary / 'variable') + self.assertEqual(again.returncode, 0, again.stderr) + self.assertIn('TULDOK_PROJECT_HOME is set, so it overrides this setting while it is set.', again.stdout) + self.assertNotIn('used before', again.stdout) # after holds no project directory + shown = json.loads(self.run_cli('home', '--set', after, '--json').stdout) + self.assertEqual((shown['before'], shown['before_projects'], shown['overridden']), + (str(self.temporary / 'third'), 0, False)) + + +class New(SettingsCase): + + def test_new_writes_tuldok_json_alone(self): + folder = self.temporary / 'projects' / 'mine' + made = self.run_cli('new', folder, '--name', 'My project', '--code-root', '../code') + self.assertEqual(made.returncode, 0, made.stderr) + self.assertEqual(os.listdir(folder), ['tuldok.json']) + document = json.loads((folder / 'tuldok.json').read_text()) + self.assertEqual(set(document), {'format', 'id', 'name', 'code_root'}) + self.assertEqual((document['format'], document['name'], document['code_root']), + ('tuldok_project_v1', 'My project', '../code')) + self.assertRegex(document['id'], '^[0-9a-f]{32}$') + self.assertIn(document['id'], made.stdout) + self.assertTrue(check_folder(folder, code=False).ok) + empty = self.temporary / 'empty' + empty.mkdir() + shown = json.loads(self.run_cli('new', empty, '--name', 'Another', '--json').stdout) + self.assertEqual((shown['folder'], shown['name'], shown['code_root']), (str(empty), 'Another', '.')) + self.assertNotEqual(shown['project_id'], document['id']) + self.assertEqual(json.loads((empty / 'tuldok.json').read_text())['code_root'], '.') + self.assertFalse(self.settings.exists()) # new needs no setting and writes none + + def test_new_refuses_and_makes_nothing(self): + taken = self.temporary / 'taken' + taken.mkdir() + (taken / 'notes.txt').write_text('mine') + file = self.temporary / 'file' + file.write_text('') + cases = { + 'a folder that is not empty': (taken, 'Name', '.'), + 'a file': (file, 'Name', '.'), + 'an empty name': (self.temporary / 'a', '', '.'), + 'a name too long': (self.temporary / 'b', 'n' * 121, '.'), + 'an absolute code root': (self.temporary / 'c', 'Name', '/code'), + } + for name, (folder, project_name, code_root) in cases.items(): + with self.subTest(name): + refused = self.run_cli('new', folder, '--name', project_name, '--code-root', code_root) + self.assertEqual(refused.returncode, 2, refused.stdout) + self.assertTrue(refused.stderr.strip()) + self.assertEqual(os.listdir(taken), ['notes.txt']) + self.assertEqual(sorted(os.listdir(self.temporary)), ['file', 'taken']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_model_graph.py b/tests/test_tuldok_project_model_graph.py new file mode 100644 index 00000000..fc32b555 --- /dev/null +++ b/tests/test_tuldok_project_model_graph.py @@ -0,0 +1,520 @@ +"""A model's structure, read from model-graph.json (docs/plans/training-and-curation/spec-t2.md, slice T2a). + +The reader is tested on graph files the tests write by hand, and on the two +committed under tests/fixtures/model_graph (written once by +tuldok_log.graph.export_graph through tests/project_structure_fixture_export.py), +with no database: a Product is a directory. The weights the change overlay +compares are .npz files written with the standard library. The operations +run through the service only when TULDOK_DATABASE_URL names a disposable +cluster. No PyTorch and no GPU are used. +""" +import copy +import hashlib +import json +import math +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import agent_api # noqa: E402 +from tuldok_project import adoption, inside, inside_archive, inside_graph, local, operations, store # noqa: E402 +from tuldok_project.service import Service # noqa: E402 +import test_tuldok_project_builds as builds_base # noqa: E402 +from test_tuldok_project_inside import OWNER, npy, zip_bytes # noqa: E402 + +FIXTURES = Path(__file__).resolve().parent / 'fixtures' / 'model_graph' +GRAPH_OPERATIONS = ('model_graph', 'model_graph_overlay', 'model_graph_compare') + + +def value(*shape): + return {'shape': list(shape), 'dtype': 'float32'} + + +def module(path, kind, parameters, *children): + return {'path': path, 'class': kind, 'parameters': parameters, 'children': list(children)} + + +def node(identifier, kind, module_path, tensors=(), outputs=(), operation=None, attributes=None): + return {'id': identifier, 'kind': kind, 'operation': operation or (kind if kind != 'operation' else 'aten.x'), + 'module': module_path, 'tensors': list(tensors), 'inputs': [], 'outputs': list(outputs), + 'attributes': attributes or {}} + + +TENSORS = {'enc.0.weight': (4, 4), 'enc.0.bias': (4,), 'enc.1.weight': (2, 4), 'enc.1.bias': (2,), + 'head.weight': (3, 2), 'head.bias': (3,)} + + +def hand_graph(): + """A model of 39 parameters: enc (a Sequential of two Linear, enc.0 and enc.1, with a ReLU run in enc itself) + and head (a Linear with a buffer), and an add in the model itself that takes a skip from enc.0 and two values + from enc (the ReLU and head).""" + tensors = {name: {'shape': list(shape), 'dtype': 'float32', 'kind': 'parameter'} for name, shape in TENSORS.items()} + tensors['head.scale'] = {'shape': [3], 'dtype': 'float32', 'kind': 'buffer'} + return { + 'format': 'tuldok_model_graph_v1', 'complete': True, 'reason': None, 'method': 'torch.export', + 'torch': '2.8.0+cpu', 'model': 'tests.Net', 'inputs': [{'name': 'x', **value(1, 4)}], + 'tensors': tensors, + 'nodes': [ + node('x', 'input', '', outputs=[value(1, 4)], operation='input'), + node('a', 'operation', 'enc.0', ['enc.0.weight', 'enc.0.bias'], [value(1, 4)], 'aten.linear.default', + {'bias': True}), + node('r', 'operation', 'enc', [], [value(1, 4)], 'aten.relu.default'), + node('b', 'operation', 'enc.1', ['enc.1.weight', 'enc.1.bias'], [value(1, 2)], 'aten.linear.default'), + node('c', 'operation', 'head', ['head.weight', 'head.bias', 'head.scale'], [value(1, 3)], + 'aten.linear.default'), + node('d', 'operation', '', [], [value(1, 3)], 'aten.add.Tensor', {'alpha': 1, 'dims': [1, 2]}), + node('out', 'output', '', outputs=[value(1, 3)], operation='output')], + 'edges': [{'from': 'x', 'to': 'a', 'input': 0}, {'from': 'a', 'to': 'r', 'input': 0}, + {'from': 'r', 'to': 'b', 'input': 0}, {'from': 'b', 'to': 'c', 'input': 0}, + {'from': 'c', 'to': 'd', 'input': 0}, {'from': 'a', 'to': 'd', 'input': 1}, + {'from': 'r', 'to': 'd', 'input': 2}, {'from': 'd', 'to': 'out', 'input': 0}], + 'modules': module('', 'tests.Net', 39, + module('enc', 'torch.nn.Sequential', 30, module('enc.0', 'torch.nn.Linear', 20), + module('enc.1', 'torch.nn.Linear', 10)), + module('head', 'torch.nn.Linear', 9)), + 'notes': []} + + +def npz(arrays): + """An .npz of {name: (shape, [values])} in float32.""" + return zip_bytes({name + '.npy': npy(shape, ' r and r -> b are inside enc and dropped; a -> d and r -> d are one edge of two + self.assertEqual(self.edges(answer), [('x', 'module:enc', 1), ('module:enc', 'module:head', 1), + ('module:head', 'd', 1), ('module:enc', 'd', 2), ('d', 'out', 1)]) + enc = answer['nodes'][1] + self.assertEqual((enc['kind'], enc['label'], enc['class'], enc['operations'], enc['parameters'], + enc['tensors'], enc['tensor_count'], enc['first']), + ('group', 'enc', 'torch.nn.Sequential', 3, 30, + ['enc.0.weight', 'enc.0.bias', 'enc.1.weight', 'enc.1.bias'], 4, 1)) + self.assertEqual(enc['inputs'], [value(1, 4)]) + self.assertEqual(enc['outputs'], [value(1, 2), value(1, 4)], 'b -> c, then a -> d (r -> d merged with it)') + self.assertEqual(answer['open'], []) + + def test_folded_at_depth_one(self): + answer = self.graph(depth=1) + self.assertEqual([item['id'] for item in answer['nodes']], + ['x', 'module:enc.0', 'r', 'module:enc.1', 'c', 'd', 'out']) + self.assertEqual(len(answer['edges']), 8 - 0) + self.assertEqual(answer['open'], ['enc', 'head']) + + def test_opened_and_closed_by_name(self): + closed = self.graph(closed=['enc']) + self.assertEqual([item['id'] for item in closed['nodes']], ['x', 'module:enc', 'c', 'd', 'out']) + self.assertEqual(closed['open'], ['head']) + # A module opened under a closed one stays inside it. + inner = self.graph(depth=0, opened=['enc.1']) + self.assertEqual([item['id'] for item in inner['nodes']], ['x', 'module:enc', 'module:head', 'd', 'out']) + opened = self.graph(depth=0, opened=['enc', 'enc.1']) + self.assertEqual([item['id'] for item in opened['nodes']], + ['x', 'module:enc.0', 'r', 'b', 'module:head', 'd', 'out']) + self.assertEqual(opened['open'], ['enc', 'enc.1']) + + def test_large_graphs_start_at_depth_one(self): + with mock.patch.object(inside_graph, 'LARGE', 4): + self.assertEqual(self.graph()['depth'], 1) + self.assertEqual(self.graph()['depth'], 64) + + def test_a_module_the_tree_does_not_list_is_its_nearest_listed_parent(self): + document = hand_graph() + document['nodes'][2]['module'] = 'enc.ghost.deeper' + self.write(document) + self.assertEqual(self.graph()['nodes'][2]['module'], 'enc') + + +class Refusals(ReaderCase): + + def check(self, document, code='unsupported', **patches): + self.write(document) + inside_graph.forget() + with mock.patch.multiple(inside_graph, **patches) if patches else _nothing(): + return self.refused(code, inside_graph.model_graph, self.product) + + def changed(self, change): + document = hand_graph() + change(document) + return document + + def test_each_refusal(self): + self.assertIn('not JSON', self.check('{"format": ')) + self.assertIn('not JSON', self.check(b'\xff\xfe'.decode('latin-1'))) + self.assertIn('not JSON', self.check('[' * 100_000 + ']' * 100_000)) + self.assertIn('not JSON', self.check(json.dumps(hand_graph()).replace('"alpha": 1', '"alpha": NaN'))) + self.assertIn('format', self.check(self.changed(lambda d: d.update(format='other_v1')))) + self.assertIn('has no modules', self.check(self.changed(lambda d: d.pop('modules')))) + self.assertIn('complete', self.check(self.changed(lambda d: d.update(complete='yes')))) + self.assertIn('used twice', self.check(self.changed(lambda d: d['nodes'][2].update(id='a')))) + self.assertIn('names no node', self.check(self.changed(lambda d: d['edges'][0].update(to='nowhere')))) + self.assertIn('kind', self.check(self.changed(lambda d: d['nodes'][1].update(kind='layer')))) + self.assertIn('sizes', self.check(self.changed(lambda d: d['nodes'][1]['outputs'][0].update(shape=[-1])))) + self.assertIn('may not begin', self.check(self.changed(lambda d: d['nodes'][1].update(id='module:a')))) + self.assertIn('start at the model', self.check(self.changed(lambda d: d['modules'].update(path='net')))) + self.assertIn('extend its parent', self.check(self.changed( + lambda d: d['modules']['children'][0]['children'][0].update(path='other.0')))) + self.assertIn('listed twice', self.check(self.changed( + lambda d: d['modules']['children'][1].update(path='enc')))) + self.assertIn('short list', self.check(self.changed( + lambda d: d['nodes'][1]['attributes'].update(nested={'a': 1})))) + self.assertIn('at most 6', self.check(hand_graph(), NODES=6)) + self.assertIn('at most 7', self.check(hand_graph(), EDGES=7)) + self.assertIn('more than 4 modules', self.check(hand_graph(), MODULES=4)) + self.assertIn('deeper than 1', self.check(hand_graph(), DEPTH=1)) + self.assertIn('at most 1 are read', self.check(hand_graph(), NODE_TENSORS=1)) + self.assertIn('bytes', self.check(hand_graph(), code='too-large', FILE_BYTES=100)) + + def test_no_file(self): + answer = inside_graph.model_graph(self.product) + self.assertEqual((answer['path'], answer['complete'], answer['nodes'], answer['modules']), + (None, False, [], None)) + self.assertTrue(set(operations.MODEL_GRAPH) <= set(answer)) + self.refused('not-found', inside_graph.model_graph_overlay, self.product, kind='parameters') + self.refused('not-found', inside_graph.model_graph_compare, self.product, other=self.product) + + def test_a_module_tree_alone(self): + document = hand_graph() + document.update(complete=False, reason='torch.export failed (data-dependent); torch.fx failed.', + method='module-tree') + document['edges'].append({'from': 'nowhere', 'to': 'x'}) # ignored with the nodes + self.write(document) + answer = inside_graph.model_graph(self.product, depth=0) + self.assertEqual((answer['complete'], answer['reason'], answer['nodes'], answer['edges']), + (False, document['reason'], [], [])) + self.assertEqual(answer['counts']['modules'], 5) + self.assertEqual(answer['modules']['parameters'], 39) + + +class _nothing: + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + +class Finding(ReaderCase): + + def test_beside_a_declared_weights_file(self): + product = self.make_product('run-2', outputs={'model': {'path': 'out/weights.npz', 'role': 'weights'}}) + Path(product.root, 'out').mkdir() + Path(product.root, 'out', 'weights.npz').write_bytes(npz(weights(lambda name, n: n))) + self.write(hand_graph(), product, 'out/model-graph.json') + self.assertEqual(inside_graph.model_graph(product)['path'], 'out/model-graph.json') + self.write(hand_graph(), product) # the top of the product comes first + self.assertEqual(inside_graph.model_graph(product)['path'], 'model-graph.json') + + def test_kept_while_unchanged(self): + path = self.write(hand_graph()) + first = inside_graph.graph_of(self.product)[1] + self.assertIs(inside_graph.graph_of(self.product)[1], first) + document = hand_graph() + document['torch'] = '2.9.0' + path.write_text(json.dumps(document) + ' ' * 7, encoding='utf-8') + self.assertEqual(inside_graph.graph_of(self.product)[1].torch, '2.9.0') + + +class Overlays(ReaderCase): + + def setUp(self): + super().setUp() + self.write(hand_graph()) + + def overlay(self, product=None, **arguments): + answer = inside_graph.model_graph_overlay(product or self.product, **arguments) + json.dumps(answer, allow_nan=False) + return answer + + def test_counts(self): + parameters = self.overlay(kind='parameters') + self.assertEqual(parameters['modules'], {'': 39, 'enc': 30, 'enc.0': 20, 'enc.1': 10, 'head': 9}) + self.assertEqual(parameters['nodes'], {'a': 20, 'r': 0, 'b': 10, 'c': 9, 'd': 0}) + self.assertEqual((parameters['unit'], parameters['low'], parameters['high']), ('parameters', 0, 39)) + outputs = self.overlay(kind='outputs') + self.assertEqual(outputs['nodes'], {'a': 4, 'r': 4, 'b': 2, 'c': 3, 'd': 3}) + self.assertEqual(outputs['modules'], {'': 16, 'enc': 10, 'enc.0': 4, 'enc.1': 2, 'head': 3}) + counted = self.overlay(kind='operations') + self.assertEqual(counted['modules'], {'': 5, 'enc': 3, 'enc.0': 1, 'enc.1': 1, 'head': 1}) + self.assertEqual(set(counted['nodes'].values()), {1}) + + def test_a_symbolic_size_has_no_output_count(self): + document = hand_graph() + document['nodes'][3]['outputs'] = [{'shape': ['s0', 2], 'dtype': 'float32'}] + self.write(document) + outputs = self.overlay(kind='outputs') + self.assertEqual((outputs['nodes']['b'], outputs['modules']['enc.1'], outputs['modules']['enc'], + outputs['modules']['head']), (None, None, None, 3)) + + def expected_change(self, here, there, names): + difference = math.fsum((a - b) ** 2 for name in names for a, b in zip(here[name][1], there[name][1])) + other = math.fsum(b * b for name in names for b in there[name][1]) + return math.sqrt(difference) / math.sqrt(other) + + def test_change_against_the_start_and_another_run(self): + start = weights(lambda name, n: 1 + n % 3) + trained = weights(lambda name, n: (1 + n % 3) * (2 if name.startswith('enc.0') else 1) + + (0.5 if name == 'head.bias' else 0)) + Path(self.product.root, 'weights.npz').write_bytes(npz(trained)) + Path(self.product.root, 'initial-weights.npz').write_bytes(npz(start)) + answer = self.overlay(kind='change', against='start') + self.assertEqual(answer['against'], {'kind': 'start', 'run': 'run-1', 'path': 'weights.npz', + 'other_run': 'run-1', 'other_path': 'initial-weights.npz'}) + for path, names in (('enc.0', ['enc.0.weight', 'enc.0.bias']), ('enc', [n for n in TENSORS if + n.startswith('enc.')]), + ('head', ['head.weight', 'head.bias']), ('', list(TENSORS))): + self.assertAlmostEqual(answer['modules'][path], self.expected_change(trained, start, names), 12, path) + self.assertEqual(answer['modules']['enc.1'], 0.0) + self.assertAlmostEqual(answer['nodes']['a'], 1.0, 12) + self.assertAlmostEqual(answer['nodes']['c'], self.expected_change(trained, start, ['head.weight', + 'head.bias']), 12) + self.assertIsNone(answer['nodes']['r'], 'r reads no tensor') + self.assertEqual((answer['low'], answer['high']), (0.0, 1.0)) + # Another run, whose weights are the start: the same numbers. + other = self.make_product('run-0') + Path(other.root, 'weights.npz').write_bytes(npz(start)) + against = self.overlay(kind='change', against='run', other=other) + self.assertEqual(against['modules'], answer['modules']) + self.assertEqual(against['against']['other_run'], 'run-0') + + def test_change_with_zero_and_missing_tensors(self): + start = weights(lambda name, n: 0 if name.startswith('head') else 1) + trained = weights(lambda name, n: 2) + del trained['enc.1.bias'] + Path(self.product.root, 'weights.npz').write_bytes(npz(trained)) + Path(self.product.root, 'initial-weights.npz').write_bytes(npz(start)) + answer = self.overlay(kind='change', against='start') + self.assertIsNone(answer['modules']['head'], 'it started at zero') + self.assertAlmostEqual(answer['modules']['enc.1'], 1.0, 12, 'over the one tensor compared') + + def test_change_refusals(self): + self.refused('not-found', self.overlay, kind='change', against='start') # no weights file + Path(self.product.root, 'weights.npz').write_bytes(npz(weights(lambda name, n: 1))) + self.refused('not-found', self.overlay, kind='change', against='start') # no initial- file + self.refused('invalid', self.overlay, kind='change', against='run') # no other run + self.refused('invalid', self.overlay, kind='change') + self.refused('invalid', self.overlay, kind='time') + + +class Comparing(ReaderCase): + + def test_modules_matched_by_path(self): + here = hand_graph() + there = copy.deepcopy(here) + there['tensors']['enc.1.weight']['shape'] = [3, 4] + there['modules']['children'][0]['children'][0]['class'] = 'torch.nn.LazyLinear' + there['modules']['children'][1] = module('tail', 'torch.nn.Linear', 9) + del there['tensors']['head.weight'], there['tensors']['head.bias'], there['tensors']['head.scale'] + there['tensors']['tail.weight'] = {'shape': [3, 2], 'dtype': 'float32', 'kind': 'parameter'} + there['nodes'][4]['module'] = 'tail' + there['nodes'][4]['tensors'] = ['tail.weight'] + other = self.make_product('run-0') + self.write(here) + self.write(there, other) + answer = inside_graph.model_graph_compare(self.product, other=other) + rows = {row['path']: row for row in answer['modules']} + self.assertEqual([row['path'] for row in answer['modules']], ['', 'enc', 'enc.0', 'enc.1', 'head', 'tail']) + self.assertEqual({path: row['status'] for path, row in rows.items()}, + {'': 'same', 'enc': 'same', 'enc.0': 'changed', 'enc.1': 'changed', 'head': 'added', + 'tail': 'missing'}) + self.assertEqual(rows['enc.1']['reshaped'], [{'name': 'enc.1.weight', 'shape': [2, 4], 'other_shape': [3, 4]}]) + self.assertEqual((rows['enc.0']['class'], rows['enc.0']['other_class']), + ('torch.nn.Linear', 'torch.nn.LazyLinear')) + self.assertEqual((rows['tail']['class'], rows['tail']['other_parameters']), (None, 9)) + self.assertEqual(answer['counts'], {'same': 2, 'changed': 2, 'added': 1, 'missing': 1}) + + +class Fixtures(ReaderCase): + """The two graphs export_graph wrote (tests/project_structure_fixture_export.py), read as Tuldok reads them.""" + + def test_the_committed_graphs(self): + products = {} + for name in ('parent', 'derived'): + products[name] = self.make_product(name) + self.write((FIXTURES / f'{name}.json').read_text(encoding='utf-8'), products[name]) + answer = inside_graph.model_graph(products[name]) + self.assertEqual((answer['complete'], answer['method'], answer['inputs'][0]['shape']), + (True, 'torch.export', [2, 1, 8, 8])) + for item in answer['nodes']: + self.assertTrue(set(item['tensors']) <= set(answer['tensors']), item['id']) + used = {tensor for item in answer['nodes'] for tensor in item['tensors']} + self.assertEqual(used, set(answer['tensors']), 'every parameter is read by some operation') + adds = [item for item in answer['nodes'] if item['operation'] == 'aten.add.Tensor'] + self.assertEqual([item['module'] for item in adds], ['blocks.0', 'blocks.1']) + edges_in = [edge for edge in answer['edges'] if edge['to'] == adds[0]['id']] + self.assertEqual(len(edges_in), 2, 'the residual add takes the block\'s input and its last convolution') + parameters = sum(math.prod(t['shape']) for t in answer['tensors'].values()) + self.assertEqual(answer['modules']['parameters'], parameters) + folded = inside_graph.model_graph(products[name], depth=0) + self.assertIn('module:blocks', [item['id'] for item in folded['nodes']]) + compared = inside_graph.model_graph_compare(products['derived'], other=products['parent']) + status = {row['path']: row['status'] for row in compared['modules']} + self.assertEqual({path for path, state in status.items() if state != 'same'}, {'head.0', 'head.2', 'gate'}) + self.assertEqual((status['head.0'], status['gate']), ('changed', 'added')) + + +# ------------------------------------------------------------------ through the service + +class GraphProducts(builds_base.BuildCase): + """recipes/fit-a adopted twice, each product with the derived fixture's graph and a weights.npz.""" + + def setUp(self): + super().setUp() + patcher = mock.patch.dict(os.environ, {local.SETTINGS_VARIABLE: str(self.temporary / 'settings.json')}) + patcher.start() + self.addCleanup(patcher.stop) + inside_archive.forget_members() + inside.forget_indexes() + inside_graph.forget() + + def declare(document): + document['run']['outputs'] = {'model': {'path': 'weights.npz', 'role': 'weights'}, + 'graph': {'path': 'model-graph.json', 'role': 'file'}} + self.edit('recipes/fit-a', declare) + graph = json.loads((FIXTURES / 'derived.json').read_text(encoding='utf-8')) + self.runs_made = [] + for scale in (1.0, 2.0): + source = self.temporary / f'weights-{scale}.npz' + source.write_bytes(zip_bytes({name + '.npy': npy(tuple(t['shape']), ' + self.assertNotIn('recipes/early-fit', references) # a blocked subject recipe is not a step + self.assertNotIn('datasets/other', references) # role cases + self.assertFalse([reference for reference in references if reference.startswith(('charts/', 'models/'))]) + + def test_nothing_is_followed_out_of_a_blocked_step(self): + self.mark('datasets/waves', 'no', 'The scenes were made by hand.') + made = self.make('recipes/stepper-k1') + self.assertEqual(_references(made), ['recipes/stepper-k1']) # not its dataset, solver or environment + self.assertEqual(made['counts'], {'build': 0, 'set-up': 0, 'blocked': 1, 'ready': 0, 'built': 0}) + self.assertEqual(made['steps'][0]['blocked_by'], [{'reference': 'datasets/waves', 'rebuild': 'no', + 'reason': 'The scenes were made by hand.'}]) + # A cause is a step when it is a target itself. + both = self.make('recipes/stepper-k1', 'datasets/waves') + self.assertEqual(sorted(_references(both)), ['datasets/waves', 'recipes/stepper-k1']) + + def test_a_blocked_subject_recipe_and_what_only_it_uses_are_not_steps(self): + extra = copy.deepcopy(self.documents['datasets/waves']) + extra['title'] = 'Extra scenes' + self.documents['datasets/extra'] = extra + self.documents['recipes/stepper-k2']['run']['inputs']['extra'] = 'datasets/extra' + self.mark('recipes/stepper-k2', 'needs-work') + made = self.make('benchmarks/rollout') + references = _references(made) + for reference in ('recipes/stepper-k2', 'datasets/extra', 'recipes/early-fit'): + self.assertNotIn(reference, references) + self.assertEqual(sorted(references), ['benchmarks/rollout', 'datasets/waves', 'environments/lab-cpu', + 'recipes/stepper-k1', 'steps/solver']) + skipped = _step(made, 'benchmarks/rollout')['subjects']['skipped'] + self.assertEqual([(item['name'], item['recipe']) for item in skipped], + [('stepper-k2', 'recipes/stepper-k2'), ('early-fit', 'recipes/early-fit')]) + self.assertEqual(made['counts']['blocked'], 0) + + def test_an_environment_alone(self): + made = self.make('environments/lab-cpu') + self.assertEqual([(step['reference'], step['action']) for step in made['steps']], + [('environments/lab-cpu', 'set-up')]) + self.assertTrue(made['buildable']) + + def test_a_chart_is_replaced_by_its_benchmark(self): + made = self.make('charts/rollout-ranking') + self.assertEqual(made['targets'], ['benchmarks/rollout']) + self.assertEqual(made['steps'], self.make('benchmarks/rollout')['steps']) + self.assertEqual(made['notes'], ['charts/rollout-ranking is drawn from benchmarks/rollout, ' + 'which is planned in its place.']) + + def test_a_model_is_not_built(self): + made = self.make('models/stepper') + self.assertEqual(made['targets'], ['models/stepper']) + self.assertEqual(made['steps'], []) + self.assertTrue(made['buildable']) + self.assertEqual(made['counts'], {'build': 0, 'set-up': 0, 'blocked': 0, 'ready': 0, 'built': 0}) + self.assertEqual(made['notes'], ['models/stepper is a model; a model is not built, ' + 'only the recipes that train it.']) + + def test_several_targets_share_their_steps(self): + made = self.make('recipes/stepper-k1', 'steps/solver', 'charts/rollout-ranking', 'benchmarks/rollout') + self.assertEqual(made['targets'], ['recipes/stepper-k1', 'steps/solver', 'benchmarks/rollout']) + references = _references(made) + self.assertEqual(len(references), len(set(references))) + self.assertEqual(sorted(references), sorted(_references(self.make('benchmarks/rollout')))) + self.assertTrue(self.make('recipes/stepper-k1', 'steps/solver')['buildable']) + + def test_every(self): + made = self.make(every=True) + self.assertTrue(made['every']) + self.assertEqual(made['targets'], ['benchmarks/rollout', 'datasets/waves', 'recipes/early-fit', + 'recipes/stepper-k1', 'recipes/stepper-k2', 'steps/solver']) + self.assertEqual(sorted(_references(made)), sorted(made['targets'] + ['environments/lab-cpu'])) + self.assertFalse(made['buildable']) # recipes/early-fit is a target + self.assertEqual(made['blocked_targets'], ['recipes/early-fit']) + + def test_an_unknown_target(self): + with self.assertRaisesRegex(ValueError, 'There is no definition datasets/nothing.'): + self.make('datasets/nothing') + + +class Steps(PlanCase): + """Order, actions and the fields of a step: section 3.""" + + def test_order_puts_each_step_after_what_it_uses(self): + made = self.make('benchmarks/rollout') + references = _references(made) + self.assertEqual([step['number'] for step in made['steps']], list(range(1, len(references) + 1))) + self.assertEqual(references, graph.order({reference: self.documents[reference] for reference in references})) + position = {reference: index for index, reference in enumerate(references)} + for edge in graph.edges(self.documents): + if edge.from_ref in position and edge.to_ref in position: + self.assertLess(position[edge.to_ref], position[edge.from_ref], edge) + + def test_every_action(self): + self.mark('recipes/stepper-k2', 'needs-work') + made = self.make('benchmarks/rollout', 'recipes/stepper-k2', 'recipes/early-fit') + actions = {step['reference']: (step['state'], step['action']) for step in made['steps']} + self.assertEqual(actions, { + 'environments/lab-cpu': ('not-set-up', 'set-up'), + 'steps/solver': ('not-built', 'build'), + 'datasets/waves': ('not-built', 'build'), + 'recipes/stepper-k1': ('not-built', 'build'), + 'recipes/stepper-k2': ('needs-work', 'blocked'), + 'recipes/early-fit': ('cannot-rebuild', 'blocked'), + 'benchmarks/rollout': ('not-built', 'build'), + }) + self.assertEqual(made['counts'], {'build': 4, 'set-up': 1, 'blocked': 2, 'ready': 0, 'built': 0}) + self.assertFalse(made['buildable']) + self.assertEqual(made['blocked_targets'], ['recipes/stepper-k2', 'recipes/early-fit']) + + def test_fields_of_a_step(self): + made = self.make('recipes/stepper-k2') + step = _step(made, 'recipes/stepper-k2') + self.assertEqual(step['kind'], 'recipe') + self.assertEqual(step['title'], 'Stepper fine-tuned from k1') + self.assertEqual(step['command'][:3], ['python', 'lab/train.py', '--start']) + self.assertIn('{inputs.start}', step['command']) # placeholders not filled in + self.assertEqual(step['environment'], 'environments/lab-cpu') + self.assertEqual(step['inputs'], {'start': {'reference': 'recipes/stepper-k1', 'output': 'weights'}, + 'training': {'reference': 'datasets/waves', 'output': None}}) + self.assertEqual(step['resources'], {name: None for name in plan.RESOURCES}) + self.assertIsNone(step['estimate']) + self.assertEqual((step['blocked_by'], step['blocked_by_more']), ([], 0)) + self.assertNotIn('subjects', step) + self.assertEqual(_step(made, 'recipes/stepper-k1')['resources'], + {'gpu': False, 'cpu_threads': 4, 'memory_gb': 2, 'wall_hours': 0.5, 'disk_gb': 0.1}) + environment = _step(made, 'environments/lab-cpu') + self.assertEqual((environment['command'], environment['environment'], environment['inputs']), (None, None, {})) + # A dataset's estimate command is never run: the estimate stays null. + self.assertIsNone(_step(made, 'datasets/waves')['estimate']) + + +class Causes(PlanCase): + """blocked_by: root causes, nearest first, at most five.""" + + def test_a_cause_two_inputs_away_and_no_blocked_ancestor_in_between(self): + self.mark('steps/solver', 'no', 'The solver source was lost.') + made = self.make('recipes/stepper-k1') + self.assertFalse(made['buildable']) + step = _step(made, 'recipes/stepper-k1') + self.assertEqual(step['action'], 'blocked') + # datasets/waves is blocked too, but its own label is not the cause; neither is a step. + self.assertEqual(step['blocked_by'], [{'reference': 'steps/solver', 'rebuild': 'no', + 'reason': 'The solver source was lost.'}]) + self.assertEqual(_references(made), ['recipes/stepper-k1']) + + def test_nearest_first_then_by_reference(self): + self.mark('recipes/stepper-k2', 'needs-work') + self.mark('steps/solver', 'no') + self.mark('recipes/stepper-k1', 'needs-work') + self.mark('datasets/waves', 'needs-work') + step = _step(self.make('recipes/stepper-k2'), 'recipes/stepper-k2') + self.assertEqual([(cause['reference'], cause['rebuild']) for cause in step['blocked_by']], [ + ('recipes/stepper-k2', 'needs-work'), # itself, no edge away + ('datasets/waves', 'needs-work'), # one input away; before recipes/ by reference + ('recipes/stepper-k1', 'needs-work'), # one input away + ('steps/solver', 'no'), # two inputs away by the shortest way + ]) + + def test_at_most_five_with_a_count_of_the_rest(self): + definitions = {'recipes/many': {'kind': 'recipe', 'title': 'Many', 'run': { + 'command': ['true'], 'inputs': {f'in_{index}': f'datasets/d{index}' for index in range(7)}}}} + for index in range(7): + definitions[f'datasets/d{index}'] = {'kind': 'dataset', 'title': f'D{index}', + 'earlier_work': {'rebuild': 'no', 'reason': f'Lost {index}.'}} + step = _step(plan.make_plan(definitions, ['recipes/many']), 'recipes/many') + self.assertEqual([cause['reference'] for cause in step['blocked_by']], + [f'datasets/d{index}' for index in range(5)]) + self.assertEqual(step['blocked_by_more'], 2) + + def test_other_edges_carry_no_cause(self): + self.documents['models/stepper']['earlier_work'] = {'rebuild': 'no', 'reason': 'Lost.'} + made = self.make('recipes/stepper-k1') + self.assertTrue(made['buildable']) + self.assertEqual(_step(made, 'recipes/stepper-k1')['action'], 'build') + + +class Subjects(PlanCase): + """A benchmark runs the subjects it can and is never blocked by one.""" + + def test_a_blocked_recipe_is_skipped_and_a_subject_without_a_recipe_runs(self): + self.documents['benchmarks/rollout']['subjects'].append({'name': 'baseline', 'title': 'Baseline'}) + made = self.make('benchmarks/rollout') + step = _step(made, 'benchmarks/rollout') + self.assertEqual(step['action'], 'build') + self.assertEqual(step['subjects'], { + 'run': ['stepper-k1', 'stepper-k2', 'baseline'], + 'skipped': [{'name': 'early-fit', 'recipe': 'recipes/early-fit', + 'reason': 'its recipe recipes/early-fit is blocked'}]}) + self.assertEqual((step['blocked_by'], step['blocked_by_more']), ([], 0)) + self.assertTrue(made['buildable']) + + def test_one_runnable_subject_is_enough(self): + self.mark('recipes/stepper-k2', 'no') + step = _step(self.make('benchmarks/rollout'), 'benchmarks/rollout') + self.assertEqual((step['action'], step['subjects']['run']), ('build', ['stepper-k1'])) + + def test_no_subject_to_run(self): + self.mark('recipes/stepper-k1', 'no') # blocks stepper-k2 through its input as well + made = self.make('benchmarks/rollout') + step = _step(made, 'benchmarks/rollout') + self.assertEqual(_references(made), ['benchmarks/rollout']) # nothing is followed out of it + self.assertEqual(step['state'], 'not-built') + self.assertEqual(step['action'], 'blocked') + self.assertEqual(step['subjects']['run'], []) + self.assertEqual(step['blocked_by'], []) + self.assertFalse(made['buildable']) + + +class Totals(PlanCase): + """Section 4: build steps only, with how many declare no figure.""" + + def test_fixture_figures(self): + made = self.make('benchmarks/rollout') + self.assertEqual((made['disk_gb'], made['disk_unknown']), (0.1, 4)) + self.assertEqual((made['wall_hours'], made['wall_unknown']), (0.6, 3)) # solver 0.1 + stepper-k1 0.5 + self.assertEqual(made['gpu_steps'], 0) + + def test_every_figure_declared_and_blocked_steps_not_counted(self): + for index, reference in enumerate(('steps/solver', 'datasets/waves', 'recipes/stepper-k1', + 'recipes/stepper-k2', 'benchmarks/rollout'), 1): + self.documents[reference]['run']['resources'] = {'gpu': index % 2 == 0, 'disk_gb': index, + 'wall_hours': index / 4} + self.mark('recipes/stepper-k2', 'needs-work') + made = self.make('benchmarks/rollout') + self.assertEqual(made['counts']['build'], 4) + self.assertEqual((made['disk_gb'], made['disk_unknown']), (1 + 2 + 3 + 5, 0)) + self.assertEqual((made['wall_hours'], made['wall_unknown']), (2.75, 0)) + self.assertEqual(made['gpu_steps'], 1) # datasets/waves; stepper-k2 is blocked + + def test_nothing_to_build(self): + made = self.make('recipes/early-fit') + self.assertEqual((made['disk_gb'], made['disk_unknown'], made['wall_hours'], made['wall_unknown'], + made['gpu_steps']), (0, 0, 0, 0, 0)) + + +@unittest.skipUnless(DATABASE, 'TULDOK_DATABASE_URL is not set; database tests need a disposable cluster.') +class BuildPlan(unittest.TestCase): + """build_plan reads the database copy and nothing else.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + base = Path(directory.name) / 'minimal' + shutil.copytree(FIXTURE, base) + self.folder = base / 'project' + self.project_id = uuid.uuid4().hex + project_file = self.folder / 'tuldok.json' + document = json.loads(project_file.read_text(encoding='utf-8')) + document['id'] = self.project_id + project_file.write_text(json.dumps(document, indent=2) + '\n', encoding='utf-8') + self.addCleanup(store.drop_project, self.project_id) + self.project = store.open_project(self.folder) + self.addCleanup(self.project.close) + + def test_the_same_plan_as_from_the_files(self): + self.assertEqual(plan.build_plan(self.project, ['benchmarks/rollout']), + plan.make_plan(_fixture_documents(), ['benchmarks/rollout'])) + self.assertEqual(plan.build_plan(self.project, [], every=True)['targets'], + plan.every_target(_fixture_documents())) + + def test_unknown_references(self): + for reference, message in (('datasets/nothing', 'There is no definition datasets/nothing.'), + ('nothing', "'nothing' is not a reference such as datasets/.")): + with self.assertRaises(ProjectError) as raised: + plan.build_plan(self.project, ['steps/solver', reference]) + self.assertEqual((raised.exception.code, str(raised.exception)), ('not-found', message)) + + def test_nothing_is_run_and_no_file_is_opened(self): + def refuse(*args, **kwargs): + raise AssertionError(f'A plan must not run or open anything: {args!r}') + + expected = plan.make_plan(_fixture_documents(), [], every=True) + patches = [mock.patch.object(subprocess, 'Popen', refuse), mock.patch.object(os, 'system', refuse), + mock.patch.object(os, 'fork', refuse), mock.patch.object(os, 'posix_spawn', refuse), + mock.patch.object(os, 'posix_spawnp', refuse), mock.patch.object(os, 'execv', refuse), + mock.patch.object(os, 'execve', refuse), mock.patch.object(os, 'open', refuse), + mock.patch.object(builtins, 'open', refuse), mock.patch.object(io, 'open', refuse)] + for patcher in patches: + patcher.start() + try: + made = plan.build_plan(self.project, [], every=True) + finally: + for patcher in patches: + patcher.stop() + self.assertEqual(made, expected) + with self.assertRaises(AssertionError): # the guard works + with mock.patch.object(builtins, 'open', refuse): + open(self.folder / 'tuldok.json') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_pumas.py b/tests/test_tuldok_project_pumas.py new file mode 100644 index 00000000..a1b3425f --- /dev/null +++ b/tests/test_tuldok_project_pumas.py @@ -0,0 +1,879 @@ +"""Generating datasets and suggesting annotations through a Pumas endpoint (spec-d5.md, slices D5a and D5b). + +Everything is asked of tests/fake_pumas_typed.py, a fake made from the +released 0.8.0 contract that this test starts on a loopback port of its own, +serving a /v1/served-models listing the test writes (the vision fixture's +models and four more). No running Pumas is contacted and this machine's +listeners are never scanned: find_endpoints is tested with the scanner +patched. The settings file and the projects home are throwaway paths. + +The check of a definition is tested on its own. Everything else needs a +database and runs only when TULDOK_DATABASE_URL names a disposable cluster: a +Git repository holding a project folder and its code side by side, whose +datasets are generated for real by builds.execute (the step runs as +python -m tuldok_project.steps generate, against the fake), and a Service +for the operations. No GPU, no environment. +""" +import base64 +import copy +import hashlib +import io +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import tempfile +import unittest +from unittest import mock +import uuid +import wave + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from PIL import Image # noqa: E402 + +import fake_pumas_typed # noqa: E402 +import gateway_discovery # noqa: E402 +from tuldok_project import annotations, builds, check, inside, local, plan, pumas, steps, store # noqa: E402 +from tuldok_project.service import Service # noqa: E402 +from tuldok_project.store import ProjectError # noqa: E402 + +DATABASE = os.environ.get('TULDOK_DATABASE_URL') +GIT = ['git', '-c', 'user.name=Tuldok Test', '-c', 'user.email=test@example.invalid', '-c', 'commit.gpgsign=false'] +RIGHTS = 'Made by a model for this test; anything may be done with it.' +FIXTURES = Path(__file__).resolve().parent / 'fixtures' +OWNER = {'actor_id': 'owner-1', 'kind': 'owner', 'label': 'The owner', + 'scopes': ['approve-execution', 'cancel', 'edit', 'execute', 'inspect']} +EDITOR = {'actor_id': 'agent-1', 'kind': 'agent', 'label': 'helper', 'scopes': ['inspect', 'edit']} +TEXT_BOUNDS = [{'option': 'max_tokens', 'minimum': 1.0, 'maximum': 4096.0}, + {'option': 'temperature', 'minimum': 0.0, 'maximum': 2.0}, + {'option': 'top_p', 'minimum': 0.0, 'maximum': 1.0}] +IMAGE_BOUNDS = [{'option': 'width', 'minimum': 1.0, 'maximum': 4294967295.0}, + {'option': 'height', 'minimum': 1.0, 'maximum': 4294967295.0}, + {'option': 'seed', 'minimum': 0.0, 'maximum': 4294967295.0}, + {'option': 'input_characters', 'minimum': 1.0, 'maximum': 4000.0}] + + +# ----------------------------------------------------------- the listing + +def _capability(name, inputs, outputs, input_modalities, output_modalities, bounds=(), reason=None, task=None): + return {'capability': name, 'semantic_task': task or name, 'input_formats': list(inputs), + 'output_formats': list(outputs), 'input_modalities': list(input_modalities), + 'output_modalities': list(output_modalities), 'streaming': False, + 'availability': {'state': 'unavailable', 'reason': reason} if reason else {'state': 'available'}, + 'option_bounds': [dict(bound) for bound in bounds]} + + +def _model(model, profile, capabilities, inputs, outputs, provider='llama_cpp'): + return {'model': model, 'profile': profile, 'model_id': f'test/{model}', 'provider': provider, + 'load_state': 'loaded', 'profile_required': False, 'modalities': {'input': inputs, 'output': outputs}, + 'capabilities': capabilities} + + +def listing(): + """The vision fixture's served models (controlled-vision on two profiles, controlled-image) and four more.""" + value = json.loads((FIXTURES / 'pumas-image-to-text-v1' / 'served-models.json').read_bytes()) + value['models'] += [ + _model('writer', 'writer-cpu', [ + _capability('chat_generation', ['messages_text'], ['text'], ['text'], ['text'], TEXT_BOUNDS), + _capability('text_generation', ['text'], ['text'], ['text'], ['text'], TEXT_BOUNDS), + _capability('image_generation', ['text'], ['png_base64'], ['text'], ['image'], IMAGE_BOUNDS, + reason='unsupported_adapter', task='text_to_image')], ['text'], ['text']), + _model('painter', 'painter-gpu', [ + _capability('image_generation', ['text'], ['png_base64'], ['text'], ['image'], IMAGE_BOUNDS, + task='text_to_image'), + _capability('chat_generation', ['messages_text'], ['text'], ['text'], ['text'], TEXT_BOUNDS, + reason='unsupported_adapter')], ['text'], ['image'], provider='torch'), + _model('decider', 'decider-cpu', [ + _capability('decision', ['decision'], ['decisions'], ['text', 'image', 'audio'], ['json'])], + ['text', 'image', 'audio'], ['json']), + _model('listener', 'listener-cpu', [ + _capability('audio_transcription', ['pcm_s16le', 'pcm_f32le'], ['text'], ['audio'], ['text'], + [{'option': 'max_output_tokens', 'minimum': 512.0, 'maximum': 512.0}])], + ['audio'], ['text'], provider='torch'), + ] + return value + + +def png(size=(16, 8), color=(20, 120, 200)): + out = io.BytesIO() + Image.new('RGB', size, color).save(out, 'PNG') + return out.getvalue() + + +def wav(seconds=0.25, rate=16000): + out = io.BytesIO() + with wave.open(out, 'wb') as sound: + sound.setnchannels(1) + sound.setsampwidth(2) + sound.setframerate(rate) + sound.writeframes(b'\x01\x00' * int(seconds * rate)) + return out.getvalue() + + +def closed_port(): + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + return probe.getsockname()[1] + + +def text_result(text): + return {'kind': 'text', 'text': text, 'finish_reason': 'stop'} + + +# ------------------------------------------------------------ definitions + +def _generate(settings, inputs=None): + run = {'step': 'generate', 'settings': {'rights': RIGHTS, **settings}} + if inputs is not None: + run['inputs'] = inputs + return {'format': 'tuldok_definition_v1', 'kind': 'dataset', 'title': 'Generated', 'run': run} + + +def _import(settings): + return {'format': 'tuldok_definition_v1', 'kind': 'dataset', 'title': 'Imported', + 'run': {'step': 'import', 'settings': {'rights': RIGHTS, **settings}}} + + +def _task(dataset, task, vocabulary=None, title='A task'): + document = {'format': 'tuldok_definition_v1', 'kind': 'annotation', 'title': title, 'dataset': dataset, + 'task': task} + if vocabulary is not None: + document['vocabulary'] = vocabulary + return document + + +DEFINITIONS = { + # The exact texts a page would write: a text dataset, and an image dataset driven by a dataset of prompts. + 'datasets/lines': _generate({'make': 'text', 'model': 'writer', 'prompt': 'Write line {n} of a short poem.', + 'count': 3, 'seed': 5, 'options': {'max_tokens': 64, 'temperature': 0.7}}), + 'datasets/prompts': _import({'source': 'data/prompts', 'pattern': '*.txt', 'lines': True}), + 'datasets/pictures': _generate({'make': 'image', 'model': 'painter', 'profile': 'painter-gpu', + 'prompt': 'A picture of {text}, number {n}.', 'per': 2, 'seed': 100, + 'options': {'width': 64, 'height': 32}}, inputs={'prompts': 'datasets/prompts'}), + 'datasets/one-line': _generate({'make': 'text', 'model': 'writer', 'prompt': 'One line.', 'count': 1}), + 'datasets/three-lines': _generate({'make': 'text', 'model': 'writer', 'prompt': 'Line {n}.', 'count': 3}), + 'datasets/too-long': _generate({'make': 'text', 'model': 'writer', 'prompt': 'Line {n}.', 'count': 2, + 'options': {'max_tokens': 99999}}), + 'datasets/nobody': _generate({'make': 'text', 'model': 'nobody', 'prompt': 'Line {n}.', 'count': 2}), + 'datasets/no-paint': _generate({'make': 'image', 'model': 'writer', 'prompt': 'Line {n}.', 'count': 2, + 'options': {'width': 8, 'height': 8}}), + 'datasets/mixed': _import({'source': 'data/mixed'}), + 'annotations/captions': _task('datasets/mixed', 'text', title='Captions, transcripts and summaries'), + 'annotations/kinds': _task('datasets/mixed', 'class', ['cat', 'dog', 'car'], title='What it is'), + 'annotations/labels': _task('datasets/mixed', 'tags', ['red', 'blue', 'round'], title='What it has'), + 'annotations/quality': _task('datasets/mixed', 'review', title='Quality'), +} + + +def write_project(root, project_id): + """root/project (the folder, code_root ../code) and root/code (the prompts and a text, an image and a sound).""" + folder, code = root / 'project', root / 'code' + for directory in ('datasets', 'annotations'): + (folder / directory).mkdir(parents=True) + (folder / 'tuldok.json').write_text(json.dumps({'format': 'tuldok_project_v1', 'id': project_id, + 'name': 'Generation test project', 'code_root': '../code'}, + indent=2) + '\n', encoding='utf-8') + for reference, document in DEFINITIONS.items(): + (folder / f'{reference}.json').write_text(json.dumps(document, indent=2) + '\n', encoding='utf-8') + (code / 'data' / 'prompts').mkdir(parents=True) + (code / 'data' / 'prompts' / 'subjects.txt').write_text('a red ball\na blue car\n', encoding='utf-8') + mixed = code / 'data' / 'mixed' + mixed.mkdir() + (mixed / 'note.txt').write_text('A long text about a dog that runs after a red ball.\n', encoding='utf-8') + (mixed / 'photo.png').write_bytes(png()) + (mixed / 'voice.wav').write_bytes(wav()) + (mixed / 'shape.xyz').write_text('0 0 0\n1 1 1\n', encoding='utf-8') + + +def records_of(directory): + with open(Path(directory) / 'records.jsonl', encoding='utf-8') as handle: + return [json.loads(line) for line in handle] + + +# ------------------------------------------------------------------ check + +class Checked(unittest.TestCase): + """check on a generate's settings and input (spec-d5.md section 3), with no database.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + self.folder = self.root / 'project' + write_project(self.root, uuid.uuid4().hex) + + def write(self, reference, document): + (self.folder / f'{reference}.json').write_text(json.dumps(document, indent=2) + '\n', encoding='utf-8') + + def problems(self, reference): + report = check.check_folder(self.folder) + return {(problem.pointer, problem.severity) for problem in report.problems + if problem.path == f'{reference}.json'} + + def test_the_project_checks(self): + report = check.check_folder(self.folder) + self.assertEqual([problem for problem in report.problems if problem.severity == 'error'], []) + + def test_settings_a_generate_refuses(self): + self.write('datasets/lines', _generate({'make': 'video', 'model': ' padded', 'profile': 'a b', + 'prompt': 'A {text} and {other} and {', 'count': 0, 'per': 2, + 'seed': -1, 'options': {'seed': 1, 'Bad': 1, 'max_tokens': 'many'}, + 'extra': True})) + found = self.problems('datasets/lines') + for pointer in ('make', 'model', 'profile', 'prompt', 'count', 'per', 'seed', 'options/seed', 'options/Bad', + 'options/max_tokens', 'extra'): + self.assertIn((f'/run/settings/{pointer}', 'error'), found) + + def test_placeholders_and_the_input(self): + self.write('datasets/lines', _generate({'make': 'text', 'model': 'writer', 'prompt': 'About {text}.', + 'count': 2})) + report = check.check_folder(self.folder) + self.assertIn('The prompt uses {text}; it may use {n} ({text} needs an input of prompts).', + [problem.message for problem in report.problems if problem.path == 'datasets/lines.json']) + self.write('datasets/lines', _generate({'make': 'text', 'model': 'writer', 'prompt': 'About {text}.', + 'count': 2}, inputs={'prompts': 'datasets/prompts'})) + self.assertIn(('/run/settings/count', 'error'), self.problems('datasets/lines')) + self.write('datasets/lines', _generate({'make': 'text', 'model': 'writer', 'prompt': 'One.'})) + self.assertIn(('/run/settings/count', 'error'), self.problems('datasets/lines')) + document = _generate({'make': 'text', 'model': 'writer', 'prompt': 'One.', 'count': 1}, + inputs={'other': 'datasets/prompts'}) + document['run']['annotations'] = ['annotations/kinds'] + self.write('datasets/lines', document) + self.assertTrue({('/run/inputs', 'error'), ('/run/annotations', 'error')} <= self.problems('datasets/lines')) + del document['run']['inputs'], document['run']['annotations'] + document['run']['command'] = ['make'] + self.write('datasets/lines', document) + self.assertIn(('/run/command', 'error'), self.problems('datasets/lines')) + + def test_a_plan_without_an_endpoint_says_so(self): + definitions = {reference: copy.deepcopy(document) for reference, document in DEFINITIONS.items()} + made = plan.make_plan(definitions, ['datasets/pictures'], endpoint=False) + steps_ = {step['reference']: step for step in made['steps']} + self.assertEqual(steps_['datasets/prompts']['action'], 'build') + self.assertEqual(steps_['datasets/pictures']['action'], 'blocked') + self.assertEqual(steps_['datasets/pictures']['command'], ['tuldok', 'generate']) + self.assertEqual(steps_['datasets/pictures']['blocked_by'], + [{'reference': 'datasets/pictures', 'rebuild': 'endpoint', 'reason': plan.NO_ENDPOINT}]) + self.assertFalse(made['buildable']) + self.assertEqual(made['blocked_targets'], ['datasets/pictures']) + self.assertIn('datasets/pictures is generated through a Pumas endpoint, and none is set on this machine.', + made['notes']) + asked = [] + made = plan.make_plan(definitions, ['datasets/prompts'], endpoint=lambda: asked.append(1) or False) + self.assertTrue(made['buildable']) + self.assertEqual(asked, []) # no generate to build: the setting is not even read + made = plan.make_plan(definitions, ['datasets/pictures'], endpoint=lambda: True) + self.assertTrue(made['buildable']) + self.assertEqual({step['action'] for step in made['steps']}, {'build'}) + + +class Settings(unittest.TestCase): + """pumas_endpoint in the settings file, beside project_home.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.file = Path(directory.name) / 'settings.json' + patcher = mock.patch.dict(os.environ, {local.SETTINGS_VARIABLE: str(self.file)}) + patcher.start() + self.addCleanup(patcher.stop) + + def test_set_kept_beside_the_home_and_cleared(self): + self.assertIsNone(local.pumas_endpoint()) + local.set_pumas_endpoint('http://127.0.0.1:8765') + local.set_project_home(str(self.file.parent / 'home')) + self.assertEqual(json.loads(self.file.read_text()), {'format': 'tuldok_settings_v1', + 'pumas_endpoint': 'http://127.0.0.1:8765', + 'project_home': str(self.file.parent / 'home')}) + self.assertEqual(self.file.stat().st_mode & 0o777, 0o600) + local.set_pumas_endpoint(None) + self.assertEqual(json.loads(self.file.read_text()), {'format': 'tuldok_settings_v1', + 'project_home': str(self.file.parent / 'home')}) + with self.assertRaises(ValueError): + local.set_pumas_endpoint('ftp://somewhere') + + def test_a_bad_value_in_the_file(self): + self.file.write_text(json.dumps({'format': 'tuldok_settings_v1', 'pumas_endpoint': 'not an address'})) + with self.assertRaises(local.SettingsError) as raised: + local.pumas_endpoint() + self.assertIn('its pumas_endpoint must be an http or https address', str(raised.exception)) + + +class Routes(unittest.TestCase): + """What a served model is offered for comes from its descriptors alone (no request).""" + + def setUp(self): + self.models = {(entry['model'], entry['profile']): entry + for entry in pumas.client.decode_served_models(json.dumps(listing()).encode())['models']} + + def capabilities(self, model, profile, task, kind): + return [route['capability'] for route in pumas.routes(self.models[model, profile], task, kind)] + + def test_section_four(self): + vision = ('controlled-vision', 'vision-cpu') + self.assertEqual(self.capabilities(*vision, 'text', 'image'), ['image_to_text']) + self.assertEqual(self.capabilities('controlled-vision', 'vision-gpu', 'text', 'image'), []) # unavailable + self.assertEqual(self.capabilities('listener', 'listener-cpu', 'text', 'audio'), ['audio_transcription']) + self.assertEqual(self.capabilities('writer', 'writer-cpu', 'text', 'text'), ['chat_generation']) + for kind in ('text', 'image', 'audio'): + self.assertEqual(self.capabilities('decider', 'decider-cpu', 'class', kind), ['decision']) + self.assertEqual(self.capabilities('decider', 'decider-cpu', 'tags', kind), ['decision']) + self.assertEqual(self.capabilities('writer', 'writer-cpu', 'class', 'text'), ['chat_generation']) + self.assertEqual(self.capabilities(*vision, 'tags', 'image'), ['image_to_text']) + self.assertEqual(self.capabilities('writer', 'writer-cpu', 'class', 'audio'), []) + for task in ('review', 'spans', 'segments'): + for model in self.models: + self.assertEqual(self.capabilities(*model, task, 'image'), []) + for task in ('boxes', 'polygons'): # spec-t4.md section 4: regions asked of a vision model as JSON + self.assertEqual(self.capabilities(*vision, task, 'image'), ['image_to_text']) + self.assertEqual(self.capabilities('writer', 'writer-cpu', task, 'image'), []) + self.assertEqual(self.capabilities(*vision, task, 'text'), []) + for kind in ('pointcloud', 'mesh', 'array', 'file'): + for model in self.models: + self.assertEqual(self.capabilities(*model, 'text', kind), []) + + def test_what_each_model_makes(self): + self.assertEqual(pumas.makes(self.models['writer', 'writer-cpu']), + [{'make': 'text', 'capability': 'chat_generation', 'seed': False, + 'options': {'max_tokens': {'minimum': 1.0, 'maximum': 4096.0}, + 'temperature': {'minimum': 0.0, 'maximum': 2.0}, + 'top_p': {'minimum': 0.0, 'maximum': 1.0}}}]) + self.assertEqual(pumas.makes(self.models['painter', 'painter-gpu']), + [{'make': 'image', 'capability': 'image_generation', 'seed': True, + 'options': {'width': {'minimum': 1.0, 'maximum': 4294967295.0}, + 'height': {'minimum': 1.0, 'maximum': 4294967295.0}}}]) + self.assertEqual(pumas.makes(self.models['decider', 'decider-cpu']), []) + self.assertEqual(pumas.makes(self.models['listener', 'listener-cpu']), []) + + +# ------------------------------------------------------------- the fake + +class Endpoint(unittest.TestCase): + """A fake endpoint, and throwaway settings and projects home.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.temporary = Path(directory.name) + self.settings = self.temporary / 'settings.json' + self.home = self.temporary / 'home' + patcher = mock.patch.dict(os.environ, {local.SETTINGS_VARIABLE: str(self.settings), + local.HOME_VARIABLE: str(self.home)}) + patcher.start() + self.addCleanup(patcher.stop) + self.server, self.base, self.fake = fake_pumas_typed.start() + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.fake['served_models'] = listing() + self.fake['advertise'] = '0.8.0' + + def posts(self): + return [body for body in self.fake['requests']] + + +class Observed(Endpoint): + """endpoint, set_endpoint and find_endpoints as pumas.py answers them, with no project.""" + + def test_endpoint_with_its_models_and_an_unavailable_capability(self): + seen = pumas.observe(self.base) + self.assertEqual((seen['address'], seen['answered'], seen['version'], seen['problem']), + (self.base, True, '0.8.0', None)) + by = {(model['model'], model['profile']): model for model in seen['models']} + self.assertEqual(len(by), 7) + writer = by['writer', 'writer-cpu'] + image = next(row for row in writer['capabilities'] if row['capability'] == 'image_generation') + self.assertEqual((image['available'], image['reason']), (False, 'unsupported_adapter')) + self.assertEqual([made['make'] for made in writer['makes']], ['text']) + self.assertEqual([made['make'] for made in by['painter', 'painter-gpu']['makes']], ['image']) + self.assertEqual(by['controlled-vision', 'vision-cpu']['modalities'], {'input': ['text', 'image'], + 'output': ['text']}) + self.assertEqual(self.posts(), []) + + def test_no_endpoint_and_one_that_does_not_answer(self): + self.assertEqual(pumas.observe(None), {'address': None, 'answered': False, 'version': None, 'models': [], + 'problem': pumas.NO_ENDPOINT}) + seen = pumas.observe(f'http://127.0.0.1:{closed_port()}') + self.assertFalse(seen['answered']) + self.assertIn('Pumas served models unavailable', seen['problem']) + with self.assertRaises(pumas.PumasError) as raised: + pumas.normalize('https://pumas.example.org') + self.assertEqual(raised.exception.code, 'invalid') + + def test_find_endpoints_with_the_scanner_patched(self): + port = int(self.base.rsplit(':', 1)[1]) + with mock.patch.object(gateway_discovery, 'listening_endpoints', + return_value=[('127.0.0.1', closed_port()), ('127.0.0.1', port)]): + found = pumas.find() + self.assertEqual(found, {'endpoints': [{'address': self.base, 'version': '0.8.0', 'component': 'pumas-rpc'}], + 'rejected': []}) + + +@unittest.skipUnless(DATABASE, 'TULDOK_DATABASE_URL is not set; database tests need a disposable cluster.') +class Project(Endpoint): + """A committed project, built for real by builds.execute against the fake, and a Service for it.""" + + def setUp(self): + super().setUp() + self.root = self.temporary / 'repository' + self.root.mkdir() + self.project_id = uuid.uuid4().hex + write_project(self.root, self.project_id) + self.folder = self.root / 'project' + self.git('init', '-q') + self.git('add', '-A') + self.git('commit', '-q', '-m', 'The project.') + self.addCleanup(store.drop_project, self.project_id) + self.service = Service(self.folder) + self.addCleanup(self.service.close) + + def git(self, *arguments): + return subprocess.run([*GIT, '-C', str(self.root), *arguments], check=True, capture_output=True, + timeout=60).stdout + + def call(self, name, arguments=None, actor=OWNER): + result = self.service.call(name, arguments or {}, actor) + json.dumps(result, allow_nan=False) + return result + + def refused(self, name, arguments=None, actor=OWNER): + with self.assertRaises(ProjectError) as raised: + self.service.call(name, arguments or {}, actor) + return raised.exception + + def set_endpoint(self): + return self.call('set_endpoint', {'address': self.base}) + + def build(self, *targets, ok=True): + with store.open_project(self.folder) as project: + made = plan.build_plan(project, list(targets)) + done = builds.execute(self.folder, made, confirm=lambda given: True) + self.assertEqual(done['ok'], ok, done) + return done + + def run_row(self, run_id): + with store.open_project(self.folder) as project: + return builds.run_details(project, run_id) + + def product(self, reference): + with store.open_project(self.folder) as project: + found = builds.products(project, reference) + return found[0] if found else None + + def stderr(self, item): + return (Path(item['directory']) / 'stderr.log').read_text(encoding='utf-8') + + +class EndpointSetting(Project): + + def test_set_cleared_and_refused_when_it_does_not_answer(self): + seen = self.set_endpoint() + self.assertEqual((seen['address'], seen['answered'], len(seen['models'])), (self.base, True, 7)) + self.assertEqual(json.loads(self.settings.read_text())['pumas_endpoint'], self.base) + self.assertEqual(self.call('endpoint', actor=EDITOR)['address'], self.base) + silent = f'http://127.0.0.1:{closed_port()}' + refused = self.refused('set_endpoint', {'address': silent}) + self.assertEqual(refused.code, 'unavailable') + self.assertTrue(str(refused).startswith(f'The Pumas endpoint at {silent} did not answer, so it was not set')) + self.assertEqual(json.loads(self.settings.read_text())['pumas_endpoint'], self.base) + self.assertEqual(self.refused('set_endpoint', {'address': 'https://pumas.example.org'}).code, 'invalid') + self.assertEqual(self.refused('set_endpoint', {'address': self.base}, actor=EDITOR).code, 'forbidden') + self.assertEqual(self.refused('find_endpoints', actor=EDITOR).code, 'forbidden') + cleared = self.call('set_endpoint', {'address': None}) + self.assertEqual((cleared['address'], cleared['answered']), (None, False)) + self.assertNotIn('pumas_endpoint', json.loads(self.settings.read_text())) + with store.open_project(self.folder) as project: + rows = project._execute('SELECT operation, outcome, detail FROM {schema}.activity ORDER BY at, id' + ).fetchall() + self.assertEqual([tuple(row) for row in rows], [('set_endpoint', 'done', 'set'), + ('set_endpoint', 'unavailable', refused.args[0][:400]), + ('set_endpoint', 'invalid', mock.ANY), + ('set_endpoint', 'done', 'cleared')]) + self.assertEqual(self.posts(), []) # asking what it serves is no model operation + + def test_find_endpoints_through_the_service(self): + port = int(self.base.rsplit(':', 1)[1]) + with mock.patch.object(gateway_discovery, 'listening_endpoints', return_value=[('127.0.0.1', port)]): + found = self.call('find_endpoints') + self.assertEqual(found['endpoints'], [{'address': self.base, 'version': '0.8.0', 'component': 'pumas-rpc'}]) + + def test_no_endpoint_set_the_plan_says_so(self): + plan_ = self.call('plan', {'targets': ['datasets/lines']}) + self.assertFalse(plan_['buildable']) + self.assertEqual(plan_['steps'][0]['blocked_by'][0]['reason'], plan.NO_ENDPOINT) + with store.open_project(self.folder) as project: + made = plan.build_plan(project, ['datasets/lines']) + with self.assertRaises(ProjectError) as raised: + builds.execute(self.folder, made, confirm=lambda given: True) + self.assertEqual(raised.exception.code, 'blocked') + # The setting goes away while a plan made with it is run: the step fails before it starts. + self.set_endpoint() + with store.open_project(self.folder) as project: + made = plan.build_plan(project, ['datasets/lines']) + self.assertTrue(made['buildable']) + self.call('set_endpoint', {'address': None}) + done = builds.execute(self.folder, made, confirm=lambda given: True) + self.assertFalse(done['ok']) + self.assertIn('No Pumas endpoint is set on this machine', done['steps'][0]['problem']) + self.assertEqual(self.posts(), []) + + +class Generated(Project): + """Datasets generated for real (D5a).""" + + def setUp(self): + super().setUp() + self.set_endpoint() + self.fake['respond'] = lambda body: text_result('made: ' + body['input']['text']) + + def test_a_text_dataset_with_its_records_hashes_and_origin(self): + done = self.build('datasets/lines') + [item] = done['steps'] + product = self.product('datasets/lines') + records = records_of(product['directory']) + self.assertEqual([record['id'] for record in records], ['g000001', 'g000002', 'g000003']) + first = records[0] + self.assertEqual(list(first), ['id', 'kind', 'text', 'sha256', 'bytes', 'rights', 'origin']) + text = 'made: Write line 1 of a short poem.' + self.assertEqual((first['kind'], first['text'], first['rights']), ('text', text, RIGHTS)) + self.assertEqual((first['sha256'], first['bytes']), (hashlib.sha256(text.encode()).hexdigest(), len(text))) + # The capability took no seed (text_generation options have none): null, and none was sent. + self.assertEqual(first['origin'], {'generated': {'model': 'writer', 'profile': 'writer-cpu', + 'capability': 'chat_generation', + 'prompt': 'Write line 1 of a short poem.', 'seed': None, + 'options': {'max_tokens': 64, 'temperature': 0.7}, + 'input': None}}) + posts = self.posts() + self.assertEqual(len(posts), 3) + self.assertEqual({key: posts[2][key] for key in ('model', 'profile', 'capability', 'input', 'output', + 'options', 'stream')}, + {'model': 'writer', 'profile': 'writer-cpu', 'capability': 'chat_generation', + 'input': {'kind': 'text', 'text': 'Write line 3 of a short poem.'}, 'output': 'text', + 'options': {'kind': 'text_generation', 'max_tokens': 64, 'temperature': 0.7}, + 'stream': False}) + row = self.run_row(item['run_id']) + self.assertEqual(row['endpoint'], {'address': self.base, 'version': '0.8.0'}) + self.assertEqual(row['argv'][3], 'generate') + self.assertIsNone(row['environment']) + listed = inside.records(inside.Product(product['run_id'], product['directory']), limit=10) + self.assertEqual((listed['total'], listed['problems'], listed['kinds']), (3, [], {'text': 3})) + self.assertEqual(listed['records'][0]['sha256'], first['sha256']) + stdout = (Path(item['directory']) / 'stdout.log').read_text(encoding='utf-8') + self.assertIn('"stage":"generate","completed":3,"total":3', stdout) + # Built again only when its definition changes. + self.assertEqual(self.call('plan', {'targets': ['datasets/lines']})['steps'][0]['action'], 'built') + # Nothing of the address is in the Git folder, nor in what was committed. + port = self.base.rsplit(':', 1)[1] + for path in self.root.rglob('*'): + if path.is_file() and '.git' not in path.parts: + data = path.read_bytes() + self.assertNotIn(self.base.encode(), data, path) + self.assertNotIn(f':{port}'.encode(), data, path) + self.assertNotIn(f'127.0.0.1:{port}'.encode(), self.git('log', '-p', '--all')) + + def test_an_image_dataset_from_a_dataset_of_prompts_with_per_and_seeds(self): + self.fake['respond'] = None + done = self.build('datasets/pictures') + self.assertEqual([item['reference'] for item in done['steps']], ['datasets/prompts', 'datasets/pictures']) + prompts = records_of(self.product('datasets/prompts')['directory']) + self.assertEqual([record['text'] for record in prompts], ['a red ball', 'a blue car']) + product = self.product('datasets/pictures') + records = records_of(product['directory']) + self.assertEqual([record['id'] for record in records], ['g000001', 'g000002', 'g000003', 'g000004']) + for number, record in enumerate(records, 1): + source = prompts[(number - 1) // 2] + self.assertEqual((record['kind'], record['path']), ('image', f'files/g{number:06d}.png')) + data = (Path(product['directory']) / record['path']).read_bytes() + self.assertEqual((record['sha256'], record['bytes']), (hashlib.sha256(data).hexdigest(), len(data))) + with Image.open(io.BytesIO(data)) as image: + self.assertEqual((image.format, image.size), ('PNG', (64, 32))) + self.assertEqual(record['origin'], {'generated': { + 'model': 'painter', 'profile': 'painter-gpu', 'capability': 'image_generation', + 'prompt': f'A picture of {source["text"]}, number {number}.', 'seed': 99 + number, + 'options': {'width': 64, 'height': 32}, 'input': source['id']}}) + self.assertEqual([body['options'] for body in self.posts()], + [{'kind': 'image_generation', 'width': 64, 'height': 32, 'seed': seed} + for seed in (100, 101, 102, 103)]) + listed = inside.records(inside.Product(product['run_id'], product['directory']), limit=10) + self.assertEqual((listed['kinds'], listed['problems']), ({'image': 4}, [])) + + def test_an_option_a_model_and_a_capability_fail_before_any_request(self): + for reference, expected in ( + ('datasets/too-long', "The option max_tokens 99999 is outside the bound 1 to 4096 that the endpoint " + "advertises for chat_generation."), + ('datasets/nobody', "The endpoint serves no model 'nobody'"), + ('datasets/no-paint', 'writer (writer-cpu) cannot make image now: image_generation is unavailable ' + '(unsupported_adapter).')): + with self.subTest(reference): + done = self.build(reference, ok=False) + self.assertEqual(done['steps'][0]['state'], 'failed') + stderr = self.stderr(done['steps'][0]) + self.assertIn('Nothing was asked: ', stderr) + self.assertIn(expected, stderr) + self.assertIsNone(self.product(reference)) + self.assertEqual(self.posts(), []) + + def test_not_admitted_is_asked_again_twice_then_fails(self): + self.fake['modes'] = ['not_admitted', 'not_admitted'] + done = self.build('datasets/one-line') + self.assertEqual(len(self.posts()), 3) + self.assertEqual(len({body['request_id'] for body in self.posts()}), 3) + self.assertEqual(len(records_of(self.product('datasets/one-line')['directory'])), 1) + self.assertIn('Item 1 was not admitted (capability_unavailable); asking again in 1 s.', + (Path(done['steps'][0]['directory']) / 'stdout.log').read_text(encoding='utf-8')) + self.fake['requests'].clear() + self.fake['modes'] = ['not_admitted'] * 3 + definition = self.folder / 'datasets' / 'one-line.json' + document = json.loads(definition.read_text()) + document['title'] = 'Generated again' + definition.write_text(json.dumps(document, indent=2) + '\n') + self.git('commit', '-q', '-am', 'Again.') + done = self.build('datasets/one-line', ok=False) + self.assertEqual(len(self.posts()), 3) + stderr = self.stderr(done['steps'][0]) + self.assertIn('Item 1 failed (asked 3 times): Pumas refused the request (capability_unavailable; not ' + 'admitted, no provider effect): fixture provider reason', stderr) + + def test_another_failure_fails_at_once_with_the_item_number(self): + self.fake['modes'] = ['success', 'unknown'] + done = self.build('datasets/three-lines', ok=False) + self.assertEqual(len(self.posts()), 2) + stderr = self.stderr(done['steps'][0]) + self.assertIn('Item 2 failed: Pumas reported provider_failure; provider outcome unknown, no automatic replay: ' + 'fixture provider reason', stderr) + # What was made stays in the failed run's directory, and is no product. + self.assertEqual(len(records_of(Path(done['steps'][0]['directory']) / 'output')), 1) + self.assertIsNone(self.product('datasets/three-lines')) + self.assertEqual(self.run_row(done['steps'][0]['run_id'])['endpoint'], + {'address': self.base, 'version': '0.8.0'}) + + +class Suggested(Project): + """suggest_annotation and suggestion_models, each row of section 4's table (D5b).""" + + def setUp(self): + super().setUp() + self.set_endpoint() + self.build('datasets/mixed') + self.ids = {record['kind']: record['id'] for record in records_of(self.product('datasets/mixed')['directory'])} + self.answers = {} + self.fake['respond'] = self.respond + + def respond(self, body): + if body['capability'] == 'decision': + return {'kind': 'decisions', 'answers': self.answers, 'input_tokens': 7} + return text_result(self.answer) + + def suggest(self, task, kind, model, profile=None, **extra): + arguments = {'task': task, 'record': self.ids[kind], 'model': model, **extra} + if profile: + arguments['profile'] = profile + return self.call('suggest_annotation', arguments, actor=EDITOR) + + def last(self): + return self.posts()[-1] + + def test_suggestion_models_offer_only_what_fits(self): + found = self.call('suggestion_models', {'task': 'captions'}, actor=EDITOR) + self.assertEqual((found['type'], found['address'], found['problem']), ('text', self.base, None)) + self.assertEqual(found['kinds'], { + 'audio': [{'model': 'listener', 'profile': 'listener-cpu', 'capability': 'audio_transcription'}], + 'image': [{'model': 'controlled-vision', 'profile': 'vision-cpu', 'capability': 'image_to_text'}], + 'pointcloud': [], + 'text': [{'model': 'controlled-vision', 'profile': 'vision-cpu', 'capability': 'chat_generation'}, + {'model': 'writer', 'profile': 'writer-cpu', 'capability': 'chat_generation'}]}) + kinds = self.call('suggestion_models', {'task': 'kinds'})['kinds'] + self.assertEqual([row['model'] for row in kinds['audio']], ['decider']) + self.assertEqual({row['model']: row['capability'] for row in kinds['image']}, + {'controlled-vision': 'image_to_text', 'decider': 'decision'}) + # No model fits: none offered. + self.assertEqual(self.call('suggestion_models', {'task': 'quality'})['kinds'], + {'audio': [], 'image': [], 'pointcloud': [], 'text': []}) + self.fake['served_models'] = {**listing(), 'models': [model for model in listing()['models'] + if model['model'] == 'writer']} + kinds = self.call('suggestion_models', {'task': 'kinds'})['kinds'] + self.assertEqual((kinds['audio'], kinds['image'], kinds['pointcloud']), ([], [], [])) + refused = self.refused('suggest_annotation', {'task': 'kinds', 'record': self.ids['audio'], 'model': 'writer'}) + self.assertEqual(refused.code, 'unavailable') + self.assertIn('cannot suggest a value of a class task for a audio record now', str(refused)) + self.call('set_endpoint', {'address': None}) + self.assertEqual(self.call('suggestion_models', {'task': 'kinds'})['problem'], pumas.NO_ENDPOINT) + self.assertEqual(self.refused('suggest_annotation', {'task': 'kinds', 'record': self.ids['text'], + 'model': 'writer'}).code, 'unavailable') + self.assertEqual(self.posts(), []) + + def test_a_caption_by_image_to_text(self): + self.answer = 'A blue rectangle.' + found = self.suggest('captions', 'image', 'controlled-vision', 'vision-cpu') + self.assertEqual(found, {'task': 'annotations/captions', 'record': self.ids['image'], + 'suggestion': {'value': 'A blue rectangle.', + 'from': {'model': 'controlled-vision', 'profile': 'vision-cpu', + 'capability': 'image_to_text'}}}) + body = self.last() + self.assertEqual((body['capability'], body['output'], body['profile']), ('image_to_text', 'text', 'vision-cpu')) + [message] = body['input']['messages'] + self.assertEqual(body['input']['kind'], 'image_messages') + self.assertEqual(message['content'][0], {'kind': 'text', 'text': pumas.DEFAULT_CAPTION}) + self.assertEqual(base64.b64decode(message['content'][1]['data_base64']), png()) + self.assertNotIn('options', body) + # The model must be named with its profile when it is served by two. + self.assertEqual(self.refused('suggest_annotation', {'task': 'captions', 'record': self.ids['image'], + 'model': 'controlled-vision'}).code, 'invalid') + + def test_a_transcript_by_audio_transcription(self): + self.answer = 'hello there' + found = self.suggest('captions', 'audio', 'listener') + self.assertEqual(found['suggestion'], {'value': 'hello there', 'from': { + 'model': 'listener', 'profile': 'listener-cpu', 'capability': 'audio_transcription'}}) + body = self.last() + clip = body['input'] + self.assertEqual({key: clip[key] for key in ('kind', 'encoding', 'sample_rate_hz', 'channels', 'sample_count')}, + {'kind': 'audio', 'encoding': 'pcm_s16le', 'sample_rate_hz': 16000, 'channels': 1, + 'sample_count': 4000}) + self.assertEqual(base64.b64decode(clip['data_base64']), b'\x01\x00' * 4000) + self.assertNotIn('options', body) + + def test_a_summary_by_chat(self): + self.answer = ' A dog chases a ball. ' + found = self.suggest('captions', 'text', 'writer', instruction='Summarize in one sentence.') + self.assertEqual(found['suggestion']['value'], 'A dog chases a ball.') + body = self.last() + self.assertEqual((body['capability'], body['input']['kind']), ('chat_generation', 'messages')) + self.assertEqual(body['input']['messages'], [ + {'role': 'system', 'content': 'Summarize in one sentence.'}, + {'role': 'user', 'content': 'A long text about a dog that runs after a red ball.\n'}]) + + def test_class_and_tags_by_a_decision_on_text_images_and_sounds(self): + self.answers = {'class': {'type': 'choice', 'choice': 'dog', 'probabilities': {'dog': 0.9}, + 'confidence': 0.9}} + for kind in ('text', 'image', 'audio'): + with self.subTest(kind): + found = self.suggest('kinds', kind, 'decider') + self.assertEqual(found['suggestion'], {'value': 'dog', 'from': { + 'model': 'decider', 'profile': 'decider-cpu', 'capability': 'decision'}}) + body = self.last() + self.assertEqual((body['output'], body['input']['kind']), ('decisions', 'decision')) + self.assertEqual(body['input']['questions'], {'class': { + 'type': 'choice', 'instructions': 'Choose the term that fits.', + 'criteria': {'cat': None, 'dog': None, 'car': None}}}) + text, image, sound = self.posts() + self.assertEqual(text['input']['state'], 'A long text about a dog that runs after a red ball.\n') + self.assertEqual(set(image['input']), {'kind', 'state', 'questions', 'images'}) + self.assertEqual(image['input']['images'], [{'encoding': 'png', + 'data_base64': base64.b64encode(png()).decode()}]) + self.assertEqual(set(sound['input']['audio'][0]), {'encoding', 'sample_rate_hz', 'channels', 'sample_count', + 'data_base64'}) + self.answers = {'t0': {'type': 'noul', 'noul': 0.9}, 't1': {'type': 'noul', 'noul': 0.2}, + 't2': {'type': 'noul', 'noul': 0.7}} + self.assertEqual(self.suggest('labels', 'image', 'decider')['suggestion']['value'], ['red', 'round']) + self.assertEqual(sorted(self.last()['input']['questions']), ['t0', 't1', 't2']) + self.assertEqual(self.last()['input']['questions']['t1'], + {'type': 'noul', 'instructions': 'Does the term "blue" apply?'}) + + def test_class_and_tags_by_chat_and_image_to_text_asked_for_terms_only(self): + self.answer = 'dog' + found = self.suggest('kinds', 'text', 'writer') + self.assertEqual(found['suggestion'], {'value': 'dog', 'from': { + 'model': 'writer', 'profile': 'writer-cpu', 'capability': 'chat_generation'}}) + self.assertIn('Answer with exactly one of these terms', self.last()['input']['messages'][0]['content']) + self.assertIn('\ncat\ndog\ncar', self.last()['input']['messages'][0]['content']) + self.answer = '"Cat."' + self.assertEqual(self.refused('suggest_annotation', {'task': 'kinds', 'record': self.ids['image'], + 'model': 'controlled-vision', 'profile': 'vision-cpu'} + ).code, 'unsuitable') + self.answer = 'cat.' + found = self.suggest('kinds', 'image', 'controlled-vision', 'vision-cpu') + self.assertEqual(found['suggestion']['value'], 'cat') + self.assertEqual(self.last()['capability'], 'image_to_text') + self.answer = 'round\nred' + self.assertEqual(self.suggest('labels', 'text', 'writer')['suggestion']['value'], ['red', 'round']) + self.answer = 'red, blue' + self.assertEqual(self.suggest('labels', 'image', 'controlled-vision', 'vision-cpu')['suggestion']['value'], + ['red', 'blue']) + + def test_an_answer_not_made_of_terms_is_refused_with_the_models_words(self): + self.answer = 'a fluffy animal, probably' + refused = self.refused('suggest_annotation', {'task': 'labels', 'record': self.ids['text'], 'model': 'writer'}, + actor=EDITOR) + self.assertEqual(refused.code, 'unsuitable') + self.assertIn("is not made of the task's terms", str(refused)) + self.assertIn('“a fluffy animal, probably”', str(refused)) + self.answers = {'class': {'type': 'choice', 'choice': 'horse', 'probabilities': {}, 'confidence': 0.5}} + refused = self.refused('suggest_annotation', {'task': 'kinds', 'record': self.ids['text'], 'model': 'decider'}) + self.assertEqual(refused.code, 'unsuitable') + self.assertIn("The term 'horse' is not in the vocabulary of the task.", str(refused)) + self.answers = {'class': {'type': 'choice'}} + self.assertEqual(self.refused('suggest_annotation', {'task': 'kinds', 'record': self.ids['text'], + 'model': 'decider'}).code, 'unsuitable') + self.answer = ' ' + self.assertEqual(self.refused('suggest_annotation', {'task': 'captions', 'record': self.ids['text'], + 'model': 'writer'}).code, 'unsuitable') + # A refusal of the endpoint is the endpoint's reason. + self.fake['modes'] = ['not_admitted'] + refused = self.refused('suggest_annotation', {'task': 'captions', 'record': self.ids['text'], + 'model': 'writer'}) + self.assertEqual(refused.code, 'unavailable') + self.assertIn('capability_unavailable', str(refused)) + # Nothing is stored by asking. + with store.open_project(self.folder) as project: + self.assertEqual(project._execute('SELECT count(*) FROM {schema}.activity').fetchone()[0], 1) + + def test_a_kept_suggestion_names_the_model_in_git(self): + self.answer = 'A blue rectangle.' + found = self.suggest('captions', 'image', 'controlled-vision', 'vision-cpu')['suggestion'] + self.call('set_annotation', {'task': 'captions', 'from': found['from'], + 'items': [{'record': self.ids['image'], 'state': 'accepted', + 'value': found['value'] + ' On white.'}]}, actor=EDITOR) + self.call('save') + [line] = (self.folder / 'annotations' / 'captions.jsonl').read_text(encoding='utf-8').splitlines() + kept = json.loads(line) + self.assertEqual((kept['value'], kept['by'], kept['from']), + ('A blue rectangle. On white.', 'agent:helper', + {'model': 'controlled-vision', 'profile': 'vision-cpu', 'capability': 'image_to_text'})) + self.assertEqual(annotations.read(line.encode() + b'\n', 'text', []).problems, []) + + +class StepAlone(Endpoint): + """generate_records as a function on directories: refusals that need no project.""" + + def setUp(self): + super().setUp() + self.output = self.temporary / 'out' + self.output.mkdir() + self.said = [] + + def generate(self, **settings): + return steps.generate_records({'rights': RIGHTS, **settings}, str(self.output), endpoint=self.base, + out=self.said.append, pauses=(0, 0)) + + def test_a_seed_past_the_bound_and_a_long_prompt_fail_before_any_request(self): + with self.assertRaises(steps.StepError) as raised: + self.generate(make='image', model='painter', prompt='x', count=3, seed=4294967294, + options={'width': 8, 'height': 8}) + self.assertIn('seed 4294967296 is outside the bound', str(raised.exception)) + with self.assertRaises(steps.StepError) as raised: + self.generate(make='image', model='painter', prompt='x' * 4001, count=1, options={'width': 8, 'height': 8}) + self.assertIn("The prompt's input_characters 4001 is outside the bound 1 to 4000", str(raised.exception)) + with self.assertRaises(steps.StepError) as raised: + self.generate(make='image', model='painter', prompt='x', count=1, options={'width': 8}) + self.assertIn('image_generation needs the options width and height.', str(raised.exception)) + with self.assertRaises(steps.StepError) as raised: + self.generate(make='text', model='writer', prompt='x', count=1, options={'width': 8}) + self.assertIn("chat_generation takes the options max_tokens, temperature, top_p, not 'width'.", + str(raised.exception)) + self.assertEqual(self.posts(), []) + + def test_a_long_text_is_written_as_a_file(self): + self.fake['respond'] = lambda body: text_result('y' * 70000) + counts = self.generate(make='text', model='writer', prompt='x', count=1) + self.assertEqual(counts['in_files'], 1) + [record] = records_of(self.output) + self.assertEqual((record['kind'], record['path'], record['bytes']), ('text', 'files/g000001.txt', 70000)) + self.assertEqual((self.output / 'files' / 'g000001.txt').read_text(), 'y' * 70000) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_result_rows.py b/tests/test_tuldok_project_result_rows.py new file mode 100644 index 00000000..3fa95af4 --- /dev/null +++ b/tests/test_tuldok_project_result_rows.py @@ -0,0 +1,536 @@ +"""Benchmark results in the database (spec-m3.md sections 1 to 4, slice T7). + +The values-file check and the summaries computed from values are tested +without a database. Everything else runs only when TULDOK_DATABASE_URL names +a disposable cluster: each test copies tests/fixtures/tuldok_project/minimal +with a fresh project id (test_tuldok_project_store.StoreCase) and drops that +project's schema afterwards. The command-line tests run the commands as a +child process, as test_tuldok_project_cli.py does. +""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from tuldok_project import check, result_rows, results # noqa: E402 +from tuldok_project.results import SummaryRow # noqa: E402 +from tuldok_project.store import ProjectError # noqa: E402 +from tuldok_project.store import ProjectError # noqa: E402 +from test_tuldok_project_store import FIXTURE, StoreCase # noqa: E402 + +REPOSITORY = Path(__file__).resolve().parents[1] +VALUES = FIXTURE / 'values' +BENCHMARK = json.loads((FIXTURE / 'project' / 'benchmarks' / 'rollout.json').read_text(encoding='utf-8')) +COMMIT = '0123456789abcdef0123456789abcdef01234567' + + +def values_file(subject='stepper-k1', values=None, **fields): + """The text of a small values file for the fixture's benchmark (the hand-computed case unless values is given).""" + if values is None: + values = [ + {'case': 'a', 'metric': 'rmse', 'value': 1.0, 'group': {'horizon': 1}}, + {'case': 'b', 'metric': 'rmse', 'value': 2.0, 'group': {'horizon': 1}}, + {'case': 'c', 'metric': 'rmse', 'value': None, 'reason': 'diverged', 'group': {'horizon': 1}}, + {'case': 'a', 'metric': 'rmse', 'value': 4.0, 'group': {'horizon': 8}}, + {'case': 'a', 'metric': 'seconds', 'value': 3.0}, + ] + document = {'format': 'tuldok_results_v1', 'benchmark': 'rollout', 'subject': subject, 'commit': COMMIT, + 'produced': '2026-11-10', 'values': values} + document.update(fields) + return json.dumps({key: value for key, value in document.items() if value is not ...}) + + +# The summaries of values_file(), by hand: (metric, group) -> (count, unavailable, mean, median, p10, p90, min, max). +HAND = { + ('rmse', ''): (3, 1, 7 / 3, 2.0, 1.2, 3.6, 1.0, 4.0), # 1, 2, 4 and one null + ('rmse', 'horizon=1'): (2, 1, 1.5, 1.5, 1.1, 1.9, 1.0, 2.0), # 1, 2 and one null + ('rmse', 'horizon=8'): (1, 0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0), + ('seconds', ''): (1, 0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0), +} + + +def find(name): + return BENCHMARK if name == 'rollout' else None + + +def summary_numbers(row): + return (row.count, row.unavailable, row.mean, row.median, row.p10, row.p90, row.min, row.max) + + +class ValuesCheck(unittest.TestCase): + """Section 1: one failing case per rule.""" + + def problems(self, text): + found, problems = result_rows.check_values(text, find, path='v.json') + self.assertIsNone(found) + self.assertTrue(problems) + self.assertTrue(all(problem.severity == 'error' and problem.path == 'v.json' for problem in problems)) + return [(problem.pointer, problem.message) for problem in problems] + + def assertPointer(self, text, pointer): + found = self.problems(text) + self.assertIn(pointer, [p for p, _ in found], found) + + def one(self, **change): + """values_file() with one value changed (None in change removes that field).""" + value = {'case': 'a', 'metric': 'rmse', 'value': 1.0, 'group': {'horizon': 1}} + value.update(change) + return values_file(values=[{k: v for k, v in value.items() if v is not ...}]) + + def test_a_good_file_passes(self): + found, problems = result_rows.check_values(values_file(), find) + self.assertEqual(problems, []) + self.assertEqual((found.benchmark, found.subject, found.commit, found.produced), + ('rollout', 'stepper-k1', COMMIT, '2026-11-10')) + self.assertEqual(found.values[0].group, 'horizon=1') + self.assertEqual(found.values[2].reason, 'diverged') + for path in sorted(VALUES.glob('*.json')): + self.assertEqual(result_rows.check_values(path.read_bytes(), find)[1], [], path.name) + + def test_strict_json(self): + self.assertPointer('{"format": "tuldok_results_v1", "format": "tuldok_results_v1"}', '') + self.assertPointer(values_file().replace('1.0', 'NaN', 1), '') + self.assertPointer('[1, 2]', '') + + def test_size_limit(self): + self.assertPointer(b' ' * (result_rows.VALUES_LIMIT + 1), '') + + def test_format(self): + self.assertPointer(values_file(format='tuldok_results_v2'), '/format') + + def test_required_fields(self): + found = self.problems(values_file(commit=...)) + self.assertIn(('', "The field 'commit' is required."), found) + + def test_unknown_fields(self): + self.assertPointer(values_file(extra=1), '/extra') + self.assertPointer(self.one(units='m'), '/values/0/units') + + def test_benchmark_and_subject(self): + self.assertPointer(values_file(benchmark='nothing'), '/benchmark') + self.assertPointer(values_file(subject='nobody'), '/subject') + + def test_commit_and_produced(self): + self.assertPointer(values_file(commit='abc'), '/commit') + self.assertPointer(values_file(produced='2026-13-01'), '/produced') + self.assertPointer(values_file(produced='yesterday'), '/produced') + + def test_values_is_a_non_empty_list(self): + self.assertPointer(values_file(values={}), '/values') + self.assertPointer(values_file(values=[]), '/values') + self.assertPointer(values_file(values=[3]), '/values/0') + + def test_case(self): + self.assertPointer(self.one(case=''), '/values/0/case') + self.assertPointer(self.one(case='x' * 201), '/values/0/case') + self.assertPointer(self.one(case=...), '/values/0/case') + + def test_metric(self): + self.assertPointer(self.one(metric='speed'), '/values/0/metric') + + def test_value(self): + self.assertPointer(self.one(value='1.0'), '/values/0/value') + self.assertPointer(self.one(value=True), '/values/0/value') + self.assertPointer(self.one(value=10 ** 400), '/values/0/value') + self.assertPointer(self.one(value=...), '/values/0') + + def test_reason(self): + self.assertPointer(self.one(value=None), '/values/0/reason') + self.assertPointer(self.one(value=None, reason=''), '/values/0/reason') + self.assertPointer(self.one(value=None, reason='x' * 401), '/values/0/reason') + self.assertPointer(self.one(reason='measured anyway'), '/values/0/reason') + + def test_group(self): + self.assertPointer(self.one(group=[1]), '/values/0/group') + self.assertPointer(self.one(group={'size': 1}), '/values/0/group/size') + self.assertPointer(self.one(group={'horizon': [1]}), '/values/0/group/horizon') + self.assertPointer(self.one(group={'horizon': None}), '/values/0/group/horizon') + self.assertPointer(self.one(group={'horizon': 'a;b'}), '/values/0/group/horizon') + self.assertPointer(self.one(group={'horizon': 'a=b'}), '/values/0/group/horizon') + + def test_group_values_are_stored_as_text(self): + for written, text in ((1, '1'), (1.5, '1.5'), (2.0, '2.0'), (True, 'true'), (False, 'false'), ('x', 'x')): + found, problems = result_rows.check_values(self.one(group={'horizon': written}), find) + self.assertEqual(problems, []) + self.assertEqual(found.values[0].group, f'horizon={text}') + + def test_case_metric_and_group_are_unique(self): + first = {'case': 'a', 'metric': 'rmse', 'value': 1.0, 'group': {'horizon': 1}} + self.assertPointer(values_file(values=[first, dict(first, value=2.0)]), '/values/1') + # The same case and metric in another group is a different value. + found, problems = result_rows.check_values(values_file(values=[first, dict(first, group={'horizon': 8})]), find) + self.assertEqual(problems, []) + + +class SummariesFromValues(unittest.TestCase): + """Section 3, which summaries exist, against hand-computed numbers.""" + + def test_hand_computed(self): + found, _ = result_rows.check_values(values_file(), find) + rows = result_rows.summaries_from_values(found) + self.assertEqual({(row.metric, row.group): summary_numbers(row) for row in rows}, HAND) + self.assertTrue(all(row.subject == 'stepper-k1' and row.commit == COMMIT and row.produced == '2026-11-10' + for row in rows)) + + def test_no_rows_for_combinations(self): + two_keys = dict(BENCHMARK, summary={'group_by': ['horizon', 'size']}) + text = values_file(values=[{'case': 'a', 'metric': 'rmse', 'value': 1.0, 'group': {'horizon': 1, 'size': 's'}}, + {'case': 'b', 'metric': 'rmse', 'value': 3.0, 'group': {'size': 's'}}]) + found, problems = result_rows.check_values(text, lambda name: two_keys) + self.assertEqual(problems, []) + self.assertEqual(found.values[0].group, 'horizon=1;size=s') + rows = {row.group: row.count for row in result_rows.summaries_from_values(found)} + self.assertEqual(rows, {'': 2, 'horizon=1': 1, 'size=s': 2}) + + def test_fixture_results_file_is_made_from_the_fixture_values(self): + rows = [] + for path in sorted(VALUES.glob('*.json')): + rows += result_rows.summaries_from_values(result_rows.check_values(path.read_bytes(), find)[0]) + self.assertEqual(results.write_results(rows), + (FIXTURE / 'project' / 'results' / 'rollout.csv').read_text(encoding='utf-8')) + + +class ResultsCase(StoreCase): + + @property + def csv(self): + return self.folder / 'results' / 'rollout.csv' + + def stored(self, project, **filters): + return {(item.row.subject, item.row.metric, item.row.group): item + for item in project.result_summaries('rollout', **filters)} + + def rewrite_csv(self, change): + """Rewrite the results file with change(rows) applied to its SummaryRow list.""" + rows = results.read_results(self.csv.read_text(encoding='utf-8')) + self.csv.write_text(results.write_results(change(rows)), encoding='utf-8') + + def replace_row(self, subject, metric, group, **fields): + def change(rows): + return [SummaryRow(**{**vars(row), **fields}) if (row.subject, row.metric, row.group) == (subject, metric, group) + else row for row in rows] + self.rewrite_csv(change) + + +class OpenResults(ResultsCase): + + def test_open_imports_every_summary_row(self): + project = self.open() + stored = self.stored(project) + file_rows = results.read_results(self.csv.read_text(encoding='utf-8')) + self.assertEqual([item.row for item in stored.values()], file_rows) + self.assertEqual({item.origin for item in stored.values()}, {'file'}) + self.assertEqual(project.results_overview(), [{ + 'benchmark': 'rollout', 'subjects_with_results': 3, 'subjects_with_values': 0, 'rows': 12, + 'rows_file': 12, 'rows_values': 0, 'differs_from_file': False}]) + self.assertEqual(project.result_values('rollout', 'stepper-k1'), {'total': 0, 'values': []}) + + def test_group_filters(self): + project = self.open() + self.assertEqual(len(project.result_summaries('rollout')), 12) + self.assertEqual({item.row.group for item in project.result_summaries('rollout', group='')}, {''}) + self.assertEqual({item.row.group for item in project.result_summaries('rollout', group='horizon')}, + {'horizon=1', 'horizon=8'}) + self.assertEqual(project.result_summaries('rollout', group='hor'), []) + self.assertEqual(len(project.result_summaries('rollout', subject='early-fit', metric='rmse')), 3) + with self.assertRaises(ProjectError) as caught: + project.result_summaries('nothing') + self.assertEqual(caught.exception.code, 'not-found') + + def test_unchanged_results_attach_without_the_checker(self): + self.close(self.open()) + before = self.dump() + checker = self.counting_checker() + self.open() + checker.assert_not_called() + self.assertEqual(self.dump(), before) + + def test_changed_results_file_is_imported(self): + self.close(self.open()) + self.replace_row('stepper-k2', 'seconds', '', median=1.4) + checker = self.counting_checker() + project = self.open() + checker.assert_called_once() + self.assertEqual(self.stored(project)['stepper-k2', 'seconds', ''].row.median, 1.4) + + def test_removed_and_added_results_file(self): + self.close(self.open()) + text = self.csv.read_text(encoding='utf-8') + self.csv.unlink() + checker = self.counting_checker() + project = self.open() + self.assertEqual(checker.call_count, 1) + self.assertEqual(project.result_summaries('rollout'), []) + self.assertFalse(project.results_overview()[0]['differs_from_file']) + self.close(project) + self.close(self.open()) # unchanged again: no check + self.assertEqual(checker.call_count, 1) + self.csv.write_text(text, encoding='utf-8') + project = self.open() + self.assertEqual(checker.call_count, 2) + self.assertEqual(len(project.result_summaries('rollout')), 12) + + def test_bad_results_file_refuses_the_open_and_changes_nothing(self): + self.close(self.open()) + before = self.dump() + self.csv.write_text(self.csv.read_text(encoding='utf-8').replace('stepper-k1,', 'nobody,', 1), encoding='utf-8') + with self.assertRaises(ProjectError) as caught: + self.open() + self.assertIn('results/rollout.csv', [problem.path for problem in caught.exception.problems]) + self.assertEqual(self.dump(), before) + + def test_floats_round_trip_exactly(self): + awkward = (0.1 + 0.2, 1e-300, 123456789.12345679, -2.5e-07) + self.replace_row('stepper-k1', 'seconds', '', mean=awkward[0], median=awkward[1], p10=awkward[2], + p90=awkward[3]) + project = self.open() + row = self.stored(project)['stepper-k1', 'seconds', ''].row + self.assertEqual((row.mean, row.median, row.p10, row.p90), awkward) + self.assertEqual(project.status()['results_unsaved'], []) # the database makes the same text again + + +class ImportValues(ResultsCase): + + def setUp(self): + super().setUp() + self.project = self.open() + + def test_import_replaces_the_subjects_values_and_summaries(self): + imported = self.project.import_values(values_file()) + self.assertEqual(imported, {'benchmark': 'rollout', 'subject': 'stepper-k1', 'values': 5, 'summaries': 4}) + stored = self.stored(self.project, subject='stepper-k1') + self.assertEqual({(metric, group): summary_numbers(item.row) for (_, metric, group), item in stored.items()}, + HAND) + self.assertEqual({item.origin for item in stored.values()}, {'values'}) + self.assertEqual({item.row.produced for item in stored.values()}, {'2026-11-10'}) + # The other subjects keep the file's rows. + self.assertEqual({item.origin for item in self.project.result_summaries('rollout', subject='stepper-k2')}, + {'file'}) + held = self.project.result_values('rollout', 'stepper-k1') + self.assertEqual(held['total'], 5) + self.assertIn({'case': 'c', 'metric': 'rmse', 'group': 'horizon=1', 'value': None, 'reason': 'diverged'}, + held['values']) + page = self.project.result_values('rollout', 'stepper-k1', metric='rmse', offset=1, limit=2) + self.assertEqual((page['total'], [row['case'] for row in page['values']]), (4, ['a', 'b'])) + overview = self.project.results_overview()[0] + self.assertEqual((overview['subjects_with_values'], overview['rows_file'], overview['rows_values'], + overview['differs_from_file']), (1, 8, 4, True)) + # A second import of the same subject replaces, it does not add. + self.project.import_values(values_file(values=[{'case': 'z', 'metric': 'seconds', 'value': 9.0}])) + self.assertEqual(self.project.result_values('rollout', 'stepper-k1')['total'], 1) + self.assertEqual(list(self.stored(self.project, subject='stepper-k1')), [('stepper-k1', 'seconds', '')]) + + def test_a_file_with_a_problem_changes_nothing(self): + before = self.dump() + bad = values_file(values=[{'case': 'a', 'metric': 'rmse', 'value': 1.0}, + {'case': 'b', 'metric': 'speed', 'value': 1.0}]) + with self.assertRaises(ProjectError) as caught: + self.project.import_values(bad, path='bad.json') + self.assertEqual(caught.exception.code, 'invalid') + self.assertEqual([(p.path, p.pointer) for p in caught.exception.problems], [('bad.json', '/values/1/metric')]) + self.assertEqual(self.dump(), before) + + def test_values_keep_their_summaries_when_the_results_file_is_reimported(self): + self.project.import_values(values_file()) + self.close(self.project) + self.replace_row('stepper-k1', 'seconds', '', median=99.0) + self.replace_row('stepper-k2', 'seconds', '', median=98.0) + project = self.open() + stored = self.stored(project) + self.assertEqual(stored['stepper-k1', 'seconds', ''].origin, 'values') + self.assertEqual(stored['stepper-k1', 'seconds', ''].row.median, 3.0) + self.assertNotIn(('stepper-k1', 'rmse', 'horizon=8'), [k for k, v in stored.items() if v.origin == 'file']) + self.assertEqual(stored['stepper-k2', 'seconds', ''].row.median, 98.0) + self.assertEqual(stored['stepper-k2', 'seconds', ''].origin, 'file') + + def test_many_values_import_quickly(self): + cases = [{'case': f'scene-{index:05d}', 'metric': metric, 'value': index * 0.001, + 'group': {'horizon': index % 2 * 7 + 1}} for index in range(10000) for metric in ('rmse', 'seconds')] + started = time.monotonic() + imported = self.project.import_values(values_file(values=cases)) + self.assertLess(time.monotonic() - started, 20) + self.assertEqual(imported['values'], 20000) + self.assertEqual(self.project.result_values('rollout', 'stepper-k1', limit=5)['total'], 20000) + self.assertEqual(self.stored(self.project)['stepper-k1', 'rmse', 'horizon=8'].row.count, 5000) + + +class SaveAndStatus(ResultsCase): + + def setUp(self): + super().setUp() + self.project = self.open() + + def test_save_writes_the_results_file_and_status_is_clean_after(self): + self.project.import_values(values_file()) + status = self.project.status() + self.assertEqual((status['results_unsaved'], status['results_changed']), (['rollout'], [])) + saved = self.project.save() + self.assertEqual((saved['results_written'], saved['results_deleted']), (['rollout'], [])) + rows = results.read_results(self.csv.read_text(encoding='utf-8')) + self.assertEqual(rows, [item.row for item in self.project.result_summaries('rollout')]) + self.assertEqual(self.project.status()['results_unsaved'], []) + self.assertTrue(check.check_folder(self.folder).ok) + self.assertEqual(self.project.save()['results_written'], []) # same text: nothing written + self.close(self.project) + checker = self.counting_checker() + self.open() + checker.assert_not_called() + + def test_values_matching_the_file_write_nothing(self): + for path in sorted(VALUES.glob('*.json')): + self.project.import_values(path.read_bytes()) + self.assertEqual(self.project.status()['results_unsaved'], []) + modified = self.csv.stat().st_mtime_ns + self.assertEqual(self.project.save()['results_written'], []) + self.assertEqual(self.csv.stat().st_mtime_ns, modified) + + def test_save_refuses_when_the_results_file_changed_on_disk(self): + self.project.import_values(values_file()) + waves = self.edited('datasets/waves', lambda d: d.update(title='Waves, edited')) + self.project.set_definition('datasets/waves', waves) + self.replace_row('stepper-k2', 'seconds', '', median=1.4) + on_disk = self.csv.read_text(encoding='utf-8') + self.assertEqual(self.project.status()['results_changed'], ['results/rollout.csv']) + with self.assertRaises(ProjectError) as caught: + self.project.save() + self.assertEqual(caught.exception.code, 'changed-on-disk') + self.assertEqual(caught.exception.references, ['results/rollout.csv']) + self.assertEqual(self.csv.read_text(encoding='utf-8'), on_disk) + self.assertNotEqual(self.read('datasets/waves'), waves) # nothing at all was written + + def test_removing_the_benchmark_removes_its_results_and_save_deletes_the_file(self): + # tuldok.json's map.chart uses charts/rollout-progression (spec-w1.md): drop it first. + with self.assertRaises(ProjectError) as caught: + self.project.remove_definition('charts/rollout-progression') + self.assertEqual((caught.exception.code, caught.exception.references), ('in-use', ['tuldok.json'])) + self.close(self.project) + project_file = self.folder / 'tuldok.json' + document = json.loads(project_file.read_text(encoding='utf-8')) + del document['map'] + project_file.write_text(json.dumps(document, indent=2) + '\n', encoding='utf-8') + self.project = self.open() + self.project.import_values(values_file()) + for chart in ('speed-accuracy', 'rollout-ranking', 'rollout-progression', 'rollout-comparison'): + self.project.remove_definition(f'charts/{chart}') + self.project.remove_definition('benchmarks/rollout') + connection = self.database() + for table in ('result_summaries', 'result_values'): + count = connection.execute(f'SELECT count(*) FROM {self.schema}.{table}').fetchone()[0] + self.assertEqual(count, 0, table) + self.assertEqual(self.project.status()['results_unsaved'], ['rollout']) + saved = self.project.save() + self.assertEqual(saved['results_deleted'], ['rollout']) + self.assertFalse(self.csv.exists()) + self.assertEqual(connection.execute(f'SELECT count(*) FROM {self.schema}.results_files').fetchone()[0], 0) + self.assertTrue(check.check_folder(self.folder).ok) + self.assertEqual(self.project.status()['results_unsaved'], []) + + def test_removing_a_subject_removes_its_rows(self): + self.project.import_values(values_file(subject='stepper-k2')) + self.project.set_definition('charts/rollout-ranking', self.edited('charts/rollout-ranking', lambda d: d.update( + subjects=['stepper-k1', 'early-fit']))) + benchmark = self.edited('benchmarks/rollout', lambda d: d.update( + subjects=[s for s in d['subjects'] if s['name'] != 'stepper-k2'])) + self.project.set_definition('benchmarks/rollout', benchmark) + self.assertEqual(self.project.result_summaries('rollout', subject='stepper-k2'), []) + self.assertEqual(self.database().execute( + f'SELECT count(*) FROM {self.schema}.result_values').fetchone()[0], 0) + self.assertEqual(len(self.project.result_summaries('rollout')), 8) + self.project.save() + self.assertTrue(check.check_folder(self.folder).ok) + self.assertNotIn('stepper-k2', self.csv.read_text(encoding='utf-8')) + + +def run(*arguments): + environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1') + return subprocess.run([sys.executable, '-m', 'tuldok_project', *map(str, arguments)], cwd=REPOSITORY, + env=environment, capture_output=True, text=True, timeout=120) + + +class Commands(ResultsCase): + + def setUp(self): + super().setUp() + self.values_files = sorted(self.base.glob('values/*.json')) + + def run_ok(self, *arguments): + done = run(*arguments) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual(done.stderr, '') + return done.stdout + + def test_results(self): + lines = self.run_ok('results', self.folder).splitlines() + self.assertEqual(lines[0].split(), ['benchmark', 'subjects', 'with', 'values', 'rows', 'from', 'file', + 'from', 'values', 'saved']) + self.assertEqual(lines[1].split(), ['rollout', '3', '0', '12', '12', '0', 'yes']) + self.assertEqual(json.loads(self.run_ok('results', self.folder, '--json'))[0]['rows'], 12) + lines = self.run_ok('results', self.folder, 'rollout', '--subject', 'stepper-k1', '--group', '').splitlines() + self.assertEqual(len(lines), 3) + self.assertEqual(lines[1].split()[:4], ['stepper-k1', 'rmse', '-', '8']) + rows = json.loads(self.run_ok('results', self.folder, 'rollout', '--metric', 'rmse', '--group', 'horizon', + '--json')) + self.assertEqual(len(rows), 6) + self.assertEqual(rows[0], {'subject': 'early-fit', 'metric': 'rmse', 'group': 'horizon=1', 'count': 4, + 'unavailable': 0, 'mean': 0.625, 'median': 0.625, 'p10': 0.325, 'p90': 0.925, + 'min': 0.25, 'max': 1.0, 'commit': COMMIT, 'produced': '2026-11-02', + 'origin': 'file'}) + missing = run('results', self.folder, 'nothing') + self.assertEqual((missing.returncode, missing.stderr), (1, 'There is no benchmark benchmarks/nothing.\n')) + + def test_values_before_and_after_import(self): + self.assertIn('holds no per-case values for stepper-k1', + self.run_ok('values', self.folder, 'rollout', '--subject', 'stepper-k1')) + self.assertEqual(json.loads(self.run_ok('values', self.folder, 'rollout', '--subject', 'stepper-k1', '--json')), + {'total': 0, 'values': []}) + output = self.run_ok('import-values', self.folder, *self.values_files) + self.assertEqual(len(output.splitlines()), 3) + self.assertIn('imported benchmarks/rollout early-fit', output) + lines = self.run_ok('values', self.folder, 'rollout', '--subject', 'early-fit', '--limit', '2').splitlines() + self.assertEqual(lines, ['case metric group value', 'scene-01 rmse horizon=1 0.25', + 'scene-01 rmse horizon=8 none (diverged before step 8)', + '2 of 12 values shown; use --limit to see more.']) + held = json.loads(self.run_ok('values', self.folder, 'rollout', '--subject', 'early-fit', '--metric', + 'seconds', '--json')) + self.assertEqual(held['total'], 4) + self.assertEqual(held['values'][1], {'case': 'scene-02', 'metric': 'seconds', 'group': '', 'value': None, + 'reason': 'timer not recorded'}) + overview = json.loads(self.run_ok('results', self.folder, '--json'))[0] + self.assertEqual((overview['rows_values'], overview['differs_from_file']), (12, False)) + self.assertEqual(self.run_ok('status', self.folder), + 'Nothing is unsaved and no file changed on disk since the last import.\n') + + def test_import_values_json_and_refusal(self): + imported = json.loads(self.run_ok('import-values', self.folder, self.values_files[0], '--json')) + self.assertEqual(imported, [{'benchmark': 'rollout', 'subject': 'early-fit', 'values': 12, 'summaries': 4}]) + bad = self.temporary / 'bad.json' + bad.write_text(values_file(subject='stepper-k2', commit='abc'), encoding='utf-8') + before = json.loads(self.run_ok('results', self.folder, '--json')) + done = run('import-values', self.folder, self.values_files[1], bad) + self.assertEqual(done.returncode, 1) + self.assertEqual(done.stdout, '') + self.assertIn(f'error {bad} /commit ', done.stderr) + self.assertIn('Nothing was imported', done.stderr) + self.assertEqual(json.loads(self.run_ok('results', self.folder, '--json')), before) + twice = run('import-values', self.folder, self.values_files[1], self.values_files[1]) + self.assertEqual(twice.returncode, 2) + self.assertIn('same benchmark and subject', twice.stderr) + + def test_save_and_status_report_results(self): + changed = self.temporary / 'k1.json' + changed.write_text(values_file(), encoding='utf-8') + self.run_ok('import-values', self.folder, changed) + self.assertEqual(self.run_ok('status', self.folder), 'results unsaved rollout\n') + self.assertEqual(self.run_ok('save', self.folder), 'wrote results/rollout.csv\n') + self.assertEqual(json.loads(self.run_ok('status', self.folder, '--json'))['results_unsaved'], []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_results.py b/tests/test_tuldok_project_results.py new file mode 100644 index 00000000..b4affca6 --- /dev/null +++ b/tests/test_tuldok_project_results.py @@ -0,0 +1,310 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tuldok_project.results import ( # noqa: E402 + COLUMNS, STATISTICS, SummaryRow, check_results, read_results, summarise, write_results, +) + +HEADER = ','.join(COLUMNS) +COMMIT = '0123456789abcdef0123456789abcdef01234567' +BENCHMARK = { + 'kind': 'benchmark', + 'metrics': [{'id': 'rmse', 'title': 'RMSE', 'better': 'lower'}, + {'id': 'step_time', 'title': 'Step time', 'better': 'lower'}], + 'subjects': [{'name': 'baseline'}, {'name': 'stepper-v2'}], + 'summary': {'group_by': ['resolution', 'case_family']}, +} +GOOD_ROW = f'baseline,rmse,,3,0,2.0,2.0,1.2,2.8,1.0,3.0,{COMMIT},2026-11-02' + + +def row(**changes): + """Return one valid SummaryRow with the given fields changed.""" + fields = dict(subject='baseline', metric='rmse', group='', count=3, unavailable=0, + mean=2.0, median=2.0, p10=1.2, p90=2.8, min=1.0, max=3.0, + commit=COMMIT, produced='2026-11-02') + fields.update(changes) + return SummaryRow(**fields) + + +def results_text(*lines): + return '\n'.join((HEADER,) + lines) + '\n' + + +def replace_cell(line, column, value): + cells = line.split(',') + cells[COLUMNS.index(column)] = value + return ','.join(cells) + + +class SummariseTests(unittest.TestCase): + def assertSummary(self, values, expected, unavailable=0): + summary = summarise(values, unavailable) + self.assertEqual(set(summary), {'count', 'unavailable', *STATISTICS}) + for name, value in expected.items(): + self.assertEqual(summary[name], value, name) + + def test_one_value(self): + self.assertSummary([5], dict(count=1, unavailable=0, mean=5.0, median=5.0, + p10=5.0, p90=5.0, min=5.0, max=5.0)) + + def test_two_values(self): + # Positions: median 0.5, p10 0.1, p90 0.9 between 1 and 3. + self.assertSummary([3, 1], dict(count=2, mean=2.0, median=2.0, p10=1.2, p90=2.8, + min=1.0, max=3.0)) + + def test_odd_count(self): + # Ascending 1..5, count - 1 = 4: median at 2, p10 at 0.4, p90 at 3.6. + self.assertSummary([3, 1, 2, 5, 4], dict(count=5, mean=3.0, median=3.0, p10=1.4, + p90=4.6, min=1.0, max=5.0)) + + def test_even_count(self): + # Ascending 1..4, count - 1 = 3: median at 1.5, p10 at 0.3, p90 at 2.7. + self.assertSummary([4, 2, 3, 1], dict(count=4, mean=2.5, median=2.5, p10=1.3, + p90=3.7, min=1.0, max=4.0)) + + def test_positions_on_whole_ranks(self): + # Eleven values 0..10: p10 at exactly 1, p90 at exactly 9. + self.assertSummary(list(range(11)), dict(count=11, mean=5.0, median=5.0, + p10=1.0, p90=9.0)) + + def test_ties(self): + # Ascending 2, 2, 2, 7: median at 1.5 is 2; p90 at 2.7 is 2 + 0.7 * 5. + self.assertSummary([2, 7, 2, 2], dict(count=4, mean=3.25, median=2.0, p10=2.0, + p90=5.5, min=2.0, max=7.0)) + + def test_negative_values(self): + # Ascending -10, -4, -1: p10 at 0.2 is -10 + 0.2 * 6; p90 at 1.8 is -4 + 0.8 * 3. + summary = summarise([-4, -1, -10]) + self.assertEqual((summary['mean'], summary['median'], summary['min'], summary['max']), + (-5.0, -4.0, -10.0, -1.0)) + self.assertAlmostEqual(summary['p10'], -8.8, places=12) + self.assertAlmostEqual(summary['p90'], -1.6, places=12) + + def test_zero_values(self): + self.assertSummary([0, 0, 0], dict(count=3, mean=0.0, median=0.0, p10=0.0, p90=0.0, + min=0.0, max=0.0)) + + def test_zero_count(self): + self.assertSummary([], dict(count=0, unavailable=3, **{name: None for name in STATISTICS}), + unavailable=3) + + def test_rejects_non_finite_values_and_bad_unavailable(self): + with self.assertRaises(ValueError): + summarise([1.0, float('nan')]) + with self.assertRaises(ValueError): + summarise([1.0], unavailable=-1) + + +class RoundTripTests(unittest.TestCase): + def test_rows_round_trip_exactly_and_are_sorted(self): + rows = [ + row(subject='stepper-v2', metric='step_time', mean=0.1 + 0.2, median=1e-05, + p10=-0.0, p90=1.5e+16, min=-3.25, max=123456789.123456789), + row(group='case_family=plume;resolution=64', commit='', produced='2026-11-02T10:30:00Z'), + row(count=0, unavailable=4, **{name: None for name in STATISTICS}), + row(metric='step_time', group='case_family=a "quoted", value'), + ] + text = write_results(rows) + self.assertTrue(text.startswith(HEADER + '\n')) + self.assertNotIn('\r', text) + self.assertIn('"case_family=a ""quoted"", value"', text) + back = read_results(text) + self.assertEqual(back, sorted(rows, key=lambda r: (r.subject, r.metric, r.group))) + self.assertEqual(write_results(back), text) + self.assertEqual(repr(back[-1].mean), repr(0.1 + 0.2)) + + def test_empty_cells_for_missing_statistics(self): + text = write_results([row(count=0, **{name: None for name in STATISTICS})]) + self.assertEqual(text.splitlines()[1], f'baseline,rmse,,0,0,,,,,,,{COMMIT},2026-11-02') + + def test_written_file_passes_the_check(self): + rows = [row(), row(subject='stepper-v2', group='resolution=32')] + self.assertEqual(check_results(write_results(rows), BENCHMARK), []) + + def test_read_rejects_malformed_files(self): + for text in ('', 'subject,metric\n', results_text('baseline,rmse'), + results_text(replace_cell(GOOD_ROW, 'count', 'three')), + results_text(replace_cell(GOOD_ROW, 'mean', 'nan')), + results_text('"baseline,rmse')): + with self.subTest(text=text), self.assertRaises(ValueError): + read_results(text) + + +class CheckResultsTests(unittest.TestCase): + def assertProblem(self, text, severity, pointer, fragment): + problems = check_results(text, BENCHMARK) + matching = [p for p in problems if p[0] == severity and p[1] == pointer and fragment in p[2]] + self.assertTrue(matching, f'no {severity} at {pointer!r} mentioning {fragment!r}: {problems}') + + def assertBadCell(self, column, value, fragment): + self.assertProblem(results_text(GOOD_ROW, replace_cell(GOOD_ROW.replace('baseline', 'stepper-v2'), + column, value)), + 'error', 'line 3', fragment) + + def test_valid_file_has_no_problems(self): + self.assertEqual(check_results(results_text(GOOD_ROW), BENCHMARK), []) + + def test_header_only_file_is_valid(self): + self.assertEqual(check_results(HEADER + '\n', BENCHMARK), []) + + # UTF-8, line ends, quoting, header. + def test_not_utf8(self): + problems = check_results(b'\xff' + results_text(GOOD_ROW).encode(), BENCHMARK) + self.assertEqual(problems, [('error', '', 'The results file is not UTF-8.')]) + + def test_utf8_bytes_are_accepted(self): + self.assertEqual(check_results(results_text(GOOD_ROW).encode('utf-8'), BENCHMARK), []) + + def test_carriage_return_line_ends(self): + self.assertProblem(results_text(GOOD_ROW).replace('\n', '\r\n'), 'error', '', '\\r') + + def test_missing_final_line_end(self): + self.assertProblem(results_text(GOOD_ROW).rstrip('\n'), 'warning', '', 'line end') + + def test_bad_quoting(self): + self.assertProblem(results_text(GOOD_ROW, 'baseline,"rm"se,,0,0,,,,,,,,2026-11-02'), + 'error', 'line 3', 'RFC 4180') + + def test_empty_file(self): + self.assertProblem('', 'error', '', 'empty') + + def test_wrong_header(self): + self.assertProblem(results_text(GOOD_ROW).replace('p10,p90', 'p90,p10'), + 'error', 'line 1', 'first line') + + def test_quoted_header_is_not_exact(self): + text = '"subject"' + results_text(GOOD_ROW)[len('subject'):] + self.assertProblem(text, 'error', 'line 1', 'first line') + + def test_wrong_cell_count(self): + self.assertProblem(results_text(GOOD_ROW + ',extra'), 'error', 'line 2', 'cells') + + def test_empty_line(self): + self.assertProblem(results_text(GOOD_ROW, '', GOOD_ROW.replace('baseline', 'stepper-v2')), + 'error', 'line 3', 'Empty lines') + + # Subject and metric. + def test_unknown_subject(self): + self.assertBadCell('subject', 'nobody', 'subject') + + def test_unknown_metric(self): + self.assertBadCell('metric', 'accuracy', 'metric') + + # Group. + def test_group_key_not_in_group_by(self): + self.assertBadCell('group', 'seed=1', 'summary.group_by') + + def test_group_keys_unsorted(self): + self.assertBadCell('group', 'resolution=64;case_family=plume', 'sorted') + + def test_group_key_repeated(self): + self.assertBadCell('group', 'resolution=32;resolution=64', 'repeat') + + def test_group_pair_without_equals(self): + self.assertBadCell('group', 'resolution', 'key=value') + + def test_group_value_with_equals(self): + self.assertBadCell('group', 'resolution=6=4', 'key=value') + + def test_group_value_with_line_break(self): + line = replace_cell(GOOD_ROW, 'group', '"resolution=6\n4"') + self.assertProblem(results_text(line), 'error', 'line 2', 'line break') + + def test_group_without_group_by_in_benchmark(self): + benchmark = dict(BENCHMARK, summary={}) + problems = check_results(results_text(replace_cell(GOOD_ROW, 'group', 'resolution=64')), benchmark) + self.assertEqual([(p[0], p[1]) for p in problems], [('error', 'line 2')]) + + # Count, unavailable and statistics. + def test_count_not_an_integer(self): + self.assertBadCell('count', '2.5', 'count') + + def test_count_negative(self): + self.assertBadCell('count', '-1', 'count') + + def test_unavailable_not_an_integer(self): + self.assertBadCell('unavailable', 'none', 'unavailable') + + def test_statistic_not_a_decimal_number(self): + self.assertBadCell('median', 'nan', 'median') + + def test_statistic_infinite(self): + self.assertBadCell('max', 'inf', 'max') + + def test_statistics_filled_when_count_is_zero(self): + self.assertBadCell('count', '0', 'empty when count is 0') + + def test_statistics_missing_when_count_above_zero(self): + self.assertBadCell('p90', '', 'missing') + + def test_statistics_outside_summary_statistics_may_be_empty(self): + benchmark = dict(BENCHMARK, summary={'statistics': ['median']}) + line = f'baseline,rmse,,3,0,,2.0,,,,,{COMMIT},2026-11-02' + self.assertEqual(check_results(results_text(line), benchmark), []) + + def test_zero_count_with_empty_statistics_is_valid(self): + line = 'baseline,rmse,,0,5,,,,,,,,2026-11-02' + self.assertEqual(check_results(results_text(line), BENCHMARK), []) + + # Commit and produced. + def test_commit_uppercase(self): + self.assertBadCell('commit', COMMIT.upper(), 'commit') + + def test_commit_short(self): + self.assertBadCell('commit', COMMIT[:7], 'commit') + + def test_produced_wrong_form(self): + self.assertBadCell('produced', '2026-11-02 10:30', 'produced') + + def test_produced_not_a_real_date(self): + self.assertBadCell('produced', '2026-02-30', 'produced') + + def test_produced_empty(self): + self.assertBadCell('produced', '', 'produced') + + def test_produced_with_time(self): + line = replace_cell(GOOD_ROW, 'produced', '2026-11-02T23:59:59Z') + self.assertEqual(check_results(results_text(line), BENCHMARK), []) + + # Uniqueness and order. + def test_duplicate_row(self): + self.assertProblem(results_text(GOOD_ROW, GOOD_ROW), 'error', 'line 3', 'repeats') + + def test_rows_out_of_order(self): + later = GOOD_ROW.replace('baseline', 'stepper-v2') + self.assertProblem(results_text(later, GOOD_ROW), 'error', 'line 3', 'out of order') + + def test_order_is_by_group_as_string(self): + first = replace_cell(GOOD_ROW, 'group', 'resolution=128') + second = replace_cell(GOOD_ROW, 'group', 'resolution=64') + self.assertEqual(check_results(results_text(first, second), BENCHMARK), []) + self.assertProblem(results_text(second, first), 'error', 'line 3', 'out of order') + + # Collecting and robustness. + def test_collects_every_problem(self): + line = 'nobody,accuracy,seed=1,x,-1,,,,,,,ABC,yesterday' + problems = check_results(results_text(line), BENCHMARK) + self.assertGreaterEqual(len(problems), 7) + self.assertTrue(all(p[:2] == ('error', 'line 2') for p in problems)) + + def test_line_numbers_follow_physical_lines(self): + quoted = replace_cell(GOOD_ROW, 'group', '"resolution=6\n4"') + bad = replace_cell(GOOD_ROW.replace('baseline', 'stepper-v2'), 'metric', 'accuracy') + self.assertProblem(results_text(quoted, bad), 'error', 'line 4', 'metric') + + def test_never_raises_on_bad_input(self): + for text, benchmark in ((None, BENCHMARK), (results_text(GOOD_ROW), None), + (results_text(GOOD_ROW), {'subjects': 'x', 'metrics': [1], + 'summary': {'group_by': 5}}), + ('\x00,"', BENCHMARK)): + with self.subTest(text=text, benchmark=benchmark): + problems = check_results(text, benchmark) + self.assertTrue(problems) + self.assertTrue(all(p[0] in ('error', 'warning') for p in problems)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_service.py b/tests/test_tuldok_project_service.py new file mode 100644 index 00000000..2f132662 --- /dev/null +++ b/tests/test_tuldok_project_service.py @@ -0,0 +1,515 @@ +"""The operations table and the service of a served project folder (spec-m7.md, sections 2, 3 and 5). + +The table is tested as data. Everything else needs a database and runs only +when TULDOK_DATABASE_URL names a disposable cluster. The service tests start +from test_tuldok_project_builds.BuildCase (a Git-committed copy of the +buildable fixture with its environment connected, dropped afterwards); the +results tests from test_tuldok_project_store.StoreCase (a copy of the +minimal fixture). Every result is passed through json.dumps(allow_nan=False). +The builds run the fixture's stand-in scripts; no GPU is used. +""" +import json +import os +from pathlib import Path +import sys +import unittest + +import jsonschema +import psycopg + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from metadata import connection_info # noqa: E402 +from tuldok_project import operations, store # noqa: E402 +from tuldok_project.service import Service # noqa: E402 +from tuldok_project.store import ProjectError # noqa: E402 +import test_tuldok_project_builds as builds_base # noqa: E402 +import test_tuldok_project_store as store_base # noqa: E402 + +ALL_SCOPES = ['admin', 'approve-execution', 'approve-review', 'cancel', 'edit', 'execute', 'expose-holdout', + 'freeze', 'inspect'] +OWNER = {'actor_id': 'owner-1', 'kind': 'owner', 'label': 'The owner', 'scopes': ALL_SCOPES} + + +def agent(*scopes): + return {'actor_id': f'agent-{"-".join(scopes) or "none"}', 'kind': 'agent', 'label': 'An agent', + 'scopes': list(scopes)} + + +EVERY_AGENT_SCOPE = agent('inspect', 'edit', 'execute', 'cancel', 'freeze', 'expose-holdout') +REQUEST_OPERATIONS = ('request_build', 'approve_build', 'decline_build', 'cancel_build', 'build_requests', + 'build_request') +SECRET = 'not-to-be-sent-7f3a' + + +class Table(unittest.TestCase): + """operations.OPERATIONS as data.""" + + def test_every_operation_is_declared_once(self): + names = [operation.name for operation in operations.OPERATIONS] + self.assertEqual(len(names), 92) # spec-t6.md section 6 (7), spec-t7.md (2), and the rows of spec-m7.md section 2, of section 2 of spec-w1.md and spec-w3.md, of section 3 of spec-d1.md and spec-d2.md, of section 5 of spec-d4.md, of sections 2 and 4 of spec-d5.md, of section 2 of spec-d6.md, of section 3 of spec-t2.md, of section 4 of spec-t1.md, of sections 2 to 4 of spec-t3.md and spec-t5.md (7), and of section 6 of spec-t4.md (4) + self.assertEqual(len(set(names)), len(names)) + for name in REQUEST_OPERATIONS: + self.assertIsNotNone(operations.by_name(name)) + self.assertIsNone(operations.by_name('nothing')) + self.assertIs(operations.by_name('summary'), operations.OPERATIONS[0]) + + def test_every_schema_is_closed_and_valid(self): + for operation in operations.OPERATIONS: + with self.subTest(operation.name): + jsonschema.Draft202012Validator.check_schema(operation.arguments) + self.assertEqual(operation.arguments['type'], 'object') + self.assertIs(operation.arguments['additionalProperties'], False) + self.assertTrue(set(operation.arguments['required']) <= set(operation.arguments['properties'])) + + def test_every_row_has_a_summary_sentence_a_rule_and_result_keys(self): + for operation in operations.OPERATIONS: + with self.subTest(operation.name): + summary = operation.summary + self.assertTrue(summary[0].isupper() and summary.endswith('.'), summary) + self.assertNotIn('. ', summary[:-1]) + self.assertIn(operation.who, operations.WHO) + self.assertIsInstance(operation.mutation, bool) + self.assertTrue(operation.result) + self.assertEqual(len(set(operation.result)), len(operation.result)) + + def test_mutations_and_owner_rows(self): + mutations = {operation.name for operation in operations.OPERATIONS if operation.mutation} + self.assertEqual(mutations, {'set_definition', 'remove_definition', 'revert_definition', 'save', 'resolve_conflict', + 'import_values', 'check_environment', 'connect_environment', + 'disconnect_environment', 'request_build', 'approve_build', 'decline_build', + 'cancel_build', 'cancel_run', 'stop_run', 'set_annotation', 'revert_annotations', + 'set_endpoint', 'set_embedder', 'keep_annotation', + 'serve_start', 'serve_approve', 'serve_stop', 'serve_values'}) + owner = {operation.name for operation in operations.OPERATIONS if operation.who == 'owner'} + self.assertEqual(owner, {'connect_environment', 'disconnect_environment', 'approve_build', 'decline_build', + 'set_endpoint', 'find_endpoints', 'set_embedder', 'serve_approve'}) + + def test_error_status(self): + self.assertEqual([operations.error_status(code) for code in + ('not-found', 'invalid', 'forbidden', 'conflict', 'unavailable', 'stale')], + [404, 400, 403, 409, 409, 409]) + + +def plain(test, result): + """The result as JSON text; fails the test when it is not plain JSON.""" + test.assertIsInstance(result, dict) + return json.dumps(result, allow_nan=False) + + +class ServiceCase(builds_base.BuildCase): + """BuildCase with a Service for the copy.""" + + def setUp(self): + super().setUp() + self.service = Service(self.folder) + self.service.start() + self.addCleanup(self.service.close) + + def call(self, name, arguments=None, actor=OWNER): + result = self.service.call(name, arguments or {}, actor) + plain(self, result) + return result + + def refused(self, name, arguments=None, actor=OWNER): + with self.assertRaises(ProjectError) as caught: + self.service.call(name, arguments or {}, actor) + return caught.exception + + def activity(self): + return self.call('activity', {'limit': 500})['items'] + + +class WhoRules(ServiceCase): + + def test_inspect(self): + self.assertEqual(self.call('summary', actor=agent('inspect'))['worker'], 'running') + unstarted = Service(self.folder) + self.addCleanup(unstarted.close) + self.assertEqual(unstarted.call('summary', {}, OWNER)['worker'], 'stopped') # before start() + self.call('summary', actor=OWNER) + self.assertEqual(self.refused('summary', actor=agent()).code, 'forbidden') + self.assertEqual(self.refused('summary', actor=agent('edit', 'cancel')).code, 'forbidden') + self.assertEqual(self.refused('summary', actor={**OWNER, 'scopes': ['edit']}).code, 'forbidden') + + def test_edit(self): + self.assertEqual(self.refused('save', actor=agent('inspect', 'cancel', 'execute')).code, 'forbidden') + self.call('save', actor=agent('edit')) + self.call('save', actor=OWNER) + + def test_cancel(self): + self.assertEqual(self.refused('cancel_run', {'run': 'abcdef12'}, actor=agent('inspect', 'edit')).code, + 'forbidden') + # Past the actor check, the run is looked for. + self.assertEqual(self.refused('cancel_run', {'run': 'abcdef12'}, actor=agent('cancel')).code, 'not-found') + self.assertEqual(self.refused('cancel_run', {'run': 'abcdef12'}, actor=OWNER).code, 'not-found') + + def test_request(self): + arguments = {'targets': ['benchmarks/score']} + self.assertEqual(self.refused('request_build', arguments, actor=agent('inspect', 'edit', 'cancel')).code, + 'forbidden') + self.assertEqual(self.refused('request_build', arguments, actor=agent('execute')).code, 'forbidden') + # Past the actor check, the request is made (an agent's waits) or refused for what it asks. + self.assertEqual(self.call('request_build', arguments, actor=agent('inspect', 'execute'))['state'], + 'awaiting-approval') + self.assertEqual(self.refused('request_build', {'targets': []}, actor=OWNER).code, 'nothing-to-build') + + def test_owner(self): + arguments = {'reference': 'environments/test-python', 'interpreter': sys.executable} + self.assertEqual(self.refused('connect_environment', arguments, actor=EVERY_AGENT_SCOPE).code, 'forbidden') + self.assertEqual(self.refused('approve_build', {'request': 'abcdef12'}, actor=EVERY_AGENT_SCOPE).code, + 'forbidden') + self.assertEqual(self.refused('connect_environment', arguments, + actor={**OWNER, 'scopes': ['inspect', 'edit']}).code, 'forbidden') + self.assertTrue(self.call('connect_environment', arguments)['ok']) + self.assertEqual(self.refused('approve_build', {'request': 'abcdef12'}).code, 'not-found') + + def test_forbidden_calls_write_no_activity(self): + self.refused('save', actor=agent('inspect')) + self.refused('disconnect_environment', {'reference': 'environments/test-python'}, actor=EVERY_AGENT_SCOPE) + self.assertEqual(self.activity(), []) + + def test_unknown_operation(self): + self.assertEqual(self.refused('drop_everything').code, 'not-found') + + +class Arguments(ServiceCase): + + def test_invalid_arguments_are_refused_with_one_problem_per_error(self): + refused = self.refused('results', {'benchmark': '', 'limit': 5000, 'colour': 'red'}) + self.assertEqual(refused.code, 'invalid') + self.assertEqual(len(refused.problems), 3) + self.assertEqual(sorted(problem.pointer for problem in refused.problems), ['/', '/benchmark', '/limit']) + self.assertEqual(self.refused('plan', {'targets': ['x'] * 51}).code, 'invalid') + self.assertEqual(self.refused('summary', ['not', 'an', 'object']).code, 'invalid') + self.assertEqual(self.refused('chart', {}).code, 'invalid') + + def test_an_invalid_mutation_is_recorded(self): + self.assertEqual(self.refused('set_definition', {'reference': 'recipes/fit-b'}).code, 'invalid') + [row] = self.activity() + self.assertEqual((row['operation'], row['target'], row['outcome']), ('set_definition', 'recipes/fit-b', + 'invalid')) + + +class Reads(ServiceCase): + """Every read operation answers with its declared keys and plain JSON.""" + + def test_every_read(self): + self.build_all() + run_id = self.runs('recipes/fit-a')[0]['id'] + arguments = { + 'summary': {}, 'status': {}, 'check': {}, 'definitions': {'kind': 'recipe'}, + 'definition': {'reference': 'recipes/fit-a'}, 'graph': {}, 'results_overview': {}, + 'results': {'benchmark': 'benchmarks/score', 'limit': 3}, 'values': {'benchmark': 'score', 'subject': 'fit-a'}, + 'chart': {'chart': 'score-ranking', 'svg': True, 'theme': 'dark'}, 'report': {'theme': 'dark'}, + 'plan': {'targets': ['benchmarks/score']}, 'estimates': {}, 'environments': {}, + 'environment': {'reference': 'environments/test-python'}, 'runs': {'limit': 2}, + 'run': {'run': run_id}, 'progress': {'run': run_id[:8]}, 'products': {}, 'activity': {}, + 'map': {}, 'note': {'path': 'missing.md'}, 'recipe_results': {'recipe': 'recipes/fit-a'}, + 'draft_branch': {'recipe': 'recipes/fit-a', 'title': 'Fit c'}, + 'records': {'run': run_id}, 'record': {'run': run_id, 'id': 'weights.json'}, 'inside': {'run': run_id}, + 'array_slice': {'run': run_id, 'path': 'weights.npy'}, + 'array_summary': {'run': run_id, 'path': 'weights.npy'}, + 'weights': {'run': run_id}, + 'weights_compare': {'run': run_id, 'path': 'weights.json', 'other_run': run_id, + 'other_path': 'weights.json'}, + 'array_tiles': {'run': run_id, 'path': 'weights.npy'}, + 'fields': {'run': run_id}, 'field_compare': {'run': run_id, 'entry': 0, 'case': 0}, + 'model_graph': {'run': run_id}, 'model_graph_overlay': {'run': run_id, 'kind': 'parameters'}, + 'model_graph_compare': {'run': run_id, 'other_run': run_id}, + 'training': {'run': run_id}, 'training_series': {'run': run_id, 'series': [{'name': 'loss'}]}, + 'training_histograms': {'run': run_id, 'name': 'w', 'kind': 'weights'}, + 'training_samples': {'run': run_id}, 'training_sample': {'run': run_id, 'index': 0}, + 'model_compare': {'recipes': ['recipes/fit-a', 'recipes/fit-b']}, 'benchmark_axes': {'benchmark': 'score'}, + 'intervals': {'benchmark': 'benchmarks/score'}, + 'served': {}, 'serve_session': {'session': 'a' * 32}, + 'serve_predict': {'model': 'fit-a', 'inputs': {}}, + 'annotation_tasks': {}, 'annotated_records': {'task': 'none-here'}, + 'annotation': {'task': 'none-here', 'record': 'r1'}, + 'annotation_sets': {'task': 'none-here'}, 'annotation_agreement': {'task': 'none-here'}, + 'annotation_compare': {'task': 'none-here', 'record': 'r1', 'sets': ['kept']}, + 'endpoint': {}, 'find_endpoints': {}, 'suggestion_models': {'task': 'none-here'}, + 'suggest_annotation': {'task': 'none-here', 'record': 'r1', 'model': 'm'}, + 'embeddings': {'dataset': 'datasets/numbers'}, + 'embedding_map': {'run': run_id, 'name': 'none-here'}, + 'embedding_neighbours': {'run': run_id, 'name': 'none-here', 'record': 'r1'}, + 'embedding_duplicates': {'run': run_id, 'name': 'none-here'}, + 'embedding_outliers': {'run': run_id, 'name': 'none-here'}, + 'embedding_leakage': {'run': run_id, 'name': 'none-here'}, + 'sweeps': {}, + 'sweep': {'sweep': 'none-here'}, + } + reads = [operation for operation in operations.OPERATIONS if not operation.mutation] + arguments['build_requests'] = {'state': 'queued', 'limit': 5} + self.assertEqual({operation.name for operation in reads} - set(arguments), {'build_request'}) + for operation in reads: + with self.subTest(operation.name): + if operation.name == 'build_request': # tested with a request in test_tuldok_project_build_requests + self.assertEqual(self.refused(operation.name, {'request': 'abcdef12'}).code, 'not-found') + continue + if operation.name in ('array_slice', 'array_summary', 'array_tiles'): # no arrays here; tested in + # test_tuldok_project_inside and test_tuldok_project_weights + self.assertEqual(self.refused(operation.name, arguments[operation.name]).code, 'not-found') + continue + if operation.name in ('model_graph_overlay', 'model_graph_compare', 'training_histograms', + 'training_sample'): # fit-a has no model-graph.json (model_graph + # answers path null) and logged nothing; test_tuldok_project_model_graph, _training + self.assertEqual(self.refused(operation.name, arguments[operation.name]).code, 'not-found') + continue + if operation.name == 'field_compare': # fit-a keeps no fields; test_tuldok_project_fields + self.assertEqual(self.refused(operation.name, arguments[operation.name]).code, 'not-found') + continue + if operation.name == 'weights_compare': # fit-a's weights are JSON; test_tuldok_project_weights + self.assertEqual(self.refused(operation.name, arguments[operation.name]).code, 'unsupported') + continue + if operation.name == 'find_endpoints': # the owner's, and it scans this machine: tested with the + # scanner patched in test_tuldok_project_pumas + self.assertEqual(self.refused(operation.name, {}, actor=agent('inspect')).code, 'forbidden') + continue + if operation.name in ('suggestion_models', 'suggest_annotation'): # no task here; asked in + # test_tuldok_project_pumas + self.assertEqual(self.refused(operation.name, arguments[operation.name]).code, 'not-found') + continue + if operation.name in ('annotated_records', 'annotation', 'annotation_sets', 'annotation_agreement', + 'annotation_compare'): # this fixture has no task; read in + # test_tuldok_project_annotation_rows and test_tuldok_project_annotation_agreement + self.assertEqual(self.refused(operation.name, arguments[operation.name]).code, 'not-found') + continue + if operation.name.startswith('embedding_'): # no embeddings here; test_tuldok_project_curation + self.assertEqual(self.refused(operation.name, arguments[operation.name]).code, 'not-found') + continue + if operation.name in ('serve_session', 'serve_predict'): # nothing served; test_tuldok_project_serving + self.assertIn(self.refused(operation.name, arguments[operation.name]).code, + ('not-found', 'not-served')) + continue + if operation.name == 'sweep': # this fixture has no sweep; read in test_tuldok_project_sweeps + self.assertEqual(self.refused(operation.name, arguments['sweep']).code, 'not-found') + continue + if operation.name == 'note': # this fixture has no notes; read in test_tuldok_project_workspace + self.assertEqual(self.refused(operation.name, arguments['note']).code, 'not-found') + continue + result = self.call(operation.name, arguments[operation.name], actor=agent('inspect')) + self.assertTrue(set(operation.result) <= set(result), set(operation.result) - set(result)) + self.assertEqual(self.activity(), []) # reads are not recorded + + def test_shapes(self): + self.build_all() + results = self.call('results', {'benchmark': 'score', 'limit': 3, 'offset': 1}) + self.assertEqual(len(results['rows']), 3) + self.assertEqual(results['total'], 4) # two subjects, two metrics, no groups + self.assertEqual(self.call('runs', {'limit': 2})['runs'][0]['reference'], 'benchmarks/score') + self.assertEqual(len(self.call('runs', {'limit': 2})['runs']), 2) + chart = self.call('chart', {'chart': 'charts/score-ranking', 'svg': True}) + self.assertTrue(chart['svg'].startswith('')) + summary = self.call('summary') + self.assertEqual(summary['folder'], str(self.folder)) + self.assertEqual(self.call('plan', {'targets': ['benchmarks/score']})['counts']['build'], 0) + progress = self.call('progress', {'run': self.runs('datasets/numbers')[0]['id']}) + self.assertTrue(progress['ended']) + self.assertEqual(set(progress['stdout']), {'text', 'offset', 'size', 'skipped'}) + later = self.call('progress', {'run': progress['run']['id'], 'stdout_offset': progress['stdout']['offset']}) + self.assertEqual(later['stdout']['text'], '') + self.assertIsInstance(self.call('runs')['runs'][0]['queued_at'], str) + self.assertEqual(self.refused('definition', {'reference': 'recipes/none'}).code, 'not-found') + self.assertEqual(self.refused('chart', {'chart': 'Not A Name'}).code, 'not-found') + + def test_run_never_returns_variable_values(self): + made = self.plan('datasets/numbers') + done = builds_base.builds.execute(self.folder, made, confirm=lambda given: True, + extra_env={'SERVICE_SECRET': SECRET}) + self.assertTrue(done['ok'], done) + run_id = done['steps'][0]['run_id'] + with self.opened() as project: + env = project._execute('SELECT env FROM {schema}.runs WHERE id = %s', (run_id,)).fetchone()[0] + self.assertEqual(env['SERVICE_SECRET'], SECRET) + result = self.call('run', {'run': run_id}) + self.assertNotIn('env', result) + self.assertEqual(result['env_names'], sorted(env)) + self.assertIn('SERVICE_SECRET', result['env_names']) + for name in ('run', 'progress', 'runs', 'products'): + given = {} if name in ('runs', 'products') else {'run': run_id} + self.assertNotIn(SECRET, plain(self, self.call(name, given)), name) + + +class Changes(ServiceCase): + + def edited(self, reference, change): + document = self.call('definition', {'reference': reference})['document'] + change(document) + return json.dumps(document, indent=2) + '\n' + + def test_set_remove_and_save(self): + text = self.edited('recipes/fit-b', lambda document: document.update(title='Fit b, edited')) + shown = self.call('set_definition', {'reference': 'recipes/fit-b', 'text': text}, actor=agent('edit')) + self.assertEqual((shown['title'], shown['unsaved'], shown['text']), ('Fit b, edited', True, text)) + self.assertEqual(self.call('remove_definition', {'reference': 'charts/score-ranking'}), + {'removed': 'charts/score-ranking'}) + status = self.call('status') + self.assertEqual((status['unsaved'], status['removed']), (['recipes/fit-b'], ['charts/score-ranking'])) + saved = self.call('save') + self.assertEqual((saved['written'], saved['deleted']), (['recipes/fit-b'], ['charts/score-ranking'])) + self.assertEqual((self.folder / 'recipes' / 'fit-b.json').read_text(encoding='utf-8'), text) + self.assertFalse((self.folder / 'charts' / 'score-ranking.json').exists()) + + refused = self.refused('set_definition', {'reference': 'recipes/fit-b', 'text': '{"format": 1}'}) + self.assertEqual(refused.code, 'invalid') + self.assertTrue(refused.problems) + self.assertEqual(self.refused('remove_definition', {'reference': 'datasets/numbers'}).code, 'in-use') + + rows = [(row['operation'], row['target'], row['outcome'], row['actor']['kind']) for row in self.activity()] + self.assertEqual(rows, [('remove_definition', 'datasets/numbers', 'in-use', 'owner'), + ('set_definition', 'recipes/fit-b', 'invalid', 'owner'), + ('save', '', 'done', 'owner'), + ('remove_definition', 'charts/score-ranking', 'done', 'owner'), + ('set_definition', 'recipes/fit-b', 'done', 'agent')]) + items = self.activity() + self.assertEqual(items[0]['detail'], str(self.refused_detail('datasets/numbers'))) + self.assertEqual(items[2]['detail'], '') + self.assertEqual(set(items[0]['actor']), {'actor_id', 'kind', 'label'}) + page = self.call('activity', {'offset': 1, 'limit': 2}) + self.assertEqual((page['total'], [item['operation'] for item in page['items']]), + (5, ['set_definition', 'save'])) + + def test_revert(self): + before = self.call('definition', {'reference': 'recipes/fit-b'}) + text = self.edited('recipes/fit-b', lambda document: document.update(title='Fit b, edited')) + self.call('set_definition', {'reference': 'recipes/fit-b', 'text': text}) + self.assertEqual(self.refused('revert_definition', {'reference': 'recipes/fit-b'}, actor=agent('inspect')).code, + 'forbidden') + self.assertEqual(self.call('revert_definition', {'reference': 'recipes/fit-b'}, actor=agent('edit')), + {'reference': 'recipes/fit-b', 'dropped': False}) + self.assertEqual(self.call('definition', {'reference': 'recipes/fit-b'}), before) + self.assertEqual(self.refused('revert_definition', {'reference': 'recipes/fit-b'}).code, 'nothing-to-revert') + self.assertEqual(self.refused('revert_definition', {'reference': 'recipes/none'}).code, 'not-found') + draft = self.call('draft_branch', {'recipe': 'fit-a', 'title': 'Fit c'}, actor=agent('inspect')) + self.assertEqual(draft['problems'], []) + self.call('set_definition', {'reference': draft['reference'], 'text': draft['text']}) + self.assertEqual(self.call('revert_definition', {'reference': draft['reference']}), + {'reference': 'recipes/fit-c', 'dropped': True}) + rows = [(row['operation'], row['target'], row['outcome'], row['actor']['kind']) for row in self.activity()] + self.assertEqual(rows, [('revert_definition', 'recipes/fit-c', 'done', 'owner'), + ('set_definition', 'recipes/fit-c', 'done', 'owner'), + ('revert_definition', 'recipes/none', 'not-found', 'owner'), + ('revert_definition', 'recipes/fit-b', 'nothing-to-revert', 'owner'), + ('revert_definition', 'recipes/fit-b', 'done', 'agent'), + ('set_definition', 'recipes/fit-b', 'done', 'owner')]) + + def refused_detail(self, reference): + return f'{reference} is used by benchmarks/score, recipes/fit-a, recipes/fit-b; change those first.' + + def test_environment_changes(self): + reference = 'environments/test-python' + self.assertTrue(self.call('check_environment', {'reference': reference}, actor=agent('edit', 'inspect'))['ok']) + gone = self.call('disconnect_environment', {'reference': reference}) + self.assertEqual(gone['disconnected'], reference) + self.assertEqual(self.call('environment', {'reference': reference})['state'], 'not-set-up') + self.assertEqual(self.refused('check_environment', {'reference': reference}).code, 'not-set-up') + outcomes = [(row['operation'], row['outcome']) for row in self.activity()] + self.assertEqual(outcomes, [('check_environment', 'not-set-up'), ('disconnect_environment', 'done'), + ('check_environment', 'done')]) + + def test_check_answers_for_a_folder_with_errors(self): + (self.folder / 'datasets' / 'broken.json').write_text('{"format": ', encoding='utf-8') + found = self.call('check') + self.assertFalse(found['ok']) + self.assertGreaterEqual(found['errors'], 1) + self.assertIn('datasets/broken.json', [problem['path'] for problem in found['problems']]) + self.assertEqual(set(found['problems'][0]), {'severity', 'path', 'pointer', 'message'}) + refused = self.refused('summary') + self.assertEqual(refused.code, 'invalid') + self.assertTrue(refused.problems) + # A mutation refused because the folder does not open is still recorded. + self.assertEqual(self.refused('save').code, 'invalid') + (self.folder / 'datasets' / 'broken.json').unlink() + [row] = self.activity() + self.assertEqual((row['operation'], row['outcome']), ('save', 'invalid')) + self.assertTrue(row['detail']) + + def test_resolve_conflict(self): + text = self.edited('recipes/fit-b', lambda document: document.update(title='Kept here')) + self.call('set_definition', {'reference': 'recipes/fit-b', 'text': text}) + path = self.folder / 'recipes' / 'fit-b.json' + on_disk = json.loads(path.read_text(encoding='utf-8')) + on_disk['title'] = 'Changed on disk' + path.write_text(json.dumps(on_disk, indent=2) + '\n', encoding='utf-8') + refused = self.refused('summary') + self.assertEqual((refused.code, refused.references), ('conflict', ['recipes/fit-b'])) + summary = self.call('resolve_conflict', {'choice': 'discard'}) + self.assertTrue({'project_id', 'worker', 'kinds'} <= set(summary)) + self.assertEqual(self.call('definition', {'reference': 'recipes/fit-b'})['title'], 'Changed on disk') + self.assertEqual(self.refused('resolve_conflict', {'choice': 'maybe'}).code, 'invalid') + outcomes = [(row['operation'], row['outcome']) for row in self.activity()] + self.assertEqual(outcomes, [('resolve_conflict', 'invalid'), ('resolve_conflict', 'done'), + ('set_definition', 'done')]) + + def test_cancel_and_stop_a_finished_run(self): + self.build('datasets/numbers') + run_id = self.runs('datasets/numbers')[0]['id'] + self.assertEqual(self.refused('cancel_run', {'run': run_id}).code, 'not-active') + stopped = self.call('stop_run', {'run': run_id}, actor=agent('cancel')) + self.assertEqual((stopped['id'], stopped['was'], stopped['state']), (run_id, 'succeeded', 'succeeded')) + rows = [(row['operation'], row['target'], row['outcome']) for row in self.activity()] + self.assertEqual(rows, [('stop_run', run_id, 'done'), ('cancel_run', run_id, 'not-active')]) + + def test_refused_request_operations_are_recorded(self): + self.refused('cancel_build', {'request': 'abcdef12'}) + self.refused('decline_build', {'request': 'abcdef12', 'reason': 'No.'}) + rows = [(row['operation'], row['target'], row['outcome']) for row in self.activity()] + self.assertEqual(rows, [('decline_build', 'abcdef12', 'not-found'), + ('cancel_build', 'abcdef12', 'not-found')]) + + +class Results(store_base.StoreCase): + """results, values and import_values on the minimal fixture, which has a results file and values files.""" + + def setUp(self): + super().setUp() + self.service = Service(self.folder) + self.addCleanup(self.service.close) + + def call(self, name, arguments=None, actor=OWNER): + result = self.service.call(name, arguments or {}, actor) + plain(self, result) + return result + + def test_results_values_and_import(self): + before = self.call('results', {'benchmark': 'rollout'}) + self.assertEqual(before['total'], len(before['rows'])) + self.assertTrue(all(row['origin'] == 'file' for row in before['rows'])) + self.assertEqual(self.call('values', {'benchmark': 'rollout', 'subject': 'stepper-k1'}), + {'rows': [], 'total': 0}) + text = (self.base / 'values' / 'rollout-stepper-k1.json').read_text(encoding='utf-8') + imported = self.call('import_values', {'text': text}, actor=agent('edit')) + self.assertEqual((imported['benchmark'], imported['subject']), ('rollout', 'stepper-k1')) + values = self.call('values', {'benchmark': 'benchmarks/rollout', 'subject': 'stepper-k1', 'limit': 2}) + self.assertEqual((len(values['rows']), values['total']), (2, imported['values'])) + grouped = self.call('results', {'benchmark': 'rollout', 'subject': 'stepper-k1', 'group': ''}) + self.assertTrue(grouped['rows'] and all(row['group'] == '' for row in grouped['rows'])) + overview = self.call('results_overview')['benchmarks'] + self.assertEqual(overview[0]['subjects_with_values'], 1) + with self.assertRaises(ProjectError) as caught: + self.service.call('import_values', {'text': '{}'}, OWNER) + self.assertEqual(caught.exception.code, 'invalid') + items = self.call('activity')['items'] + self.assertEqual([(row['operation'], row['outcome']) for row in items], + [('import_values', 'invalid'), ('import_values', 'done')]) + + +class ActivityTable(store_base.StoreCase): + + def test_an_older_schema_gets_the_table(self): + self.close(self.open()) + with psycopg.connect(connection_info(), autocommit=True) as connection: + connection.execute(f'DROP TABLE {self.schema}.activity') + self.close(self.open()) + with psycopg.connect(connection_info(), autocommit=True) as connection: + found = connection.execute("SELECT 1 FROM pg_tables WHERE schemaname = %s AND tablename = 'activity'", + (self.schema,)).fetchone() + self.assertIsNotNone(found) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_serving.py b/tests/test_tuldok_project_serving.py new file mode 100644 index 00000000..65924372 --- /dev/null +++ b/tests/test_tuldok_project_serving.py @@ -0,0 +1,604 @@ +"""Serving a trained model (docs/plans/training-and-curation/spec-t6.md). + +The declaration, the protocol's checks, the adapter's encoding and the +supervisor need no database: the supervisor tests serve a stand-in module +(pure Python, written by the test) with the interpreter running the tests, +under a temporary TULDOK_PROJECT_HOME, and check after each test that no +process of a session is left. The end-to-end tests need a database and run +only when TULDOK_DATABASE_URL names a disposable cluster: they start from +test_tuldok_project_http.Served (a Git-committed copy of the buildable fixture +served by the app on port 0, an owner, an agent and a reader credential), add +serve.py and a serve block to fit-a, build it with the fixture's stand-in +script, serve it and call it through POST /api/serve/fit-a/predict. No GPU, +no NumPy, nothing downloaded. +""" +import base64 +import http.client +import json +import math +import os +from pathlib import Path +import struct +import subprocess +import sys +import tempfile +import textwrap +import time +import types +import unittest +from unittest import mock +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from tuldok_project import check, documents, launcher, local, serve_adapter, serve_protocol, serving # noqa: E402 +from tuldok_project.serve_protocol import ServeError # noqa: E402 +import test_tuldok_project_builds as builds_base # noqa: E402 +import test_tuldok_project_http as http_base # noqa: E402 + +OWNER = {'actor_id': 'owner-1', 'kind': 'owner', 'label': 'The owner', 'scopes': ['inspect', 'approve-execution']} +AGENT = {'actor_id': 'agent-1', 'kind': 'agent', 'label': 'An agent', 'scopes': ['inspect', 'execute', 'cancel']} + +STAND_IN = textwrap.dedent(''' + """A stand-in model: y = constant + x; it can be told to fail or hang through its inputs.""" + import json, os, time + + def load(product): + settings = json.load(open(os.path.join(product, 'weights.json'))) + if settings.get('fail'): + raise RuntimeError('the weights are broken on purpose') + if settings.get('hang'): + time.sleep(60) + print('loaded the stand-in') + return settings + + def flat(value): + if hasattr(value, 'ravel'): + return [float(v) for v in value.ravel()] + if isinstance(value, list): + return [v for item in value for v in flat(item)] + return [float(value)] + + def predict(model, inputs): + print('a stray print from predict') + if inputs.get('fail'): + raise ValueError('asked to fail') + if inputs.get('hang'): + time.sleep(60) + if 'a' in inputs: + return {'sum': sum(flat(inputs['a'])), 'echo': inputs['a']} + return {'y': [model['constant'] + x for x in inputs['x']]} +''') + + +def array(shape, values): + return {'array': 'float32', 'shape': list(shape), + 'base64': base64.b64encode(struct.pack('<%df' % len(values), *values)).decode('ascii')} + + +def floats(encoded): + raw = base64.b64decode(encoded['base64']) + return list(struct.unpack('<%df' % (len(raw) // 4), raw)) + + +class Definition: + def __init__(self, reference, document): + self.reference, self.document = reference, document + self.kind = documents.KINDS[reference.split('/')[0]] + self.path = reference + '.json' + + +RECIPE = {'format': 'tuldok_definition_v1', 'kind': 'recipe', 'title': 'Served', + 'run': {'command': ['train'], 'environment': 'environments/py'}, + 'serve': {'module_file': 'serve.py', 'load': 'load', 'example': {'x': [1]}}} + + +class Declaration(unittest.TestCase): + + def problems(self, document, extra=()): + found = [(pointer, message) for pointer, message in documents.schema_problems('recipe', document)] + definitions = {'recipes/served': Definition('recipes/served', document), + 'environments/py': Definition('environments/py', {})} + definitions.update({reference: Definition(reference, {}) for reference in extra}) + check.check_serve(definitions, lambda severity, path, pointer, message: found.append((pointer, message))) + return found + + def test_a_valid_declaration(self): + self.assertEqual(self.problems(RECIPE), []) + document = json.loads(json.dumps(RECIPE)) + document['serve'].update(backend='local', function='answer', environment='environments/other', + idle_minutes=5, start_seconds=10, request_seconds=0.5, gpu=False) + self.assertEqual(self.problems(document, ['environments/other']), []) + self.assertIn(('/serve/module_file', 'serve.py'), list(check._code_paths(document))) + + def test_what_is_refused(self): + def changed(**serve): + document = json.loads(json.dumps(RECIPE)) + document['serve'].update(serve) + return [pointer for pointer, _message in self.problems(document)] + self.assertTrue(changed(function='2bad')) + self.assertTrue(changed(backend='pumas')) # the seam is there; only local is built + self.assertTrue(changed(idle_minutes=0)) + self.assertTrue(changed(request_seconds=601)) + self.assertTrue(changed(example=[1])) + self.assertEqual(changed(module_file='serve.txt'), ['/serve/module_file']) + self.assertEqual(changed(environment='environments/none'), ['/serve/environment']) + document = json.loads(json.dumps(RECIPE)) + del document['run']['environment'] + self.assertEqual([pointer for pointer, _ in self.problems(document)], ['/serve']) + document['serve']['environment'] = 'environments/py' + self.assertEqual(self.problems(document), []) + document = json.loads(json.dumps(RECIPE)) + del document['serve']['module_file'] + self.assertTrue(self.problems(document)) + + +class Protocol(unittest.TestCase): + + def test_a_request(self): + inputs = {'x': [[1, 2], [3, 4]], 'a': array([2, 2], [1, 2, 3, 4]), 'name': 'x'} + self.assertIs(serve_protocol.check_request({'inputs': inputs}), inputs) + self.assertEqual(serve_protocol.check_request({'inputs': {'a': array([], [5])}})['a']['shape'], []) + for body in ({}, {'inputs': []}, {'inputs': {}, 'more': 1}, [1], {'inputs': {'x': float('nan')}}): + with self.subTest(body=body), self.assertRaises(ServeError) as refused: + serve_protocol.check_request(body) + self.assertEqual(refused.exception.code, 'invalid') + + def test_a_malformed_array(self): + good = array([3], [1, 2, 3]) + for wrong in ({**good, 'array': 'float64'}, {**good, 'shape': [2]}, {**good, 'shape': [-1]}, + {**good, 'shape': [True]}, {**good, 'shape': [1] * 9}, {**good, 'base64': 'no t!'}, + {**good, 'shape': [1 << 13, 1 << 12]}): + with self.subTest(wrong=wrong), self.assertRaises(ServeError) as refused: + serve_protocol.check_request({'inputs': {'a': wrong}}) + self.assertEqual(refused.exception.code, 'invalid') + + def test_depth(self): + value = 1 + for _ in range(serve_protocol.DEPTH): + value = [value] + with self.assertRaises(ServeError): + serve_protocol.check_request({'inputs': {'x': value}}) + serve_protocol.check_request({'inputs': {'x': value[0][0]}}) + + def test_an_answer(self): + line = json.dumps({'id': 3, 'ok': True, 'outputs': {'y': array([1], [2.0])}, 'model_seconds': 0.1}).encode() + self.assertEqual(serve_protocol.check_answer(line, 3)['outputs']['y']['shape'], [1]) + for wrong, number in ((b'not json', 1), (line, 4), (b'{"id": 3, "ok": true, "model_seconds": 1}', 3), + (b'{"id": 3, "ok": false, "model_seconds": 1}', 3), + (b'{"id": 3, "ok": true, "outputs": 1, "model_seconds": -1}', 3), + (b'{"id": 3, "ok": true, "outputs": NaN, "model_seconds": 1}', 3)): + with self.subTest(wrong=wrong), self.assertRaises(ServeError) as refused: + serve_protocol.check_answer(wrong, number) + self.assertEqual(refused.exception.code, 'serve-failed') + + def test_the_summary_against_its_own_arithmetic(self): + totals = [5.0, 1.0, 4.0, 2.0, 3.0, 10.0] + lines = [{'n': i + 1, 'at': 100.0 + i * 0.5, 'ok': True, 'queue_ms': 0.0, 'model_ms': t / 2, 'total_ms': t} + for i, t in enumerate(totals)] + [{'n': 7, 'at': 104.0, 'ok': False, 'total_ms': 99.0}] + found = serve_protocol.summary(lines) + ordered = sorted(totals) + + def at(p): + position = p * (len(ordered) - 1) + low = int(position) + return ordered[low] + (position - low) * (ordered[min(low + 1, 5)] - ordered[low]) + self.assertEqual((found['count'], found['answered'], found['failed']), (7, 6, 1)) + self.assertEqual(found['total_ms']['p50'], at(0.5)) + self.assertAlmostEqual(found['total_ms']['p95'], 0.25 * 5 + 0.75 * 10) + self.assertEqual(found['total_ms']['p99'], at(0.99)) + self.assertEqual(found['total_ms']['mean'], sum(totals) / 6) + self.assertEqual(found['total_ms']['max'], 10.0) + self.assertEqual(found['span_seconds'], 2.5) + self.assertEqual(found['throughput_per_second'], 5 / 2.5) + self.assertIsNone(serve_protocol.summary([])['total_ms']) + self.assertEqual(serve_protocol.measure(lines[0], 'total_s'), 0.005) + + +class Adapter(unittest.TestCase): + """serve_adapter's encoding and decoding, without NumPy (none is installed here).""" + + def test_decoding_without_numpy(self): + self.assertEqual(serve_adapter.decode({'a': array([2, 3], [0, 1, 2, 3, 4, 5]), 'b': [[1, 2]]}, + use_numpy=False), + {'a': [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], 'b': [[1, 2]]}) + self.assertEqual(serve_adapter.decode(array([], [7]), use_numpy=False), 7.0) + self.assertEqual(serve_adapter.decode(array([0, 2], []), use_numpy=False), []) + + def test_encoding(self): + self.assertEqual(serve_adapter.encode({'t': (1, 2.5), 'n': None}), {'t': [1, 2.5], 'n': None}) + for wrong in ({'x': float('inf')}, {1: 2}, {'x': object()}): + with self.subTest(wrong=wrong), self.assertRaises(ValueError): + serve_adapter.encode(wrong) + + class Fake: # what the adapter takes for a NumPy array: shape, dtype, tobytes and astype + shape, dtype, ndim = (2,), 'float64', 1 + + def astype(self, kind): + return types.SimpleNamespace(tobytes=lambda order: struct.pack('<2f', 1.5, -2.0)) + + def tobytes(self): + return b'' + encoded = serve_adapter.encode({'w': Fake()})['w'] + self.assertEqual((encoded['array'], encoded['shape'], floats(encoded)), ('float32', [2], [1.5, -2.0])) + + def test_an_answer_of_the_adapter(self): + line = json.dumps({'id': 2, 'inputs': {'x': [1, 2]}}).encode() + answer = serve_adapter.answer(lambda model, inputs: {'y': [v * model for v in inputs['x']]}, 3, line) + self.assertEqual((answer['id'], answer['ok'], answer['outputs']), (2, True, {'y': [3, 6]})) + failed = serve_adapter.answer(lambda model, inputs: 1 / 0, 3, line) + self.assertEqual((failed['ok'], failed['error']), (False, 'ZeroDivisionError: division by zero')) + self.assertFalse(serve_adapter.answer(lambda m, i: 1, 3, b'{nope')['ok']) + + +def _alive(pid): + try: + with open(f'/proc/{pid}/stat', 'rb') as handle: + return handle.read().rsplit(b')', 1)[1].split()[0] != b'Z' + except OSError: + return False + + +class SupervisorCase(unittest.TestCase): + """A Supervisor under a temporary home, a stand-in module and product; no process left after a test.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + patcher = mock.patch.dict(os.environ, {local.HOME_VARIABLE: str(self.root / 'home')}) + patcher.start() + self.addCleanup(patcher.stop) + for name, value in (('WATCH_SECONDS', 0.1), ('STOP_GRACE', 2.0)): + patcher = mock.patch.object(serving, name, value) + patcher.start() + self.addCleanup(patcher.stop) + (self.root / 'code').mkdir() + (self.root / 'code' / 'serve.py').write_text(STAND_IN, encoding='utf-8') + self.supervisor = serving.Supervisor('a' * 32) + self.addCleanup(self.no_process_left) + self.addCleanup(self.supervisor.close) + + def no_process_left(self): + for session in self.supervisor._sessions.values(): + if session.pid is not None: + self.assertFalse(_alive(session.pid), f'the process of session {session.id} is left') + self.assertEqual(launcher.strays('serve-' + session.id), []) + + def launch(self, weights=None, idle_minutes=30, most=2, actor=OWNER, awaiting=False, **serve): + product = self.root / ('product-' + str(len(list(self.root.glob('product-*'))))) + product.mkdir() + (product / 'weights.json').write_text(json.dumps(weights or {'constant': 2.0}), encoding='utf-8') + declared = {'module_file': 'serve.py', 'load': 'load', **serve} + session = self.supervisor.open_session('recipes/fit-a', 'Fit a', actor=actor, scratch=False, backend='local', + idle_minutes=idle_minutes, most=most, awaiting=awaiting) + made = serving.Launch(model='recipes/fit-a', serve=declared, product=str(product), product_run='r' * 32, + environment='environments/py', interpreter=sys.executable, + code_root=str(self.root / 'code'), commit='c' * 40, scratch=False) + if awaiting: + return session, made + return self.supervisor.launch(session['id'], made) + + def wait(self, session_id, *states, seconds=20): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + found = self.supervisor.get(session_id) + if found['state'] in states: + return found + time.sleep(0.05) + self.fail(f'session {session_id} stayed {found["state"]}') + + def predict(self, inputs, model='recipes/fit-a'): + return self.supervisor.predict(model, inputs, time.perf_counter()) + + +class Supervisor(SupervisorCase): + + def test_start_ready_predict_stop(self): + session = self.launch() + self.assertEqual(session['state'], 'starting') + ready = self.wait(session['id'], 'ready') + self.assertIsInstance(ready['pid'], int) + self.assertEqual(ready['python'].split('.')[:2], [str(sys.version_info[0]), str(sys.version_info[1])]) + answer = self.predict({'x': [1, 2.5]}) + self.assertEqual(answer['outputs'], {'y': [3.0, 4.5]}) + self.assertEqual(set(answer['timing']), {'queue_ms', 'model_ms', 'total_ms'}) + self.assertLessEqual(answer['timing']['model_ms'], answer['timing']['total_ms']) + encoded = self.predict({'a': array([2, 2], [1, 2, 3, 4.5])})['outputs'] + self.assertEqual(encoded['sum'], 10.5) + # Without NumPy the array arrives as nested lists and goes back as lists; with NumPy (as on CI) predict gets a + # float32 array and the answer encodes it again as an array. + try: + import numpy # noqa: F401 + self.assertEqual(encoded['echo'], array([2, 2], [1, 2, 3, 4.5])) + except ImportError: + self.assertEqual(encoded['echo'], [[1.0, 2.0], [3.0, 4.5]]) + with self.assertRaises(ServeError) as refused: + self.predict({'fail': True}) + self.assertEqual(refused.exception.code, 'predict-failed') + self.assertIn('ValueError: asked to fail', str(refused.exception)) + self.assertEqual(self.supervisor.get(session['id'])['state'], 'ready') + self.assertIn('a stray print from predict', self.supervisor.get(session['id'])['output_tail']) + lines = self.supervisor.requests(session['id']) + self.assertEqual([(line['n'], line['ok'], line['status']) for line in lines], + [(1, True, 200), (2, True, 200), (3, False, 422)]) + pid = ready['pid'] + stopped = self.supervisor.stop(session['id'], OWNER) + self.assertEqual((stopped['state'], stopped['reason']), ('stopped', 'asked')) + self.assertFalse(_alive(pid)) + with self.assertRaises(ServeError) as refused: + self.predict({'x': [1]}) + self.assertEqual(refused.exception.code, 'not-served') + on_disk = json.loads((self.root / 'home' / ('a' * 32) / 'serving' / session['id'] / 'session.json') + .read_text(encoding='utf-8')) + self.assertEqual((on_disk['state'], on_disk['requests'], on_disk['failed_requests']), ('stopped', 3, 1)) + + def test_a_load_that_fails_or_hangs(self): + failed = self.wait(self.launch({'fail': True})['id'], 'failed') + self.assertIn('RuntimeError: the weights are broken on purpose', failed['problem']) + self.assertIn('Traceback', failed['output_tail']) + hung = self.wait(self.launch({'constant': 1, 'hang': True}, start_seconds=1)['id'], 'failed') + self.assertIn('did not load within 1 seconds', hung['problem']) + time.sleep(0.3) + + def test_a_request_that_hangs(self): + session = self.launch(request_seconds=0.5) + self.wait(session['id'], 'ready') + with self.assertRaises(ServeError) as refused: + self.predict({'hang': True}) + self.assertEqual(refused.exception.code, 'timed-out') + failed = self.supervisor.get(session['id']) + self.assertEqual(failed['state'], 'failed') + self.assertIn('more than 0.5 seconds', failed['problem']) + self.assertEqual(self.supervisor.requests(session['id'])[-1]['status'], 504) + + def test_an_idle_model_stops(self): + session = self.launch(idle_minutes=0.5 / 60) + self.wait(session['id'], 'ready') + stopped = self.wait(session['id'], 'stopped', seconds=10) + self.assertEqual(stopped['reason'], 'idle') + + def test_the_limit_and_one_session_a_model(self): + first = self.launch(most=1) + with self.assertRaises(ServeError) as refused: + self.launch(most=1) + self.assertEqual(refused.exception.code, 'already-served') + with self.assertRaises(ServeError) as refused: + self.supervisor.open_session('recipes/fit-b', 'Fit b', actor=OWNER, scratch=False, backend='local', + idle_minutes=1, most=1, awaiting=False) + self.assertEqual(refused.exception.code, 'limit') + self.supervisor.stop(first['id'], OWNER) + self.supervisor.open_session('recipes/fit-b', 'Fit b', actor=OWNER, scratch=False, backend='local', + idle_minutes=1, most=1, awaiting=True) # awaiting takes no place + + def test_approval(self): + session, made = self.launch(actor=AGENT, awaiting=True) + self.assertEqual((session['state'], session['requested_by']['kind']), ('awaiting-approval', 'agent')) + self.assertIsNotNone(session['expires_at']) + with self.assertRaises(ServeError) as refused: + self.predict({'x': [1]}) + self.assertEqual(refused.exception.code, 'not-ready') + self.assertEqual(self.supervisor.awaiting(session['id']), ('recipes/fit-a', False)) + self.supervisor.approve(session['id'], OWNER, 2) + self.supervisor.launch(session['id'], made) + ready = self.wait(session['id'], 'ready') + self.assertEqual(ready['decided_by']['kind'], 'owner') + self.supervisor.stop(session['id'], AGENT) + declined, _made = self.launch(actor=AGENT, awaiting=True) + self.assertEqual(self.supervisor.stop(declined['id'], OWNER)['state'], 'declined') + with self.assertRaises(ServeError) as refused: + self.supervisor.approve(declined['id'], OWNER, 2) + self.assertEqual(refused.exception.code, 'not-awaiting') + expiring, _made = self.launch(actor=AGENT, awaiting=True) + self.supervisor._sessions[expiring['id']].expires_at = serving._now() + self.assertEqual(self.supervisor.get(expiring['id'])['state'], 'expired') + + def test_a_session_left_by_a_tuldok_that_is_gone(self): + session = self.launch() + self.wait(session['id'], 'ready') + self.supervisor.close() + again = serving.Supervisor('a' * 32) + found = again.get(session['id']) + self.assertEqual(found['state'], 'stopped') + self.assertEqual([item['id'] for item in again.sessions()], [session['id']]) + path = self.root / 'home' / ('a' * 32) / 'serving' / session['id'] / 'session.json' + document = json.loads(path.read_text(encoding='utf-8')) + document['state'] = 'ready' + path.write_text(json.dumps(document), encoding='utf-8') + self.assertEqual((again.get(session['id'])['state'], again.get(session['id'])['reason']), + ('stopped', 'Tuldok stopped')) + + +SERVE_PY = textwrap.dedent(''' + import json, os + + def load(product): + with open(os.path.join(product, 'weights.json')) as handle: + return json.load(handle) + + def predict(model, inputs): + if 'a' in inputs: + return {'sum': sum(v for row in inputs['a'] for v in row)} + return {'y': [model['constant'] + x for x in inputs['x']]} +''') +LATENCY = {'format': 'tuldok_definition_v1', 'kind': 'benchmark', 'title': 'Served latency', + 'earlier_work': {'rebuild': 'no', 'reason': 'Its values come from serving sessions (serve_values).'}, + 'metrics': [{'id': 'total_ms', 'title': 'Total time', 'units': 'ms', 'better': 'lower'}, + {'id': 'model_ms', 'title': 'Model time', 'units': 'ms', 'better': 'lower'}, + {'id': 'wait', 'title': 'Queue time', 'units': 's', 'better': 'lower'}], + 'subjects': [{'name': 'fit-a-served', 'recipe': 'recipes/fit-a'}, {'name': 'other', 'recipe': 'recipes/fit-b'}]} + + +class EndToEnd(http_base.Served): + """fit-a with serve.py, built, served by the app and called over HTTP.""" + + def setUp(self): + super().setUp() + settings = Path(self.temporary) / 'settings.json' + patcher = mock.patch.dict(os.environ, {local.SETTINGS_VARIABLE: str(settings)}) + patcher.start() + self.addCleanup(patcher.stop) + self.write_code('serve.py', SERVE_PY) + self.edit('recipes/fit-a', lambda d: d.update(serve={'module_file': 'serve.py', 'load': 'load', + 'example': {'x': [1, 2]}})) + (self.folder / 'benchmarks' / 'served-latency.json').write_text(json.dumps(LATENCY, indent=2) + '\n', + encoding='utf-8') + for arguments in (('add', '-A'), ('commit', '-q', '-m', 'Serve fit-a.')): # self.root is the address here + subprocess.run([*builds_base.GIT, '-C', str(Path(self.temporary) / 'buildable'), *arguments], check=True, + capture_output=True, timeout=60) + done = self.build('recipes/fit-a') + self.assertTrue(done['ok'], done) + with self.opened() as project: + weights = json.loads(Path(project._execute( + "SELECT directory FROM {schema}.products WHERE reference = 'recipes/fit-a' AND current").fetchone()[0], + 'weights.json').read_text(encoding='utf-8')) + self.constant = weights['constant'] + self.addCleanup(self.no_process_left) + + def no_process_left(self): + self.host.service.serving.close() # what host.close does next; the processes are looked at after it + for session in self.host.service.serving._sessions.values(): + if session.pid is not None: + self.assertFalse(_alive(session.pid)) + self.assertEqual(launcher.strays('serve-' + session.id), []) + + def predict(self, body, token=None, *, raw=None, name='fit-a', content_type='application/json'): + headers = {'Content-Type': content_type} + if token: + headers['Authorization'] = 'Bearer ' + token + data = raw if raw is not None else json.dumps(body).encode() + request = Request(f'{self.root}/api/serve/{name}/predict', data=data, headers=headers) + try: + response = urlopen(request, timeout=30) + except HTTPError as error: + response = error + with response: + return response.status, json.loads(response.read()) + + def ready(self, session, token=None): + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + found = self.ok('serve_session', {'session': session}, token) + if found['state'] != 'starting': + return found + time.sleep(0.05) + self.fail('the model did not become ready') + + def test_served_through_the_endpoint_and_turned_into_a_benchmark_subject(self): + servable = self.ok('served')['servable'] + self.assertEqual([(item['model'], item['built'], item['example']) for item in servable], + [('recipes/fit-a', True, {'x': [1, 2]})]) + # The reader may predict but not start; the owner starts at once. + status, value = self.call('serve_start', {'model': 'fit-a'}, self.reader['token']) + self.assertEqual((status, value['code']), (403, 'forbidden')) + session = self.ok('serve_start', {'model': 'fit-a'}, self.owner.token) + self.assertEqual((session['state'], session['model'], session['backend']), ('starting', 'recipes/fit-a', + 'local')) + self.assertEqual(len(session['commit']), 40) + found = self.ready(session['id']) + self.assertEqual(found['state'], 'ready', found) + # No credential: refused; a reader's: answered. + status, value = self.predict({'inputs': {'x': [1, 2]}}) + self.assertEqual(status, 401, value) + status, value = self.predict({'inputs': {'x': [1, 2]}}, 'not-a-credential') + self.assertEqual(status, 401, value) + for _ in range(5): + status, value = self.predict({'inputs': {'x': [1, 2]}}, self.reader['token']) + self.assertEqual(status, 200, value) + self.assertEqual(value['outputs'], {'y': [self.constant + 1, self.constant + 2]}) + self.assertEqual((value['model'], value['session']), ('recipes/fit-a', session['id'])) + status, value = self.predict({'inputs': {'a': array([2, 2], [1, 2, 3, 4])}}, self.reader['token']) + self.assertEqual((status, value['outputs']), (200, {'sum': 10.0})) + # Refusals. + self.assertEqual(self.predict({'x': [1]}, self.reader['token'])[1]['code'], 'invalid') + self.assertEqual(self.predict(None, self.reader['token'], raw=b'{')[0], 400) + self.assertEqual(self.predict({'inputs': {}}, self.reader['token'], content_type='text/plain')[0], 400) + self.assertEqual(self.predict({'inputs': {'a': {**array([2], [1, 2]), 'shape': [3]}}}, + self.reader['token'])[0], 400) + status, value = self.predict({'inputs': {'x': 'no'}}, self.reader['token']) + self.assertEqual((status, value['code']), (422, 'predict-failed')) + status, value = self.predict({'inputs': {}}, self.reader['token'], name='fit-b') + self.assertEqual((status, value['code']), (404, 'not-served')) + connection = http.client.HTTPConnection('127.0.0.1', self.server.server_port, timeout=30) + connection.putrequest('POST', '/api/serve/fit-a/predict') + for key, value in (('Authorization', 'Bearer ' + self.reader['token']), ('Content-Type', 'application/json'), + ('Content-Length', str(serve_protocol.REQUEST_LIMIT + 1))): + connection.putheader(key, value) + connection.endheaders() # the body is never sent: the length alone is refused + response = connection.getresponse() + self.assertEqual((response.status, json.loads(response.read())['code']), (413, 'too-large')) + connection.close() + # Through the operation too. + answer = self.ok('serve_predict', {'model': 'recipes/fit-a', 'inputs': {'x': [0]}}, self.reader['token']) + self.assertEqual(answer['outputs'], {'y': [self.constant]}) + # The timings: the summary, then the benchmark's subject. + found = self.ok('serve_session', {'session': session['id']}) + self.assertEqual((found['summary']['count'], found['summary']['answered'], found['summary']['failed']), + (8, 7, 1)) + self.assertEqual(len(found['recent']), 8) + status, value = self.call('serve_values', {'session': session['id'], 'benchmark': 'served-latency', + 'subject': 'other'}) + self.assertEqual((status, value['code']), (400, 'invalid')) + status, value = self.call('serve_values', {'session': session['id'], 'benchmark': 'served-latency', + 'subject': 'fit-a-served'}, self.reader['token']) + self.assertEqual(status, 403) + done = self.ok('serve_values', {'session': session['id'], 'benchmark': 'served-latency', + 'subject': 'fit-a-served', 'metrics': {'total_ms': 'total_ms', + 'model_ms': 'model_ms', + 'wait': 'queue_s'}}) + self.assertEqual((done['benchmark'], done['subject']), ('served-latency', 'fit-a-served')) + self.assertTrue(Path(done['file']).is_file()) + lines = self.host.service.serving.requests(session['id']) + totals = sorted(line['total_ms'] for line in lines if line['ok']) + rows = {row['metric']: row for row in self.ok('results', {'benchmark': 'served-latency'})['rows']} + self.assertEqual((rows['total_ms']['count'], rows['total_ms']['unavailable']), (7, 1)) + self.assertAlmostEqual(rows['total_ms']['median'], totals[3]) + self.assertAlmostEqual(rows['total_ms']['min'], totals[0]) + self.assertAlmostEqual(rows['wait']['max'], max(line['queue_ms'] for line in lines if line['ok']) / 1000) + values = self.ok('values', {'benchmark': 'served-latency', 'subject': 'fit-a-served', 'limit': 100}) + self.assertEqual(values['total'], 24) + self.assertIn('request-000001', {value['case'] for value in values['rows']}) + # Stop: nothing answers; the activity names what was done. + stopped = self.ok('serve_stop', {'session': session['id']}, self.owner.token) + self.assertEqual(stopped['state'], 'stopped') + self.assertEqual(self.predict({'inputs': {'x': [1]}}, self.reader['token'])[0], 404) + done = [(item['operation'], item['target'], item['outcome']) for item in self.activity() + if item['operation'].startswith('serve')] + self.assertIn(('serve_start', 'recipes/fit-a', 'done'), done) + self.assertIn(('serve_stop', session['id'], 'done'), done) + + def test_an_agent_asks_and_the_owner_approves(self): + session = self.ok('serve_start', {'model': 'recipes/fit-a'}) + self.assertEqual(session['state'], 'awaiting-approval') + status, value = self.call('serve_approve', {'session': session['id']}) + self.assertEqual((status, value['code']), (403, 'forbidden')) + status, value = self.predict({'inputs': {'x': [1]}}, self.agent['token']) + self.assertEqual((status, value['code']), (409, 'not-ready')) + approved = self.ok('serve_approve', {'session': session['id']}, self.owner.token) + self.assertEqual(approved['state'], 'starting') + self.assertEqual(self.ready(session['id'])['state'], 'ready') + self.assertEqual(self.predict({'inputs': {'x': [1]}}, self.agent['token'])[0], 200) + self.assertEqual(self.ok('serve_stop', {'session': session['id']})['state'], 'stopped') # cancel scope + again = self.ok('serve_start', {'model': 'fit-a'}) + self.assertEqual(self.ok('serve_stop', {'session': again['id']}, self.owner.token)['state'], 'declined') + + def test_the_limit_of_the_machine_and_what_cannot_be_served(self): + Path(os.environ[local.SETTINGS_VARIABLE]).write_text( + json.dumps({'format': 'tuldok_settings_v1', 'serve_most': 1}), encoding='utf-8') + status, value = self.call('serve_start', {'model': 'fit-b'}, self.owner.token) + self.assertEqual((status, value['code']), (409, 'not-servable')) + status, value = self.call('serve_start', {'model': 'nothing'}, self.owner.token) + self.assertEqual(status, 404) + session = self.ok('serve_start', {'model': 'fit-a'}, self.owner.token) + status, value = self.call('serve_start', {'model': 'fit-a'}, self.owner.token) + self.assertEqual((status, value['code']), (409, 'already-served')) + self.assertEqual(self.ok('served')['most'], 1) + self.ready(session['id']) + self.host.service.close() # Tuldok stopping stops what it serves + self.assertEqual(self.ok('serve_session', {'session': session['id']})['state'], 'stopped') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_snapshot.py b/tests/test_tuldok_project_snapshot.py new file mode 100644 index 00000000..e0ad4622 --- /dev/null +++ b/tests/test_tuldok_project_snapshot.py @@ -0,0 +1,381 @@ +"""The code a run sees: snapshots of one commit, scratch runs, and what changed since a commit. + +Each test builds a small Git repository in a temporary directory whose path +contains a space, with user.name and user.email set in that repository only, +and points TULDOK_PROJECT_HOME at a temporary directory so nothing is written +under the owner's home. Snapshots are read-only; the clean-up makes them +writable again before the temporary directory is removed. +""" +import hashlib +import io +import os +from pathlib import Path +import stat +import subprocess +import sys +import tarfile +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tuldok_project import local, snapshot # noqa: E402 +from tuldok_project.snapshot import SnapshotError # noqa: E402 + +PROJECT_ID = '0123456789abcdef0123456789abcdef' + + +class Repository(unittest.TestCase): + """A fresh repository with a committed tree under 'Linux Software/repo', and an empty local home.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.addCleanup(self._make_writable, Path(directory.name)) + self.base = Path(directory.name) / 'Linux Software' + patcher = mock.patch.dict(os.environ, {local.HOME_VARIABLE: str(self.base / 'tuldok home')}) + patcher.start() + self.addCleanup(patcher.stop) + self.home = Path(local.home(PROJECT_ID)) + self.repo = self.base / 'repo' + self.repo.mkdir(parents=True) + self.git('init', '-q', '-b', 'main') + self.git('config', 'user.name', 'Test') + self.git('config', 'user.email', 'test@example.invalid') + self.git('config', 'commit.gpgsign', 'false') + self.write('README.md', 'top\n') + self.write('lab/train.py', 'print("train")\n') + self.write('lab/run.sh', '#!/bin/sh\necho run\n', executable=True) + self.write('lab/data dir/table one.csv', 'a,b\n1,2\n') + self.write('configs/k1.json', '{}\n') + self.write('other/notes.txt', 'unrelated\n') + os.symlink('train.py', self.repo / 'lab' / 'link.py') + self.commit('first') + + @staticmethod + def _make_writable(root): + for current, directories, files in os.walk(root): + for name in directories: + path = os.path.join(current, name) + if not os.path.islink(path): + os.chmod(path, 0o755) + if root.exists(): + os.chmod(root, 0o755) + + def git(self, *arguments, cwd=None): + environment = {key: value for key, value in os.environ.items() if not key.startswith('GIT_')} + environment.update(GIT_CONFIG_GLOBAL=os.devnull, GIT_CONFIG_NOSYSTEM='1') + done = subprocess.run(['git', '-C', str(cwd or self.repo), *arguments], capture_output=True, text=True, + env=environment, check=True) + return done.stdout.strip() + + def write(self, relative, text, *, executable=False): + path = self.repo / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + if executable: + path.chmod(0o755) + + def commit(self, message): + self.git('add', '-A') + self.git('commit', '-q', '-m', message) + return self.git('rev-parse', 'HEAD') + + def head(self): + return self.git('rev-parse', 'HEAD') + + def prepare(self, code_root=None, code_paths=None, **options): + return snapshot.prepare(code_root or self.repo, code_paths, home=self.home, **options) + + +class Prepare(Repository): + def test_clean_repository_makes_a_snapshot_of_the_commit(self): + made = self.prepare() + commit = self.head() + self.assertEqual(made.commit, commit) + self.assertFalse(made.scratch) + self.assertEqual(made.directory, str(self.home / 'code' / commit)) + self.assertEqual(made.code_root, made.directory) + self.assertEqual(made.prefix, '') + root = Path(made.code_root) + self.assertEqual((root / 'lab' / 'train.py').read_text(), 'print("train")\n') + self.assertEqual((root / 'lab' / 'data dir' / 'table one.csv').read_text(), 'a,b\n1,2\n') + self.assertTrue((root / 'other' / 'notes.txt').exists()) + self.assertEqual(os.readlink(root / 'lab' / 'link.py'), 'train.py') + self.assertEqual(sorted(p.name for p in (self.home / 'code').iterdir()), [commit], 'no partial directory left') + + def test_code_root_in_a_subdirectory_keeps_the_repository_layout(self): + made = self.prepare(self.repo / 'lab') + self.assertEqual(made.prefix, 'lab') + self.assertEqual(made.code_root, os.path.join(made.directory, 'lab')) + self.assertTrue((Path(made.code_root) / 'train.py').is_file()) + self.assertEqual(sorted(os.listdir(made.directory)), ['lab'], 'only the code root is extracted') + + def test_code_paths_limit_what_is_extracted(self): + made = self.prepare(code_paths=['lab/train.py', 'configs']) + root = Path(made.code_root) + self.assertTrue((root / 'lab' / 'train.py').is_file()) + self.assertTrue((root / 'configs' / 'k1.json').is_file()) + self.assertFalse((root / 'lab' / 'run.sh').exists()) + self.assertFalse((root / 'other').exists()) + self.assertFalse((root / 'README.md').exists()) + + def test_code_paths_are_relative_to_a_code_root_in_a_subdirectory(self): + made = self.prepare(self.repo / 'lab', ['data dir']) + self.assertTrue((Path(made.code_root) / 'data dir' / 'table one.csv').is_file()) + self.assertFalse((Path(made.code_root) / 'train.py').exists()) + + def test_code_path_missing_from_the_commit_is_a_clear_error(self): + with self.assertRaises(SnapshotError) as caught: + self.prepare(code_paths=['nowhere']) + self.assertEqual(caught.exception.code, 'git-failed') + self.assertIn('archive', str(caught.exception)) + self.assertIn('nowhere', str(caught.exception)) + self.assertEqual(os.listdir(self.home / 'code'), [], 'the partial directory is removed') + + def test_malformed_code_paths_are_refused(self): + for value in (['../x'], ['a//b'], [], ['lab', 'lab'], ['/abs']): + with self.subTest(value), self.assertRaises(SnapshotError) as caught: + self.prepare(code_paths=value) + self.assertEqual(caught.exception.code, 'invalid') + + def test_snapshot_is_read_only(self): + made = self.prepare() + root = Path(made.code_root) + mode = lambda path: stat.S_IMODE(os.lstat(path).st_mode) # noqa: E731 + self.assertEqual(mode(root), 0o555) + self.assertEqual(mode(root / 'lab'), 0o555) + self.assertEqual(mode(root / 'lab' / 'data dir'), 0o555) + self.assertEqual(mode(root / 'lab' / 'train.py'), 0o444) + self.assertEqual(mode(root / 'lab' / 'run.sh'), 0o555) + if os.geteuid() != 0: + with self.assertRaises(PermissionError): + (root / 'lab' / 'new.py').write_text('x') + + def test_dirty_tracked_file_is_refused_and_named(self): + self.write('lab/train.py', 'print("changed")\n') + with self.assertRaises(SnapshotError) as caught: + self.prepare() + self.assertEqual(caught.exception.code, 'dirty') + self.assertEqual(caught.exception.files, ['lab/train.py']) + self.assertIn('lab/train.py', str(caught.exception)) + self.assertFalse((self.home / 'code').exists(), 'nothing is made for a refused run') + + def test_staged_and_deleted_files_are_dirty_and_named_relative_to_the_code_root(self): + self.write('lab/data dir/table one.csv', 'changed\n') + self.git('add', 'lab/data dir/table one.csv') + (self.repo / 'lab' / 'run.sh').unlink() + with self.assertRaises(SnapshotError) as caught: + self.prepare(self.repo / 'lab') + self.assertEqual(sorted(caught.exception.files), ['data dir/table one.csv', 'run.sh']) + + def test_refusal_names_at_most_ten_files(self): + for index in range(12): + self.write(f'many/f{index:02}.txt', 'a\n') + self.commit('many') + for index in range(12): + self.write(f'many/f{index:02}.txt', 'b\n') + with self.assertRaises(SnapshotError) as caught: + self.prepare() + message = str(caught.exception) + self.assertIn('many/f09.txt', message) + self.assertNotIn('many/f10.txt', message) + self.assertIn('and 2 more', message) + self.assertEqual(len(caught.exception.files), 12) + + def test_changes_outside_the_code_paths_do_not_make_the_run_dirty(self): + self.write('other/notes.txt', 'changed\n') + made = self.prepare(code_paths=['lab']) + self.assertTrue((Path(made.code_root) / 'lab' / 'train.py').is_file()) + with self.assertRaises(SnapshotError): + self.prepare() + + def test_untracked_file_is_ignored(self): + self.write('lab/scratch.py', 'untracked\n') + made = self.prepare() + self.assertFalse((Path(made.code_root) / 'lab' / 'scratch.py').exists()) + + def test_scratch_uses_the_code_root_in_place_and_writes_nothing(self): + self.write('lab/train.py', 'print("changed")\n') + with mock.patch.object(snapshot.subprocess, 'run', wraps=subprocess.run) as run: + made = self.prepare(self.repo / 'lab', scratch=True) + self.assertTrue(made.scratch) + self.assertEqual(made.code_root, str(self.repo / 'lab')) + self.assertIsNone(made.directory) + self.assertEqual(made.commit, self.head()) + self.assertFalse(self.home.exists(), 'a scratch run does not touch the snapshot directory') + self.assertFalse(any('archive' in call.args[0] for call in run.call_args_list)) + + def test_existing_snapshot_is_reused_without_running_git_archive(self): + first = self.prepare() + with mock.patch.object(snapshot.subprocess, 'run', wraps=subprocess.run) as run: + second = self.prepare() + self.assertEqual(first, second) + commands = [call.args[0] for call in run.call_args_list] + self.assertTrue(commands, 'Git is still asked for the commit') + self.assertFalse(any('archive' in command for command in commands), commands) + + def test_different_code_paths_at_one_commit_make_different_snapshots(self): + commit = self.head() + narrow = self.prepare(code_paths=['configs']) + wide = self.prepare(code_paths=['configs', 'lab']) + self.assertNotEqual(narrow.directory, wide.directory) + self.assertRegex(os.path.basename(narrow.directory), f'^{commit}-[0-9a-f]{{12}}$') + self.assertFalse((Path(narrow.code_root) / 'lab').exists()) + self.assertTrue((Path(wide.code_root) / 'lab' / 'train.py').is_file()) + whole = self.prepare() + self.assertEqual(os.path.basename(whole.directory), commit) + self.assertEqual(len(os.listdir(self.home / 'code')), 3) + + def test_same_code_paths_in_another_order_share_one_snapshot(self): + first = self.prepare(code_paths=['lab', 'configs']) + with mock.patch.object(snapshot.subprocess, 'run', wraps=subprocess.run) as run: + second = self.prepare(code_paths=['configs', 'lab']) + self.assertEqual(first.directory, second.directory) + self.assertFalse(any('archive' in call.args[0] for call in run.call_args_list)) + digest = hashlib.sha256(b'configs\nlab').hexdigest()[:12] + self.assertEqual(os.path.basename(first.directory), f'{self.head()}-{digest}') + + def test_leftover_partial_directory_is_not_taken_for_a_snapshot(self): + commit = self.head() + leftover = self.home / 'code' / f'.partial-{commit}-crashed' + leftover.mkdir(parents=True) + (leftover / 'half.txt').write_text('half') + made = self.prepare() + self.assertEqual(made.directory, str(self.home / 'code' / commit)) + self.assertTrue((Path(made.code_root) / 'lab' / 'train.py').is_file()) + self.assertFalse((Path(made.code_root) / 'half.txt').exists()) + + def test_build_that_loses_the_race_uses_the_existing_snapshot(self): + commit = self.head() + real_rename = os.rename + renames = [] + + def rename_after_another_build(source, target): + # Another build finishes first: its complete snapshot is in place before ours is renamed. + renames.append(source) + if len(renames) == 1: + snapshot.prepare(self.repo, None, home=self.home) + return real_rename(source, target) + + with mock.patch.object(snapshot.os, 'rename', side_effect=rename_after_another_build): + made = self.prepare() + self.assertEqual(made.directory, str(self.home / 'code' / commit)) + self.assertEqual(len(renames), 2, 'both builds tried to put their snapshot in place') + self.assertEqual(os.listdir(self.home / 'code'), [commit], 'the losing build discarded its directory') + self.assertTrue((Path(made.code_root) / 'lab' / 'train.py').is_file()) + + def test_directory_outside_a_work_tree_is_refused(self): + outside = self.base / 'not a repo' + outside.mkdir() + with self.assertRaises(SnapshotError) as caught: + self.prepare(outside) + self.assertEqual(caught.exception.code, 'not-in-git') + self.assertIn('scratch', str(caught.exception)) + made = self.prepare(outside, scratch=True) + self.assertTrue(made.scratch) + self.assertEqual(made.commit, '') + + def test_repository_without_a_commit_is_a_clear_error(self): + empty = self.base / 'empty repo' + empty.mkdir() + self.git('init', '-q', cwd=empty) + with self.assertRaises(SnapshotError) as caught: + self.prepare(empty) + self.assertEqual(caught.exception.code, 'no-commit') + self.assertTrue(str(caught.exception).startswith('Git could not read the commit HEAD names')) + + def test_archive_entries_outside_the_snapshot_are_refused(self): + cases = (('../escape.txt', tarfile.REGTYPE), ('/abs.txt', tarfile.REGTYPE), + ('dev', tarfile.CHRTYPE), ('out', tarfile.SYMTYPE)) + for index, (name, kind) in enumerate(cases): + with self.subTest(name): + target = self.base / f'extract {index}' + target.mkdir() + archive = tempfile.TemporaryFile() + self.addCleanup(archive.close) + with tarfile.open(fileobj=archive, mode='w') as tar: + member = tarfile.TarInfo(name) + member.type = kind + if kind == tarfile.SYMTYPE: + member.linkname = '../../outside' + tar.addfile(member, None if kind != tarfile.REGTYPE else io.BytesIO(b'')) + archive.seek(0) + with self.assertRaises(SnapshotError) as caught: + snapshot._extract(archive, str(target)) + self.assertEqual(caught.exception.code, 'unsafe-archive') + self.assertFalse((self.base / 'escape.txt').exists()) + + def test_symbolic_link_inside_the_archive_is_kept(self): + made = self.prepare(code_paths=['lab']) + link = Path(made.code_root) / 'lab' / 'link.py' + self.assertTrue(link.is_symlink()) + self.assertEqual(link.read_text(), 'print("train")\n') + + def test_git_failure_is_an_error_naming_its_purpose(self): + with mock.patch.object(snapshot.subprocess, 'run', + side_effect=subprocess.TimeoutExpired(['git'], snapshot.GIT_SECONDS)): + with self.assertRaises(SnapshotError) as caught: + self.prepare() + self.assertEqual(caught.exception.code, 'not-in-git') + with mock.patch.object(snapshot.subprocess, 'run', side_effect=FileNotFoundError('git')): + with self.assertRaises(SnapshotError) as caught: + snapshot._git(self.repo, 'read the commit', 'rev-parse', 'HEAD') + self.assertIn('read the commit', str(caught.exception)) + self.assertIn('not found', str(caught.exception)) + + def test_git_is_run_without_a_shell_with_locks_off_and_a_time_limit(self): + with mock.patch.object(snapshot.subprocess, 'run', wraps=subprocess.run) as run: + self.prepare() + for call in run.call_args_list: + command = call.args[0] + self.assertIsInstance(command, list) + self.assertEqual(command[:3], ['git', '-C', command[2]]) + self.assertFalse(call.kwargs.get('shell')) + self.assertEqual(call.kwargs['env']['GIT_OPTIONAL_LOCKS'], '0') + self.assertEqual(call.kwargs['timeout'], 600 if 'archive' in command else 60) + self.assertTrue(any('archive' in call.args[0] for call in run.call_args_list)) + + +class ChangedSince(Repository): + def test_no_change(self): + self.assertEqual(snapshot.changed_since(self.repo, self.head()), set()) + + def test_committed_change_relative_to_the_code_root(self): + before = self.head() + self.write('lab/train.py', 'print("new")\n') + self.write('other/notes.txt', 'changed\n') + self.commit('second') + self.assertEqual(snapshot.changed_since(self.repo, before), {'lab/train.py', 'other/notes.txt'}) + self.assertEqual(snapshot.changed_since(self.repo / 'lab', before), {'train.py'}) + + def test_uncommitted_change_to_a_tracked_file(self): + before = self.head() + self.write('lab/data dir/table one.csv', 'changed\n') + self.write('lab/untracked.py', 'new\n') + self.write('other/notes.txt', 'changed\n') + self.assertEqual(snapshot.changed_since(self.repo / 'lab', before), {'data dir/table one.csv'}) + + def test_renamed_file_names_both_paths(self): + before = self.head() + self.git('mv', 'lab/train.py', 'lab/fit.py') + self.assertEqual(snapshot.changed_since(self.repo / 'lab', before), {'train.py', 'fit.py'}) + self.commit('rename') + self.assertEqual(snapshot.changed_since(self.repo / 'lab', before), {'train.py', 'fit.py'}) + + def test_commit_git_no_longer_has(self): + with self.assertRaises(SnapshotError) as caught: + snapshot.changed_since(self.repo, 'f' * 40) + self.assertEqual(caught.exception.code, 'unknown-commit') + self.assertIn('f' * 40, str(caught.exception)) + + def test_commit_must_be_a_full_id(self): + for value in ('HEAD', '--output=x', 'abc'): + with self.subTest(value), self.assertRaises(SnapshotError) as caught: + snapshot.changed_since(self.repo, value) + self.assertEqual(caught.exception.code, 'invalid') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_steps.py b/tests/test_tuldok_project_steps.py new file mode 100644 index 00000000..8566def4 --- /dev/null +++ b/tests/test_tuldok_project_steps.py @@ -0,0 +1,584 @@ +"""The steps Tuldok provides (docs/plans/definition-first-projects/spec-d4.md, section 6, slice D4c). + +import and freeze are first tested as functions on directories the test +writes. Then a project is built for real through builds.execute: a Git +repository holding the project folder and its code side by side (code_root +../code, so the line files are read from the commit and not from the +snapshot), an import from a relative source, two tasks with annotations, a +third task on the same dataset, and a freeze. Those tests run only when +TULDOK_DATABASE_URL names a disposable cluster. No GPU, no environment: a step +runs with Tuldok's own interpreter, and the code root holds a decoy +tuldok_project/steps.py that fails if it is ever imported. +""" +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest import mock +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from tuldok_project import (annotations, build_requests, builds, check, inside, local, plan, steps, # noqa: E402 + store) + +DATABASE = os.environ.get('TULDOK_DATABASE_URL') +GIT = ['git', '-c', 'user.name=Tuldok Test', '-c', 'user.email=test@example.invalid', '-c', 'commit.gpgsign=false'] +RIGHTS = 'Made by the test; anything may be done with them.' +AT = '2026-11-13T10:00:00Z' +PLY_FACES = b'ply\nformat ascii 1.0\nelement vertex 3\nproperty float x\nproperty float y\nproperty float z\n' \ + b'element face 1\nproperty list uchar int vertex_indices\nend_header\n0 0 0\n1 0 0\n0 1 0\n3 0 1 2\n' +PLY_POINTS = b'ply\nformat ascii 1.0\nelement vertex 1\nproperty float x\nproperty float y\nproperty float z\n' \ + b'end_header\n0 0 0\n' + + +def _sha(data): + return hashlib.sha256(data).hexdigest() + + +def _tree(root): + """A source tree holding a file of each kind, nested files, names that make the same id, and links.""" + files = { + 'notes/a.txt': b'first line\n\n \nsecond line\r\n', + 'notes/b.jsonl': b'{"text": "from json"}\n{"other": 1}\n[1, 2]\n', + 'images/cat.png': b'\x89PNG\r\n\x1a\nnot really', + 'sounds/hello.wav': b'RIFF....WAVE', + 'shapes/cloud.xyz': b'0 0 0\n1 1 1\n', + 'shapes/mesh.obj': b'v 0 0 0\nv 1 0 0\nv 0 1 0\nf 1 2 3\n', + 'shapes/faces.ply': PLY_FACES, + 'shapes/points.ply': PLY_POINTS, + 'arrays/x.npy': b'\x93NUMPY fake', + 'other/blob.bin': b'\x00\x01\x02', + 'same id/a b.txt': b'one\n', + 'same id/a_b.txt': b'two\n', + 'odd/bang!name.txt': b'bang\n', + } + for name, data in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return files + + +def _records(output): + with open(Path(output) / 'records.jsonl', encoding='utf-8') as handle: + return [json.loads(line) for line in handle] + + +def _row(record, state, value, sha256=None, by='owner'): + return {'record': record, 'sha256': sha256, 'state': state, 'value': value, 'by': by, 'at': AT} + + +class Shared(unittest.TestCase): + """What steps.py repeats from inside.py, standard library alone, stays the same.""" + + def test_kinds_patterns_and_ids(self): + self.assertEqual(steps.EXTENSIONS, inside.EXTENSIONS) + self.assertEqual(steps.KINDS, inside.KINDS) + for pattern in ('**/*', '*.png', 'a/**/b?.txt', '**', 'x/*/y'): + self.assertEqual(steps.pattern_regex(pattern).pattern, inside.pattern_regex(pattern).pattern) + self.assertEqual(steps.id_from('a b/c!d/é.png'), 'a_b__c_d___.png') + self.assertEqual(len(steps.id_from('x' * 300)), 200) + used = set() + self.assertEqual([steps.unique(name, used, 200) for name in ('a.png', 'a.png', 'a.png', 'a-2.png')], + ['a.png', 'a-2.png', 'a-3.png', 'a-2-2.png']) + long = 'y' * 196 + '.png' + self.assertEqual(len(steps.unique(long, {long}, 200)), 200) + + +class Import(unittest.TestCase): + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.temporary = Path(directory.name) + self.code = self.temporary / 'code' + self.source = self.code / 'data' + self.files = _tree(self.source) + outside = self.temporary / 'outside' + outside.mkdir() + (outside / 'secret.txt').write_text('never imported', encoding='utf-8') + os.symlink(outside / 'secret.txt', self.source / 'notes' / 'leaving.txt') + os.symlink(outside, self.source / 'leaving-folder') + os.symlink(self.source / 'images' / 'cat.png', self.source / 'other' / 'cat-again.png') + self.said = [] + + def output(self, name='out'): + path = self.temporary / name + path.mkdir() + return path + + def run_import(self, settings, *, name='out', code=True): + output = self.output(name) + counts = steps.import_files({'rights': RIGHTS, **settings}, str(output), + code_root=str(self.code) if code else None, out=self.said.append) + return output, counts, _records(output) + + def test_every_kind_from_a_relative_source(self): + output, counts, records = self.run_import({'source': 'data'}) + by_origin = {record['origin']['path']: record for record in records} + kinds = {path: record['kind'] for path, record in by_origin.items()} + self.assertEqual(kinds['images/cat.png'], 'image') + self.assertEqual(kinds['sounds/hello.wav'], 'audio') + self.assertEqual(kinds['shapes/cloud.xyz'], 'pointcloud') + self.assertEqual(kinds['shapes/points.ply'], 'pointcloud') + self.assertEqual(kinds['shapes/mesh.obj'], 'mesh') + self.assertEqual(kinds['shapes/faces.ply'], 'mesh') + self.assertEqual(kinds['arrays/x.npy'], 'array') + self.assertEqual(kinds['other/blob.bin'], 'file') + self.assertEqual(kinds['notes/a.txt'], 'text') + # A link inside the source is imported where it stands; links leaving it are skipped and counted. + self.assertIn('other/cat-again.png', by_origin) + self.assertNotIn('notes/leaving.txt', by_origin) + self.assertEqual(counts['links_out'], 2) + self.assertTrue(any('2 symbolic links leaving the source were skipped.' in line for line in self.said)) + self.assertEqual(len(records), len(self.files) + 1) + for path, data in self.files.items(): + record = by_origin[path] + self.assertEqual((record['sha256'], record['bytes'], record['rights']), (_sha(data), len(data), RIGHTS)) + placed = output / record['path'] + self.assertEqual(placed.read_bytes(), data) + self.assertEqual(os.stat(placed).st_ino, os.stat(self.source / path).st_ino) # hard-linked + self.assertEqual(list(record), ['id', 'kind', 'path', 'sha256', 'bytes', 'rights', 'origin']) + # Ids from paths, made unique; a name holding '!' is placed where an address cannot read it as an archive. + self.assertEqual(by_origin['images/cat.png']['id'], 'images__cat.png') + self.assertEqual(by_origin['same id/a b.txt']['id'], 'same_id__a_b.txt') + self.assertEqual(by_origin['same id/a_b.txt']['id'], 'same_id__a_b-2.txt') + self.assertEqual(by_origin['odd/bang!name.txt']['path'], 'files/odd/bang_name.txt') + self.assertEqual(by_origin['images/cat.png']['path'], 'files/images/cat.png') + self.assertEqual(len({record['id'] for record in records}), len(records)) + # Nothing outside the output was written, nothing in the source changed. + self.assertEqual(sorted(os.listdir(output)), ['files', 'records.jsonl']) + self.assertEqual((self.source / 'notes' / 'a.txt').read_bytes(), self.files['notes/a.txt']) + + def test_by_pattern_with_a_kind_from_an_absolute_source(self): + _, counts, records = self.run_import({'source': str(self.source), 'pattern': 'shapes/*.ply', 'kind': 'mesh'}, + code=False) + self.assertEqual([(record['origin']['path'], record['kind']) for record in records], + [('shapes/faces.ply', 'mesh'), ('shapes/points.ply', 'mesh')]) + self.assertEqual(counts['links_out'], 1) # the folder link is not a file the pattern can take + + def test_lines(self): + _, counts, records = self.run_import({'source': 'data', 'pattern': 'notes/*', 'lines': True}) + texts = [(record['id'], record['text'], record['origin']) for record in records] + self.assertEqual(texts, [ + ('notes__a.txt__1', 'first line', {'path': 'notes/a.txt', 'line': 1}), + ('notes__a.txt__4', 'second line', {'path': 'notes/a.txt', 'line': 4}), + ('notes__b.jsonl__1', 'from json', {'path': 'notes/b.jsonl', 'line': 1}), + ('notes__b.jsonl__2', '{"other": 1}', {'path': 'notes/b.jsonl', 'line': 2}), + ('notes__b.jsonl__3', '[1, 2]', {'path': 'notes/b.jsonl', 'line': 3})]) + for record in records: + self.assertEqual(record['kind'], 'text') + self.assertNotIn('path', record) + self.assertEqual(record['sha256'], _sha(record['text'].encode('utf-8'))) + self.assertEqual(record['bytes'], len(record['text'].encode('utf-8'))) + self.assertEqual(counts['lines'], 5) + + def test_the_product_is_read_by_the_readers_with_its_hashes(self): + output, _, records = self.run_import({'source': 'data', 'pattern': '**/*.txt', 'lines': True}) + product = inside.Product('run-import', str(output)) + listed = inside.records(product, limit=200) + self.assertEqual(listed['source'], 'records.jsonl') + self.assertEqual(listed['problems'], []) + self.assertEqual(listed['total'], len(records)) + shown = {record['id']: record for record in listed['records']} + for record in records: + self.assertEqual(shown[record['id']]['sha256'], record['sha256']) + self.assertEqual(shown[record['id']]['bytes'], record['bytes']) + self.assertIsNone(shown[record['id']]['meta']) + one = inside.record(product, records[0]['id']) + self.assertEqual(one['content']['text'], records[0]['text']) + self.assertEqual(one['record']['sha256'], records[0]['sha256']) + + def test_a_source_that_leaves_the_code_root(self): + os.symlink(self.temporary / 'outside', self.code / 'away') + with self.assertRaisesRegex(steps.StepError, 'outside the code root'): + steps.import_files({'source': 'away', 'rights': RIGHTS}, str(self.output()), code_root=str(self.code), + out=self.said.append) + + +class Freeze(unittest.TestCase): + """freeze on an imported product of ten text records, with two tasks.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.temporary = Path(directory.name) + source = self.temporary / 'source' + source.mkdir() + for number in range(10): + (source / f'r{number}.txt').write_text(f'record {number}\n', encoding='utf-8') + self.product = self.temporary / 'product' + self.product.mkdir() + steps.import_files({'source': str(source), 'rights': RIGHTS}, str(self.product), out=lambda text: None) + self.records = {record['id']: record for record in _records(self.product)} + self.count = 0 + + def lines(self, name, rows): + path = self.temporary / f'{name}.jsonl' + data = annotations.write(rows) + path.write_bytes(data) + return str(path), _sha(data) + + def task(self, name, rows, task='class', vocabulary=('good', 'bad')): + path, digest = self.lines(name, rows) + return {'reference': f'annotations/{name}', 'task': task, 'vocabulary': list(vocabulary), 'lines': path, + 'sha256': digest} + + def freeze(self, settings, tasks): + self.count += 1 + output = self.temporary / f'frozen-{self.count}' + output.mkdir() + said = [] + counts = steps.freeze_records(settings, str(output), records=str(self.product), tasks=tasks, + out=said.append) + return output, counts, _records(output), said + + def accepted_all(self): + return [_row(identifier, 'accepted', 'good', record['sha256']) + for identifier, record in self.records.items()] + + def test_splits_by_the_shares_the_same_for_a_seed(self): + tasks = [self.task('quality', self.accepted_all())] + settings = {'seed': 7, 'shares': {'train': 8, 'validation': 1, 'test': 1}} + output, counts, frozen, _ = self.freeze(settings, tasks) + self.assertEqual([record['split'] for record in frozen], ['test'] + ['train'] * 8 + ['validation']) + self.assertEqual(counts['included'], 10) + again = self.freeze(settings, tasks)[2] + self.assertEqual(frozen, again) + other = self.freeze({**settings, 'seed': 8}, tasks)[2] + self.assertNotEqual([record['id'] for record in frozen], [record['id'] for record in other]) + self.assertEqual(sorted(record['id'] for record in frozen), sorted(record['id'] for record in other)) + # The order is the SHA-256 of the seed and the record's hash. + expected = sorted(self.records.values(), key=lambda record: steps.order_key(7, record['sha256'], record['id'])) + self.assertEqual([record['id'] for record in frozen], [record['id'] for record in expected]) + for record in frozen: + source = self.records[record['id']] + self.assertEqual(record['sha256'], source['sha256']) + self.assertEqual(record['annotations'], {'quality': {'state': 'accepted', 'value': 'good'}}) + self.assertEqual(os.stat(output / record['path']).st_ino, os.stat(self.product / source['path']).st_ino) + self.assertEqual(list(record)[-2:], ['split', 'annotations']) + + def test_includes_by_state_and_leaves_out_stale_ones(self): + ids = sorted(self.records) + rows = [_row(ids[0], 'accepted', 'good', self.records[ids[0]]['sha256']), + _row(ids[1], 'accepted', 'bad', None), # tied by id alone + _row(ids[2], 'needs-work', 'bad', self.records[ids[2]]['sha256']), + _row(ids[3], 'rejected', None, self.records[ids[3]]['sha256']), + _row(ids[4], 'accepted', 'good', '0' * 64), # its content differs + _row('gone', 'accepted', 'good', '1' * 64), # its record is gone + _row(ids[5], 'draft', None, self.records[ids[5]]['sha256'])] + captions = [_row(ids[0], 'draft', 'a caption', self.records[ids[0]]['sha256']), + _row(ids[1], 'accepted', 'stale caption', '2' * 64), + _row(ids[9], 'accepted', 'not taken', None)] + tasks = [self.task('quality', rows), self.task('captions', captions, task='text', vocabulary=())] + _, counts, frozen, said = self.freeze({'seed': 1, 'shares': {'all': 1}}, tasks) + self.assertEqual(sorted(record['id'] for record in frozen), [ids[0], ids[1]]) + carried = {record['id']: record['annotations'] for record in frozen} + self.assertEqual(carried[ids[0]], {'quality': {'state': 'accepted', 'value': 'good'}, + 'captions': {'state': 'draft', 'value': 'a caption'}}) + self.assertEqual(carried[ids[1]], {'quality': {'state': 'accepted', 'value': 'bad'}}) + self.assertEqual((counts['stale'], counts['excluded'], counts['unannotated'], counts['stale_other']), + (2, 3, 4, 1)) + self.assertIn('2 annotations of quality are stale (the record is gone or its content differs) and were ' + 'left out.', said) + self.assertTrue(any(line.startswith('{"format":"tuldok_progress_v1","stage":"freeze"') for line in said)) + wider = self.freeze({'seed': 1, 'shares': {'all': 1}, 'include': ['accepted', 'needs-work', 'draft']}, + tasks)[2] + self.assertEqual(sorted(record['id'] for record in wider), [ids[0], ids[1], ids[2], ids[5]]) + + def test_a_changed_line_file_is_refused(self): + task = self.task('quality', self.accepted_all()) + task['sha256'] = '3' * 64 + with self.assertRaisesRegex(steps.StepError, 'not the file the build read'): + self.freeze({'seed': 1, 'shares': {'all': 1}}, [task]) + + +class Checked(unittest.TestCase): + """check: the step's settings, inputs and tasks (section 6), on a folder written by the test.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + self.folder = self.root / 'project' + _write_project(self.root, uuid.uuid4().hex) + + def problems(self): + report = check.check_folder(self.folder) + return {(problem.severity, problem.path, problem.pointer, problem.message) for problem in report.problems} + + def write(self, reference, document): + path = self.folder / f'{reference}.json' + path.parent.mkdir(exist_ok=True) + path.write_text(json.dumps(document, indent=2) + '\n', encoding='utf-8') + + def test_the_fixture_checks(self): + self.assertEqual({problem for problem in self.problems() if problem[0] == 'error'}, set()) + + def test_an_absolute_source_is_a_warning(self): + self.write('datasets/raw', _import_definition({'source': str(self.root / 'code' / 'data')})) + self.assertIn(('warning', 'datasets/raw.json', '/run/settings/source', + f'The source {self.root / "code" / "data"} is absolute, so this dataset can be built on this ' + 'machine only; give a path relative to the code root.'), self.problems()) + + def test_settings_and_fields_a_step_does_not_take(self): + document = _import_definition({'source': 'missing', 'kind': 'video', 'lines': 'yes', 'extra': 1, + 'pattern': '../x'}) + del document['run']['settings']['rights'] + document['run'].update(command=['python', 'x.py'], environment='environments/none', + outputs={'data': {'path': 'x', 'role': 'data'}}) + self.write('datasets/raw', document) + found = {(pointer, severity) for severity, path, pointer, _ in self.problems() if path == 'datasets/raw.json'} + for pointer in ('/run/settings/rights', '/run/settings/kind', '/run/settings/lines', '/run/settings/extra', + '/run/settings/pattern', '/run/settings/source', '/run/command', '/run/environment', + '/run/outputs'): + self.assertIn((pointer, 'error'), found) + + def test_a_freeze_by_a_task_of_another_dataset(self): + self.write('datasets/elsewhere', _import_definition({'source': 'data'})) + self.write('annotations/other-place', {'format': 'tuldok_definition_v1', 'kind': 'annotation', + 'title': 'Elsewhere', 'dataset': 'datasets/elsewhere', + 'task': 'review'}) + frozen = _freeze_definition(['annotations/quality', 'annotations/other-place']) + frozen['run']['settings'] = {'seed': 1.5, 'shares': {'train': 0}, 'include': ['accepted', 'accepted']} + self.write('datasets/frozen', frozen) + problems = self.problems() + self.assertIn(('error', 'datasets/frozen.json', '/run/annotations/1', + 'annotations/other-place annotates datasets/elsewhere, not datasets/raw, the input of this ' + 'freeze; name a task of its input.'), problems) + found = {pointer for severity, path, pointer, _ in problems if path == 'datasets/frozen.json'} + self.assertTrue({'/run/settings/seed', '/run/settings/shares', '/run/settings/include'} <= found) + + def test_a_plan_shows_the_step_and_notes_a_task_elsewhere(self): + definitions = {path.parent.name + '/' + path.stem: json.loads(path.read_text(encoding='utf-8')) + for path in self.folder.glob('*/*.json')} + definitions['annotations/quality']['dataset'] = 'datasets/frozen' + made = plan.make_plan(definitions, ['datasets/frozen']) + shown = {step['reference']: (step['command'], step['environment'], step['step'], step['annotations']) + for step in made['steps']} + self.assertEqual(shown, {'datasets/raw': (['tuldok', 'import'], None, 'import', []), + 'datasets/frozen': (['tuldok', 'freeze'], None, 'freeze', + ['annotations/quality', 'annotations/captions'])}) + self.assertIn('datasets/frozen freezes by annotations/quality, which annotates datasets/frozen, not its ' + 'input datasets/raw; it fails when built.', made['notes']) + + +# ------------------------------------------------------------- a project + +def _import_definition(settings): + return {'format': 'tuldok_definition_v1', 'kind': 'dataset', 'title': 'Imported', + 'run': {'step': 'import', 'settings': {'rights': RIGHTS, **settings}}} + + +def _freeze_definition(tasks): + return {'format': 'tuldok_definition_v1', 'kind': 'dataset', 'title': 'Frozen', + 'run': {'step': 'freeze', 'inputs': {'records': 'datasets/raw'}, 'annotations': tasks, + 'settings': {'seed': 11, 'shares': {'train': 3, 'test': 1}}}} + + +def _write_project(root, project_id): + """root/project (the folder, code_root ../code) and root/code (a source tree, and a decoy of Tuldok's steps).""" + folder, code = root / 'project', root / 'code' + for directory in ('datasets', 'annotations'): + (folder / directory).mkdir(parents=True) + (folder / 'tuldok.json').write_text(json.dumps({'format': 'tuldok_project_v1', 'id': project_id, + 'name': 'Steps test project', 'code_root': '../code'}, + indent=2) + '\n', encoding='utf-8') + definitions = { + 'datasets/raw': _import_definition({'source': 'data', 'pattern': '**/*.txt'}), + 'datasets/frozen': _freeze_definition(['annotations/quality', 'annotations/captions']), + 'annotations/quality': {'format': 'tuldok_definition_v1', 'kind': 'annotation', 'title': 'Quality', + 'dataset': 'datasets/raw', 'task': 'class', 'vocabulary': ['good', 'bad']}, + 'annotations/captions': {'format': 'tuldok_definition_v1', 'kind': 'annotation', 'title': 'Captions', + 'dataset': 'datasets/raw', 'task': 'text'}, + 'annotations/unrelated': {'format': 'tuldok_definition_v1', 'kind': 'annotation', 'title': 'Unrelated', + 'dataset': 'datasets/raw', 'task': 'review'}, + } + for reference, document in definitions.items(): + (folder / f'{reference}.json').write_text(json.dumps(document, indent=2) + '\n', encoding='utf-8') + for number in range(8): + path = code / 'data' / f'item-{number}.txt' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f'item {number}\n', encoding='utf-8') + decoy = code / 'tuldok_project' + decoy.mkdir() + (decoy / '__init__.py').write_text('', encoding='utf-8') + (decoy / 'steps.py').write_text('import sys\nprint("the snapshot\'s code was imported", file=sys.stderr)\n' + 'sys.exit(3)\n', encoding='utf-8') + + +def _writable(root): + for current, directories, _files in os.walk(root): + for name in directories: + path = os.path.join(current, name) + if not os.path.islink(path): + os.chmod(path, 0o755) + + +@unittest.skipUnless(DATABASE, 'TULDOK_DATABASE_URL is not set; database tests need a disposable cluster.') +class Built(unittest.TestCase): + """import and freeze built for real by builds.execute, and the frozen dataset's out-of-date state.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) / 'repository' + self.root.mkdir() + self.home = Path(directory.name) / 'home' + self.addCleanup(_writable, self.home) + patcher = mock.patch.dict(os.environ, {local.HOME_VARIABLE: str(self.home)}) + patcher.start() + self.addCleanup(patcher.stop) + self.project_id = uuid.uuid4().hex + _write_project(self.root, self.project_id) + self.folder = self.root / 'project' + self.git('init', '-q') + self.commit('The project.') + self.addCleanup(store.drop_project, self.project_id) + + def git(self, *arguments): + subprocess.run([*GIT, '-C', str(self.root), *arguments], check=True, capture_output=True, timeout=60) + + def commit(self, message): + self.git('add', '-A') + self.git('commit', '-q', '-m', message) + + def opened(self): + return store.open_project(self.folder) + + def build(self, *targets, scratch=False): + with self.opened() as project: + made = plan.build_plan(project, list(targets)) + done = builds.execute(self.folder, made, confirm=lambda given: True, scratch=scratch) + self.assertTrue(done['ok'], done) + return done + + def status(self): + with self.opened() as project: + return builds.status(project) + + def product(self, reference): + with self.opened() as project: + found = builds.products(project, reference) + return found[0] + + def write_lines(self, name, rows): + (self.folder / 'annotations' / f'{name}.jsonl').write_bytes(annotations.write(rows)) + + def test_import_then_freeze_and_out_of_date_by_the_line_files(self): + done = self.build('datasets/raw') + run = done['steps'][0] + with self.opened() as project: + row = builds.run_details(project, run['run_id']) + # Tuldok's interpreter and code; no environment; the snapshot's decoy never imported. + self.assertEqual(row['argv'][:5], [sys.executable, '-m', 'tuldok_project.steps', 'import', + os.path.join(row['directory'], 'settings.json')]) + self.assertIsNone(row['environment']) + self.assertEqual((row['env']['PYTHONSAFEPATH'], row['env']['PYTHONPATH']), ('1', builds.TULDOK_ROOT)) + stderr = (Path(row['directory']) / 'stderr.log').read_text(encoding='utf-8') + self.assertNotIn('snapshot', stderr) + raw = self.product('datasets/raw') + records = {record['id']: record for record in _records(raw['directory'])} + self.assertEqual(sorted(records), [f'item-{number}.txt' for number in range(8)]) + event = (Path(row['directory']) / 'stdout.log').read_text(encoding='utf-8') + self.assertIn('"stage":"import","completed":8,"total":8', event) + + ids = sorted(records) + self.write_lines('quality', [_row(identifier, 'accepted', 'good', records[identifier]['sha256']) + for identifier in ids[:6]] + [_row(ids[6], 'rejected', None)]) + self.write_lines('captions', [_row(ids[0], 'accepted', 'the first item', records[ids[0]]['sha256'])]) + self.write_lines('unrelated', [_row(ids[0], 'accepted', None)]) + self.commit('Annotations.') + done = self.build('datasets/frozen') + self.assertEqual([step['reference'] for step in done['steps']], ['datasets/frozen']) + frozen_product = self.product('datasets/frozen') + frozen = _records(frozen_product['directory']) + self.assertEqual([record['split'] for record in frozen], ['test'] + ['train'] * 5) # 6 cut 1:3 + self.assertEqual({record['id'] for record in frozen}, set(ids[:6])) + first = next(record for record in frozen if record['id'] == ids[0]) + self.assertEqual(first['annotations'], {'quality': {'state': 'accepted', 'value': 'good'}, + 'captions': {'state': 'accepted', 'value': 'the first item'}}) + with self.opened() as project: + row = builds.run_details(project, frozen_product['run_id']) + covered = build_requests.covers(project, plan.build_plan(project, ['datasets/frozen']), + ['datasets/frozen'], every=False, scratch=False) + quality = (self.folder / 'annotations' / 'quality.jsonl').read_bytes() + self.assertEqual(row['annotations']['annotations/quality']['sha256'], _sha(quality)) + self.assertEqual(row['annotations']['annotations/quality']['file'], 'annotations/quality.jsonl') + self.assertEqual(set(row['annotations']), {'annotations/quality', 'annotations/captions'}) + self.assertIn('annotations/quality', covered['definitions']) + self.assertIn('annotations/captions', covered['definitions']) + self.assertIsNone(row['environment']) + self.assertEqual(self.status()['datasets/frozen']['state'], 'built') + # The frozen product is read by spec-d1's readers, with its splits and annotations. + listed = inside.records(inside.Product(frozen_product['run_id'], frozen_product['directory']), limit=200) + self.assertEqual((listed['splits'], listed['problems']), ({'train': 5, 'test': 1}, [])) + self.assertEqual(next(record for record in listed['records'] if record['id'] == ids[0])['annotations'], + first['annotations']) + + # Another task's file changes: still built. + self.write_lines('unrelated', [_row(ids[1], 'accepted', None)]) + self.commit('Another task.') + self.assertEqual(self.status()['datasets/frozen']['state'], 'built') + # One of its tasks' files changes but is not committed: the commit a build would use is unchanged. + self.write_lines('quality', [_row(ids[0], 'rejected', None)]) + self.assertEqual(self.status()['datasets/frozen']['state'], 'built') + self.commit('Fewer accepted.') + entry = self.status()['datasets/frozen'] + self.assertEqual(entry['state'], 'out-of-date') + self.assertEqual(entry['causes'], [f'the annotations of annotations/quality (annotations/quality.jsonl) changed ' + f'since run {frozen_product["run_id"][:12]}']) + self.assertEqual(self.status()['datasets/raw']['state'], 'built') + done = self.build('datasets/frozen') + self.assertEqual(_records(self.product('datasets/frozen')['directory']), []) + self.assertEqual(self.status()['datasets/frozen']['state'], 'built') + # A file removed is a cause too. + os.remove(self.folder / 'annotations' / 'captions.jsonl') + self.commit('No captions.') + self.assertIn('are gone since run', self.status()['datasets/frozen']['causes'][0]) + + def test_the_source_changing_puts_an_import_out_of_date(self): + self.build('datasets/raw') + (self.root / 'code' / 'data' / 'item-0.txt').write_text('changed\n', encoding='utf-8') + self.commit('A source file changed.') + causes = self.status()['datasets/raw']['causes'] + self.assertEqual(len(causes), 1) + self.assertTrue(causes[0].startswith('its source data changed since commit '), causes) + self.assertTrue(causes[0].endswith(': data/item-0.txt'), causes) + + def test_a_scratch_freeze_reads_the_working_tree(self): + self.build('datasets/raw') + records = {record['id']: record for record in _records(self.product('datasets/raw')['directory'])} + self.write_lines('quality', [_row(identifier, 'accepted', 'good', record['sha256']) + for identifier, record in records.items()]) + self.build('datasets/frozen', scratch=True) + self.assertEqual(len(_records(self.product('datasets/frozen')['directory'])), 8) + self.assertEqual(self.status()['datasets/frozen']['state'], 'built') + self.write_lines('quality', [_row('item-0.txt', 'accepted', 'bad', records['item-0.txt']['sha256'])]) + self.assertIn('annotations/quality', self.status()['datasets/frozen']['causes'][0]) + + def test_a_step_is_cancelled_like_any_run(self): + # The cancel file is written before the custodian first looks, while the step is still starting. + for number in range(200): + (self.root / 'code' / 'data' / f'bulk-{number:05}.txt').write_text(str(number), encoding='utf-8') + self.commit('Many files.') + with self.opened() as project: + made = plan.build_plan(project, ['datasets/raw']) + + def on_event(kind, item): + if kind == 'start': + Path(item['directory'], 'cancel').touch() + + done = builds.execute(self.folder, made, confirm=lambda given: True, on_event=on_event) + self.assertFalse(done['ok']) + self.assertEqual(done['steps'][0]['state'], 'cancelled') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_store.py b/tests/test_tuldok_project_store.py new file mode 100644 index 00000000..c3a7804a --- /dev/null +++ b/tests/test_tuldok_project_store.py @@ -0,0 +1,656 @@ +"""The database copy of a project folder: open, states, edit and save (spec sections 4.1 to 4.5). + +Database tests run only when TULDOK_DATABASE_URL names a disposable cluster +(see docs/plans/definition-first-projects/spec-m1-m2.md, section 1). Each test +copies tests/fixtures/tuldok_project/minimal into a temporary directory, gives +it a fresh project id and drops that project's schema afterwards, so tests +share no state. The states of section 4.3 are also tested as a pure function. +""" +import json +import os +from pathlib import Path +import shutil +import sys +import tempfile +import unittest +from unittest import mock +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import psycopg # noqa: E402 + +from metadata import connection_info # noqa: E402 +from tuldok_project import check, graph, store # noqa: E402 +from tuldok_project.store import ProjectError # noqa: E402 + +REPOSITORY = Path(__file__).resolve().parents[1] +FIXTURE = REPOSITORY / 'tests' / 'fixtures' / 'tuldok_project' / 'minimal' +DATABASE = os.environ.get('TULDOK_DATABASE_URL') + +FIXTURE_STATES = { + 'annotations/wave-quality': 'none', # a task is never built (spec-d4.md) + 'benchmarks/rollout': 'not-built', # its early-fit subject carries nothing + 'charts/rollout-comparison': 'none', + 'charts/rollout-progression': 'none', + 'charts/rollout-ranking': 'none', + 'charts/speed-accuracy': 'none', + 'datasets/waves': 'not-built', + 'environments/lab-cpu': 'not-set-up', + 'models/stepper': 'none', + 'recipes/early-fit': 'cannot-rebuild', + 'recipes/stepper-k1': 'not-built', + 'recipes/stepper-k2': 'not-built', + 'steps/solver': 'not-built', +} + +CLEAN_STATUS = {'unsaved': [], 'removed': [], 'added': [], 'changed': [], 'missing': [], 'irregular': [], + 'results_unsaved': [], 'results_changed': [], 'annotations_unsaved': [], 'annotations_changed': []} + + + +def _fixture_documents(): + project = FIXTURE / 'project' + found = {} + for path in sorted(project.glob('*/*.json')): + found[f'{path.parent.name}/{path.stem}'] = json.loads(path.read_text(encoding='utf-8')) + return found + + +class States(unittest.TestCase): + """graph.states, section 4.3, without a database.""" + + def setUp(self): + self.documents = _fixture_documents() + + def mark(self, reference, rebuild): + self.documents[reference]['earlier_work'] = {'rebuild': rebuild, 'reason': 'Test.'} + + def test_fixture(self): + self.assertEqual(graph.states(self.documents), FIXTURE_STATES) + + def test_needs_work_travels_up_through_inputs(self): + self.mark('steps/solver', 'needs-work') + states = graph.states(self.documents) + for reference in ('steps/solver', 'datasets/waves', 'recipes/stepper-k1', 'recipes/stepper-k2', + 'benchmarks/rollout'): + self.assertEqual(states[reference], 'needs-work', reference) + self.assertEqual(states['recipes/early-fit'], 'cannot-rebuild') + self.assertEqual(states['environments/lab-cpu'], 'not-set-up') + + def test_cannot_rebuild_travels_up_and_wins_over_needs_work(self): + self.mark('steps/solver', 'needs-work') + self.mark('recipes/stepper-k1', 'no') + states = graph.states(self.documents) + self.assertEqual(states['datasets/waves'], 'needs-work') + self.assertEqual(states['recipes/stepper-k1'], 'cannot-rebuild') + self.assertEqual(states['recipes/stepper-k2'], 'cannot-rebuild') # k1 is no, waves needs-work + self.assertEqual(states['benchmarks/rollout'], 'needs-work') # k1 is only a subject of it + + def test_only_input_edges_carry_states(self): + self.mark('models/stepper', 'no') # used by recipes through "model", not an input + states = graph.states(self.documents) + self.assertEqual(states['models/stepper'], 'cannot-rebuild') + self.assertEqual(states['recipes/stepper-k1'], 'not-built') + + def test_cycle_is_tolerated(self): + self.documents['steps/solver']['run']['inputs'] = {'scenes': 'datasets/waves'} + self.mark('datasets/waves', 'needs-work') + states = graph.states(self.documents) + self.assertEqual(states['steps/solver'], 'needs-work') + self.assertEqual(states['datasets/waves'], 'needs-work') + + +@unittest.skipUnless(DATABASE, 'TULDOK_DATABASE_URL is not set; database tests need a disposable cluster.') +class StoreCase(unittest.TestCase): + """A fresh copy of the fixture with its own project id, dropped afterwards.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.temporary = Path(directory.name) + self.base = self.temporary / 'minimal' + shutil.copytree(FIXTURE, self.base) + self.folder = self.base / 'project' + self.project_id = uuid.uuid4().hex + project_file = self.folder / 'tuldok.json' + document = json.loads(project_file.read_text(encoding='utf-8')) + document['id'] = self.project_id + project_file.write_text(json.dumps(document, indent=2) + '\n', encoding='utf-8') + self.schema = f'project_{self.project_id}' + self.opened = [] + self.addCleanup(self._cleanup) + + def _cleanup(self): + for project in self.opened: + project.close() + store.drop_project(self.project_id) + + def open(self, folder=None, **options): + project = store.open_project(folder or self.folder, **options) + self.opened.append(project) + return project + + def close(self, project): + project.close() + self.opened.remove(project) + + def database(self): + connection = psycopg.connect(connection_info(), autocommit=True) + self.addCleanup(connection.close) + return connection + + def dump(self): + """Every row of the project's tables, for asserting that nothing changed.""" + connection = self.database() + if connection.execute('SELECT 1 FROM pg_namespace WHERE nspname = %s', (self.schema,)).fetchone() is None: + return None + tables = {} + for table in ('state', 'definitions', 'edges', 'results_files', 'result_summaries', 'result_values'): + query = psycopg.sql.SQL('SELECT * FROM {}.{} ORDER BY 1, 2').format( + psycopg.sql.Identifier(self.schema), psycopg.sql.Identifier(table)) + tables[table] = connection.execute(query).fetchall() + return tables + + def read(self, reference): + return (self.folder / f'{reference}.json').read_text(encoding='utf-8') + + def write(self, reference, text, folder=None): + path = (folder or self.folder) / f'{reference}.json' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding='utf-8') + + def edited(self, reference, change, folder=None): + """The file's document with change(document) applied, as indented text.""" + document = json.loads(((folder or self.folder) / f'{reference}.json').read_text(encoding='utf-8')) + change(document) + return json.dumps(document, indent=4) + '\n' + + def counting_checker(self): + patcher = mock.patch.object(check, 'check_folder', wraps=check.check_folder) + checker = patcher.start() + self.addCleanup(patcher.stop) + return checker + + +class Open(StoreCase): + + def test_first_open_imports_every_definition(self): + project = self.open() + summary = project.summary() + self.assertEqual(summary['project_id'], self.project_id) + self.assertEqual(summary['folder'], os.path.realpath(self.folder)) + self.assertEqual(summary['code_root'], '../code') + self.assertEqual(summary['kinds'], {'annotation': 1, 'benchmark': 1, 'chart': 4, 'dataset': 1, + 'environment': 1, 'model': 1, 'recipe': 3, 'step': 1}) + self.assertEqual(summary['states'], {'cannot-rebuild': 1, 'none': 6, 'not-built': 5, 'not-set-up': 1}) + rows = project.definitions() + self.assertEqual({row['reference']: row['state'] for row in rows}, FIXTURE_STATES) + self.assertEqual([row['reference'] for row in project.definitions(kind='chart')], + ['charts/rollout-comparison', 'charts/rollout-progression', 'charts/rollout-ranking', + 'charts/speed-accuracy']) + + shown = project.definition('datasets/waves') + self.assertEqual(shown['text'], self.read('datasets/waves')) # the exact file text + self.assertEqual(shown['document']['run']['settings']['layout'], {'grid': [16, 16]}) + self.assertEqual(shown['uses'], [{'reference': 'environments/lab-cpu', 'role': 'environment'}, + {'reference': 'steps/solver', 'role': 'input:solver'}]) + self.assertIn({'reference': 'recipes/stepper-k1', 'role': 'input:training'}, shown['used_by']) + self.assertIn({'reference': 'benchmarks/rollout', 'role': 'cases'}, shown['used_by']) + + drawn = project.graph() + self.assertEqual(len(drawn['nodes']), 13) + self.assertIn({'from': 'recipes/stepper-k2', 'to': 'recipes/stepper-k1', 'role': 'input:start'}, drawn['edges']) + self.assertEqual(len(drawn['edges']), len(check.check_folder(self.folder).edges)) + order = project.order() + self.assertLess(order.index('steps/solver'), order.index('datasets/waves')) + self.assertLess(order.index('recipes/stepper-k1'), order.index('recipes/stepper-k2')) + self.assertEqual(project.status(), CLEAN_STATUS) + + def test_second_open_of_unchanged_folder_does_not_run_the_checker(self): + self.close(self.open()) + before = self.dump() + checker = self.counting_checker() + project = self.open() + checker.assert_not_called() + self.assertEqual(self.dump(), before) + self.assertEqual(len(project.definitions()), 13) + + def test_changed_added_and_removed_files_are_imported(self): + self.close(self.open()) + self.write('datasets/waves', self.edited('datasets/waves', lambda d: d.update(title='Waves again'))) + added = self.edited('charts/speed-accuracy', lambda d: d.update(title='Seconds against error')) + self.write('charts/seconds', added) + (self.folder / 'charts' / 'speed-accuracy.json').unlink() + checker = self.counting_checker() + project = self.open() + checker.assert_called_once() + references = {row['reference']: row for row in project.definitions()} + self.assertEqual(references['datasets/waves']['title'], 'Waves again') + self.assertEqual(project.definition('charts/seconds')['text'], added) + self.assertNotIn('charts/speed-accuracy', references) + self.assertNotIn('charts/speed-accuracy', [edge['from'] for edge in project.graph()['edges']]) + self.assertIn('charts/seconds', [edge['from'] for edge in project.graph()['edges']]) + + def test_folder_with_errors_leaves_the_database_unchanged(self): + self.close(self.open()) + before = self.dump() + self.write('datasets/waves', self.edited('datasets/waves', lambda d: d.pop('title'))) + with self.assertRaises(ProjectError) as caught: + self.open() + self.assertEqual(caught.exception.code, 'invalid') + self.assertIn(('datasets/waves.json', '/title'), + [(p.path, p.pointer) for p in caught.exception.problems if p.severity == 'error']) + self.assertEqual(self.dump(), before) + + def test_first_open_with_errors_creates_nothing(self): + self.write('recipes/stepper-k2', self.edited('recipes/stepper-k2', + lambda d: d.update(model='models/missing'))) + with self.assertRaises(ProjectError) as caught: + self.open() + self.assertEqual([(p.path, p.pointer) for p in caught.exception.problems], + [('recipes/stepper-k2.json', '/model')]) + self.assertIsNone(self.dump()) + + def test_second_path_requires_rebind(self): + self.close(self.open()) + moved = self.temporary / 'moved' + shutil.copytree(self.base, moved) + with self.assertRaises(ProjectError) as caught: + self.open(moved / 'project') + self.assertEqual(caught.exception.code, 'bound-elsewhere') + self.assertIn(os.path.realpath(self.folder), str(caught.exception)) + self.assertIn(os.path.realpath(moved / 'project'), str(caught.exception)) + checker = self.counting_checker() + project = self.open(moved / 'project', rebind=True) + checker.assert_not_called() # same files: rebinding attaches without a check + self.assertEqual(project.summary()['folder'], os.path.realpath(moved / 'project')) + self.close(project) + with self.assertRaises(ProjectError): + self.open() # the original path is now the other one + + def test_rebind_to_a_folder_with_different_files_imports_them(self): + self.close(self.open()) + moved = self.temporary / 'moved' + shutil.copytree(self.base, moved) + self.write('steps/solver', self.edited('steps/solver', lambda d: d.update(title='Moved solver'), + folder=moved / 'project'), folder=moved / 'project') + project = self.open(moved / 'project', rebind=True) + self.assertEqual(project.definition('steps/solver')['title'], 'Moved solver') + + def test_a_project_is_open_once(self): + self.open() + with self.assertRaises(ProjectError) as caught: + self.open() + self.assertEqual(caught.exception.code, 'busy') + with self.assertRaises(ProjectError): + store.drop_project(self.project_id) + + def test_states_travel_up_in_the_store(self): + self.write('steps/solver', self.edited('steps/solver', lambda d: d.update( + earlier_work={'rebuild': 'needs-work', 'reason': 'Its command needs a new flag.'}))) + self.write('recipes/stepper-k1', self.edited('recipes/stepper-k1', lambda d: d.update( + earlier_work={'rebuild': 'no', 'reason': 'Its code is lost.'}))) + project = self.open() + states = {row['reference']: row['state'] for row in project.definitions()} + self.assertEqual(states['datasets/waves'], 'needs-work') + self.assertEqual(states['benchmarks/rollout'], 'needs-work') + self.assertEqual(states['recipes/stepper-k2'], 'cannot-rebuild') + self.assertEqual(project.definition('recipes/stepper-k2')['state'], 'cannot-rebuild') + + +class Edit(StoreCase): + + def setUp(self): + super().setUp() + self.project = self.open() + + def assert_refused(self, reference, text, *expected): + before = self.dump() + with self.assertRaises(ProjectError) as caught: + self.project.set_definition(reference, text) + self.assertEqual(caught.exception.code, 'invalid') + found = [(p.path, p.pointer) for p in caught.exception.problems] + for item in expected: + self.assertIn(item, found) + self.assertEqual(self.dump(), before) + return caught.exception + + def test_set_definition_stores_the_exact_text_and_its_edges(self): + extra = json.dumps({'format': 'tuldok_definition_v1', 'kind': 'step', 'title': 'Extra', + 'run': {'command': ['true'], 'outputs': {'mesh': {'path': 'mesh.bin', 'role': 'file'}}}}) + self.project.set_definition('steps/extra', extra) + waves = self.edited('datasets/waves', lambda d: d['run']['inputs'].update( + mesh={'from': 'steps/extra', 'output': 'mesh'})) + self.project.set_definition('datasets/waves', waves) + shown = self.project.definition('datasets/waves') + self.assertEqual(shown['text'], waves) + self.assertTrue(shown['unsaved']) + self.assertIn({'reference': 'steps/extra', 'role': 'input:mesh'}, shown['uses']) + self.assertEqual(self.project.definition('steps/extra')['used_by'], + [{'reference': 'datasets/waves', 'role': 'input:mesh'}]) + self.assertEqual(self.project.status()['unsaved'], ['datasets/waves', 'steps/extra']) + self.assertLess(self.project.order().index('steps/extra'), self.project.order().index('datasets/waves')) + + def test_set_definition_changes_states(self): + self.project.set_definition('steps/solver', self.edited('steps/solver', lambda d: d.update( + earlier_work={'rebuild': 'needs-work', 'reason': 'Its command needs a new flag.'}))) + self.assertEqual(self.project.definition('recipes/stepper-k2')['state'], 'needs-work') + + def test_refused_when_not_json(self): + self.assert_refused('datasets/waves', '{"format": "tuldok_definition_v1",', ('datasets/waves.json', '')) + + def test_refused_when_the_document_alone_is_wrong(self): + self.assert_refused('datasets/waves', self.edited('datasets/waves', lambda d: d.pop('title')), + ('datasets/waves.json', '/title')) + self.assert_refused('datasets/waves', self.edited('datasets/waves', lambda d: d.update(kind='step')), + ('datasets/waves.json', '/kind')) + self.assert_refused('datasets/waves', self.edited('datasets/waves', lambda d: d['run'].update( + sources=['tools/missing.py'])), ('datasets/waves.json', '/run/sources/0')) + + def test_refused_when_a_reference_is_missing(self): + self.assert_refused('recipes/stepper-k1', self.edited('recipes/stepper-k1', lambda d: d.update( + model='models/missing')), ('recipes/stepper-k1.json', '/model')) + + def test_refused_when_a_named_output_is_missing(self): + self.assert_refused('recipes/stepper-k2', self.edited('recipes/stepper-k2', lambda d: d['run']['inputs'].update( + start={'from': 'recipes/stepper-k1', 'output': 'nothing'})), + ('recipes/stepper-k2.json', '/run/inputs/start/output')) + # Taking an output away from a producer breaks the definition that names it. + self.assert_refused('steps/solver', self.edited('steps/solver', lambda d: d['run']['outputs'].pop('binary')), + ('datasets/waves.json', '/run/inputs/solver/output')) + + def test_refused_when_it_makes_a_cycle(self): + error = self.assert_refused('steps/solver', self.edited('steps/solver', lambda d: d['run'].update( + inputs={'scenes': 'datasets/waves'})), ('steps/solver.json', '/run/inputs/scenes')) + self.assertTrue(any('cycle' in p.message for p in error.problems)) + + def test_refused_when_a_chart_no_longer_fits_its_benchmark(self): + self.assert_refused('benchmarks/rollout', self.edited('benchmarks/rollout', lambda d: d['metrics'].pop(1)), + ('charts/speed-accuracy.json', '/metric_x')) + + def test_refused_for_a_bad_reference(self): + with self.assertRaises(ProjectError) as caught: + self.project.set_definition('things/waves', self.read('datasets/waves')) + self.assertEqual(caught.exception.code, 'not-found') + + def test_remove_definition(self): + self.project.remove_definition('charts/speed-accuracy') + row = {r['reference']: r for r in self.project.definitions()}['charts/speed-accuracy'] + self.assertTrue(row['removed']) + self.assertNotIn('charts/speed-accuracy', [n['reference'] for n in self.project.graph()['nodes']]) + self.assertEqual(self.project.status()['removed'], ['charts/speed-accuracy']) + self.assertEqual(self.project.summary()['kinds']['chart'], 3) + + def test_remove_refused_while_used(self): + before = self.dump() + with self.assertRaises(ProjectError) as caught: + self.project.remove_definition('datasets/waves') + self.assertEqual(caught.exception.code, 'in-use') + self.assertEqual(caught.exception.references, ['annotations/wave-quality', 'benchmarks/rollout', + 'recipes/stepper-k1', 'recipes/stepper-k2']) + self.assertEqual(self.dump(), before) + + def test_removing_a_user_frees_what_it_used(self): + # tuldok.json's map.chart uses charts/rollout-progression (spec-w1.md): drop it first. + self.close(self.project) + project_file = self.folder / 'tuldok.json' + document = json.loads(project_file.read_text(encoding='utf-8')) + del document['map'] + project_file.write_text(json.dumps(document, indent=2) + '\n', encoding='utf-8') + self.project = self.open() + for chart in ('speed-accuracy', 'rollout-ranking', 'rollout-progression', 'rollout-comparison'): + self.project.remove_definition(f'charts/{chart}') + self.project.remove_definition('benchmarks/rollout') + self.assertTrue({r['reference']: r for r in self.project.definitions()}['benchmarks/rollout']['removed']) + + def test_open_refuses_a_folder_that_no_longer_fits_unsaved_edits(self): + self.project.set_definition('datasets/waves', self.edited('datasets/waves', lambda d: d['run']['inputs'].update( + log={'from': 'steps/solver', 'output': 'log'}))) + self.close(self.project) + self.write('steps/solver', self.edited('steps/solver', lambda d: d['run']['outputs'].pop('log'))) + self.assertTrue(check.check_folder(self.folder).ok) # the folder alone is fine + before = self.dump() + with self.assertRaises(ProjectError) as caught: + self.open() + self.assertEqual(caught.exception.code, 'invalid') + self.assertIn(('datasets/waves.json', '/run/inputs/log/output'), + [(p.path, p.pointer) for p in caught.exception.problems]) + self.assertEqual(self.dump(), before) + + def test_removing_a_definition_never_saved_forgets_it(self): + extra = json.dumps({'format': 'tuldok_definition_v1', 'kind': 'model', 'title': 'Extra'}) + self.project.set_definition('models/extra', extra) + self.project.remove_definition('models/extra') + self.assertNotIn('models/extra', [r['reference'] for r in self.project.definitions()]) + + +class Save(StoreCase): + + def setUp(self): + super().setUp() + self.project = self.open() + + def test_save_writes_and_deletes_files(self): + waves = self.edited('datasets/waves', lambda d: d.update(title='Waves, saved')) + self.project.set_definition('datasets/waves', waves) + extra = '{"format":"tuldok_definition_v1","kind":"model","title":"Extra"}' + self.project.set_definition('models/extra', extra) + self.project.remove_definition('charts/speed-accuracy') + result = self.project.save() + self.assertEqual(result, {'written': ['datasets/waves', 'models/extra'], 'deleted': ['charts/speed-accuracy'], + 'results_written': [], 'results_deleted': [], + 'annotations_written': [], 'annotations_deleted': []}) + self.assertEqual(self.read('datasets/waves'), waves) + self.assertEqual(self.read('models/extra'), extra) + self.assertFalse((self.folder / 'charts' / 'speed-accuracy.json').exists()) + self.assertEqual(sorted(p.name for p in (self.folder / 'datasets').iterdir()), ['waves.json']) + self.assertEqual(self.project.status(), CLEAN_STATUS) + self.assertTrue(check.check_folder(self.folder).ok) + # What was saved is what the database records, so the next open attaches without a check. + self.close(self.project) + checker = self.counting_checker() + reopened = self.open() + checker.assert_not_called() + self.assertFalse(reopened.definition('datasets/waves')['unsaved']) + + def test_save_refuses_when_a_file_changed_on_disk(self): + self.project.set_definition('steps/solver', self.edited('steps/solver', lambda d: d.update(title='Solver 2'))) + self.project.set_definition('datasets/waves', self.edited('datasets/waves', lambda d: d.update(title='W2'))) + self.project.remove_definition('charts/speed-accuracy') + on_disk = self.edited('datasets/waves', lambda d: d.update(title='Changed by hand')) + self.write('datasets/waves', on_disk) + solver = self.read('steps/solver') + self.assertEqual(self.project.status()['changed'], ['datasets/waves']) + with self.assertRaises(ProjectError) as caught: + self.project.save() + self.assertEqual(caught.exception.code, 'changed-on-disk') + self.assertEqual(caught.exception.references, ['datasets/waves']) + self.assertEqual(self.read('datasets/waves'), on_disk) + self.assertEqual(self.read('steps/solver'), solver) + self.assertTrue((self.folder / 'charts' / 'speed-accuracy.json').exists()) + self.assertTrue(self.project.definition('steps/solver')['unsaved']) + + def test_save_refuses_when_a_new_file_appeared_on_disk(self): + self.project.set_definition('models/extra', '{"format":"tuldok_definition_v1","kind":"model","title":"Extra"}') + self.write('models/extra', '{"format":"tuldok_definition_v1","kind":"model","title":"By hand"}') + with self.assertRaises(ProjectError) as caught: + self.project.save() + self.assertEqual(caught.exception.references, ['models/extra']) + + +class Conflict(StoreCase): + + def setUp(self): + super().setUp() + project = self.open() + self.edit = self.edited('datasets/waves', lambda d: d.update(title='Edited in Tuldok')) + project.set_definition('datasets/waves', self.edit) + self.close(project) + self.on_disk = self.edited('datasets/waves', lambda d: d.update(title='Edited by hand')) + self.write('datasets/waves', self.on_disk) + + def test_refused_without_a_choice(self): + before = self.dump() + with self.assertRaises(ProjectError) as caught: + self.open() + self.assertEqual(caught.exception.code, 'conflict') + self.assertEqual(caught.exception.references, ['datasets/waves']) + self.assertEqual(self.dump(), before) + + def test_keep_holds_the_edit_and_save_then_overwrites(self): + project = self.open(on_conflict='keep') + shown = project.definition('datasets/waves') + self.assertEqual(shown['text'], self.edit) + self.assertTrue(shown['unsaved']) + self.assertEqual(project.status()['changed'], []) + project.save() + self.assertEqual(self.read('datasets/waves'), self.edit) + + def test_discard_takes_the_file(self): + project = self.open(on_conflict='discard') + shown = project.definition('datasets/waves') + self.assertEqual(shown['text'], self.on_disk) + self.assertFalse(shown['unsaved']) + + def test_file_removed_on_disk_with_an_unsaved_edit(self): + project = self.open(on_conflict='discard') + chart = self.edited('charts/speed-accuracy', lambda d: d.update(title='Edited chart')) + project.set_definition('charts/speed-accuracy', chart) + self.close(project) + (self.folder / 'charts' / 'speed-accuracy.json').unlink() + with self.assertRaises(ProjectError) as caught: + self.open() + self.assertEqual(caught.exception.references, ['charts/speed-accuracy']) + project = self.open(on_conflict='keep') + self.assertTrue(project.definition('charts/speed-accuracy')['unsaved']) + project.save() + self.assertEqual(self.read('charts/speed-accuracy'), chart) + self.close(project) + (self.folder / 'charts' / 'speed-accuracy.json').unlink() + project = self.open(on_conflict='discard') # the row is clean now: no conflict, the file is just gone + self.assertNotIn('charts/speed-accuracy', [r['reference'] for r in project.definitions()]) + + +class Revert(StoreCase): + """revert_definition and definition_problems (spec-w3.md section 2).""" + + def setUp(self): + super().setUp() + self.project = self.open() + + def dump(self): + """Every row, in an order that does not depend on the order rows were written.""" + return {table: sorted(rows, key=repr) for table, rows in super().dump().items()} + + def refused(self, reference, code): + before = self.dump() + with self.assertRaises(ProjectError) as caught: + self.project.revert_definition(reference) + self.assertEqual(caught.exception.code, code) + self.assertEqual(self.dump(), before) + return caught.exception + + def test_an_unsaved_edit_gives_way_to_the_file(self): + before = self.dump() + self.project.set_definition('recipes/stepper-k2', self.edited('recipes/stepper-k2', lambda d: d.update( + title='Edited', change='component', derived_from='recipes/early-fit'))) + self.assertNotEqual(self.dump(), before) + self.assertEqual(self.project.revert_definition('recipes/stepper-k2'), + {'reference': 'recipes/stepper-k2', 'dropped': False}) + self.assertEqual(self.dump(), before) # text, document, title, kind, file digest, flags and edges + self.assertEqual(self.project.status(), CLEAN_STATUS) + + def test_a_removed_definition_comes_back(self): + before = self.dump() + self.project.remove_definition('charts/speed-accuracy') + self.project.revert_definition('charts/speed-accuracy') + self.assertEqual(self.dump(), before) + self.assertIn('charts/speed-accuracy', [node['reference'] for node in self.project.graph()['nodes']]) + + def test_the_file_as_it_is_now_on_disk_is_taken(self): + self.project.remove_definition('charts/speed-accuracy') + text = self.edited('charts/speed-accuracy', lambda d: d.update(title='Changed on disk')) + self.write('charts/speed-accuracy', text) + self.project.revert_definition('charts/speed-accuracy') + shown = self.project.definition('charts/speed-accuracy') + self.assertEqual((shown['text'], shown['title'], shown['unsaved'], shown['removed']), + (text, 'Changed on disk', False, False)) + self.assertEqual(self.project.status()['changed'], []) # its digest is the file's now + + def test_a_removed_benchmark_gets_its_result_rows_back(self): + self.close(self.project) + project_file = self.folder / 'tuldok.json' + document = json.loads(project_file.read_text(encoding='utf-8')) + del document['map'] + project_file.write_text(json.dumps(document, indent=2) + '\n', encoding='utf-8') + self.project = self.open() + before = self.dump() + for chart in ('speed-accuracy', 'rollout-ranking', 'rollout-progression', 'rollout-comparison'): + self.project.remove_definition(f'charts/{chart}') + self.project.remove_definition('benchmarks/rollout') + self.assertEqual(self.dump()['result_summaries'], []) + self.project.revert_definition('benchmarks/rollout') + for chart in ('speed-accuracy', 'rollout-ranking', 'rollout-progression', 'rollout-comparison'): + self.project.revert_definition(f'charts/{chart}') + self.assertEqual(self.dump(), before) + self.assertEqual(self.project.status(), CLEAN_STATUS) + + def test_a_definition_never_saved_is_dropped(self): + extra = json.dumps({'format': 'tuldok_definition_v1', 'kind': 'model', 'title': 'Extra'}) + self.project.set_definition('models/extra', extra) + user = self.edited('recipes/stepper-k1', lambda d: d.update(model='models/extra')) + self.project.set_definition('recipes/stepper-k1', user) + refused = self.refused('models/extra', 'in-use') + self.assertEqual(refused.references, ['recipes/stepper-k1']) + self.project.revert_definition('recipes/stepper-k1') + self.assertEqual(self.project.revert_definition('models/extra'), {'reference': 'models/extra', 'dropped': True}) + self.assertNotIn('models/extra', [row['reference'] for row in self.project.definitions()]) + self.assertEqual(self.project.status(), CLEAN_STATUS) + + def test_a_file_that_no_longer_checks_is_refused(self): + self.project.set_definition('recipes/stepper-k2', self.edited('recipes/stepper-k2', lambda d: d.update( + title='Edited'))) + self.write('recipes/stepper-k2', self.edited('recipes/stepper-k2', lambda d: d.update(model='models/missing'))) + refused = self.refused('recipes/stepper-k2', 'invalid') + self.assertIn(('recipes/stepper-k2.json', '/model'), [(p.path, p.pointer) for p in refused.problems]) + self.write('recipes/stepper-k2', '{"format": ') + self.assertEqual(self.refused('recipes/stepper-k2', 'invalid').problems[0].path, 'recipes/stepper-k2.json') + (self.folder / 'recipes' / 'stepper-k2.json').unlink() + (self.folder / 'recipes' / 'stepper-k2.json').symlink_to(self.folder / 'recipes' / 'stepper-k1.json') + self.refused('recipes/stepper-k2', 'invalid') + + def test_nothing_to_revert_and_not_found(self): + self.refused('recipes/stepper-k1', 'nothing-to-revert') + self.refused('recipes/missing', 'not-found') + self.refused('things/waves', 'not-found') + + def test_definition_problems_stores_nothing_and_has_warnings(self): + before = self.dump() + text = self.edited('recipes/early-fit', lambda d: d.update(run={ + 'command': ['sh', 'missing.sh'], 'sources': ['missing.sh']})) + problems = self.project.definition_problems('recipes/early-fit', text) + self.assertEqual([(p.severity, p.pointer) for p in problems], [('warning', '/run/sources/0')]) + problems = self.project.definition_problems('recipes/new', self.edited('recipes/stepper-k1', lambda d: d.update( + model='models/missing'))) + self.assertEqual([(p.severity, p.path, p.pointer) for p in problems], [('error', 'recipes/new.json', '/model')]) + self.assertEqual(self.project.definition_problems('recipes/new', '{')[0].path, 'recipes/new.json') + self.assertEqual(self.dump(), before) + + +class Drop(StoreCase): + + def test_drop_project(self): + self.close(self.open()) + self.assertIsNotNone(self.dump()) + store.drop_project(self.project_id) + self.assertIsNone(self.dump()) + store.drop_project(self.project_id) # dropping what is not there is fine + + def test_drop_refuses_a_bad_id(self): + with self.assertRaises(ValueError): + store.drop_project('project; drop') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_sweeps.py b/tests/test_tuldok_project_sweeps.py new file mode 100644 index 00000000..6a6a4484 --- /dev/null +++ b/tests/test_tuldok_project_sweeps.py @@ -0,0 +1,655 @@ +"""Sweeps (docs/plans/training-and-curation/spec-t7.md): the definition, the schedule, halving, trials and the runs. + +The schedule, halving's decisions and a trial's text are checked against the +test's own arithmetic without a database. The rest needs a database and runs +only when TULDOK_DATABASE_URL names a disposable cluster: a project written +by the test, committed to a Git repository of its own, whose recipe runs the +stand-in tests/project_sweep_stand_in.py (plain Python, logging through +tuldok_log, about a second a trial, its loss falling to a floor its settings +decide) with this interpreter. No GPU is used. +""" +import hashlib +import itertools +import json +import math +import os +from pathlib import Path +import random +import shutil +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import agent_api # noqa: E402 +from tuldok_project import (builds, check, environments, launcher, local, operations, plan, # noqa: E402 + service as service_module, store, sweeps, training) +from tuldok_project.service import Service # noqa: E402 +from tuldok_project.store import ProjectError # noqa: E402 + +REPOSITORY = Path(__file__).resolve().parents[1] +STAND_IN = Path(__file__).resolve().parent / 'project_sweep_stand_in.py' +DATABASE = os.environ.get('TULDOK_DATABASE_URL') +GIT = ['git', '-c', 'user.name=Tuldok Test', '-c', 'user.email=test@example.invalid', '-c', 'commit.gpgsign=false'] +ALL_SCOPES = ['admin', 'approve-execution', 'approve-review', 'cancel', 'edit', 'execute', 'expose-holdout', + 'freeze', 'inspect'] +OWNER = {'actor_id': 'owner-1', 'kind': 'owner', 'label': 'The owner', 'scopes': ALL_SCOPES} +AGENT = {'actor_id': 'agent-1', 'kind': 'agent', 'label': 'An agent', 'scopes': ['inspect', 'execute', 'cancel']} +ENDED = ('succeeded', 'failed', 'cancelled', 'declined', 'expired') + + +def r6(value): + return float(f'{value:.6g}') + + +def floor(learning_rate, width): + """The stand-in's floor (tests/project_sweep_stand_in.py).""" + return 0.1 * (math.log10(learning_rate) + 2) ** 2 + 1 / width + + +def recipe(settings, group=None): + document = {'format': 'tuldok_definition_v1', 'kind': 'recipe', 'title': 'The stand-in', + 'run': {'command': ['python', 'stand_in.py', '--settings', '{settings}', '--out', '{output}'], + 'environment': 'environments/here', 'settings': settings, + 'env': {'PYTHONPATH': str(REPOSITORY)}, + 'outputs': {'weights': {'path': 'weights.json', 'role': 'weights'}}, + 'sources': ['stand_in.py']}} + if group: + document['group'] = group + return document + + +def sweep(**fields): + document = {'format': 'tuldok_definition_v1', 'kind': 'sweep', 'title': 'Learning rate and width', + 'recipe': 'recipes/base', + 'parameters': [{'path': 'learning_rate', 'low': 0.001, 'high': 0.1, 'scale': 'log'}, + {'path': 'width', 'values': [4, 8, 16, 32]}], + 'strategy': {'kind': 'random', 'seed': 7}, + 'objectives': [{'metric': 'loss', 'better': 'lower'}], + 'budget': {'trials': 4}} + document.update(fields) + return document + + +# ======================================================================= the schedule + +class Schedule(unittest.TestCase): + + def test_a_grid_is_the_product_in_order_with_its_points_rounded(self): + document = sweep(parameters=[{'path': 'a', 'values': ['x', 'y']}, + {'path': 'b', 'low': 0.001, 'high': 0.1, 'scale': 'log', 'count': 3}, + {'path': 'c', 'low': 1, 'high': 2, 'integer': True, 'count': 5}], + strategy={'kind': 'grid'}) + b = [r6(math.exp(math.log(0.001) + i * (math.log(0.1) - math.log(0.001)) / 2)) for i in range(3)] + c = [] + for i in range(5): # 1, 1.25, 1.5, 1.75, 2 -> floor(x + 0.5): 1, 1, 2, 2, 2 + whole = math.floor(1 + i * 0.25 + 0.5) + if whole not in c: + c.append(whole) + self.assertEqual(c, [1, 2]) + expected = [{'a': a, 'b': bv, 'c': cv} for a, bv, cv in itertools.product(['x', 'y'], b, c)] + self.assertEqual(list(sweeps.schedule(document)), expected) + self.assertEqual(b, [0.001, 0.01, 0.1]) + + def test_random_draws_follow_the_seed_and_drop_repeats(self): + document = sweep() + rng = random.Random(7) + expected, seen = [], set() + while len(expected) < 30: + u = rng.random() + lr = r6(math.exp(math.log(0.001) + u * (math.log(0.1) - math.log(0.001)))) + width = [4, 8, 16, 32][rng.randrange(4)] + if (lr, width) in seen: + continue + seen.add((lr, width)) + expected.append({'learning_rate': lr, 'width': width}) + self.assertEqual(list(itertools.islice(sweeps.schedule(document), 30)), expected) + + def test_a_small_space_is_exhausted(self): + document = sweep(parameters=[{'path': 'width', 'values': [4, 8]}, {'path': 'deep', 'values': [True, False]}]) + drawn = list(sweeps.schedule(document)) + self.assertEqual(len(drawn), 4) + self.assertEqual({(item['width'], item['deep']) for item in drawn}, + {(4, True), (4, False), (8, True), (8, False)}) + + def test_one_and_one_point_zero_are_the_same_draw(self): + document = sweep(parameters=[{'path': 'x', 'values': [1, 1.0]}]) + self.assertEqual(len(list(sweeps.schedule(document))), 1) + + +# ======================================================================= halving + +class FakeConnection: + def __init__(self): + self.calls = [] + + def execute(self, query, parameters=None): + self.calls.append(parameters) + + +def write_log(directory, lines): + path = Path(directory) / 'output' / 'training' + path.mkdir(parents=True, exist_ok=True) + with open(path / 'metrics.jsonl', 'a', encoding='utf-8') as handle: + for line in lines: + handle.write(json.dumps(line) + '\n') + + +class Halving(unittest.TestCase): + STRATEGY = {'kind': 'halving', 'seed': 1, 'metric': 'loss', 'better': 'lower', 'min': 2, 'factor': 2} + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.base = Path(directory.name) + self.addCleanup(training.forget) + + def watch(self, name, recorded, lines, strategy=None): + """A watcher of a trial whose log holds lines, read once as the runner reads it.""" + document = sweep(strategy=strategy or self.STRATEGY) + directory = self.base / name + write_log(directory, lines) + watch = sweeps._Watch(document, FakeConnection(), 's', 'run', 1, recorded, None, time.monotonic) + watch.directory, watch.output, watch.run_id = str(directory), str(directory / 'output'), name + watch.look() + return watch + + def test_rungs_and_their_crossing(self): + self.assertEqual(sweeps.rung_points(self.STRATEGY)[:4], [2, 4, 8, 16]) + self.assertEqual(sweeps.rung_points({**self.STRATEGY, 'unit': 'time', 'min': 0.5, 'factor': 3})[:3], + [0.5, 1.5, 4.5]) + rungs = sweeps.Rungs([2, 4, 8]) + self.assertEqual(rungs.feed(1, 0.9), []) + self.assertEqual(rungs.feed(5, 0.7), [(0, 2, 0.7), (1, 4, 0.7)]) # one line crossing two rungs + self.assertEqual(rungs.feed(5, 0.6), []) + self.assertEqual(rungs.feed(9, None), [(2, 8, None)]) + + def test_goes_on(self): + self.assertEqual(sweeps.goes_on(5.0, [], 'lower', 3), (True, 1, 1)) + self.assertEqual(sweeps.goes_on(5.0, [1.0], 'lower', 3), (True, 2, 1)) # n < factor + self.assertEqual(sweeps.goes_on(5.0, [1.0, 2.0], 'lower', 3), (False, 3, 1)) + self.assertEqual(sweeps.goes_on(0.5, [1.0, 2.0], 'lower', 3), (True, 3, 1)) + self.assertEqual(sweeps.goes_on(1.0, [1.0, 2.0], 'lower', 3), (True, 3, 1)) # a tie is not better + self.assertEqual(sweeps.goes_on(1.5, [1.0, 2.0, 3.0], 'lower', 3), (True, 4, 2)) + self.assertEqual(sweeps.goes_on(1.5, [1.0, 2.0, 3.0], 'higher', 3), (False, 4, 2)) + + def test_decisions_on_logs(self): + recorded = {} + # The value at a rung is the latest finite value on the line reaching it or before: here the line of + # step 3 (no loss) reaches rung 2 with 0.8, the loss of step 1. + first = self.watch('a', recorded, [{'step': 1, 'time': 0.1, 'loss': 0.8}, {'step': 3, 'time': 0.2}, + {'step': 4, 'time': 0.3, 'loss': 0.6}]) + self.assertEqual([(r['rung'], r['value'], r['goes_on']) for r in first.reached], [(0, 0.8, True), (1, 0.6, True)]) + self.assertIsNone(first.reason) + # A second trial worse at rung 2: n = 2 = factor, kept 1, one better: pruned, its cancel file written. + second = self.watch('b', recorded, [{'step': 2, 'time': 0.1, 'loss': 0.9}, {'step': 4, 'time': 0.2, 'loss': 0.1}]) + self.assertEqual([(r['rung'], r['value'], r['n'], r['kept'], r['goes_on']) for r in second.reached], + [(0, 0.9, 2, 1, False)]) + self.assertEqual(second.reason, 'pruned') + self.assertTrue((self.base / 'b' / launcher.CANCEL_FILE).exists()) + self.assertIn('rung 1 (step 2)', second.problem) + # A third, better at both rungs: n = 3, kept 2. + third = self.watch('c', recorded, [{'step': 2, 'time': 0.1, 'loss': 0.5}, {'step': 4, 'time': 0.2, 'loss': 0.4}]) + self.assertEqual([(r['n'], r['kept'], r['goes_on']) for r in third.reached], [(3, 2, True), (2, 1, True)]) + self.assertEqual(recorded, {0: [0.8, 0.9, 0.5], 1: [0.6, 0.4]}) + # A trial with no value of the metric records nothing and goes on. + fourth = self.watch('d', recorded, [{'step': 2, 'time': 0.1, 'other': 1.0}]) + self.assertEqual(fourth.reached, []) + self.assertIsNone(fourth.reason) + + def test_appended_lines_are_read_on(self): + recorded = {0: [0.1]} + watch = self.watch('a', recorded, [{'step': 1, 'time': 0.1, 'loss': 0.9}]) + self.assertEqual(watch.reached, []) + write_log(self.base / 'a', [{'step': 2, 'time': 0.2, 'loss': 0.95}]) + watch.look() + self.assertEqual(watch.reason, 'pruned') + self.assertEqual(watch.latest, {'loss': 0.95}) + + def test_minutes_stop_a_trial(self): + document = sweep() + directory = self.base / 'late' + write_log(directory, [{'step': 1, 'time': 0.1, 'loss': 0.9}]) + watch = sweeps._Watch(document, FakeConnection(), 's', 'run', 1, {}, 10.0, lambda: 11.0) + watch.directory, watch.output, watch.run_id = str(directory), str(directory / 'output'), 'late' + watch.look() + self.assertEqual(watch.reason, 'minutes') + self.assertTrue((directory / launcher.CANCEL_FILE).exists()) + + +# ======================================================================= a trial's definition + +class Trial(unittest.TestCase): + + def test_the_text_of_a_trial(self): + base = recipe({'training': {'learning_rate': 0.01, 'schedule': [{'lr': 1}, {'lr': 2}]}, 'width': 8}, + group='small') + base['card'] = 'notes/x.md' + base['labels'] = {'short': 'B'} + base['compared_with'] = [{'recipe': 'recipes/other', 'explanation': 'x'}] + base['model'] = 'models/m' + document = sweep(title='A search') + values = {'training.learning_rate': 0.003, 'training.schedule.1.lr': 5, 'width': 16} + text = sweeps.trial_text('sweeps/s', document, 'recipes/base', base, 3, values) + made = json.loads(text) + self.assertTrue(text.endswith('}\n')) + self.assertEqual(made['title'], 'A search · trial 3') + self.assertEqual((made['derived_from'], made['change'], made['group'], made['model'], made['labels']), + ('recipes/base', 'sweep-trial', 'small', 'models/m', {'short': 'T3'})) + self.assertEqual(made['sweep'], {'definition': 'sweeps/s', 'trial': 3, 'values': values}) + self.assertEqual(made['run']['settings'], {'training': {'learning_rate': 0.003, + 'schedule': [{'lr': 1}, {'lr': 5}]}, 'width': 16}) + self.assertNotIn('card', made) + self.assertNotIn('compared_with', made) + self.assertEqual(base['run']['settings']['width'], 8) # the recipe is not changed + self.assertEqual(made['description'], + 'Trial 3 of sweeps/s: training.learning_rate = 0.003, training.schedule.1.lr = 5, width = 16.') + + def test_names_never_overwrite(self): + self.assertEqual(sweeps.place_trial({}, 'sweeps/s', 7, 'A'), ('recipes/s-007', False)) + self.assertEqual(sweeps.place_trial({'recipes/s-007': 'A'}, 'sweeps/s', 7, 'A'), ('recipes/s-007', True)) + self.assertEqual(sweeps.place_trial({'recipes/s-007': 'B'}, 'sweeps/s', 7, 'A'), ('recipes/s-007-2', False)) + self.assertEqual(sweeps.place_trial({'recipes/s-007': 'B', 'recipes/s-007-2': 'C'}, 'sweeps/s', 7, 'A'), + ('recipes/s-007-3', False)) + self.assertEqual(sweeps.place_trial({}, 'sweeps/' + 'x' * 120, 1000, 'A')[0], 'recipes/' + 'x' * 100 + '-1000') + + def test_paths(self): + settings = {'a': {'b': [1, {'c': 2}]}, 'n': 3} + self.assertIsNone(sweeps.path_problem(settings, 'a.b.1.c')) + self.assertIsNone(sweeps.path_problem(settings, 'a.new')) + self.assertIn('not in the recipe', sweeps.path_problem(settings, 'x.y')) + self.assertIn('not an object or a list', sweeps.path_problem(settings, 'n.y')) + self.assertIn('past its end', sweeps.path_problem(settings, 'a.b.2')) + + +# ======================================================================= the checker + +class Checking(unittest.TestCase): + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.folder = Path(directory.name) / 'project' + for part in ('recipes', 'sweeps', 'environments'): + (self.folder / part).mkdir(parents=True) + (self.folder / 'tuldok.json').write_text(json.dumps({'format': 'tuldok_project_v1', 'id': 'a' * 32, + 'name': 'Sweeps', 'code_root': '.'})) + self.write('environments/here', {'format': 'tuldok_definition_v1', 'kind': 'environment', 'title': 'Here', + 'python': '3.12', 'device': 'cpu', 'packages': {'jsonschema': '4.26.0'}}) + self.write('recipes/base', recipe({'learning_rate': 0.01, 'width': 8, 'nested': {'depth': 2}})) + + def write(self, reference, document): + (self.folder / f'{reference}.json').write_text(json.dumps(document, indent=2) + '\n') + + def problems(self, document, name='s'): + self.write(f'sweeps/{name}', document) + report = check.check_folder(self.folder, code=False) + return [(p.severity, p.path, p.pointer, p.message) for p in report.problems] + + def test_a_good_sweep_checks(self): + self.assertEqual(self.problems(sweep()), []) + + def test_the_schema(self): + found = self.problems(sweep(parameters=[{'path': 'learning_rate', 'values': [1], 'low': 0, 'high': 1}], + strategy={'kind': 'halving', 'seed': 1}, objectives=[], budget={'trials': 0, + 'concurrency': 2})) + pointers = {pointer for _, _, pointer, _ in found} + self.assertTrue({'/parameters/0', '/strategy/metric', '/objectives', '/budget/trials', '/budget/concurrency'} + <= pointers, found) + + def test_rules_beyond_the_schema(self): + found = self.problems(sweep(parameters=[{'path': 'learning_rate', 'low': 1, 'high': 0.5}, + {'path': 'learning_rate', 'low': -1, 'high': 1, 'scale': 'log'}, + {'path': 'width.x', 'values': [1, 1]}, + {'path': 'missing.y', 'values': [1]}, + {'path': 'steps', 'low': 1.5, 'high': 3, 'integer': True}], + objectives=[{'metric': 'loss', 'better': 'lower'}, + {'metric': 'loss', 'better': 'higher'}])) + messages = [(pointer, message) for _, _, pointer, message in found] + self.assertIn(('/parameters/0', 'low (1) must be less than high (0.5).'), messages) + self.assertIn(('/parameters/1/path', 'learning_rate is varied twice; keep one parameter for it.'), messages) + self.assertIn(('/parameters/1/low', 'A log scale needs low above 0.'), messages) + self.assertIn(('/parameters/2/values', 'The values repeat; list each once.'), messages) + self.assertIn(('/parameters/2/path', "width is not an object or a list in the recipe's settings."), messages) + self.assertIn(('/parameters/3/path', "missing is not in the recipe's settings."), messages) + self.assertIn(('/parameters/4', 'An integer range needs whole numbers for low and high.'), messages) + self.assertIn(('/objectives', 'The two objectives name the same metric; name two.'), messages) + + def test_a_grid(self): + found = self.problems(sweep(strategy={'kind': 'grid'})) + self.assertIn(('error', 'sweeps/s.json', '/parameters/0', + 'A grid takes count points of a range; add count (2 to 64).'), found) + found = self.problems(sweep(strategy={'kind': 'grid'}, + parameters=[{'path': 'learning_rate', 'low': 0.001, 'high': 0.1, 'count': 3}, + {'path': 'width', 'values': [4, 8]}])) + self.assertEqual(found, [('warning', 'sweeps/s.json', '/budget/trials', + 'The grid has 6 points and the budget 4; the first 4 in order are run.')]) + found = self.problems(sweep(strategy={'kind': 'grid'}, + parameters=[{'path': f'p{i}', 'low': 0, 'high': 1, 'count': 64} for i in range(3)])) + self.assertIn('at most 100,000', found[0][3]) + + def test_the_recipe(self): + found = self.problems(sweep(recipe='recipes/none')) + self.assertTrue(any(pointer == '/recipe' for _, _, pointer, _ in found), found) + filed = recipe({}) + del filed['run']['settings'] + filed['run']['settings_file'] = 'settings.json' + self.write('recipes/filed', filed) + found = self.problems(sweep(recipe='recipes/filed')) + self.assertEqual(len(found), 1) + self.assertIn('its settings are in a file', found[0][3]) + + def test_a_trial_names_a_sweep(self): + self.write('sweeps/s', sweep()) + trial = json.loads(sweeps.trial_text('sweeps/s', sweep(), 'recipes/base', recipe({'learning_rate': 0.01}), 1, + {'learning_rate': 0.02})) + self.write('recipes/s-001', trial) + report = check.check_folder(self.folder, code=False) + self.assertEqual(report.problems, []) + roles = {(edge.from_ref, edge.to_ref, edge.role) for edge in report.edges} + self.assertTrue({('recipes/s-001', 'sweeps/s', 'sweep'), ('sweeps/s', 'recipes/base', 'varies'), + ('recipes/s-001', 'recipes/base', 'derived')} <= roles) + trial['sweep']['definition'] = 'sweeps/gone' + self.write('recipes/s-001', trial) + report = check.check_folder(self.folder, code=False) + self.assertEqual([(p.path, p.pointer) for p in report.problems], [('recipes/s-001.json', '/sweep/definition')]) + + +# ======================================================================= with a database + +def _writable(root): + for current, directories, _files in os.walk(root): + for name in directories: + os.chmod(os.path.join(current, name), 0o755) + if os.path.isdir(root): + os.chmod(root, 0o755) + + +@unittest.skipUnless(DATABASE, 'TULDOK_DATABASE_URL is not set; database tests need a disposable cluster.') +class SweepCase(unittest.TestCase): + """A project written by the test: the stand-in as recipes/base, committed, its environment connected.""" + + SETTINGS = {'learning_rate': 0.01, 'width': 8, 'steps': 40, 'sleep': 0.03} + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.temporary = Path(directory.name) + self.home = self.temporary / 'home' + self.addCleanup(_writable, self.home) + patcher = mock.patch.dict(os.environ, {local.HOME_VARIABLE: str(self.home)}) + patcher.start() + self.addCleanup(patcher.stop) + patcher = mock.patch.object(service_module, 'POLL_SECONDS', 0.1) + patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(training.forget) + self.root = self.temporary / 'repo' + self.folder = self.root / 'project' + code = self.root / 'code' + code.mkdir(parents=True) + shutil.copy(STAND_IN, code / 'stand_in.py') + for part in ('recipes', 'sweeps', 'environments'): + (self.folder / part).mkdir(parents=True) + self.project_id = __import__('uuid').uuid4().hex + (self.folder / 'tuldok.json').write_text(json.dumps({ + 'format': 'tuldok_project_v1', 'id': self.project_id, 'name': 'Sweeps', 'code_root': '../code', + 'groups': [{'id': 'small', 'title': 'Small models'}]}, indent=2) + '\n') + self.write('environments/here', {'format': 'tuldok_definition_v1', 'kind': 'environment', 'title': 'Here', + 'python': f'{sys.version_info[0]}.{sys.version_info[1]}', 'device': 'cpu', + 'packages': {'jsonschema': '4.26.0'}}) + self.write('recipes/base', recipe(dict(self.SETTINGS), group='small')) + self.git('init', '-q') + self.commit('The project.') + self.addCleanup(store.drop_project, self.project_id) + with builds.opened(self.folder) as project: + self.assertTrue(environments.connect(project, 'environments/here', sys.executable)['ok']) + self.addCleanup(self.stop_everything) + + def stop_everything(self): + try: + with builds.opened(self.folder) as project: + run_ids = [row[0] for row in project._execute('SELECT id FROM {schema}.runs').fetchall()] + except Exception: + run_ids = [] + for run_id in run_ids: + launcher.stop_strays(run_id) + + def write(self, reference, document): + (self.folder / f'{reference}.json').write_text(json.dumps(document, indent=2) + '\n') + + def git(self, *arguments): + subprocess.run([*GIT, '-C', str(self.root), *arguments], check=True, capture_output=True, timeout=60) + + def commit(self, message): + self.git('add', '-A') + self.git('commit', '-q', '-m', message) + + def keep(self, reference, document): + """Keep a definition through the store and save it, as the page does, then commit the folder.""" + with builds.opened(self.folder) as project: + project.set_definition(reference, json.dumps(document, indent=2) + '\n') + project.save() + self.commit(f'{reference}.') + + def service(self): + made = Service(self.folder) + self.addCleanup(made.close) + made.start() + return made + + def wait_for(self, served, request_id, seconds=120): + deadline = time.monotonic() + seconds + while True: + shown = served.call('build_request', {'request': request_id}, OWNER) + if shown['state'] in ENDED: + return shown + self.assertLess(time.monotonic(), deadline, f'request still {shown["state"]}') + time.sleep(0.1) + + +class Planning(SweepCase): + + def test_a_sweep_is_planned_alone_and_covers_its_recipe(self): + self.keep('sweeps/s', sweep()) + with builds.opened(self.folder) as project: + made = plan.build_plan(project, ['sweeps/s']) + self.assertEqual(made['targets'], ['sweeps/s']) + self.assertEqual([(step['reference'], step['kind'], step['action']) for step in made['steps']], + [('environments/here', 'environment', 'ready'), ('sweeps/s', 'sweep', 'build')]) + step = made['steps'][-1] + self.assertEqual((step['recipe'], step['trials'], step['state']), ('recipes/base', 4, 'not-built')) + self.assertTrue(made['buildable']) + self.assertEqual(made['counts']['build'], 1) + from tuldok_project import build_requests + covered = build_requests.covers(project, made, ['sweeps/s'], every=False, scratch=False) + self.assertEqual(set(covered['definitions']), {'environments/here', 'sweeps/s', 'recipes/base'}) + self.assertEqual(covered['steps'], [{'reference': 'sweeps/s', 'subjects': []}]) + for targets, every in ((['sweeps/s', 'recipes/base'], False), (['sweeps/s'], True)): + with self.assertRaises(ProjectError) as refused: + plan.build_plan(project, targets, every=every) + self.assertEqual(refused.exception.code, 'invalid') + + +class Running(SweepCase): + + def test_halving_through_a_service_approved_once(self): + strategy = {'kind': 'halving', 'seed': 3, 'metric': 'loss', 'better': 'lower', 'min': 2, 'factor': 2} + self.keep('sweeps/s', sweep(strategy=strategy, budget={'trials': 6}, + objectives=[{'metric': 'loss', 'better': 'lower'}, + {'metric': 'time', 'better': 'lower', 'title': 'Seconds'}])) + served = self.service() + asked = served.call('request_build', {'targets': ['sweeps/s']}, AGENT) + self.assertEqual(asked['state'], 'awaiting-approval') + self.assertEqual(served.call('sweep', {'sweep': 's'}, OWNER)['request']['state'], 'awaiting-approval') + served.call('approve_build', {'request': asked['id']}, OWNER) + done = self.wait_for(served, asked['id']) + self.assertEqual(done['state'], 'succeeded', done['problem']) + shown = served.call('sweep', {'sweep': 'sweeps/s'}, OWNER) + json.dumps(shown, allow_nan=False) + trials = shown['trials'] + # The test's arithmetic: the schedule, each trial's floor, and halving's rule over the rungs 2, 4, 8, 16, 32. + values = list(itertools.islice(sweeps.schedule(sweep(strategy=strategy)), 6)) + self.assertEqual([trial['values'] for trial in trials], values) + recorded, expected = {}, [] + for item in values: + bottom = floor(item['learning_rate'], item['width']) + state = 'completed' + for rung, point in enumerate([2, 4, 8, 16, 32]): + value = bottom + math.exp(-point / 8) + held = recorded.setdefault(rung, []) + n = len(held) + 1 + better = sum(1 for other in held if other < value) + held.append(value) + if n >= 2 and better >= math.ceil(n / 2): + state = 'pruned' + break + expected.append(state) + self.assertIn('pruned', expected) + self.assertEqual([trial['state'] for trial in trials], expected) + self.assertEqual(shown['run']['counts']['pruned'], expected.count('pruned')) + self.assertEqual((shown['run']['state'], shown['run']['ended'], shown['state']), ('finished', 'trials', 'built')) + for trial, item in zip(trials, values): + if trial['state'] == 'completed': + self.assertAlmostEqual(trial['objectives']['loss'], floor(item['learning_rate'], item['width']) + + math.exp(-40 / 8)) + else: + self.assertTrue(trial['problem'].startswith('Pruned by successive halving')) + self.assertLess(trial['latest']['step'], 40) + completed = [trial for trial in trials if trial['state'] == 'completed'] + best = min(completed, key=lambda trial: trial['objectives']['loss']) + self.assertEqual(shown['best'], {'number': best['number'], 'reference': best['reference'], + 'value': best['objectives']['loss']}) + self.assertTrue(set(shown['frontier']) <= {trial['number'] for trial in completed}) + self.assertIn(best['number'], shown['frontier']) + # The trials as axes: the objectives and the two parameters, one subject a trial. + axes = shown['axes'] + self.assertEqual([metric['id'] for metric in axes['metrics']], ['loss', 'time']) + self.assertEqual([(axis['path'], axis['kind']) for axis in axes['settings']], + [('learning_rate', 'number'), ('width', 'number')]) + self.assertEqual(len(axes['subjects']), 6) + # One approval: one request, one plan id over every run, every run of a trial. + runs = served.call('runs', {}, OWNER)['runs'] + self.assertEqual({run['plan_id'] for run in runs}, {done['plan_id']}) + self.assertEqual(sorted(run['reference'] for run in runs), sorted(trial['reference'] for trial in trials)) + self.assertEqual(len(done['runs']), 6) + pruned = [trial for trial in trials if trial['state'] == 'pruned'] + for trial in pruned: + run = served.call('run', {'run': trial['run_id']}, OWNER) + self.assertEqual(run['state'], 'cancelled') + self.assertEqual(run['problem'], trial['problem']) + # Trials are recipes, unsaved, derived from the recipe, grouped under the sweep on the map. + status = served.call('status', {}, OWNER) + self.assertEqual(status['unsaved'], sorted(trial['reference'] for trial in trials)) + trial = served.call('definition', {'reference': trials[0]['reference']}, OWNER) + self.assertEqual((trial['document']['derived_from'], trial['document']['change'], + trial['document']['sweep']['values']), ('recipes/base', 'sweep-trial', values[0])) + made = served.call('map', {}, OWNER) + group = [g for g in made['groups'] if g['id'] == 'sweeps/s'] + self.assertEqual(group, [{'id': 'sweeps/s', 'title': 'Learning rate and width', 'parent': 'small', + 'description': 'The trials of sweeps/s, a search over the settings of recipes/base.', + 'models': 6, 'sweep': True}]) + models = {model['reference']: model for model in made['models']} + self.assertEqual({models[trial['reference']]['group'] for trial in trials}, {'sweeps/s'}) + self.assertNotIn('sweep', models['recipes/base']) # only a trial and a sweep's group carry it + self.assertEqual((models[trials[0]['reference']]['sweep'], models[trials[0]['reference']]['derived_from']), + ('sweeps/s', 'recipes/base')) + listed = served.call('sweeps', {}, OWNER)['sweeps'] + self.assertEqual([(item['reference'], item['state'], item['run']['counts']['completed']) for item in listed], + [('sweeps/s', 'built', expected.count('completed'))]) + # Saved, the trials are files of the folder. + served.call('save', {}, OWNER) + self.assertTrue((self.folder / (trials[0]['reference'] + '.json')).exists()) + # Built already: asking again is refused. + self.assertEqual(self.refused(served, 'request_build', {'targets': ['sweeps/s']}), 'nothing-to-build') + + def refused(self, served, name, arguments): + with self.assertRaises(ProjectError) as refused: + served.call(name, arguments, OWNER) + return refused.exception.code + + def test_minutes_stop_a_sweep(self): + self.write('recipes/base', recipe({**self.SETTINGS, 'steps': 30, 'sleep': 0.1}, group='small')) + self.commit('Slower.') + self.keep('sweeps/m', sweep(budget={'trials': 10, 'minutes': 0.06})) + served = self.service() + asked = served.call('request_build', {'targets': ['sweeps/m']}, OWNER) + self.assertEqual(asked['state'], 'queued') + started = time.monotonic() + done = self.wait_for(served, asked['id']) + self.assertEqual(done['state'], 'succeeded', done['problem']) + shown = served.call('sweep', {'sweep': 'm'}, OWNER) + self.assertEqual((shown['run']['state'], shown['run']['ended']), ('finished', 'minutes')) + states = [trial['state'] for trial in shown['trials']] + self.assertEqual(states[-1], 'stopped', states) + self.assertLess(len(states), 10) + self.assertLess(time.monotonic() - started, 30) + self.assertEqual(shown['trials'][-1]['problem'], "The sweep's minutes were spent while it ran.") + + def test_a_grid_reuses_its_built_trials(self): + document = sweep(strategy={'kind': 'grid'}, parameters=[{'path': 'width', 'values': [4, 8, 16]}], + budget={'trials': 2}) + self.write('recipes/base', recipe({**self.SETTINGS, 'steps': 4, 'sleep': 0.01}, group='small')) + self.commit('Quicker.') + self.keep('sweeps/g', document) + with builds.opened(self.folder) as project: + made = plan.build_plan(project, ['sweeps/g']) + first = sweeps.execute(self.folder, made) + self.assertTrue(first['ok']) + self.assertEqual([item['reference'] for item in first['steps']], ['recipes/g-001', 'recipes/g-002']) + with builds.opened(self.folder) as project: + self.assertEqual(plan.build_plan(project, ['sweeps/g'])['steps'][-1]['action'], 'built') + # Another trial taking the name of the third with other settings is never overwritten. + other = json.loads(project.definition('recipes/g-001')['text']) + other['title'] = 'Someone else' + project.set_definition('recipes/g-003', json.dumps(other, indent=2) + '\n') + document['budget'] = {'trials': 3} + self.keep('sweeps/g', document) + with builds.opened(self.folder) as project: + made = plan.build_plan(project, ['sweeps/g']) + self.assertEqual(made['steps'][-1]['state'], 'not-built') + second = sweeps.execute(self.folder, made) + self.assertEqual([item['reference'] for item in second['steps']], ['recipes/g-003-2']) + with builds.opened(self.folder) as project: + shown = sweeps.show(project, 'g') + self.assertEqual([(t['reference'], t['state'], t['reused']) for t in shown['trials']], + [('recipes/g-001', 'completed', True), ('recipes/g-002', 'completed', True), + ('recipes/g-003-2', 'completed', False)]) + self.assertEqual(len(shown['runs']), 2) + self.assertEqual(json.loads(project.definition('recipes/g-003')['text'])['title'], 'Someone else') + self.assertEqual(json.loads(project.definition('recipes/g-003-2')['text'])['sweep']['values'], + {'width': 16}) + + def test_a_stale_definition_is_refused(self): + self.keep('sweeps/s', sweep()) + with builds.opened(self.folder) as project: + made = plan.build_plan(project, ['sweeps/s']) + from tuldok_project import build_requests + covered = build_requests.covers(project, made, ['sweeps/s'], every=False, scratch=False) + covered['definitions']['recipes/base'] = '0' * 64 + with self.assertRaises(ProjectError) as refused: + sweeps.execute(self.folder, made, expect=covered) + self.assertEqual(refused.exception.code, 'stale') + + +class Published(unittest.TestCase): + + def test_the_two_reads_are_in_the_project_folder_profile_and_not_in_the_catalog(self): + for name in ('sweeps', 'sweep'): + operation = operations.by_name(name) + self.assertEqual((operation.who, operation.mutation), ('inspect', False)) + folder = {item['id'] for item in agent_api.capabilities('b' * 32, profile='project_folder')['operations']} + catalog = agent_api.capabilities('b' * 32) + self.assertTrue({'project.sweeps', 'project.sweep'} <= folder) + self.assertFalse({'project.sweeps', 'project.sweep'} & {item['id'] for item in catalog['operations']}) + self.assertEqual((len(catalog['operations']), hashlib.sha256(agent_api.encoded(catalog)).hexdigest()), + (110, 'ad5cc7ac91646b39b7a20b9dfca5ec79ed96ded4b5e36732fcf1145f1cc24460')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tuldok_project_training.py b/tests/test_tuldok_project_training.py new file mode 100644 index 00000000..2bc56450 --- /dev/null +++ b/tests/test_tuldok_project_training.py @@ -0,0 +1,392 @@ +"""What a training script logged, read while it is written (docs/plans/training-and-curation/spec-t1.md, slice T1a). + +The readers are tested on files the tests write with the standard library and +need no database. The operations need one and run only when +TULDOK_DATABASE_URL names a disposable cluster: they build recipes/fit-a (as +test_tuldok_project_builds does), write training files into its run's output, +and read them with the run marked running and queued. No GPU is used. +""" +import hashlib +import json +import math +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import agent_api # noqa: E402 +from tuldok_project import builds, local, operations, store, training # noqa: E402 +from tuldok_project.service import Service # noqa: E402 +import test_tuldok_project_builds as builds_base # noqa: E402 +from test_tuldok_project_inside import OWNER, floats, npy, png # noqa: E402 + +TRAINING = ('training', 'training_series', 'training_histograms', 'training_samples', 'training_sample') + + +def line(value): + return (json.dumps(value) + '\n').encode() + + +class TrainingCase(unittest.TestCase): + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = os.path.realpath(directory.name) + os.makedirs(os.path.join(self.root, 'training', 'samples')) + training.forget() + self.addCleanup(training.forget) + self.run = training.Run('run-1', 'running', self.root) + + def path(self, name): + return os.path.join(self.root, 'training', name) + + def append(self, name, data): + with open(self.path(name), 'ab') as handle: + handle.write(data) + + def overview(self): + got = training.overview(self.run) + json.dumps(got, allow_nan=False) + return got + + def series(self, *wanted, most=1000): + got = training.series(self.run, [w if isinstance(w, dict) else {'name': w} for w in wanted], most=most) + json.dumps(got, allow_nan=False) + return got + + def refused(self, code, call, *arguments, **options): + with self.assertRaises(store.ProjectError) as caught: + call(*arguments, **options) + self.assertEqual(caught.exception.code, code, str(caught.exception)) + + +class Metrics(TrainingCase): + + def test_nothing_logged(self): + got = training.overview(training.Run('run-2', 'queued', None)) + self.assertEqual((got['present'], got['series'], got['lines'], got['run']['active']), (False, [], 0, True)) + os.rmdir(os.path.join(self.root, 'training', 'samples')) + os.rmdir(os.path.join(self.root, 'training')) + self.assertFalse(self.overview()['present']) + self.refused('not-found', training.series, training.Run('run-2', 'queued', None), [{'name': 'loss'}]) + + def test_series_gaps_other_keys_and_bad_lines(self): + self.append('metrics.jsonl', b''.join([ + line({'step': 0, 'time': 0.5, 'epoch': 0, 'loss': 2.0, 'lr': 0.1, 'note': 'warm-up'}), + line({'step': 1, 'time': 1.0, 'loss': None, 'lr': 0.1}), + b'{"step": 2, "time": 1.5, "loss": NaN, "lr": 0.1, "val/loss": 3.0}\n', + b'not json\n', + line({'step': -1, 'time': 2.0, 'loss': 1.0}), + line({'step': 3, 'loss': 1.0}), + line([1, 2]), + line({'step': 4, 'time': 2.5, 'loss': 1.0, 'lr': 0.05, 'flag': True, 'big': 10 ** 400}), + b'\n', + b'{"step": 5, "time": 3.0, "loss": 0.5' + b' ' * (training.LINE_BYTES + 10) + b'}\n', + ])) + got = self.overview() + by = {s['name']: s for s in got['series']} + self.assertEqual(list(by), ['loss', 'lr', 'val/loss', 'big']) + self.assertEqual((by['loss']['points'], by['loss']['nonfinite'], by['loss']['min'], by['loss']['max'], + by['loss']['last'], by['loss']['last_step']), (4, 2, 1.0, 2.0, 1.0, 4)) + self.assertEqual((by['lr']['points'], by['lr']['last']), (4, 0.05)) + self.assertEqual((by['val/loss']['points'], by['val/loss']['first_step']), (1, 2)) + self.assertEqual((by['big']['nonfinite'], by['big']['last']), (1, None)) + self.assertEqual(got['other_keys'], ['note', 'flag']) + self.assertEqual((got['lines'], got['last_step'], got['last_time']), (4, 4, 2.5)) + self.assertEqual(got['bad_lines'], 5) + self.assertEqual([p['line'] for p in got['problems']], [4, 5, 6, 7, 10]) + self.assertTrue(all(p['file'] == 'training/metrics.jsonl' for p in got['problems'])) + [loss] = self.series('loss')['series'] + self.assertEqual(loss['buckets'], [[0, 0, 0.5, 2.0, 2.0, 2.0], [1, 1, 1.0, None, None, None], + [2, 2, 1.5, None, None, None], [4, 4, 2.5, 1.0, 1.0, 1.0]]) + self.assertEqual(self.series('missing')['series'][0]['points'], 0) + + def test_a_partial_last_line_waits_for_its_end(self): + self.append('metrics.jsonl', line({'step': 0, 'time': 0.0, 'loss': 1.0}) + b'{"step": 1, "time": 0.1, "lo') + got = self.overview() + self.assertEqual((got['lines'], got['bad_lines']), (1, 0)) + self.append('metrics.jsonl', b'ss": 0.5}\n' + line({'step': 2, 'time': 0.2, 'loss': 0.25})) + got = self.overview() + self.assertEqual((got['lines'], got['bad_lines'], got['series'][0]['last']), (3, 0, 0.25)) + self.assertEqual([b[3] for b in self.series('loss')['series'][0]['buckets']], [1.0, 0.5, 0.25]) + + def test_a_line_too_long_while_it_is_written(self): + self.append('metrics.jsonl', line({'step': 0, 'time': 0.0, 'loss': 1.0}) + b'{"x": "' + + b'a' * (training.LINE_BYTES + 1)) + self.assertEqual(self.overview()['bad_lines'], 1) + self.append('metrics.jsonl', b'a' * 100 + b'"}\n' + line({'step': 1, 'time': 0.1, 'loss': 0.5})) + got = self.overview() + self.assertEqual((got['lines'], got['bad_lines'], got['series'][0]['points']), (2, 1, 2)) + + def test_a_file_replaced_or_shortened_is_read_again(self): + self.append('metrics.jsonl', b''.join(line({'step': i, 'time': i, 'loss': i}) for i in range(5))) + self.assertEqual(self.overview()['lines'], 5) + with open(self.path('metrics.jsonl'), 'wb') as handle: # the same inode, shorter + handle.write(line({'step': 0, 'time': 0, 'loss': 9.0})) + got = self.overview() + self.assertEqual((got['lines'], got['series'][0]['last']), (1, 9.0)) + replacement = self.path('other') + with open(replacement, 'wb') as handle: # another file, longer + handle.write(b''.join(line({'step': i, 'time': i, 'acc': i}) for i in range(7))) + os.replace(replacement, self.path('metrics.jsonl')) + got = self.overview() + self.assertEqual(([s['name'] for s in got['series']], got['lines']), (['acc'], 7)) + + def test_buckets_of_a_long_series_and_since(self): + values = [math.sin(i / 7) * 10 + i for i in range(1000)] + self.append('metrics.jsonl', b''.join(line({'step': 10 * i, 'time': i / 4, 'loss': v}) + for i, v in enumerate(values))) + [raw] = self.series('loss', most=1000)['series'] + self.assertEqual((raw['size'], len(raw['buckets'])), (1, 1000)) + self.assertEqual([b[3] for b in raw['buckets']], values) + [got] = self.series('loss', most=100)['series'] + self.assertEqual((got['size'], got['first'], len(got['buckets'])), (16, 0, 63)) + for number, bucket in enumerate(got['buckets']): + part = values[number * 16:(number + 1) * 16] + self.assertEqual(bucket[:3], [10 * number * 16, 10 * (number * 16 + len(part) - 1), + (number * 16 + len(part) - 1) / 4]) + self.assertAlmostEqual(bucket[3], math.fsum(part) / len(part), places=12) + self.assertEqual(bucket[4:], [min(part), max(part)]) + # The page holds 63 buckets of 16; more points arrive; it asks from its last one. + self.append('metrics.jsonl', b''.join(line({'step': 10 * i, 'time': i / 4, 'loss': float(i)}) + for i in range(1000, 1020))) + [more] = self.series({'name': 'loss', 'size': 16, 'since': 62}, most=100)['series'] + self.assertEqual((more['size'], more['first'], len(more['buckets']), more['points']), (16, 62, 2, 1020)) + self.assertEqual(more['buckets'][-1][1], 10190) + # Past 1,600 points the size doubles: the page is sent everything again, from 0. + self.append('metrics.jsonl', b''.join(line({'step': 10 * i, 'time': i / 4, 'loss': 1.0}) + for i in range(1020, 1700))) + [again] = self.series({'name': 'loss', 'size': 16, 'since': 63}, most=100)['series'] + self.assertEqual((again['size'], again['first'], len(again['buckets'])), (32, 0, 54)) + self.assertEqual(training.bucket_size(0, 10), 1) + self.assertEqual(training.bucket_size(11, 10), 2) + + def test_the_bound_on_lines(self): + with mock.patch.object(training._Metrics, 'line_most', 3): + self.append('metrics.jsonl', b''.join(line({'step': i, 'time': i, 'loss': i}) for i in range(6))) + got = self.overview() + self.assertEqual((got['lines'], got['truncated']), (3, True)) + + def test_the_bound_on_series(self): + with mock.patch.object(training, 'SERIES', 2): + self.append('metrics.jsonl', line({'step': 0, 'time': 0, 'a': 1, 'b': 2, 'c': 3})) + got = self.overview() + self.assertEqual([s['name'] for s in got['series']], ['a', 'b']) + self.assertEqual((got['bad_lines'], len(got['problems'])), (0, 1)) + + def test_a_file_outside_the_output_is_not_read(self): + outside = tempfile.NamedTemporaryFile('wb', delete=False) + self.addCleanup(os.unlink, outside.name) + outside.write(line({'step': 0, 'time': 0, 'secret': 1})) + outside.close() + os.symlink(outside.name, self.path('metrics.jsonl')) + self.assertEqual(self.overview()['series'], []) + + +class Histograms(TrainingCase): + + def histogram(self, step, edges, counts, name='layer.weight', kind='weights', **extra): + self.append('histograms.jsonl', line({'step': step, 'time': step / 10, 'name': name, 'kind': kind, + 'edges': edges, 'counts': counts, **extra})) + + def test_rows_re_binned_onto_one_grid(self): + self.histogram(0, [0.0, 1.0, 2.0], [4, 6]) + self.histogram(10, [1.0, 3.0, 4.0], [10, 2], nonfinite=3) + self.histogram(10, [0.0, 1.0], [5], kind='gradients') + self.append('histograms.jsonl', b'{"step": 1}\n' + line({'step': 1, 'time': 0, 'name': 'x', 'kind': 'weights', + 'edges': [1, 0], 'counts': [1]})) + got = self.overview() + self.assertEqual([(h['name'], h['kind'], h['rows'], h['last_step']) for h in got['histograms']], + [('layer.weight', 'weights', 2, 10), ('layer.weight', 'gradients', 1, 10)]) + self.assertEqual((got['histogram_lines'], got['bad_lines']), (3, 2)) + h = training.histograms(self.run, name='layer.weight', kind='weights', bins=4) + json.dumps(h, allow_nan=False) + self.assertEqual((h['low'], h['high'], h['bins'], h['stride'], h['nonfinite']), (0.0, 4.0, 4, 1, 3)) + self.assertEqual(h['rows'], [{'step': 0, 'time': 0.0}, {'step': 10, 'time': 1.0}]) + self.assertEqual(h['counts'], [[4.0, 6.0, 0.0, 0.0], [0.0, 5.0, 5.0, 2.0]]) + self.assertEqual(h['totals'], [10.0, 12.0]) + h = training.histograms(self.run, name='layer.weight', kind='weights', bins=8) + self.assertEqual(h['counts'][0], [2.0, 2.0, 3.0, 3.0, 0.0, 0.0, 0.0, 0.0]) + self.refused('not-found', training.histograms, self.run, name='nothing', kind='weights') + + def test_rows_chosen_past_most_keep_the_first_and_the_last(self): + for step in range(50): + self.histogram(step, [0.0, 1.0], [step]) + h = training.histograms(self.run, name='layer.weight', kind='weights', most=7, bins=2) + steps = [row['step'] for row in h['rows']] + self.assertEqual((len(steps), steps[0], steps[-1], h['stride']), (7, 0, 49, 49 / 6)) + self.assertEqual(steps, sorted(set(steps))) + + def test_rebin_keeps_the_total(self): + counts = training.rebin([-1.0, -0.3, 0.2, 0.9, 0.9, 2.5], [3, 5, 7, 2, 11], -1.0, 2.5, 7) + self.assertAlmostEqual(math.fsum(counts), 28.0, places=12) + self.assertEqual(training.rebin([1.0, 1.0], [4], 1.0, 1.0, 5), [0.0, 0.0, 4.0, 0.0, 0.0]) + + +class Samples(TrainingCase): + + def sample(self, step, name, path, data=None): + if data is not None: + with open(os.path.join(self.root, 'training', path), 'wb') as handle: + handle.write(data) + self.append('samples.jsonl', line({'step': step, 'time': step / 10, 'name': name, 'path': path})) + + def test_images_arrays_missing_files_and_bad_paths(self): + self.sample(10, 'prediction', 'samples/000000010-prediction.png', png(5, 3)) + self.sample(10, 'field', 'samples/000000010-field.npy', npy((2, 3), ' (struct code, NPY descr) +TYPES = {'F16': ('e', '({state:'prepared',candidate_id:CANDIDATE,trial:{format:'trial_v1',id:'1'.repeat(32),study_id:STUDY,material:{kind:'recipe_material',catalog_id:P,id:C,sha256:C},replicate:0},graph:{id:'2'.repeat(32),revision:1,nodes:[],edges:[]},model:{kind:'model',catalog_id:P,id:ACTOR,revision:1,sha256:C},runtime_snapshot_sha256:C}); -const recovery=()=>({...state(),state:'recovery_required',available_actions:['open'],recovery:{action:'open',root:'fixture',directory:'alpha',reason:'projection_unavailable',request_key:'pending-preparation-key'}}); -(async()=>{ - { - const h=mount();await tick();for(const [id,value] of Object.entries({'project-candidate-study':STUDY,'project-candidate-id':CANDIDATE,'project-candidate-revision':'1','project-candidate-directory':'package','project-candidate-runtime':C}))h.input(id,value);h.nodes.get('project-candidate-kind').value='package'; - h.answers.push(({options})=>{const body=JSON.parse(options.body);assert.equal(body.study_id,undefined);assert.equal(body.expected_checkpoint,C);assert.equal(body.runtime_snapshot_sha256,C);assert.equal(body.candidate_id,CANDIDATE);assert.ok(!Object.hasOwn(body,'approval_id'));assert.equal(h.nodes.get('project-candidate-recovery-key').value,body.request_key);return response(bound(prepared(),NEXT));});await h.submit('project-candidate-prepare-form');assert.equal(h.nodes.get('project-files-error').textContent,'');assert.match(h.nodes.get('project-files-status').textContent,/No Run was launched/); - } - { - const h=mount();await tick();h.answers.push(response(recovery()));await h.click('project-files-refresh'); - assert.equal(h.nodes.get('project-candidate-prepare-fields').disabled,true);assert.equal(h.nodes.get('project-candidate-reconcile').disabled,false);assert.equal(h.nodes.get('project-candidate-abandon').disabled,false); - h.input('project-candidate-recovery-study',STUDY);h.input('project-candidate-recovery-key','pending-preparation-key'); - h.answers.push(({options})=>{const body=JSON.parse(options.body);assert.deepEqual(Object.keys(body).sort(),['binding','expected_checkpoint','preparation_key'].sort());assert.equal(body.preparation_key,'pending-preparation-key');return response(bound(prepared(),NEXT));});await h.click('project-candidate-reconcile');assert.equal(h.nodes.get('project-files-error').textContent,'');assert.match(h.nodes.get('project-files-status').textContent,/no writer was replayed/); - } - { - const h=mount();await tick();h.answers.push(response(recovery()));await h.click('project-files-refresh');h.input('project-candidate-recovery-study',STUDY);h.input('project-candidate-recovery-key','pending-preparation-key'); - const before=h.requests.length;await h.click('project-candidate-abandon');assert.equal(h.requests.length,before);assert.match(h.nodes.get('project-files-error').textContent,/explicit reason/); - h.input('project-candidate-abandon-reason','Scientist explicitly abandons stopped partial writer Ω'); - h.answers.push(({options})=>{const body=JSON.parse(options.body);assert.notEqual(body.request_key,body.preparation_key);return response(bound({format:'project_candidate_preparation_abandon_v1',origin_catalog_id:P,study_id:STUDY,preparation_key:body.preparation_key,preparation_operation:'study_execution.prepare_package',input_sha256:C,candidate:{kind:'candidate',catalog_id:P,id:CANDIDATE,sha256:NEXT},actor_id:ACTOR,reason:body.reason,observed_checkpoint:C,occurred_at:'2026-10-22T12:00:00+00:00',state:'abandoned'},NEXT));});await h.click('project-candidate-abandon');assert.equal(h.nodes.get('project-files-error').textContent,'');assert.match(h.nodes.get('project-files-status').textContent,/original staging and scientific facts remain/); - } - console.log('Candidate controller: 3 selected groups PASS'); -})().catch(error=>{console.error(error);process.exitCode=1;}); diff --git a/tests/test_workspace_project_file_deletion.cjs b/tests/test_workspace_project_file_deletion.cjs deleted file mode 100644 index 846f944b..00000000 --- a/tests/test_workspace_project_file_deletion.cjs +++ /dev/null @@ -1,43 +0,0 @@ -'use strict'; -// Controlled response checks; actual filesystem/HTTP proof is a separate gate. -const assert=require('node:assert/strict'),crypto=require('node:crypto'); -const {mount,bound,response,tick,P,ACTOR,C,NEXT}=require('./test_workspace_project_files_m4.cjs'); -const path='notes/scientist.md',text='Ω authored note\0before\r\n',sha=crypto.createHash('sha256').update(text).digest('hex'); -const id=crypto.createHash('sha256').update(P+'\0note\0'+path).digest('hex'),stamp='2026-10-22T12:00:00+00:00'; -const positive={revision:1,sha256:sha,bytes:Buffer.byteLength(text),text,adopted_at:stamp,actor_id:ACTOR}; -const deleted={revision:2,deleted:true,sha256:null,bytes:0,text:null,adopted_at:stamp,actor_id:ACTOR}; -const selection=version=>({id,project_id:P,kind:'note',path,version}); -const missing={id,kind:'note',path,revision:1,expected_sha256:sha,expected_bytes:positive.bytes,observed_sha256:null,observed_bytes:null,state:'missing',reason:'missing'}; -const tombstone={...missing,revision:2,expected_sha256:null,expected_bytes:0,state:'deleted',reason:null}; -const page=item=>({items:[item],offset:0,limit:40,total:1}); -async function chooseMissing(h){h.answers.push(response(bound(page(missing))));await h.click('project-working-load');h.answers.push(response(bound(selection(positive))));await h.row('project-working-list');} -(async()=>{ - { - const h=mount();await tick();await chooseMissing(h); - assert.equal(h.nodes.get('project-working-delete-fields').disabled,false); - assert.equal(h.nodes.get('project-working-text').value,text); - assert.match(h.nodes.get('project-working-delete-summary').textContent,new RegExp(sha)); - assert.ok(!h.requests.some(r=>r.url.endsWith('/adopt-deletion')),'Inspection must never adopt deletion'); - const before=h.requests.length;await h.submit('project-working-delete-form');assert.equal(h.requests.length,before);assert.match(h.nodes.get('project-files-error').textContent,/explicitly confirm/); - h.nodes.get('project-working-delete-confirm').checked=true; - h.nodes.get('project-working-delete-confirm').listeners.get('change')(); - h.answers.push(({url,options})=>{assert.equal(url,'/api/workspace/project-files/files/adopt-deletion');const body=JSON.parse(options.body);assert.equal(body.binding,'b'.repeat(32));assert.equal(body.expected_checkpoint,C);assert.equal(body.path,path);assert.equal(body.expected_revision,1);assert.equal(body.expected_sha256,sha);assert.deepEqual(Object.keys(body).sort(),['binding','expected_checkpoint','expected_revision','expected_sha256','path','request_key']);return response(bound({id,project_id:P,kind:'note',path,revision:2,sha256:null,bytes:0,deleted:true,adopted_at:stamp,changed:true},NEXT));},response(bound(page(tombstone),NEXT)),response(bound(selection(deleted),NEXT))); - await h.submit('project-working-delete-form');assert.equal(h.nodes.get('project-files-error').textContent,'');assert.equal(h.nodes.get('project-working-text').value,'');assert.match(h.nodes.get('project-working-summary').textContent,/note deletion retained/);assert.equal(h.nodes.get('project-working-delete-fields').disabled,true); - const metadata=v=>Object.fromEntries(Object.entries(v).filter(([k])=>k!=='text')); - h.answers.push(response(bound({id,items:[metadata(positive),metadata(deleted)],offset:0,limit:40,total:2},NEXT))); - await h.click('project-working-history');const labels=h.nodes.get('project-working-history-list').querySelectorAll('button').map(b=>b.textContent);assert.match(labels[1],/note deletion/);assert.doesNotMatch(labels[1],/null/); - h.answers.push(response(bound(selection(positive),NEXT)));await h.row('project-working-history-list');assert.equal(h.nodes.get('project-working-text').value,text);assert.equal(h.nodes.get('project-working-delete-fields').disabled,true); - } - { - const h=mount();await tick();await chooseMissing(h); - h.answers.push(response(bound(page({...missing,state:'unchanged',observed_sha256:sha,observed_bytes:positive.bytes,reason:null}))));await h.click('project-working-load'); - assert.equal(h.nodes.get('project-working-delete-fields').disabled,true,'Fresh inspection invalidates prior missing-note choice'); - h.nodes.get('project-working-delete-confirm').checked=true;const before=h.requests.length;await h.submit('project-working-delete-form');assert.equal(h.requests.length,before); - } - { - const h=mount();await tick();await chooseMissing(h);h.nodes.get('project-working-delete-confirm').checked=true; - h.answers.push(response(bound({id,project_id:P,kind:'note',path,revision:2,sha256:sha,bytes:0,deleted:true,adopted_at:stamp,changed:true},NEXT))); - await h.submit('project-working-delete-form');assert.match(h.nodes.get('project-files-error').textContent,/differs/);assert.equal(h.nodes.get('project-working-text').value,text,'Malformed deletion ACK cannot replace captured text'); - } - console.log('PASS 3 note-deletion controller groups: explicit missing-note confirmation, exact CAS, tombstone/history, refresh invalidation and malformed ACK refusal'); -})().catch(error=>{console.error(error);process.exitCode=1;}); diff --git a/tests/test_workspace_project_files.cjs b/tests/test_workspace_project_files.cjs deleted file mode 100644 index 86582638..00000000 --- a/tests/test_workspace_project_files.cjs +++ /dev/null @@ -1,267 +0,0 @@ -// Deterministic projection/lifecycle evidence; real keyboard/layout evidence is -// owned by browser_project_files.cjs, not this deliberately small DOM substitute. -'use strict'; -const assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'),vm=require('node:vm'),crypto=require('node:crypto'); -const html=fs.readFileSync(path.join(__dirname,'../static/project-files.html'),'utf8'); -const source=fs.readFileSync(path.join(__dirname,'../static/project-files.js'),'utf8'); -const pins=JSON.parse(source.match(/PROJECT_FILES_CONTRACTS_START \*\/\s*([\s\S]*?)\s*\/\* PROJECT_FILES_CONTRACTS_END/)[1]); -const capabilities={format:'agent_capabilities_v1',envelope_version:1,catalog_id:'a'.repeat(32),operations:Object.entries(pins).map(([id,sha256])=>({id,availability:'available',contracts:[{version:1,sha256,input_schema:{},output_schema:{},scopes:['inspect'],authority:'agent',effects:'read',retry:'read',bounds:{request_bytes:1048576,response_bytes:16777216,inline_decoded_bytes:524288,json_depth:64}}]})),limits:{request_bytes:1048576,response_bytes:16777216,max_operations:256}}; -const B='b'.repeat(32),P='c'.repeat(32),R='d'.repeat(32),C='e'.repeat(64),N='f'.repeat(64); -const actions=['new','open','save','close','query','get','import_text','annotate']; -const state=(binding=B,checkpoint=C,opened=true)=>({format:'project_files_state_v1',binding,state:opened?'open':'no_project_open',roots:['fixture'],project:opened?{project_id:P,name:'Text project',checkpoint}:null,available_actions:opened?actions:['new','open']}); -const record=(name='Greeting',revision=1)=>({id:R,kind:'text',name,text:'Hello world',groups:['source'],parents:[],provenance:{method:'import',rights:'unknown',normalization:'NFC; LF newlines'},task:'text_classification',annotation:{label:'greeting'},review:'draft',revision,source_revision:1,created_at:'2026-10-22T12:00:00+00:00',updated_at:'2026-10-22T12:00:00+00:00'}); -const summary=value=>Object.fromEntries(Object.entries(value).filter(([key])=>['id','kind','name','task','review','revision','source_revision','created_at','updated_at'].includes(key))); -const records=(binding=B,checkpoint=C,items=[record()],next_after_id=null)=>({binding,project_id:P,checkpoint,items:items.map(summary),next_after_id}); -const result=(binding=B,checkpoint=C,value=record())=>({binding,project_id:P,checkpoint,record:value,history:[value],next_history_before_revision:null}); -const reply=(value,status=200)=>new Response(JSON.stringify(value),{status,headers:{'Content-Type':'application/json'}}); -const defer=()=>{let resolve;const promise=new Promise(done=>{resolve=done;});return {promise,resolve};}; -const tick=async()=>{await new Promise(done=>setImmediate(done));await new Promise(done=>setImmediate(done));}; -function mount(initial=[reply(state()),reply(records())],capabilityValue=capabilities){ - const nodes=new Map(),events=new Map(),requests=[],answers=[...initial],discoveries=[];let focused=null; - class Node{ - constructor(tag='div'){this.tagName=tag;this.value='';this.children=[];this.dataset={};this.attrs={};this.listeners=new Map();this.disabled=false;this.hidden=false;this.textContent='';} - append(...values){this.children.push(...values);if(this.tagName==='select'&&!this.value&&values.length)this.value=values[0].value;} - replaceChildren(...values){this.children=[...values];if(this.tagName==='select')this.value=values[0]?.value??'';} - setAttribute(name,value){this.attrs[name]=value;} - addEventListener(name,handler){this.listeners.set(name,handler);} - focus(){focused=this;} - querySelectorAll(tag){return this.children.flatMap(child=>[...(child.tagName===tag?[child]:[]),...child.querySelectorAll(tag)]);} - } - for(const match of html.matchAll(/<([a-z][a-z0-9]*)\b([^>]*\bid="([^"]+)"[^>]*)>/g)){ - const node=new Node(match[1]);node.value=match[2].match(/\bvalue="([^"]*)"/)?.[1]??'';node.disabled=/\bdisabled\b/.test(match[2]);nodes.set(match[3],node); - } - nodes.get('annotation-task').value='text_classification';nodes.get('record-review').value='draft';nodes.get('project-explore-kind').value='data_query';nodes.get('project-source-kind').value='image';nodes.get('project-data-review').value='draft'; - const context={document:{getElementById:id=>{assert.ok(nodes.has(id),'Unknown node '+id);return nodes.get(id);},createElement:tag=>new Node(tag)},URLSearchParams,TextEncoder,crypto,console,window:null}; - context.window=context;context.TuldokAuth={canReview:()=>true,fetch:(url,options)=>{if(url==='/api/agent/capabilities'){discoveries.push({url,options});return Promise.resolve(reply(capabilityValue));}const header=options.headers?.['Tuldok-Operation-Contract'];assert.ok(header,'Every project call selects an exact contract');const [operation,version,digest]=header.split('/');assert.equal(version,'1');assert.equal(digest,pins[operation]);requests.push({url,options});assert.ok(answers.length,'Unexpected fetch '+url);return Promise.resolve(answers.shift());}}; - context.addEventListener=(name,handler)=>events.set(name,handler); - // View modules have separate real-browser evidence; this fixture owns native record/control lifetimes. - context.ScientificFieldInspector=()=>({clear(){},show(){}}); - context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); - context.ProjectMeasurements=()=>({setContext(){},clear(){}}); - context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); - vm.createContext(context);vm.runInContext(source,context); - return {nodes,requests,answers,events,discoveries,focused:()=>focused, - input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, - async click(id){await nodes.get(id).onclick();await tick();}, - async submit(id){nodes.get(id).onsubmit({preventDefault(){}});await tick();}}; -} -(async()=>{ - // The actual browser controller transports the current binding and checkpoint - // on every read/edit, then adopts only the acknowledged checkpoint. - { - const h=mount();await tick();assert.equal(h.nodes.get('project-files-error').textContent,'');assert.equal(h.discoveries.length,1);assert.equal(h.nodes.get('text-import-fields').disabled,false); - h.input('text-name','Authored note');h.input('text-content',''); - h.answers.push(reply(result(B,N,record('Authored note'))),reply(records(B,N,[record('Authored note')]))); - await h.submit('text-import-form'); - assert.ok(h.requests[2],h.nodes.get('project-files-error').textContent+' '+JSON.stringify(h.requests));const sent=JSON.parse(h.requests[2].options.body); - assert.equal(sent.binding,B);assert.equal(sent.expected_checkpoint,C);assert.equal(sent.asset.text,'');assert.match(sent.request_key,/^[a-f0-9]{32}$/); - assert.equal(h.nodes.get('text-content').value,'');assert.equal(h.nodes.get('project-files-dirty').textContent,'No unsubmitted edits.'); - assert.ok(h.requests[3].url.includes('checkpoint='+N));assert.equal(h.nodes.get('record-heading').textContent,'Authored note'); - assert.equal(h.focused(),h.nodes.get('record-heading')); - h.input('annotation-value','revised');h.answers.push(reply(result(B,C,{...record('Authored note',2),annotation:{label:'revised'}})),reply(records(B,C))); - await h.submit('annotation-form'); - const annotation=JSON.parse(h.requests[4].options.body);assert.equal(annotation.expected_checkpoint,N);assert.equal(annotation.annotation.revision,1);assert.equal(annotation.annotation.source_revision,1);assert.equal(annotation.annotation.annotation.label,'revised'); - h.answers.push(reply(state()),reply(records()));await h.click('project-files-save'); - assert.equal(JSON.parse(h.requests[6].options.body).expected_checkpoint,C);assert.equal(h.nodes.get('record-heading').textContent,'Authored note');assert.equal(h.focused(),h.nodes.get('project-files-save'));assert.equal(h.nodes.get('project-files-save').disabled,false); - h.answers.push(reply(state(P,C,false)));await h.click('project-files-close'); - assert.equal(h.nodes.get('text-import-fields').disabled,true); - h.nodes.get('project-files-directory').value='copied-project';h.answers.push(reply(state(B,C,true)),reply(records()));await h.click('project-files-open'); - const opening=JSON.parse(h.requests[9].options.body);assert.equal(opening.binding,P);assert.equal(opening.expected_checkpoint,null);assert.equal(opening.directory,'copied-project'); - } - // Unsubmitted drafts cannot disappear through lifecycle actions and warn on - // navigation; explicit discard is the only local destructive transition. - { - const h=mount();await tick();h.input('text-content','Keep this draft'); - const count=h.requests.length;await h.click('project-files-close');assert.equal(h.requests.length,count);assert.equal(h.nodes.get('text-content').value,'Keep this draft'); - assert.equal(h.focused(),h.nodes.get('project-files-error'));assert.match(h.nodes.get('project-files-error').textContent,/unsent or unacknowledged/); - let prevented=false;const event={preventDefault(){prevented=true;}};h.events.get('beforeunload')(event);assert.ok(prevented);assert.equal(event.returnValue,''); - await h.click('project-files-discard');assert.equal(h.nodes.get('text-content').value,'');assert.equal(h.nodes.get('project-files-discard').disabled,true); - } - // A session replacement supersedes a delayed response. It cannot display the - // old record even when the old request later succeeds with a valid payload. - { - const h=mount();await tick();const delayed=defer();h.answers.push(delayed.promise); - const button=h.nodes.get('records-list').querySelectorAll('button')[0],old=button.onclick();await tick(); - h.answers.push(reply(state(P,N)),reply(records(P,N,[record('New session record')])));h.events.get('tuldok-session')();await tick(); - delayed.resolve(reply(result(B,C,record('Obsolete record'))));await old;await tick(); - assert.equal(h.nodes.get('record-text').value,'');assert.notEqual(h.nodes.get('record-heading').textContent,'Obsolete record'); - assert.match(h.nodes.get('records-list').querySelectorAll('button')[0].textContent,/New session record/); - } - // Unknown acknowledgement retains edits, fences writes, and exposes recovery. - { - const h=mount();await tick();h.input('text-name','Draft');h.input('text-content','Unique draft'); - h.answers.push(Promise.reject(Error('connection lost')));await h.submit('text-import-form'); - assert.equal(h.nodes.get('text-content').value,'Unique draft');assert.match(h.nodes.get('project-files-error').textContent,/may already be saved/);assert.equal(h.nodes.get('import-text').disabled,true); - h.answers.push(reply(state()),reply(records()));await h.click('project-files-refresh');assert.equal(h.nodes.get('text-content').value,'Unique draft');assert.equal(h.nodes.get('import-text').disabled,false); - } - // History and record continuation are explicit, checkpoint-bound reads. - // Loading an older history page never rewrites an unsubmitted annotation. - { - const h=mount([reply(state()),reply(records(B,C,[record()],R))]);await tick(); - const other={...record('Second record'),id:'e'.repeat(32)};h.answers.push(reply(records(B,C,[other])));await h.click('records-more'); - assert.equal(h.nodes.get('records-list').querySelectorAll('button').length,2);assert.ok(h.requests.at(-1).url.includes('after_id='+R));assert.equal(h.nodes.get('records-more').hidden,true); - const latest=record('Greeting',2);h.answers.push(reply({...result(B,C,latest),next_history_before_revision:2}),reply(records()));await h.nodes.get('records-list').querySelectorAll('button')[0].onclick();await tick(); - h.input('annotation-value','Unsaved annotation');h.answers.push(reply({...result(B,C,latest),history:[record()]}));await h.click('record-history-more'); - assert.ok(h.requests.at(-1).url.includes('before_revision=2'));assert.equal(h.nodes.get('record-history').children.length,2);assert.equal(h.nodes.get('record-history-more').hidden,true);assert.equal(h.nodes.get('annotation-value').value,'Unsaved annotation'); - } - // A committed outcome receipt survives the error projection. Recovery Open - // targets only the declared directory and preserves the in-tab source draft. - { - const h=mount();await tick();h.input('text-name','Retained draft');h.input('text-content','May be committed'); - const receipt={committed:true,selected:true,project_id:P,checkpoint:N,request_key:'file-request-0001'}; - h.answers.push(reply({code:'projection_unavailable',error:'Saved; local view unavailable.',receipt},503));await h.submit('text-import-form'); - assert.match(h.nodes.get('project-files-receipt').textContent,/Saved to project/);assert.equal(h.nodes.get('text-content').value,'May be committed'); - const recovery={...state(),state:'recovery_required',available_actions:['open'],recovery:{action:'open',root:'fixture',directory:'original-project',reason:'projection_unavailable',request_key:'file-request-0001'}}; - h.answers.push(reply(recovery));await h.click('project-files-refresh');assert.equal(h.nodes.get('project-files-open').disabled,false);assert.equal(h.nodes.get('project-files-directory').disabled,true);assert.equal(h.nodes.get('project-files-directory').value,'original-project'); - h.answers.push(reply(state(P,N)),reply(records(P,N)));await h.click('project-files-open');assert.equal(h.nodes.get('text-content').value,'May be committed');assert.equal(JSON.parse(h.requests.at(-2).options.body).directory,'original-project');assert.equal(h.nodes.get('project-files-discard').disabled,false); - h.answers.push(reply(result(P,N)),reply(records(P,N)));await h.nodes.get('records-list').querySelectorAll('button')[0].onclick();await tick();assert.equal(h.nodes.get('record-text').value,'Hello world');assert.equal(h.nodes.get('text-content').value,'May be committed'); - } - // Incompatible capability pins stop dispatch rather than trusting remote - // schemas or silently downgrading the operation selection. - { - const incompatible=JSON.parse(JSON.stringify(capabilities));incompatible.operations.find(item=>item.id==='project_files.status').contracts[0].sha256='0'.repeat(64); - const h=mount([],incompatible);await tick();assert.equal(h.requests.length,0);assert.match(h.nodes.get('project-files-error').textContent,/No compatible project status contract/);assert.equal(h.nodes.get('project-files-open').disabled,true); - } - // Invalid future/unknown fields are errors, never an empty successful project. - { - const h=mount([reply({...state(),future:true})]);await tick();assert.match(h.nodes.get('project-files-error').textContent,/Unsupported project response/);assert.equal(h.nodes.get('project-files-new').disabled,true); - } - // A stale checkpoint error retains annotation text and focuses its recovery - // message. Refresh can update identity without silently reattaching the draft. - { - const h=mount();await tick();h.answers.push(reply(result()),reply(records()));await h.nodes.get('records-list').querySelectorAll('button')[0].onclick();await tick(); - h.input('annotation-value','Never lose this label');h.answers.push(reply({code:'stale_checkpoint',error:'Reload current checkpoint.'},409));await h.submit('annotation-form'); - assert.equal(h.nodes.get('annotation-value').value,'Never lose this label');assert.equal(h.nodes.get('annotation-save').disabled,true);assert.match(h.nodes.get('project-files-error').textContent,/stale_checkpoint/); - h.answers.push(reply(state(B,N)),reply(records(B,N)));await h.click('project-files-refresh');assert.equal(h.nodes.get('annotation-value').value,'Never lose this label');assert.equal(h.nodes.get('annotation-fields').disabled,true); - } - // M2 explores actual data through fixed contracts, preserves independent - // drafts and reads a bounded history page at the exact selected checkpoint. - { - const h=mount();await tick(); - const boundValue=(value,checkpoint=C)=>({binding:B,project_id:P,checkpoint,value}); - const data={...record(),source_available:true,source_lineage_known:true}; - h.answers.push(reply(boundValue({items:[data],offset:0,limit:40,total:1,analysis:{},criteria:{}}))); - await h.click('project-explore-load'); - assert.match(h.requests.at(-1).url,/\/data\?binding=/); - h.answers.push(reply(boundValue(data)));await h.nodes.get('project-explore-list').querySelectorAll('button')[0].onclick();await tick(); - assert.equal(h.nodes.get('project-data-note-form').hidden,false);assert.equal(h.nodes.get('project-metadata-form').hidden,true); - assert.equal(h.nodes.get('project-detail-title').textContent,'Greeting'); - h.answers.push(reply(boundValue({items:[data],offset:0,limit:1,total:2}))); - await h.click('project-detail-history');assert.match(h.requests.at(-1).url,/limit=1/);assert.equal(h.nodes.get('project-detail-history-more').hidden,false); - h.answers.push(reply(boundValue({items:[data],offset:1,limit:1,total:2}))); - await h.click('project-detail-history-more');assert.match(h.requests.at(-1).url,/offset=1/);assert.equal(h.nodes.get('project-detail-history-more').hidden,true); - h.input('project-search-name','Keep this independent draft');h.input('project-rights-note','Authored permission note'); - h.answers.push(reply(boundValue({record:{...data,revision:2},changed:true},N)),reply(records(B,N))); - await h.submit('project-rights-form');const request=h.requests.at(-2),body=JSON.parse(request.options.body); - assert.equal(request.options.headers['Tuldok-Operation-Contract'].split('/')[0],'project_files.data_rights'); - assert.equal(body.expected_checkpoint,C);assert.equal(body.revision,1);assert.equal(body.note,'Authored permission note');assert.match(body.request_key,/^[a-f0-9]{32}$/); - assert.match(h.nodes.get('project-files-dirty').textContent,/unsent or unacknowledged/);assert.equal(h.nodes.get('project-search-name').value,'Keep this independent draft'); - assert.equal(h.nodes.get('project-rights-fields').disabled,true,'Old selected metadata is fenced after publication'); - await h.click('project-files-discard');assert.equal(h.nodes.get('project-search-name').value,''); - } - // Missing/corrupt artifact observations are readable facts. A superseded M2 - // request cannot attach its payload or authoring metadata to a new binding. - { - const h=mount();await tick();h.nodes.get('project-explore-kind').value='artifacts'; - const facts={sha256:C,bytes:99,availability:{status:'corrupt',integrity:'mismatch',reason:'Exact unique bytes differ'}}; - h.answers.push(reply({binding:B,project_id:P,checkpoint:C,value:{items:[facts],offset:0,limit:40,total:1}})); - await h.click('project-explore-load');await h.nodes.get('project-explore-list').querySelectorAll('button')[0].onclick();await tick(); - const strings=node=>[node.textContent,...node.children.flatMap(strings)]; - assert.ok(strings(h.nodes.get('project-detail-facts')).includes('corrupt')); - assert.ok(strings(h.nodes.get('project-detail-facts')).includes('Exact unique bytes differ')); - h.nodes.get('project-explore-kind').value='data_query';const delayed=defer();h.answers.push(delayed.promise);const loading=h.click('project-explore-load');await tick(); - h.answers.push(reply(state(P,N)),reply(records(P,N)));h.events.get('tuldok-session')();await tick(); - delayed.resolve(reply({binding:B,project_id:P,checkpoint:C,value:{items:[record('Obsolete history')],offset:0,limit:40,total:1}}));await loading;await tick(); - assert.equal(h.nodes.get('project-explore-list').children.length,0); - assert.notEqual(h.nodes.get('project-detail-title').textContent,'Obsolete history'); - } - // Model creation copies the current owner's complete inert template. Source - // registration selects declared scientific references without user hash edits. - { - const h=mount();await tick();const boundValue=(value,checkpoint=C)=>({binding:B,project_id:P,checkpoint,value}); - const template={name:'Owner template',task:{kind:'task',catalog_id:P,id:'classification',revision:1,sha256:C},kind:'template',specification:{template:'owned'},source_selector:null,preprocessing_policy:{owned:true},output_policy:{owned:true}}; - h.answers.push(reply(boundValue({recipes:[{model_template:template}]})));await h.click('project-definitions-load'); - h.input('project-model-create-name','My scientific model');h.answers.push(reply(boundValue({id:R,name:'My scientific model',revision:1},N)),reply(records(B,N))); - await h.submit('project-model-create-form');const created=JSON.parse(h.requests.at(-2).options.body); - assert.equal(created.name,'My scientific model');assert.deepEqual(created.specification,template.specification);assert.equal(created.source_selector,null);assert.equal(created.expected_checkpoint,C);assert.equal(created.task.catalog_id,P); - h.input('project-register-path','fixture/tuldok.json');h.answers.push(reply(boundValue({project_id:R,origin_catalog_id:P,manifest_sha256:C},C)),reply(boundValue({items:[{id:'measurements',label:'Captured scientific fields',sha256:N,observed_sha256:N,status:'available',integrity:'verified'}],offset:0,limit:40,total:1},C))); - await h.submit('project-register-form');assert.equal(h.nodes.get('project-scientific-fields').disabled,false); - h.input('project-scientific-name','Observed fields');h.answers.push(reply(boundValue({id:R,kind:'field_sequence'},N)),reply(records(B,N))); - await h.submit('project-scientific-form');const imported=JSON.parse(h.requests.at(-2).options.body); - assert.equal(imported.reference_id,'measurements');assert.equal(imported.expected_sha256,N);assert.equal(imported.snapshot.project_id,R);assert.equal(imported.snapshot.manifest_sha256,C);assert.equal(imported.expected_checkpoint,C);assert.equal(imported.document.name,'Observed fields'); - assert.ok(h.requests.every(request=>!/(execute|submit-external|capture-runtime)/.test(request.url))); - } - // Resolving an explicitly selected missing artifact imports a trusted-root - // file by identity; its durable observation is followed by a fresh status read. - { - const h=mount();await tick();h.nodes.get('project-explore-kind').value='artifacts'; - const boundValue=(value,checkpoint=C)=>({binding:B,project_id:P,checkpoint,value}); - h.answers.push(reply(boundValue({items:[{format:'project_artifact_v1',sha256:C,bytes:9,availability:{state:'missing'}}],offset:0,limit:40,total:1}))); - await h.click('project-explore-load');await h.nodes.get('project-explore-list').querySelectorAll('button')[0].onclick();await tick(); - assert.equal(h.nodes.get('project-artifact-resolve-form').hidden,false);assert.equal(h.nodes.get('project-artifact-root').value,'fixture'); - h.input('project-artifact-path','unique/observation.bin');h.answers.push(reply(boundValue({format:'project_artifact_resolution_v1',sha256:C,bytes:9,actor_id:P,observed_at:'2026-10-22T12:00:00+00:00'},N)),reply(boundValue({state:'verified',expected_sha256:C,expected_bytes:9,observed_sha256:C,observed_bytes:9},N))); - await h.submit('project-artifact-resolve-form'); - const resolve=h.requests.at(-2),body=JSON.parse(resolve.options.body),status=h.requests.at(-1); - assert.equal(resolve.options.headers['Tuldok-Operation-Contract'].split('/')[0],'project_files.artifact_resolve'); - assert.equal(body.root,'fixture');assert.equal(body.path,'unique/observation.bin');assert.equal(body.sha256,C);assert.equal(body.expected_checkpoint,C);assert.match(body.request_key,/^[a-f0-9]{32}$/); - assert.equal(status.options.headers['Tuldok-Operation-Contract'].split('/')[0],'project_files.artifact_status');assert.equal(JSON.parse(status.options.body).checkpoint,N); - assert.equal(h.nodes.get('project-artifact-resolve-form').hidden,true);assert.equal(h.nodes.get('project-artifact-path').value,'');assert.equal(h.nodes.get('project-files-dirty').textContent,'No unsubmitted edits.'); - } - // Native saved selections contain items without pagination fields. Their - // successful create and get results must stay distinct from page contracts. - { - const h=mount();await tick();const boundValue=(value,checkpoint=C)=>({binding:B,project_id:P,checkpoint,value}); - const data={...record(),source_available:true},selection={id:P,name:'Exact revision',revision:1,mode:'fixed',schema_version:1,items:[{id:R,revision:1,source_revision:1}]}; - h.answers.push(reply(boundValue({items:[data],total:1,offset:0,limit:40,analysis:{},criteria:{}})));await h.click('project-explore-load'); - h.answers.push(reply(boundValue(data)));await h.nodes.get('project-explore-list').querySelectorAll('button')[0].onclick();await tick(); - h.input('project-selection-name','Exact revision');h.answers.push(reply(boundValue(selection,N)),reply(records(B,N))); - await h.submit('project-selection-form');assert.equal(h.nodes.get('project-files-error').textContent,'');assert.equal(h.nodes.get('project-files-dirty').textContent,'No unsubmitted edits.'); - assert.equal(JSON.parse(h.requests.at(-2).options.body).expected_checkpoint,C);assert.ok(h.requests.at(-1).url.includes('checkpoint='+N)); - h.nodes.get('project-explore-kind').value='selections';h.answers.push(reply(boundValue({items:[selection],total:1,offset:0,limit:40},N)));await h.click('project-explore-load'); - h.answers.push(reply(boundValue(selection,N)));await h.nodes.get('project-explore-list').querySelectorAll('button')[0].onclick();await tick(); - assert.equal(h.nodes.get('project-files-error').textContent,'');assert.equal(h.nodes.get('project-detail-title').textContent,'Exact revision'); - // Strict continuation checks still apply to actual pages. - h.answers.push(reply(boundValue({items:[selection],total:2,offset:0,limit:40},N)));await h.click('project-explore-load');assert.match(h.nodes.get('project-files-error').textContent,/Saved page differs/); - } - // Unpinned references use the owner's actual observation and retain their - // unverified meaning. A declared digest mismatch is never selectable/imported. - { - const h=mount();await tick();const boundValue=(value,checkpoint=C)=>({binding:B,project_id:P,checkpoint,value}); - const refs=[{id:'unpinned',label:'Observed data',status:'available',integrity:'unverified',observed_sha256:N}, - {id:'mismatch',label:'Changed declared data',status:'available',integrity:'mismatch',sha256:C,observed_sha256:N}, - {id:'missing',label:'Missing declared data',status:'missing',integrity:'unverified',sha256:C}]; - h.input('project-register-path','fixture/tuldok.json');h.answers.push(reply(boundValue({project_id:R,origin_catalog_id:P,manifest_sha256:C},N)),reply(boundValue({items:refs,total:3,offset:0,limit:40},N))); - await h.submit('project-register-form');const choices=h.nodes.get('project-scientific-reference').children; - assert.equal(choices[0].disabled,false);assert.match(choices[0].textContent,/unverified; not pinned/);assert.equal(choices[1].disabled,true);assert.match(choices[1].textContent,/mismatch/);assert.equal(choices[2].disabled,true); - h.nodes.get('project-scientific-reference').value='mismatch';const before=h.requests.length;await h.submit('project-scientific-form');assert.equal(h.requests.length,before);assert.match(h.nodes.get('project-files-error').textContent,/Declared digests must match/); - h.nodes.get('project-scientific-reference').value='unpinned';h.input('project-scientific-name','Observed fields');h.answers.push(reply(boundValue({id:R,kind:'field_sequence'},C)),reply(records(B,C))); - await h.submit('project-scientific-form');assert.equal(h.nodes.get('project-files-error').textContent,'');const imported=JSON.parse(h.requests.at(-2).options.body); - assert.equal(imported.reference_id,'unpinned');assert.equal(imported.expected_sha256,N);assert.equal(imported.expected_checkpoint,N); - } - // Current material review, local grant and durable admission are separate - // named actions. Reconciliation has no fabricated receipt key. - { - const h=mount();await tick();const boundValue=(value,checkpoint=C)=>({binding:B,project_id:P,checkpoint,value}); - assert.equal(h.requests.length,2,'Opening status never starts execution'); - h.answers.push(reply(boundValue({format:'project_execution_health_v1',active_run_ids:[],active_count:0,offset:0,limit:40,local_owner_active:false,available:true,drain_required:false})));await h.click('project-execution-health');assert.match(h.nodes.get('project-execution-summary').textContent,/0 current/); - h.answers.push(reply(boundValue({items:[{id:R,name:'Explicit recipe',revision:3,recipe:'external_recipe'}],total:1,offset:0,limit:40})));await h.click('project-execution-graphs'); - h.answers.push(reply(boundValue({graph_revision:3,material_sha256:N,material:{runtime:'current observed runtime'}})));await h.submit('project-execution-preflight-form');assert.equal(h.nodes.get('project-execution-submit-fields').disabled,true); - h.input('project-execution-consumer',P);await h.submit('project-execution-approval-form');assert.match(h.nodes.get('project-files-error').textContent,/Explicitly approve/); - const grant={id:R,catalog_id:P,approver_actor_id:P,approver_credential_id:null,consumer_actor_id:P,material_sha256:N,created_at:1,expires_at:3601,revoked_at:null}; - h.nodes.get('project-execution-confirm').checked=true;h.answers.push(reply(boundValue({format:'project_current_execution_approval_v1',current_local:true,grant},N)));await h.submit('project-execution-approval-form');assert.equal(h.nodes.get('project-files-error').textContent,'');assert.equal(h.nodes.get('project-execution-submit-fields').disabled,false);assert.equal(h.nodes.get('project-files-dirty').textContent,'No unsubmitted edits.'); - const approval=JSON.parse(h.requests.at(-1).options.body);assert.equal(approval.expected_checkpoint,C);assert.equal(approval.expected_digest,N);assert.equal(approval.consumer_actor_id,P); - h.answers.push(reply(boundValue({format:'project_run_admission_v1',origin_catalog_id:P,run_id:R,request_sha256:N},C)));await h.submit('project-execution-submit-form');assert.match(h.nodes.get('project-files-status').textContent,/durable intent/);assert.equal(h.nodes.get('project-execution-submit-fields').disabled,true);assert.equal(JSON.parse(h.requests.at(-1).options.body).approval_id,R); - h.answers.push(reply(boundValue({format:'project_run_admission_v1',origin_catalog_id:P,run_id:R,request_sha256:N},N)));await h.submit('project-execution-cancel-form');assert.match(h.nodes.get('project-files-status').textContent,/intent saved/); - h.answers.push(reply(boundValue({format:'project_execution_reconciliation_v1',origin_catalog_id:P,run_ids:[R],available:true},C)));await h.click('project-execution-reconcile');const reconciled=JSON.parse(h.requests.at(-1).options.body);assert.deepEqual(Object.keys(reconciled).sort(),['binding','expected_checkpoint']);assert.equal(reconciled.expected_checkpoint,N);assert.match(h.nodes.get('project-files-status').textContent,/reconciled and drained/); - } - // A typed model request uses the saved source revisions and an explicit - // endpoint. A missing acknowledgement preserves its authored request. - { - const h=mount();await tick();const boundValue=(value,checkpoint=C)=>({binding:B,project_id:P,checkpoint,value}); - const source={...record(),source_available:true};h.answers.push(reply(boundValue({items:[source],total:1,offset:0,limit:40,analysis:{},criteria:{}})));await h.click('project-explore-load');h.answers.push(reply(boundValue(source)));await h.nodes.get('project-explore-list').querySelectorAll('button')[0].onclick();await tick(); - h.nodes.get('project-producer-task').value='text_proposal_history';h.answers.push(reply(boundValue(source)));await h.click('project-producer-source-load'); - h.input('project-producer-url','http://127.0.0.1:12345');h.input('project-producer-model','exact-current-model');h.input('project-producer-instruction','Classify the exact frozen source');h.input('project-producer-labels','accepted\nrejected');h.nodes.get('project-producer-transport').value='typed';h.nodes.get('project-producer-confirm').checked=true; - h.answers.push(new Response('{broken',{status:200}));await h.submit('project-producer-start-form');const sent=JSON.parse(h.requests.at(-1).options.body);assert.equal(sent.family,'text_proposal_history');assert.equal(sent.document.source_id,R);assert.equal(sent.document.revision,1);assert.equal(sent.document.seed,null);assert.equal(sent.document.protocol,'pumas_typed_v1');assert.deepEqual(sent.document.labels,['accepted','rejected']);assert.match(h.nodes.get('project-files-error').textContent,/Acknowledgement unavailable/);assert.equal(h.nodes.get('project-producer-instruction').value,'Classify the exact frozen source');assert.equal(h.nodes.get('project-files-close').disabled,true); - } - console.log('PASS project-files controller: checkpoint requests, submit/history projection, dirty preservation, superseded replies, typed errors, unknown acknowledgements'); -})().catch(error=>{console.error(error);process.exitCode=1;}); diff --git a/tests/test_workspace_project_files_m4.cjs b/tests/test_workspace_project_files_m4.cjs deleted file mode 100644 index 80e343ca..00000000 --- a/tests/test_workspace_project_files_m4.cjs +++ /dev/null @@ -1,102 +0,0 @@ -// M4 controller fault boundaries. This small DOM substitute proves requests and -// inert projection; keyboard/layout and actual storage require real browser QA. -'use strict'; -const assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'),vm=require('node:vm'),crypto=require('node:crypto'); -const html=fs.readFileSync(path.join(__dirname,'../static/project-files.html'),'utf8'),source=fs.readFileSync(path.join(__dirname,'../static/project-files.js'),'utf8'); -const pins=JSON.parse(source.match(/PROJECT_FILES_CONTRACTS_START \*\/\s*([\s\S]*?)\s*\/\* PROJECT_FILES_CONTRACTS_END/)[1]); -const versionPins=JSON.parse(source.match(/PROJECT_FILES_VERSIONS_START \*\/\s*([\s\S]*?)\s*\/\* PROJECT_FILES_VERSIONS_END/)[1]); -const B='b'.repeat(32),P='c'.repeat(32),ACTOR='d'.repeat(32),FORK='a'.repeat(32),C='e'.repeat(64),NEXT='f'.repeat(64); -const actions=['new','open','save','close','query','get','import_text','annotate']; -const capabilities={format:'agent_capabilities_v1',envelope_version:1,catalog_id:'a'.repeat(32),operations:Object.entries(pins).map(([id,sha256])=>({id,availability:'available',contracts:[{version:1,sha256,input_schema:{},output_schema:{},scopes:['inspect'],authority:'agent',effects:'read',retry:'read',bounds:{request_bytes:1048576,response_bytes:16777216,inline_decoded_bytes:524288,json_depth:64}}]})),limits:{request_bytes:1048576,response_bytes:16777216,max_operations:256}}; -const state=(binding=B,checkpoint=C,project=P,name='Text project')=>({format:'project_files_state_v1',binding,state:'open',roots:['fixture'],project:{project_id:project,name,checkpoint},available_actions:actions}); -const records=(binding=B,checkpoint=C,project=P)=>({binding,project_id:project,checkpoint,items:[],next_after_id:null}); -const bound=(value,checkpoint=C)=>({binding:B,project_id:P,checkpoint,value}); -const response=(value,status=200)=>new Response(JSON.stringify(value),{status,headers:{'Content-Type':'application/json'}}); -const tick=async()=>{for(let n=0;n<4;n++)await new Promise(done=>setImmediate(done));}; -const hash=raw=>crypto.createHash('sha256').update(raw).digest('hex'),fileID=hash(P+'\0script\0scripts/inert.py'); -const version=(text='print("only text")\0Ω\r\n',revision=1)=>({revision,sha256:hash(Buffer.from(text)),bytes:Buffer.byteLength(text),adopted_at:'2026-10-22T12:00:00+00:00',actor_id:ACTOR,text}); -const selected=(text,revision)=>({id:fileID,project_id:P,kind:'script',path:'scripts/inert.py',version:version(text,revision)}); -function mount(capabilityValue=capabilities){ - const nodes=new Map(),requests=[],answers=[response(state()),response(records())];let focused=null; - class Element{ - constructor(tag){Object.assign(this,{tagName:tag,value:'',children:[],dataset:{},attrs:{},listeners:new Map(),disabled:false,hidden:false,textContent:''});} - append(...children){this.children.push(...children);if(this.tagName==='select'&&!this.value)this.value=children[0]?.value??'';} - replaceChildren(...children){this.children=children;if(this.tagName==='select')this.value=children[0]?.value??'';} - setAttribute(key,value){this.attrs[key]=value;} - addEventListener(name,handler){this.listeners.set(name,handler);} - focus(){focused=this;} - querySelectorAll(tag){return this.children.flatMap(child=>[...(child.tagName===tag?[child]:[]),...child.querySelectorAll(tag)]);} - } - for(const match of html.matchAll(/<([a-z][a-z0-9]*)\b([^>]*\bid="([^"]+)"[^>]*)>/g)){ - const node=new Element(match[1]);node.value=match[2].match(/\bvalue="([^"]*)"/)?.[1]??'';nodes.set(match[3],node); - } - for(const [id,value] of Object.entries({'annotation-task':'text_classification','record-review':'draft','project-explore-kind':'data_query','project-source-kind':'image','project-data-review':'draft','project-working-kind':'script'}))nodes.get(id).value=value; - const context={document:{getElementById:id=>{assert.ok(nodes.has(id),id);return nodes.get(id);},createElement:tag=>new Element(tag)},URLSearchParams,TextEncoder,TextDecoder,crypto,console,window:null};context.window=context; - context.addEventListener=()=>{}; - context.TuldokAuth={canReview:()=>true,fetch:async(url,options)=>{ - if(url==='/api/agent/capabilities')return response(capabilityValue); - const [operation,version,digest]=options.headers['Tuldok-Operation-Contract'].split('/');assert.equal(digest,versionPins[operation]?.[version]??pins[operation]); - const request={url,options};requests.push(request);assert.ok(answers.length,'Unexpected fetch '+url); - const next=answers.shift();return typeof next==='function'?next(request):next; - }}; - // View modules have separate real-browser evidence; this fixture owns native record/control lifetimes. - context.ScientificFieldInspector=()=>({clear(){},show(){}}); - context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); - context.ProjectMeasurements=()=>({setContext(){},clear(){}}); - context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); - vm.createContext(context);vm.runInContext(source,context); - return {nodes,requests,answers,focused:()=>focused, - input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, - async click(id){await nodes.get(id).onclick();await tick();}, - async submit(id){nodes.get(id).onsubmit({preventDefault(){}});await tick();for(let n=0;n<100&&nodes.get('project-save-as-fields').disabled&&!nodes.get('project-files-error').textContent;n++)await new Promise(done=>setTimeout(done,2));}, - async row(id,index=0){await nodes.get(id).querySelectorAll('button')[index].onclick();await tick();}}; -} -const inspection=()=>{const v=version();return {id:fileID,kind:'script',path:'scripts/inert.py',revision:1,expected_sha256:v.sha256,expected_bytes:v.bytes,observed_sha256:v.sha256,observed_bytes:v.bytes,state:'unchanged',reason:null};}; -module.exports={mount,state,records,bound,response,tick,B,P,ACTOR,C,NEXT,capabilities,versionPins}; -if(require.main===module)(async()=>{ - // Save As changes only after a correlated independent selection receipt. - { - const h=mount();await tick();h.input('project-save-as-directory','independent');h.input('project-save-as-name','Fork Ω'); - h.answers.push(({options})=>{const body=JSON.parse(options.body);assert.equal(body.binding,B);assert.equal(body.expected_checkpoint,C);assert.equal(body.root,'fixture');return response({state:state(ACTOR,NEXT,FORK,'Fork Ω'),fork:{source_project_id:P,source_checkpoint:C,project_id:FORK,checkpoint:NEXT,root:body.root,directory:body.directory,request_key:body.request_key,reconciled:false,mapping_count:2,copied_count:1,missing_regenerable_count:1}});},response(records(ACTOR,NEXT,FORK))); - await h.submit('project-save-as-form');assert.equal(h.nodes.get('project-files-error').textContent,'');assert.match(h.nodes.get('project-save-as-summary').textContent,/2 editable mappings/);assert.equal(h.focused(),h.nodes.get('project-fork-load')); - assert.ok(h.requests.at(-1).url.includes('binding='+ACTOR));assert.ok(h.requests.at(-1).url.includes('checkpoint='+NEXT)); - } - // Lost acknowledgement retains the exact source/destination/key for explicit - // retry, even after status refresh observes a different selection. - { - const h=mount();await tick();h.input('project-save-as-directory','pending');h.input('project-save-as-name','Pending'); - h.answers.push(()=>Promise.reject(Error('lost reply')));await h.submit('project-save-as-form');const original=JSON.parse(h.requests.at(-1).options.body); - assert.match(h.nodes.get('project-files-error').textContent,/may already be saved/);assert.equal(h.nodes.get('project-save-as-directory').value,'pending'); - h.answers.push(response(state(ACTOR,NEXT,FORK,'Pending')),response(records(ACTOR,NEXT,FORK)));await h.click('project-files-refresh'); - h.input('project-save-as-directory','different');const before=h.requests.length;await h.submit('project-save-as-form');assert.equal(h.requests.length,before);assert.match(h.nodes.get('project-files-error').textContent,/pending Save As destination/); - h.input('project-save-as-directory','pending');h.answers.push(({options})=>{assert.deepEqual(JSON.parse(options.body),original);return response({error:'Inspect original source before recovery.',code:'stale_binding'},409);});await h.submit('project-save-as-form');assert.equal(h.requests.length,before+1); - } - // Captured scripts render only after exact byte/revision checks. A corrupt - // reply cannot replace the prior visible revision, and no script is evaluated. - { - const h=mount();await tick();h.answers.push(response(bound({items:[inspection()],offset:0,limit:40,total:1})));await h.click('project-working-load'); - h.answers.push(response(bound(selected())));await h.row('project-working-list');assert.equal(h.nodes.get('project-working-text').value,version().text);assert.equal(h.nodes.get('project-working-path').value,'scripts/inert.py'); - const meta=version();delete meta.text;h.answers.push(response(bound({id:fileID,items:[meta],offset:0,limit:40,total:1})));await h.click('project-working-history'); - assert.equal(h.nodes.get('project-working-history-list').children.length,1);assert.equal(h.nodes.get('project-working-history-more').hidden,true); - const corrupt=selected('changed',1);corrupt.version.sha256=C;h.answers.push(response(bound(corrupt)));await h.row('project-working-history-list'); - assert.equal(h.nodes.get('project-working-text').value,version().text);assert.match(h.nodes.get('project-files-error').textContent,/exact bytes/); - } - // Explicit adoption transports the reviewed SHA and optimistic revision. The - // current file is read by the server owner; the browser sends no source text. - { - const h=mount();await tick();const v=version('new exact text',1); - for(const [id,value] of Object.entries({'project-working-path':'scripts/inert.py','project-working-revision':'0','project-working-sha':v.sha256}))h.input(id,value); - h.answers.push(({options})=>{const body=JSON.parse(options.body);assert.equal(body.expected_checkpoint,C);assert.equal(body.expected_revision,0);assert.equal(body.expected_sha256,v.sha256);assert.equal(body.path,'scripts/inert.py');assert.ok(!Object.hasOwn(body,'text'));return response(bound({id:fileID,project_id:P,kind:'script',path:body.path,revision:1,sha256:v.sha256,bytes:v.bytes,adopted_at:v.adopted_at,changed:true},NEXT));},response(bound({items:[],offset:0,limit:40,total:0},NEXT)),response(bound(selected('new exact text',1),NEXT))); - await h.submit('project-working-adopt-form');assert.equal(h.nodes.get('project-files-error').textContent,'');assert.equal(h.nodes.get('project-working-text').value,'new exact text');assert.equal(h.nodes.get('project-working-revision').value,'1');assert.match(h.nodes.get('project-files-status').textContent,/remains inert/); - } - // Fork metadata pages and unresolved binding details remain read-only. The - // original definition is fetched only after a user selects that exact row. - { - const h=mount();await tick();const origin={project_id:FORK,checkpoint:NEXT,family:'model_definition',id:ACTOR,revision:2,created_at:'2026-10-22T12:00:00+00:00'},current={format:'project_material_fork_intent_v1',catalog_id:P,id:ACTOR,revision:1,name:'Retained