diff --git a/agent_api.py b/agent_api.py index 4bcd9ce6..90a534c9 100644 --- a/agent_api.py +++ b/agent_api.py @@ -952,6 +952,23 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non required=[*list(_MEASUREMENT_SELECTION),'metric_id'],output=_MEASUREMENT_BOUND(_MEASUREMENT_QUERY)) _project_file_operation('measurement_inspect','GET','/measurements/{packet_sha256}/observations/{observation_id}', dict(_MEASUREMENT_SELECTION,observation_id=_MEASUREMENT_ID),output=_MEASUREMENT_BOUND(_MEASUREMENT_INSPECTION)) + +_COMPARISON_QUERY = obj(dict( + {key:_MEASUREMENT_QUERY['properties'][key] for key in ('metric','definition','filters','subject_ids','total','offset','limit','sampling')}, + subject_ids=array(_MEASUREMENT_ID,256,1), + summary=array(obj(dict(_MEASUREMENT_SUMMARY['properties'],source=obj(_MEASUREMENT_SELECTION))),256), + rows=array(obj(dict(_MEASUREMENT_OBSERVATION['properties'],source=obj(_MEASUREMENT_SELECTION))),100))) +_COMPARISON_VALUE = obj({'format':const('retained_measurement_comparison_v1'), + 'scope':_MEASUREMENT_SCOPE,'scope_definition':{'type':'object'}, + 'dimensions':_MEASUREMENT_HEADER['properties']['dimensions'],'metrics':array(_MEASUREMENT_METRIC,32,1), + 'definition_source':obj(_MEASUREMENT_SELECTION),'case_population_sha256':SHA, + 'case_count':integer(1,4096),'observation_count':integer(1,100000), + 'subjects':array(obj(dict(_MEASUREMENT_SUBJECT['properties'],source=obj(_MEASUREMENT_SELECTION))),256,1), + 'query':{'oneOf':[_COMPARISON_QUERY,_NULL]}}) +_project_file_operation('measurement_compare','POST','/measurements/compare', + dict(sources=array(obj(_MEASUREMENT_SELECTION),16,1),metric_id=_MEASUREMENT_ID, + subject_ids=array(_MEASUREMENT_ID,256,1),filters=_MEASUREMENT_DIMENSIONS,offset=SAFE_NAT,limit=integer(1,100)), + required=['sources'],output=obj({'sources':array(_MEASUREMENT_SOURCE,16,1),'value':_COMPARISON_VALUE})) _project_file_operation('data_annotate', 'POST', '/data/{record_id}', { 'record_id': ID, 'revision': SAFE_POS, 'source_revision': SAFE_POS, 'task': {'enum': list(ANNOTATIONS)}, 'annotation': JSON_OBJECT, @@ -1312,6 +1329,17 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non scopes=('inspect','edit'),mutation=True,description='Create an independent idle project fork at an explicit configured local destination; retain inert origins and exact copy completeness.') _OPERATIONS[-1]['outputSchema']['oneOf'][1]['properties']['result']['properties']['receipt']=PROJECT_FILES_RECEIPT PROJECT_FILES_M4.add('project_files.save_as');_OWNER_OPERATION_IDS.add('project_files.save_as');_MODEL_RECEIPT_IDS.add('project_files.save_as') +# Composition is a new owner-only operation; existing Save As pins/state stay frozen. +_PROJECT_COMPOSITION_SOURCE=obj({'root':text(160),'directory':text(512),'project_id':ID,'checkpoint':SHA}) +model_operation('project_files.compose','POST','/api/workspace/project-files/compose', + dict(_PROJECT_FILES_MUTATION,root=text(160),directory=text(512),name=text(200), + sources=array(_PROJECT_COMPOSITION_SOURCE,15,1)), + output=obj({'state':PROJECT_FILES_STATE,'fork':_PROJECT_FORK_RESULT, + 'sources':array(obj({'project_id':ID,'checkpoint':SHA}),16,2)}), + scopes=('inspect','edit'),mutation=True, + description='Compose explicit retained project checkpoints with the current editable primary into one independent inert project.') +_OPERATIONS[-1]['outputSchema']['oneOf'][1]['properties']['result']['properties']['receipt']=PROJECT_FILES_RECEIPT +PROJECT_FILES_M4.add('project_files.compose');_OWNER_OPERATION_IDS.add('project_files.compose');_MODEL_RECEIPT_IDS.add('project_files.compose') _PROJECT_FILE_INSPECTION=obj({'id':SHA,'kind':_PROJECT_FILE_KIND,'path':text(512),'revision':SAFE_POS, 'expected_sha256':{'oneOf':[SHA,{'type':'null'}]},'expected_bytes':integer(0,1048576), 'observed_sha256':{'oneOf':[SHA,{'type':'null'}]},'observed_bytes':{'oneOf':[integer(0,1048576),{'type':'null'}]}, @@ -1327,6 +1355,9 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non output=obj(dict(_PROJECT_FILES_PAGE['properties'],id=SHA,items=array({'oneOf':[obj(_PROJECT_FILE_VERSION_META),obj(_PROJECT_FILE_DELETED_META)]},40))),milestone=4) _project_file_operation('file_adopt','POST','/files/adopt',{'kind':_PROJECT_FILE_KIND,'path':text(512), 'expected_revision':SAFE_NAT,'expected_sha256':SHA},output=_PROJECT_FILE_RESULT,mutation=True,milestone=4) +_project_file_operation('files_adopt','POST','/files/adopt-many',{'items':array(obj({'kind':_PROJECT_FILE_KIND, + 'path':text(512),'expected_revision':SAFE_NAT,'expected_sha256':SHA}),40,1)}, + output=obj({'items':array(_PROJECT_FILE_RESULT,40,1),'changed_count':integer(0,40)}),mutation=True,milestone=4) _project_file_operation('file_adopt_deletion','POST','/files/adopt-deletion',{'path':text(512), 'expected_revision':SAFE_POS,'expected_sha256':SHA},output=obj(dict(_PROJECT_FILE_RESULT['properties'], kind=const('note'),revision=integer(2,9007199254740991),deleted=const(True),sha256={'type':'null'},bytes=const(0),changed=const(True))), @@ -1440,6 +1471,13 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non _LOOKUP = {key: operation for operation in _OPERATIONS for key in (operation['operation_id'], operation['mcp_name'])} # Explicit locally implemented versions. No remotely supplied code or routes. _CONTRACTS = {op['operation_id']: {1: copy.deepcopy(op)} for op in _OPERATIONS} +# Keep the installed v1 bounds/hash exact. Version 2 admits complete larger +# comparisons; each individual packet and every response page retain their caps. +_comparison_capacity=copy.deepcopy(_CONTRACTS['project_files.measurement_compare'][1]) +_comparison_value=_comparison_capacity['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties'] +_comparison_value['observation_count']=integer(1,250000) +_comparison_value['query']['oneOf'][0]['properties']['total']=integer(0,250000) +_CONTRACTS['project_files.measurement_compare'][2]=_comparison_capacity # v1 remains the closed Run-only current custody view. v2 additionally exposes # bounded original local native phases; imported history never populates it. _native_health=copy.deepcopy(_CONTRACTS['project_files.execution_health'][1]) @@ -1943,6 +1981,12 @@ def operation(name, version=None): return copy.deepcopy(versions[max(versions) if version is None else version]) +def measurement_comparison_row_limit(version=None): + """Return the selected immutable public contract's comparison budget.""" + selected=operation('project_files.measurement_compare',version) + return selected['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties']['observation_count']['maximum'] + + def _native_validation_result_ok(value, revision): """Check the finite public truth fields without interpreting native runtime data. @@ -1974,7 +2018,7 @@ def _finite(value, depth=0, *, nul_path=None, _path=()): if depth > DEPTH_LIMIT: raise AgentAPIError('JSON nesting exceeds the transport bound.') if isinstance(value, str): - if any(0xD800 <= ord(ch) <= 0xDFFF for ch in value) or ('\x00' in value and _path != nul_path): + if (not str.isascii(value) and any(0xD800 <= ord(ch) <= 0xDFFF for ch in value)) or ('\x00' in value and _path != nul_path): raise AgentAPIError('Text contains an invalid code point.') elif isinstance(value, float) and not math.isfinite(value): raise AgentAPIError('JSON numbers must be finite.') @@ -2351,6 +2395,8 @@ def _private_file(path, maximum): class AgentClient: """No catalog access, automatic retries, redirects, proxies or owner identity.""" + MAX_TIMEOUT_SECONDS = 300 + def __init__(self, endpoint, token, expected_catalog_id, timeout=120): try: parsed = urlsplit(endpoint) @@ -2360,7 +2406,7 @@ def __init__(self, endpoint, token, expected_catalog_id, timeout=120): raise ValueError() if not re.fullmatch(r'[A-Za-z0-9_-]{20,4096}', token) or not re.fullmatch('[a-f0-9]{32}', expected_catalog_id): raise ValueError() - if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= 300: + if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= self.MAX_TIMEOUT_SECONDS: raise ValueError() except (ValueError, TypeError): raise AgentAPIError('Invalid fixed loopback client configuration.', 'configuration') from None @@ -2650,7 +2696,7 @@ def call(self, name, arguments, *, observer=None, observation=None): elif (value['binding']!=arguments['binding'] or value['receipt']['request_key']!=arguments['request_key'] or value['receipt']['report_sha256']!=arguments['report_sha256']): raise AgentAPIError('Extraction receipt belongs to another controller report or request.','transport',502) - if op['operation_id']=='project_files.save_as' and result['status']<400: + if op['operation_id'] in ('project_files.save_as','project_files.compose') and result['status']<400: value=result['result'];fork=value['fork'];state=value['state'] if (state['state']!='open' or state['project']['project_id']!=fork['project_id'] or state['project']['checkpoint']!=fork['checkpoint'] or state['project']['name']!=arguments['name'] or state['binding']==arguments['binding'] @@ -2658,6 +2704,11 @@ def call(self, name, arguments, *, observer=None, observation=None): or fork['source_checkpoint']!=arguments['expected_checkpoint'] or any(fork[field]!=arguments[field] for field in ('root','directory','request_key'))): raise AgentAPIError('Save As result differs from its source, destination or selected state.','transport',502) + if op['operation_id']=='project_files.compose': + expected=[dict(project_id=fork['source_project_id'],checkpoint=arguments['expected_checkpoint'])] + expected.extend({key:item[key] for key in ('project_id','checkpoint')} for item in arguments['sources']) + if value['sources']!=expected: + raise AgentAPIError('Composition result differs from its exact ordered sources.','transport',502) elif op['operation_id'] in PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4: value = result['result'] if result['status'] >= 400: @@ -2671,6 +2722,18 @@ def call(self, name, arguments, *, observer=None, observation=None): raise AgentAPIError('Project read response belongs to a different checkpoint.', 'transport', 502) page = value['value'] action=op['operation_id'].removeprefix('project_files.') + if action=='files_adopt': + if (len(page['items'])!=len(arguments['items']) + or page['changed_count']!=sum(item['changed'] for item in page['items']) + or sum(item['bytes'] for item in page['items'])>8*1024*1024): + raise AgentAPIError('Batch adoption count or capture bound differs.','transport',502) + for item,reviewed in zip(page['items'],arguments['items']): + identity=hashlib.sha256((value['project_id']+'\0'+reviewed['kind']+'\0'+reviewed['path']).encode()).hexdigest() + if (item['id']!=identity or item['project_id']!=value['project_id'] + or item['kind']!=reviewed['kind'] or item['path']!=reviewed['path'] + or item['sha256']!=reviewed['expected_sha256'] + or item['revision']!=reviewed['expected_revision']+int(item['changed'])): + raise AgentAPIError('Batch adoption differs from its reviewed files.','transport',502) if action in ('file_get','file_adopt','file_adopt_deletion'): kind,path=page['kind'],page['path'] expected_id=hashlib.sha256((value['project_id']+'\0'+kind+'\0'+path).encode()).hexdigest() @@ -2968,6 +3031,10 @@ def call(self, name, arguments, *, observer=None, observation=None): class ProjectFilesOwnerClient(AgentClient): """Explicit owner file-project access; no unrelated owner operations.""" + # Large retained projects may outlast the generic transport's five minutes. + # Only an explicit owner configuration opts into a longer absolute wait. + MAX_TIMEOUT_SECONDS = 7200 + OPERATIONS = frozenset('project_files.' + action for action in ('status', 'new', 'open', 'save', 'close', 'query', 'get', 'import_text', 'annotate')) | PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4 @@ -3025,6 +3092,8 @@ def _validate_measurement_selection(action, page, arguments, project_id): """Correlate fixed scalar replies; scientific validity remains upstream.""" def refuse(): raise AgentAPIError('Retained measurements differ from the selected source or continuation.', 'transport', 502) + if action=='measurement_compare': + return _validate_comparison_selection(page, arguments) if action=='measurement_sources': offset,limit=arguments.get('offset',0),arguments.get('limit',40) keys=[(d['origin_catalog_id'],d['id']) for d in page['items']] @@ -3057,6 +3126,50 @@ def refuse(): or any(r['metric_id']!=arguments['metric_id'] or r['subject_id'] not in subjects for r in result['rows'])):refuse() +def _validate_comparison_selection(page, arguments): + """Bind aggregate replies to reviewed packets, subjects and exact page intent.""" + def refuse(): + raise AgentAPIError('Measurement comparison differs from its selected packets or continuation.', 'transport', 502) + def key(locator): + return locator['origin_catalog_id'], locator['packet_sha256'] + selected = arguments['sources'] + keys = [key(item) for item in selected] + documents, value = page['sources'], page['value'] + if (len(keys)!=len(set(keys)) or len(documents)!=len(keys) + or [(doc['origin_catalog_id'],doc['id']) for doc in documents]!=keys + or any(doc['blob']['sha256']!=doc['id'] or doc['source']['measured_sha256']!=doc['id'] for doc in documents) + or sum(doc['blob']['bytes'] for doc in documents)>128*1024*1024 + or value['definition_source']!=selected[0]):refuse() + subjects = value['subjects'] + identities = [subject['id'] for subject in subjects] + if (len(identities)!=len(set(identities)) or any(key(subject['source']) not in keys for subject in subjects) + or value['observation_count']!=len(subjects)*value['case_count']*len(value['metrics'])):refuse() + owners = {subject['id']:subject['source'] for subject in subjects} + query = value['query'] + if 'metric_id' not in arguments: + if query is not None:refuse() + return + if query is None:refuse() + wanted = arguments.get('subject_ids', identities) + offset,limit = arguments.get('offset',0),arguments.get('limit',50) + metrics = [metric for metric in value['metrics'] if metric['id']==arguments['metric_id']] + if (len(metrics)!=1 or query['metric']!=metrics[0] or query['subject_ids']!=wanted + or len(wanted)!=len(set(wanted)) or any(identity not in owners for identity in wanted) + or query['filters']!=arguments.get('filters',{}) + or query['offset']!=offset or query['limit']!=limit + or len(query['rows'])!=min(limit,max(0,query['total']-offset)) + or [summary['subject_id'] for summary in query['summary']]!=wanted + or sum(summary['count'] for summary in query['summary'])!=query['total']):refuse() + for summary in query['summary']: + if (summary['source']!=owners[summary['subject_id']] + or sum(summary['states'].values())!=summary['count'] + or (summary['maximum'] is None)!=(summary['states']['measured']==0)):refuse() + for row in query['rows']: + if (row['subject_id'] not in wanted or row['metric_id']!=arguments['metric_id'] + or row['source']!=owners[row['subject_id']] + or (row['value'] is None)!=(row['state']!='measured')):refuse() + + class PublicationOwnerClient(AgentClient): """Explicit owner publication actions only; ordinary AgentClient stays agent-only.""" OPERATIONS = frozenset(('runs.publication_status', 'runs.recover_publication', 'runs.cancel_publication', diff --git a/app.py b/app.py index b7a59bb2..f818cc0b 100644 --- a/app.py +++ b/app.py @@ -486,9 +486,15 @@ def _access_body(self): except (ValueError, workbench.WorkbenchError): raise access.AccessError('Access requests require a bounded JSON object.', 'invalid', 400) from None + def _catalog_busy(self): + return self.reply({'error': 'Tuldok is busy completing another operation. Try again after it finishes.', + 'code': 'storage'}, 503) + def _access_error(self, error): try: dataset.access.denial(error, 'http.request', urlsplit(self.path).path[:512], anonymous=True) + except CatalogBusy: + return self._catalog_busy() except CommitOutcomeUnknown: return self.reply({'error': 'Commit outcome is unknown; inspect state before retrying.', 'code': 'commit_unknown'}, 503) except psycopg.Error: @@ -583,6 +589,8 @@ def _handle_GET(self): return self.reply({'error': str(error), 'code': error.code}, error.status) except access.AccessError as error: return self._access_error(error) + except CatalogBusy: + return self._catalog_busy() except CommitOutcomeUnknown: return self.reply({'error': 'Commit outcome is unknown; inspect state before retrying.', 'code': 'commit_unknown'}, 503) except psycopg.Error: @@ -633,6 +641,8 @@ def _handle_POST(self): return self.reply({'error': str(error), 'code': error.code}, error.status) except access.AccessError as error: return self._access_error(error) + except CatalogBusy: + return self._catalog_busy() except CommitOutcomeUnknown: return self.reply({'error': 'Commit outcome is unknown; inspect state before retrying.', 'code': 'commit_unknown'}, 503) except psycopg.Error: @@ -909,6 +919,7 @@ def _get(self): '/workspaces.js': ('workspaces.js', 'text/javascript'), '/experiment-graph.js': ('experiment-graph.js', 'text/javascript'), '/project-progression.js': ('project-progression.js', 'text/javascript'), + '/project-model-portfolio.js': ('project-model-portfolio.js', 'text/javascript'), '/project-measurements.js': ('project-measurements.js', 'text/javascript'), '/project-measurements.css': ('project-measurements.css', 'text/css'), '/experiment-lineage.js': ('experiment-lineage.js', 'text/javascript'), diff --git a/docs/contracts/project-files.md b/docs/contracts/project-files.md index 7959d37d..387c9bc6 100644 --- a/docs/contracts/project-files.md +++ b/docs/contracts/project-files.md @@ -32,9 +32,12 @@ request receipts and a SHA256 checkpoint. Its digest covers canonical bytes of the document with `checkpoint` omitted. The store supports only its declared format and family; malformed and future versions fail before projection writes. The canonical checkpoint index is limited to 64 MiB, separately from the -100,000-record, 40 MiB-per-record and 256 MiB-total-record limits. Commit checks +100,000-record, 40 MiB-per-record and 2 GiB-total-record limits. The aggregate +budget covers canonical record bytes, excluding artifact payloads; scientific +catalogs retain original runtime/file observations across many Runs. Commit +and cold Open enforce the same aggregate budget. Commit checks the index size before staging immutable records. This format is unchanged; -older installations with the former 16 MiB index limit cannot open larger +older installations with the former 16 MiB index or 256 MiB aggregate limit cannot open larger checkpoints and must be upgraded before consuming them. Verified stopped-source backups and ordinary catalog backups share identical @@ -102,6 +105,16 @@ annotations; human review still requires the owner. Every project action is boun to an opaque workspace generation and its expected canonical checkpoint. Use `ProjectFilesOwnerClient` only for the narrow owner project-file operations. +Its explicit `timeout` configuration may be up to 7,200 seconds for large retained +projects; the default remains 120 seconds. The absolute request watchdog still +applies and a transport timeout never authorizes an automatic mutation retry. +Generic clients retain their 300-second maximum. This wait budget is not an +execution deadline or a cancellation request. + +Browser contract tables derive from the local `agent_api` registry. Regenerate +them with `python scripts/generate_project_files_contracts.py`; add `--check` +to verify that the installed v1 pins, all supported version pins and fixed +routes match that registry. Remote schemas remain opaque discovery data. `ProjectFilesClient` wraps a negotiated client. CLI domain `file-*` commands take `--arguments` JSON files; lifecycle commands use explicit flags and select `--owner-project-files` with a diff --git a/docs/contracts/project-model-portfolio.md b/docs/contracts/project-model-portfolio.md new file mode 100644 index 00000000..d7a0a991 --- /dev/null +++ b/docs/contracts/project-model-portfolio.md @@ -0,0 +1,87 @@ +# Authored model organization + +The project workspace can display an adopted `.portfolio.json` note as an +authored guide to model fits/components. This is separate from **Experiment +progression**, which retains genuine Study/Trial ancestry. Group edges mean +authored grouping; they do not establish derivation, admission or scientific +acceptance. Relationship notes have explicit comparison/parameter meanings and +source excerpts. Missing historical Trial ancestry stays unknown. + +## Retained document + +`project_model_portfolio_v1` is a UTF-8 JSON note of at most1MiB with exactly: + +- `format`: the fixed string above; `project_id`: the owning32hex project ID; +- `title` (1–200characters), `description` (1–8,000characters), and + `authorship`: `retrospective`; +- `groups`: at most64records with exact fields `id`, `label`, `parent`, + `description`. IDs are unique, parent is null or another group, and grouping + is acyclic; +- `models`:1–1,000records with exact fields `id`, `label`, `group_id`, `aliases`, + `review`, `card`, `summary`. Each unique ID names an authored model entry, + not necessarily a complete generator, Trial or unique parameter state; +- `relationships`: at most2,000records with exact fields `from`, `to`, `kind`, + `explanation`, `evidence_model`, `excerpt`. All model references exist, endpoints + differ, and each ordered endpoint/kind tuple is unique. + +Group/model IDs contain1–160ASCII letters/digits or `._:/-`, beginning with a +letter/digit. Group labels are at most200characters, model labels/aliases500. +Each model has at most128unique aliases. A card reference has exact fields +`file_id` (64hex), `path` (a relative `notes/` path up to512characters without +empty/dot/dot-dot components), positive integer `revision`, and64hex `sha256`. +These are exact captured note references, never source-reading capabilities. + +Review is `source_reviewed` or `review_pending`. Reviewed summaries have exactly +six nonempty strings, each up to16,384characters: `method`, `purpose`, +`experiment`, `outcome`, `lesson`, `reproduction`. Pending summaries are null. +This status is an author's source-review statement, not numerical qualification. +Relationship kind is `comparison` or `parameter_derivation`; the latter points +from derived model to parameter source and must be acyclic. Explanation is at +most8,000characters and exact excerpt at most16,384. Comparisons are authored +context, not permission to combine incompatible metrics/populations. + +## Ownership and reads + +The ordinary project-file owner captures/revises the note, authenticates reads, +verifies captured UTF-8 bytes/hashes and retains previous revisions. There is no +new canonical family, SQL table, execution grant, recipe or mutation interface. +Existing `files_inspect` and `file_get` contracts are unchanged. Save/load keeps +the original guide and referenced note history. A fork whose new project/note +identities differ requires an explicitly re-authored guide; the reader refuses +foreign identities instead of retargeting them silently. + +`ProjectModelPortfolioDocument` owns the presentation-document decoder and +cross-reference/literal checks. `ProjectModelPortfolio` owns selection, +presentation and its pending reads. Its injected `read` must be the host's +already negotiated, current-authority, byte-verified project-file reader. +The host owns transport/hash validation and context identity. The controller +checks the exact model card ID/path/revision/SHA/project, and that all six +summary strings and that card's relationship excerpts occur literally in its +captured text before displaying a selected summary. Text is inert; no markup, +scripts, parameter loading, scientific scoring or source execution occurs. + +Only explicit actions read notes/cards. Discovery visits40adopted files per +page; no hidden scan/retry/polling occurs. The guide holds at most1,000models, +shows20per graph window with their ancestor groups, and reports exact total, +reviewed and filtered counts. Group/name/alias/review filters, paging and +textual relationship controls select the same entries as graph buttons. +Paragraphs wrap; exact source text remains in a disclosure. + +The host's binding/project/checkpoint/credential/generation context owns every +read. A context change, permission loss or teardown aborts pending work, clears +old content and restores focus to the host status when needed. Late replies +cannot publish into another context. Errors show their declared diagnostic and +clear old detail instead of displaying a previous model as the new selection. +Unsupported format or foreign-project binding is `unsupported`; malformed +document/reference/summary is `invalid`; transport/availability failures retain +their owning diagnostic. No success or empty-data fallback substitutes for a +failed read. + +## Verification scope + +Pure codec tests cover contract decisions using literal fixtures. Controlled +read and real-browser tests cover selection, unavailable/obsolete responses, +repeated invocation, focus, keyboard activation and wrapping on desktop/mobile. +Those fixtures do not establish real project capture or save/load. Native +acceptance additionally requires an actual adopted guide, exact original cards, +current authority, real project reads and retained checkpoint/custody evidence. diff --git a/docs/runbooks/project-composition.md b/docs/runbooks/project-composition.md new file mode 100644 index 00000000..76757fd6 --- /dev/null +++ b/docs/runbooks/project-composition.md @@ -0,0 +1,54 @@ +# Compose retained projects + +The owner operation `project_files.compose` creates one independent project from +an open editable primary and one or more explicit retained project checkpoints. +It preserves the original scientific evidence and forks the primary editable +notes, data and definitions through the same owners as Save As. Secondary drafts +and operation receipts remain inert historical sources. No source project is +modified and no imported program, runtime or credential becomes active. + +Negotiate the operation's version-1 contract through the native project owner +client. The route is `POST /api/workspace/project-files/compose`. In addition to +the current `binding`, `expected_checkpoint` and one fresh `request_key`, supply: + +- `root`, `directory` and `name` for a new destination under a configured root; +- `sources`, an ordered list of additional selections, each containing `root`, + `directory`, `project_id` and `checkpoint`. + +The current project is implicitly the first source. At most 15 additional +sources are accepted by the HTTP contract. All source projects must be idle and +available for native exclusive store ownership. The primary's captured working +files must be clean. The destination cannot contain, lie inside or alias any +source, and existing unrelated destinations are never overwritten. + +Equal canonical envelopes share one retained representation. Same-ID conflicts +are rejected except for Run documents that differ only in their historical +approval grant/consumption while retaining the same admission and every other +field. The complete alternate document is retained as a native Run observation; +it does not become a second Run or create a disclosure. Both documents remain +subject to the history owner's proof and payload validation. + +The persisted `project_composition_v1` record binds the ordered source identities +and exact canonical envelope hashes to the destination's ordinary fork overlay. +A source membership explicitly identifies any alternate Run observation needed +to recover that source's document. Its decoder rejects missing, altered or +contradictory memberships. Subsequent ordinary forks preserve and archive this +provenance. Older software that does not support this family must reject the +project explicitly; records must not be dropped for compatibility. + +Payloads are verified and copied as independent ordinary files. A missing copy +can be supplied by another source with the same verified declaration; a corrupt +selected source is rejected. Unique bytes cannot be omitted. Only dependencies +classified as regenerable by their domain owners can remain explicitly missing. + +On success, `state` describes the newly selected project, `fork` reports the +primary origin and copy completeness, and `sources` lists every selected source +identity/checkpoint in order. The local owner installation retains its own +credentials. Existing Save As and lifecycle-state contracts are unchanged. + +Retain a request before dispatch. An interrupted publication can leave an inert +stage or a complete unselected destination. Inspect the original request and +owner state before reconciling with the same key and arguments. Native receipts +bind reconciliation to the complete ordered source set. A timeout does not +justify another request key or deleting a retained stage. Selection failure +returns a durable unselected receipt while preserving the current primary. diff --git a/project_artifacts.py b/project_artifacts.py index 743ba5ce..fe2a0ee0 100644 --- a/project_artifacts.py +++ b/project_artifacts.py @@ -247,12 +247,34 @@ def status(self, declaration): self._ensure_open() try: with self.store.root_directory() as root: - try: - with self._source(root, declaration) as source: - return _observe(source, declaration) - except _Missing: - return dict(state='missing', expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], - observed_sha256=None, observed_bytes=None) + return self._status_at(root, declaration) + except OSError as error: + _io(error) + + def _status_at(self, root, declaration): + try: + with self._source(root, declaration) as source: + return _observe(source, declaration) + except _Missing: + return dict(state='missing', expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], + observed_sha256=None, observed_bytes=None) + + def status_many(self, declarations): + """Hash each explicit artifact under one before/after-verified root. + + Unlike presence inspection, these observations verify present payload + hashes. An unchanged canonical closure is proved for the entire group; + every individual file retains its anchored source-location checks. + """ + if type(declarations) is not list: + _fail('Artifact status groups must be explicit finite lists.') + declarations = [validate_declaration(value) for value in declarations] + with self._mutex: + self._ensure_open() + try: + with self.store.root_directory(verify_after=True) as root: + result = [self._status_at(root, value) for value in declarations] + return result except OSError as error: _io(error) @@ -327,36 +349,74 @@ def _snapshot(self, declaration): snapshot = None try: with self.store.root_directory() as root: - local = staging = None - try: - with self._source(root, declaration) as source: - local = _directory(root, '.tuldok-local', create=True) - staging = _directory(local, 'artifact-snapshots', create=True) - temporary = '.snapshot-' + uuid.uuid4().hex + '.tmp' - output = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, - 0o600, dir_fd=staging) - try: - _verified(_observe(source, declaration, output=output)) - os.fchmod(output, 0o400) - os.fsync(output) - finally: - os.close(output) - snapshot = os.open(temporary, FILE_FLAGS, dir_fd=staging) - _info(snapshot, rooted=True) - os.unlink(temporary, dir_fd=staging) - except _Missing as error: - raise ProjectArtifactError('Required artifact bytes are missing.', 'unavailable', 404) from error - finally: - if staging is not None: - os.close(staging) - if local is not None: - os.close(local) + snapshot = self._snapshot_at(root, declaration) return snapshot except BaseException: if snapshot is not None: os.close(snapshot) raise + def _snapshot_at(self, root, declaration): + snapshot = None + try: + local = staging = None + try: + with self._source(root, declaration) as source: + local = _directory(root, '.tuldok-local', create=True) + staging = _directory(local, 'artifact-snapshots', create=True) + temporary = '.snapshot-' + uuid.uuid4().hex + '.tmp' + output = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, dir_fd=staging) + try: + _verified(_observe(source, declaration, output=output)) + os.fchmod(output, 0o400) + os.fsync(output) + finally: + os.close(output) + snapshot = os.open(temporary, FILE_FLAGS, dir_fd=staging) + _info(snapshot, rooted=True) + os.unlink(temporary, dir_fd=staging) + except _Missing as error: + raise ProjectArtifactError('Required artifact bytes are missing.', 'unavailable', 404) from error + finally: + if staging is not None: + os.close(staging) + if local is not None: + os.close(local) + return snapshot + except BaseException: + if snapshot is not None: + os.close(snapshot) + raise + + def copy_many_to(self, destination, declarations): + """Copy a finite inventory with one before/after-verified source root. + + Each payload uses its own verified unlinked snapshot and ordinary atomic + destination publication. Only one temporary payload is retained at once. + A later failure can leave earlier verified destination blobs inert; the + caller owns publication of the complete destination project. + """ + if not isinstance(destination, ProjectArtifacts) or destination is self: + _fail('Artifact copying requires a distinct destination owner.') + if type(declarations) is not list: + _fail('Artifact copy groups must be explicit finite lists.') + declarations = [validate_declaration(value) for value in declarations] + first, second = sorted((self, destination), key=id) + with first._mutex, second._mutex: + self._ensure_open() + destination._ensure_open() + try: + with self.store.root_directory(verify_after=True) as root: + for declaration in declarations: + snapshot = self._snapshot_at(root, declaration) + try: + destination.publish_fd(snapshot, declaration, maximum_bytes=declaration['bytes']) + finally: + os.close(snapshot) + except OSError as error: + _io(error) + @contextmanager def open_verified(self, declaration, *, maximum_bytes=None): """Borrow a verified unlinked read-only snapshot, not a mutable source fd.""" diff --git a/project_composition.py b/project_composition.py new file mode 100644 index 00000000..53bc04f3 --- /dev/null +++ b/project_composition.py @@ -0,0 +1,303 @@ +"""Pure retained-source assembly for native independent project composition. + +Sources are ordered: the primary supplies the base envelope for an overlapping +identity. Exact duplicates share custody. Only admission-proof grant/consumption +variants of otherwise byte-identical Runs can coexist as native Run observations. +No source root, catalog permission, runtime or editable overlay is activated here. +The lifecycle owner must validate each source and the complete final candidate, +clone primary edits and publish all required bytes through the native owners. +""" +import hashlib + +from project_fork import ARCHIVE_FAMILIES, _sha, _uuid, decode_fork +from project_history import capture_run_observation +from project_store import (MAX_RECORDS, MAX_TOTAL_BYTES, ProjectStoreError, + _encode, _records) + + +class ProjectCompositionError(ProjectStoreError): + pass + + +def _fail(message, code='invalid', status=400): + raise ProjectCompositionError(message, code, status) + + +def _record_sha(record): + return hashlib.sha256(_encode(record)).hexdigest() + + +def _approval_variant(original, alternate): + if original['family'] != 'run_history': + return False + left, right = original['document'], alternate['document'] + if _encode({k:v for k,v in left.items() if k != 'admission_proof'}) != _encode( + {k:v for k,v in right.items() if k != 'admission_proof'}): + return False + left, right = left.get('admission_proof'), right.get('admission_proof') + fields = {'grant', 'consumption', 'admission'} + return (type(left) is dict and type(right) is dict and set(left) == fields and set(right) == fields + and _encode(left['admission']) == _encode(right['admission'])) + + +def merge_retained_sources(snapshots): + """Return owned originals, exact source memberships and inactive dispositions. + + This internal construction step is not a publication or a fresh domain + admission boundary. Each source must already have passed its domain owners. + Store shape/size checks are retained here. Sources may be supplied lazily + to keep one temporary source snapshot in memory at a time. Same-ID conflicts outside the one + explicit Run-proof case are rejected; nothing is renamed or overwritten. + Membership digests cover complete canonical envelopes, not Python equality. + """ + try: + snapshots = iter(snapshots) + except TypeError: + _fail('Composition requires explicit complete source snapshots.') + projects, indexed, hashes, sizes = set(), {}, {}, {} + memberships, archived = [], set() + for snapshot in snapshots: + if type(snapshot) is not dict or set(snapshot) != {'project_id','name','checkpoint','records'}: + _fail('Composition requires complete canonical source snapshots.') + _uuid(snapshot['project_id']); _sha(snapshot['checkpoint']) + if snapshot['project_id'] in projects: + _fail('Composition source project identity repeats.', 'conflict', 409) + projects.add(snapshot['project_id']) + if type(snapshot['name']) is not str or not snapshot['name'].strip() or len(snapshot['name']) > 200: + _fail('Composition source name differs from the project contract.') + previous = decode_fork(snapshot['records'], project_id=snapshot['project_id']) + if previous is not None: + archived.update((item['family'], item['id']) for item in previous['archived']) + membership = dict(project_id=snapshot['project_id'], name=snapshot['name'], + checkpoint=snapshot['checkpoint'], records={}, run_observations={}) + if len(snapshot['records']) > MAX_RECORDS: + _fail('Composition source exceeds the project record budget.', 'too_large', 413) + source_bytes = 0 + for original in snapshot['records']: + # Validate/copy one envelope at a time. Holding another complete + # source-sized serialized/deep-copy list is unnecessary for custody. + family, identity, document, raw = _records([original])[0] + source_bytes += len(raw) + if source_bytes > MAX_TOTAL_BYTES: + _fail('Composition source exceeds the project byte budget.', 'too_large', 413) + record = dict(family=family, id=identity, document=document) + key = family, identity + sha = _record_sha(record) + membership['records'].setdefault(family, {})[identity] = sha + if family in ARCHIVE_FAMILIES: + archived.add(key) + if key not in indexed: + indexed[key], hashes[key], sizes[key] = record, sha, len(raw) + elif hashes[key] != sha: + if not _approval_variant(indexed[key], record): + _fail('Composition has conflicting retained envelopes: '+family+'/'+identity+'.', 'conflict', 409) + observation = capture_run_observation(document) + observation_key = observation['family'], observation['id'] + observation_sha = _record_sha(observation) + if observation_key in indexed and hashes[observation_key] != observation_sha: + _fail('Composition Run observation identity conflicts.', 'conflict', 409) + indexed[observation_key], hashes[observation_key] = observation, observation_sha + sizes[observation_key] = len(_records([observation])[0][3]) + membership['run_observations'][identity] = observation['id'] + memberships.append(membership) + del snapshot + if len(projects) < 2: + _fail('Composition requires at least two explicit complete source snapshots.') + if len(indexed) > MAX_RECORDS or sum(sizes.values()) > MAX_TOTAL_BYTES: + _fail('Composed retained sources exceed the project record budget.', 'too_large', 413) + return dict(records=[indexed[key] for key in sorted(indexed)], sources=memberships, + archived=[dict(family=family,id=identity) for family,identity in sorted(archived)]) + + +FAMILY = 'project_composition' +FORMAT = 'project_composition_v1' + + +def composition_input(sources, directory, name, request_key): + return dict(format='project_composition_request_v1', + sources=[dict(project_id=s['project_id'],checkpoint=s['checkpoint']) for s in sources], + directory=directory,name=name,request_key=request_key) + + +def _request_digest(sources, directory, name, request_key): + return hashlib.sha256(_encode(composition_input(sources,directory,name,request_key))).hexdigest() + + +def _shape(value, fields, label): + if type(value) is not dict or set(value) != set(fields): + _fail('Invalid composition '+label+' fields.') + + +def _decode_memberships(document, indexed): + from project_store import REQUEST_KEY, _identifier, _relative_parts + _shape(document,('format','destination_project_id','directory','name','request_key','request_sha256','sources'),'provenance') + if document['format'] != FORMAT: + _fail('Unsupported composition provenance format.','unsupported',422) + _uuid(document['destination_project_id']);_relative_parts(document['directory']) + if type(document['name']) is not str or not document['name'].strip() or len(document['name'])>200: + _fail('Invalid composition name.') + if type(document['request_key']) is not str or not REQUEST_KEY.fullmatch(document['request_key']): + _fail('Invalid composition request key.') + _sha(document['request_sha256']) + sources=document['sources'] + if type(sources) is not list or len(sources)<2: + _fail('Composition provenance needs at least two sources.') + seen=set() + for source in sources: + _shape(source,('project_id','name','checkpoint','records','run_observations'),'source membership') + _uuid(source['project_id']);_sha(source['checkpoint']) + if source['project_id'] in seen or source['project_id']==document['destination_project_id']: + _fail('Composition source identities repeat or alias the destination.') + seen.add(source['project_id']) + if type(source['name']) is not str or not source['name'].strip() or len(source['name'])>200: + _fail('Invalid composition source name.') + if type(source['records']) is not dict or type(source['run_observations']) is not dict: + _fail('Composition membership must use explicit inventories.') + runs=source['records'].get('run_history',{}) + if type(runs) is not dict:_fail('Composition Run membership must be an object.') + for identity,sha in source['run_observations'].items(): + _identifier(identity,'Composition original Run ID');_sha(sha) + if not set(source['run_observations'])<=set(runs): + _fail('Composition has an observation mapping without an original Run.') + for family,members in source['records'].items(): + _identifier(family,'Composition member family') + if type(members) is not dict or not members: + _fail('Composition family membership must be a nonempty object.') + for identity,sha in members.items(): + _identifier(identity,'Composition member ID');_sha(sha) + alternate=source['run_observations'].get(identity) if family=='run_history' else None + if alternate is None: + retained=indexed.get((family,identity)) + if retained is None:_fail('Composition source member is missing.','unavailable',409) + else: + from project_history import _observation_document + _sha(alternate) + wrapped=indexed.get(('run_observation',alternate)) + if wrapped is None:_fail('Composition source observation is missing.','unavailable',409) + observed=_observation_document(wrapped) + retained=dict(family='run_history',id=identity,document=observed['observation']) + base=indexed.get(('run_history',identity)) + if (base is None or not _approval_variant(base,retained) + or _record_sha(base)==_record_sha(retained)): + _fail('Composition alternate observation is not an admission-proof-only variant.') + if _record_sha(retained)!=sha: + _fail('Composition source membership differs from its exact retained envelope.') + if document['request_sha256']!=_request_digest(sources,document['directory'],document['name'],document['request_key']): + _fail('Composition request receipt differs from its exact sources.') + + +def decode_records(records, *, project_id=None): + """Validate inert source provenance and its independently owned fork binding.""" + from project_fork import _decode_document + indexed={(r['family'],r['id']):r for r in records} + if len(indexed)!=len(records):_fail('Composition record identity repeats.') + result={} + for record in records: + if record['family']!=FAMILY:continue + document=record['document'];_decode_memberships(document,indexed) + if record['id']!=document['destination_project_id']: + _fail('Composition envelope belongs to a different destination.') + fork=indexed.get(('project_fork',record['id'])) + if fork is None:_fail('Composition has no native fork overlay.','unavailable',409) + _decode_document(fork['document'],record['id'],indexed) + overlay=fork['document'];primary=document['sources'][0] + if (overlay['source']!={key:primary[key] for key in ('project_id','checkpoint')} + or any(overlay[key]!=document[key] for key in ('directory','name','request_key'))): + _fail('Composition provenance differs from its primary fork binding.') + archived={(r['family'],r['id']) for r in overlay['archived']} + required={(family,identity) for source in document['sources'] for family,members in source['records'].items() + for identity in members if family in ARCHIVE_FAMILIES} + if not required<=archived: + _fail('Composition must archive all source editable values and operation receipts.') + result[record['id']]=document + if project_id is not None: + current=decode_fork(records,project_id=project_id) + archived=set() if current is None else {(r['family'],r['id']) for r in current['archived']} + if any(identity!=project_id and (FAMILY,identity) not in archived for identity in result): + _fail('Prior composition provenance must remain explicitly archived.') + return result + + +def validate_source_memberships(records, snapshots, destination_project_id): + """Bind a constructed candidate to the actual owner-selected complete inputs.""" + document=next((r['document'] for r in records if r['family']==FAMILY and r['id']==destination_project_id),None) + if document is None:_fail('Composition candidate has no destination provenance.') + if len(document['sources'])!=len(snapshots):_fail('Composition candidate source count differs.') + for membership,snapshot in zip(document['sources'],snapshots): + if any(membership[key]!=snapshot[key] for key in ('project_id','name','checkpoint')): + _fail('Composition candidate belongs to different source identities.') + expected={} + for record in snapshot['records']: + expected.setdefault(record['family'],{})[record['id']]=_record_sha(record) + if membership['records']!=expected: + _fail('Composition candidate does not preserve the complete source membership.') + _decode_memberships(document,{(r['family'],r['id']):r for r in records}) + + +def build_candidate(snapshots, destination_project_id, *, created_at, id_factory, + directory, name, request_key): + """Preserve all sources and independently fork only the primary editable state.""" + import copy + from project_fork_composition import build_candidate as fork_candidate, artifact_plan + from project_fork import ID + merged=merge_retained_sources(snapshots) + _uuid(destination_project_id) + used={s['project_id'] for s in snapshots} + if destination_project_id in used:_fail('Composition destination must have a fresh identity.') + used.update(r['id'] for r in merged['records'] if ID.fullmatch(r['id'])) + used.update(r['document']['current']['id'] for r in merged['records'] + if r['family'] in ('model_definition','pipeline_graph','recipe_definition')) + used.add(destination_project_id) + def allocate(family,identity,revision): + value=id_factory(family,identity,revision);_uuid(value) + if value in used:_fail('Composition allocator reused a source or allocated identity.') + used.add(value);return value + fork=fork_candidate(snapshots[0],destination_project_id,created_at=created_at,id_factory=allocate) + indexed={(r['family'],r['id']):r for r in merged['records']} + for record in fork['records']: + key=record['family'],record['id'] + if key in indexed: + if _record_sha(indexed[key])!=_record_sha(record):_fail('Primary fork changed an original envelope.') + else:indexed[key]=record + document=dict(format=FORMAT,destination_project_id=destination_project_id,directory=directory,name=name, + request_key=request_key,request_sha256=_request_digest(merged['sources'],directory,name,request_key), + sources=merged['sources']) + record=dict(family=FAMILY,id=destination_project_id,document=document) + if (FAMILY,destination_project_id) in indexed:_fail('Composition provenance identity collides.') + indexed[(FAMILY,destination_project_id)]=record + records=[indexed[key] for key in sorted(indexed)] + validate_source_memberships(records,snapshots,destination_project_id) + archived={(r['family'],r['id']) for r in merged['archived']+fork['archived']} + # Every input has its own valid current fork. The combined records receive + # their one new current overlay only in the publication owner. Ask each + # source owner for its preservation policy before combining those roots; + # independent editable forks introduce no new payload identities. + plans={} + inventories=[fork['artifacts']] + inventories.extend(artifact_plan(source['records']) for source in snapshots[1:]) + for inventory in inventories: + for edge in inventory: + sha=edge['declaration']['sha256'];previous=plans.get(sha) + if previous is not None and previous['declaration']!=edge['declaration']: + _fail('Composition source artifact declarations conflict.') + if previous is None or edge['preservation']=='unique': + plans[sha]=copy.deepcopy(edge) + return dict(records=records,mapping=copy.deepcopy(fork['mapping']), + archived=[dict(family=f,id=i) for f,i in sorted(archived)], + artifacts=[plans[sha] for sha in sorted(plans)]) + + +def publish_composition(primary_store, primary_artifacts, retained_sources, trusted_root_fd, directory, + *, root_path, name, expected_checkpoint, request_key, quiescent, + created_at, id_factory, validate_candidate): + """Publish through the native fork lifecycle with explicit additional owners.""" + from functools import partial + from project_fork import _publish_project + if type(retained_sources) not in (list,tuple) or not retained_sources: + _fail('Composition requires at least one explicit secondary source owner.') + return _publish_project(primary_store,primary_artifacts,trusted_root_fd,directory, + root_path=root_path,name=name,expected_checkpoint=expected_checkpoint,request_key=request_key, + quiescent=quiescent,retained_sources=retained_sources, + build_candidate=partial(build_candidate,created_at=created_at,id_factory=id_factory, + directory=directory,name=name,request_key=request_key), + validate_candidate=validate_candidate) diff --git a/project_custody.py b/project_custody.py index 2e0a97aa..1a188f1d 100644 --- a/project_custody.py +++ b/project_custody.py @@ -197,7 +197,7 @@ def _directory(self, create=False): finally:os.close(parent) def _binding(self, store, projection_id): - snapshot=store.snapshot() + snapshot=store.summary() with store.root_directory() as descriptor: info=os.fstat(descriptor) return dict(host_catalog_id=self.host_catalog_id,project_id=snapshot['project_id'], @@ -247,7 +247,7 @@ def __init__(self, owner, store, binding): persisted=value is not None if value is None: value=dict(format='project_local_custody_v1',**binding, - project_name=store.snapshot()['name'],selected_checkpoint=store.snapshot()['checkpoint'], + project_name=store.summary()['name'],selected_checkpoint=store.summary()['checkpoint'], attempted=None,runs=[],producers=[],runtime_effects=[],package_effects=[]) if any(value[key]!=item for key,item in binding.items()):_fail('Original physical custody binding differs.') self.value=validate(value) diff --git a/project_data.py b/project_data.py index 4a48dd38..ca70bab0 100644 --- a/project_data.py +++ b/project_data.py @@ -1001,7 +1001,7 @@ def _publish(self, raw, kind, role, media_type): declaration = dict(format='project_artifact_v1', sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) self.artifacts.publish_bytes(raw, declaration) self._pending_artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=self.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=self.session.summary['project_id'], sha256=declaration['sha256']), media_type=media_type, kind=kind, role=role, preservation='unique', required_for=['inspect', 'release']) def _require_record(self, record_id, *, extended=False): @@ -1028,7 +1028,7 @@ def _mutate(self, operation, target, body, expected_checkpoint, key, invoke): fail('Data mutation body must be an object.') self._permission(operation, target, body) with self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() if operation in ('data.annotate','data.correct_rights_note','data.save_image_source','data.delete_image_source'): self._require_record(target,extended=operation!='data.annotate') legacy = self._legacy_replay(operation, target, body, key) @@ -1158,10 +1158,10 @@ def freeze_release(self, body, expected_checkpoint, key): def _read(self, action, invoke, target=None): with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() self._sync_sources() result = invoke() - self.session.check() + self.session.check_current() return result def query(self, options=None): @@ -1399,7 +1399,7 @@ def restore(self, snapshot=None): self.db.execute('INSERT INTO saved_searches VALUES (%s,%s,%s,%s,%s,%s)', (record_id,r['name'],r['revision'],encode_retained(r['criteria']),r['created_at'],r['updated_at'])) elif family == 'dataset_exposure': self.db.execute('INSERT INTO workspace_exposures VALUES (%s,%s,%s,%s,%s,%s,%s)', tuple(doc[k] for k in ('id','record_id','source_hash','reason','run_id','release_id','created_at'))) - self.session.check() + self.session.check_current() self._materialize_images_and_releases(owned) return snapshot diff --git a/project_dispatch.py b/project_dispatch.py index bdbdc250..de519135 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -37,8 +37,8 @@ 'execution_study_prepare_package','execution_study_prepare_verifier','execution_study_abandon_preparation','execution_study_reconcile_preparation')) _RUNTIME_ACTIONS=frozenset(('runtime_sources','runtime_inspect','runtime_capture','runtime_status','runtime_abandon')) _MEASUREMENT_ACTIONS=frozenset(('measurement_import','measurement_sources','measurement_metadata', - 'measurement_subjects','measurement_query','measurement_inspect')) -_M4_ACTIONS=frozenset(('files_inspect','file_get','file_history','file_adopt','file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')) + 'measurement_subjects','measurement_query','measurement_inspect','measurement_compare')) +_M4_ACTIONS=frozenset(('files_inspect','file_get','file_history','file_adopt','files_adopt','file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')) SUPPORTED_OPERATIONS = frozenset('project_files.' + action for action in _DATA_ACTIONS | _MATERIAL_ACTIONS | _HISTORY_ACTIONS | _OTHER_ACTIONS | _EXECUTION_ACTIONS | _M4_ACTIONS | _RUNTIME_ACTIONS | _MEASUREMENT_ACTIONS) @@ -149,7 +149,7 @@ def _data(projection, action, a): def _definitions(projection, family, a): docs = projection.materials._documents() values = [doc['current'] for (kind, _), doc in docs.items() - if kind == family and doc['origin_catalog_id'] == projection.session.snapshot['project_id']] + if kind == family and doc['origin_catalog_id'] == projection.session.summary['project_id']] return _page(sorted(values, key=lambda item: item['id']), a) @@ -365,10 +365,21 @@ def _m4(projection,action,a): session=projection.session if action=='file_adopt': return projection.files.adopt(_body(a),a['expected_checkpoint'],a['request_key']) + if action=='files_adopt': + return projection.files.adopt_many(_body(a),a['expected_checkpoint'],a['request_key']) if action=='file_adopt_deletion': return projection.files.adopt_deletion(_body(a),a['expected_checkpoint'],a['request_key']) with session.access.lock,session.lock,session.db.lock: - session.access.require('inspect','project_files.'+action);snapshot=session.check() + session.access.require('inspect','project_files.'+action) + if action=='fork_info': + snapshot=session.check() + from project_fork import decode_fork + overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) + return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], + mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, + copied_count=len(overlay['artifacts']['copied']) if overlay else 0, + missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) + session.check_current() if action=='files_inspect':return _page(projection.files.inspect()['items'],a) if action=='file_get':return projection.files.read(a['file_id'],a.get('revision')) if action=='file_history': @@ -388,13 +399,6 @@ def _m4(projection,action,a): items=[dict(item,requires_reselection=[]) for item in value['editable']] items.extend(dict(family=item['family'],id=item['id'],requires_reselection=item['fields']) for item in value['requires_reselection']) return _page(sorted(items,key=lambda item:(item['family'],item['id'])),a) - if action=='fork_info': - from project_fork import decode_fork - overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) - return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], - mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, - copied_count=len(overlay['artifacts']['copied']) if overlay else 0, - missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) raise AssertionError('Closed M4 dispatch is incomplete.') @@ -438,6 +442,7 @@ def _measurements(projection,action,a): if action=='measurement_import': return projection.source_adapter.import_measurements(_body(a),a['expected_checkpoint'],a['request_key']) if action=='measurement_sources':return owner.sources(offset=a.get('offset',0),limit=a.get('limit',40)) + if action=='measurement_compare':return owner.compare(a['sources'],**_body(a,'sources')) origin,packet=a['origin_catalog_id'],a['packet_sha256'] if action=='measurement_metadata':return owner.metadata(origin,packet) if action=='measurement_subjects':return owner.subjects(origin,packet,offset=a.get('offset',0),limit=a.get('limit',40)) diff --git a/project_execution.py b/project_execution.py index ca5c36d0..0e4941c6 100644 --- a/project_execution.py +++ b/project_execution.py @@ -52,7 +52,7 @@ def publication_started(self): def confirmed(self): if not self.started:raise RuntimeError('File publication must have a durable start before confirmation.') - execution=self.execution;execution.session.check() + execution=self.execution;execution.session.check_current() if execution.db.in_transaction: raise RuntimeError('File publication confirms only after its canonical and outer SQL acknowledgement.') if execution.custody is not None: @@ -164,7 +164,7 @@ def register_custody(self, identity, handle): def before_launch(self, identity): with self.db.lock: - self.session.check();run=self.runs._load(identity);self._proof(run) + self.session.check_current();run=self.runs._load(identity);self._proof(run) self.runtime_access.require_job('runs',identity,run['admission_id'],run['request_sha256']) if self.custody is not None: with self._journal_guard():self.custody.launch_intent(identity) @@ -176,7 +176,7 @@ def _publication_intent(self, before, records, key, sha): def _local_owner(self, *, recover=True): """Construct execution only after the caller checked exact authority.""" - self.session.check() + self.session.check_current() if self._runs is None: from runs import Runs from pipeline import Pipelines @@ -296,7 +296,7 @@ def _restore_file_writer(self, kind, request_key, input_sha256, content_sha256, authority.require(scope,action,owner_only=True) if self.db.in_transaction: raise RuntimeError('File reconciliation requires no outer SQL transaction.') - if self.session.snapshot['checkpoint']!=expected_checkpoint: + if self.session.summary['checkpoint']!=expected_checkpoint: raise WorkbenchError('Expected current file checkpoint changed.','stale_project',409) if self.custody is None: raise WorkbenchError('No original physical file custody exists.','unavailable',409) @@ -398,7 +398,7 @@ def _producer_authority(self, family, identity, admission_id, request_digest): def register_producer(self, family, identity, admission_id, request_digest): with self.db.lock: - self.session.check();self._producer_authority(family,identity,admission_id,request_digest) + self.session.check_current();self._producer_authority(family,identity,admission_id,request_digest) if self.custody is not None: with self._journal_guard(): if not self.custody.has_pending:self.custody.acknowledge(self.session.snapshot) @@ -406,7 +406,7 @@ def register_producer(self, family, identity, admission_id, request_digest): def producer_effect_intent(self, family, identity): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -416,7 +416,7 @@ def producer_effect_intent(self, family, identity): def update_producer_outcome(self, family, identity, provider_outcome, worker_drained): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -464,7 +464,7 @@ def _authorize_observation(self): # The domain owner has already checked submit/cancel authority; # recheck live host credentials without inventing a worker owner. self.runtime_access.require('inspect', 'runs.observe', identity) - if proof['admission']['catalog_id'] != self.session.snapshot['project_id']: + if proof['admission']['catalog_id'] != self.session.summary['project_id']: raise AccessError('Local custody belongs to another project.') def _capture_run(self, identity): @@ -591,7 +591,7 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt def preflight_external(self, graph_id, revision): self.runtime_access.require('inspect', 'pipeline.preflight_external', graph_id) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() return self.projection.pipelines.preflight_external(graph_id, revision) def native_preflight(self,graph_id,revision,expected_checkpoint,key): @@ -615,7 +615,7 @@ def native_submit(self,body,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.submit',body['graph_id'],owner_only=True) effective=dict(body,actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.submit',body['graph_id'],effective,expected_checkpoint,key) if replay is not None:return replay with self.projection.native_preparation.current_graph(self.projection.pipelines,body['graph_id'], @@ -626,14 +626,14 @@ def invoke(): context.consume_admission() owner=self._local_owner();public=owner.submit(dict(body,request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.submit',body['graph_id'],effective,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.submit',body['graph_id'],effective,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_retry(self,run_id,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.retry',run_id,owner_only=True) body=dict(actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.retry',run_id,body,expected_checkpoint,key) if replay is not None:return replay if self._runs is None:raise WorkbenchError('Retry requires the actual current local Run owner.','unavailable',409) @@ -646,7 +646,7 @@ def invoke(): context.consume_admission() owner=self.runs;public=owner.retry(run_id,dict(request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.retry',run_id,body,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.retry',run_id,body,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_reconcile(self,phase_id,expected_checkpoint): @@ -655,9 +655,9 @@ def native_reconcile(self,phase_id,expected_checkpoint): def submit_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require_external_submission() with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: # A retained request never recreates a local Run/admission or wakes @@ -678,9 +678,9 @@ def invoke(): def approve_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require('approve-execution', 'runs.approve_external', owner_only=True) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: value = prior['result'] diff --git a/project_extraction.py b/project_extraction.py index 79872c6d..461b0d0f 100644 --- a/project_extraction.py +++ b/project_extraction.py @@ -18,7 +18,7 @@ import project_material as material_codec import project_history as history_codec import project_producers as producer_codec -from projects import decode_retained_snapshot, manifest, decode as decode_manifest, relative_path +from projects import decode_retained_snapshot, decode_retained_manifest, decode as decode_manifest, relative_path from project_files import file_id, MAX_FILE_BYTES MAX_PAYLOAD = 40 * 1024 * 1024 @@ -60,6 +60,8 @@ def __init__(self, source): self.origin = data_codec.identity(self.db.dataset_id) self.records, self.inventory, self.dependencies = {}, {}, {} self.blockers, self.warnings, self.working_files = [], [], [] + # Preserve emitted order without scanning prior runtime diagnostics. + self._issue_keys = {False: set(), True: set()} self.anchor = None self.queue, self.visited, self.queued = deque(), set(), set() self.snapshot_by_id = {} @@ -67,7 +69,7 @@ def __init__(self, source): self.payload_gaps = [] self.deleted_intents = set() self.session = SimpleNamespace(snapshot=source.snapshot, db=self.db, - artifacts=source.artifacts, check=lambda: None) + artifacts=source.artifacts, check_current=lambda: None) # Workbench and each metadata-view constructor only assign references; # none runs SQL, creates directories or constructs Dataset/Jobs/Runs. # Never call get/history: those public owners may enroll images. @@ -121,10 +123,13 @@ def need(self, family, identity, parent=None, reason='required scientific depend return target def issue(self, key, code, detail, *, warning=False): - value = dict(family=key[0], id=key[1], code=code, detail=str(detail)[:1000]) - destination = self.warnings if warning else self.blockers - if value not in destination: - destination.append(value) + detail = str(detail)[:1000] + identity = key[0], key[1], code, detail + seen = self._issue_keys[bool(warning)] + if identity not in seen: + seen.add(identity) + destination = self.warnings if warning else self.blockers + destination.append(dict(family=key[0], id=key[1], code=code, detail=detail)) def add(self, record): key = record['family'], record['id'] @@ -374,11 +379,15 @@ def capture_snapshot(self, sha, key): if len(matches) != 1: _missing('An exact selected project snapshot is not available in this pinned source.') saved = matches[0] - root = self.source.roots.get(saved['root']) - if root is None or list(root[1]) != saved['root_identity']: - _missing('The retained registration has no matching current explicit source-root capability.') - raw = self.source.read_root(saved['root'], saved['manifest_path'], 65536) - if hashlib.sha256(raw).hexdigest() != sha or manifest(decode_manifest(raw)) != saved['manifest']: + # Explicitly recovered bytes retain the original registration as inert + # history. Its former directory/inode is not current read authority. + raw = self.source.read_bound_payload(sha, maximum_bytes=65536) + if raw is None: + root = self.source.roots.get(saved['root']) + if root is None or list(root[1]) != saved['root_identity']: + _missing('The retained registration has no matching current explicit source-root capability.') + raw = self.source.read_root(saved['root'], saved['manifest_path'], 65536) + if hashlib.sha256(raw).hexdigest() != sha or decode_retained_manifest(decode_manifest(raw)) != saved['manifest']: raise WorkbenchError('Original registered manifest bytes differ.', 'corrupt_payload', 409) blob = self.raw(key, raw, 'project_manifest') record = dict(family='project_snapshot', id=key[1], document=dict(format=material_codec.FAMILIES['project_snapshot'], @@ -751,8 +760,7 @@ def validate(self): if self.blockers:return try: data_codec.decode_data_records(records) - material_codec.decode_records(records) - materials = material_codec.ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=self.origin, records=records), check=lambda: None)) + materials = material_codec.MaterialSnapshot(records) history_codec.decode_records(records, materials=materials) producer_codec.decode_records(records, artifacts=self.source.artifacts) except WorkbenchError as error: diff --git a/project_files.py b/project_files.py index a1de61fc..ad0f6b3f 100644 --- a/project_files.py +++ b/project_files.py @@ -5,6 +5,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import errno import hashlib import os @@ -18,6 +19,8 @@ DELETION_FORMAT='project_file_v2' MAX_FILE_BYTES=1024*1024 MAX_FILES=1000 +MAX_ADOPT_ITEMS=40 +MAX_ADOPT_BYTES=8*1024*1024 KINDS={'script':'scripts','note':'notes'} VERSION_FIELDS={'revision','sha256','bytes','text','adopted_at','actor_id'} RESULT_FIELDS={'id','project_id','kind','path','revision','sha256','bytes','adopted_at','changed'} @@ -126,7 +129,21 @@ def _result(envelope,changed): return result -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): + if operation=='files.adopt_many': + closed(result,{'items','changed_count'},'batch adoption result') + if type(result['items']) is not list or not 1<=len(result['items'])<=MAX_ADOPT_ITEMS: + fail('Invalid batch adoption result size.') + seen=set();projects=set() + for item in result['items']: + _validate_result('files.adopt',item,lookup) + if item['id'] in seen:fail('Batch adoption result repeats a file.') + seen.add(item['id']);projects.add(item['project_id']) + if (len(projects)!=1 or type(result['changed_count']) is not int + or result['changed_count']!=sum(item['changed'] for item in result['items']) + or sum(item['bytes'] for item in result['items'])>MAX_ADOPT_BYTES): + fail('Batch adoption result count or project differs.') + return True if operation not in ('files.adopt','files.adopt_deletion'):return False deleted=operation=='files.adopt_deletion' closed(result,RESULT_FIELDS|{'deleted'} if deleted else RESULT_FIELDS,'adoption result') @@ -137,14 +154,34 @@ def validate_operation_result(operation,result,records=None): else:sha(result['sha256']) _date(result['adopted_at']) if type(result['revision']) is not int or not 1<=result['revision']<=2**53-1 or type(result['bytes']) is not int or not 0<=result['bytes']<=MAX_FILE_BYTES:fail('Invalid adoption result bounds.') - if records is not None: - owned=decode_records(records) - envelope=next((r for r in owned if r['id']==result['id']),None) + if lookup is not None: + envelope=lookup(result['id']) if envelope is None or not any((value.get('deleted') is True)==deleted and all(value[key]==result[key] for key in ('revision','sha256','bytes','adopted_at')) for value in envelope['document']['history']): fail('Adoption result is not an exact retained working-file revision.') return True +def receipt_validator(records): + """Own one decoded candidate's file histories for its receipt checks. + + The returned validator has no live Session or publication authority. It + owns copied records, so caller mutation cannot change this fixed view. + A different candidate requires a new validator. + """ + owned = {record['id']: record for record in decode_records(records)} + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + # Keep standalone shape/error ordering, and share only this call's lookup + # across batch items. There is no retained cache between invocations. + @cache + def owned(): + return {record['id']: record for record in decode_records(records)} + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class _WorkingError(Exception): def __init__(self,state):self.state=state @@ -215,39 +252,51 @@ def _read(root,parts): os.close(parent) +def inspect_captured(records, project_id, root): + """Observe captured working files using only a locked project capability.""" + return _inspect_validated(decode_records(records), project_id, root) + + +def _inspect_validated(records, project_id, root): + """Consume full-decoder output or fresh values from a validated Session.""" + owned=[r for r in records if r['document']['project_id']==project_id] + items=[] + for envelope in sorted(owned,key=lambda r:r['document']['path']): + doc=envelope['document'];value=doc['current'] + observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], + expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) + try: + parts=_path(doc['kind'],doc['path']) + if value.get('deleted') is True and _missing(root,parts): + observation['state']='deleted' + else: + if value.get('deleted') is True:observation['state']='changed' + _,observed,size=_read(root,parts) + observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') + except _WorkingError as error: + if value.get('deleted') is True: + observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) + else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) + items.append(observation) + return items + + class ProjectFiles: def __init__(self,session): self.session,self.access,self.db,self.store=session,session.access,session.db,session.store def inspect(self): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.inspect');snapshot=self.session.check() - owned=[r for r in decode_records(snapshot['records']) if r['document']['project_id']==snapshot['project_id']];items=[] + self.access.require('inspect','files.inspect');snapshot=self.session.check_records(FAMILY) with self.store.root_directory() as root: - for envelope in sorted(owned,key=lambda r:r['document']['path']): - doc=envelope['document'];value=doc['current'] - observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], - expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) - try: - parts=_path(doc['kind'],doc['path']) - if value.get('deleted') is True and _missing(root,parts): - observation['state']='deleted' - else: - if value.get('deleted') is True:observation['state']='changed' - _,observed,size=_read(root,parts) - observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') - except _WorkingError as error: - if value.get('deleted') is True: - observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) - else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) - items.append(observation) - self.session.check() + items=_inspect_validated(snapshot['records'],snapshot['project_id'],root) + self.session.check_current() return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) def read(self,record_id,revision=None): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.read',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.read',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) values=envelope['document']['history'] if revision is None:value=values[-1] @@ -257,8 +306,8 @@ def read(self,record_id,revision=None): def history(self,record_id): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.history',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.history',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) return copy.deepcopy(envelope['document']['history']) @@ -275,38 +324,86 @@ def require_clean(self): def materialize_captured(records,project_id,root_fd): return materialize_captured(records,project_id,root_fd) - def adopt(self,body,expected_checkpoint,key): + def _adoption_intent(self,body): closed(body,{'kind','path','expected_revision','expected_sha256'},'working-file adoption') - parts=_path(body['kind'],body['path']);project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) + parts=_path(body['kind'],body['path']);project_id=self.session.summary['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 0<=body['expected_revision']<=2**53-1:fail('Invalid expected adoption revision.') + return dict(body),parts,record_id + + def _capture_adoption(self,intent,prior,root): + body,parts,record_id=intent + revision=prior['document']['current']['revision'] if prior else 0 + if revision!=body['expected_revision']:fail('Working-file adoption revision changed.','conflict',409) + try:text,observed,size=_read(root,parts) + except _WorkingError as error:fail('Working file is '+error.state+'.','unavailable' if error.state=='missing' else 'invalid',404 if error.state=='missing' else 400) + if observed!=body['expected_sha256']:fail('Working file changed after review.','conflict',409) + if prior is not None and observed==prior['document']['current']['sha256']:return _result(prior,False),None,size + actor=self.access.require('edit','files.adopt',record_id) + value=dict(revision=revision+1,sha256=observed,bytes=size,text=text,adopted_at=timestamp(),actor_id=actor.actor_id) + history=copy.deepcopy(prior['document']['history']) if prior else [];history.append(value) + envelope=dict(family=FAMILY,id=record_id,document=dict(format=prior['document']['format'] if prior else FORMAT, + project_id=self.session.summary['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), + history=history,origin=copy.deepcopy(prior['document']['origin']) if prior else None)) + return _result(envelope,True),envelope,size + + def adopt(self,body,expected_checkpoint,key): + intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.summary['project_id'] pending=[] with self.access.lock: self.access.require('edit','files.adopt',record_id) def invoke(): snapshot=self.session.snapshot;owned=[r for r in decode_records(snapshot['records']) if r['document']['project_id']==project_id] prior=next((r for r in owned if r['id']==record_id),None) - revision=prior['document']['current']['revision'] if prior else 0 - if revision!=body['expected_revision']:fail('Working-file adoption revision changed.','conflict',409) if prior is None and len(owned)>=MAX_FILES:fail('Project already tracks 1,000 working files.','too_large',413) with self.store.root_directory() as root: - try:text,observed,size=_read(root,parts) - except _WorkingError as error:fail('Working file is '+error.state+'.','unavailable' if error.state=='missing' else 'invalid',404 if error.state=='missing' else 400) - if observed!=body['expected_sha256']:fail('Working file changed after review.','conflict',409) - if prior is not None and observed==prior['document']['current']['sha256']:return _result(prior,False) - actor=self.access.require('edit','files.adopt',record_id) - value=dict(revision=revision+1,sha256=observed,bytes=size,text=text,adopted_at=timestamp(),actor_id=actor.actor_id) - history=copy.deepcopy(prior['document']['history']) if prior else [];history.append(value) - envelope=dict(family=FAMILY,id=record_id,document=dict(format=prior['document']['format'] if prior else FORMAT,project_id=project_id,kind=body['kind'],path=body['path'],current=copy.deepcopy(value),history=history,origin=copy.deepcopy(prior['document']['origin']) if prior else None)) - decode_records([r for r in snapshot['records'] if not (r['family']==FAMILY and r['id']==record_id)]+[envelope]);pending.append(envelope) - return _result(envelope,True) + result,envelope,_=self._capture_adoption(intent,prior,root) + if envelope is not None: + decode_records([r for r in snapshot['records'] if not (r['family']==FAMILY and r['id']==record_id)]+[envelope]);pending.append(envelope) + return result return self.session.mutate('files.adopt',record_id,body,expected_checkpoint,key,invoke, lambda result,before:dict(records=copy.deepcopy(pending),result=copy.deepcopy(result))) + def adopt_many(self,body,expected_checkpoint,key): + """Capture one bounded, all-or-nothing group through ordinary adoption. + + Every file retains its permission/revision/hash and anchored read checks. + One verified root scope and one canonical publication cover the group. + No working file is rewritten, even when another input rejects the group. + """ + closed(body,{'items'},'batch working-file adoption') + if type(body['items']) is not list or not 1<=len(body['items'])<=MAX_ADOPT_ITEMS: + fail('Choose 1 through 40 working files for adoption.','too_large',413) + intents=[self._adoption_intent(item) for item in body['items']] + ids=[intent[2] for intent in intents] + if len(set(ids))!=len(ids):fail('Batch adoption repeats a working-file path.') + body=dict(items=[intent[0] for intent in intents]);project_id=self.session.summary['project_id'] + target=digest('\0'.join(ids).encode());pending=[] + with self.access.lock: + for identity in ids:self.access.require('edit','files.adopt',identity) + def invoke(): + snapshot=self.session.snapshot + owned={r['id']:r for r in decode_records(snapshot['records']) if r['document']['project_id']==project_id} + if len(owned)+sum(identity not in owned for identity in ids)>MAX_FILES: + fail('Project already tracks 1,000 working files.','too_large',413) + results=[];total=0 + with self.store.root_directory(verify_after=True) as root: + for intent in intents: + result,envelope,size=self._capture_adoption(intent,owned.get(intent[2]),root) + total+=size + if total>MAX_ADOPT_BYTES:fail('Batch working-file capture exceeds 8 MiB.','too_large',413) + if envelope is not None:pending.append(envelope) + results.append(result) + changed={r['id'] for r in pending} + decode_records([r for r in snapshot['records'] if not (r['family']==FAMILY and r['id'] in changed)]+pending) + return dict(items=results,changed_count=sum(item['changed'] for item in results)) + return self.session.mutate('files.adopt_many',target,body,expected_checkpoint,key,invoke, + lambda result,before:dict(records=copy.deepcopy(pending),result=copy.deepcopy(result))) + def adopt_deletion(self,body,expected_checkpoint,key): closed(body,{'path','expected_revision','expected_sha256'},'note deletion adoption') parts=_path('note',body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 1<=body['expected_revision']<=2**53-1:fail('Invalid expected note revision.') - project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] + project_id=self.session.summary['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] with self.access.lock: self.access.require('edit','files.adopt_deletion',record_id) def invoke(): diff --git a/project_fork.py b/project_fork.py index ff75fec4..f35b9812 100644 --- a/project_fork.py +++ b/project_fork.py @@ -24,7 +24,7 @@ # formerly active values/receipts become inactive through the fork overlay. ARCHIVE_FAMILIES = frozenset(('workbench_text', 'workbench_record', 'workbench_response', 'workbench_preference', 'saved_selection', 'saved_search', 'model_definition', - 'pipeline_graph', 'recipe_definition', 'project_file', 'project_operation', FAMILY)) + 'pipeline_graph', 'recipe_definition', 'project_file', 'project_operation', 'project_composition', FAMILY)) ID = re.compile(r'^[a-f0-9]{32}$') SHA = re.compile(r'^[a-f0-9]{64}$') @@ -174,10 +174,16 @@ def decode_fork(records, *, project_id=None): return copy.deepcopy(overlays[current]) -def active_records(records, *, project_id=None): +def active_records(records, *, project_id=None, families=None): + """Own selected active copies while validating the complete fork history.""" + if families is not None: + if type(families) is not frozenset: + _fail('Active record families must be an explicit frozenset.') + for family in families:_identifier(family,'Active record family') overlay = decode_fork(records, project_id=project_id) archived = set() if overlay is None else {(item['family'], item['id']) for item in overlay['archived']} - return [copy.deepcopy(item) for item in records if (item['family'], item['id']) not in archived] + return [copy.deepcopy(item) for item in records + if (families is None or item['family'] in families) and (item['family'], item['id']) not in archived] def _inside(candidate_fd, ancestor_fd): @@ -236,189 +242,288 @@ def _result(snapshot, overlay, reconciled): return dict(snapshot=copy.deepcopy(snapshot), fork=copy.deepcopy(overlay), reconciled=reconciled) -def _reconcile_existing(source_store, source, trusted_root_fd, relative, root_path, - expected_input, request_key, digest, validate_candidate): +def _reconcile_existing(sources, trusted_root_fd, relative, root_path, + expected_input, request_key, digest, validate_candidate, *, composed): with ProjectStore.open_at(trusted_root_fd, relative, root_path=root_path) as destination: snapshot = destination.snapshot() overlay = decode_fork(snapshot['records'], project_id=snapshot['project_id']) - if (overlay is None or _input(overlay['source'], overlay['directory'], overlay['name'], overlay['request_key']) != expected_input - or overlay['request_sha256'] != digest): + if composed: + from project_composition import decode_records, composition_input + document = decode_records(snapshot['records'],project_id=snapshot['project_id']).get(snapshot['project_id']) + observed_input = None if document is None else composition_input(document['sources'],document['directory'],document['name'],document['request_key']) + observed_digest = None if document is None else document['request_sha256'] + else: + observed_input = None if overlay is None else _input(overlay['source'],overlay['directory'],overlay['name'],overlay['request_key']) + observed_digest = None if overlay is None else overlay['request_sha256'] + if observed_input != expected_input or observed_digest != digest: _fail('Existing destination does not belong to this exact fork request.', 'conflict', 409) if destination.reconcile(request_key, digest) is None: _fail('Existing destination lacks the canonical fork publication receipt.', 'conflict', 409) - validate_candidate(copy.deepcopy(snapshot['records']), snapshot['project_id']) declarations = _declarations(snapshot['records']) with ProjectArtifacts(destination) as artifacts: + validate_candidate(copy.deepcopy(snapshot['records']),snapshot['project_id'],artifacts) for identity in overlay['artifacts']['copied']: if artifacts.status(declarations[identity])['state'] != 'verified': _fail('Previously copied fork payloads are missing or corrupt.', 'unavailable', 409) - _source_current(source_store, source) - # Rereading reconciles identity; this barrier resolves a lost parent - # fsync acknowledgement without guessing durability from mere presence. + for item in sources: + _source_current(item['store'],item['identity']) destination.save(snapshot['checkpoint']) - return _result(destination.snapshot(), overlay, True) + result = _result(destination.snapshot(),overlay,True) + if composed:result['composition']=copy.deepcopy(document) + return result def publish_fork(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, name, expected_checkpoint, request_key, quiescent, build_candidate, validate_candidate): - """Publish a complete independent directory while preserving the source. - - Caller must hold its lifecycle/authorization guard and prove quiescence. - build_candidate(source_snapshot, destination_project_id) is one pure owned - composition callback returning exactly records/mapping/archived/artifacts. - It preserves every original envelope and appends fresh domain-constructed - values. validate_candidate(records, project_id) owns semantic closure. - No temporary or final destination is deleted after failed/unknown outcomes. + """Publish an independent single-source fork through the shared lifecycle.""" + if not callable(validate_candidate): + _fail('Fork requires complete validation.','unavailable',503) + return _publish_project(source_store,source_artifacts,trusted_root_fd,relative, + root_path=root_path,name=name,expected_checkpoint=expected_checkpoint,request_key=request_key, + quiescent=quiescent,build_candidate=build_candidate, + validate_candidate=lambda records,identity,_artifacts:validate_candidate(records,identity)) + + +def recover_committed_publication(trusted_root_fd, staging, relative, *, root_path, + expected_project_id, expected_checkpoint, + expected_request, quiescent): + """Publish one explicitly selected committed stage without rebuilding it. + + This operator recovery accepts a frozen historical snapshot, not a new read + of its original sources. The caller owns current filesystem authority and + cessation of the former process. The store's exclusive root lock, complete + domain validation, exact original request/commit receipt, payload custody, + and clean captured files are required before a no-overwrite rename. No SQL + projection, source permission, source runtime or worker is restored here. + After an uncertain rename, repeat this exact recovery to inspect the final + destination; never repeat composition or create another stage. + """ + if quiescent is not True: + _fail('Recovery requires proven cessation of the former publication owner.', 'busy', 409) + _uuid(expected_project_id);_sha(expected_checkpoint) + stage_parts=_relative_parts(staging);parts=_relative_parts(relative) + if (stage_parts[:-1]!=parts[:-1] + or not re.fullmatch(r'\.tuldok-fork-[a-f0-9]{32}',stage_parts[-1]) + or stage_parts==parts): + _fail('Recovery requires an explicit fork stage beside its original destination.') + if type(expected_request) is not dict or expected_request.get('directory')!=relative: + _fail('Recovery requires the exact original publication request.') + key=expected_request.get('request_key') + if type(key) is not str or not REQUEST_KEY.fullmatch(key): + _fail('Recovery requires the original publication request key.') + digest=_digest(expected_request) + published=False + try: + with anchored_directory(trusted_root_fd,'/'.join(parts[:-1]),root_path=root_path) as parent: + def exists(name): + try:os.stat(name,dir_fd=parent,follow_symlinks=False);return True + except FileNotFoundError:return False + stage_exists,final_exists=exists(stage_parts[-1]),exists(parts[-1]) + if stage_exists and final_exists: + _fail('Both stage and destination exist; neither was replaced.', 'conflict', 409) + if not stage_exists and not final_exists: + _fail('Neither the selected stage nor its destination is available.', 'unavailable', 409) + selected=relative if final_exists else staging + with ProjectStore.open_at(trusted_root_fd,selected,root_path=root_path) as store: + snapshot=store.snapshot() + if (snapshot['project_id']!=expected_project_id + or snapshot['checkpoint']!=expected_checkpoint): + _fail('Recovery project identity or checkpoint differs.', 'stale', 409) + overlay=decode_fork(snapshot['records'],project_id=expected_project_id) + if overlay is None:_fail('Recovery requires a committed independent publication.') + from project_composition import decode_records, composition_input + composition=decode_records(snapshot['records'],project_id=expected_project_id).get(expected_project_id) + observed=(composition_input(composition['sources'],composition['directory'],composition['name'],composition['request_key']) + if composition is not None else _input(overlay['source'],overlay['directory'],overlay['name'],overlay['request_key'])) + if observed!=expected_request or store.reconcile(key,digest) is None: + _fail('Committed stage does not match the original request and receipt.', 'conflict', 409) + from project_projection import validate_project_records + from project_files import inspect_captured + declarations=_declarations(snapshot['records']) + with ProjectArtifacts(store) as artifacts: + validate_project_records(snapshot['records'],project_id=expected_project_id,artifacts=artifacts) + checks=artifacts.status_many([declarations[identity] for identity in overlay['artifacts']['copied']]) + if any(item['state']!='verified' for item in checks): + _fail('Committed publication payloads are missing or corrupt.', 'unavailable', 409) + with store.root_directory() as project_root: + if any(item['state'] not in ('unchanged','deleted') + for item in inspect_captured(snapshot['records'],expected_project_id,project_root)): + _fail('Committed stage has changed or missing working files.', 'working_files_changed', 409) + store.save(expected_checkpoint) + if not final_exists: + # Recheck the selected directory entry while the store root + # remains locked; an external rename must not substitute it. + with store.root_directory() as project_root: + entry=os.stat(stage_parts[-1],dir_fd=parent,follow_symlinks=False) + actual=os.fstat(project_root) + if (entry.st_dev,entry.st_ino)!=(actual.st_dev,actual.st_ino): + _fail('Selected stage directory changed.', 'stale', 409) + _rename_noreplace(parent,stage_parts[-1],parts[-1]);published=True + os.fsync(parent) + return dict(project_id=expected_project_id,checkpoint=expected_checkpoint, + directory=relative,request_key=key,reconciled=final_exists, + payloads_copied=0,selected=False) + except BaseException as error: + # Anchored directory owners also translate IO failures. Preserve the + # uncertain post-rename outcome through either exception representation. + if published: + raise ProjectForkError('Committed publication was renamed but acknowledgement is uncertain; reconcile this exact recovery.', + 'unknown',503) from error + if isinstance(error,OSError): + code='conflict' if error.errno==errno.EEXIST else 'unavailable' + raise ProjectForkError('Committed publication recovery is unavailable; no stage was rebuilt.', + code,503 if code!='conflict' else 409) from error + raise + + +def _publish_project(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, + name, expected_checkpoint, request_key, quiescent, build_candidate, + validate_candidate, retained_sources=()): + """One native stage/commit/no-clobber lifecycle for forks and composition. + + The workspace owns authorization, quiescence and all borrowed source owners. + Every source is pinned and rechecked. Composed candidates additionally prove + complete source provenance. Validators receive the actual source/destination + artifact owner; no imported runtime, permission or source path is activated. + No failed or unknown stage is deleted or selected by this owner. """ if quiescent is not True: _fail('Fork requires proven quiescence under the workspace lifecycle guard.', 'busy', 409) if not callable(build_candidate) or not callable(validate_candidate): _fail('Fork requires pure domain construction and complete validation.', 'unavailable', 503) - if type(request_key) is not str or not REQUEST_KEY.fullmatch(request_key): - _fail('Invalid fork request key.') + if type(retained_sources) not in (tuple,list):_fail('Explicit retained source owners required.') + if type(request_key) is not str or not REQUEST_KEY.fullmatch(request_key):_fail('Invalid fork request key.') _sha(expected_checkpoint) - if type(name) is not str or not name.strip() or len(name) > 200: - _fail('Fork name must contain 1–200 characters.') - _encode(name) - parts = _relative_parts(relative) - snapshot = source_store.snapshot() - if snapshot['checkpoint'] != expected_checkpoint: - _fail('Expected source checkpoint changed.', 'stale', 409) - validate_candidate(copy.deepcopy(snapshot['records']), snapshot['project_id']) - source = dict(project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint']) - expected_input = _input(source, relative, name, request_key) - digest = _digest(expected_input) - source_root = staged = artifacts = None - staging_relative = None - overlay = committed = None - published = False + if type(name) is not str or not name.strip() or len(name)>200:_fail('Fork name must contain 1–200 characters.') + _encode(name);parts=_relative_parts(relative) + owners=[dict(store=source_store,artifacts=source_artifacts,checkpoint=expected_checkpoint)] + for item in retained_sources: + _shape(item,('store','artifacts','checkpoint'),'retained source owner') + _sha(item['checkpoint']);owners.append(dict(item)) + sources=[];snapshots=[];seen=set() + for item in owners: + snapshot=item['store'].snapshot() + if snapshot['checkpoint']!=item['checkpoint']:_fail('Expected source checkpoint changed.','stale',409) + if snapshot['project_id'] in seen:_fail('Retained source project identity repeats.','conflict',409) + seen.add(snapshot['project_id']) + validate_candidate(copy.deepcopy(snapshot['records']),snapshot['project_id'],item['artifacts']) + identity={key:snapshot[key] for key in ('project_id','checkpoint')} + sources.append(dict(store=item['store'],artifacts=item['artifacts'],identity=identity)) + snapshots.append(snapshot) + composed=bool(retained_sources);source=sources[0]['identity'] + if composed: + from project_composition import composition_input + expected_input=composition_input(snapshots,relative,name,request_key) + else:expected_input=_input(source,relative,name,request_key) + digest=_digest(expected_input) + source_roots=[];staged=artifacts=None;staging_relative=None;committed=None;published=False + def sources_current(): + for item in sources:_source_current(item['store'],item['identity']) try: - with source_store.root_directory() as borrowed: - source_root = os.dup(borrowed) - with anchored_directory(trusted_root_fd, '/'.join(parts[:-1]), root_path=root_path) as parent: - if _inside(parent, source_root): - _fail('Fork destination cannot overlap the source project.') - try: - existing = os.stat(parts[-1], dir_fd=parent, follow_symlinks=False) - except FileNotFoundError: - existing = None + for item in sources: + with item['store'].root_directory() as borrowed:source_roots.append(os.dup(borrowed)) + with anchored_directory(trusted_root_fd,'/'.join(parts[:-1]),root_path=root_path) as parent: + if any(_inside(parent,root) for root in source_roots):_fail('Fork destination cannot overlap any source project.') + try:existing=os.stat(parts[-1],dir_fd=parent,follow_symlinks=False) + except FileNotFoundError:existing=None if existing is not None: - if not stat.S_ISDIR(existing.st_mode): - _fail('Fork destination is an existing or unsafe filesystem object.', 'conflict', 409) - selected = os.open(parts[-1], DIR_FLAGS, dir_fd=parent) + if not stat.S_ISDIR(existing.st_mode):_fail('Fork destination is an existing or unsafe filesystem object.','conflict',409) + selected=os.open(parts[-1],DIR_FLAGS,dir_fd=parent) try: - if _inside(source_root, selected) or _inside(selected, source_root): - _fail('Fork destination aliases or overlaps the source project.') - finally: - os.close(selected) - return _reconcile_existing(source_store, source, trusted_root_fd, relative, root_path, - expected_input, request_key, digest, validate_candidate) - staging_name = '.tuldok-fork-' + uuid.uuid4().hex - staging_relative = '/'.join([*parts[:-1], staging_name]) - staged = ProjectStore.create_at(trusted_root_fd, staging_relative, name, - 'fork-stage.' + hashlib.sha256(request_key.encode()).hexdigest(), digest, root_path=root_path) - destination_id = staged.snapshot()['project_id'] - if destination_id == source['project_id']: - _fail('Fork destination identity must be new.') - candidate = build_candidate(copy.deepcopy(snapshot), destination_id) - _shape(candidate, ('records', 'mapping', 'archived', 'artifacts'), 'owned candidate') - if type(candidate['records']) is not list: - _fail('Fork candidate records must be a list.') - records = copy.deepcopy(candidate['records']) - indexed = {} + if any(_inside(root,selected) or _inside(selected,root) for root in source_roots): + _fail('Fork destination aliases or overlaps a source project.') + finally:os.close(selected) + return _reconcile_existing(sources,trusted_root_fd,relative,root_path, + expected_input,request_key,digest,validate_candidate,composed=composed) + staging_name='.tuldok-fork-'+uuid.uuid4().hex + staging_relative='/'.join([*parts[:-1],staging_name]) + staged=ProjectStore.create_at(trusted_root_fd,staging_relative,name, + 'fork-stage.'+hashlib.sha256(request_key.encode()).hexdigest(),digest,root_path=root_path) + destination_id=staged.snapshot()['project_id'] + if destination_id in seen:_fail('Fork destination identity must be new.') + candidate=build_candidate(copy.deepcopy(snapshots if composed else snapshots[0]),destination_id) + _shape(candidate,('records','mapping','archived','artifacts'),'owned candidate') + if type(candidate['records']) is not list:_fail('Fork candidate records must be a list.') + records=copy.deepcopy(candidate['records']);indexed={} for item in records: - _shape(item, ('family', 'id', 'document'), 'candidate envelope') - _identifier(item['family'], 'Fork candidate family') - _identifier(item['id'], 'Fork candidate record ID') - identity = item['family'], item['id'] - if identity in indexed: - _fail('Fork candidate repeats a record identity.') - indexed[identity] = item - source_index = {(item['family'], item['id']): item for item in snapshot['records']} - for identity, original in source_index.items(): - if identity not in indexed or _encode(indexed[identity]) != _encode(original): - _fail('Fork must preserve every original canonical envelope exactly.') - archived = [_locator(item) for item in candidate['archived']] - if not set(archived) <= set(source_index): - _fail('Fork may archive only retained source records.') - required = {identity for identity in source_index if identity[0] in ARCHIVE_FAMILIES} - previous = decode_fork(snapshot['records'], project_id=source['project_id']) - if previous is not None: - required.update(_locator(item) for item in previous['archived']) - if not required <= set(archived): - _fail('Fork must archive every source editable value and operation receipt.') - declarations = _declarations(records) - if declarations != _declarations(snapshot['records']): - _fail('Fork must retain the source payload inventory without regenerating bytes.') - plan = _artifact_plan(candidate['artifacts'], declarations) - copied, missing = [], [] - for identity in sorted(plan): - status = source_artifacts.status(declarations[identity])['state'] - if status == 'corrupt': - _fail('Source fork payload is corrupt.', 'invalid', 400) - if status == 'missing': - if plan[identity] == 'unique': - _fail('Irreplaceable source bytes are missing; complete fork is unavailable.', 'unavailable', 409) - missing.append(identity) - elif status == 'verified': - copied.append(identity) - else: - _fail('Fork requires verified payload custody.', 'unavailable', 409) - overlay = dict(format=FORMAT, source=source, destination_project_id=destination_id, - request_key=request_key, request_sha256=digest, directory=relative, name=name, - mapping=copy.deepcopy(candidate['mapping']), archived=copy.deepcopy(candidate['archived']), - artifacts=dict(copied=copied, missing_regenerable=missing)) - records.append(dict(family=FAMILY, id=destination_id, document=overlay)) - decode_fork(records, project_id=destination_id) - validate_candidate(copy.deepcopy(records), destination_id) - _source_current(source_store, source) - artifacts = ProjectArtifacts(staged) - for identity in copied: - value = declarations[identity] - with source_artifacts.open_verified(value, maximum_bytes=value['bytes']) as source_fd: - artifacts.publish_fd(source_fd, value, maximum_bytes=value['bytes']) - # Ordinary authored files are explicitly adopted text. Only a new - # task-owned fork stage materializes them; Open never calls this. + _shape(item,('family','id','document'),'candidate envelope') + _identifier(item['family'],'Fork candidate family');_identifier(item['id'],'Fork candidate record ID') + key=item['family'],item['id'] + if key in indexed:_fail('Fork candidate repeats a record identity.') + indexed[key]=item + source_index={(r['family'],r['id']):r for snapshot in snapshots for r in snapshot['records']} + if composed: + from project_composition import validate_source_memberships + validate_source_memberships(records,snapshots,destination_id) + else: + for identity,original in source_index.items(): + if identity not in indexed or _encode(indexed[identity])!=_encode(original): + _fail('Fork must preserve every original canonical envelope exactly.') + archived=[_locator(item) for item in candidate['archived']] + if not set(archived)<=set(source_index):_fail('Fork may archive only retained source records.') + required={key for key in source_index if key[0] in ARCHIVE_FAMILIES} + for snapshot in snapshots: + previous=decode_fork(snapshot['records'],project_id=snapshot['project_id']) + if previous is not None:required.update(_locator(item) for item in previous['archived']) + if not required<=set(archived):_fail('Fork must archive every source editable value and operation receipt.') + declarations=_declarations(records);expected_declarations={} + for snapshot in snapshots: + for identity,value in _declarations(snapshot['records']).items(): + if identity in expected_declarations and expected_declarations[identity]!=value: + _fail('Source artifact declarations conflict.') + expected_declarations[identity]=value + if declarations!=expected_declarations:_fail('Fork must retain the source payload inventory without regenerating bytes.') + plan=_artifact_plan(candidate['artifacts'],declarations);selected={} + for number,item in enumerate(sources): + own=_declarations(snapshots[number]['records']);identities=sorted(own) + observations=item['artifacts'].status_many([own[key] for key in identities]) + for identity,observation in zip(identities,observations): + status=observation['state'] + if status=='corrupt':_fail('Source fork payload is corrupt.','invalid',400) + if status=='verified':selected.setdefault(identity,number) + elif status!='missing':_fail('Fork requires verified payload custody.','unavailable',409) + copied=sorted(selected);missing=sorted(set(plan)-set(selected)) + if any(plan[key]=='unique' for key in missing): + _fail('Irreplaceable source bytes are missing; complete fork is unavailable.','unavailable',409) + overlay=dict(format=FORMAT,source=source,destination_project_id=destination_id, + request_key=request_key,request_sha256=_digest(_input(source,relative,name,request_key)),directory=relative,name=name, + mapping=copy.deepcopy(candidate['mapping']),archived=copy.deepcopy(candidate['archived']), + artifacts=dict(copied=copied,missing_regenerable=missing)) + records.append(dict(family=FAMILY,id=destination_id,document=overlay)) + decode_fork(records,project_id=destination_id) + validate_candidate(copy.deepcopy(records),destination_id,source_artifacts) + sources_current();artifacts=ProjectArtifacts(staged) + for number,item in enumerate(sources): + group=[declarations[key] for key in copied if selected[key]==number] + if group:item['artifacts'].copy_many_to(artifacts,group) from project_files import materialize_captured - with staged.root_directory() as destination_root: - materialize_captured(records, destination_id, destination_root) - _source_current(source_store, source) - validate_candidate(copy.deepcopy(records), destination_id) - committed = staged.commit(records, staged.snapshot()['checkpoint'], request_key, digest) - staged.save(committed['checkpoint']) - _source_current(source_store, source) - # All named content and its one canonical pointer are durable. The - # destination appears in one atomic no-overwrite directory rename. - try: - _rename_noreplace(parent, staging_name, parts[-1]) + with staged.root_directory() as destination_root:materialize_captured(records,destination_id,destination_root) + sources_current();validate_candidate(copy.deepcopy(records),destination_id,artifacts) + committed=staged.commit(records,staged.snapshot()['checkpoint'],request_key,digest) + staged.save(committed['checkpoint']);sources_current() + try:_rename_noreplace(parent,staging_name,parts[-1]) except FileExistsError as error: - raise ProjectForkError('Fork destination appeared during publication; it was not overwritten.', 'conflict', 409) from error - published = True - os.fsync(parent) - _source_current(source_store, source) - return _result(committed, overlay, False) + raise ProjectForkError('Fork destination appeared during publication; it was not overwritten.','conflict',409) from error + published=True;os.fsync(parent);sources_current() + result=_result(committed,overlay,False) + if composed: + from project_composition import decode_records + result['composition']=copy.deepcopy(decode_records(records,project_id=destination_id)[destination_id]) + return result except BaseException as error: if published: - failure = ProjectForkError('Fork destination was published but acknowledgement is uncertain; retry this exact request.', - 'unknown', 503) + failure=ProjectForkError('Fork destination was published but acknowledgement is uncertain; retry this exact request.','unknown',503) if committed is not None: - failure.receipt = dict(committed=True, selected=False, project_id=committed['project_id'], - checkpoint=committed['checkpoint'], request_key=request_key, directory=relative) - raise failure from error - if isinstance(error, OSError): - code = 'conflict' if error.errno == errno.EEXIST else 'unavailable' - failure = ProjectForkError('Fork publication is unavailable; retained staging has not been selected.', code, - 409 if code == 'conflict' else 503) - failure.staging_directory = staging_relative + failure.receipt=dict(committed=True,selected=False,project_id=committed['project_id'],checkpoint=committed['checkpoint'],request_key=request_key,directory=relative) raise failure from error - if staging_relative is not None: - error.staging_directory = staging_relative + if isinstance(error,OSError): + code='conflict' if error.errno==errno.EEXIST else 'unavailable' + failure=ProjectForkError('Fork publication is unavailable; retained staging has not been selected.',code,409 if code=='conflict' else 503) + failure.staging_directory=staging_relative;raise failure from error + if staging_relative is not None:error.staging_directory=staging_relative raise finally: - if artifacts is not None: - artifacts.close() - if staged is not None: - staged.close() - if source_root is not None: - os.close(source_root) + if artifacts is not None:artifacts.close() + if staged is not None:staged.close() + for descriptor in source_roots:os.close(descriptor) diff --git a/project_history.py b/project_history.py index 2fec6803..b69eccce 100644 --- a/project_history.py +++ b/project_history.py @@ -273,6 +273,21 @@ def _observation_document(record): return doc +def capture_run_observation(document): + """Retain one already validated Run-history document as inert exact evidence. + + This creates no disclosure fact or execution permission. The complete + history decoder still owns Run, admission and payload closure validation. + """ + from project_store import _encode as canonical + sha=hashlib.sha256(canonical(document)).hexdigest() + record={'family':'run_observation','id':sha,'document':{ + 'format':FAMILY_FORMATS['run_observation'],'origin_catalog_id':document['origin_catalog_id'], + 'run_id':document['run']['id'],'sha256':sha,'observation':copy.deepcopy(document)}} + _observation_document(record) + return record + + def declarations(record): if record['family']=='run_observation': return declarations({'family':'run_history','document':record['document']['observation']}) @@ -735,16 +750,29 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): from project_material import ProjectMaterial materials=ProjectMaterial(session) self.materials=materials;self.authorize_disclosure=authorize_disclosure + self._decoded_checkpoint=None + self._decoded_facts=None def _facts(self): - self.session.check() - return decode_records(self.session.snapshot['records'],materials=self.materials,artifacts=self.artifacts) + self.session.check_current() + summary=self.session.summary + checkpoint=(summary['project_id'],summary['checkpoint']) + if self._decoded_checkpoint==checkpoint: + return self._decoded_facts + records=self.session.snapshot['records'] + facts=decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) + # Current canonical bytes are guarded on every call. Only this owner's + # private decoded metadata is retained; returned values and permissions + # remain the responsibility of each read below. + self._decoded_facts=facts + self._decoded_checkpoint=checkpoint + return facts def _authorize(self,action,target=None): self.access.require('inspect',action,target) def _origin(self,origin): - return self.session.snapshot['project_id'] if origin is None else origin + return self.session.summary['project_id'] if origin is None else origin def _record(self,collection,identity,origin=None): origin=self._origin(origin);identifier(origin,'Origin catalog');identifier(identity) @@ -924,11 +952,9 @@ def invoke(): current=self.access.require('expose-holdout','history.disclose',run_id) if current.actor_id!=actor_id:_invalid('Disclosure caller changed.','conflict') doc=self._record(self._facts().runs,run_id,body['origin_catalog_id']) - from project_store import _encode as canonical - sha=hashlib.sha256(canonical(doc)).hexdigest() - captured.append({'family':'run_observation','id':sha,'document':{ - 'format':FAMILY_FORMATS['run_observation'],'origin_catalog_id':body['origin_catalog_id'], - 'run_id':run_id,'sha256':sha,'observation':copy.deepcopy(doc)}}) + observation=capture_run_observation(doc) + sha=observation['id'] + captured.append(observation) return validate_disclosure_fact({'format':'project_disclosure_v2','origin_catalog_id':body['origin_catalog_id'], 'run_id':run_id,'payload_set_sha256':payload_set_sha256(doc['origin_catalog_id'],run_id,doc['payloads']), 'population':'all','reason':body['reason'],'actor_id':actor_id,'occurred_at':timestamp(), @@ -1149,7 +1175,7 @@ def create_study(self,body,expected_checkpoint,request_key): closed(body,('name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']);policy=validate_retained_policy(body['policy']) def build(facts): - origin=self.session.snapshot['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) + origin=self.session.summary['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) prior=next((key for key,value in facts.policies.items() if key[0]==origin and key[3]==sha),None) policy_id=prior[1] if prior else uuid.uuid4().hex ref={'kind':'selection_policy','catalog_id':origin,'id':policy_id,'revision':1,'sha256':sha} @@ -1166,7 +1192,7 @@ def save_study(self,study_id,body,expected_checkpoint,request_key): closed(body,('revision','name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['policy'],'selection_policy',origin) policy=facts.policies.get((origin,ref['id'],ref['revision'],ref['sha256'])) @@ -1187,7 +1213,7 @@ def prepare_trial_from_material(self,study_id,body,expected_checkpoint,request_k if request_key not in operation_records(self.session.snapshot['records']): self.materials.recipe(body['material'],verify=True) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['material'],'recipe_material',origin);self.materials.recipe(ref) @@ -1210,7 +1236,7 @@ def select_candidate(self,study_id,body,expected_checkpoint,request_key): identifier(body['trial_id']);identifier(body['run_id']) # Expensive semantic proof precedes the write transaction. All exact # associations and selected policy are checked again under publication. - origin=self.session.snapshot['project_id'] + origin=self.session.summary['project_id'] self.access.require('edit','studies.select_candidate',study_id) evidence=(self.evidence(body['run_id'],{},origin_catalog_id=origin) if request_key not in operation_records(self.session.snapshot['records']) else None) @@ -1240,7 +1266,7 @@ def build(facts): def record_lineage(self,body,expected_checkpoint,request_key): closed(body,('trial','intent'));body=checked_json(body,INTENT_LIMIT+4096) def build(facts): - origin=self.session.snapshot['project_id'];trial=facts.trial_ref(body['trial']) + origin=self.session.summary['project_id'];trial=facts.trial_ref(body['trial']) if body['trial']['catalog_id']!=origin:_invalid('Inherited Trials remain immutable historical evidence.','conflict') doc=copy.deepcopy(facts.trials[(origin,trial['id'])]) if doc['lineage'] is not None:_invalid('Trial lineage is already recorded and immutable.','conflict') diff --git a/project_material.py b/project_material.py index 530ce34a..fbfa0189 100644 --- a/project_material.py +++ b/project_material.py @@ -277,8 +277,8 @@ def decode_runtime_material(value, *, catalog_id): from recipe_process import validate_cpu_snapshot_manifest,encoded manifest=validate_cpu_snapshot_manifest(descriptor,value['manifest']) elif descriptor.get('profile') == 'managed_torch_python_v1': - from torch_runtime import verify_descriptor,encoded - manifest=verify_descriptor(descriptor,value['manifest']) + from torch_runtime import verify_retained_descriptor,encoded + manifest=verify_retained_descriptor(descriptor,value['manifest']) else: raise WorkbenchError('Unsupported retained runtime profile.', 'unsupported', 409) raw=encoded(manifest) if len(raw)!=descriptor['manifest_bytes'] or hashlib.sha256(raw).hexdigest()!=descriptor['manifest_sha256']: @@ -315,15 +315,63 @@ def validate_operation_result(operation, result, records=None): return True +def _retained_value(get_documents, family, ref): + catalog_id = _catalog(ref['catalog_id']) + reference(ref, {'model_material':'model', 'recipe_material':'recipe_material', + 'runtime_material':'runtime'}[family], catalog_id) + key = catalog_id + '.' + ref['sha256'] + document = get_documents().get((family, key)) + if document is None: + raise WorkbenchError('Retained material is unavailable.', 'unavailable', 409) + if document['value']['ref'] != ref: + raise WorkbenchError('Retained material reference differs.', 'invalid', 400) + return copy.deepcopy(document['value']) + + +class MaterialSnapshot: + """Owned, validated material facts for one fixed checkpoint read. + + The decoder copies the selected documents. Neither input mutation nor a + returned value can alter this view. It has no live Session, artifact reader, + publication operation or current execution authority. + """ + @_boundary + def __init__(self, records): + self.__documents = decode_records(records) + + @_boundary + def recipe(self, ref): + return _retained_value(lambda: self.__documents, 'recipe_material', ref) + + @_boundary + def runtime(self, ref): + return _retained_value(lambda: self.__documents, 'runtime_material', ref) + + def artifact_blobs(self): + for document in self.__documents.values(): + for edge in document.get('artifacts', []): + yield copy.deepcopy(edge['blob']) + + class ProjectMaterial: def __init__(self, session): self.session = session self.owner_access = _MaterialAccess(session) if hasattr(session,'access') else None def _documents(self): - self.session.check() + self.session.check_current() return decode_records(self.session.snapshot['records']) + def read_view(self, records): + """Own validated material facts from one guarded checkpoint read. + + The history reader supplies its selected records. This view carries no + live publication, verification or execution capability; later reads + must select their own checkpoint and construct a new view. + """ + self.session.check_current() + return MaterialSnapshot(records) + @_boundary def intents(self): """Inspect unresolved active definitions; they confer no execution binding.""" @@ -333,7 +381,7 @@ def intents(self): from project_material_fork import is_intent documents = self._documents() active = {(r['family'], r['id']) for r in active_records( - self.session.snapshot['records'], project_id=self.session.snapshot['project_id'])} + self.session.snapshot['records'], project_id=self.session.summary['project_id'])} return [dict(family=family, id=locator, current=copy.deepcopy(doc['current']), fork_origin=copy.deepcopy(doc['fork_origin'])) for (family, locator), doc in sorted(documents.items()) @@ -344,7 +392,7 @@ def intent_capabilities(self): intents = self.intents() unresolved = {(r['family'],r['id']) for r in intents} from project_fork import active_records - active = active_records(self.session.snapshot['records'], project_id=self.session.snapshot['project_id']) + active = active_records(self.session.snapshot['records'], project_id=self.session.summary['project_id']) return dict(requires_reselection=[dict(family=r['family'],id=r['id'],fields=r['current']['required_reselection']) for r in intents], editable=[dict(family=r['family'],id=r['id']) for r in active if r['family'] in EDITABLE and (r['family'],r['id']) not in unresolved]) @@ -356,13 +404,7 @@ def _require_resolved(self, family, identity): @_boundary def _value(self, family, ref): - catalog_id = _catalog(ref['catalog_id']) - reference(ref,{'model_material':'model','recipe_material':'recipe_material','runtime_material':'runtime'}[family],catalog_id) - key = catalog_id + '.' + ref['sha256'] - document = self._documents().get((family, key)) - if document is None: raise WorkbenchError('Retained material is unavailable.', 'unavailable', 409) - if document['value']['ref'] != ref: raise WorkbenchError('Retained material reference differs.', 'invalid', 400) - return copy.deepcopy(document['value']) + return _retained_value(self._documents, family, ref) @_boundary def model(self, ref, *, verify=False): @@ -398,8 +440,8 @@ def runtime(self, ref, *, verify=False): from recipe_process import decode, encoded, validate_cpu_manifest manifest = validate_cpu_manifest(decode(raw)) else: - from torch_runtime import decode, encoded, validate_manifest - manifest = validate_manifest(decode(raw)) + from torch_runtime import decode, encoded, validate_retained_manifest + manifest = validate_retained_manifest(decode(raw)) if encoded(manifest) != raw: raise WorkbenchError('Retained runtime manifest encoding differs.', 'invalid', 400) return value @@ -411,15 +453,15 @@ def snapshot_ref(self, project_id, manifest_sha256, *, catalog_id=None, verify=F if len(candidates) != 1: raise WorkbenchError('Retained project snapshot is unavailable or ambiguous.', 'unavailable', 409) doc = candidates[0] if verify: - from projects import decode, manifest + from projects import decode, decode_retained_manifest edge = doc['artifacts'][0]['blob'] raw = self.session.artifacts.read_blob(edge, maximum_bytes=64*1024) - if manifest(decode(raw)) != doc['value']['manifest']: + if decode_retained_manifest(decode(raw)) != doc['value']['manifest']: raise WorkbenchError('Retained project manifest differs.', 'invalid', 400) return copy.deepcopy(doc['value']) def capture_definition(self, family, value, before_records, *, action, actor_id=None, audit_facts=None): - catalog_id = self.session.snapshot['project_id'] + catalog_id = self.session.summary['project_id'] value = _definition(family, value, catalog_id) key = catalog_id + '.' + value['id'] prior = decode_records(before_records).get((family, key)) @@ -539,17 +581,17 @@ def capture_recipe_dependencies(self,record,before_records,*,runtime_manifest=No from recipe_process import decode,validate_cpu_manifest manifest=validate_cpu_manifest(decode(manifest_raw)) else: - from torch_runtime import decode,validate_manifest - manifest=validate_manifest(decode(manifest_raw)) + from torch_runtime import decode,validate_retained_manifest + manifest=validate_retained_manifest(decode(manifest_raw)) runtime=self.capture_runtime(dict(ref=ref,descriptor=descriptor, environment=None if descriptor is not None else environment,manifest=manifest),manifest_raw) return [*self.declarations(runtime),runtime] @_boundary def capture_snapshot(self, value, raw, *, catalog_id): - from projects import decode_retained_snapshot,decode,manifest + from projects import decode_retained_snapshot,decode,decode_retained_manifest value=decode_retained_snapshot(value);_catalog(catalog_id) - if type(raw) is not bytes or len(raw)>64*1024 or hashlib.sha256(raw).hexdigest()!=value['manifest_sha256'] or manifest(decode(raw))!=value['manifest']: + if type(raw) is not bytes or len(raw)>64*1024 or hashlib.sha256(raw).hexdigest()!=value['manifest_sha256'] or decode_retained_manifest(decode(raw))!=value['manifest']: raise WorkbenchError('Exact original project manifest bytes differ.') key=catalog_id+'.'+value['manifest_sha256'] prior=decode_records(self.session.snapshot['records']).get(('project_snapshot',key)) diff --git a/project_measurements.py b/project_measurements.py index b97bca56..e64a307e 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -6,11 +6,13 @@ """ import copy from datetime import datetime +from functools import cache, partial from project_artifacts import validate_declaration from project_material import decode_records as material_records from projects import ID, NAME, SHA, relative_path -from retained_measurements import MeasurementTable, MAX_BYTES, MAX_RESPONSE_BYTES, encode, text +from retained_measurements import (MeasurementTable, MeasurementComparison, MAX_BYTES, MAX_RESPONSE_BYTES, + MAX_COMPARISON_SOURCES, MAX_COMPARISON_BYTES, encode, text) from workbench import WorkbenchError, timestamp @@ -100,33 +102,55 @@ def decode_records(records, *, artifacts=None): return result -def validate_operation_result(operation, result, records=None): +def _validate_result(operation, result, lookup=None): if operation != 'measurements.import': return False _document(result) - if records is not None: + if lookup is not None: identity = result['origin_catalog_id'] + '.' + result['id'] - if decode_records(records).get(identity) != result: + if lookup(identity) != result: raise WorkbenchError('Measurement receipt has no exact retained source.') return True +def receipt_validator(records, *, artifacts=None): + """Own one candidate's validated source bindings and packet custody. + + Decode and copy every retained measurement once, including current available + packet bytes when supplied. The returned receipt checker is fixed to those + facts; it carries no live source or publication capability. + """ + owned = decode_records(records, artifacts=artifacts) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return decode_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class ProjectMeasurements: def __init__(self, session): self.session, self.artifacts = session, session.artifacts self.access, self.db = session.access, session.db def _documents(self): - return decode_records(self.session.snapshot['records']) + # Session adoption/publication already validates the complete candidate. + # Verify current canonical bytes, then own only the measurement family. + records = self.session.check_records(FAMILY)['records'] + return {item['id']: item['document'] for item in records} def import_packet(self, raw, source, expected_checkpoint, key): """Called only after SourceAdapter's explicit current rooted capture.""" with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'measurements.import') - self.session.check() + self.session.check_current() table = MeasurementTable(raw) blob = dict(sha256=table.sha256, bytes=len(raw)) - origin = self.session.snapshot['project_id'] + origin = self.session.summary['project_id'] request = dict(blob=blob, source=copy.deepcopy(source)) def invoke(): @@ -151,11 +175,11 @@ def sources(self, *, offset=0, limit=40): raise WorkbenchError('Invalid measurement source page.') with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', 'measurements.sources') - self.session.check() + self.session.check_current() docs = sorted(self._documents().values(), key=lambda d: (d['origin_catalog_id'], d['id'])) items = [dict(doc, availability=self.artifacts.inspect(declaration(doc['blob']))) for doc in docs[offset:offset+limit]] - self.session.check() + self.session.check_current() self.access.require('inspect', 'measurements.sources') return self._bounded(dict(items=items, total=len(docs), offset=offset, limit=limit)) @@ -171,7 +195,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): target = origin_catalog_id + '.' + packet_sha256 with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() doc = self._documents().get(target) if doc is None: raise WorkbenchError('Measurement source is unavailable.', 'not_found', 404) @@ -180,7 +204,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): if table.metadata()['title'] != doc['title']: raise WorkbenchError('Measurement title differs from the exact retained packet.') value = self._bounded(dict(source=doc, value=select(table))) - self.session.check() + self.session.check_current() self.access.require('inspect', action, target) return value @@ -198,3 +222,39 @@ def query(self, origin_catalog_id, packet_sha256, metric_id, **selection): def inspect(self, origin_catalog_id, packet_sha256, observation_id): return self._read(origin_catalog_id, packet_sha256, 'measurements.inspect', lambda table: table.inspect(observation_id)) + + def compare(self, sources, metric_id=None, **selection): + """Read one checkpoint's bounded group with every original source intact.""" + if type(sources) is not list or not 1 <= len(sources) <= MAX_COMPARISON_SOURCES: + raise WorkbenchError('Choose 1 through 16 native measurement sources.', 'too_large', 413) + targets = [] + for source in sources: + _closed(source, ('origin_catalog_id','packet_sha256')) + _identity(source['origin_catalog_id'], ID) + _identity(source['packet_sha256'], SHA) + targets.append(source['origin_catalog_id'] + '.' + source['packet_sha256']) + if len(set(targets)) != len(targets): + raise WorkbenchError('Native comparison repeats a measurement source.') + with self.access.lock, self.session.lock, self.db.lock: + for target in targets: + self.access.require('inspect', 'measurements.compare', target) + self.session.check_current() + documents = self._documents() + if any(target not in documents for target in targets): + raise WorkbenchError('A comparison source is unavailable.', 'not_found', 404) + chosen = [documents[target] for target in targets] + if sum(doc['blob']['bytes'] for doc in chosen) > MAX_COMPARISON_BYTES: + raise WorkbenchError('Native comparison exceeds 128 MiB of original packets.', 'too_large', 413) + tables = [] + for locator, doc in zip(sources, chosen): + raw = self.artifacts.read_blob(doc['blob'], maximum_bytes=MAX_BYTES) + table = MeasurementTable(raw) + if table.metadata()['title'] != doc['title']: + raise WorkbenchError('Comparison title differs from its original packet.') + tables.append((locator, table)) + value = MeasurementComparison(tables).read(metric_id, **selection) + result = self._bounded(dict(sources=chosen, value=value)) + self.session.check_current() + for target in targets: + self.access.require('inspect', 'measurements.compare', target) + return result diff --git a/project_native_execution.py b/project_native_execution.py index 832a990f..5895e4d7 100644 --- a/project_native_execution.py +++ b/project_native_execution.py @@ -452,7 +452,7 @@ def identity(info): def _publish(self, document, state, *, helper=None, receipt_sha256=None): with self.db.lock,self.db.transaction(): - self.session.check();actor=self._authorize(document['phase_id']) + self.session.check_current();actor=self._authorize(document['phase_id']) document['events'].append(dict(state=state,at=timestamp(),actor_id=actor.actor_id, helper=copy.deepcopy(helper),receipt_sha256=receipt_sha256)) self._dirty[document['phase_id']]=copy.deepcopy(document) @@ -473,7 +473,7 @@ def reconcile(self,phase_id,expected_checkpoint): execution=self.execution;journal=execution.custody if journal is None:_fail('There is no original local native custody.','unavailable') with self.db.lock: - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') local=journal.refresh() selected=next((row for row in local.get('native_phases',[]) if row['phase_id']==phase_id),None) if selected is None:_fail('A retained phase alone cannot select current native custody.','unavailable') @@ -501,7 +501,7 @@ def reconcile(self,phase_id,expected_checkpoint): if self._active:_fail('Current native owner has not finished its bounded observation.','busy') with execution.access.lock,self.session.lock,self.db.lock: self.access.require('cancel','native_execution.reconcile',phase_id,owner_only=True) - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') if journal.refresh()!=local:_fail('Original native custody changed while draining.','stale_project') binding=journal.binding if (binding['project_id']!=self.db.dataset_id or binding['projection_id']!=self.db.projection_id @@ -788,7 +788,7 @@ def _probe_result(recipe, inputs, evidence): def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, run_id=None, job=None, phase_id=None): if self.db.in_transaction:_fail('Native phase cannot launch inside a metadata transaction.','busy') with self.db.lock: - self.session.check() + self.session.check_current() if run_id is None and self._foreground_phase is not None: _fail('Another explicit native preparation phase is active.','busy') if not self.db.claim_runtime('project_native_phases'):_fail('Another current native phase owner is active.','busy') @@ -818,7 +818,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru self._publish(document,'prepared') if self.execution.custody is None:_fail('Native launch requires its original physical custody journal.','unavailable') with self.db.lock,self.execution._journal_guard(): - self._dispatch_authorized(phase_id);self.session.check() + self._dispatch_authorized(phase_id);self.session.check_current() if not self.execution.custody.has_pending:self.execution.custody.acknowledge(self.session.snapshot) self.execution.custody.native_prepared(document) self.execution.custody.native_launch_intent(phase_id) @@ -847,7 +847,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru # Recheck actual authority after both durable ACKs. A # saved GO intent is never permission to retry this send. with self.db.lock: - self.session.check();self._dispatch_authorized(phase_id) + self.session.check_current();self._dispatch_authorized(phase_id) if not self.execution.custody.has_pending:_fail('Native local custody disappeared.','unknown') prepared.check_current_runtime() handle.go(handle.ready) diff --git a/project_native_preparation.py b/project_native_preparation.py index 85a0b1be..24113f63 100644 --- a/project_native_preparation.py +++ b/project_native_preparation.py @@ -188,7 +188,7 @@ def validate_operation_result(operation,result,records=None): _fail('Native intent source or frozen input differs from its graph.','invalid') from types import SimpleNamespace from project_material import ProjectMaterial - retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check=lambda:None)) + retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None)) if result['model'] is not None:retained.model(result['model']) if result['composition'] is not None: composition=retained.recipe(result['composition'])['material'] @@ -230,7 +230,7 @@ def __init__(self,adapter,owner,graph,model,composition,purpose,key): def _check(self,recipe): if not self._active or _CURRENT.get() is not self or recipe!=self.graph['recipe']: _fail('An explicit current native preparation is required.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.probe_native',recipe,owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Native preparation caller changed.','conflict') if (os.environ.get('TULDOK_TRAINING_PYTHON') or sys.executable)!=self.configured_executable: @@ -262,7 +262,7 @@ def probe(self,owner,recipe): if self._probe is None: if self.adapter.db.in_transaction:_fail('Native probe cannot run inside a metadata transaction.','unsupported') self.authorize_probe(recipe) - result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.snapshot['checkpoint'], + result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.summary['checkpoint'], request_key=self.key+'.probe',preparation=self) self._probe=copy.deepcopy(result['runtime']);self.phase=copy.deepcopy(result['phase']) return copy.deepcopy(self._probe) @@ -297,7 +297,7 @@ def __init__(self,adapter,owner,graph): def consume_admission(self): if not self._active or self._consumed:_fail('Current integrity qualification has already been consumed.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.snapshot',self.graph['id'],owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Current integrity caller changed.','conflict') if self.owner.get(self.graph['id'])!=self.graph or self.owner.definition_identity(self.recipe)!=self.definition: @@ -333,8 +333,8 @@ def _scope(self,owner,graph,model,composition,purpose,expected_checkpoint,key): if self.db.in_transaction:_fail('Native preparation must precede the metadata transaction.','unsupported') if _CURRENT.get() is not None:_fail('A native preparation is already active.','busy') self.access.require('edit','native_preparation.'+purpose,graph['id'],owner_only=True) - self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + self.session.check_current() + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') context=NativePreparationContext(self,owner,graph,model,composition,purpose,_key(purpose,key)) result=dict(format='project_native_preparation_intent_v1',origin_catalog_id=self.session.project_id, actor_id=context.actor.actor_id,purpose=purpose,graph=dict(id=graph['id'],revision=graph['revision'], @@ -361,22 +361,22 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): if graph['revision']!=revision:_fail('Graph changed before native validation.','stale') import release_validation if adapter_for_graph(graph) is release_validation: - self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + self.session.check_current() + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') value=owner.validate_graph(graph) return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') with self._scope(owner,graph,None,None,'validation',expected_checkpoint,key) as context: value=owner.validate_graph(graph);context._check(graph['recipe']) - return self.session.mutate('native_preparation.validate',graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate('native_preparation.validate',graph_id,body,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') @contextmanager def current_graph(self,owner,graph_id,revision,expected_checkpoint,key): if self.db.in_transaction:_fail('Current native validation must precede admission.','unsupported') self.access.require('edit','native_preparation.admission',graph_id,owner_only=True) - self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + self.session.check_current() + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') graph=owner.get(graph_id) if graph['revision']!=revision:_fail('Graph changed before native admission.','stale') import release_validation @@ -413,7 +413,7 @@ def capture(value,before): record=self.materials.capture_recipe_material(value,owner.material_path) return dict(records=self.materials._coalesce([record,*self.materials.declarations(record), *self.materials.capture_recipe_dependencies(record,before)]),result=value) - return self.session.mutate(operation,graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate(operation,graph_id,body,self.session.summary['checkpoint'],key, lambda:owner._retain_material(prepared),capture) finally: if prepared['staging'].exists():shutil.rmtree(prepared['staging']) diff --git a/project_producer_execution.py b/project_producer_execution.py index b4456aba..e3729233 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -73,7 +73,7 @@ def _family(self, family): return family def _owner(self, family): - self._family(family);self.session.check() + self._family(family);self.session.check_current() if self._fenced: raise WorkbenchError('Current producer custody is uncertain. Reopen for inert inspection.', 'outcome_unknown', 409) self._refuse_unknown_provider() @@ -264,7 +264,7 @@ def _settle_drained(self): # Every effectful request is preceded by canonical uncertainty; # a terminal job with no such request is known not admitted. outcome='not_admitted' - observation=(outcome,True,self.session.snapshot['checkpoint']) + observation=(outcome,True,self.session.summary['checkpoint']) if self._journal_observations.get(identity_key)!=observation: self.execution.update_producer_outcome(family,producer_id,outcome,True) self._journal_observations[identity_key]=observation @@ -429,7 +429,7 @@ def status(self, *, offset=0, limit=40): raise WorkbenchError('Invalid current producer health page.') with self.session.access.lock,self.session.lock,self.db.lock: self.session.access.require('inspect','producers.health') - self.session.check() + self.session.check_current() self._settle_drained() items=[] for family,owner in self._owners.items(): diff --git a/project_producers.py b/project_producers.py index 79b97f20..8e052b34 100644 --- a/project_producers.py +++ b/project_producers.py @@ -592,9 +592,9 @@ def __init__(self, session): def _read(self, action, invoke): with self.session.access.lock, self.session.lock, self.session.db.lock: self.session.access.require('inspect', action) - self.session.check() + self.session.check_current() result = invoke(self.session.snapshot['records']) - self.session.check() + self.session.check_current() return result def list(self, family, *, origin_catalog_id=None, offset=0, limit=40): @@ -915,7 +915,7 @@ def _publish(owner, raw, role, media='application/json'): fail('Conflicting exact producer payload bytes.') owner._pending_payloads[declaration['sha256']] = raw owner._artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=owner.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=owner.session.summary['project_id'], sha256=declaration['sha256']), media_type=media, kind='producer_evidence', role=role, preservation='unique', required_for=['verify_producer']) diff --git a/project_projection.py b/project_projection.py index e8a81a40..25b6bf28 100644 --- a/project_projection.py +++ b/project_projection.py @@ -15,8 +15,7 @@ from project_history import ProjectHistory, FAMILY_FORMATS as HISTORY_FAMILIES from project_history import decode_records as decode_history_records from project_history import validate_operation_result as validate_history_result -from project_material import ProjectMaterial, FAMILIES as MATERIAL_FAMILIES -from project_material import decode_records as decode_material_records +from project_material import ProjectMaterial, MaterialSnapshot, FAMILIES as MATERIAL_FAMILIES from project_material import validate_operation_result as validate_material_result from project_producers import ProjectProducers, FAMILIES as PRODUCER_FAMILIES from project_producers import decode_records as decode_producer_records @@ -26,14 +25,17 @@ from project_producer_execution import ProjectProducerExecution, validate_operation_result as validate_producer_result from project_files import ProjectFiles, decode_records as decode_file_records from project_files import validate_operation_result as validate_file_result +from project_files import receipt_validator as file_receipt_validator from project_study_execution import ProjectStudyExecution, validate_operation_result as validate_study_execution_result from project_runtime import ProjectRuntime, validate_operation_result as validate_runtime_result from project_native_execution import FAMILY as NATIVE_FAMILY, decode_records as decode_native_records from project_native_preparation import ProjectNativePreparation, validate_operation_result as validate_preparation_result from project_sources import SourceAdapter, validate_operation_result as validate_source_result +from project_sources import receipt_validator as source_receipt_validator from project_measurements import ProjectMeasurements, FAMILY as MEASUREMENT_FAMILY from project_measurements import decode_records as decode_measurement_records from project_measurements import validate_operation_result as validate_measurement_result +from project_measurements import receipt_validator as measurement_receipt_validator from workbench import WorkbenchError from workbench import timestamp @@ -59,12 +61,66 @@ def validate_artifact_resolution(operation, result, records): return True +def validate_project_records(records, *, project_id, artifacts): + """Validate one inert candidate through every owning domain, without SQL reconstruction.""" + claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( + MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) + identities = set() + declarations = {} + for item in records: + identity = (item['family'], item['id']) + if item['family'] not in claimed or identity in identities: + raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) + identities.add(identity) + if item['family'] == 'project_artifact': + declaration = validate_declaration(item['document']) + if declaration['sha256'] != item['id']: + raise WorkbenchError('Artifact inventory identity differs.') + declarations[item['id']] = declaration + decode_data_records(records) + from project_fork import decode_fork + decode_fork(records, project_id=project_id) + from project_composition import decode_records as decode_compositions + decode_compositions(records, project_id=project_id) + from project_transfer import decode_records as decode_extraction_records + decode_extraction_records(records) + operation_records(records, project_id=project_id) + validate_file_receipt = file_receipt_validator(records) + decode_native_records(records,artifacts=artifacts) + materials = MaterialSnapshot(records) + for blob in materials.artifact_blobs(): + if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): + raise WorkbenchError('Material artifact declaration is missing or inconsistent.') + # Resolve every cross-owner reference against THIS candidate, rather + # than a resolver that still sees the preceding committed checkpoint. + decode_producer_records(records, artifacts=artifacts) + decode_history_records(records, materials=materials, artifacts=artifacts) + validate_measurement_receipt = measurement_receipt_validator(records, artifacts=artifacts) + validate_source_receipt = source_receipt_validator(records) + for receipt in operation_documents(records): + operation, result = receipt['operation'], receipt['result'] + if (validate_data_result(operation, result, records) + or validate_material_result(operation, result, records) + or validate_history_result(operation, result, records) + or validate_source_receipt(operation, result) + or validate_measurement_receipt(operation, result) + or validate_execution_result(operation, result, records) + or validate_producer_result(operation, receipt['target'], result, records) + or validate_file_receipt(operation, result) + or validate_study_execution_result(operation, result, records) + or validate_runtime_result(operation, result, records) + or validate_preparation_result(operation, result, records) + or validate_artifact_resolution(operation, result, records)): + continue + raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + + class ProjectProjection: def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None): self.store, self.db, self.access = store, db, access self.custody = custody self.pending_original = [] - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.artifacts = ProjectArtifacts(store) self.sources = None self._disclosures = set() @@ -127,59 +183,10 @@ def schema(self): def execute(self, *args, **kwargs): return self.db.execute(*args, **kwargs) - def validate_records(self, records, *, project_id=None): - project_id = self.project_id if project_id is None else project_id - claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( - MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) - identities = set() - declarations = {} - for item in records: - identity = (item['family'], item['id']) - if item['family'] not in claimed or identity in identities: - raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) - identities.add(identity) - if item['family'] == 'project_artifact': - declaration = validate_declaration(item['document']) - if declaration['sha256'] != item['id']: - raise WorkbenchError('Artifact inventory identity differs.') - declarations[item['id']] = declaration - decode_data_records(records) - from project_fork import decode_fork - decode_fork(records, project_id=project_id) - from project_transfer import decode_records as decode_extraction_records - decode_extraction_records(records) - operation_records(records, project_id=project_id) - decode_file_records(records) - decode_native_records(records,artifacts=self.artifacts) - materials = decode_material_records(records) - for document in materials.values(): - for edge in document.get('artifacts', []): - blob = edge['blob'] - if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): - raise WorkbenchError('Material artifact declaration is missing or inconsistent.') - # Resolve every cross-owner reference against THIS candidate, rather - # than a resolver that still sees the preceding committed checkpoint. - candidate = SimpleNamespace(snapshot=dict(project_id=project_id, records=records), - artifacts=self.artifacts, check=lambda: None) - decode_producer_records(records, artifacts=self.artifacts) - decode_history_records(records, materials=ProjectMaterial(candidate), artifacts=self.artifacts) - decode_measurement_records(records, artifacts=self.artifacts) - for receipt in operation_documents(records): - operation, result = receipt['operation'], receipt['result'] - if (validate_data_result(operation, result, records) - or validate_material_result(operation, result, records) - or validate_history_result(operation, result, records) - or validate_source_result(operation, result, records) - or validate_measurement_result(operation, result, records) - or validate_execution_result(operation, result, records) - or validate_producer_result(operation, receipt['target'], result, records) - or validate_file_result(operation, result, records) - or validate_study_execution_result(operation, result, records) - or validate_runtime_result(operation, result, records) - or validate_preparation_result(operation, result, records) - or validate_artifact_resolution(operation, result, records)): - continue - raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + def validate_records(self, records, *, project_id=None, artifacts=None): + return validate_project_records(records, + project_id=self.project_id if project_id is None else project_id, + artifacts=self.artifacts if artifacts is None else artifacts) def resolve_artifact(self, body, expected_checkpoint, key): """Explicitly supply exact bytes from a configured current local root.""" @@ -187,7 +194,7 @@ def resolve_artifact(self, body, expected_checkpoint, key): raise WorkbenchError('Choose one declared artifact and a current rooted file.') with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'artifacts.resolve', body['sha256'], owner_only=True) - self.session.check() + self.session.check_current() declaration = next((item['document'] for item in self.session.snapshot['records'] if item['family'] == 'project_artifact' and item['id'] == body['sha256']), None) if declaration is None: @@ -243,7 +250,7 @@ def runtime_configuration(self, request): def runtime_sources(self): with self.access.lock, self.session.lock, self.db.lock: self.access.require('admin', 'runtime.sources', owner_only=True) - self.session.check() + self.session.check_current() return dict(sources=sorted(self._runtime_sources)) def authorize_disclosure(self, run, population): @@ -310,14 +317,15 @@ def available(self): return self.projection.session.available def _tag(self, value): - snapshot = self.projection.session.snapshot - return dict(value, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint']) + summary = self.projection.session.summary + return dict(value, project_id=summary['project_id'], checkpoint=summary['checkpoint']) def records(self): - self.projection.session.check() + self.projection.session.check_current() from project_fork import active_records snapshot = self.projection.session.snapshot - records = active_records(snapshot['records'], project_id=snapshot['project_id']) + records = active_records(snapshot['records'], project_id=snapshot['project_id'], + families=frozenset(('workbench_record', 'workbench_text'))) return self._tag(dict(items=[copy.deepcopy(item['document']['current']) for item in records if item['family'] in ('workbench_record', 'workbench_text') and item['document']['current']['kind'] == 'text'])) diff --git a/project_runtime.py b/project_runtime.py index a94cc6e6..d63a9eb5 100644 --- a/project_runtime.py +++ b/project_runtime.py @@ -156,7 +156,7 @@ def _authorize(self, action): def inspect(self, config_request): """Read an explicitly selected current source; never provision or execute.""" with self.access.lock, self.session.lock, self.db.lock: - self._authorize('runtime_snapshots.inspect'); self.session.check() + self._authorize('runtime_snapshots.inspect'); self.session.check_current() _body(config_request, '0'*64, 0) with self.execution.runtime_inspection(): return self.owner.inspect(self.resolve_configuration(copy.deepcopy(config_request))) @@ -165,7 +165,7 @@ def status(self, key): """Read historical outcome only; captured does not claim local readiness.""" _key(key) with self.access.lock, self.session.lock, self.db.lock: - self.access.require('inspect', 'runtime.capture_status'); self.session.check() + self.access.require('inspect', 'runtime.capture_status'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) intent = receipts.get(_intent_key(key)); complete = receipts.get(key) if intent is not None and intent['operation'] != 'runtime.capture_intent': intent = None @@ -185,7 +185,7 @@ def recover_capture(self, config_request, expected_inventory_sha256, expected_ch body_sha = _intent_input(body) self._authorize('runtime_snapshots.create') self.execution.prepare_runtime_reconciliation(key, body_sha, expected_inventory_sha256, expected_checkpoint) - return self.capture(config_request, expected_inventory_sha256, self.session.snapshot['checkpoint'], key, + return self.capture(config_request, expected_inventory_sha256, self.session.summary['checkpoint'], key, reserve_bytes=reserve_bytes) def abandon(self, capture_key, reason, expected_checkpoint, key): @@ -212,7 +212,7 @@ def forbidden(): _fail('No original local runtime copy exists to abandon.', 'una _fail('Runtime intent differs from its exact original local writer.', 'conflict', 409) if capture_key in receipts: _fail('Capture completed; reconcile its acknowledgement with the exact capture request.', 'conflict', 409) - observed = self.session.snapshot['checkpoint'] + observed = self.session.summary['checkpoint'] def invoke(): current = self._authorize('runtime.capture_abandon') if current.actor_id != actor.actor_id: _fail('Runtime abandonment caller changed.', 'conflict', 409) @@ -248,7 +248,7 @@ def capture(self, config_request, expected_inventory_sha256, expected_checkpoint body_sha = _intent_input(body) intent = None with self.access.lock, self.session.lock, self.db.lock: - actor = self._authorize('runtime_snapshots.create'); self.session.check() + actor = self._authorize('runtime_snapshots.create'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) if any(r['operation']=='runtime.abandon' and r['result']['capture_key']==key for r in receipts.values()): _fail('This capture was abandoned; use a fresh key for a new measurement.', 'conflict', 409) @@ -273,7 +273,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), invalid, lambda *args: invalid(), scope='admin') else: - if expected_checkpoint != self.session.snapshot['checkpoint']: + if expected_checkpoint != self.session.summary['checkpoint']: _fail('Expected project checkpoint changed.', 'stale_project', 409) config = self.resolve_configuration(copy.deepcopy(config_request)) observed = self.owner.inspect(config) @@ -281,7 +281,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 _fail('Runtime inventory changed after inspection.', 'stale', 409) intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key), + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key), lambda result,before:dict(records=[],result=result), scope='admin') guard.publication_started() if os.path.lexists(self.owner.path / expected_inventory_sha256): @@ -294,7 +294,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 from pipeline import digest from pathlib import Path raw = read_regular(Path(descriptor['root'])/'manifest.json', MAX_MANIFEST) - runtime_ref = dict(kind='runtime',catalog_id=self.session.snapshot['project_id'], + runtime_ref = dict(kind='runtime',catalog_id=self.session.summary['project_id'], id=digest(descriptor),sha256=digest(descriptor)) value = dict(ref=runtime_ref,descriptor=descriptor,environment=None,manifest=decode(raw)) observed = dict(inventory_sha256=descriptor['manifest_sha256'],file_count=descriptor['file_count'],total_bytes=descriptor['total_bytes']) @@ -302,8 +302,8 @@ def capture(result, before): record = self.materials.capture_runtime(value, raw) return dict(records=[*self.materials.declarations(record),record],result=result) result = self.session.mutate('runtime.capture', expected_inventory_sha256, body, - self.session.snapshot['checkpoint'], key, - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') + self.session.summary['checkpoint'], key, + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') guard.confirmed() return result except BaseException as error: diff --git a/project_session.py b/project_session.py index 46c106cd..aa471c1e 100644 --- a/project_session.py +++ b/project_session.py @@ -9,6 +9,7 @@ import threading import agent_api +from project_store import copy_validated_json from workbench import WorkbenchError @@ -65,7 +66,8 @@ def operation_records(records, *, project_id=None): if project_id is None and overlay is not None: project_id = overlay['destination_project_id'] result = {} - for value in operation_documents(active_records(records, project_id=project_id)): + for value in operation_documents(active_records(records, project_id=project_id, + families=frozenset({OPERATION_FAMILY}))): if overlay is not None and value['format'] == 'project_operation_v1': raise ProjectSessionError('A fork must retain legacy operation receipts as archived historical facts.') if project_id is not None and value.get('project_id', project_id) != project_id: @@ -85,7 +87,7 @@ def __init__(self, store, db, access, artifacts, *, validate_records): self._validate = validate_records self.available = False self._snapshot = None - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.extra_capture = None self.publication_intent = None self.runtime_access = None @@ -176,10 +178,16 @@ def reconcile_selection(self, snapshot): self._recovery_records = None self._recovery_request = None + @property + def summary(self): + """Scalar metadata for the same bound checkpoint as snapshot.""" + with self.lock: + return {key: self._snapshot[key] for key in ('project_id', 'name', 'checkpoint')} + @property def snapshot(self): with self.lock: - return copy.deepcopy(self._snapshot) + return copy_validated_json(self._snapshot) def _validated(self, records): operation_records(records, project_id=self.project_id) @@ -196,18 +204,45 @@ def adopt(self, snapshot): current = self.store.snapshot() if current != snapshot: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._snapshot = copy.deepcopy(snapshot) + self._snapshot = copy_validated_json(snapshot) self.available = True - def check(self): - """Check freshness while a caller holds host/session/projection guards.""" + def _checked(self, read): if not self.available: raise ProjectSessionError('Reopen this project to reconstruct its projection.', 'projection_unavailable', 503) - current = self.store.snapshot() + current = read() if current['checkpoint'] != self._snapshot['checkpoint']: raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) return current + def check_current(self): + """Verify actual canonical bytes without returning an unused record graph. + + The caller holds the same host/session/projection guards as check(). + Store.summary reads and hashes all current records before returning. + """ + self._checked(self.store.summary) + + def check(self): + """Return independent current records under caller-owned projection guards.""" + return self._checked(self.store.snapshot) + + def check_records(self, family): + """Verify every canonical record, then copy one bound record family. + + The complete snapshot, including cross-family closure, was validated + and isolated by adopt/publication. Returned copies retain that validity + inside the trusted owner boundary; they are not new imported raw input. + Exact current bytes and checkpoint must still match before using it. + Callers hold the same ordered guards as check(); no cached alias escapes. + """ + if type(family) is not str: + raise TypeError('Record-family selection requires a string.') + current = self._checked(self.store.summary) + current['records'] = copy.deepcopy([ + item for item in self._snapshot['records'] if item['family'] == family]) + return current + def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capture, *, scope='edit'): """Invoke existing policy, capture fixed-owner upserts, publish, commit SQL. diff --git a/project_sources.py b/project_sources.py index 08939eba..a17b02a5 100644 --- a/project_sources.py +++ b/project_sources.py @@ -6,6 +6,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import hashlib import os @@ -36,7 +37,7 @@ def _summary(saved, origin): schema_version=saved['manifest']['schema_version']) -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): """Validate the fixed source receipt against its exact retained snapshot.""" if operation!='source.register':return False try: @@ -50,8 +51,8 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Source registration requires an attributed timestamp.') if datetime.fromisoformat(result['registered_at']).utcoffset() is None: raise WorkbenchError('Source registration timestamp requires an offset.') - if records is not None: - document=material_records(records).get(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) + if lookup is not None: + document=lookup(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) expected={key:result[key] for key in ('origin_catalog_id','project_id','manifest_sha256','name','schema_version')} if document is None or _summary(document['value'],document['origin_catalog_id'])!=expected: raise WorkbenchError('Source receipt differs from its exact retained snapshot.') @@ -61,6 +62,20 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Malformed retained source operation result.') from None +def receipt_validator(records): + """Own one candidate's retained material facts for source receipt checks.""" + owned = material_records(records) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return material_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class SourceAdapter: def __init__(self,projection): self.projection=projection @@ -117,9 +132,9 @@ def register(self,body,expected_checkpoint,key): self._mutation_identity(expected_checkpoint,key) with self.access.lock,self.session.lock,self.db.lock: actor=self.access.require('edit','source.register',owner_only=True) - self.session.check() + self.session.check_current() saved,raw=self._read_registration(body) - origin=self.session.snapshot['project_id'];invoked=False + origin=self.session.summary['project_id'];invoked=False def invoke(): nonlocal invoked invoked=True;self._bind(saved,raw) @@ -153,7 +168,7 @@ def references(self,snapshotref,offset=0,limit=40): raise WorkbenchError('Invalid bounded source reference page.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('inspect','source.references',snapshot['project_id']) - self.session.check() + self.session.check_current() historical=self.materials.snapshot_ref(snapshot['project_id'],snapshot['manifest_sha256'], catalog_id=snapshot['origin_catalog_id']) try:current=self.sources._saved(snapshot['project_id'],extensions=True) @@ -171,7 +186,7 @@ def references(self,snapshotref,offset=0,limit=40): selected=current['manifest']['data'][offset:offset+limit];budget=[INSPECTION_LIMIT] items=[self.sources._inspect(current['root'],copy.deepcopy(ref),budget,DATA_LIMIT) for ref in selected] self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) - self.session.check();self.access.require('inspect','source.references',snapshot['project_id']) + self.session.check_current();self.access.require('inspect','source.references',snapshot['project_id']) return dict(items=items,total=len(current['manifest']['data']),offset=offset,limit=limit) def import_scientific(self,body,expected_checkpoint,key): @@ -183,7 +198,7 @@ def import_scientific(self,body,expected_checkpoint,key): if type(body['document']) is not dict:raise WorkbenchError('Scientific import metadata must be an object.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_scientific',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],DATA_LIMIT) # ProjectData owns the one canonical mutation and native parser; # current rooted reads above never become an imported capability. @@ -204,7 +219,7 @@ def _capture_reference(self,snapshot,reference_id,expected_sha256,maximum_bytes) expected_sha256,maximum_bytes) if source['manifest_sha256']!=snapshot['manifest_sha256']: raise WorkbenchError('Current source snapshot changed during capture.','stale',409) - self.session.check() + self.session.check_current() return raw,source def import_measurements(self,body,expected_checkpoint,key): @@ -216,7 +231,7 @@ def import_measurements(self,body,expected_checkpoint,key): from retained_measurements import MAX_BYTES with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_measurements',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],MAX_BYTES) source=dict(source,origin_catalog_id=snapshot['origin_catalog_id']) self.access.require('edit','source.import_measurements',snapshot['project_id']) diff --git a/project_store.py b/project_store.py index 0531b8ea..d079ca3f 100644 --- a/project_store.py +++ b/project_store.py @@ -54,6 +54,7 @@ 'project_artifact': 'project_artifact_v1', 'project_operation': 'project_operation_v1', 'project_fork': 'project_fork_v1', + 'project_composition': 'project_composition_v1', 'project_file': 'project_file_v1', 'project_extraction': 'project_extraction_v1', 'generation_history': 'project_generation_history_v1', @@ -62,7 +63,9 @@ 'text_proposal_history': 'project_text_proposal_history_v1', } MAX_RECORDS = 100_000 -MAX_TOTAL_BYTES = 256 * 1024 * 1024 +# Scientific history can retain original runtime/file observations in hundreds +# of Runs. Bound the whole local checkpoint separately from each document. +MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 IDENTIFIER = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$') REQUEST_KEY = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') DOMAIN_FORMAT = re.compile(r'^[a-z][a-z0-9_]{0,95}_v([1-9][0-9]*)$') @@ -121,6 +124,17 @@ def _encode(value): allow_nan=False) + '\n').encode('utf-8') +def copy_validated_json(value): + """Own another tree of already validated JSON without Python graph copying. + + Canonical Store/Session values contain only JSON scalars, lists and string + keyed dictionaries. This is a copy operation inside that proof boundary, + never an input decoder or a substitute for freshness/codec validation. + """ + return json.loads(json.dumps(value, ensure_ascii=True, allow_nan=False, + separators=(',', ':'))) + + def _manifest_raw(manifest): raw = _encode(manifest) if len(raw) > MAX_MANIFEST_BYTES: @@ -465,6 +479,7 @@ def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path): self._origin_path = Path(origin_path) self._meta_fd = None self._manifest = None + self._read_cache = None self._mutex = threading.RLock() self._unknown = False try: @@ -626,12 +641,12 @@ def root_directory(self, *, verify_after=False): if type(verify_after) is not bool: _fail('Root read verification must be an explicit boolean.') with self._mutex: - before = self.snapshot() + before = self._current()[0]['checkpoint'] descriptor = os.dup(self._root_fd) try: yield descriptor self._binding() - if verify_after and self.snapshot()['checkpoint'] != before['checkpoint']: + if verify_after and self._current()[0]['checkpoint'] != before: _fail('Project checkpoint changed during the bounded root read.', 'stale', 409) finally: os.close(descriptor) @@ -639,6 +654,17 @@ def root_directory(self, *, verify_after=False): def _load(self): self._binding() raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) + cached = self._read_cache + if cached is not None and raw == cached[0]: + manifest, loaded = cached[1:] + # Reuse decoding only after checking every actual canonical file. + # Stat identity is not content identity; corruption still fails. + for entry in manifest['inventory']: + self._verified_record_bytes(entry) + if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: + _fail('Project checkpoint changed while reading.', 'stale', 409) + self._binding() + return manifest, loaded manifest = _decode(raw) if type(manifest) is not dict: _fail('Project checkpoint must be an object.') @@ -680,9 +706,7 @@ def _load(self): total += entry['bytes'] if total > MAX_TOTAL_BYTES: _fail('Project records exceed their total byte limit.') - record_raw = self._record_file(family, record_id, digest) - if len(record_raw) != entry['bytes'] or _sha(record_raw) != digest: - _fail('Canonical record bytes changed or are corrupt.', 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + record_raw = self._verified_record_bytes(entry) document = _decode(record_raw) if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') @@ -706,8 +730,16 @@ def _load(self): if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: _fail('Project checkpoint changed while reading.', 'stale', 409) self._binding() + self._read_cache = (raw, manifest, loaded) return manifest, loaded + def _verified_record_bytes(self, entry): + raw = self._record_file(entry['family'], entry['id'], entry['sha256']) + if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: + _fail('Canonical record bytes changed or are corrupt.', + 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + return raw + def _record_file(self, family, record_id, digest, raw=None, sync=False): fds = [] try: @@ -717,14 +749,23 @@ def _record_file(self, family, record_id, digest, raw=None, sync=False): fds.append(parent) name = digest + '.json' if raw is not None: - temporary = '.record-' + uuid.uuid4().hex + '.tmp' - _write_file(parent, temporary, raw) try: - _rename_noreplace(parent, temporary, name) - except FileExistsError: - if _read(parent, name, MAX_RECORD_BYTES) != raw: + existing = _read(parent, name, MAX_RECORD_BYTES) + except FileNotFoundError: + temporary = '.record-' + uuid.uuid4().hex + '.tmp' + _write_file(parent, temporary, raw) + try: + _rename_noreplace(parent, temporary, name) + except FileExistsError: + if _read(parent, name, MAX_RECORD_BYTES) != raw: + _fail('Immutable project revision already has different bytes.') + os.unlink(temporary, dir_fd=parent) + else: + if existing != raw: _fail('Immutable project revision already has different bytes.') - os.unlink(temporary, dir_fd=parent) + # Existing revisions can come from a copied project or an + # interrupted earlier commit. Retain all ordinary sync checks; + # only the disposable write/rename/unlink cycle is avoided. sync = True if sync: _sync_file(parent, name) @@ -763,17 +804,27 @@ def _publish(self, manifest, creating=False): def _snapshot(self, manifest, records): return dict(project_id=manifest['project_id'], name=manifest['name'], - checkpoint=manifest['checkpoint'], records=copy.deepcopy(records)) + checkpoint=manifest['checkpoint'], records=copy_validated_json(records)) + + def _current(self): + try: + manifest, records = self._load() + if manifest != self._manifest: + _fail('Project checkpoint changed; close and reopen.', 'stale', 409) + return manifest, records + except OSError as exc: + _io_error(exc, 'read project snapshot') + + def summary(self): + """Return scalar metadata after the same complete freshness check.""" + with self._mutex: + manifest, _records = self._current() + return {key: manifest[key] for key in ('project_id', 'name', 'checkpoint')} def snapshot(self): with self._mutex: - try: - manifest, records = self._load() - if manifest != self._manifest: - _fail('Project checkpoint changed; close and reopen.', 'stale', 409) - return self._snapshot(manifest, records) - except OSError as exc: - _io_error(exc, 'read project snapshot') + manifest, records = self._current() + return self._snapshot(manifest, records) def reconcile(self, request_key, request_sha256): """Inspect retry identity before a domain mutation; never repeat effects.""" @@ -872,6 +923,7 @@ def recover_checkpoint(self): def close(self): with self._mutex: + self._read_cache = None if self._meta_fd is not None: os.close(self._meta_fd) self._meta_fd = None diff --git a/project_study_execution.py b/project_study_execution.py index 9e19b3c2..2133bbce 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -16,7 +16,7 @@ from pipeline import closed, identifier, digest from project_history import (FAMILY_FORMATS, _audit, _sha, envelope, decode_records as history_records) -from project_material import ProjectMaterial +from project_material import ProjectMaterial, MaterialSnapshot from project_session import operation_records from workbench import WorkbenchError, encode, timestamp @@ -36,8 +36,7 @@ def _invalid(message,code='invalid'): def _facts(records): - view=SimpleNamespace(snapshot={'records':records},check=lambda:None) - return history_records(records,materials=ProjectMaterial(view)) + return history_records(records,materials=MaterialSnapshot(records)) def _intent_key(operation,key): @@ -143,7 +142,7 @@ def _validate_preparation(operation,result,records): trial=result['trial'];origin=trial['material']['catalog_id'] candidate=facts.candidates[(origin,result['candidate_id'])]['candidate'] if candidate['study_id']!=trial['study_id']:_invalid('Preparation candidate belongs to another Study.') - materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check=lambda:None)) + materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check_current=lambda:None)) composition=materials.recipe(trial['material'])['material'] if (composition['model']!=result['model'] or composition['recipe_id']!=result['graph']['id'] or composition['recipe_revision']!=result['graph']['revision']): @@ -203,7 +202,7 @@ def validate_operation_result(operation,result,records=None): if saved is None or any(saved[key]!=reservation[key] for key in RESERVATION_FIELDS if key not in ('state','settled')): _invalid('Admission receipt has no exact canonical reservation.') trial=facts.trials[(origin,result['trial_id'])]['trial'];run=facts.runs[(origin,result['run_id'])]['run'] - view=SimpleNamespace(snapshot={'records':records},check=lambda:None) + view=SimpleNamespace(snapshot={'records':records},check_current=lambda:None) validate_retained_reservation(reservation,trial=trial, composition=ProjectMaterial(view).recipe(trial['material']),run=run) elif operation=='study_execution.stop': @@ -268,7 +267,7 @@ def __init__(self,adapter,phase): def __getattr__(self,name):return getattr(self.source,name) def open(self,body): return self.adapter.projection.source_adapter.register(body, - self.adapter.session.snapshot['checkpoint'],self.phase+'-source') + self.adapter.session.summary['checkpoint'],self.phase+'-source') class _PreparationModels: @@ -279,11 +278,11 @@ def _read_blob(self,ref,item,maximum=40*1024**2):return self.adapter.materials.b def create(self,body): adapter=self.adapter return adapter.materials.mutate_definition('model_definition','models.create',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) + adapter.session.summary['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) def publish(self,identity,body): adapter=self.adapter return adapter.materials.publish_model(adapter.projection.models,identity,body, - adapter.session.snapshot['checkpoint'],self.phase+'-publish') + adapter.session.summary['checkpoint'],self.phase+'-publish') class _PreparationPipelines: @@ -296,7 +295,7 @@ def recipe_material(self,ref,*,verify=False):return self.adapter.materials.recip def create_external(self,body): adapter=self.adapter return adapter.materials.mutate_definition('pipeline_graph','graphs.create_external',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) + adapter.session.summary['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) def prepare_external_material(self,*args,**kwargs):return self.owner.prepare_external_material(*args,**kwargs) def _retain_material(self,prepared): adapter=self.adapter;value=self.owner._retain_material(prepared) @@ -311,7 +310,7 @@ def __init__(self,projection,execution): self.projection,self.execution=projection,execution self.session,self.db,self.history=projection.session,projection.db,projection.history self.materials=projection.materials;self.access=execution.runtime_access - self.origin=self.session.snapshot['project_id'] + self.origin=self.session.summary['project_id'] self._dirty=set();self._revisions={};self._epochs={};self._pending_epochs={} self._package_records=[] self.context=copy.copy(projection.context) @@ -325,12 +324,12 @@ def __init__(self,projection,execution): def _canonical(self,records=None): records=self.session.snapshot['records'] if records is None else records - return history_records(records,materials=self.materials) + return history_records(records,materials=self.materials.read_view(records)) def restore_authoring(self,records=None): """Reconstruct intent only, under the current project guard.""" with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check();facts=self._canonical(records) + self.session.check_current();facts=self._canonical(records) with self.db.transaction(): for (origin,identity),doc in sorted(facts.studies.items()): if origin!=self.origin:continue @@ -574,7 +573,7 @@ def approve(self,study_id,body,expected_checkpoint,key): validate_operation_result(prior['operation'],prior['result'],self.session.snapshot['records']) self._current_grant(prior['result'],current.actor_id) else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self._materialize_trial(study_id,body['trial_id'],body['expected_material_sha256']) def invoke(): grant=self.studies.approve(study_id,body) @@ -590,7 +589,7 @@ def admit(self,study_id,body,expected_checkpoint,key): authority=self.access.require_external_submission() with self.session.access.lock,self.session.lock,self.db.lock: builtin=None - self.session.check();self.refresh_authoring() + self.session.check_current();self.refresh_authoring() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: result=prior['result'] @@ -598,7 +597,7 @@ def admit(self,study_id,body,expected_checkpoint,key): 'SELECT run_id FROM model_attempt_reservations WHERE run_id=%s',(result.get('run_id'),)).fetchone() is None): _invalid('A historical admission receipt cannot create a current local attempt.','unavailable') else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self.require_current_epoch(self.studies._study(study_id)) self.reject_historical_request(study_id,authority.actor_id,key,None) trial=self._materialize_trial(study_id,body['trial_id']) @@ -627,7 +626,7 @@ def invoke(): # Qualification may launch the explicit probe. It finishes # before Session opens the finite admission transaction. with preparation.admission(pipelines,ref,expected_checkpoint,key): - return self._mutate('study_execution.admit',study_id,body,self.session.snapshot['checkpoint'],key,invoke,'inspect') + return self._mutate('study_execution.admit',study_id,body,self.session.summary['checkpoint'],key,invoke,'inspect') return self._mutate('study_execution.admit',study_id,body,expected_checkpoint,key,invoke,'inspect') def stop(self,study_id,body,expected_checkpoint,key): @@ -646,7 +645,7 @@ def invoke(): # cancellation phase. Receipt replay does not repeat effects. for run_id in invoked: cancel_key='study-stop-'+hashlib.sha256((key+':'+run_id).encode()).hexdigest() - try:self.execution.cancel_run(run_id,self.session.snapshot['checkpoint'],cancel_key) + try:self.execution.cancel_run(run_id,self.session.summary['checkpoint'],cancel_key) except BaseException as error: error.study_stop_receipt=copy.deepcopy(result) raise @@ -684,7 +683,7 @@ def reconcile_preparation(self,study_id,body,expected_checkpoint): if complete.get('native_receipt'): effective=_preparation_body(intent,body['preparation_key']) effective.pop('request_key');effective['actor_id']=intent['actor_id'] - result=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],body['preparation_key'], + result=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],body['preparation_key'], lambda:copy.deepcopy(complete['result']),'cancel') else:result=copy.deepcopy(complete['result']) guard.confirmed() @@ -704,7 +703,7 @@ def abandon_preparation(self,study_id,body,expected_checkpoint,key): operation,current_intent,complete=_preparation_reference(self.session.snapshot['records'],study_id,body['preparation_key']) if current_intent!=intent:_invalid('Preparation intent changed during guarded reconciliation.','conflict') if complete is not None:_invalid('Preparation is complete; reconcile its acknowledgement without abandoning science.','conflict') - observed_checkpoint=self.session.snapshot['checkpoint'] + observed_checkpoint=self.session.summary['checkpoint'] def invoke(): actor=self.access.require('cancel','studies.abandon_preparation',study_id,owner_only=True) if actor.actor_id!=current.actor_id:_invalid('Abandonment caller changed.','conflict') @@ -735,14 +734,14 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): input_sha=digest(dict(study_id=study_id,body=effective)) intent_key=_intent_key(operation,key);phase=intent_key.removesuffix('-intent') with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check() + self.session.check_current() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: # Current authority is checked above. Historical completion # acknowledges science and cannot repeat filesystem effects. return self._mutate(operation,study_id,effective,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Completed preparation replay invoked effects.')),'edit') - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') receipts=operation_records(self.session.snapshot['records']) if any(_intent_key(other,key) in receipts for other in PREPARATIONS if other!=operation): _invalid('Preparation key already identifies another preparation operation.','conflict') @@ -779,7 +778,7 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): publication=SimpleNamespace(runtime_snapshot=snapshot,before_write=guard.publication_started, read_recipe_blob=self.materials.blob) result=_prepare_operation(studies,study_id,dict(body,request_key=key),kind,publication=publication) - value=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],key, + value=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(result),'edit') guard.confirmed() return value diff --git a/project_text.py b/project_text.py index d9502cb6..fc778528 100644 --- a/project_text.py +++ b/project_text.py @@ -252,7 +252,7 @@ def restore(self, snapshot=None): encode(row['annotation']) if row['annotation'] is not None else None, row['review'], row['revision'], row['created_at'], row['updated_at'])) for state in doc['history']: self._workbench._history(state) - if self.store.snapshot()['checkpoint'] != current['checkpoint']: + if self.store.summary()['checkpoint'] != current['checkpoint']: raise ProjectTextError('Project changed during reconstruction.', 'stale_project', 409) self._snapshot, self.available = current, True return copy.deepcopy(current) diff --git a/project_transfer.py b/project_transfer.py index 932d0963..91848559 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -226,6 +226,18 @@ def resolve_payload(self,sha256,size): raw=self.read_root(location['root'],location['relative'],MAX_SOURCE_FILE) if len(raw)!=size or digest(raw)!=sha256:fail('External source payload differs from its declaration.') return raw + def read_bound_payload(self,sha256,*,maximum_bytes=MAX_SOURCE_FILE): + """Read exact operator-bound bytes; None means no explicit binding. + + Historical paths or root identities never create this capability. A + changed, oversized or unavailable bound file remains an error, even if + another working file exists at the original historical location. + """ + self.require();sha(sha256) + location=self._payload_locations.get(sha256) + if location is None:return None + if location['bytes']>maximum_bytes:fail('Bound original source exceeds its owner bound.','too_large',413) + return self.resolve_payload(sha256,location['bytes']) @contextmanager def payload_fd(self,sha256,size): sha(sha256) @@ -708,7 +720,7 @@ def replay(opened): try: store=ProjectStore.create_at(destination_parent_fd,stage_name,name,'extraction-stage.'+digest(request_key.encode())[:32],request_digest,root_path=root_path) with store.root_directory() as owned_fd:stage_identity=_fd_identity(owned_fd) - destination=store.snapshot()['project_id'];created_at=timestamp() + destination=store.summary()['project_id'];created_at=timestamp() from project_data_fork import fork_records as fork_data from project_material_fork import fork_records as fork_material original=copy.deepcopy(records);mappings=[] @@ -743,7 +755,7 @@ def replay(opened): with source.artifacts.open_verified(value,maximum_bytes=value['bytes']) as fd:artifacts.publish_fd(fd,value,maximum_bytes=value['bytes']) with store.root_directory() as fd:materialize_captured(records,destination,fd) _validate_candidate(records,destination,artifacts) - committed=store.commit(records,store.snapshot()['checkpoint'],request_key,request_digest) + committed=store.commit(records,store.summary()['checkpoint'],request_key,request_digest) verification=_cold_verify(store,source) store.save(committed['checkpoint']);_assert_unchanged(source) _rename_noreplace(destination_parent_fd,stage_name,directory);published=True;os.fsync(destination_parent_fd) diff --git a/project_transfer_controller.py b/project_transfer_controller.py index 7362e0ea..3ece224e 100644 --- a/project_transfer_controller.py +++ b/project_transfer_controller.py @@ -252,7 +252,7 @@ def extract(self, binding, report_sha256, destination_root, directory, name, req fail('Recover the selected workspace before reconciling extraction.', 'projection_unavailable', 503) guards.enter_context(projection.session.lock) guards.enter_context(projection.db.lock) - projection.session.check() + projection.session.check_current() opened = self.workspace._store receipt = self._transfer.extract(self._selection, report_sha256, destination_fd, directory, name, request_key, opened_store=opened) except BaseException as error: diff --git a/project_workspace.py b/project_workspace.py index 83f9597b..b006dbe2 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -5,6 +5,7 @@ The installation catalog holds local credentials; it is never portable content. """ import copy +from contextlib import ExitStack import hashlib import json import os @@ -121,7 +122,7 @@ def _state(self): project = None if self._recovery is None and self._store is not None: try: - snapshot = self._store.snapshot() + snapshot = self._store.summary() except ProjectStoreError as error: if error.code not in ('unknown', 'outcome_unknown', 'stale', 'stale_project', 'invalid', 'unsupported', 'unavailable'): @@ -183,7 +184,7 @@ def _bound(self, arguments, *, may_be_empty=False, check_checkpoint=True, allow_ if arguments.get('expected_checkpoint') is not None: raise WorkbenchError('The empty workspace has no checkpoint.', 'stale_project', 409) return - snapshot = self._store.snapshot() + snapshot = self._store.summary() expected = arguments.get('expected_checkpoint', arguments.get('checkpoint')) if check_checkpoint and expected != snapshot['checkpoint']: raise WorkbenchError('The project checkpoint changed. Refresh before acting.', 'stale_project', 409) @@ -205,7 +206,7 @@ def _root(self, alias, relative): def _projection_for(self, store): identity = uuid.uuid4().hex path = self.path / '.project-projections' / identity - snapshot = store.snapshot() + snapshot = store.summary() projection = Metadata(path, dsn=self._dsn, dataset_id=snapshot['project_id'], projection_id=identity) try: owner = ProjectProjection(store, projection, self.access, @@ -226,7 +227,7 @@ def _retained_projection_for(self, store, projection_id): if type(projection_id) is not str or not re.fullmatch('[a-f0-9]{32}', projection_id): raise WorkbenchError('Invalid original local projection identity.', 'unavailable', 409) db = Metadata(self.path / '.project-projections' / projection_id, dsn=self._dsn, - create=False, dataset_id=store.snapshot()['project_id'], projection_id=projection_id) + create=False, dataset_id=store.summary()['project_id'], projection_id=projection_id) try: return ProjectProjection(store, db, self.access, {alias: value[0] for alias, value in self._roots.items()}, @@ -242,7 +243,7 @@ def _detach(self): self._projection.execution.assert_quiescent() if self._store is not None: # Prove pending project publications and their full closure first. - self._store.save(self._store.snapshot()['checkpoint']) + self._store.save(self._store.summary()['checkpoint']) if self._projection is not None: self._projection.close() if self._store is not None: @@ -271,14 +272,14 @@ def _recover_current(self, arguments): store = projection = None try: store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) - snapshot = store.snapshot() + snapshot = store.summary() if snapshot['project_id'] != self._last_project['project_id']: raise WorkbenchError('Recovery destination has a different project identity.', 'stale_project', 409) # A reread reconciles identity; only a successful durability barrier # resolves a prior checkpoint/directory fsync failure. store.save(snapshot['checkpoint']) projection, text = self._projection_for(store) - snapshot = store.snapshot() + snapshot = store.summary() if self._projection is not None: self._projection.close() self._store, self._projection, self._text = store, projection, text @@ -331,7 +332,7 @@ def _select(self, action, arguments): if store.reconcile(arguments['request_key'], digest) is None: raise error reconciled = True - snapshot = store.snapshot() + snapshot = store.summary() if reconciled: store.save(snapshot['checkpoint']) created = snapshot @@ -341,7 +342,7 @@ def _select(self, action, arguments): self._detach() self._store, self._projection, self._text = store, projection, text self._selection = selection - snapshot = store.snapshot() + snapshot = store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} self._recovery = None self._binding = uuid.uuid4().hex @@ -365,7 +366,7 @@ def _select(self, action, arguments): raise def _record_result(self, record, *, before_revision=None, history_limit=1): - snapshot = self._store.snapshot() + snapshot = self._store.summary() history = [item for item in self._text.history(record['id']) if item['revision'] <= record['revision'] and (before_revision is None or item['revision'] < before_revision)] @@ -377,6 +378,12 @@ def _record_result(self, record, *, before_revision=None, history_limit=1): next_history_before_revision=page[-1]['revision'] if len(history) > len(page) else None) def _save_as(self, arguments): + return self._publish_independent(arguments) + + def _compose(self, arguments): + return self._publish_independent(arguments, composed=True) + + def _publish_independent(self, arguments, *, composed=False): """Publish a new independent project, then select a fresh inert cache.""" from functools import partial from project_fork import publish_fork @@ -390,13 +397,35 @@ def _save_as(self, arguments): self._store.save(arguments['expected_checkpoint']) published = None incoming = projection = None + retained = ExitStack() try: - published = publish_fork(self._store, source.artifacts, root_fd, arguments['directory'], - root_path=root_path, name=arguments['name'], - expected_checkpoint=arguments['expected_checkpoint'], request_key=arguments['request_key'], - quiescent=True, build_candidate=partial(build_candidate, created_at=timestamp(), - id_factory=lambda _family, _source_id, _revision: uuid.uuid4().hex), - validate_candidate=lambda records, identity: source.validate_records(records, project_id=identity)) + if composed: + from project_composition import publish_composition + from project_artifacts import ProjectArtifacts + others=[];identities={self._store.summary()['project_id']} + for selected in arguments['sources']: + if selected['project_id'] in identities: + raise WorkbenchError('Composition source identity repeats.','conflict',409) + identities.add(selected['project_id']) + directory,descriptor=self._root(selected['root'],selected['directory']) + store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory)) + actual=store.summary() + if actual['project_id']!=selected['project_id'] or actual['checkpoint']!=selected['checkpoint']: + raise WorkbenchError('Selected composition source checkpoint differs.','stale_project',409) + artifacts=retained.enter_context(ProjectArtifacts(store)) + others.append(dict(store=store,artifacts=artifacts,checkpoint=selected['checkpoint'])) + published=publish_composition(self._store,source.artifacts,others,root_fd,arguments['directory'], + root_path=root_path,name=arguments['name'],expected_checkpoint=arguments['expected_checkpoint'], + request_key=arguments['request_key'],quiescent=True,created_at=timestamp(), + id_factory=lambda *_:uuid.uuid4().hex, + validate_candidate=lambda records,identity,artifacts:source.validate_records(records,project_id=identity,artifacts=artifacts)) + else: + published = publish_fork(self._store, source.artifacts, root_fd, arguments['directory'], + root_path=root_path, name=arguments['name'], + expected_checkpoint=arguments['expected_checkpoint'], request_key=arguments['request_key'], + quiescent=True, build_candidate=partial(build_candidate, created_at=timestamp(), + id_factory=lambda _family, _source_id, _revision: uuid.uuid4().hex), + validate_candidate=lambda records, identity: source.validate_records(records, project_id=identity)) # A filesystem editor can act outside the workspace lock. Leave the # complete fork unselected if its source working files changed. source.files.require_clean() @@ -415,7 +444,10 @@ def _save_as(self, arguments): request_key=arguments['request_key'], reconciled=published['reconciled'], mapping_count=len(overlay['mapping']), copied_count=len(overlay['artifacts']['copied']), missing_regenerable_count=len(overlay['artifacts']['missing_regenerable'])) - return dict(state=self._state(), fork=summary) + result=dict(state=self._state(), fork=summary) + if composed: + result['sources']=[{key:item[key] for key in ('project_id','checkpoint')} for item in published['composition']['sources']] + return result except BaseException as error: if projection is not None and projection is not self._projection: projection.close() @@ -430,11 +462,13 @@ def _save_as(self, arguments): elif receipt is not None: receipt = dict(receipt, root=arguments['root']) if receipt is not None: - failure = WorkbenchError('Independent project was published but could not be selected; retry this exact Save As.', + failure = WorkbenchError('Independent project was published but could not be selected; reconcile this exact request.', 'project_selection_unavailable', 503) failure.receipt = receipt raise failure from error raise + finally: + retained.close() def dispatch(self, name, arguments, *, selected_version=None): """Pin workspace -> current host authority -> projection through return. @@ -457,7 +491,7 @@ def dispatch(self, name, arguments, *, selected_version=None): if scope == 'execute' and action in ('execution_submit_external', 'execution_study_admit'): self.access.require_external_submission() else: - self.access.require(scope, name, owner_only=action in ('new', 'open', 'save', 'save_as', 'close')) + self.access.require(scope, name, owner_only=action in ('new', 'open', 'save', 'save_as', 'compose', 'close')) if selected_version == 1 and action in ('get', 'annotate'): self._bound(arguments, check_checkpoint=action == 'get') record = self._projection.data._require_record(arguments['record_id']) @@ -466,7 +500,7 @@ def dispatch(self, name, arguments, *, selected_version=None): 'unsupported', 409) if name in agent_api.PROJECT_FILES_M4 and not M4_ENABLED: raise WorkbenchError('This project capability is not yet available.', 'unavailable', 409) - if name in (agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | agent_api.PROJECT_FILES_M4) - {'project_files.save_as'}: + if name in (agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | agent_api.PROJECT_FILES_M4) - {'project_files.save_as','project_files.compose'}: if ((name in agent_api.PROJECT_FILES_M2 and not M2_ENABLED) or (name in agent_api.PROJECT_FILES_M3 and not M3_ENABLED) or (name in agent_api.PROJECT_FILES_M4 and not M4_ENABLED)): @@ -475,13 +509,17 @@ def dispatch(self, name, arguments, *, selected_version=None): from project_dispatch import dispatch try: value = dispatch(self._projection, name, arguments) + if (name=='project_files.measurement_compare' + and value['value']['observation_count']>agent_api.measurement_comparison_row_limit(selected_version)): + raise WorkbenchError('Comparison exceeds the selected API version observation budget. Select comparison contract version 2 for larger complete populations.', + 'too_large',413) if name == 'project_files.execution_health' and selected_version == 1: value = {key: item for key, item in value.items() if key != 'native_phases'} except BaseException as error: if getattr(error, 'code', None) in ('unknown', 'outcome_unknown', 'projection_unavailable'): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} result = dict(binding=self._binding, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint'], value=value) @@ -496,7 +534,7 @@ def dispatch(self, name, arguments, *, selected_version=None): if prior is not None: if prior[0] != signature: raise WorkbenchError('Request key already has different input.', 'conflict', 409) - prior_state = prior[1]['state'] if action == 'save_as' else prior[1] + prior_state = prior[1]['state'] if action in ('save_as','compose') else prior[1] if prior_state != self._state(): raise WorkbenchError('This lifecycle receipt belongs to an obsolete workspace state.', 'stale_project', 409) result = copy.deepcopy(prior[1]) @@ -510,6 +548,8 @@ def dispatch(self, name, arguments, *, selected_version=None): result = self._state() elif action == 'save_as': result = self._save_as(arguments) + elif action == 'compose': + result = self._compose(arguments) elif action == 'close': self._bound(arguments) self._detach() @@ -542,11 +582,11 @@ def dispatch(self, name, arguments, *, selected_version=None): raise WorkbenchError('Unsupported project-file operation.', 'unsupported', 400) except BaseException as error: code = getattr(error, 'code', None) - if action != 'save_as' and code in ('unknown', 'outcome_unknown', 'projection_unavailable'): + if action not in ('save_as','compose') and code in ('unknown', 'outcome_unknown', 'projection_unavailable'): reason = 'unknown_publication' if code in ('unknown', 'outcome_unknown') else 'projection_unavailable' self._recovery_required(reason, arguments.get('request_key')) raise - if arguments.get('request_key') and action in ('new', 'open', 'save', 'save_as', 'close'): + if arguments.get('request_key') and action in ('new', 'open', 'save', 'save_as', 'compose', 'close'): self._receipts[key] = (signature, copy.deepcopy(result)) if len(agent_api.encoded(result)) > agent_api.RESPONSE_LIMIT: raise WorkbenchError('This project result exceeds the synchronous response bound; canonical work is retained.', @@ -589,14 +629,14 @@ def _dispatch_draining(self, name, arguments): raise WorkbenchError('The retained physical recovery owner changed.', 'local_custody_unavailable', 409) recovering = True if incoming is not projection and name == 'project_files.execution_reconcile_current': - value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.snapshot['checkpoint']) + value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.summary['checkpoint']) else: internal = arguments if recovery_operation and recovering and name != 'project_files.execution_reconcile_current': # Public CAS above uses the last acknowledged workspace # view. Only this qualified local owner sees its exact # selected/attempted checkpoint for finite recovery. - internal = dict(arguments, expected_checkpoint=incoming.session.snapshot['checkpoint']) + internal = dict(arguments, expected_checkpoint=incoming.session.summary['checkpoint']) if name == 'project_files.runtime_capture' and recovering: options = {'reserve_bytes': internal['reserve_bytes']} if 'reserve_bytes' in internal else {} value = incoming.runtime.recover_capture({'source': internal['source']}, @@ -620,7 +660,7 @@ def _dispatch_draining(self, name, arguments): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise with self.access.lock, projection.session.lock, projection.db.lock: - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} if name in _RECOVERY_OPERATIONS: self._recovery = None diff --git a/projects.py b/projects.py index 4a55f852..5860d846 100644 --- a/projects.py +++ b/projects.py @@ -105,6 +105,22 @@ def manifest(value): return value +def decode_retained_manifest(value): + """Keep the closed schema1 activity field as inert historical content. + + Current registration still uses manifest(), which does not accept this + retired field. Reading a retained manifest grants no activity-file access. + """ + import copy + if type(value) is dict and value.get('schema_version') == 1 and 'activity_path' in value: + relative_path(value['activity_path']) + current = {key: item for key, item in value.items() if key != 'activity_path'} + manifest(current) + else: + manifest(value) + return copy.deepcopy(value) + + def decode_retained_snapshot(value): """Validate immutable registration facts without binding any current root.""" import copy @@ -115,7 +131,7 @@ def decode_retained_snapshot(value): if type(value['manifest_sha256']) is not str or not SHA.fullmatch(value['manifest_sha256']):raise WorkbenchError('Invalid retained manifest hash.') root=value['root_identity'] if type(root) is not list or len(root)!=2 or any(type(number) is not int or number<0 for number in root):raise WorkbenchError('Invalid historical root identity.') - document=manifest(value['manifest']) + document=decode_retained_manifest(value['manifest']) if document['id']!=value['id'] or document['name']!=value['name']:raise WorkbenchError('Retained snapshot manifest identity differs.') return copy.deepcopy(value) diff --git a/recipe_contract.py b/recipe_contract.py index 2c1f85fc..2c3de494 100644 --- a/recipe_contract.py +++ b/recipe_contract.py @@ -830,8 +830,23 @@ def prepare(projects, graph, execution_sha256, profile=None, *, releases=None, r 'graph_revision':graph['revision'], 'execution_sha256':execution_sha256}, captured +def _legacy_torch_policy(material): + return (material['schema_version']==1 and material['recipe']['schema_version']==1 + and material['recipe']['runtime']==TORCH_PROFILE + and material['environment']=={'LANG':'C.UTF-8'}) + + def validate_material(material): """Cold structural custody checks; no filesystem access or current-code comparison.""" + return _validate_material(material, retained=False) + + +def validate_retained_material(material): + """Validate current and original schema1 execution facts without admission.""" + return _validate_material(material, retained=True) + + +def _validate_material(material, *, retained): closed(material, ('schema_version', 'catalog_id', 'project', 'graph_execution_sha256', 'recipe', 'parameters', 'inputs', 'captures', 'runtime', 'host_files', 'permissions', 'environment', 'resources'), ('native_policies','cpu_snapshot','torch_runtime','extension_binding')) if type(material['schema_version']) is not int or material['schema_version'] not in (1,2,3) or not isinstance(material['catalog_id'], str) or not material['catalog_id']: @@ -856,7 +871,11 @@ def receipt(value): raise WorkbenchError('Material leaves no generated context capacity.') if encode(validate_parameters(definition, material['parameters'])) != encode(material['parameters']): raise WorkbenchError('Retained parameters must include every typed value.') - if material['resources'] != definition['resources'] or material['environment'] != ({} if definition['runtime']==TORCH_PROFILE else {'LANG':'C.UTF-8'}) or material['permissions'] != {'filesystem':'trusted-local', 'network':'not_isolated'}: + expected_environment = {} if definition['runtime']==TORCH_PROFILE else {'LANG':'C.UTF-8'} + legacy_environment = retained and _legacy_torch_policy(material) + if (material['resources'] != definition['resources'] + or material['environment'] != expected_environment and not legacy_environment + or material['permissions'] != {'filesystem':'trusted-local', 'network':'not_isolated'}): raise WorkbenchError('Material execution policy is inconsistent.') if not isinstance(material['inputs'], dict) or set(material['inputs']) != set(definition['inputs']): raise WorkbenchError('Material input inventory differs.') expected = {'source/'+item['path']:item['sha256'] for item in definition['sources']} @@ -991,6 +1010,10 @@ def receipt(value): def spec_for_material(material, staging, *, context=None): """Pure reconstruction: retained approvals never depend on current source/runtime.""" validate_material(material) + return _spec_for_material(material, staging, context=context) + + +def _spec_for_material(material, staging, *, context=None): staging = Path(staging).absolute() output = staging/'output' definition = material['recipe']; runtime = material['runtime'] @@ -1041,6 +1064,23 @@ def validate_spec(material, spec, staging, *, context=None): if is_evidence(material['recipe']) and context is None: raise WorkbenchError('Final model-evidence specification requires authoritative attempt context.') expected = spec_for_material(material, staging, context=context) + return _validate_spec(expected, spec) + + +def validate_retained_spec(material, spec, staging, *, context=None): + """Prove an exact historical spec; never select this path for new work.""" + validate_retained_material(material) + if is_evidence(material['recipe']) and context is None: + raise WorkbenchError('Final model-evidence specification requires authoritative attempt context.') + expected = _spec_for_material(material, staging, context=context) + if _legacy_torch_policy(material): + # This original policy predates descriptor duplication in the spec. + # Its descriptor remains mandatory in the exact retained material. + expected.pop('torch_runtime') + return _validate_spec(expected, spec) + + +def _validate_spec(expected, spec): actual = {key:value for key,value in spec.items() if key not in ('attempt_id', 'request_sha256')} if encode(actual) != encode(expected): raise WorkbenchError('Execution specification differs from the approved material.', 'conflict', 409) diff --git a/retained_measurements.py b/retained_measurements.py index 11188802..4044b017 100644 --- a/retained_measurements.py +++ b/retained_measurements.py @@ -23,6 +23,10 @@ MAX_SOURCE_TOTAL = 16 * 1024 * 1024 MAX_RESPONSE_BYTES = 256 * 1024 MAX_ROWS = 100000 +MAX_COMPARISON_SOURCES = 16 +MAX_COMPARISON_SUBJECTS = 256 +MAX_COMPARISON_BYTES = 128 * 1024 * 1024 +MAX_COMPARISON_ROWS = 250000 STATES = ('measured', 'failed', 'unsupported', 'unavailable') ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') SHA = re.compile(r'^[a-f0-9]{64}$') @@ -137,6 +141,7 @@ def __init__(self, raw): if type(raw) is not bytes or not 0 < len(raw) <= MAX_BYTES: fail('A bounded complete byte snapshot is required.', 'too_large', 413) self.sha256 = hashlib.sha256(raw).hexdigest() + self.bytes = len(raw) doc = decode(raw) exact(doc, ('format', 'schema_version', 'title', 'scope', 'dimensions', 'subjects', 'cases', 'metrics', 'observations', 'sources')) @@ -348,3 +353,112 @@ def _inspection_reference(self,reference): state='exceeds_inspection_bound';value=None return {'reference':copy.deepcopy(reference),'source_sha256':self._sources[reference['source_id']]['sha256'], 'state':state,'value':value} + + +class MeasurementComparison: + """One bounded scientific comparison over immutable, source-bound tables. + + The native custody owner supplies validated locator/table pairs. Compatibility + depends on original definitions and actual case evidence, not display names. + Returned rows keep their packet locator; no observations or counts are pooled + across subjects. Pagination affects rows only, never the maximum denominator. + """ + def __init__(self, tables): + if type(tables) is not list or not 1 <= len(tables) <= MAX_COMPARISON_SOURCES: + fail('Choose 1 through 16 comparison sources.', 'too_large', 413) + self._tables = [] + self._subjects = {} + seen = set() + basis = None + observations = raw_bytes = 0 + for locator, table in tables: + exact(locator, ('origin_catalog_id', 'packet_sha256')) + if (type(locator['origin_catalog_id']) is not str + or not re.fullmatch('[a-f0-9]{32}', locator['origin_catalog_id']) + or not isinstance(table, MeasurementTable) or locator['packet_sha256'] != table.sha256): + fail('Comparison source differs from its validated packet.') + key = (locator['origin_catalog_id'], locator['packet_sha256']) + if key in seen: + fail('Comparison repeats a source.') + seen.add(key) + raw_bytes += table.bytes + if raw_bytes > MAX_COMPARISON_BYTES: + fail('Comparison exceeds 128 MiB of original packets.', 'too_large', 413) + metadata = table.metadata() + cases = [] + for case in table._cases.values(): + reference = case['reference'] + evidence = table._reference(reference) + cases.append(dict(id=case['id'], dimensions=case['dimensions'], evidence=evidence, + opaque_source_sha256=table._sources[reference['source_id']]['sha256'] + if reference['pointer'] is None else None)) + current = dict(scope=metadata['scope_definition'], dimensions=metadata['dimensions'], + metrics=[table._definition(metric, ('id','label','units')) for metric in table._metrics.values()], + cases=cases) + # Canonical source bytes distinguish Boolean/numeric values too; + # ordinary Python equality is not the scientific identity relation. + identity = digest(current) + if basis is None: + basis = identity + self._header = dict(format='retained_measurement_comparison_v1', + scope=metadata['scope'], scope_definition=metadata['scope_definition'], + dimensions=metadata['dimensions'], metrics=metadata['metrics'], + definition_source=copy.deepcopy(locator), case_population_sha256=digest(cases), + case_count=metadata['case_count']) + elif identity != basis: + fail('Comparison sources have different scope, metric, dimension or original case definitions.', 'incompatible', 409) + for subject in table._subjects.values(): + if subject['id'] in self._subjects: + fail('Comparison sources overlap a subject identity.', 'conflict', 409) + self._subjects[subject['id']] = dict(copy.deepcopy(subject), source=copy.deepcopy(locator)) + observations += metadata['observation_count'] + if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_COMPARISON_ROWS: + fail('Comparison exceeds 256 subjects or 250,000 observations.', 'too_large', 413) + self._tables.append((copy.deepcopy(locator), table)) + self._header.update(subjects=list(self._subjects.values()), observation_count=observations) + + def read(self, metric_id=None, *, subject_ids=None, filters=None, offset=0, limit=50): + result = copy.deepcopy(self._header) + result['query'] = None + if metric_id is None: + if subject_ids is not None or filters is not None or offset != 0 or limit != 50: + fail('A comparison selection requires an explicit metric.') + return self._bounded(result) + selected = list(self._subjects) if subject_ids is None else subject_ids + if (type(selected) is not list or not selected + or any(type(value) is not str or value not in self._subjects for value in selected) + or len(set(selected)) != len(selected)): + fail('Choose existing unique comparison subjects.') + if type(offset) is not int or not 0 <= offset <= 2**53-1 or type(limit) is not int or not 1 <= limit <= 100: + fail('Comparison page bounds must be finite integers.') + summaries = {} + rows = [] + total = 0 + template = None + for locator, table in self._tables: + own = [identity for identity in selected if identity in table._subjects] + if not own: + continue + page = table.query(metric_id, subject_ids=own, filters=filters, offset=0, limit=1) + if template is None: + template = page + for summary in page['summary']: + summaries[summary['subject_id']] = dict(summary, source=copy.deepcopy(locator)) + start = max(0, offset-total) + if total+page['total'] > offset and len(rows) < limit: + current = table.query(metric_id, subject_ids=own, filters=filters, + offset=start, limit=limit-len(rows)) + rows.extend(dict(row, source=copy.deepcopy(locator)) for row in current['rows']) + total += page['total'] + metric = next(item for item in self._header['metrics'] if item['id'] == metric_id) + result['query'] = dict(metric=copy.deepcopy(metric), definition=template['definition'], + filters=template['filters'], subject_ids=list(selected), + summary=[summaries[identity] for identity in selected], rows=rows, + total=total, offset=offset, limit=limit, sampling='none') + return self._bounded(result) + + @staticmethod + def _bounded(value): + if len(encode(value)) > MAX_RESPONSE_BYTES: + fail('Comparison response exceeds its bound; choose fewer sources or a smaller page.', 'too_large', 413) + return value diff --git a/runs.py b/runs.py index 8d8a9398..c4cfcfc7 100644 --- a/runs.py +++ b/runs.py @@ -193,7 +193,7 @@ def validate_retained_run(run, *, catalog_id, run_id, request_key, request_sha25 or Path(declared).parent.name != 'workspace-runs'): raise ValueError('Invalid retained external staging location.') admitted_context=extension_context - recipe_contract.validate_spec(run['material'], run['spec'], declared,context=admitted_context) + recipe_contract.validate_retained_spec(run['material'], run['spec'], declared,context=admitted_context) if (run['graph'].get('recipe') != 'external_recipe' or run['material']['catalog_id'] != catalog_id or digest(run['material']) != run['material_sha256'] diff --git a/runtime_snapshots.py b/runtime_snapshots.py index 09ce69ca..e1795760 100644 --- a/runtime_snapshots.py +++ b/runtime_snapshots.py @@ -316,7 +316,14 @@ def create(self,config,expected_inventory_sha256,*,quiesced=False,reserve_bytes= try: for index,row in enumerate(manifest['files']): if index%128==0 and shutil.disk_usage(self.path).free1}, + 'ROUTES':{name:{'method':agent_api.operation(name)['method'],'path':agent_api.operation(name)['path'], + 'mutation':agent_api.operation(name)['side_effects']} + for name in sorted(agent_api.PROJECT_FILES_M2|agent_api.PROJECT_FILES_M3|agent_api.PROJECT_FILES_M4)}} + for label,value in tables.items(): + pattern=r'(/\* PROJECT_FILES_'+label+r'_START \*/\s*)(.*?)(\s*/\* PROJECT_FILES_'+label+r'_END \*/)' + generated=json.dumps(value,sort_keys=True,indent=2) + original,count=re.subn(pattern,lambda m:m[1]+generated+m[3],original,flags=re.S) + if count!=1:raise ValueError('Exactly one installed '+label+' table required') + return original + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--check',action='store_true') + args=parser.parse_args();path=ROOT/'static/project-files.js';original=path.read_text();expected=render(original) + if args.check: + if original!=expected:raise SystemExit('Installed project-file browser contracts are stale') + elif original!=expected:path.write_text(expected) diff --git a/static/project-files.css b/static/project-files.css index 515d9ca5..c18e2e30 100644 --- a/static/project-files.css +++ b/static/project-files.css @@ -10,3 +10,13 @@ #project-runtime-files,#project-runtime-summary,#project-runtime-pending,#project-runtime-layout,#project-runtime-status{overflow-wrap:anywhere;min-width:0}#project-runtime-layout,#project-runtime-status{white-space:pre-wrap;max-height:32rem;overflow:auto} #project-candidate-preparation-result,#project-candidate-preparation-status{overflow-wrap:anywhere;min-width:0}#project-candidate-preparation-result{white-space:pre-wrap;max-height:32rem;overflow:auto} + +#project-portfolio-source,#project-portfolio-detail,#project-portfolio-count{overflow-wrap:anywhere;min-width:0} +#project-portfolio-detail p{white-space:pre-wrap;max-width:85ch} +#project-portfolio-detail pre{white-space:pre-wrap;overflow-wrap:anywhere;max-height:32rem;overflow:auto} +#project-portfolio-detail details{margin:1rem 0} +#project-portfolio-detail h4{margin-bottom:.25rem} +#project-portfolio-detail h3{scroll-margin-top:1rem} +#project-portfolio-filter-form{display:grid;grid-template-columns:1fr 2fr 1fr;gap:.7rem;align-items:end} +#project-portfolio-filter-form button{justify-self:start} +@media(max-width:760px){#project-portfolio-filter-form{grid-template-columns:1fr}} diff --git a/static/project-files.html b/static/project-files.html index de9c6f1a..52f1ccca 100644 --- a/static/project-files.html +++ b/static/project-files.html @@ -14,6 +14,7 @@ + @@ -50,6 +51,7 @@

Project file

No unsubmitted edits.

+

Prepare a Candidate package or verifier

Choose a current Study, Candidate and captured CPU runtime. Preparation saves an inert Trial and exact sources; approval and Run admission remain separate actions.

diff --git a/static/project-files.js b/static/project-files.js index 708658ed..3146719e 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -14,6 +14,7 @@ "project_files.candidate": "c519ff04bc06d5e538d36b7f278862d8f282d0e7b97dd7f262444c384a50f854", "project_files.candidate_select": "f043be6923760a208ca50d02a1ff4b60eadf09d48a034acdb18749806a537080", "project_files.close": "95d0099ec2a378a1bbb65676799dd3baae2f36b838301f7aa165b116fba5edf6", + "project_files.compose": "b1dcbbd4ad7ebe17b17e11238f99a93408da24d0d411527e187900cc0f089105", "project_files.data_annotate": "289270f5cb946765c5ced22fbbd18765bb55743145014c81cf54d7a6ba3025ea", "project_files.data_availability": "92332c04973a4fa45d47e384717152ebc4a9ca08e7eb487669688ccd2ed42cbb", "project_files.data_get": "0d19505995cbd8c12e3938cd89cf05955a2a63488a220711d29dd24b27c0c8c8", @@ -66,6 +67,7 @@ "project_files.file_adopt_deletion": "7b6d661856a614d1b83d8011d1ffc7f9e2619853495faf9250888739f880685c", "project_files.file_get": "a22c27e2fd890ffcb479e0aa8ea40a0cb32d06d1e5750b970b40f27915fe4d24", "project_files.file_history": "c83477f7b0f8616c96ad79cebaf0eb3887fe289567f5d5ceb1d1502bebed7423", + "project_files.files_adopt": "62215260bf40bfa5db2521f908413bb6dfe47656fb8ea30b38039d637e5daf51", "project_files.files_inspect": "0c2ff3ed93109ea19b5c92ffa24ac8f67e4bcfa3fe386921e984b5dfa3bfe7eb", "project_files.fork_info": "958bc1aa269b5459c7e2d9c6bcb6725c06f7055502d73c49a95330cc4065aea4", "project_files.get": "6b0f4e16a00aa119b86cbb5493be0bac945ff08a5e63773a8f2c2f9296658d5b", @@ -82,6 +84,13 @@ "project_files.material_intent_capabilities": "86e90e2d0f094beb1c277a680da9a35e28d78c827390ac1c4e51fc3003c1ce32", "project_files.material_intents": "6431fd2780d706c77d2c4dd82f15794017368fd3bade28ac62bbdff5c8b0c547", "project_files.materials": "92a6a6a1750fed89aef9574ab6e82be2c6c6351bfa89fbfcb1f3866d4ea3cb13", + "project_files.measurement_compare": "abbb5e406a3c95155e90f08e8a071f330425190b25275f0f6512de08b8afb929", + "project_files.measurement_import": "c8e500396138fc60a31fc916c441bb95454f410231a6508609a7ebba5cd31f18", + "project_files.measurement_inspect": "fcbe9846ca49c3da4713b54a3cb4a8237189d8eea5799f275d33992ef268914f", + "project_files.measurement_metadata": "5a92ed5259f8876a3c5b8fd1886a9e0f44f0aea8e53d7f63902ac765ddac0de7", + "project_files.measurement_query": "65b1456a6870dd6d4b7499669777a2c66157142ccc27cac1864a995664220e1c", + "project_files.measurement_sources": "7805e9c5a6b2e2ab566ca134c18697116c89e08a1991b5192b0b5c152a34041d", + "project_files.measurement_subjects": "37bd3f717c7158e34b9c221bfa1dbce79490b9d55248803d13dab2ea17b35ea5", "project_files.model_create": "34cf790fb2a3355ccab7c3a6c7174ced8dae861812d9479582fd8d5a9212745c", "project_files.model_get": "95cc198ab6f084b596324f7251803ad4670a9f8e786e1a9639f1b9ed041798c3", "project_files.model_material": "46afde2b06e0e5b08565f12ec86d2d7ec43ef6b63d97c86c264fc9c2f451f20b", @@ -142,13 +151,7 @@ "project_files.study_create": "8d2aac7c073a6a7b3ebb41227b5b466cf03606625404f93a59b325038a4ad7a1", "project_files.study_save": "5bf3fb814588fd7d9c35836e9d964a64481b3cc4169caf76cad6ce776ff0fe15", "project_files.trial": "79afa4bed6d0907f3d4116cc04cb784e94e5a24ab89725ec24f5d7bce70e869f", - "project_files.trial_prepare": "e1f6973536fd4c356c4a92fc674d6766f7df0d8a113b8ee71bb05d0adc40b04f", - "project_files.measurement_import": "c8e500396138fc60a31fc916c441bb95454f410231a6508609a7ebba5cd31f18", - "project_files.measurement_sources": "7805e9c5a6b2e2ab566ca134c18697116c89e08a1991b5192b0b5c152a34041d", - "project_files.measurement_metadata": "5a92ed5259f8876a3c5b8fd1886a9e0f44f0aea8e53d7f63902ac765ddac0de7", - "project_files.measurement_subjects": "37bd3f717c7158e34b9c221bfa1dbce79490b9d55248803d13dab2ea17b35ea5", - "project_files.measurement_query": "65b1456a6870dd6d4b7499669777a2c66157142ccc27cac1864a995664220e1c", - "project_files.measurement_inspect": "fcbe9846ca49c3da4713b54a3cb4a8237189d8eea5799f275d33992ef268914f" + "project_files.trial_prepare": "e1f6973536fd4c356c4a92fc674d6766f7df0d8a113b8ee71bb05d0adc40b04f" } /* PROJECT_FILES_CONTRACTS_END */; // Generated fixed operation routes from agent_api; remote schemas are never executed. @@ -165,6 +168,10 @@ "project_files.get": { "1": "6b0f4e16a00aa119b86cbb5493be0bac945ff08a5e63773a8f2c2f9296658d5b", "2": "fdc0858fc03a0866bb743b04251ea5f9a3c1e6be1b211366db7e0652166736ea" + }, + "project_files.measurement_compare": { + "1": "abbb5e406a3c95155e90f08e8a071f330425190b25275f0f6512de08b8afb929", + "2": "d2de9763a9109d0e4599f50e038a3fda0e6bf7c9c60684434825d0108019bc6f" } } /* PROJECT_FILES_VERSIONS_END */; @@ -174,668 +181,683 @@ { "project_files.artifact_resolve": { "method": "POST", - "path": "/api/workspace/project-files/artifacts/resolve", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/artifacts/resolve" }, "project_files.artifact_status": { "method": "POST", - "path": "/api/workspace/project-files/artifacts/status", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/artifacts/status" }, "project_files.artifacts": { "method": "GET", - "path": "/api/workspace/project-files/artifacts", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/artifacts" }, "project_files.candidate": { "method": "GET", - "path": "/api/workspace/project-files/candidates/{candidate_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/candidates/{candidate_id}" }, "project_files.candidate_select": { "method": "POST", - "path": "/api/workspace/project-files/studies/{study_id}/candidates", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies/{study_id}/candidates" + }, + "project_files.compose": { + "method": "POST", + "mutation": true, + "path": "/api/workspace/project-files/compose" }, "project_files.data_annotate": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}" }, "project_files.data_availability": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/availability", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/availability" }, "project_files.data_get": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}" }, "project_files.data_history": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/history" }, "project_files.data_import_audio": { "method": "POST", - "path": "/api/workspace/project-files/data/import-audio", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-audio" }, "project_files.data_import_image": { "method": "POST", - "path": "/api/workspace/project-files/data/import-image", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-image" }, "project_files.data_import_mesh": { "method": "POST", - "path": "/api/workspace/project-files/data/import-mesh", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-mesh" }, "project_files.data_import_pointcloud": { "method": "POST", - "path": "/api/workspace/project-files/data/import-pointcloud", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-pointcloud" }, "project_files.data_import_scientific": { "method": "POST", - "path": "/api/workspace/project-files/data/import-scientific", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-scientific" }, "project_files.data_import_sequence": { "method": "POST", - "path": "/api/workspace/project-files/data/import-sequence", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-sequence" }, "project_files.data_import_text": { "method": "POST", - "path": "/api/workspace/project-files/data/import-text", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-text" }, "project_files.data_import_volume": { "method": "POST", - "path": "/api/workspace/project-files/data/import-volume", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-volume" }, "project_files.data_preference_delete": { "method": "POST", - "path": "/api/workspace/project-files/preferences/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/preferences/delete" }, "project_files.data_preference_history": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/preference-history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/preference-history" }, "project_files.data_preference_save": { "method": "POST", - "path": "/api/workspace/project-files/preferences", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/preferences" }, "project_files.data_preferences": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/preferences", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/preferences" }, "project_files.data_query": { "method": "GET", - "path": "/api/workspace/project-files/data", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data" }, "project_files.data_response_history": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/response-history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/response-history" }, "project_files.data_response_save": { "method": "POST", - "path": "/api/workspace/project-files/responses", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/responses" }, "project_files.data_responses": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/responses", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/responses" }, "project_files.data_rights": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}/rights", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}/rights" }, "project_files.data_scientific": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/scientific", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/scientific" }, "project_files.data_source_delete": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}/source/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}/source/delete" }, "project_files.data_source_save": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}/source", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}/source" }, "project_files.definitions": { "method": "GET", - "path": "/api/workspace/project-files/definitions", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/definitions" }, "project_files.dependencies": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}/dependencies", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/dependencies" }, "project_files.evidence": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/evidence", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/evidence" }, "project_files.execution_approve_external": { "method": "POST", - "path": "/api/workspace/project-files/execution/approvals", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/approvals" }, "project_files.execution_cancel": { "method": "POST", - "path": "/api/workspace/project-files/execution/runs/{run_id}/cancel", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/runs/{run_id}/cancel" }, "project_files.execution_generation_resume": { "method": "POST", - "path": "/api/workspace/project-files/execution/generation/{producer_id}/resume", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/generation/{producer_id}/resume" }, "project_files.execution_grounded_review": { "method": "POST", - "path": "/api/workspace/project-files/execution/grounded/{producer_id}/review", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/grounded/{producer_id}/review" }, "project_files.execution_health": { "method": "GET", - "path": "/api/workspace/project-files/execution", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/execution" }, "project_files.execution_preflight_external": { "method": "POST", - "path": "/api/workspace/project-files/execution/preflight", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/execution/preflight" }, "project_files.execution_producer_cancel": { "method": "POST", - "path": "/api/workspace/project-files/execution/producers/{producer_id}/cancel", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/producers/{producer_id}/cancel" }, "project_files.execution_producer_decide": { "method": "POST", - "path": "/api/workspace/project-files/execution/producers/{producer_id}/decide", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/producers/{producer_id}/decide" }, "project_files.execution_producer_health": { "method": "GET", - "path": "/api/workspace/project-files/execution/producers/health", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/execution/producers/health" }, "project_files.execution_producer_start": { "method": "POST", - "path": "/api/workspace/project-files/execution/producers", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/producers" }, "project_files.execution_reconcile_current": { "method": "POST", - "path": "/api/workspace/project-files/execution/reconcile-current", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/reconcile-current" }, "project_files.execution_study_abandon_preparation": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/abandon", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/abandon" }, "project_files.execution_study_admit": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/runs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/runs" }, "project_files.execution_study_approve": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/approvals", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/approvals" }, "project_files.execution_study_continue": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/continue", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/continue" }, "project_files.execution_study_prepare_package": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/packages", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/packages" }, "project_files.execution_study_prepare_verifier": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/verifiers", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/verifiers" }, "project_files.execution_study_reconcile_preparation": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/reconcile", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/reconcile" }, "project_files.execution_study_stop": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/stop", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/stop" }, "project_files.execution_submit_external": { "method": "POST", - "path": "/api/workspace/project-files/execution/runs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/runs" }, "project_files.expose_holdout": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/disclose", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/runs/{run_id}/disclose" }, "project_files.file_adopt": { "method": "POST", - "path": "/api/workspace/project-files/files/adopt", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/files/adopt" }, "project_files.file_adopt_deletion": { "method": "POST", - "path": "/api/workspace/project-files/files/adopt-deletion", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/files/adopt-deletion" }, "project_files.file_get": { "method": "GET", - "path": "/api/workspace/project-files/files/{file_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/files/{file_id}" }, "project_files.file_history": { "method": "GET", - "path": "/api/workspace/project-files/files/{file_id}/history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/files/{file_id}/history" + }, + "project_files.files_adopt": { + "method": "POST", + "mutation": true, + "path": "/api/workspace/project-files/files/adopt-many" }, "project_files.files_inspect": { "method": "GET", - "path": "/api/workspace/project-files/files", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/files" }, "project_files.fork_info": { "method": "GET", - "path": "/api/workspace/project-files/fork", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/fork" }, "project_files.graph_clone": { "method": "POST", - "path": "/api/workspace/project-files/graphs/{graph_id}/clone", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs/{graph_id}/clone" }, "project_files.graph_create": { "method": "POST", - "path": "/api/workspace/project-files/graphs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs" }, "project_files.graph_create_external": { "method": "POST", - "path": "/api/workspace/project-files/graphs/external", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs/external" }, "project_files.graph_get": { "method": "GET", - "path": "/api/workspace/project-files/graphs/{graph_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/graphs/{graph_id}" }, "project_files.graph_save": { "method": "POST", - "path": "/api/workspace/project-files/graphs/{graph_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs/{graph_id}" }, "project_files.graphs": { "method": "GET", - "path": "/api/workspace/project-files/graphs", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/graphs" }, "project_files.lineage": { "method": "GET", - "path": "/api/workspace/project-files/lineage", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/lineage" }, "project_files.lineage_record": { "method": "POST", - "path": "/api/workspace/project-files/lineage", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/lineage" }, "project_files.material_intent": { "method": "GET", - "path": "/api/workspace/project-files/material-intents/{family}/{definition_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/material-intents/{family}/{definition_id}" }, "project_files.material_intent_capabilities": { "method": "GET", - "path": "/api/workspace/project-files/material-intents/capabilities", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/material-intents/capabilities" }, "project_files.material_intents": { "method": "GET", - "path": "/api/workspace/project-files/material-intents", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/material-intents" }, "project_files.materials": { "method": "GET", - "path": "/api/workspace/project-files/materials", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials" + }, + "project_files.measurement_compare": { + "method": "POST", + "mutation": false, + "path": "/api/workspace/project-files/measurements/compare" + }, + "project_files.measurement_import": { + "method": "POST", + "mutation": true, + "path": "/api/workspace/project-files/measurements/import" + }, + "project_files.measurement_inspect": { + "method": "GET", + "mutation": false, + "path": "/api/workspace/project-files/measurements/{packet_sha256}/observations/{observation_id}" + }, + "project_files.measurement_metadata": { + "method": "GET", + "mutation": false, + "path": "/api/workspace/project-files/measurements/{packet_sha256}" + }, + "project_files.measurement_query": { + "method": "POST", + "mutation": false, + "path": "/api/workspace/project-files/measurements/{packet_sha256}/query" + }, + "project_files.measurement_sources": { + "method": "GET", + "mutation": false, + "path": "/api/workspace/project-files/measurements" + }, + "project_files.measurement_subjects": { + "method": "GET", + "mutation": false, + "path": "/api/workspace/project-files/measurements/{packet_sha256}/subjects" }, "project_files.model_create": { "method": "POST", - "path": "/api/workspace/project-files/models", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/models" }, "project_files.model_get": { "method": "GET", - "path": "/api/workspace/project-files/models/{model_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/models/{model_id}" }, "project_files.model_material": { "method": "POST", - "path": "/api/workspace/project-files/materials/model", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/model" }, "project_files.model_publish": { "method": "POST", - "path": "/api/workspace/project-files/models/{model_id}/publish", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/models/{model_id}/publish" }, "project_files.model_save": { "method": "POST", - "path": "/api/workspace/project-files/models/{model_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/models/{model_id}" }, "project_files.models": { "method": "GET", - "path": "/api/workspace/project-files/models", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/models" }, "project_files.native_preflight": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/preflight", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/preflight" }, "project_files.native_reconcile": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/phases/{phase_id}/reconcile", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/phases/{phase_id}/reconcile" }, "project_files.native_retry": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/runs/{run_id}/retry", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/runs/{run_id}/retry" }, "project_files.native_submit": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/runs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/runs" }, "project_files.package_state": { "method": "GET", - "path": "/api/workspace/project-files/candidates/{candidate_id}/package-state", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/candidates/{candidate_id}/package-state" }, "project_files.policies": { "method": "GET", - "path": "/api/workspace/project-files/policies", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/policies" }, "project_files.policy": { "method": "POST", - "path": "/api/workspace/project-files/policies/get", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/policies/get" }, "project_files.producer": { "method": "GET", - "path": "/api/workspace/project-files/producers/{origin_catalog_id}/{producer_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/producers/{origin_catalog_id}/{producer_id}" }, "project_files.producers": { "method": "GET", - "path": "/api/workspace/project-files/producers", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/producers" }, "project_files.progression": { "method": "GET", - "path": "/api/workspace/project-files/progression", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/progression" }, "project_files.progression_origins": { "method": "GET", - "path": "/api/workspace/project-files/progression/origins", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/progression/origins" }, "project_files.publication_result": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}/publication-result", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/publication-result" }, "project_files.recipe_create": { "method": "POST", - "path": "/api/workspace/project-files/recipes", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/recipes" }, "project_files.recipe_get": { "method": "GET", - "path": "/api/workspace/project-files/recipes/{recipe_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/recipes/{recipe_id}" }, "project_files.recipe_material": { "method": "POST", - "path": "/api/workspace/project-files/materials/recipe", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/recipe" }, "project_files.recipe_prepare": { "method": "POST", - "path": "/api/workspace/project-files/recipes/{graph_id}/prepare", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/recipes/{graph_id}/prepare" }, "project_files.recipe_save": { "method": "POST", - "path": "/api/workspace/project-files/recipes/{recipe_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/recipes/{recipe_id}" }, "project_files.recipes": { "method": "GET", - "path": "/api/workspace/project-files/recipes", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/recipes" }, "project_files.release_freeze": { "method": "POST", - "path": "/api/workspace/project-files/releases/freeze", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/releases/freeze" }, "project_files.release_preview": { "method": "POST", - "path": "/api/workspace/project-files/releases/preview", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/releases/preview" }, "project_files.retained_analysis": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/analysis", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/analysis" }, "project_files.retained_analysis_export": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/analysis/export", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/analysis/export" }, "project_files.run": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}" }, "project_files.run_document_segment": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}/document-segment", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/document-segment" }, "project_files.runs": { "method": "GET", - "path": "/api/workspace/project-files/runs", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs" }, "project_files.runtime_abandon": { "method": "POST", - "path": "/api/workspace/project-files/runtime/abandon", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/runtime/abandon" }, "project_files.runtime_capture": { "method": "POST", - "path": "/api/workspace/project-files/runtime/capture", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/runtime/capture" }, "project_files.runtime_inspect": { "method": "GET", - "path": "/api/workspace/project-files/runtime/inspect", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runtime/inspect" }, "project_files.runtime_material": { "method": "POST", - "path": "/api/workspace/project-files/materials/runtime", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/runtime" }, "project_files.runtime_sources": { "method": "GET", - "path": "/api/workspace/project-files/runtime/sources", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runtime/sources" }, "project_files.runtime_status": { "method": "GET", - "path": "/api/workspace/project-files/runtime/status", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runtime/status" }, "project_files.save_as": { "method": "POST", - "path": "/api/workspace/project-files/save-as", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/save-as" }, "project_files.search_create": { "method": "POST", - "path": "/api/workspace/project-files/searches", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/searches" }, "project_files.search_delete": { "method": "POST", - "path": "/api/workspace/project-files/searches/{search_id}/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/searches/{search_id}/delete" }, "project_files.search_rename": { "method": "POST", - "path": "/api/workspace/project-files/searches/{search_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/searches/{search_id}" }, "project_files.searches": { "method": "GET", - "path": "/api/workspace/project-files/searches", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/searches" }, "project_files.selection_create": { "method": "POST", - "path": "/api/workspace/project-files/selections", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/selections" }, "project_files.selection_delete": { "method": "POST", - "path": "/api/workspace/project-files/selections/{selection_id}/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/selections/{selection_id}/delete" }, "project_files.selection_get": { "method": "GET", - "path": "/api/workspace/project-files/selections/{selection_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/selections/{selection_id}" }, "project_files.selection_rename": { "method": "POST", - "path": "/api/workspace/project-files/selections/{selection_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/selections/{selection_id}" }, "project_files.selections": { "method": "GET", - "path": "/api/workspace/project-files/selections", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/selections" }, "project_files.snapshot": { "method": "POST", - "path": "/api/workspace/project-files/materials/snapshot", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/snapshot" }, "project_files.source_references": { "method": "GET", - "path": "/api/workspace/project-files/sources/{project_id}/references", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/sources/{project_id}/references" }, "project_files.source_register": { "method": "POST", - "path": "/api/workspace/project-files/sources/register", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/sources/register" }, "project_files.studies": { "method": "GET", - "path": "/api/workspace/project-files/studies", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/studies" }, "project_files.study": { "method": "GET", - "path": "/api/workspace/project-files/studies/{study_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/studies/{study_id}" }, "project_files.study_create": { "method": "POST", - "path": "/api/workspace/project-files/studies", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies" }, "project_files.study_save": { "method": "POST", - "path": "/api/workspace/project-files/studies/{study_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies/{study_id}" }, "project_files.trial": { "method": "GET", - "path": "/api/workspace/project-files/trials/{trial_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/trials/{trial_id}" }, "project_files.trial_prepare": { "method": "POST", - "path": "/api/workspace/project-files/studies/{study_id}/trials", - "mutation": true - }, - "project_files.measurement_import": { - "method": "POST", - "path": "/api/workspace/project-files/measurements/import", - "mutation": true - }, - "project_files.measurement_inspect": { - "method": "GET", - "path": "/api/workspace/project-files/measurements/{packet_sha256}/observations/{observation_id}", - "mutation": false - }, - "project_files.measurement_metadata": { - "method": "GET", - "path": "/api/workspace/project-files/measurements/{packet_sha256}", - "mutation": false - }, - "project_files.measurement_query": { - "method": "POST", - "path": "/api/workspace/project-files/measurements/{packet_sha256}/query", - "mutation": false - }, - "project_files.measurement_sources": { - "method": "GET", - "path": "/api/workspace/project-files/measurements", - "mutation": false - }, - "project_files.measurement_subjects": { - "method": "GET", - "path": "/api/workspace/project-files/measurements/{packet_sha256}/subjects", - "mutation": false + "mutation": true, + "path": "/api/workspace/project-files/studies/{study_id}/trials" } } /* PROJECT_FILES_ROUTES_END */; @@ -880,6 +902,17 @@ onContextFocusLost:()=>$('project-files-status').focus() }); window.addEventListener('pagehide',()=>projectProgression.clear()); + const projectPortfolio=window.ProjectModelPortfolio({ + root:$('project-model-portfolio'), + async read(action,args,signal){ + if(!m2Allowed(action))throw Error('Model organization inspection is unavailable for the selected project.'); + const value=await m2Request(action,args,epoch,{signal}); + if(value===null)throw new DOMException('Project selection changed.','AbortError'); + return value; + }, + onContextFocusLost:()=>$('project-files-status').focus() + }); + window.addEventListener('pagehide',()=>projectPortfolio.clear()); const projectMeasurements=window.ProjectMeasurements({ root:$('project-measurements'), async read(action,args,signal){ @@ -1145,10 +1178,12 @@ } const m2Allowed=action=>!busy&&!blocked&&state?.state==='open'&&selectedContracts.has('project_files.'+action); function m2Controls(){ + projectPortfolio.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), + project_id:state?.project?.project_id,enabled:m2Allowed('files_inspect')&&m2Allowed('file_get')}); projectProgression.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), project_id:state?.project?.project_id,enabled:m2Allowed('progression')&&m2Allowed('progression_origins')}); projectMeasurements.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), - enabled:['measurement_sources','measurement_metadata','measurement_subjects','measurement_query','measurement_inspect'].every(m2Allowed)}); + enabled:['measurement_sources','measurement_compare','measurement_inspect'].every(m2Allowed)}); if(blocked||state?.state!=='open'||scientificEpoch!==null&&(scientificEpoch!==epoch||explored?.checkpoint!==checkpoint()))clearScientific(); const action=$('project-explore-kind').value; $('project-explore-load').disabled=!m2Allowed(action); diff --git a/static/project-measurements.js b/static/project-measurements.js index 7f99a043..64e3f9c0 100644 --- a/static/project-measurements.js +++ b/static/project-measurements.js @@ -20,11 +20,11 @@ details.append(node('summary', title), node('pre', JSON.stringify(value, null, 2))); return details; } window.ProjectMeasurements = function({root, read, onContextFocusLost = () => {}}) { - root.innerHTML = '

Retained measurements

Compare exact saved scalars and inspect their original cases. Maxima use every matching case; failed and unavailable observations remain counted.

'; + root.innerHTML = '

Retained measurements

Compare exact saved scalars and inspect their original cases. Maxima use every matching case; failed and unavailable observations remain counted.

'; const $ = name => root.querySelector('#project-measurements-' + name); let context = null, generation = 0, plotGeneration = 0, pending, busy = false, pendingFocus = false; - let sources = [], sourceOffset = 0, sourceTotal = 0, selected = null, descriptor = null, metadata = null; - let subjects = [], subjectOffset = 0, subjectTotal = 0, filters = new Map(), applied = null, query = null; + let sources = [], sourceOffset = 0, sourceTotal = 0, selected = null, descriptors = null, metadata = null; + let subjects = [], subjectOffset = 0, subjectTotal = 0, filters = new Map(), applied = null, query = null, selectedSubjects = new Set(); const renderer = window.RetainedChart.mount($('chart'), spec, {onError: error => { const ownsFocus = root.contains(document.activeElement); clearResults(); $('status').textContent = error.message; controls(); if (ownsFocus) $('compare').focus(); @@ -37,15 +37,17 @@ function controls() { for (const control of root.querySelectorAll('input,select,button')) control.disabled = !context?.enabled || busy; $('sources-prev').hidden = sourceOffset === 0; $('sources-next').hidden = sourceOffset + sources.length >= sourceTotal; - $('source').disabled ||= !sources.length; $('load').disabled ||= !$('source').value; - $('subjects-prev').hidden = subjectOffset === 0; $('subjects-next').hidden = subjectOffset + subjects.length >= subjectTotal; - $('compare').disabled ||= !metadata || !$('subjects').selectedOptions.length; + $('source').disabled ||= !sources.length; $('source-all').disabled ||= !sources.length || sources.length > 16; + $('load').disabled ||= !$('source').selectedOptions.length || $('source').selectedOptions.length > 16; + $('subjects-prev').hidden = subjectOffset === 0; $('subjects-next').hidden = subjectOffset + 40 >= subjectTotal; + $('compare').disabled ||= !metadata || !selectedSubjects.size; + $('subjects-all').disabled ||= !metadata; $('subjects-clear').disabled ||= !metadata; $('prev').disabled ||= !query || query.offset === 0; $('next').disabled ||= !query || query.offset + query.rows.length >= query.total; root.setAttribute('aria-busy', String(busy)); } function clearSource() { - selected = descriptor = metadata = null; subjects = []; subjectOffset = subjectTotal = 0; filters.clear(); + selected = descriptors = metadata = null; subjects = []; selectedSubjects.clear(); subjectOffset = subjectTotal = 0; filters.clear(); $('form').hidden = true; $('context').replaceChildren(); $('filters').replaceChildren(); $('metric').replaceChildren(); $('subjects').replaceChildren(); clearResults(); } @@ -84,34 +86,68 @@ seen.add(key); } sources = value.items; sourceOffset = offset; sourceTotal = value.total; $('source').replaceChildren(); - for (const item of sources) { + for (const [index, item] of sources.entries()) { const option = node('option', item.title + ' · ' + item.id.slice(0, 12) + (item.availability?.state === 'missing' ? ' · missing packet' : '')); - option.value = sourceKey(item); $('source').append(option); + option.value = sourceKey(item); option.selected = index === 0; $('source').append(option); } $('source-page').textContent = `${sources.length ? offset + 1 : 0}–${offset + sources.length} of ${sourceTotal} sources`; - $('status').textContent = sources.length ? 'Choose a source and load its exact retained metadata.' : 'No retained measurement sources in this project.'; + $('status').textContent = sources.length ? 'Choose compatible sources and load their exact retained metadata.' : 'No retained measurement sources in this project.'; return $('source'); }); } - function checkRead(value, source = selected) { - if (!value || value.source?.origin_catalog_id !== source?.origin_catalog_id || value.source?.id !== source?.id - || value.source.blob?.sha256 !== source.id || value.source.source?.measured_sha256 !== source.id - || descriptor && (value.source.blob.bytes !== descriptor.blob.bytes || value.source.title !== descriptor.title) - || new TextEncoder().encode(JSON.stringify(value)).length > 256 * 1024) throw Error('Measurement reply differs from the selected packet.'); - return value.value; + const same = (left, right) => { + if (Object.is(left, right)) return true; + if (!left || !right || typeof left !== 'object' || typeof right !== 'object' || Array.isArray(left) !== Array.isArray(right)) return false; + const keys = Object.keys(left); + return keys.length === Object.keys(right).length && keys.every(key => Object.hasOwn(right, key) && same(left[key], right[key])); + }; + const locatorKey = value => value.origin_catalog_id + '.' + value.packet_sha256; + const selection = () => ({sources:selected.map(item => ({origin_catalog_id:item.origin_catalog_id, packet_sha256:item.id}))}); + function checkComparison(response) { + if (!response || !Array.isArray(response.sources) || response.sources.length !== selected.length + || new TextEncoder().encode(JSON.stringify(response)).length > 256 * 1024) throw Error('Comparison differs from selected sources.'); + for (const [index, source] of response.sources.entries()) { + const expected = selected[index]; + if (source.origin_catalog_id !== expected.origin_catalog_id || source.id !== expected.id + || source.title !== expected.title || source.blob?.sha256 !== source.id || source.blob.bytes !== expected.blob.bytes + || source.source?.measured_sha256 !== source.id) throw Error('Comparison source identity changed.'); + } + if (descriptors && !same(descriptors, response.sources)) throw Error('Original comparison capture facts changed.'); + const value = response.value, keys = selected.map(sourceKey); + if (value?.format !== 'retained_measurement_comparison_v1' || !hash(value.case_population_sha256) + || !Array.isArray(value.subjects) || !value.subjects.length || value.subjects.length > 256 + || !natural(value.case_count) || !value.case_count || !natural(value.observation_count) + || !Array.isArray(value.metrics) || !value.metrics.length || value.metrics.length > 32 + || !Array.isArray(value.dimensions) || value.dimensions.length > 32 + || value.observation_count !== value.subjects.length * value.case_count * value.metrics.length + || locatorKey(value.definition_source) !== keys[0] + || value.subjects.some(item => !identity(item.id) || typeof item.label !== 'string' || !keys.includes(locatorKey(item.source))) + || new Set(value.subjects.map(item => item.id)).size !== value.subjects.length) throw Error('Invalid comparison metadata.'); + if (metadata && ['scope','scope_definition','dimensions','metrics','definition_source','case_population_sha256','case_count','observation_count','subjects'].some(key => !same(value[key], metadata[key]))) throw Error('Comparison population changed.'); + return value; + } + function checkRead(response, source) { + const original = descriptors.find(item => sourceKey(item) === locatorKey(source)); + if (!original || response?.source?.origin_catalog_id !== source.origin_catalog_id || response.source.id !== source.packet_sha256 + || response.source.blob?.sha256 !== source.packet_sha256 || !same(response.source, original) + || new TextEncoder().encode(JSON.stringify(response)).length > 256 * 1024) throw Error('Observation differs from its original packet.'); + return response.value; } - const selection = () => ({origin_catalog_id: selected.origin_catalog_id, packet_sha256: selected.id}); - async function subjectPage(offset, active, signal) { - const value = checkRead(await read('measurement_subjects', {...selection(), offset, limit: 40}, signal)); if (!active()) return; - page(value, offset, 40, value.items); - if (value.total !== metadata.subject_count) throw Error('Subject page differs from the retained packet population.'); - if (value.items.some(item => !identity(item.id) || typeof item.label !== 'string') - || new Set(value.items.map(item => item.id)).size !== value.items.length) throw Error('Invalid retained subjects.'); - subjects = value.items; subjectOffset = offset; subjectTotal = value.total; $('subjects').replaceChildren(); - for (const item of subjects) { const option = node('option', item.label + ' · ' + item.id); option.value = item.id; option.selected = true; $('subjects').append(option); } - $('subject-page').textContent = `Subjects ${subjects.length ? offset + 1 : 0}–${offset + subjects.length} of ${subjectTotal}. Only explicitly selected subjects on this page will be compared.`; + function subjectPage(offset) { + if (!metadata || !natural(offset) || offset > subjectTotal) throw Error('Invalid comparison subject page.'); + subjectOffset = offset; $('subjects').replaceChildren(); + const items = subjects.slice(offset, offset + 40); + for (const item of items) { + const option = node('option', item.label + ' · ' + item.id); option.value = item.id; + option.selected = selectedSubjects.has(item.id); $('subjects').append(option); + } + subjectSelectionStatus(); return $('subjects'); } + function subjectSelectionStatus() { + const count = Math.min(40, subjectTotal - subjectOffset); + $('subject-page').textContent = `Subjects ${count ? subjectOffset + 1 : 0}–${subjectOffset + count} of ${subjectTotal}. ${selectedSubjects.size} selected across all pages; selection is retained when paging.`; + } function dimensions() { filters.clear(); $('filters').replaceChildren(); for (const [index, dimension] of metadata.dimensions.entries()) { @@ -158,12 +194,13 @@ for (const text of headings) { const cell = node('th', text); cell.scope = 'col'; row.append(cell); } head.append(row); result.append(node('caption', caption), head, body); return {result, body}; } - async function inspect(observation_id) { + async function inspect(observation_id, source) { clearInspection(); return perform(async(active, signal) => { - const value = checkRead(await read('measurement_inspect', {...selection(), observation_id}, signal)); if (!active()) return; - if (value.packet_sha256 !== selected.id || value.observation?.id !== observation_id - || value.observation.metric_id !== applied?.metric_id || !applied.subject_ids.includes(value.observation.subject_id)) throw Error('Observation differs from the selected comparison.'); + const value = checkRead(await read('measurement_inspect', {...source, observation_id}, signal), source); if (!active()) return; + if (value.packet_sha256 !== source.packet_sha256 || value.observation?.id !== observation_id + || value.observation.metric_id !== applied?.metric_id || !applied.subject_ids.includes(value.observation.subject_id) + || !same(subjects.find(item => item.id === value.observation.subject_id)?.source, source)) throw Error('Observation differs from the selected comparison.'); const heading = node('h3', value.subject.label + ' · ' + value.case.id); heading.tabIndex = -1; const observation = value.observation; $('inspection').append(heading, node('p', `${value.metric.label} · ${value.metric.units} · ${observation.state} · ${observation.value === null ? observation.reason : display(observation.value)}`), @@ -175,27 +212,31 @@ }); } async function showQuery(value, request, active, signal, authority) { - if (value.packet_sha256 !== selected.id || value.metric?.id !== request.metric_id + if (!value || value.metric?.id !== request.metric_id + || !same(value.metric, metadata.metrics.find(metric => metric.id === request.metric_id)) || JSON.stringify(value.subject_ids) !== JSON.stringify(request.subject_ids) || !value.filters || Object.keys(value.filters).length !== Object.keys(request.filters).length || Object.entries(request.filters).some(([key, wanted]) => !Object.hasOwn(value.filters, key) || value.filters[key] !== wanted) || !Array.isArray(value.summary) || value.summary.length !== request.subject_ids.length || value.sampling !== 'none') throw Error('Measurement comparison differs from its requested selection.'); page(value, request.offset, request.limit, value.rows); + if (value.summary.reduce((count, summary) => count + summary.count, 0) !== value.total) throw Error('Comparison denominator differs from its complete summaries.'); const maxima = table('Maximum over all matching cases, including every observation status', ['Subject / execution attributes','Maximum','Measured','Failed','Unsupported','Unavailable']); - const points = []; + const points = [], inspections = new Map(); for (const [index, summary] of value.summary.entries()) { if (summary.subject_id !== request.subject_ids[index] || !natural(summary.count) || STATES.some(state => !natural(summary.states?.[state])) || STATES.reduce((sum, state) => sum + summary.states[state], 0) !== summary.count || (summary.maximum === null) !== (summary.states.measured === 0)) throw Error('Invalid complete measurement summary.'); - const subject = subjects.find(item => item.id === summary.subject_id); if (!subject) throw Error('Subject is outside the selected page.'); + const subject = subjects.find(item => item.id === summary.subject_id); if (!subject || !same(subject.source, summary.source)) throw Error('Subject differs from its original comparison packet.'); const row = node('tr'), label = node('td'); label.append(node('strong', subject.label), attributes('Execution attributes', subject.dimensions)); const maximum = node('td'); if (summary.maximum === null) maximum.textContent = 'No measured value'; else { const point = summary.maximum; if (!finite(point.value) || !identity(point.id) || !identity(point.case_id)) throw Error('Invalid retained maximum.'); - maximum.append(button(display(point.value) + ' · inspect maximum for ' + subject.label, () => void inspect(point.id))); - points.push({subject_id:subject.id, value:point.value, observation_id:point.id}); + maximum.append(button(display(point.value) + ' · inspect maximum for ' + subject.label, () => void inspect(point.id, summary.source))); + const token = JSON.stringify([summary.source.origin_catalog_id, summary.source.packet_sha256, point.id]); + inspections.set(token, {id:point.id, source:summary.source}); + points.push({subject_id:subject.id, value:point.value, observation_id:token}); } row.append(label, maximum, ...STATES.map(state => node('td', String(summary.states[state])))); maxima.body.append(row); } @@ -204,7 +245,8 @@ if (!request.subject_ids.includes(observation.subject_id) || observation.metric_id !== request.metric_id || !identity(observation.id) || !STATES.includes(observation.state) || (observation.state === 'measured' ? !finite(observation.value) : observation.value !== null)) throw Error('Invalid retained observation row.'); const subject = subjects.find(item => item.id === observation.subject_id), row = node('tr'), detail = node('td'); - detail.append(button('Inspect ' + observation.id, () => void inspect(observation.id)), attributes('Case and execution attributes', observation.dimensions)); + if (!subject || !same(subject.source, observation.source)) throw Error('Observation belongs to another packet.'); + detail.append(button('Inspect ' + observation.id, () => void inspect(observation.id, observation.source)), attributes('Case and execution attributes', observation.dimensions)); row.append(node('td', subject.label), node('td', observation.case_id), node('td', observation.value === null ? observation.state + ': ' + observation.reason : display(observation.value)), detail); observations.body.append(row); } query = value; applied = structuredClone(request); $('results').hidden = false; @@ -214,44 +256,59 @@ const plotTicket = ++plotGeneration; if (points.length) await renderer.render({maxima:points}, {signal, isCurrent: () => context?.enabled && context.key === authority && plotTicket === plotGeneration, - onSignal:{measurement_select: selection => { const id = selection?.observation_id?.[0]; if (points.some(point => point.observation_id === id)) void inspect(id); }}}); + onSignal:{measurement_select: selection => { const id = selection?.observation_id?.[0]; const original = inspections.get(id); if (original) void inspect(original.id, original.source); }}}); else { renderer.clear(); $('chart').append(node('p', 'No measured values for this selection. Missing and failed observations remain counted below.')); } if (active()) { $('heading').focus(); $('status').textContent = 'Exact retained comparison. Metric and scope definitions are available above.'; } } function compare(offset = 0, saved = null) { return perform(async(active, signal, authority) => { const request = saved ? {...saved, offset} : {metric_id:$('metric').value, - subject_ids:[...$('subjects').selectedOptions].map(option => option.value), filters:filterValues(), offset, limit:50}; + subject_ids:subjects.filter(item => selectedSubjects.has(item.id)).map(item => item.id), filters:filterValues(), offset, limit:50}; if (!request.subject_ids.length) throw Error('Choose at least one subject.'); clearResults(); - const value = checkRead(await read('measurement_query', {...selection(), ...request}, signal)); if (!active()) return; + const value = checkComparison(await read('measurement_compare', {...selection(), ...request}, signal)).query; if (!active()) return; await showQuery(value, request, active, signal, authority); }); } $('sources').onclick = () => void loadSources(); $('sources-prev').onclick = () => void loadSources(Math.max(0, sourceOffset - 40)); $('sources-next').onclick = () => void loadSources(sourceOffset + 40); - $('source').onchange = () => { invalidate(); clearSource(); controls(); }; + $('source').onchange = () => { invalidate(); clearSource(); $('status').textContent = 'Load the selected compatible sources.'; controls(); }; $('source-form').onsubmit = event => { event.preventDefault(); void perform(async(active, signal) => { - const chosen = sources.find(source => sourceKey(source) === $('source').value); clearSource(); selected = chosen; - if (!selected) throw Error('Choose a retained measurement source.'); - const response = await read('measurement_metadata', selection(), signal); if (!active()) return; - metadata = checkRead(response); descriptor = response.source; - if (metadata.packet_sha256 !== selected.id || metadata.title !== selected.title || !Array.isArray(metadata.metrics) || metadata.metrics.length > 32 - || !Array.isArray(metadata.dimensions) || metadata.dimensions.length > 32) throw Error('Invalid exact measurement metadata.'); - $('context').append(node('h3', metadata.title), node('p', metadata.scope.label + ' · population: ' + metadata.scope.population), + const chosen = [...$('source').selectedOptions].map(option => sources.find(item => sourceKey(item) === option.value)); + clearSource(); selected = chosen; + if (!selected.length || selected.length > 16 || selected.some(item => !item)) throw Error('Choose 1 through 16 retained sources.'); + const response = await read('measurement_compare', selection(), signal); if (!active()) return; + metadata = checkComparison(response); descriptors = response.sources; + if (metadata.query !== null) throw Error('Metadata reply unexpectedly contains a query.'); + subjects = metadata.subjects; subjectTotal = subjects.length; selectedSubjects = new Set(subjects.map(item => item.id)); + $('context').append(node('h3', `Comparison of ${selected.length} retained sources`), node('p', metadata.scope.label + ' · population: ' + metadata.scope.population), node('p', typeof metadata.scope_definition.task === 'string' ? metadata.scope_definition.task : 'Interpret results using the retained scope definition.'), - jsonDetails('Exact scope, metric definitions and packet source', {metadata, source:descriptor})); + jsonDetails('Exact compatible scope, metric definitions and original packet sources', {metadata, sources:descriptors})); for (const metric of metadata.metrics) { const option = node('option', metric.label + ' · ' + metric.units); option.value = metric.id; $('metric').append(option); } - dimensions(); await subjectPage(0, active, signal); if (!active()) return; + dimensions(); subjectPage(0); if (!active()) return; $('form').hidden = false; $('status').textContent = 'Choose a metric, subjects and optional exact attribute filters.'; return $('metric'); }); }; - $('metric').onchange = invalidate; $('subjects').onchange = invalidate; + $('source-all').onclick = () => { + for (const option of $('source').options) option.selected = true; + $('source').onchange(); + }; + $('metric').onchange = invalidate; + $('subjects').onchange = () => { + for (const option of $('subjects').options) { + if (option.selected) selectedSubjects.add(option.value); else selectedSubjects.delete(option.value); + } + invalidate(); subjectSelectionStatus(); controls(); + }; + for (const [name, all] of [['subjects-all',true],['subjects-clear',false]]) $(name).onclick = () => { + selectedSubjects = new Set(all ? subjects.map(item => item.id) : []); + invalidate(); subjectPage(subjectOffset); controls(); + }; for (const [name, direction] of [['subjects-prev',-1],['subjects-next',1]]) $(name).onclick = () => { - clearResults(); void perform((active, signal) => subjectPage(subjectOffset + direction * 40, active, signal)); + void perform(() => subjectPage(subjectOffset + direction * 40)); }; $('form').onsubmit = event => { event.preventDefault(); void compare(); }; $('prev').onclick = () => void compare(Math.max(0, query.offset - query.limit), applied); diff --git a/static/project-model-portfolio.js b/static/project-model-portfolio.js new file mode 100644 index 00000000..4f1b7fa3 --- /dev/null +++ b/static/project-model-portfolio.js @@ -0,0 +1,188 @@ +'use strict'; +// Authored organization is a retained note, not Trial lineage or qualification. +// This module owns its document contract, finite view and cancellable reads. +(() => { + const fields=['method','purpose','experiment','outcome','lesson','reproduction']; + const exact=(value,keys)=>value&&typeof value==='object'&&!Array.isArray(value)&& + Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); + const text=(value,limit)=>typeof value==='string'&&value.length>0&&value.length<=limit; + const identity=value=>text(value,160)&&/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value); + const sha=value=>typeof value==='string'&&/^[a-f0-9]{64}$/.test(value); + const natural=value=>Number.isSafeInteger(value)&&value>=0; + function fail(message,code='invalid'){throw Object.assign(Error(message),{code});} + function noteRef(value){ + if(!exact(value,['file_id','path','revision','sha256'])||!sha(value.file_id)||!sha(value.sha256)|| + !natural(value.revision)||value.revision<1||!text(value.path,512)||!value.path.startsWith('notes/')|| + value.path.split('/').some(part=>!part||part==='.'||part==='..'))fail('Invalid captured model-note reference.'); + } + function decode(raw){ + if(typeof raw!=='string'||new TextEncoder().encode(raw).length>1048576)fail('Organization note exceeds its retained text limit.'); + let value;try{value=JSON.parse(raw);}catch{fail('Organization note is not valid JSON.');} + if(value?.format!=='project_model_portfolio_v1')fail('Unsupported model-organization document format.','unsupported'); + if(!exact(value,['format','project_id','title','description','authorship','groups','models','relationships'])|| + !/^[a-f0-9]{32}$/.test(value.project_id)||!text(value.title,200)||!text(value.description,8000)|| + value.authorship!=='retrospective'||!Array.isArray(value.groups)||value.groups.length>64|| + !Array.isArray(value.models)||!value.models.length||value.models.length>1000|| + !Array.isArray(value.relationships)||value.relationships.length>2000)fail('Invalid model-organization document.'); + const groups=new Map(),models=new Map(); + for(const group of value.groups){ + if(!exact(group,['id','label','parent','description'])||!identity(group.id)||groups.has(group.id)|| + !text(group.label,200)||!text(group.description,8000)||group.parent!==null&&!identity(group.parent))fail('Invalid or duplicate organization group.'); + groups.set(group.id,group); + } + for(const group of groups.values()){ + const visited=new Set([group.id]);let parent=group.parent; + while(parent!==null){if(!groups.has(parent))fail('Organization group parent is unavailable.'); + if(visited.has(parent))fail('Organization grouping contains a cycle.');visited.add(parent);parent=groups.get(parent).parent;} + } + for(const model of value.models){ + if(!exact(model,['id','label','group_id','aliases','review','card','summary'])||!identity(model.id)||models.has(model.id)|| + !text(model.label,500)||!groups.has(model.group_id)||!Array.isArray(model.aliases)||model.aliases.length>128|| + new Set(model.aliases).size!==model.aliases.length||model.aliases.some(alias=>!text(alias,500))|| + !['source_reviewed','review_pending'].includes(model.review))fail('Invalid or duplicate organized model.'); + noteRef(model.card); + if(model.review==='source_reviewed'){ + if(!exact(model.summary,fields)||fields.some(field=>!text(model.summary[field],16384)))fail('Reviewed model requires all six authored summary fields.'); + }else if(model.summary!==null)fail('Pending review cannot expose a reviewed summary.'); + models.set(model.id,model); + } + const pairs=new Set(); + for(const relation of value.relationships){ + if(!exact(relation,['from','to','kind','explanation','evidence_model','excerpt'])|| + !models.has(relation.from)||!models.has(relation.to)||relation.from===relation.to|| + !['comparison','parameter_derivation'].includes(relation.kind)||!text(relation.explanation,8000)|| + !models.has(relation.evidence_model)||!text(relation.excerpt,16384))fail('Invalid authored model relationship.'); + const key=JSON.stringify([relation.from,relation.to,relation.kind]);if(pairs.has(key))fail('Duplicate authored model relationship.');pairs.add(key); + } + const parents=new Map(),complete=new Set(),visiting=new Set(); + for(const relation of value.relationships)if(relation.kind==='parameter_derivation'){ + if(!parents.has(relation.from))parents.set(relation.from,[]);parents.get(relation.from).push(relation.to); + } + function visit(id){if(complete.has(id))return;if(visiting.has(id))fail('Authored parameter derivation contains a cycle.'); + visiting.add(id);for(const parent of parents.get(id)||[])visit(parent);visiting.delete(id);complete.add(id);} + for(const id of parents.keys())visit(id); + return value; + } + function checkCard(document,model,result){ + const ref=model.card,version=result?.version; + if(result?.project_id!==document.project_id||result.id!==ref.file_id||result.path!==ref.path||result.kind!=='note'|| + version?.deleted===true||version?.revision!==ref.revision||version.sha256!==ref.sha256||typeof version.text!=='string') + fail('Model card differs from its exact retained organization reference.'); + if(model.summary&&fields.some(field=>!version.text.includes(model.summary[field])))fail('Authored summary differs from the captured model card.'); + for(const relation of document.relationships)if(relation.evidence_model===model.id&&!version.text.includes(relation.excerpt)) + fail('Authored relationship excerpt differs from its captured model card.'); + return version.text; + } + window.ProjectModelPortfolioDocument=Object.freeze({decode,checkCard}); + const node=(tag,value)=>{const element=document.createElement(tag);if(value!==undefined)element.textContent=value;return element;}; + window.ProjectModelPortfolio=function({root,read,onContextFocusLost=()=>{}}){ + root.innerHTML='

Authored model organization

Browse a retained guide to models and their methods. Grouping and retrospective comparison notes are separate from recorded Trial ancestry and scientific qualification.

'; + const $=name=>root.querySelector('#project-portfolio-'+name); + let context=null,generation=0,pending=null,busy=false,notes=new Map(),noteOffset=0,noteTotal=0; + let guide=null,groups=new Map(),models=new Map(),applied={group:'',search:'',review:''},offset=0,selected=''; + const pageSize=20; + function controls(){ + const enabled=!!context?.enabled; + for(const name of ['notes','notes-more','note','load','group','search','review','prev','next'])$(name).disabled=!enabled||busy; + $('note').disabled ||= !notes.size;$('load').disabled ||= !notes.has($('note').value); + $('notes-more').hidden=noteOffset>=noteTotal;$('notes-more').disabled ||= noteOffset>=noteTotal; + $('prev').disabled ||= !guide||offset===0;$('next').disabled ||= !guide||offset+pageSize>=filtered().length; + for(const button of root.querySelectorAll('#project-portfolio-filter-form button,[data-lineage-node],#project-portfolio-detail button'))button.disabled=!enabled||busy; + root.setAttribute('aria-busy',String(busy)); + } + function clearGuide(){guide=null;groups.clear();models.clear();selected='';offset=0;applied={group:'',search:'',review:''};$('content').hidden=true;$('detail').replaceChildren();$('graph').replaceChildren();$('search').value='';$('review').value='';$('group').replaceChildren();} + function reset(){++generation;pending?.abort();pending=null;busy=false;notes.clear();noteOffset=noteTotal=0;$('note').replaceChildren();$('note-count').textContent='';$('status').textContent='';clearGuide();controls();} + async function perform(work){ + if(!context?.enabled||busy)return; + const ticket=++generation,key=context.key;pending=new AbortController();const signal=pending.signal;busy=true;controls(); + const active=()=>ticket===generation&&context?.enabled&&context.key===key&&!signal.aborted; + try{await work(active,signal);}catch(error){if(active()){$('detail').replaceChildren();$('status').textContent=(error.code?error.code+': ':'')+error.message;$('status').focus();}} + finally{if(active()){busy=false;pending=null;controls();}} + } + async function findNotes(more=false){ + return perform(async(active,signal)=>{ + const requested=more?noteOffset:0,value=await read('files_inspect',{offset:requested,limit:40},signal);if(!active())return; + if(value?.offset!==requested||value.limit!==40||!natural(value.total)||!Array.isArray(value.items)|| + value.items.length!==Math.min(40,Math.max(0,value.total-requested)))fail('Organization note page differs from its continuation.'); + if(!more){clearGuide();notes.clear();$('note').replaceChildren();} + for(const item of value.items)if(item.kind==='note'&&item.path?.endsWith('.portfolio.json')){ + if(!sha(item.id)||!natural(item.revision)||item.revision<1||!sha(item.expected_sha256)||notes.has(item.id))fail('Invalid organization note selection.'); + notes.set(item.id,structuredClone(item));const option=node('option',item.path+' · revision '+item.revision);option.value=item.id;$('note').append(option); + } + noteOffset=requested+value.items.length;noteTotal=value.total; + $('note-count').textContent=`${notes.size} organization notes found; inspected ${noteOffset} of ${noteTotal} adopted files.`; + $('status').textContent=notes.size?'Choose a retained organization note.':'No organization note on these pages. Adopt a .portfolio.json note or inspect the next page.'; + }); + } + function inGroup(model,wanted){let id=model.group_id;while(id!==null){if(id===wanted)return true;id=groups.get(id).parent;}return false;} + function filtered(){if(!guide)return [];return guide.models.filter(model=>(!applied.group||inGroup(model,applied.group))&& + (!applied.review||model.review===applied.review)&&(!applied.search||[model.label,...model.aliases].some(value=>value.toLocaleLowerCase().includes(applied.search))));} + function render(){ + if(!guide)return; + const focused=$('graph').contains(document.activeElement)?document.activeElement.id:null; + const matching=filtered(),shown=matching.slice(offset,offset+pageSize),included=new Set(); + for(const model of shown){let id=model.group_id;while(id!==null){included.add(id);id=groups.get(id).parent;}} + const graphNodes=[...included].map(id=>{const group=groups.get(id);return {id:'group/'+id,title:group.label,recordedAt:'',parent:group.parent===null?null:'group/'+group.parent,baseline:null,lines:['Authored method group']};}); + graphNodes.push(...shown.map(model=>({id:'model/'+model.id,title:model.label,recordedAt:'',parent:'group/'+model.group_id,baseline:null, + lines:[model.aliases.slice(0,3).join(' · ')||model.id,model.review==='source_reviewed'?'Source reviewed · authored summary':'Source review pending']}))); + $('graph').innerHTML=window.WorkspaceExperimentGraph.render(graphNodes,{selected:selected?'model/'+selected:'',zoom:1}).html; + for(const button of $('graph').querySelectorAll('[data-lineage-node]')){ + const value=button.dataset.lineageNode; + button.onclick=()=>{if(value.startsWith('model/'))void selectModel(value.slice(6));else{$('group').value=value.slice(6);applyFilters();}}; + } + $('count').textContent=`${guide.models.length} models in this guide; ${guide.models.filter(model=>model.review==='source_reviewed').length} source reviewed. ${matching.length} match the applied filters. Showing ${shown.length?offset+1:0}–${offset+shown.length}.`; + controls(); + if(focused){const replacement=$('graph').querySelector('#'+CSS.escape(focused)); + if(replacement)replacement.focus({preventScroll:true});else{$('count').tabIndex=-1;$('count').focus();}} + } + function applyFilters(){if(!guide||busy)return;applied={group:$('group').value,search:$('search').value.trim().toLocaleLowerCase(),review:$('review').value}; + if(applied.group&&!groups.has(applied.group))return;offset=0;selected='';$('detail').replaceChildren();render();$('status').textContent='Applied model filters.';} + async function loadGuide(){ + const choice=notes.get($('note').value);if(!choice)return; + return perform(async(active,signal)=>{ + clearGuide();const value=await read('file_get',{file_id:choice.id,revision:choice.revision},signal);if(!active())return; + if(value?.id!==choice.id||value.project_id!==context.project_id||value.kind!=='note'||value.path!==choice.path|| + value.version?.revision!==choice.revision||value.version.sha256!==choice.expected_sha256||value.version.deleted===true)fail('Organization note differs from the selected captured revision.'); + const parsed=decode(value.version.text); + if(parsed.project_id!==context.project_id)fail('This guide belongs to another project; re-author its captured note bindings.','unsupported'); + guide=parsed;groups=new Map(guide.groups.map(group=>[group.id,group]));models=new Map(guide.models.map(model=>[model.id,model])); + $('title').textContent=guide.title;$('description').textContent=guide.description; + $('source').textContent=`Retrospective authored guide · ${choice.path} · revision ${choice.revision} · SHA256 ${choice.expected_sha256}`; + const all=node('option','All groups');all.value='';$('group').append(all); + for(const group of guide.groups){const option=node('option',group.label);option.value=group.id;$('group').append(option);} + $('content').hidden=false;render();$('status').textContent='Loaded retained authored model organization.'; + }); + } + async function selectModel(id){ + const model=models.get(id);if(!model||busy)return; + return perform(async(active,signal)=>{ + if(!filtered().some(item=>item.id===id)){ + applied={group:'',search:'',review:''};$('group').value='';$('search').value='';$('review').value=''; + } + offset=Math.floor(filtered().findIndex(item=>item.id===id)/pageSize)*pageSize; + selected=id;$('detail').replaceChildren();render(); + const value=await read('file_get',{file_id:model.card.file_id,revision:model.card.revision},signal);if(!active())return; + const captured=checkCard(guide,model,value),heading=node('h3',model.label);heading.id='project-portfolio-model-heading';heading.tabIndex=-1;$('detail').append(heading); + $('detail').append(node('p',(model.aliases.join(' · ')||model.id)+' · '+(model.review==='source_reviewed'?'Source reviewed':'Source review pending'))); + if(model.summary){const labels={method:'Method',purpose:'Purpose',experiment:'Training and experiment',outcome:'Observed outcome',lesson:'Lesson',reproduction:'Reproduction'}; + for(const field of fields){$('detail').append(node('h4',labels[field]),node('p',model.summary[field]));}} + else $('detail').append(node('p','A source-reviewed interpretation has not been recorded for this fit. Its original captured card remains available below.')); + const relations=guide.relationships.filter(relation=>relation.from===id||relation.to===id); + if(relations.length){$('detail').append(node('h4','Authored model relationships')); + for(const relation of relations){const box=node('div'),other=models.get(relation.from===id?relation.to:relation.from),button=node('button', + (relation.kind==='comparison'?'Comparison: ':relation.from===id?'Parameters from: ':'Derived model: ')+other.label);button.type='button';button.onclick=()=>void selectModel(other.id); + box.append(button,node('p',relation.explanation));const evidence=node('details');evidence.append(node('summary','Exact retained relationship excerpt'),node('p',relation.excerpt)); + const source=node('button','Read evidence card: '+models.get(relation.evidence_model).label);source.type='button';source.onclick=()=>void selectModel(relation.evidence_model);evidence.append(source);box.append(evidence);$('detail').append(box);}} + const raw=node('details'),summary=node('summary','Exact captured card and original receipts');raw.append(summary,node('p',`${model.card.path} · revision ${model.card.revision} · SHA256 ${model.card.sha256}`),node('pre',captured));$('detail').append(raw); + $('status').textContent='Exact retained model summary and captured evidence.';heading.focus(); + }); + } + $('notes').onclick=()=>void findNotes();$('notes-more').onclick=()=>void findNotes(true); + $('note').onchange=()=>{clearGuide();controls();}; + $('source-form').onsubmit=event=>{event.preventDefault();void loadGuide();}; + $('filter-form').onsubmit=event=>{event.preventDefault();applyFilters();}; + for(const [name,direction] of [['prev',-1],['next',1]])$(name).onclick=()=>{if(busy||!guide)return;offset=Math.max(0,offset+direction*pageSize);selected='';$('detail').replaceChildren();render();}; + controls(); + return {setContext(value){if(value.key!==context?.key||context?.enabled&&!value.enabled){const owned=root.contains(document.activeElement);context=value;reset();if(owned)onContextFocusLost();}else context=value;root.hidden=!value.enabled;controls();},clear(){context=null;reset();root.hidden=true;}}; + }; +})(); diff --git a/tests/browser_model_portfolio.cjs b/tests/browser_model_portfolio.cjs new file mode 100644 index 00000000..851e650f --- /dev/null +++ b/tests/browser_model_portfolio.cjs @@ -0,0 +1,129 @@ +'use strict'; +// Real Chromium interaction/layout and controlled read-lifecycle evidence. +// This isolated static fixture does not claim native project persistence. +const fs=require('node:fs'),path=require('node:path'),http=require('node:http'),crypto=require('node:crypto'); +const {spawn}=require('node:child_process'); +const {waitForDebugger}=require('./browser_startup.cjs'),{cdpSession}=require('./browser_cdp_session.cjs'); +const output=path.resolve(process.argv[2]||''),engine=process.env.TULDOK_BROWSER_EXECUTABLE; +if(!process.argv[2]||!engine||fs.existsSync(output))throw Error('Explicit fresh output directory and browser executable required'); +fs.mkdirSync(output,{recursive:true,mode:0o700}); +const fixture=structuredClone(require('./model_portfolio_fixture.cjs')); +const hash=value=>crypto.createHash('sha256').update(value).digest('hex'); +const first=fixture.document.models[0]; +const oldMethod=first.summary.method;first.summary.method+=' '+('A physical pressure field with explicit case and work scope. '.repeat(60)); +first.label='First model '; +fixture.cards[0].version.text=fixture.cards[0].version.text.replace(oldMethod,first.summary.method); +fixture.cards[0].version.sha256=first.card.sha256=hash(fixture.cards[0].version.text); +fixture.cards[0].version.bytes=Buffer.byteLength(fixture.cards[0].version.text); +for(let index=0;index<20;index++){ + const model=structuredClone(fixture.document.models[1]);model.id='extra-'+index;model.label='Extra model '+index;model.aliases=[]; + model.card.path='notes/extra-'+index+'.md';model.card.file_id=hash(fixture.project+'\0note\0'+model.card.path); + fixture.document.models.push(model);fixture.cards.push({...structuredClone(fixture.cards[1]),id:model.card.file_id,path:model.card.path}); +} +fixture.guide.version.text=JSON.stringify(fixture.document); +fixture.guide.version.sha256=fixture.inspection.expected_sha256=fixture.inspection.observed_sha256=hash(fixture.guide.version.text); +fixture.guide.version.bytes=fixture.inspection.expected_bytes=fixture.inspection.observed_bytes=Buffer.byteLength(fixture.guide.version.text); +const inline=JSON.stringify(fixture).replaceAll('<','\\u003c'); +const html=`Authored organization fixture
`; +const staticRoot=path.join(__dirname,'../static'),allowed=new Map([ + ['/project-files.css',['project-files.css','text/css']],['/experiment-lineage.css',['experiment-lineage.css','text/css']], + ['/experiment-graph.js',['experiment-graph.js','text/javascript']],['/project-model-portfolio.js',['project-model-portfolio.js','text/javascript']]]); +const server=http.createServer((request,response)=>{ + if(request.url==='/'){response.writeHead(200,{'Content-Type':'text/html; charset=utf-8'});response.end(html);return;} + const asset=allowed.get(request.url);if(!asset){response.writeHead(404);response.end();return;} + response.writeHead(200,{'Content-Type':asset[1]});response.end(fs.readFileSync(path.join(staticRoot,asset[0]))); +}); +let child,session; +const report={status:'starting',engine,verification_only:true,native_persistence_claim:false,graphics:'Headless Chromium with GPU disabled; no model or rendering speed claim.',checks:[],errors:[],browser_stopped:false}; +const sourceFiles=['app.py','static/project-files.css','static/project-files.html','static/project-files.js','static/project-model-portfolio.js', + 'docs/contracts/project-model-portfolio.md','tests/model_portfolio_fixture.cjs','tests/test_project_model_portfolio.cjs', + 'tests/browser_model_portfolio.cjs','tests/test_project_model_portfolio_http.py','tests/test_workspace_project_files.cjs','tests/test_workspace_project_files_m4.cjs']; +report.source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); +const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); +async function main(){ + await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); + const origin='http://127.0.0.1:'+server.address().port; + try{ + const flags=['--headless','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,1000','--user-data-dir='+path.join(output,'profile'),'about:blank']; + report.flags=flags;report.origin=origin; + const log=fs.openSync(path.join(output,'browser.log'),'w');child=spawn(engine,flags,{stdio:['ignore',log,log]});fs.closeSync(log); + const port=await waitForDebugger(child,path.join(output,'profile/DevToolsActivePort')); + const targets=await(await fetch('http://127.0.0.1:'+port+'/json/list')).json(),target=targets.find(target=>target.type==='page'); + const socket=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{socket.addEventListener('open',resolve,{once:true});socket.addEventListener('error',reject,{once:true});}); + session=cdpSession(socket,{onEvent:event=>{if(event.method==='Runtime.exceptionThrown')report.errors.push(event.params.exceptionDetails.text);}}); + const send=session.send; + await send('Page.enable');await send('Runtime.enable'); + async function evaluate(expression){const result=await send('Runtime.evaluate',{expression,returnByValue:true,awaitPromise:true});if(result.exceptionDetails)throw Error(result.exceptionDetails.exception?.description||result.exceptionDetails.text);return result.result.value;} + async function until(expression){const end=Date.now()+15000;while(Date.now(){const element=document.querySelector(${JSON.stringify(selector)});if(!element||element.disabled)throw Error('Unavailable fixture control');element.scrollIntoView({block:'center'});element.focus();})()`); + await send('Input.dispatchKeyEvent',{type:'keyDown',key:'Enter',code:'Enter',windowsVirtualKeyCode:13,text:'\r'});await send('Input.dispatchKeyEvent',{type:'keyUp',key:'Enter',code:'Enter',windowsVirtualKeyCode:13});} + async function fill(selector,value){await evaluate(`(()=>{const element=document.querySelector(${JSON.stringify(selector)});element.value=${JSON.stringify(value)};element.dispatchEvent(new Event('input',{bubbles:true}));element.dispatchEvent(new Event('change',{bubbles:true}));})()`);} + async function screenshot(name){const shot=await send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,name),Buffer.from(shot.data,'base64'));} + async function load(){await keyClick('#project-portfolio-notes');await until("document.querySelector('#project-portfolio-note').options.length===1");await keyClick('#project-portfolio-load');await until("document.querySelector('#project-portfolio-status').textContent==='Loaded retained authored model organization.'");} + async function selectFirst(){await keyClick('[data-lineage-node="model/fit-first"]');await until("document.querySelector('#project-portfolio-status').textContent==='Exact retained model summary and captured evidence.'");} + await send('Page.navigate',{url:origin+'/'});await until("!!window.fixtureUI"); + await load();await check('readCalls.length===2','Guide loading reads no model card or payload'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('22 models in this guide; 1 source reviewed')",'Exact complete and reviewed guide counts'); + const point=await evaluate("(()=>{const element=document.querySelector('[data-lineage-node=\"group/pressure\"]');element.scrollIntoView({block:'center'});const box=element.getBoundingClientRect();return {x:box.x+box.width/2,y:box.y+box.height/2};})()"); + await send('Input.dispatchMouseEvent',{type:'mousePressed',button:'left',clickCount:1,...point});await send('Input.dispatchMouseEvent',{type:'mouseReleased',button:'left',clickCount:1,...point}); + await check("document.querySelector('#project-portfolio-group').value==='pressure'&&document.activeElement.dataset.lineageNode==='group/pressure'",'Pointer group activation applies the group and retains visible keyboard focus'); + await check("!document.querySelector('#root img')&&!window.injected",'Authored graph labels remain inert text'); + await keyClick('#project-portfolio-next');await check("document.querySelector('#project-portfolio-count').textContent.includes('Showing 21–22.')",'Bounded model paging uses the full population'); + await keyClick('#project-portfolio-prev');await selectFirst(); + await check("document.activeElement.id==='project-portfolio-model-heading'",'Keyboard selection moves focus to selected summary'); + await check("document.querySelector('#project-portfolio-detail').innerText.includes(fixtureData.document.models[0].summary.method)",'Complete long method paragraph is readable retained text'); + await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Desktop detail wraps without horizontal overflow'); + await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('desktop-summary.png'); + await send('Emulation.setDeviceMetricsOverride',{width:390,height:844,deviceScaleFactor:1,mobile:true}); + await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Mobile detail wraps without horizontal overflow'); + await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('mobile-summary.png'); + await send('Emulation.clearDeviceMetricsOverride'); + await fill('#project-portfolio-search','C107');await keyClick('#project-portfolio-filter-form button'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('1 match the applied filters')&&document.querySelector('#project-portfolio-next').disabled",'Alias filtering has exact counts and paging'); + await selectFirst();await keyClick('#project-portfolio-detail button');await until("document.querySelector('#project-portfolio-model-heading')?.textContent==='Second model'"); + await check("document.querySelector('#project-portfolio-detail').innerText.includes('A source-reviewed interpretation has not been recorded')",'Pending review does not invent a method summary'); + await check("document.querySelector('[data-lineage-node=\"model/fit-second\"]').getAttribute('aria-pressed')==='true'",'Off-filter relationship navigation brings selected model into the graph window'); + await selectFirst();await evaluate('window.failNext=true');await keyClick('#project-portfolio-detail button'); + await until("document.querySelector('#project-portfolio-status').textContent==='unavailable: Retained card unavailable.'"); + await check("document.querySelector('#project-portfolio-detail').children.length===0",'Unavailable card clears the previous model detail'); + await evaluate('window.corruptNext=true');await keyClick('[data-lineage-node="model/fit-first"]'); + await until("document.querySelector('#project-portfolio-status').textContent==='invalid: Model card differs from its exact retained organization reference.'"); + await check("document.querySelector('#project-portfolio-detail').children.length===0",'Foreign card revision is refused without stale detail'); + await evaluate('window.holdNext=true');await keyClick('[data-lineage-node="model/fit-first"]');await until('!!window.releaseHeld'); + await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:true});window.releaseHeld()"); + await check("window.heldSignal.aborted&&document.querySelector('#project-portfolio-content').hidden",'Context replacement aborts and clears pending view'); + await load();await check("document.querySelector('#project-portfolio-detail').children.length===0",'Reload after supersession starts without old detail'); + await selectFirst();await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:false})"); + await check("document.querySelector('#root').hidden&&document.querySelector('#project-portfolio-detail').children.length===0&&document.activeElement.id==='outside'",'Permission loss clears private view and restores host focus'); + await evaluate("fixtureUI.setContext({key:'three',project_id:fixtureData.project,enabled:true})");await load(); + await fill('#project-portfolio-search','No match');await keyClick('#project-portfolio-filter-form button'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('0 match the applied filters. Showing 0–0.')",'Empty filter result retains exact zero denominator'); + await check('readCalls.every(call=>[\'files_inspect\',\'file_get\'].includes(call.action))','All interactions use read-only project-file actions'); + if(report.errors.length)throw Error('Browser runtime exception');report.status='passed_authored_portfolio_real_browser_component'; + }catch(error){report.status='failed_authored_portfolio_real_browser_component';report.error=String(error); + if(session){try{const state=await session.send('Runtime.evaluate',{expression:'document.body.innerText',returnByValue:true});report.failure_page=state.result?.value; + const shot=await session.send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,'failed-page.png'),Buffer.from(shot.data,'base64'));}catch(captureError){report.failure_capture_error=String(captureError);}} + throw error;} + finally{session?.close();if(child&&child.exitCode===null&&child.signalCode===null){child.kill('SIGTERM');for(let i=0;i<100&&child.exitCode===null&&child.signalCode===null;i++)await pause(50);if(child.exitCode===null&&child.signalCode===null)child.kill('SIGKILL');} + report.browser_stopped=!child||child.exitCode!==null||child.signalCode!==null;await new Promise(resolve=>server.close(resolve)); + report.end_source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); + report.source_stable_through_tests=JSON.stringify(report.source_sha256)===JSON.stringify(report.end_source_sha256); + if(!report.source_stable_through_tests){report.status='failed_authored_portfolio_real_browser_component';report.errors.push('Source changed during browser qualification');process.exitCode=1;} + report.producer_sha256=hash(fs.readFileSync(__filename));fs.writeFileSync(path.join(output,'report.json'),JSON.stringify(report,null,2)+'\n');} + console.log(report.status,report.checks.length,'checks'); +} +main().catch(error=>{console.error(String(error));process.exitCode=1;}); diff --git a/tests/model_portfolio_fixture.cjs b/tests/model_portfolio_fixture.cjs new file mode 100644 index 00000000..cb63135a --- /dev/null +++ b/tests/model_portfolio_fixture.cjs @@ -0,0 +1,16 @@ +'use strict'; +// Literal presentation-document fixtures, independent of the decoder. +const crypto=require('node:crypto'); +const hash=value=>crypto.createHash('sha256').update(value).digest('hex'); +const project='a'.repeat(32),path='notes/00-methods.portfolio.json'; +const summary={method:'A small pressure correction model.',purpose:'Measure one pressure correction.',experiment:'Two retained training scenes.',outcome:'Retained diagnostic result; not a rollout qualification.',lesson:'Compare work and accuracy on the same cases.',reproduction:'Use the retained source recipe; clean retraining remains pending.'}; +const cardText=Object.values(summary).join('\n\n')+'\nOriginal receipt bytes remain here.\n'; +const ref=(path,text)=>({file_id:hash(project+'\0note\0'+path),path,revision:2,sha256:hash(text)}); +const first={id:'fit-first',label:'First model',group_id:'pressure',aliases:['C107'],review:'source_reviewed',card:ref('notes/first.md',cardText),summary}; +const second={id:'fit-second',label:'Second model',group_id:'pressure',aliases:['C106'],review:'review_pending',card:ref('notes/second.md','Original pending card.\n'),summary:null}; +const document={format:'project_model_portfolio_v1',project_id:project,title:'Retained fit guide',description:'Authored retrospective organization; original Trial history is unchanged.',authorship:'retrospective',groups:[{id:'pressure',label:'Pressure methods',parent:null,description:'Pressure requests, not complete simulation steps.'}],models:[first,second],relationships:[{from:first.id,to:second.id,kind:'comparison',explanation:'Authored comparison with the earlier control.',evidence_model:first.id,excerpt:summary.lesson}]}; +const result=(model,text)=>({id:model.card.file_id,project_id:project,kind:'note',path:model.card.path,version:{revision:2,sha256:hash(text),bytes:Buffer.byteLength(text),text,actor_id:'b'.repeat(32),adopted_at:'2026-10-25T00:00:00+00:00'}}); +const cards=[result(first,cardText),result(second,'Original pending card.\n')]; +const raw=JSON.stringify(document),guideRef=ref(path,raw),guide={...result({card:guideRef},raw),version:{...result({card:guideRef},raw).version,revision:1}}; +const inspection={id:guideRef.file_id,kind:'note',path,revision:1,expected_sha256:guideRef.sha256,expected_bytes:Buffer.byteLength(raw),observed_sha256:guideRef.sha256,observed_bytes:Buffer.byteLength(raw),state:'unchanged',reason:null}; +module.exports={project,document,summary,cards,guide,inspection}; diff --git a/tests/test_access_http.py b/tests/test_access_http.py index cb9e5366..9f73fe3a 100644 --- a/tests/test_access_http.py +++ b/tests/test_access_http.py @@ -4,6 +4,7 @@ import threading import unittest import uuid +from unittest.mock import patch from http.server import ThreadingHTTPServer from urllib.error import HTTPError from urllib.request import Request, urlopen @@ -67,6 +68,31 @@ def test_denial_audit_storage_outage_still_returns_structured_error(self): status, body, _ = self.request('/api/access/session', headers={'Authorization': 'Bearer'}) self.assertEqual((status, body['code']), (503, 'storage')) + def test_busy_catalog_authentication_returns_json_and_keeps_login_code_usable(self): + code = self.dataset.access._local_owner_token() + held, release = threading.Event(), threading.Event() + def hold_catalog(): + with self.dataset.access.lock: + held.set() + release.wait(5) + holder = threading.Thread(target=hold_catalog) + holder.start() + try: + self.assertTrue(held.wait(2)) + with patch('metadata.CATALOG_LOCAL_TIMEOUT', .05): + for body, headers in (({'token':code}, {'Origin':self.root}), (None, self.owner.headers), + (None, {'Authorization':'Bearer'}), + ({'token':code}, {'Origin':'http://elsewhere.invalid'})): + status, value, _ = self.request('/api/access/session', body, headers) + self.assertEqual((status, value['code']), (503, 'storage')) + self.assertIn('busy completing another operation', value['error']) + finally: + release.set() + holder.join(5) + self.assertFalse(holder.is_alive()) + status, identity, _ = self.request('/api/access/session', {'token':code}, {'Origin':self.root}) + self.assertEqual((status, identity['kind']), (200, 'owner')) + def test_agent_review_and_execution_denied_before_mutation(self): with self.owner.bound(self.dataset): agent = self.dataset.access.issue({'label': 'Editor', 'scopes': ['inspect', 'edit', 'execute']}) diff --git a/tests/test_agent_api.py b/tests/test_agent_api.py index 7ccee78f..004121b4 100644 --- a/tests/test_agent_api.py +++ b/tests/test_agent_api.py @@ -50,6 +50,26 @@ def sample(schema): class RegistryTests(unittest.TestCase): + def test_ascii_shortcut_preserves_unicode_and_scoped_nul_rejection(self): + for codepoint in range(128): + value=chr(codepoint) + if codepoint==0: + with self.assertRaises(api.AgentAPIError):api.decode(json.dumps(value)) + else: + self.assertEqual(api.decode(json.dumps(value)),value) + for codepoint in range(0xd800,0xe000): + with self.assertRaises(api.AgentAPIError):api.decode(json.dumps(chr(codepoint))) + for codepoint in (0x80,0x7ff,0x800,0xd7ff,0xe000,0xffff,0x10000,0x10ffff): + value=chr(codepoint) + self.assertEqual(api.decode(json.dumps(value,ensure_ascii=False)),value) + allowed={'entry':'left\x00right'} + self.assertEqual(api.decode(json.dumps(allowed),nul_path=('entry',)),allowed) + for denied in ({'entry':'ok','other':'\x00'},{'\x00':'ok'}): + with self.assertRaises(api.AgentAPIError):api.decode(json.dumps(denied),nul_path=('entry',)) + class FalseAscii(str): + def isascii(self):return True + with self.assertRaises(api.AgentAPIError):api._finite(FalseAscii('\ud800')) + def test_discovery_is_stdlib_only_in_cold_process(self): code = 'import sys;sys.path.insert(0,sys.argv[1]);import agent_api;agent_api.discovery();assert not any(x in sys.modules for x in ("jsonschema","app","workbench","mcp","psycopg"))' subprocess.run([sys.executable, '-I', '-S', '-c', code, str(Path(api.__file__).parent)], check=True, capture_output=True, timeout=10) @@ -264,6 +284,14 @@ def test_absolute_header_deadline_includes_trickle(self): with self.assertRaises(api.AgentAPIError): client.call('data.query',{}) self.assertLess(time.monotonic()-start,1) + def test_owner_transport_keeps_the_selected_absolute_wait_and_never_retries(self): + with server('trickle') as (endpoint,calls): + client=api.ProjectFilesOwnerClient(endpoint,TOKEN,CATALOG,timeout=.15) + start=time.monotonic() + with self.assertRaises(api.AgentAPIError):client._request('GET','/api/workbench/records') + self.assertLess(time.monotonic()-start,1) + self.assertEqual(len(calls),1) + class OutputTests(unittest.TestCase): @unittest.skipUnless(HAS_JSONSCHEMA, 'Optional agent jsonschema dependency is not installed.') diff --git a/tests/test_measurement_comparison.py b/tests/test_measurement_comparison.py new file mode 100644 index 00000000..16418875 --- /dev/null +++ b/tests/test_measurement_comparison.py @@ -0,0 +1,119 @@ +"""Scientific composition of real retained scalar tables, no native transport.""" +import copy +import struct +import unittest +from unittest.mock import patch + +import retained_measurements as owner +from retained_measurements import MeasurementComparison, MeasurementTable, encode +from test_retained_measurements import fixture, source +from workbench import WorkbenchError + + +def partition(subject): + document = fixture() + document['title'] = 'Original partition ' + subject + document['subjects'] = [item for item in document['subjects'] if item['id'] == subject] + document['observations'] = [row for row in document['observations'] if row['subject_id'] == subject] + return document + + +def selected(document, origin='a'*32): + table = MeasurementTable(encode(document)) + return dict(origin_catalog_id=origin, packet_sha256=table.sha256), table + + +class ComparisonTests(unittest.TestCase): + def setUp(self): + self.inputs = [selected(partition('a')), selected(partition('b'))] + self.comparison = MeasurementComparison(self.inputs) + + def test_complete_maxima_and_missing_denominators_keep_original_packet(self): + result = self.comparison.read('error') + self.assertEqual(len(result['subjects']), 2) + self.assertEqual(result['case_count'], 2) + self.assertEqual(result['observation_count'], 8) + self.assertEqual([s['maximum']['value'] for s in result['query']['summary']], [2e-15, .02]) + for row in result['query']['rows']: + locator, original = next(item for item in self.inputs if item[0] == row['source']) + evidence = original.inspect(row['id']) + self.assertEqual(struct.pack('!d', row['value']), struct.pack('!d', evidence['scalar_evidence']['value'])) + missing = self.comparison.read('latency')['query'] + self.assertEqual(missing['total'], 4) + self.assertEqual([s['states'] for s in missing['summary']], [ + dict(measured=0, failed=0, unsupported=0, unavailable=2), + dict(measured=0, failed=1, unsupported=0, unavailable=1)]) + self.assertTrue(all(s['maximum'] is None for s in missing['summary'])) + + def test_global_pages_span_packets_without_changing_maxima(self): + full = self.comparison.read('error')['query'] + rows = [] + for offset in range(5): + page = self.comparison.read('error', offset=offset, limit=1)['query'] + self.assertEqual(page['summary'], full['summary']) + self.assertEqual(page['total'], 4) + rows.extend(page['rows']) + self.assertEqual(rows, full['rows']) + self.assertEqual([r['subject_id'] for r in rows], ['a','a','b','b']) + ordered = self.comparison.read('error', subject_ids=['b','a'])['query'] + self.assertEqual([s['subject_id'] for s in ordered['summary']], ['b','a']) + + def test_density_domain_device_and_unknown_filters_are_separate(self): + density = self.comparison.read('error', filters={'cpm':16})['query'] + domain = self.comparison.read('error', filters={'grid_x':16})['query'] + self.assertEqual([s['maximum']['value'] for s in density['summary']], [1e-17,.01]) + self.assertEqual([s['maximum']['value'] for s in domain['summary']], [2e-15,.02]) + cpu = self.comparison.read('error', filters={'device':'CPU'})['query'] + self.assertEqual([s['count'] for s in cpu['summary']], [0,2]) + unknown = self.comparison.read('error', filters={'device':None})['query'] + self.assertEqual(unknown['total'], 0) + self.assertTrue(all(s['maximum'] is None for s in unknown['summary'])) + + def test_changed_original_case_evidence_or_dimension_meaning_refuses_comparison(self): + changed = partition('b') + context = fixture()['sources'][0] + import base64, json + original = json.loads(base64.b64decode(context['data_base64'])) + original['different_original_case'] = True + changed['sources'][0] = source('context', original) + for document in (changed, partition('b')): + if document is not changed: + document['dimensions'][0]['units'] = 'different units' + with self.assertRaises(WorkbenchError) as error: + MeasurementComparison([self.inputs[0], selected(document)]) + self.assertEqual(error.exception.code, 'incompatible') + + def test_overlapping_subjects_and_repeated_sources_are_explicit_errors(self): + with self.assertRaises(WorkbenchError): MeasurementComparison([self.inputs[0], self.inputs[0]]) + other = partition('a'); other['title'] = 'Different packet, same subject' + with self.assertRaises(WorkbenchError) as error: + MeasurementComparison([self.inputs[0], selected(other)]) + self.assertEqual(error.exception.code, 'conflict') + + def test_bounds_and_invalid_selections_never_return_partial_results(self): + for constant, maximum in [('MAX_COMPARISON_BYTES', 1), ('MAX_COMPARISON_SUBJECTS', 1), ('MAX_COMPARISON_ROWS', 7)]: + with patch.object(owner, constant, maximum), self.assertRaises(WorkbenchError): + MeasurementComparison(self.inputs) + with self.assertRaises(WorkbenchError): MeasurementComparison(self.inputs * 9) + for arguments in ({'subject_ids':[['bad']]}, {'subject_ids':[]}, {'subject_ids':['a','a']}, + {'subject_ids':['missing']}, {'offset':True}, {'limit':101}): + with self.subTest(arguments=arguments), self.assertRaises(WorkbenchError): + self.comparison.read('error', **arguments) + with self.assertRaises(WorkbenchError): self.comparison.read(filters={}) + + def test_packet_and_combined_observation_budgets_are_separate(self): + # Both real source tables have four observations. A packet limit of + # four still permits their complete eight-observation comparison. + with patch.object(owner,'MAX_ROWS',4),patch.object(owner,'MAX_COMPARISON_ROWS',8,create=True): + self.assertEqual(MeasurementComparison(self.inputs).read()['observation_count'],8) + with self.assertRaises(WorkbenchError):MeasurementTable(encode(fixture())) + with patch.object(owner,'MAX_COMPARISON_ROWS',7,create=True),self.assertRaises(WorkbenchError): + MeasurementComparison(self.inputs) + + def test_returned_metadata_and_query_values_have_no_mutable_owner_alias(self): + expected = self.comparison.read('error') + result = self.comparison.read('error') + result['query']['metric']['label'] = 'Changed' + result['query']['summary'][0]['states']['measured'] = 99 + result['subjects'][0]['source']['origin_catalog_id'] = 'b'*32 + self.assertEqual(self.comparison.read('error'), expected) diff --git a/tests/test_project_artifacts.py b/tests/test_project_artifacts.py index ecd81586..ad06e375 100644 --- a/tests/test_project_artifacts.py +++ b/tests/test_project_artifacts.py @@ -57,12 +57,111 @@ def commit(self, values, key='payloads'): return self.store.commit(records, self.store.snapshot()['checkpoint'], key, request(records)) class ProjectArtifactTests(_ArtifactFixture, unittest.TestCase): + def test_copy_group_keeps_exact_independent_snapshots_with_two_source_closure_checks(self): + raws = [b'first exact output', b'second exact output'] + values = [declaration(raw) for raw in raws] + for raw, value in zip(raws, values): + self.artifacts.publish_bytes(raw, value) + self.commit(values) + target = self.path / 'copy' + with ProjectStore.create(target, 'Copy', 'copy', 'b' * 64) as store, ProjectArtifacts(store) as destination: + publish = destination.publish_fd + def mutate_source_after_snapshot(fd, value, **options): + # A mutable original source must not be the fd consumed by the + # destination. Its already-verified snapshot stays independent. + original = self.payload(value) + original.unlink() + original.write_bytes(b'changed after verified capture') + return publish(fd, value, **options) + with patch.object(destination, 'publish_fd', side_effect=mutate_source_after_snapshot): + with patch.object(self.store, '_current', wraps=self.store._current) as checks: + self.artifacts.copy_many_to(destination, values) + self.assertEqual(checks.call_count, 2) + for raw, value in zip(raws, values): + copied = target / 'artifacts/sha256' / value['sha256'] + self.assertEqual(copied.read_bytes(), raw) + self.assertEqual(copied.stat().st_nlink, 1) + self.assertNotEqual(copied.stat().st_ino, self.payload(value).stat().st_ino) + + def test_copy_group_rejects_source_canonical_corruption_before_success(self): + value = declaration(b'present') + self.artifacts.publish_bytes(b'present', value) + self.commit([value]) + manifest = json.loads((self.project / '.tuldok/project.json').read_bytes()) + record = self.project / '.tuldok' / manifest['inventory'][0]['path'] + with ProjectStore.create(self.path / 'copy', 'Copy', 'copy', 'b' * 64) as store, ProjectArtifacts(store) as destination: + publish = destination.publish_fd + def corrupt_after_copy(*args, **options): + result = publish(*args, **options) + record.write_bytes(b'{"corrupt":true}\n') + return result + with patch.object(destination, 'publish_fd', side_effect=corrupt_after_copy): + self.assert_code('stale', self.artifacts.copy_many_to, destination, [value]) + + def test_copy_group_prevalidates_inventory_and_preserves_missing_payload_failure(self): + value, missing = declaration(b'present'), declaration(b'missing') + self.artifacts.publish_bytes(b'present', value) + self.commit([value, missing]) + with ProjectStore.create(self.path / 'copy', 'Copy', 'copy', 'b' * 64) as store, ProjectArtifacts(store) as destination: + with patch.object(destination, 'publish_fd', wraps=destination.publish_fd) as publish: + self.assert_code('invalid', self.artifacts.copy_many_to, destination, [value, dict(missing, bytes=True)]) + publish.assert_not_called() + self.assert_code('unavailable', self.artifacts.copy_many_to, destination, [value, missing]) + self.assertEqual(destination.status(value)['state'], 'verified') + self.assertEqual(destination.status(missing)['state'], 'missing') + + def test_status_group_retains_exact_hash_observations_with_two_closure_checks(self): + values = [declaration(b'present'), declaration(b'missing'), declaration(b'correct')] + self.artifacts.publish_bytes(b'present', values[0]) + self.artifacts.publish_bytes(b'correct', values[2]) + self.commit(values) + self.payload(values[2]).unlink() + self.payload(values[2]).write_bytes(b'changed') + expected = [self.artifacts.status(value) for value in values] + with patch.object(self.store, '_current', wraps=self.store._current) as checks: + actual = self.artifacts.status_many(values) + self.assertEqual(actual, expected) + self.assertEqual([item['state'] for item in actual], ['verified', 'missing', 'corrupt']) + self.assertEqual(checks.call_count, 2) + self.assertEqual(actual[0]['observed_sha256'], values[0]['sha256']) + self.assertEqual(actual[2]['observed_sha256'], hashlib.sha256(b'changed').hexdigest()) + + def test_status_group_rejects_canonical_corruption_during_observation(self): + value = declaration(b'present') + self.artifacts.publish_bytes(b'present', value) + self.commit([value]) + manifest = json.loads((self.project / '.tuldok/project.json').read_bytes()) + record = self.project / '.tuldok' / manifest['inventory'][0]['path'] + original = self.artifacts._source + @contextmanager + def corrupt_after_observation(*args): + with original(*args) as source: + yield source + record.write_bytes(b'{"corrupt":true}\n') + with patch.object(self.artifacts, '_source', side_effect=corrupt_after_observation): + self.assert_code('stale', self.artifacts.status_many, [value]) + + def test_status_group_rejects_checkpoint_change_and_unsafe_aliases(self): + value = declaration(b'present') + self.artifacts.publish_bytes(b'present', value) + self.commit([value]) + original = self.artifacts._source + @contextmanager + def change_checkpoint(*args): + with original(*args) as source: + yield source + self.commit([], key='new-checkpoint') + with patch.object(self.artifacts, '_source', side_effect=change_checkpoint): + self.assert_code('stale', self.artifacts.status_many, [value]) + os.link(self.payload(value), self.path / 'outside-alias') + self.assert_code('invalid', self.artifacts.status_many, [value]) + def test_presence_batch_matches_individual_observations_with_two_closure_checks(self): values=[declaration(b'present'),declaration(b'missing'),declaration(b'other')] self.artifacts.publish_bytes(b'present',values[0]) self.commit(values) expected=[self.artifacts.inspect(value) for value in values] - with patch.object(self.store,'snapshot',wraps=self.store.snapshot) as checks: + with patch.object(self.store,'_current',wraps=self.store._current) as checks: actual=self.artifacts.inspect_many(values) self.assertEqual(actual,expected) self.assertEqual(checks.call_count,2) diff --git a/tests/test_project_composition.py b/tests/test_project_composition.py new file mode 100644 index 00000000..c20a90df --- /dev/null +++ b/tests/test_project_composition.py @@ -0,0 +1,136 @@ +"""Exact retained-source assembly; native history remains the proof oracle.""" +import copy +import hashlib +import unittest +from unittest.mock import patch + +from project_composition import ProjectCompositionError, merge_retained_sources +from project_history import capture_run, declarations, decode_records +from project_store import _encode +from test_project_history import ORIGIN, legacy_run + + +def snapshot(identity, records): + return dict(project_id=identity*32, name='Source '+identity, checkpoint=identity*64, records=records) + + +def approved_history(): + run=legacy_run();run['approval_id']='9'*32 + run['release_id']=run['release_sha256']=hashlib.sha256(b'reference input').hexdigest() + run['release_bytes']=len(b'reference input') + grant=dict(id=run['approval_id'],catalog_id=ORIGIN,approver_actor_id='a'*32,approver_credential_id='b'*32, + consumer_actor_id='5'*32,material_sha256='f'*64,created_at=1,expires_at=100,revoked_at=None) + consumption=dict(grant_id=grant['id'],run_id=run['id'],admission_id=run['admission_id'], + consumer_credential_id='6'*32,request_sha256=run['request_sha256'],consumed_at=2) + effects={'pipeline.inspect_release':[run['release_id']],'dataset_releases.locate':[run['release_id']], + 'pipeline.probe':['image_classification'],'pipeline.probe_native':['image_classification'], + 'pipeline.definition_identity':['image_classification'],'pipeline.exposures':[None],'runs.update':[run['id']]} + admission=dict(id=run['admission_id'],catalog_id=ORIGIN,actor_id='5'*32,credential_id='6'*32, + kind='runs',job_id=run['id'],request_digest=run['request_sha256'],effects=effects,created_at=1) + record=capture_run(run,admission_proof=dict(grant=grant,consumption=consumption,admission=admission),payloads=[]) + records=[record,*declarations(record)] + decode_records(records) + return records + + +class RetainedCompositionTests(unittest.TestCase): + def test_exact_duplicate_is_shared_and_each_source_membership_is_retained(self): + records=approved_history();left=snapshot('1',records);right=snapshot('2',copy.deepcopy(records)) + result=merge_retained_sources([left,right]) + self.assertEqual(result['records'],sorted(records,key=lambda item:(item['family'],item['id']))) + self.assertEqual(len(result['sources']),2) + for source in result['sources']: + for record in records: + expected=hashlib.sha256(_encode(record)).hexdigest() + self.assertEqual(source['records'][record['family']][record['id']],expected) + self.assertEqual(source['run_observations'],{}) + + def test_alternate_grant_is_exact_native_observation_without_duplicate_run(self): + original=approved_history();alternate=copy.deepcopy(original) + alternate[0]['document']['admission_proof']['grant']['revoked_at']=3 + alternate[0]['document']['admission_proof']['consumption']['consumed_at']=3 + decode_records(alternate) + with patch('os.open',side_effect=AssertionError('Pure construction has no filesystem access')): + result=merge_retained_sources([snapshot('1',original),snapshot('2',alternate),snapshot('3',copy.deepcopy(alternate))]) + facts=decode_records(result['records']) + self.assertEqual(len(facts.runs),1) + self.assertEqual(len(facts.observations),1) + base=next(r for r in result['records'] if r['family']=='run_history') + observation=next(r for r in result['records'] if r['family']=='run_observation') + self.assertEqual(_encode(base),_encode(original[0])) + self.assertEqual(_encode(observation['document']['observation']),_encode(alternate[0]['document'])) + expected=hashlib.sha256(_encode(alternate[0]['document'])).hexdigest() + self.assertEqual(observation['id'],expected) + for source in result['sources'][1:]: + self.assertEqual(source['run_observations'],{original[0]['id']:expected}) + self.assertFalse(result['sources'][0]['run_observations']) + + def test_broader_run_conflicts_are_rejected_at_the_named_conflict_boundary(self): + original=approved_history() + for field in ('metrics','population','admission'): + alternate=copy.deepcopy(original) + if field=='metrics':alternate[0]['document']['run']['metrics']={'accuracy':0.5} + elif field=='population':alternate[0]['document']['payloads'][0]['population']='test' + else:alternate[0]['document']['admission_proof']['admission']['created_at']=2 + decode_records(alternate) + with self.subTest(field=field),self.assertRaises(ProjectCompositionError) as caught: + merge_retained_sources([snapshot('1',original),snapshot('2',alternate)]) + self.assertEqual(caught.exception.code,'conflict') + self.assertIn('run_history/'+original[0]['id'],str(caught.exception)) + + def test_equal_python_numbers_do_not_hide_different_canonical_bytes(self): + original=approved_history();original[0]['document']['run']['parameters']={'scale':1} + alternate=copy.deepcopy(original);alternate[0]['document']['run']['parameters']['scale']=1.0 + self.assertEqual(original,alternate) + decode_records(original);decode_records(alternate) + with self.assertRaises(ProjectCompositionError) as caught: + merge_retained_sources([snapshot('1',original),snapshot('2',alternate)]) + self.assertEqual(caught.exception.code,'conflict') + + def test_other_family_conflict_and_repeated_source_project_are_rejected(self): + original=approved_history();alternate=copy.deepcopy(original) + alternate[1]['document']['bytes']+=1 + with self.assertRaises(ProjectCompositionError) as caught: + merge_retained_sources([snapshot('1',original),snapshot('2',alternate)]) + self.assertEqual(caught.exception.code,'conflict') + self.assertIn('project_artifact/',str(caught.exception)) + with self.assertRaises(ProjectCompositionError) as repeated: + merge_retained_sources([snapshot('1',original),snapshot('1',original)]) + self.assertEqual(repeated.exception.code,'conflict') + self.assertIn('source project identity repeats',str(repeated.exception)) + + def test_source_unique_policy_overrides_another_sources_runtime_dependency(self): + from project_composition import build_candidate + from test_project_fork_composition import runtime_records + import uuid + sha=hashlib.sha256(b'Z').hexdigest() + runtime,_,_=runtime_records('9'*32,shared_sha=sha,shared_bytes=1) + declaration=next(r for r in runtime if r['family']=='project_artifact' and r['id']==sha) + candidate=build_candidate([snapshot('1',runtime),snapshot('2',[copy.deepcopy(declaration)])], + '3'*32,created_at='2026-10-24T12:00:00+00:00',id_factory=lambda *_:uuid.uuid4().hex, + directory='combined',name='Conservative source custody',request_key='shared-policy') + policies={edge['declaration']['sha256']:edge['preservation'] for edge in candidate['artifacts']} + self.assertEqual(policies[sha],'unique') + + def test_source_snapshots_can_be_loaded_lazily(self): + records=approved_history() + def sources(): + yield snapshot('1',records) + yield snapshot('2',copy.deepcopy(records)) + self.assertEqual(len(merge_retained_sources(sources())['sources']),2) + with self.assertRaises(ProjectCompositionError) as caught: + merge_retained_sources(iter([snapshot('1',records)])) + self.assertEqual(caught.exception.code,'invalid') + self.assertIn('at least two',str(caught.exception)) + + def test_output_owns_its_documents_and_does_not_mutate_sources(self): + original=approved_history();before=copy.deepcopy(original) + result=merge_retained_sources([snapshot('1',original),snapshot('2',copy.deepcopy(original))]) + result['records'][0]['document']['bytes']=123 + self.assertEqual(original,before) + original[0]['document']['run']['metrics']={'accuracy':0.25} + retained=next(r for r in result['records'] if r['family']=='run_history') + self.assertIsNone(retained['document']['run']['metrics']) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_project_composition_publication.py b/tests/test_project_composition_publication.py new file mode 100644 index 00000000..0bb6d881 --- /dev/null +++ b/tests/test_project_composition_publication.py @@ -0,0 +1,172 @@ +"""Real native stage/custody/reconciliation across multiple independent sources.""" +import copy +import errno +import hashlib +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import uuid + +from project_artifacts import ProjectArtifacts +from project_composition import decode_records, publish_composition +from project_fork import decode_fork, publish_fork +from project_fork_composition import build_candidate as fork_candidate +from project_history import decode_records as history_records +from project_store import DIR_FLAGS, ProjectStore, ProjectStoreError +from test_project_composition import approved_history + +NOW='2026-10-24T12:00:00+00:00' + + +def validate(records,identity,artifacts): + decode_fork(records,project_id=identity) + decode_records(records,project_id=identity) + history_records(records,artifacts=artifacts) + + +class CompositionPublicationTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory(prefix='native-composition-');self.addCleanup(self.tmp.cleanup) + self.root=Path(self.tmp.name);self.fd=os.open(self.root,DIR_FLAGS);self.addCleanup(os.close,self.fd) + self.stores=[];self.artifacts=[];self.snapshots=[];self.unique=[] + for number in range(2): + store=ProjectStore.create(self.root/('source'+str(number)),'Source '+str(number),'create','a'*64) + self.addCleanup(store.close);artifacts=ProjectArtifacts(store);self.addCleanup(artifacts.close) + records=approved_history() + if number:records[0]['document']['admission_proof']['grant']['revoked_at']=3 + unique=('unique source '+str(number)).encode();sha=hashlib.sha256(unique).hexdigest() + declaration=dict(format='project_artifact_v1',sha256=sha,bytes=len(unique)) + records.append(dict(family='project_artifact',id=sha,document=declaration)) + for raw in (b'reference input',unique): + artifacts.publish_bytes(raw,dict(format='project_artifact_v1',sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw))) + snap=store.commit(records,store.snapshot()['checkpoint'],'seed','b'*64) + self.stores.append(store);self.artifacts.append(artifacts);self.snapshots.append(snap);self.unique.append(sha) + + def publish(self,directory='combined',**options): + arguments=dict(root_path=self.root,name='Combined',expected_checkpoint=self.snapshots[0]['checkpoint'], + request_key='compose-exact',quiescent=True,created_at=NOW,id_factory=lambda *_:uuid.uuid4().hex,validate_candidate=validate) + arguments.update(options) + return publish_composition(self.stores[0],self.artifacts[0], + [dict(store=self.stores[1],artifacts=self.artifacts[1],checkpoint=self.snapshots[1]['checkpoint'])], + self.fd,directory,**arguments) + + def assert_code(self,code,call): + with self.assertRaises(ProjectStoreError) as caught:call() + self.assertEqual(caught.exception.code,code) + return caught.exception + + def test_complete_publication_preserves_both_sources_and_independent_payloads(self): + before=copy.deepcopy(self.snapshots) + result=self.publish();snapshot=result['snapshot'];identity=snapshot['project_id'] + self.assertFalse(result['reconciled']) + self.assertEqual(len(history_records(snapshot['records']).runs),1) + self.assertEqual(len(history_records(snapshot['records']).observations),1) + self.assertEqual(len(decode_records(snapshot['records'],project_id=identity)[identity]['sources']),2) + with ProjectStore.open(self.root/'combined') as destination,ProjectArtifacts(destination) as owner: + for sha in result['fork']['artifacts']['copied']: + source=next(s.path/'artifacts/sha256'/sha for s in self.stores if (s.path/'artifacts/sha256'/sha).exists()) + target=destination.path/'artifacts/sha256'/sha + self.assertEqual(target.read_bytes(),source.read_bytes()) + self.assertNotEqual((source.stat().st_dev,source.stat().st_ino),(target.stat().st_dev,target.stat().st_ino)) + self.assertEqual(target.stat().st_nlink,1) + self.assertEqual([s.snapshot() for s in self.stores],before) + + def test_already_forked_sources_keep_their_independent_histories(self): + for number in range(2): + result=publish_fork(self.stores[number],self.artifacts[number],self.fd,'forked-source-'+str(number), + root_path=self.root,name='Already forked source',expected_checkpoint=self.snapshots[number]['checkpoint'], + request_key='prepare-fork-'+str(number),quiescent=True, + build_candidate=lambda snap,dest:fork_candidate(snap,dest,created_at=NOW,id_factory=lambda *_:uuid.uuid4().hex), + validate_candidate=lambda records,identity:validate(records,identity,None)) + store=ProjectStore.open(self.root/('forked-source-'+str(number))) + artifacts=ProjectArtifacts(store);self.addCleanup(store.close);self.addCleanup(artifacts.close) + self.stores[number]=store;self.artifacts[number]=artifacts;self.snapshots[number]=result['snapshot'] + source_overlays=[next(r for r in s['records'] if r['family']=='project_fork') for s in self.snapshots] + result=self.publish();records=result['snapshot']['records'];identity=result['snapshot']['project_id'] + self.assertTrue(all(old in records for old in source_overlays)) + self.assertTrue(all(dict(family='project_fork',id=old['id']) in result['fork']['archived'] for old in source_overlays)) + self.assertEqual(decode_fork(records,project_id=identity)['destination_project_id'],identity) + self.assertIn(identity,decode_records(records,project_id=identity)) + + def test_missing_unique_secondary_bytes_leave_only_retained_staging(self): + (self.stores[1].path/'artifacts/sha256'/self.unique[1]).unlink() + self.assert_code('unavailable',self.publish) + self.assertFalse((self.root/'combined').exists()) + self.assertTrue(list(self.root.glob('.tuldok-fork-*'))) + + def test_missing_duplicate_is_satisfied_by_verified_primary_bytes(self): + sha=hashlib.sha256(b'reference input').hexdigest() + (self.stores[1].path/'artifacts/sha256'/sha).unlink() + result=self.publish() + self.assertIn(sha,result['fork']['artifacts']['copied']) + self.assertEqual((self.root/'combined/artifacts/sha256'/sha).read_bytes(),b'reference input') + + def test_corrupt_shared_secondary_copy_does_not_fall_back_to_good_primary(self): + sha=hashlib.sha256(b'reference input').hexdigest() + target=self.stores[1].path/'artifacts/sha256'/sha + target.chmod(0o600);target.write_bytes(b'wrong') + self.assert_code('invalid',self.publish) + self.assertFalse((self.root/'combined').exists()) + + def test_secondary_checkpoint_change_during_copy_prevents_publication(self): + copy_many=self.artifacts[0].copy_many_to + def changed(destination,declarations): + copy_many(destination,declarations) + source=self.stores[1];snapshot=source.snapshot() + source.commit(snapshot['records'],snapshot['checkpoint'],'new-source-receipt','c'*64) + with patch.object(self.artifacts[0],'copy_many_to',side_effect=changed): + self.assert_code('stale',self.publish) + self.assertFalse((self.root/'combined').exists()) + + def test_lost_publication_ack_reconciles_exact_sources_without_reconstruction(self): + fsync=os.fsync;injected=False + def fail_after_publish(fd): + nonlocal injected + if (self.root/'combined').exists() and not injected: + injected=True;raise OSError(errno.EIO,'bounded lost-ack fixture') + fsync(fd) + with patch('project_fork.os.fsync',side_effect=fail_after_publish): + failure=self.assert_code('unknown',self.publish) + self.assertTrue(failure.receipt['committed']) + with patch('project_composition.build_candidate',side_effect=AssertionError('No replayed construction')): + replay=self.publish() + self.assertTrue(replay['reconciled']) + self.assertEqual(replay['snapshot']['checkpoint'],failure.receipt['checkpoint']) + self.assert_code('conflict',lambda:self.publish(name='Changed input')) + + def test_destination_cannot_overlap_secondary_source(self): + self.assert_code('invalid',lambda:self.publish('source1/nested')) + self.assertFalse((self.stores[1].path/'nested').exists()) + + def test_persisted_membership_tampering_is_rejected(self): + result=self.publish();records=copy.deepcopy(result['snapshot']['records']) + doc=next(r['document'] for r in records if r['family']=='project_composition') + source=doc['sources'][1];family=next(iter(source['records']));identity=next(iter(source['records'][family])) + source['records'][family][identity]='0'*64 + error=self.assert_code('invalid',lambda:decode_records(records,project_id=result['snapshot']['project_id'])) + self.assertIn('exact retained envelope',str(error)) + + def test_malformed_persisted_run_memberships_have_typed_invalid_outcomes(self): + result=self.publish() + for kind in ('non-object','null-observation'): + records=copy.deepcopy(result['snapshot']['records']) + doc=next(r['document'] for r in records if r['family']=='project_composition') + source=doc['sources'][1] + if kind=='non-object':source['records']['run_history']=1 + else:source['run_observations'][next(iter(source['records']['run_history']))]=None + with self.subTest(kind=kind):self.assert_code('invalid',lambda:decode_records(records)) + + def test_future_ordinary_fork_preserves_and_archives_composition_provenance(self): + first=self.publish() + with ProjectStore.open(self.root/'combined') as source,ProjectArtifacts(source) as artifacts: + result=publish_fork(source,artifacts,self.fd,'descendant',root_path=self.root,name='Descendant', + expected_checkpoint=first['snapshot']['checkpoint'],request_key='fork-composition',quiescent=True, + build_candidate=lambda snap,dest:fork_candidate(snap,dest,created_at=NOW,id_factory=lambda *_:uuid.uuid4().hex), + validate_candidate=lambda records,identity:validate(records,identity,artifacts)) + self.assertIn(dict(family='project_composition',id=first['snapshot']['project_id']),result['fork']['archived']) + self.assertIn(first['snapshot']['project_id'],decode_records(result['snapshot']['records'],project_id=result['snapshot']['project_id'])) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_project_composition_workspace.py b/tests/test_project_composition_workspace.py new file mode 100644 index 00000000..af5134e3 --- /dev/null +++ b/tests/test_project_composition_workspace.py @@ -0,0 +1,87 @@ +"""Owner-authorized native composition, editable notes and selection recovery.""" +import copy +import hashlib +import os +import unittest +from unittest.mock import patch + +import agent_api +from access import AccessError +from project_artifacts import ProjectArtifacts +from project_composition import decode_records +from project_files import file_id +from project_store import ProjectStore +from workbench import WorkbenchError +from test_project_composition import approved_history +from test_project_workspace import ProjectWorkspaceTests + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'),'Requires disposable PostgreSQL') +class CompositionWorkspaceTests(unittest.TestCase): + def setUp(self): + self.fixture=ProjectWorkspaceTests('test_lifecycle_replay_is_idempotent_only_while_resulting_state_is_current') + self.fixture.setUp();self.addCleanup(self.fixture.doCleanups) + f=self.fixture;created,_=f.create();self.primary_id=created['project']['project_id'] + for revision,text in enumerate(('Original decision Ω\r\n','Updated decision Ω\r\n')): + raw=text.encode();target=f.external/'alpha/notes/decision.md';target.parent.mkdir(exist_ok=True) + target.write_bytes(raw) + f.call('file_adopt',f.mutation(kind='note',path='notes/decision.md',expected_revision=revision, + expected_sha256=hashlib.sha256(raw).hexdigest())) + self.primary=f.workspace._store.snapshot() + with ProjectStore.create(f.external/'retained','Retained source','create','a'*64) as store,ProjectArtifacts(store) as artifacts: + records=approved_history();raw=b'reference input' + artifacts.publish_bytes(raw,dict(format='project_artifact_v1',sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw))) + self.secondary=store.commit(records,store.snapshot()['checkpoint'],'seed','b'*64) + self.arguments=f.mutation(root='fixture',directory='combined',name='Combined scientific history',sources=[dict( + root='fixture',directory='retained',project_id=self.secondary['project_id'],checkpoint=self.secondary['checkpoint'])]) + + def compose(self):return self.fixture.call('compose',self.arguments) + + def test_native_owner_composition_preserves_note_history_and_selects_editable_primary_fork(self): + f=self.fixture + with patch('subprocess.Popen',side_effect=AssertionError('Imported projects never execute')): + result=self.compose() + state=result['state'];self.assertEqual(state['state'],'open') + self.assertNotEqual(state['project']['project_id'],self.primary_id) + self.assertEqual(result['sources'],[{key:s[key] for key in ('project_id','checkpoint')} for s in (self.primary,self.secondary)]) + snapshot=f.workspace._store.snapshot();identity=snapshot['project_id'] + decode_records(snapshot['records'],project_id=identity) + old_note=next(r for r in self.primary['records'] if r['family']=='project_file') + self.assertIn(old_note,snapshot['records']);self.assertEqual(len(old_note['document']['history']),2) + adopted=file_id(identity,'note','notes/decision.md') + value=f.call('file_get',dict(binding=state['binding'],checkpoint=state['project']['checkpoint'],file_id=adopted))['value'] + self.assertEqual(value['version']['text'],'Updated decision Ω\r\n') + raw=b'Next independent decision\n';target=f.external/'combined/notes/decision.md';target.write_bytes(raw) + updated=f.call('file_adopt',f.mutation(kind='note',path='notes/decision.md',expected_revision=value['version']['revision'], + expected_sha256=hashlib.sha256(raw).hexdigest())) + self.assertEqual(updated['value']['revision'],value['version']['revision']+1) + agent_api.validate_output('project_files.compose',dict(status=200,result=result),version=1) + + def test_failed_selection_preserves_primary_and_exact_retry_reconciles_destination(self): + f=self.fixture;project=f.workspace._projection_for + def fail_new(store): + if store.path.name=='combined':raise WorkbenchError('Bounded selection fixture failure.','unavailable',503) + return project(store) + with patch.object(f.workspace,'_projection_for',side_effect=fail_new),self.assertRaises(WorkbenchError) as caught:self.compose() + self.assertEqual(caught.exception.code,'project_selection_unavailable') + self.assertTrue(caught.exception.receipt['committed']);self.assertFalse(caught.exception.receipt['selected']) + self.assertEqual(f.state()['project']['project_id'],self.primary_id) + replay=self.compose();self.assertTrue(replay['fork']['reconciled']) + self.assertEqual(replay['state']['project']['project_id'],caught.exception.receipt['project_id']) + + def test_wrong_secondary_checkpoint_is_rejected_before_destination_publication(self): + self.arguments['sources'][0]['checkpoint']='0'*64 + with self.assertRaises(WorkbenchError) as caught:self.compose() + self.assertEqual(caught.exception.code,'stale_project') + self.assertFalse((self.fixture.external/'combined').exists()) + self.assertEqual(self.fixture.state()['project']['project_id'],self.primary_id) + + def test_anonymous_caller_is_denied_before_opening_retained_sources(self): + with patch.object(ProjectStore,'open_at',side_effect=AssertionError('No unauthorized filesystem selection')): + with self.assertRaises(AccessError) as caught: + self.fixture.workspace.dispatch('project_files.compose',self.arguments) + self.assertEqual(caught.exception.status,401) + self.assertFalse((self.fixture.external/'combined').exists()) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_project_dispatch.py b/tests/test_project_dispatch.py index f55de4cb..1aee28e5 100644 --- a/tests/test_project_dispatch.py +++ b/tests/test_project_dispatch.py @@ -30,7 +30,8 @@ def require(self, *args, **kwargs): def projection(): access = Authority() - session = SimpleNamespace(access=access, check=Mock(), snapshot={'project_id': PROJECT, 'records': []}) + session = SimpleNamespace(access=access, check=Mock(), check_current=Mock(), snapshot={'project_id': PROJECT, 'records': []}, + summary={'project_id': PROJECT, 'name': 'Dispatch fixture', 'checkpoint': CHECKPOINT}) return SimpleNamespace(session=session, data=Mock(), materials=Mock(), history=Mock(), context=SimpleNamespace(models=Mock(), pipelines=Mock()), artifacts=Mock(), producers=Mock(), source_adapter=Mock(), runtime_manifest=Mock(), disclose=Mock(), resolve_artifact=Mock(), execution=SimpleNamespace(study_execution=Mock(), producer_execution=Mock(), health=Mock(), preflight_external=Mock(), approve_external=Mock(), submit_external=Mock(), cancel_run=Mock(), reconcile_current=Mock())) @@ -38,7 +39,7 @@ def projection(): class Dispatch(unittest.TestCase): def test_supported_operations_match_the_finite_public_contract(self): - self.assertEqual(SUPPORTED_OPERATIONS, agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | (agent_api.PROJECT_FILES_M4 - {'project_files.save_as'})) + self.assertEqual(SUPPORTED_OPERATIONS, agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | (agent_api.PROJECT_FILES_M4 - {'project_files.save_as', 'project_files.compose'})) with self.assertRaises(WorkbenchError) as failure: dispatch(object(), 'project_files.runs_execute', {}) self.assertEqual(failure.exception.code, 'unsupported') diff --git a/tests/test_project_extraction.py b/tests/test_project_extraction.py index 59d4ab79..34e43176 100644 --- a/tests/test_project_extraction.py +++ b/tests/test_project_extraction.py @@ -9,6 +9,7 @@ import os from pathlib import Path import tempfile +import threading from types import SimpleNamespace import unittest from unittest.mock import patch @@ -19,15 +20,57 @@ from access import Access from metadata import Metadata from model_definitions import ModelDefinitions +from pipeline import digest from project_data import decode_data_records -from project_extraction import capture_legacy +from project_extraction import _Capture, capture_legacy from project_material import decode_records as decode_material -from project_transfer import ReadonlyCatalog, SourceView +from project_transfer import CapturedArtifacts, ReadonlyCatalog, SourceView from projects import Projects from saved_searches import SavedSearches from workbench import Workbench, WorkbenchError +class CaptureDiagnostics(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-capture-diagnostics-') + self.addCleanup(temporary.cleanup) + # These constructors assign owner references only. This fixture never + # calls a data reader, SQL, a model runtime or the source capability. + self.source = SimpleNamespace( + db=SimpleNamespace(dataset_id='a' * 32, path=Path(temporary.name), lock=threading.RLock()), + snapshot=dict(project_id='a' * 32, records=[]), artifacts=CapturedArtifacts()) + self.capture = _Capture(self.source) + + def test_severity_parent_family_and_code_keep_distinct_diagnostics(self): + capture = self.capture + parent = 'run_history', 'b' * 32 + capture.issue(parent, 'missing_source', 'Original source unavailable', warning=True) + capture.issue(parent, 'missing_source', 'Original source unavailable', warning=True) + capture.issue(parent, 'missing_source', 'Original source unavailable') + capture.issue(('study_history', parent[1]), 'missing_source', 'Original source unavailable', warning=True) + capture.issue(('run_history', 'c' * 32), 'missing_source', 'Original source unavailable', warning=True) + capture.issue(parent, 'unsupported', 'Original source unavailable', warning=True) + self.assertEqual(capture.warnings, [ + dict(family='run_history', id=parent[1], code='missing_source', detail='Original source unavailable'), + dict(family='study_history', id=parent[1], code='missing_source', detail='Original source unavailable'), + dict(family='run_history', id='c' * 32, code='missing_source', detail='Original source unavailable'), + dict(family='run_history', id=parent[1], code='unsupported', detail='Original source unavailable')]) + self.assertEqual(capture.blockers, [capture.warnings[0]]) + + def test_retained_detail_normalization_and_first_observation_order(self): + capture = self.capture + parent = 'run_history', 'b' * 32 + prefix = 'λ' * 1000 + capture.issue(parent, 'missing_source', prefix + 'first', warning=True) + capture.issue(parent, 'missing_source', prefix + 'second', warning=True) + capture.issue(parent, 'missing_source', 42, warning=True) + capture.issue(parent, 'missing_source', '42', warning=True) + self.assertEqual(capture.warnings, [ + dict(family='run_history', id=parent[1], code='missing_source', detail=prefix), + dict(family='run_history', id=parent[1], code='missing_source', detail='42')]) + self.assertEqual(capture.blockers, []) + + @unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') class LegacyCapture(unittest.TestCase): def setUp(self): @@ -85,6 +128,70 @@ def capture(self, source, roots): patch('runs.Runs.__init__', side_effect=AssertionError('Extraction constructed Runs')): return capture_legacy(source, selection) + def bind_manifest_copy(self, source): + recovered = self.root / 'recovered-original' + recovered.mkdir() + path = recovered / 'manifest.json' + path.write_bytes(self.manifest_raw) + descriptor = os.open(recovered, os.O_RDONLY | os.O_DIRECTORY) + info = os.fstat(descriptor) + source.roots['recovered'] = descriptor, [info.st_dev, info.st_ino] + source.bind_payload(hashlib.sha256(self.manifest_raw).hexdigest(), len(self.manifest_raw), + 'recovered', 'manifest.json') + return path + + def test_exact_legacy_activity_manifest_is_captured_without_reading_activity(self): + self.manifest['activity_path']='activity-must-remain-inert.json' + self.manifest_raw=(json.dumps(self.manifest,ensure_ascii=False,indent=3)+'\n\n').encode() + saved=json.loads(self.dataset.db.execute('SELECT document FROM workspace_projects WHERE id=%s', + (self.project_id,)).fetchone()[0]) + saved['manifest']=copy.deepcopy(self.manifest) + saved['manifest_sha256']=hashlib.sha256(self.manifest_raw).hexdigest() + self.dataset.db.execute('UPDATE workspace_projects SET document=%s WHERE id=%s', + (json.dumps(saved),self.project_id)) + self.selection['project_snapshot']['manifest_sha256']=saved['manifest_sha256'] + source=self.view();self.bind_manifest_copy(source) + fd,_=source.roots.pop('fixture');os.close(fd) + captured=self.capture(source,{}) + self.assertEqual(captured['blockers'],[]) + snapshot=next(row['document'] for row in captured['records'] if row['family']=='project_snapshot') + self.assertEqual(snapshot['value'],saved) + self.assertEqual(len(snapshot['artifacts']),1) + + def test_explicit_original_manifest_binding_keeps_history_without_old_root_authority(self): + source = self.view() + self.bind_manifest_copy(source) + descriptor, _ = source.roots.pop('fixture') + os.close(descriptor) + captured = self.capture(source, {}) + self.assertEqual(captured['blockers'], []) + snapshot = next(row['document'] for row in captured['records'] if row['family'] == 'project_snapshot') + self.assertEqual(snapshot['value']['root'], 'fixture') + self.assertEqual(snapshot['value']['manifest'], self.manifest) + declaration = next(row['document'] for row in captured['records'] if row['family'] == 'project_artifact') + self.assertEqual(source.artifacts.read_blob(declaration), self.manifest_raw) + self.assertNotIn('fixture', source.roots) + + def test_explicit_original_manifest_bytes_do_not_read_changed_working_manifest(self): + source = self.view() + self.bind_manifest_copy(source) + changed = b'{"unrelated_current_working_file":true}\n' + (self.project_root / 'source.json').write_bytes(changed) + captured = self.capture(source, {}) + self.assertEqual(captured['blockers'], []) + declaration = next(row['document'] for row in captured['records'] if row['family'] == 'project_artifact') + self.assertEqual(source.artifacts.read_blob(declaration), self.manifest_raw) + self.assertEqual((self.project_root / 'source.json').read_bytes(), changed) + + def test_changed_explicit_manifest_binding_blocks_instead_of_using_working_file(self): + source = self.view() + recovered = self.bind_manifest_copy(source) + recovered.write_bytes(b'changed after explicit binding') + captured = self.capture(source, {}) + self.assertTrue(any(row['code'] == 'stale_source' for row in captured['blockers'])) + self.assertFalse(any(row['family'] == 'project_snapshot' for row in captured['records'])) + self.assertEqual((self.project_root / 'source.json').read_bytes(), self.manifest_raw) + def test_text_responses_history_and_unrelated_inventory_are_read_only(self): selected = self.text();unrelated = self.text('Unrelated original data') response_id = uuid.uuid4().hex @@ -392,7 +499,7 @@ def test_two_study_lineage_reservation_candidate_and_run_science_are_captured(se before = self.files();source = self.view() captured = self.capture(source, {'studies': [child['id']]}) self.assertEqual(captured['blockers'], [], captured['blockers']) - view = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check=lambda: None) + view = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check_current=lambda: None) from project_material import ProjectMaterial facts = decode_history(captured['records'], materials=ProjectMaterial(view)) self.assertEqual(set(facts.studies), {(self.db.dataset_id, parent['id']), (self.db.dataset_id, child['id'])}) @@ -533,7 +640,7 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure captured = self.capture(source, {'runs': [consumer['id']]}) self.assertEqual(captured['blockers'], [], captured['blockers']) facts = decode_records(captured['records'], materials=ProjectMaterial(SimpleNamespace( - snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check=lambda: None))) + snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check_current=lambda: None))) self.assertEqual(set(facts.runs), {(self.db.dataset_id, run['id']) for run in (producer, middle, consumer)}) for downstream, upstream in ((consumer, middle), (middle, producer)): self.assertIn(dict(source=dict(family='run_history', id=self.db.dataset_id + '.' + downstream['id']), @@ -544,7 +651,8 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure # A fresh inert owner reader reconstructs the exact upstream custody # receipt from retained facts/private captured bytes, never source paths. session = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=copy.deepcopy(captured['records'])), - access=self.access, artifacts=source.artifacts, check=lambda: None) + summary=dict(project_id=self.db.dataset_id, checkpoint=digest(captured['records'])), + access=self.access, artifacts=source.artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda run, population: self.access.require('expose-holdout', 'runs.expose_test', run['id'])) with patch.object(source, 'read_catalog', side_effect=AssertionError('No original catalog reread')), \ diff --git a/tests/test_project_files.py b/tests/test_project_files.py index 3fbdcfab..f512d54e 100644 --- a/tests/test_project_files.py +++ b/tests/test_project_files.py @@ -7,6 +7,7 @@ from unittest.mock import patch import test_project_data as data_fixture +import project_files as files_module from project_files import (ProjectFiles, decode_records, validate_operation_result, file_id, fork_records, materialize_captured, MAX_FILE_BYTES, MAX_FILES) from project_session import ProjectSession, operation_records @@ -39,6 +40,98 @@ def adopt(self,path,text,revision=0,kind='script',key=None): return self.files.adopt(dict(kind=kind,path=path,expected_revision=revision, expected_sha256=hashlib.sha256(raw).hexdigest()),self.session.snapshot['checkpoint'],key or 'files-'+str(self.ordinal)) + def intent(self,path,text,revision=0,kind='note'): + raw=text.encode('utf-8');target=self.root/path;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(raw) + return dict(kind=kind,path=path,expected_revision=revision,expected_sha256=hashlib.sha256(raw).hexdigest()) + + def test_inspection_never_copies_the_complete_record_graph(self): + self.adopt('notes/inspection.md', 'Exact retained scientific note', kind='note') + expected = self.files.inspect() + with patch.object(self.session.store, '_snapshot', + wraps=self.session.store._snapshot) as snapshots: + observed = self.files.inspect() + self.assertEqual(observed, expected) + self.assertEqual(snapshots.call_count, 0, + 'File inspection needs only independent file records and complete freshness') + + def test_batch_captures_exact_files_in_one_canonical_mutation(self): + items=[self.intent('notes/a.md','Exact Ω\r\n'),self.intent('notes/empty.md','')] + before=self.session.snapshot['checkpoint'] + with patch.object(self.session,'mutate',wraps=self.session.mutate) as mutations: + result=self.files.adopt_many(dict(items=items),before,'batch') + self.assertEqual(mutations.call_count,1);self.assertEqual(result['changed_count'],2) + self.assertEqual([self.files.read(r['id'])['version']['text'] for r in result['items']],['Exact Ω\r\n','']) + operations=list(operation_records(self.session.snapshot['records']).values()) + self.assertEqual(sum(o['operation']=='files.adopt_many' for o in operations),1) + self.assertTrue(validate_operation_result('files.adopt_many',result,self.session.snapshot['records'])) + + def test_batch_changed_missing_or_repeated_file_never_partially_publishes(self): + first=self.intent('notes/a.md','First');second=self.intent('notes/b.md','Second') + before=copy.deepcopy(self.session.snapshot) + for label,items in [('hash',[first,dict(second,expected_sha256='0'*64)]), + ('duplicate',[first,first]),('revision',[first,dict(second,expected_revision=1)])]: + with self.subTest(label=label),self.assertRaises(WorkbenchError): + self.files.adopt_many(dict(items=items),before['checkpoint'],'reject-'+label) + self.assertEqual(self.session.snapshot,before) + (self.root/'notes/b.md').unlink() + with self.assertRaises(WorkbenchError):self.files.adopt_many(dict(items=[first,second]),before['checkpoint'],'missing') + self.assertEqual(self.session.snapshot,before);self.assertEqual(decode_records(self.session.snapshot['records']),[]) + + def test_batch_permission_precedes_file_reads_and_replay_preserves_exact_history(self): + items=[self.intent('notes/a.md','Original'),self.intent('notes/b.md','Other')] + token=self.fixture.access.issue(dict(label='Batch inspect only',scopes=['inspect']))['token'] + with self.fixture.access.bind(token),patch.object(files_module,'_read',side_effect=AssertionError('Unauthorized read')): + with self.assertRaises(AccessError):self.files.adopt_many(dict(items=items),self.session.snapshot['checkpoint'],'deny') + before=self.session.snapshot['checkpoint'];result=self.files.adopt_many(dict(items=items),before,'original-batch') + saved=copy.deepcopy(self.session.snapshot);(self.root/'notes/a.md').write_text('Unadopted edit') + with patch.object(files_module,'_read',side_effect=AssertionError('Replay must not recapture')): + self.assertEqual(self.files.adopt_many(dict(items=items),'0'*64,'original-batch'),result) + self.assertEqual(self.session.snapshot,saved) + self.intent('notes/a.md','Original');noop=[dict(item,expected_revision=1) for item in items] + value=self.files.adopt_many(dict(items=noop),self.session.snapshot['checkpoint'],'noop-batch') + self.assertEqual(value['changed_count'],0);self.assertTrue(all(len(self.files.history(r['id']))==1 for r in value['items'])) + + def test_batch_item_and_total_byte_bounds_keep_checkpoint(self): + before=copy.deepcopy(self.session.snapshot) + with self.assertRaises(WorkbenchError):self.files.adopt_many(dict(items=[]),before['checkpoint'],'empty') + items=[dict(kind='note',path=f'notes/{i}.md',expected_revision=0,expected_sha256='0'*64) for i in range(41)] + with self.assertRaises(WorkbenchError):self.files.adopt_many(dict(items=items),before['checkpoint'],'too-many') + large=[self.intent(f'notes/large-{i}.md','x'*MAX_FILE_BYTES) for i in range(9)] + with self.assertRaises(WorkbenchError):self.files.adopt_many(dict(items=large),before['checkpoint'],'aggregate-too-large') + self.assertEqual(self.session.snapshot,before) + + def test_batch_unsafe_last_file_and_project_capacity_never_publish(self): + first=self.intent('notes/a.md','First');last=self.intent('notes/b.md','Second') + before=copy.deepcopy(self.session.snapshot);path=self.root/'notes/b.md' + for label in ('symlink','utf8'): + path.unlink() + if label=='symlink':path.symlink_to(self.root/'notes/a.md') + else:path.write_bytes(b'\xff') + with self.subTest(label=label),self.assertRaises(WorkbenchError): + self.files.adopt_many(dict(items=[first,last]),before['checkpoint'],'unsafe-'+label) + self.assertEqual(self.session.snapshot,before) + self.intent('notes/b.md','Second') + with patch.object(files_module,'MAX_FILES',1),self.assertRaises(WorkbenchError): + self.files.adopt_many(dict(items=[first,last]),before['checkpoint'],'capacity') + self.assertEqual(self.session.snapshot,before) + + def test_batch_revision_history_and_forged_receipt_are_bound_to_retained_bytes(self): + item=self.intent('notes/a.md','First');body=dict(items=[item]) + first=self.files.adopt_many(body,self.session.snapshot['checkpoint'],'first-batch') + second=self.intent('notes/a.md','Second',1) + result=self.files.adopt_many(dict(items=[second]),self.session.snapshot['checkpoint'],'second-batch') + self.assertEqual(self.files.read(result['items'][0]['id'])['version']['revision'],2) + records=self.session.snapshot['records'] + self.assertTrue(validate_operation_result('files.adopt_many',first,records)) + for label in ('hash','count','duplicate','project'): + bad=copy.deepcopy(result) + if label=='hash':bad['items'][0]['sha256']='0'*64 + elif label=='count':bad['changed_count']=0 + elif label=='duplicate':bad['items'].append(copy.deepcopy(bad['items'][0]));bad['changed_count']=2 + else:bad['items'][0]['project_id']='f'*32 + with self.subTest(label=label),self.assertRaises(WorkbenchError): + validate_operation_result('files.adopt_many',bad,records) + def test_adoption_exact_utf8_history_changed_missing_and_cold_no_restore(self): source='#!/usr/bin/env python3\r\n# authored cafe\u0301 😀\r\n' first=self.adopt('scripts/authored.py',source,key='first') diff --git a/tests/test_project_files_clients.py b/tests/test_project_files_clients.py index e692dc45..2233b259 100644 --- a/tests/test_project_files_clients.py +++ b/tests/test_project_files_clients.py @@ -272,6 +272,9 @@ def test_browser_contract_pins_are_current_registry_output(self): self.assertEqual(pins, {name: api.contract(name)['sha256'] for name in api._CONTRACTS if name.startswith('project_files.')}) for name, digest in pins.items(): self.assertEqual(api.selected_header(name), name + '/1/' + digest) + versions=json.loads(re.search(r'/\* PROJECT_FILES_VERSIONS_START \*/\s*(.*?)\s*/\* PROJECT_FILES_VERSIONS_END \*/',source,re.S)[1]) + self.assertEqual(versions,{name:{str(v):api.contract(name,v)['sha256'] for v in sorted(values)} + for name,values in api._CONTRACTS.items() if name.startswith('project_files.') and len(values)>1}) def test_reads_pin_binding_checkpoint_and_path_record_without_body(self): arguments = {'record_id': RECORD, 'binding': BINDING, 'checkpoint': CHECKPOINT} @@ -352,6 +355,22 @@ def test_closed_record_output_rejects_unknown_and_inconsistent_history(self): class Consumers(unittest.TestCase): + def test_explicit_owner_wait_budget_preserves_generic_bounds_and_private_configuration(self): + endpoint,token='http://127.0.0.1:12345','fixture-agent-token-for-client-tests' + self.assertEqual(api.ProjectFilesOwnerClient(endpoint,token,CATALOG).timeout,120) + self.assertEqual(api.ProjectFilesOwnerClient(endpoint,token,CATALOG,timeout=7200).timeout,7200) + for client_type,invalid in ((api.AgentClient,301),(api.ProjectFilesOwnerClient,7201), + (api.ProjectFilesOwnerClient,float('inf')),(api.ProjectFilesOwnerClient,True)): + with self.subTest(client=client_type.__name__,timeout=invalid),self.assertRaises(api.AgentAPIError): + client_type(endpoint,token,CATALOG,timeout=invalid) + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);credential=root/'token';configuration=root/'config.json' + credential.write_text(token);credential.chmod(0o600) + configuration.write_text(json.dumps(dict(endpoint=endpoint,catalog_id=CATALOG, + token_file=str(credential),timeout=7200)));configuration.chmod(0o600) + self.assertEqual(api.ProjectFilesOwnerClient.from_config(configuration).timeout,7200) + with self.assertRaises(api.AgentAPIError):api.AgentClient.from_config(configuration) + def test_owner_file_client_keeps_unrelated_operations_outside_authority(self): raw = client({'status': 200, 'result': state()}, owner=True) self.assertEqual(ProjectFilesClient(raw).save(**bound()), state()) diff --git a/tests/test_project_files_fork_contracts.py b/tests/test_project_files_fork_contracts.py index 309f8ed2..b731de2c 100644 --- a/tests/test_project_files_fork_contracts.py +++ b/tests/test_project_files_fork_contracts.py @@ -35,8 +35,9 @@ def test_old_pins_stay_exact_and_only_v2_accepts_closed_fork_origins(self): health = 'project_files.execution_health' self.assertEqual(api.contract(health,1)['sha256'], 'c3b1d0017f5a49cf69c3a4c5c33e1d3db9846404d12627563d97ef40de7404a5') + comparison = 'project_files.measurement_compare' self.assertEqual(versions,{name:{str(v):api.contract(name,v)['sha256'] for v in (1,2)} - for name in (*expected, health)}) + for name in (*expected, health, comparison)}) def test_sdk_selects_v2_and_falls_back_to_exact_v1_without_falsifying_provenance(self): value=fork_result();raw=client({'status':200,'result':value});sdk=ProjectFilesClient(raw) diff --git a/tests/test_project_files_m4_clients.py b/tests/test_project_files_m4_clients.py index adf2cc56..90af3962 100644 --- a/tests/test_project_files_m4_clients.py +++ b/tests/test_project_files_m4_clients.py @@ -58,6 +58,20 @@ def ready(reply, *, owner=False): class M4Clients(unittest.TestCase): + def test_batch_reply_must_match_every_reviewed_file_and_changed_count(self): + text='Exact note\n';raw=text.encode();path='notes/first.md' + item=dict(id=identity('note',path),project_id=PROJECT,kind='note',path=path, + revision=1,sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw),adopted_at=STAMP,changed=True) + arguments=dict(WRITE,items=[dict(kind='note',path=path,expected_revision=0,expected_sha256=item['sha256'])]) + reply=dict(status=200,result=bound(dict(items=[item],changed_count=1),NEXT)) + self.assertEqual(ProjectFilesClient(ready(reply)).files_adopt(**arguments),reply['result']) + for field,value in [('project_id','f'*32),('sha256','a'*64),('path','notes/other.md'),('revision',2)]: + bad=copy.deepcopy(reply);bad['result']['value']['items'][0][field]=value + with self.subTest(field=field),self.assertRaises(api.AgentAPIError): + ProjectFilesClient(ready(bad)).files_adopt(**arguments) + bad=copy.deepcopy(reply);bad['result']['value']['changed_count']=0 + with self.assertRaises(api.AgentAPIError):ProjectFilesClient(ready(bad)).files_adopt(**arguments) + def test_capability_profiles_fit_unchanged_budget_and_negotiate_only_intersections(self): native = api.capabilities(CATALOG) project = api.capabilities(CATALOG, profile='project') @@ -70,10 +84,12 @@ def test_capability_profiles_fit_unchanged_budget_and_negotiate_only_intersectio with self.assertRaises(api.AgentAPIError): api.capabilities(CATALOG, profile='arbitrary') self.assertEqual(api.REGISTRY_DIGEST, api._LEGACY_DIGEST) - def test_ten_operations_have_fixed_consumers_and_closed_inputs(self): - self.assertEqual(len(api.PROJECT_FILES_M4), 10) + def test_complete_m4_operations_have_fixed_consumers_and_closed_inputs(self): + self.assertEqual(api.PROJECT_FILES_M4,{'project_files.'+name for name in ( + 'compose','save_as','files_inspect','file_get','file_history','file_adopt','files_adopt', + 'file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')}) self.assertEqual(project_dispatch.SUPPORTED_OPERATIONS & api.PROJECT_FILES_M4, - api.PROJECT_FILES_M4 - {'project_files.save_as'}) + api.PROJECT_FILES_M4 - {'project_files.save_as','project_files.compose'}) client = ProjectFilesClient(Mock()) parser = project_cli.parser() for operation in api.PROJECT_FILES_M4: @@ -215,13 +231,25 @@ def test_dispatch_projects_metadata_pages_without_captured_text_or_definition_bu import threading access=SimpleNamespace(lock=threading.RLock(),require=Mock()) session=SimpleNamespace(access=access,lock=threading.RLock(),db=SimpleNamespace(lock=threading.RLock()), - check=Mock(return_value={'project_id':PROJECT,'records':[]})) + check=Mock(side_effect=AssertionError('Unused full dispatcher snapshot')),check_current=Mock()) files=SimpleNamespace(history=Mock(return_value=[version('large text '+str(i),i+1) for i in range(42)])) projection=SimpleNamespace(session=session,files=files) page=project_dispatch.dispatch(projection,'project_files.file_history',dict(READ,file_id=identity(),offset=40,limit=2)) self.assertEqual((page['total'],[item['revision'] for item in page['items']]),(42,[41,42])) self.assertTrue(all('text' not in item for item in page['items'])) + session.check_current.assert_called_once_with() api.validate_output('project_files.file_history',dict(status=200,result=bound(page))) + def test_dispatch_fork_info_retains_the_complete_snapshot_contract(self): + import threading + access=SimpleNamespace(lock=threading.RLock(),require=Mock()) + session=SimpleNamespace(access=access,lock=threading.RLock(),db=SimpleNamespace(lock=threading.RLock()), + check=Mock(return_value={'project_id':PROJECT,'records':[]}), + check_current=Mock(side_effect=AssertionError('Fork must read its records'))) + value=project_dispatch.dispatch(SimpleNamespace(session=session),'project_files.fork_info',dict(READ,offset=0,limit=40)) + session.check.assert_called_once_with() + self.assertEqual(value,dict(source=None,project_id=PROJECT, + mapping=dict(items=[],offset=0,limit=40,total=0),archived_count=0,copied_count=0,missing_regenerable_count=0)) + if __name__=='__main__':unittest.main() diff --git a/tests/test_project_files_m4_http.py b/tests/test_project_files_m4_http.py index a520c416..8d6b0a6f 100644 --- a/tests/test_project_files_m4_http.py +++ b/tests/test_project_files_m4_http.py @@ -10,6 +10,7 @@ import project_workspace from project_store import ProjectStoreError +from workbench import WorkbenchError import test_project_files_http as m1 import test_project_files_m2_http as m2 @@ -39,6 +40,25 @@ def fork(self,**fields): self.assertEqual(status,200,value) return arguments,value + def test_batch_adoption_uses_real_http_and_survives_independent_fork(self): + self.create();root=self.external/'alpha';(root/'notes').mkdir();items=[] + for name,raw in [('first.md',b'Exact first note\r\n'),('second.md','Exact Ω note\n'.encode())]: + (root/'notes'/name).write_bytes(raw) + items.append(dict(kind='note',path='notes/'+name,expected_revision=0,expected_sha256=hashlib.sha256(raw).hexdigest())) + result=self.call('files_adopt',items=items) + self.assertEqual(result['changed_count'],2);self.assertEqual(len(self.call('files_inspect')['items']),2) + with patch.object(self.workspace._store,'_snapshot',side_effect=AssertionError('Unrelated full graph copy')): + self.assertEqual(len(self.call('files_inspect')['items']),2) + selected=self.call('file_get',file_id=result['items'][0]['id']) + self.assertEqual(selected['version']['text'].encode(),b'Exact first note\r\n') + history=self.call('file_history',file_id=result['items'][0]['id']) + self.assertEqual([v['revision'] for v in history['items']],[1]) + source=copy.deepcopy(self.workspace._projection.session.snapshot) + self.fork() + current=self.workspace._projection.session.snapshot;indexed={(r['family'],r['id']):r for r in current['records']} + for record in source['records']:self.assertEqual(indexed[(record['family'],record['id'])],record) + for item in items:self.assertEqual((self.external/'fork'/item['path']).read_bytes(),(root/item['path']).read_bytes()) + def test_nonempty_independent_fork_exact_science_new_editable_ids_and_same_request_key(self): self.create() row=self.call('data_import_text',text='Unique observation café 山',name='Original', @@ -57,6 +77,18 @@ def test_nonempty_independent_fork_exact_science_new_editable_ids_and_same_reque current=self.workspace._projection.session.snapshot indexed={(r['family'],r['id']):r for r in current['records']} for r in original['records']:self.assertEqual(indexed[(r['family'],r['id'])],r) + forged=copy.deepcopy(current) + inherited=next(r['document'] for r in forged['records'] if r['family']=='project_file' + and r['document']['project_id']==current['project_id']) + forged_version=inherited['history'][0];forged_text='Forged inherited script\n' + forged_version.update(text=forged_text,bytes=len(forged_text.encode()),sha256=hashlib.sha256(forged_text.encode()).hexdigest()) + inherited['current']=copy.deepcopy(inherited['history'][-1]) + from project_files import decode_records + with self.assertRaisesRegex(WorkbenchError,'Working-file fork requires its exact archived history'): + decode_records(forged['records']) + with self.assertRaisesRegex(WorkbenchError,'Working-file fork requires its exact archived history'): + self.workspace._projection.session.adopt(forged) + self.assertEqual(self.workspace._projection.session.snapshot,current) active=self.call('data_query')['items'] self.assertEqual(len(active),1);self.assertNotEqual(active[0]['id'],row['id']) self.assertEqual(self.call('data_get',record_id=active[0]['id'])['text'],row['text']) diff --git a/tests/test_project_fork.py b/tests/test_project_fork.py index 946f8b75..0683eef9 100644 --- a/tests/test_project_fork.py +++ b/tests/test_project_fork.py @@ -53,7 +53,7 @@ def setUp(self): capture.mkdir(parents=True) for name, raw in payloads.items(): (capture / name).write_bytes(raw) - session = SimpleNamespace(snapshot=self.source.snapshot(), artifacts=self.artifacts, check=lambda: None) + session = SimpleNamespace(snapshot=self.source.snapshot(), artifacts=self.artifacts, check_current=lambda: None) self.material = ProjectMaterial(session) self.record = self.material.capture_model_material(self.material_value, local) records = [*self.material.declarations(self.record), self.record] @@ -90,7 +90,7 @@ def test_complete_fork_retains_exact_original_material_hashes_and_copies_without self.assertEqual(overlay['source'], dict(project_id=self.origin, checkpoint=self.original['checkpoint'])) self.assertEqual(next(item for item in snapshot['records'] if item['family'] == 'model_material'), self.record) with ProjectStore.open(self.root / 'fork') as destination, ProjectArtifacts(destination) as artifacts: - reader = ProjectMaterial(SimpleNamespace(snapshot=destination.snapshot(), artifacts=artifacts, check=lambda: None)) + reader = ProjectMaterial(SimpleNamespace(snapshot=destination.snapshot(), artifacts=artifacts, check_current=lambda: None)) self.assertEqual(reader.model(self.material_value['ref'], verify=True), self.material_value) source_ref = self.material_value['material']['source_closure'][0]['blob'] self.assertEqual(reader.blob(self.material_value['ref'], source_ref), SOURCE) @@ -120,6 +120,59 @@ def test_exact_retry_reconciles_and_changed_body_or_unrelated_destination_never_ self.assert_code('conflict', self.fork, 'unrelated') self.assertEqual((other / '.tuldok/project.json').read_bytes(), unrelated) + def test_missing_runtime_inventory_does_not_repeat_full_source_closure_per_entry(self): + # Each snapshot validates the complete canonical file closure. Missing + # runtime declarations must not multiply that work by inventory length. + snapshot = self.source.snapshot + with patch.object(self.source, 'snapshot', wraps=snapshot) as scans: + self.fork('small-inventory') + baseline = scans.call_count + records = copy.deepcopy(self.original['records']) + runtime = set() + for index in range(32): + raw = ('unmaterialized runtime dependency %d' % index).encode() + sha = hashlib.sha256(raw).hexdigest() + runtime.add(sha) + records.append(dict(family='project_artifact', id=sha, + document=dict(format='project_artifact_v1', sha256=sha, bytes=len(raw)))) + self.original = self.source.commit(records, self.original['checkpoint'], + 'larger-runtime-inventory', body(records)) + def candidate(source, destination): + value = self.candidate(source, destination) + for edge in value['artifacts']: + if edge['declaration']['sha256'] in runtime: + edge['preservation'] = 'regenerable' + return value + with patch.object(self.source, 'snapshot', wraps=snapshot) as scans: + result = self.fork('large-inventory', build_candidate=candidate) + self.assertEqual(set(result['fork']['artifacts']['missing_regenerable']), runtime) + self.assertLessEqual(scans.call_count, baseline + 4, + 'Full source closure scans must stay bounded when missing runtime inventory grows') + + def test_present_payload_copy_does_not_repeat_full_source_closure_per_file(self): + snapshot = self.source.snapshot + with patch.object(self.source, 'snapshot', wraps=snapshot) as scans: + self.fork('small-payload-set') + baseline = scans.call_count + records = copy.deepcopy(self.original['records']) + payloads = {} + for index in range(8): + raw = ('retained unique output %d' % index).encode() + sha = hashlib.sha256(raw).hexdigest() + value = dict(format='project_artifact_v1', sha256=sha, bytes=len(raw)) + self.artifacts.publish_bytes(raw, value) + records.append(dict(family='project_artifact', id=sha, document=value)) + payloads[sha] = raw + self.original = self.source.commit(records, self.original['checkpoint'], + 'larger-payload-set', body(records)) + with patch.object(self.source, 'snapshot', wraps=snapshot) as scans: + result = self.fork('large-payload-set') + for sha, raw in payloads.items(): + self.assertIn(sha, result['fork']['artifacts']['copied']) + self.assertEqual((self.root / 'large-payload-set/artifacts/sha256' / sha).read_bytes(), raw) + self.assertLessEqual(scans.call_count, baseline + 4, + 'Full source closure scans must stay bounded when present payload count grows') + def test_quiescence_checkpoint_overlap_and_symlink_guards_precede_publication(self): before = self.canonical_bytes() self.assert_code('busy', self.fork, quiescent=False) @@ -318,5 +371,28 @@ def test_current_overlay_filters_receipts_and_original_editables_without_rewriti self.assert_code('invalid', decode_fork, bad) +class ActiveRecordSelectionTests(unittest.TestCase): + def test_family_selection_keeps_independent_values_and_default_membership(self): + records=[dict(family=family,id='a'*32,document=dict(format='fixture_v1',values=[1])) + for family in ('project_operation','workbench_text')] + self.assertEqual(active_records(records),records) + selected=active_records(records,families=frozenset({'project_operation'})) + self.assertEqual(selected,[records[0]]) + selected[0]['document']['values'].append(2) + self.assertEqual(records[0]['document']['values'],[1]) + self.assertEqual(active_records(records,families=frozenset()),[]) + + def test_family_selection_still_validates_excluded_fork_structure(self): + record=dict(family='workbench_text',id='a'*32,document={'format':'workbench_text_v1'}) + with self.assertRaisesRegex(ProjectForkError,'Duplicate'): + active_records([record,copy.deepcopy(record)],families=frozenset({'project_operation'})) + + def test_family_selection_rejects_implicit_or_invalid_selectors(self): + for invalid in ('project_operation',[],frozenset({1}),frozenset({'bad/family'})): + with self.subTest(selector=invalid),self.assertRaises(ProjectStoreError) as caught: + active_records([],families=invalid) + self.assertEqual(caught.exception.code,'invalid') + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_project_fork_composition.py b/tests/test_project_fork_composition.py index b5fcf831..28e48361 100644 --- a/tests/test_project_fork_composition.py +++ b/tests/test_project_fork_composition.py @@ -232,7 +232,7 @@ def setUp(self): def validate(self,records): decode_data_records(records);material_records(records);file_records(records);producer_records(records) - history_records(records,materials=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check=lambda:None))) + history_records(records,materials=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None))) for receipt in operation_documents(records): operation,result=receipt['operation'],receipt['result'] if not (data_result(operation,result,records) or file_result(operation,result,records) diff --git a/tests/test_project_history.py b/tests/test_project_history.py index c505f663..c42dd135 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -60,6 +60,8 @@ def trial(identity,study_id,origin=ORIGIN,parent=None,baseline=None): class Materials: """Only material lookup is substituted; history/lineage validators are real.""" + def read_view(self, records): + return self def recipe(self,ref,*,verify=False): if ref!={'kind':'recipe_material','catalog_id':ORIGIN,'id':MATERIAL,'sha256':MATERIAL}: raise WorkbenchError('Unknown exact fixture material.','unavailable',409) @@ -83,6 +85,47 @@ def legacy_run(origin=ORIGIN): class HistoryCodecTests(unittest.TestCase): + def test_progression_reuses_one_decoded_checkpoint_and_keeps_returned_values_independent(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + checks=[] + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER,checkpoint='c'*64), + summary=dict(project_id=OTHER,checkpoint='c'*64),artifacts=None, + access=SimpleNamespace(require=lambda *args,**kwargs:None), + check_current=lambda:checks.append('actual current-state guard')) + history=ProjectHistory(session,materials=Materials()) + with patch('project_history.decode_records',wraps=decode_records) as decoder: + first=history.progression(ORIGIN,offset=0,limit=20) + first['nodes'][0]['study']['name']='Caller mutation' + first['nodes'][0]['trial']['replicate']=99 + second=history.progression(ORIGIN,offset=0,limit=20) + self.assertEqual(decoder.call_count,1) + self.assertEqual(checks,['actual current-state guard']*2) + self.assertEqual(second['nodes'][0]['study']['name'],'Exact intent λ') + self.assertEqual(second['nodes'][0]['trial']['replicate'],0) + + def test_history_cache_refreshes_on_checkpoint_change_and_does_not_cache_decode_failure(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), + summary=dict(project_id=OTHER,checkpoint='c'*64),artifacts=None, + access=SimpleNamespace(require=lambda *args,**kwargs:None),check_current=lambda:None) + history=ProjectHistory(session,materials=Materials()) + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'Exact intent λ') + updated=copy.deepcopy(records) + for value in (updated[0]['document']['study'],updated[0]['document']['revisions'][0]): + value['name']='New checkpoint name' + session.snapshot=dict(records=updated,project_id=OTHER) + session.summary=dict(project_id=OTHER,checkpoint='d'*64) + with patch('project_history.decode_records',side_effect=WorkbenchError('Decode unavailable','unavailable',503)): + with self.assertRaisesRegex(WorkbenchError,'Decode unavailable') as failed: + history.progression(ORIGIN) + self.assertEqual(failed.exception.code,'unavailable') + with patch('project_history.decode_records',wraps=decode_records) as decoder: + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'New checkpoint name') + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'New checkpoint name') + self.assertEqual(decoder.call_count,1) + def test_fork_preserves_archived_disclosure_observation_without_current_permission(self): from project_fork import _digest,_input from project_session import operation_records @@ -124,6 +167,24 @@ def test_cross_study_parent_baseline_and_origin_namespace_remain_exact(self): changed=copy.deepcopy(records);changed[-1]['document']['lineage']['intent']['baseline']['catalog_id']=OTHER with self.assertRaises(WorkbenchError):decode_records(changed,materials=Materials()) + def test_history_read_uses_one_material_view_of_the_selected_records(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + observed=[] + class Provider: + def read_view(self, selected): + observed.append(selected) + return Materials() + def recipe(self, *args, **kwargs): + raise AssertionError('History reread the live material catalog') + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), + summary=dict(project_id=OTHER,checkpoint='c'*64), + artifacts=None,access=SimpleNamespace(),check_current=lambda:None) + facts=ProjectHistory(session,materials=Provider())._facts() + self.assertEqual(len(observed),1) + self.assertIs(observed[0],records) + self.assertEqual(facts.trials[(ORIGIN,identity)]['trial'],records[1]['document']['trial']) + def test_closure_omissions_cycles_and_history_completeness_are_rejected(self): parent_id='9'*32;child_id='a'*32 ref=lambda identity:{'catalog_id':ORIGIN,'trial_id':identity,'material_sha256':MATERIAL} @@ -293,6 +354,32 @@ def start(self): self.host,self.owner,self.session=host,owner,session self.history=ProjectHistory(session,materials=Materials()) + def test_cached_history_still_reads_actual_bytes_and_rejects_restored_mtime_corruption(self): + from project_store import ProjectStoreError + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + before=self.store.snapshot() + snapshot=self.store.commit(records,before['checkpoint'],'cache-corruption-fixture',digest(records)) + self.session.adopt(snapshot) + with self.owner.bound(self.host): + expected=self.history.progression(ORIGIN) + header=json.loads((self.store.path/'.tuldok/project.json').read_bytes()) + entry=next(item for item in header['inventory'] if item['family']=='study_history') + path=self.store.path/'.tuldok'/entry['path'] + original=path.read_bytes();info=path.stat() + corrupt=original.replace(b'Exact intent',b'Wrong intent',1) + self.assertNotEqual(corrupt,original);self.assertEqual(len(corrupt),len(original)) + try: + path.write_bytes(corrupt) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + with self.assertRaisesRegex(ProjectStoreError,'Canonical record bytes changed or are corrupt') as failed: + self.history.progression(ORIGIN) + self.assertEqual((failed.exception.code,failed.exception.status),('stale',409)) + finally: + path.write_bytes(original) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + self.assertEqual(self.history.progression(ORIGIN),expected) + def test_progression_uses_original_namespaces_exact_edges_and_bounded_pages(self): parent='a'*32;child='b'*32 ref={'catalog_id':ORIGIN,'trial_id':parent,'material_sha256':MATERIAL} @@ -370,6 +457,7 @@ def test_metadata_edits_cold_reopen_keep_history_without_active_authority(self): def test_trial_and_cross_study_lineage_checkpoint_preserve_previous_revision(self): class OwnedMaterial: def __init__(self,origin):self.origin=origin + def read_view(self,records):return self def recipe(self,ref,*,verify=False): self.ref=ref if ref!={'kind':'recipe_material','catalog_id':self.origin,'id':MATERIAL,'sha256':MATERIAL}: @@ -633,7 +721,7 @@ def test_cold_actual_composed_proof_keeps_origin_without_live_authority(self): root=self.path/'portable-history';store=ProjectStore.create(root,'Actual cold history','new','f'*64) self.addCleanup(store.close) artifacts=ProjectArtifacts(store);self.addCleanup(artifacts.close) - view=SimpleNamespace(snapshot={'project_id':store.snapshot()['project_id'],'records':[]},artifacts=artifacts,check=lambda:None) + view=SimpleNamespace(snapshot={'project_id':store.snapshot()['project_id'],'records':[]},artifacts=artifacts,check_current=lambda:None) material=ProjectMaterial(view) model_record=material.capture_model_material(model,self.models.path) recipe_record=material.capture_recipe_material(composition,self.dataset.pipelines.material_path) diff --git a/tests/test_project_material.py b/tests/test_project_material.py index 8893d6ae..9d69056c 100644 --- a/tests/test_project_material.py +++ b/tests/test_project_material.py @@ -20,7 +20,7 @@ from model_definitions import ModelDefinitions, blob, decode_model_draft, decode_model_material, verify_model_closure from pipeline import digest, decode_recipe_material, execution_digest, verify_recipe_closure from project_artifacts import ProjectArtifacts, ProjectArtifactError, validate_declaration -from project_material import ProjectMaterial, decode_records, validate_operation_result +from project_material import ProjectMaterial, MaterialSnapshot, decode_records, validate_operation_result from project_store import ProjectStore from project_session import ProjectSession, operation_records from workbench import WorkbenchError @@ -55,7 +55,8 @@ def setUp(self): self.addCleanup(lambda:self.store.close()) self.artifacts=ProjectArtifacts(self.store);self.addCleanup(lambda:self.artifacts.close()) self.catalog=self.store.snapshot()['project_id'] - self.session=SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,check=lambda:None) + self.session=SimpleNamespace(snapshot=self.store.snapshot(),summary=self.store.summary(), + artifacts=self.artifacts,check_current=lambda:None) self.material=ProjectMaterial(self.session) def capture(self): @@ -87,7 +88,7 @@ def test_cold_relocation_uses_canonical_artifacts_after_local_capture_deleted(se self.artifacts.close();self.store.close();shutil.rmtree(root) moved=self.root/'moved';(self.root/'project').rename(moved) with ProjectStore.open(moved) as store,ProjectArtifacts(store) as artifacts: - material=ProjectMaterial(SimpleNamespace(snapshot=store.snapshot(),artifacts=artifacts,check=lambda:None)) + material=ProjectMaterial(SimpleNamespace(snapshot=store.snapshot(),artifacts=artifacts,check_current=lambda:None)) self.assertEqual(store.snapshot(),snapshot) self.assertEqual(material.model(value['ref'],verify=True),value) source=value['material']['source_closure'][0]['blob'] @@ -123,6 +124,45 @@ def test_runtime_environment_is_historical_and_does_not_probe_origin_paths(self) with patch('subprocess.Popen',side_effect=AssertionError('no process')),patch('os.stat',side_effect=AssertionError('no host paths')): self.assertEqual(self.material.runtime(ref,verify=True)['environment'],environment) + def test_fixed_material_snapshot_owns_input_and_returns_independent_values(self): + environment={'python':'/unavailable/original/python','packages':{'neutral':'1.0'},'definition_files':{'adapter.py':'a'*64}} + sha=digest(environment);ref=dict(kind='runtime',catalog_id=self.catalog,id=sha,sha256=sha) + original=dict(ref=ref,descriptor=None,environment=environment,manifest=None) + record=self.material.capture_runtime(original) + records=[record];view=MaterialSnapshot(records) + records[0]['document']['value']['environment']['packages']['neutral']='changed input' + value=view.runtime(ref);self.assertEqual(value,original) + value['environment']['packages']['neutral']='changed return' + self.assertEqual(view.runtime(ref),original) + with self.assertRaises(WorkbenchError):MaterialSnapshot(records) + + def test_fixed_material_snapshot_checks_complete_reference_identity(self): + environment={'python':'/unavailable/original/python','packages':{},'definition_files':{}} + sha=digest(environment);ref=dict(kind='runtime',catalog_id=self.catalog,id=sha,sha256=sha) + record=self.material.capture_runtime(dict(ref=ref,descriptor=None,environment=environment,manifest=None)) + view=MaterialSnapshot([record]) + for change in ({'catalog_id':'f'*32},{'sha256':'f'*64},{'id':'f'*64},{'kind':'recipe_material'}): + with self.assertRaises(WorkbenchError):view.runtime(dict(ref,**change)) + + def test_invalid_live_material_reference_precedes_snapshot_access(self): + with patch.object(self.session,'check_current',side_effect=AssertionError('Invalid reference accessed snapshot')): + with self.assertRaises(WorkbenchError):self.material.runtime({'catalog_id':'invalid'}) + + def test_material_read_view_uses_supplied_records_without_resnapshotting(self): + environment={'python':'/unavailable/original/python','packages':{},'definition_files':{}} + sha=digest(environment);ref=dict(kind='runtime',catalog_id=self.catalog,id=sha,sha256=sha) + original=dict(ref=ref,descriptor=None,environment=environment,manifest=None) + records=[self.material.capture_runtime(original)] + self.assertEqual(self.session.snapshot['records'],[]) + view=self.material.read_view(records) + self.assertEqual(view.runtime(ref),original) + self.assertEqual(self.session.snapshot['records'],[]) + + def test_material_read_view_rechecks_session_availability(self): + with patch.object(self.session,'check_current',side_effect=WorkbenchError('Stale checkpoint.','stale',409)): + with self.assertRaisesRegex(WorkbenchError,'Stale checkpoint'): + self.material.read_view([]) + def test_operation_result_requires_exact_retained_revision_or_material(self): first=definition(self.catalog);record=self.material.capture_definition('model_definition',first,[],action='models.create') changed=copy.deepcopy(first);changed.update(revision=2,name='Changed') @@ -185,6 +225,27 @@ def supplied(descriptor):seen.append(descriptor);return raw self.assertEqual(restored[-1],record) self.assertEqual(self.material.runtime(ref,verify=True),value) + def test_legacy_activity_path_is_retained_without_becoming_a_current_manifest(self): + from projects import manifest as current_manifest, decode_retained_snapshot + manifest = dict(schema_version=1, id='c'*32, name='Archived activity', + data=[], recipes=[], evidence=[], requirements=[], activity_path='old/activity.json') + raw = (json.dumps(manifest, indent=3)+'\n\n').encode() + saved = dict(id=manifest['id'], name=manifest['name'], root='gone', root_identity=[1,2], + manifest_path='original.json', manifest_sha256=hashlib.sha256(raw).hexdigest(), manifest=manifest) + with self.assertRaises(WorkbenchError): current_manifest(manifest) + with patch('subprocess.Popen', side_effect=AssertionError('Retained activity is inert')): + self.assertEqual(decode_retained_snapshot(saved),saved) + record=self.material.capture_snapshot(saved,raw,catalog_id=self.catalog) + self.session.snapshot=dict(self.session.snapshot,records=[*self.material.declarations(record),record]) + self.assertEqual(self.material.snapshot_ref(manifest['id'],saved['manifest_sha256'],verify=True),saved) + documents=decode_records(self.session.snapshot['records']) + self.assertEqual(documents[('project_snapshot',self.catalog+'.'+saved['manifest_sha256'])]['value'],saved) + for bad_path in ('../escape.json','/absolute.json','a\\b.json',False): + bad=copy.deepcopy(saved);bad['manifest']['activity_path']=bad_path + with self.assertRaises(WorkbenchError): decode_retained_snapshot(bad) + bad=copy.deepcopy(saved);bad['manifest']['unknown_activity_field']='anything' + with self.assertRaises(WorkbenchError): decode_retained_snapshot(bad) + def test_original_snapshot_manifest_whitespace_is_not_reserialized_for_identity(self): manifest=dict(schema_version=1,id='c'*32,name='Snapshot',data=[],recipes=[],evidence=[],requirements=[]) import json @@ -238,6 +299,15 @@ def read(ref,item,maximum):return (payloads if ref['kind']=='model' else recipe_ dependencies=self.material.capture_recipe_dependencies(dict(document={'value':retained}),[]) self.assertEqual(dependencies[-1]['document']['value']['environment'],runtime) self.assertEqual(self.material.capture_recipe_dependencies(dict(document={'value':retained}),dependencies),[]) + recipe_root=self.root/'retained-recipe';target=recipe_root/retained['ref']['sha256'];target.mkdir(parents=True) + for name,raw in recipe_payloads.items():(target/name).write_bytes(raw) + record=self.material.capture_recipe_material(retained,recipe_root) + view=MaterialSnapshot([record,*dependencies]) + self.assertEqual(view.recipe(retained['ref']),retained) + expected=[edge['blob'] for item in [record,*dependencies] for edge in item['document']['artifacts']] + self.assertEqual(list(view.artifact_blobs()),expected) + changed=list(view.artifact_blobs());changed[0]['bytes']+=1 + self.assertEqual(list(view.artifact_blobs()),expected) for field,replacement in (('data',{}),('executable_sha256','wrong'),('captures',[dict(item,media_type='text/plain')])): bad=copy.deepcopy(retained);bad['material'][field]=replacement changed=digest(bad['material']);bad['ref'].update(id=changed,sha256=changed) @@ -291,7 +361,7 @@ def test_retained_one_fit_readers_match_directory_without_source_paths_or_infere patch('subprocess.Popen',side_effect=AssertionError('No process')): expected=evidence.verify(root,run,metrics) self.assertEqual(evidence.verify_blobs(run,metrics,read_blob=lambda name,maximum:payloads[name]),expected) - material=ProjectMaterial(SimpleNamespace(check=lambda:None)) + material=ProjectMaterial(SimpleNamespace(check_current=lambda:None)) result=material.evidence(run,read_blob=lambda name,maximum:payloads[name]) self.assertEqual(result['state'],'verified') checkpoint=result['checkpoint_index']['entries'][0]['checkpoint']['sha256'] @@ -380,7 +450,7 @@ def test_real_deferred_sql_failure_cannot_delete_canonical_source_material(self) self.assertEqual(error.exception.code,'projection_unavailable') snapshot=self.store.snapshot();retained=next(item for item in snapshot['records'] if item['family']=='model_material')['document']['value'] self.assertFalse((self.models.path/retained['ref']['sha256']).exists(),'Legacy rollback removes only disposable projection cache') - cold=ProjectMaterial(SimpleNamespace(snapshot=snapshot,artifacts=self.artifacts,check=lambda:None)) + cold=ProjectMaterial(SimpleNamespace(snapshot=snapshot,artifacts=self.artifacts,check_current=lambda:None)) self.assertEqual(cold.model(retained['ref'],verify=True),retained) self.assertEqual(cold.blob(retained['ref'],retained['material']['source_closure'][0]['blob']),SOURCE) @@ -391,7 +461,7 @@ def test_restore_is_inert_and_explicit_preparation_rebuilds_only_projection_cach draft=self.create();value=self.material.publish_model(self.models,draft['id'],{'revision':1},self.session.snapshot['checkpoint'],'cold-model') cold_db=Metadata(self.path/'cold-projection',dataset_id=self.db.dataset_id,projection_id=uuid.uuid4().hex) self.resources.append(cold_db);self.schemas.append(cold_db.schema) - cold=ProjectMaterial(SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,db=cold_db,check=lambda:None)) + cold=ProjectMaterial(SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,db=cold_db,check_current=lambda:None)) with patch('model_definitions.ModelDefinitions',side_effect=AssertionError('No mutable owner on Open')), \ patch('pipeline.Pipelines',side_effect=AssertionError('No mutable owner on Open')), \ patch('subprocess.Popen',side_effect=AssertionError('No process')): diff --git a/tests/test_project_material_fork.py b/tests/test_project_material_fork.py index 89681263..fb27a597 100644 --- a/tests/test_project_material_fork.py +++ b/tests/test_project_material_fork.py @@ -52,7 +52,8 @@ def overlay(records, made, source, checkpoint, destination): class MaterialForkCodecs(unittest.TestCase): def setUp(self): self.source=uuid.uuid4().hex;self.checkpoint='a'*64 - self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]),check=lambda:None) + self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]), + summary=dict(project_id=self.source,checkpoint=self.checkpoint),check_current=lambda:None) self.material=ProjectMaterial(self.session) def capture(self,family,current): @@ -108,7 +109,8 @@ def test_bound_drafts_preserve_full_configuration_as_visible_nonexecutable_inten ('pipeline_graph',graph(external=True)),('recipe_definition',recipe(self.source))] captured=[self.capture(f,v) for f,v in drafts] made,combined,destination=self.fork() - session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined),check=lambda:None) + session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined), + summary=dict(project_id=destination),check_current=lambda:None) reader=ProjectMaterial(session) intents=reader.intents() self.assertEqual(len(intents),4) @@ -167,7 +169,8 @@ def test_captured_source_selector_stays_in_inert_intent_and_cannot_publish(self) doc=made['records'][0]['document'] self.assertEqual(doc['current']['required_reselection'],['source_selector']) self.assertEqual(doc['current']['definition']['source_selector'],value['source_selector']) - reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined),check=lambda:None)) + reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined), + summary=dict(project_id=destination),check_current=lambda:None)) with self.assertRaises(WorkbenchError) as caught: reader.publish_model(SimpleNamespace(publish=lambda *args: self.fail('Intent reached owner publication')), doc['current']['id'],{'revision':1},'a'*64,'forbidden-intent') diff --git a/tests/test_project_measurement_comparison_http.py b/tests/test_project_measurement_comparison_http.py new file mode 100644 index 00000000..df5f9667 --- /dev/null +++ b/tests/test_project_measurement_comparison_http.py @@ -0,0 +1,73 @@ +"""Real native HTTP source import, atomic comparison and scalar inspection.""" +import hashlib +import json +import os +import unittest +from unittest.mock import patch + +import agent_api +from workbench import WorkbenchError + +import test_project_files_http as m1 +import test_project_files_m2_http as m2 +from retained_measurements import encode +from test_measurement_comparison import partition + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class ComparisonHTTP(unittest.TestCase): + cleanup_owned=m1.ProjectFilesHTTP.cleanup_owned + select_host=m1.ProjectFilesHTTP.select_host + request=m1.ProjectFilesHTTP.request + state=m1.ProjectFilesHTTP.state + envelope=m1.ProjectFilesHTTP.envelope + action=m1.ProjectFilesHTTP.action + create=m1.ProjectFilesHTTP.create + arguments=m2.ProjectFilesM2HTTP.arguments + transport=m2.ProjectFilesM2HTTP.transport + call=m2.ProjectFilesM2HTTP.call + def setUp(self): + self.operations=[] + m1.ProjectFilesHTTP.setUp(self) + + def test_two_original_sources_compare_and_inspect_through_real_HTTP(self): + self.create() + sources=[] + for subject, identity in [('a','b'*32),('b','c'*32)]: + raw=encode(partition(subject));digest=hashlib.sha256(raw).hexdigest() + filename=subject+'-measurements.json';manifest_name=subject+'-source.json' + (self.external/filename).write_bytes(raw) + manifest=dict(schema_version=1,id=identity,name='Original '+subject, + data=[dict(id='scalars',label='Retained scalars',path=filename,sha256=digest)], + recipes=[],evidence=[],requirements=[]) + (self.external/manifest_name).write_text(json.dumps(manifest)) + registered=self.call('source_register',root='fixture',manifest_path=manifest_name) + snapshot={key:registered[key] for key in ('origin_catalog_id','project_id','manifest_sha256')} + admitted=self.call('measurement_import',snapshot=snapshot,reference_id='scalars',expected_sha256=digest) + sources.append(dict(origin_catalog_id=admitted['origin_catalog_id'],packet_sha256=digest)) + before=self.workspace._projection.session.snapshot + metadata=self.call('measurement_compare',sources=sources)['value'] + self.assertEqual([s['id'] for s in metadata['subjects']],['a','b']) + self.assertIsNone(metadata['query']) + query=self.call('measurement_compare',sources=sources,metric_id='error',offset=1,limit=2)['value']['query'] + self.assertEqual(query['total'],4) + self.assertEqual([s['maximum']['value'] for s in query['summary']],[2e-15,.02]) + self.assertEqual([r['subject_id'] for r in query['rows']],['a','b']) + maximum=query['summary'][1] + inspected=self.call('measurement_inspect',**maximum['source'],observation_id=maximum['maximum']['id'])['value'] + self.assertEqual(inspected['scalar_evidence']['value'],.02) + self.assertEqual(inspected['case']['dimensions']['cpm'],8) + missing=self.call('measurement_compare',sources=sources,metric_id='latency')['value']['query'] + self.assertTrue(all(s['maximum'] is None for s in missing['summary'])) + self.assertEqual(self.workspace._projection.session.snapshot,before) + # Exercise version-specific admission at the real workspace/SQL owner + # with small declared fixture budgets; no partial comparison escapes. + arguments=self.arguments('measurement_compare',sources=sources) + with self.owner.bound(self.workspace),patch.object(agent_api,'measurement_comparison_row_limit', + side_effect=lambda version:7 if version==1 else 8): + with self.assertRaises(WorkbenchError) as error: + self.workspace.dispatch('project_files.measurement_compare',arguments,selected_version=1) + self.assertEqual(error.exception.code,'too_large') + value=self.workspace.dispatch('project_files.measurement_compare',arguments,selected_version=2) + self.assertEqual(value['value']['value']['observation_count'],8) + self.assertEqual(self.workspace._projection.session.snapshot,before) diff --git a/tests/test_project_measurement_reads.py b/tests/test_project_measurement_reads.py new file mode 100644 index 00000000..1365d63a --- /dev/null +++ b/tests/test_project_measurement_reads.py @@ -0,0 +1,106 @@ +"""Read the adopted measurement family without cloning unrelated retained facts. + +Real canonical files and real session freshness guards; no SQL reconstruction +or packet import is claimed by this focused fixture. +""" +import copy +import json +import os +from pathlib import Path +import tempfile +import threading +import unittest +from unittest.mock import patch + +import project_measurements +import project_session +from project_projection import validate_project_records +from project_store import ProjectStore, ProjectStoreError +from test_project_receipt_validation import candidate, PROJECT + + +class MeasurementReadTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-measurement-read-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name)/'project' + self.store = ProjectStore.create(self.root, 'Measurement read fixture', 'new', 'a'*64) + self.addCleanup(self.store.close) + self.records = candidate() + self.records.append(dict(family='project_artifact', id='f'*64, + document=dict(format='project_artifact_v1', sha256='f'*64, bytes=7))) + validate_project_records(self.records, project_id=PROJECT, artifacts=None) + snapshot = self.store.commit(self.records, self.store.summary()['checkpoint'], 'seed', 'b'*64) + self.session = object.__new__(project_session.ProjectSession) + self.session.store, self.session.available = self.store, True + self.session._snapshot, self.session.lock = copy.deepcopy(snapshot), threading.RLock() + self.owner = object.__new__(project_measurements.ProjectMeasurements) + self.owner.session = self.session + self.expected = {r['id']: copy.deepcopy(r['document']) for r in self.records + if r['family'] == project_measurements.FAMILY} + + def test_documents_are_literal_independent_and_do_not_copy_unrelated_graph(self): + original = project_session.copy_validated_json + unrelated = [] + def observe(value): + if isinstance(value, dict) and 'records' in value: + unrelated.extend(r['id'] for r in value['records'] if r['id'] == 'f'*64) + return original(value) + with patch.object(project_session, 'copy_validated_json', observe): + result = self.owner._documents() + self.assertEqual(result, self.expected) + self.assertEqual(unrelated, [], 'Measurement read copied an unrelated retained envelope') + next(iter(result.values()))['source']['path'] = 'caller-edited.json' + self.assertEqual(self.owner._documents(), self.expected) + self.assertEqual(self.session._snapshot, self.store.snapshot()) + + def test_lookup_reads_every_canonical_file_without_redecoding_adopted_material(self): + with patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ + patch.object(project_measurements, 'material_records', + side_effect=AssertionError('Read-only lookup redecoded adopted material')): + self.assertEqual(self.owner._documents(), self.expected) + self.assertEqual(reads.call_count, len(self.records), 'All current files must still be read and hashed') + + def test_unavailable_projection_fails_before_any_store_read(self): + self.session.available = False + with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')): + with self.assertRaises(project_session.ProjectSessionError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'projection_unavailable') + + def test_other_family_publication_requires_reopen(self): + changed = copy.deepcopy(self.records) + changed[-1]['document']['bytes'] = 8 + self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c'*64) + with self.assertRaises(project_session.ProjectSessionError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'stale_project') + + def test_same_length_unselected_corruption_with_restored_mtime_is_rejected(self): + inventory = json.loads((self.root/'.tuldok/project.json').read_bytes())['inventory'] + entry = next(r for r in inventory if r['id'] == 'f'*64) + path = self.root/'.tuldok'/entry['path'] + before, info = path.read_bytes(), path.stat() + changed = before.replace(b'"bytes": 7', b'"bytes": 8') + self.assertNotEqual(changed, before) + self.assertEqual(len(changed), len(before)) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + with self.assertRaises(ProjectStoreError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'stale') + + def test_family_can_be_empty_and_returned_snapshot_contract_stays_complete(self): + records = [r for r in self.records if r['family'] not in (project_measurements.FAMILY,'project_operation')] + validate_project_records(records, project_id=PROJECT, artifacts=None) + self.session._snapshot = self.store.commit(records, self.store.summary()['checkpoint'], 'empty', 'd'*64) + self.assertEqual(self.owner._documents(), {}) + snapshot = self.session.snapshot + self.assertEqual(snapshot, self.store.snapshot()) + snapshot['records'].clear() + self.assertEqual(self.session.snapshot, self.store.snapshot()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_measurements_clients.py b/tests/test_project_measurements_clients.py index bc415313..079f6ab2 100644 --- a/tests/test_project_measurements_clients.py +++ b/tests/test_project_measurements_clients.py @@ -4,7 +4,7 @@ from unittest.mock import patch import agent_api as api -from retained_measurements import MeasurementTable, encode +from retained_measurements import MeasurementTable, MeasurementComparison, encode from test_project_files_clients import BINDING, PROJECT, CHECKPOINT, OTHER, bound, client from test_retained_measurements import fixture from tuldok_client import ProjectFilesClient @@ -24,6 +24,33 @@ def sdk(self,value): return ProjectFilesClient(client({'status':200,'result':dict(binding=BINDING,project_id=PROJECT, checkpoint=CHECKPOINT,value=value)})) + def test_comparison_capacity_is_versioned_without_changing_packet_headers(self): + from retained_measurements import MAX_COMPARISON_ROWS,MAX_ROWS + self.assertEqual(api.measurement_comparison_row_limit(1),100000) + self.assertEqual(api.measurement_comparison_row_limit(2),250000) + self.assertEqual(api.measurement_comparison_row_limit(2),MAX_COMPARISON_ROWS) + self.assertEqual(MAX_ROWS,100000) + self.assertEqual(api._MEASUREMENT_HEADER['properties']['observation_count']['maximum'],100000) + self.assertNotEqual(api.contract('project_files.measurement_compare',1)['sha256'], + api.contract('project_files.measurement_compare',2)['sha256']) + + def test_group_comparison_header_and_query_correlate_every_original_source(self): + locator=dict(origin_catalog_id=PROJECT,packet_sha256=self.table.sha256) + comparison=MeasurementComparison([(locator,self.table)]) + args=dict(binding=BINDING,checkpoint=CHECKPOINT,sources=[locator]) + page=dict(sources=[self.source],value=comparison.read()) + self.assertEqual(self.sdk(page).measurement_compare(**args)['value'],page) + query=dict(sources=[self.source],value=comparison.read('error',offset=1,limit=1)) + request=dict(args,metric_id='error',offset=1,limit=1) + self.assertEqual(self.sdk(query).measurement_compare(**request)['value'],query) + for mutate in (lambda p:p['sources'][0].update(id=OTHER), + lambda p:p['value']['query'].update(offset=0), + lambda p:p['value']['query']['summary'][0]['states'].update(measured=99), + lambda p:p['value']['query']['rows'][0]['source'].update(packet_sha256=OTHER), + lambda p:p['value']['subjects'][0]['source'].update(origin_catalog_id='f'*32)): + bad=copy.deepcopy(query);mutate(bad) + with self.assertRaises(api.AgentAPIError):self.sdk(bad).measurement_compare(**request) + def test_capability_bound_applies_to_the_actual_advertised_profile(self): sizes={profile:len(api.capabilities(PROJECT,profile=profile)['operations']) for profile in ('catalog','project')} maximum=max(sizes.values()) diff --git a/tests/test_project_model_portfolio.cjs b/tests/test_project_model_portfolio.cjs new file mode 100644 index 00000000..704e4035 --- /dev/null +++ b/tests/test_project_model_portfolio.cjs @@ -0,0 +1,47 @@ +'use strict'; +const assert=require('node:assert/strict'),fs=require('node:fs'),vm=require('node:vm'),path=require('node:path'); +const fixture=require('./model_portfolio_fixture.cjs'),window={}; +vm.runInNewContext(fs.readFileSync(path.join(__dirname,'../static/project-model-portfolio.js'),'utf8'),{window,TextEncoder}); +const {decode,checkCard}=window.ProjectModelPortfolioDocument; +const valid=()=>structuredClone(fixture.document),parse=value=>decode(JSON.stringify(value)); +assert.deepEqual(JSON.parse(JSON.stringify(parse(valid()))),fixture.document); +let checks=1; +function bad(change,message,code='invalid'){ + const value=valid();change(value); + assert.throws(()=>parse(value),error=>error.code===code&&error.message===message);checks++; +} +bad(d=>{d.format='other';},'Unsupported model-organization document format.','unsupported'); +bad(d=>{d.extra=true;},'Invalid model-organization document.'); +bad(d=>{d.models=[];},'Invalid model-organization document.'); +bad(d=>{d.groups.push(structuredClone(d.groups[0]));},'Invalid or duplicate organization group.'); +bad(d=>{d.groups[0].parent='missing';},'Organization group parent is unavailable.'); +bad(d=>{d.groups[0].parent='pressure';},'Organization grouping contains a cycle.'); +bad(d=>{d.models.push(structuredClone(d.models[0]));},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].group_id='missing';},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].aliases.push('C107');},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].review='qualified';},'Invalid or duplicate organized model.'); +bad(d=>{delete d.models[0].summary.method;},'Reviewed model requires all six authored summary fields.'); +bad(d=>{d.models[1].summary={};},'Pending review cannot expose a reviewed summary.'); +bad(d=>{d.models[0].card.path='notes/../private';},'Invalid captured model-note reference.'); +bad(d=>{d.models[0].card.revision=0;},'Invalid captured model-note reference.'); +bad(d=>{d.relationships[0].to='unknown';},'Invalid authored model relationship.'); +bad(d=>{d.relationships[0].to=d.relationships[0].from;},'Invalid authored model relationship.'); +bad(d=>{d.relationships[0].kind='trial_parent';},'Invalid authored model relationship.'); +bad(d=>{d.relationships.push(structuredClone(d.relationships[0]));},'Duplicate authored model relationship.'); +bad(d=>{const a=d.relationships[0];a.kind='parameter_derivation';d.relationships.push({...a,from:a.to,to:a.from});},'Authored parameter derivation contains a cycle.'); +assert.throws(()=>decode('not JSON'),error=>error.code==='invalid'&&error.message==='Organization note is not valid JSON.');checks++; +assert.throws(()=>decode(' '.repeat(1048577)),error=>error.code==='invalid'&&error.message==='Organization note exceeds its retained text limit.');checks++; +assert.equal(checkCard(valid(),fixture.document.models[0],fixture.cards[0]),fixture.cards[0].version.text);checks++; +for(const change of [r=>{r.project_id='c'.repeat(32);},r=>{r.id='d'.repeat(64);},r=>{r.path='notes/other.md';},r=>{r.version.revision=3;},r=>{r.version.sha256='e'.repeat(64);},r=>{r.version.deleted=true;}]){ + const result=structuredClone(fixture.cards[0]);change(result); + assert.throws(()=>checkCard(valid(),fixture.document.models[0],result),error=>error.code==='invalid'&&error.message==='Model card differs from its exact retained organization reference.');checks++; +} +{ + const value=valid();value.models[0].summary.method='Not present in the captured card.'; + assert.throws(()=>checkCard(value,value.models[0],fixture.cards[0]),error=>error.message==='Authored summary differs from the captured model card.');checks++; +} +{ + const value=valid();value.relationships[0].excerpt='Not present in the captured card.'; + assert.throws(()=>checkCard(value,value.models[0],fixture.cards[0]),error=>error.message==='Authored relationship excerpt differs from its captured model card.');checks++; +} +console.log('PASS authored portfolio document:',checks,'literal contract/reference decisions; no native capture or browser claim.'); diff --git a/tests/test_project_model_portfolio_http.py b/tests/test_project_model_portfolio_http.py new file mode 100644 index 00000000..8fbda68f --- /dev/null +++ b/tests/test_project_model_portfolio_http.py @@ -0,0 +1,84 @@ +"""Real authenticated note transport/cold reload; no model or recipe execution.""" +import hashlib +import json +import os +from pathlib import Path +import unittest +from unittest.mock import patch + +import project_workspace +import test_project_files_http as m1 +import test_project_files_m2_http as m2 + +ROOT = Path(__file__).resolve().parents[1] + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class ModelPortfolioHTTP(unittest.TestCase): + cleanup_owned = m1.ProjectFilesHTTP.cleanup_owned + select_host = m1.ProjectFilesHTTP.select_host + request = m1.ProjectFilesHTTP.request + state = m1.ProjectFilesHTTP.state + envelope = m1.ProjectFilesHTTP.envelope + action = m1.ProjectFilesHTTP.action + create = m1.ProjectFilesHTTP.create + arguments = m2.ProjectFilesM2HTTP.arguments + transport = m2.ProjectFilesM2HTTP.transport + call = m2.ProjectFilesM2HTTP.call + + def setUp(self): + self.operations = [] + m1.ProjectFilesHTTP.setUp(self) + gate = patch.object(project_workspace, 'M4_ENABLED', True) + gate.start() + self.addCleanup(gate.stop) + + def test_native_static_module_exact_note_revision_and_cold_reload(self): + status, html = self.request('/project-files') + self.assertEqual(status, 200) + self.assertIn(b'/project-model-portfolio.js', html) + self.assertIn(b'id="project-model-portfolio"', html) + status, script = self.request('/project-model-portfolio.js') + self.assertEqual(status, 200) + self.assertEqual(script, (ROOT/'static/project-model-portfolio.js').read_bytes()) + opened = self.create() + root = self.external/'alpha' + (root/'notes').mkdir() + paragraphs = {key: 'Literal retained '+key+' paragraph.' for key in + ('method','purpose','experiment','outcome','lesson','reproduction')} + card_text = '\n\n'.join(paragraphs.values())+'\nOriginal evidence remains.\n' + (root/'notes/model.md').write_text(card_text) + card = self.call('file_adopt',kind='note',path='notes/model.md',expected_revision=0, + expected_sha256=hashlib.sha256(card_text.encode()).hexdigest()) + guide = dict(format='project_model_portfolio_v1',project_id=opened['project']['project_id'], + title='Authored retained fixture',description='Synthetic presentation fixture; no scientific quality claim.', + authorship='retrospective',groups=[dict(id='pressure',label='Pressure methods',parent=None,description='An authored group.')], + models=[dict(id='fixture-fit',label='One retained fit',group_id='pressure',aliases=['Fixture'],review='source_reviewed', + card=dict(file_id=card['id'],path='notes/model.md',revision=1,sha256=card['sha256']),summary=paragraphs)],relationships=[]) + raw=(json.dumps(guide,ensure_ascii=False)+'\n').encode() + (root/'notes/00-fixture.portfolio.json').write_bytes(raw) + saved=self.call('file_adopt',kind='note',path='notes/00-fixture.portfolio.json',expected_revision=0, + expected_sha256=hashlib.sha256(raw).hexdigest()) + observed=self.call('file_get',file_id=saved['id'],revision=1) + self.assertEqual(observed['version']['text'].encode(),raw) + self.assertEqual(json.loads(observed['version']['text']),guide) + changed='A later authored card revision.\n' + (root/'notes/model.md').write_text(changed) + self.call('file_adopt',kind='note',path='notes/model.md',expected_revision=1, + expected_sha256=hashlib.sha256(changed.encode()).hexdigest()) + self.assertEqual(self.call('file_get',file_id=card['id'],revision=1)['version']['text'],card_text) + self.assertEqual(self.call('file_get',file_id=card['id'])['version']['text'],changed) + self.action('save') + expected=(root/'.tuldok/project.json').read_bytes() + self.action('close') + cold=os.environ.get('TULDOK_PROJECT_FILES_COLD_DATABASE_URL') + self.assertTrue(cold,'Independent cold role required for retained-note acceptance') + self.select_host(self.path/'cold-control',{'fixture':self.external},cold) + self.action('open',root='fixture',directory='alpha') + self.assertEqual((root/'.tuldok/project.json').read_bytes(),expected) + self.assertEqual(self.call('file_get',file_id=saved['id'],revision=1)['version']['text'].encode(),raw) + retained=self.call('file_get',file_id=card['id'],revision=1) + self.assertEqual(retained['version']['text'],card_text) + self.assertEqual(retained['project_id'],guide['project_id']) + self.assertEqual(self.call('file_get',file_id=card['id'])['version']['text'],changed) + self.action('close') diff --git a/tests/test_project_native_extraction.py b/tests/test_project_native_extraction.py index c2c3372b..291e2247 100644 --- a/tests/test_project_native_extraction.py +++ b/tests/test_project_native_extraction.py @@ -86,7 +86,7 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact captured = self.capture_without_host(run, source) self.assertEqual(captured['blockers'], [], captured['blockers']) materials = ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, - records=captured['records']), check=lambda: None)) + records=captured['records']), check_current=lambda: None)) facts = decode_records(captured['records'], materials=materials) saved = facts.runs[(self.db.dataset_id, run['id'])] self.assertEqual(saved['run']['runtime'], run['runtime']) @@ -115,7 +115,10 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact for name in originals: (stage / name).unlink() cold = ProjectStore.open(destination); self.addCleanup(cold.close) cold_artifacts = ProjectArtifacts(cold); self.addCleanup(cold_artifacts.close) - session = SimpleNamespace(snapshot=cold.snapshot(), access=self.access, artifacts=cold_artifacts, check=lambda: None) + snapshot = cold.snapshot() + session = SimpleNamespace(snapshot=snapshot, + summary={key:snapshot[key] for key in ('project_id','checkpoint')}, + access=self.access, artifacts=cold_artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda *args: None) for name, raw in originals.items(): self.assertEqual(history.artifact(run['id'], 'source/' + name, origin_catalog_id=self.db.dataset_id), raw) diff --git a/tests/test_project_native_process.py b/tests/test_project_native_process.py index 287de66a..5df959c4 100644 --- a/tests/test_project_native_process.py +++ b/tests/test_project_native_process.py @@ -5,6 +5,7 @@ import os from pathlib import Path import select +import shutil import signal import sys import tempfile @@ -311,8 +312,11 @@ def test_current_interpreter_alias_drift_before_launch_cannot_select_another_int stage=self.root/'venv-alias-drift' prepared=native.prepare(stage,phase='probe',attempt_id=uuid.uuid4().hex,request_sha256='d'*64, runtime=runtime,worker='text_training.py',sources={'text_training.py':WORKER},wall_seconds=8) - Path(executable).unlink();Path(executable).symlink_to(Path('/usr/bin/python3').resolve()) - with self.assertRaises(custody.ProcessError) as error:prepared.launch(stage/'custody') + replacement=self.root/'replacement-python' + shutil.copy2(runtime['executable_source'],replacement) + self.assertNotEqual(replacement.resolve(),Path(executable).resolve()) + Path(executable).unlink();Path(executable).symlink_to(replacement) + with self.assertRaises(custody.ProcessError) as error:self.launch(prepared) self.assertEqual(error.exception.code,'stale');self.assertFalse((stage/'custody').exists()) def test_spawn_callback_is_not_reached_for_current_config_alias_or_capture_refusal(self): diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index d08aaadc..3f1e3e98 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -6,7 +6,7 @@ from pathlib import Path import tempfile import unittest -from unittest.mock import patch +from unittest.mock import PropertyMock, patch import uuid import psycopg @@ -120,6 +120,25 @@ def test_unknown_receipt_is_rejected_before_invoke_and_sql_or_checkpoint_changes self.assertEqual(p.store.snapshot(), before) self.assertEqual(p.db.execute('SELECT COUNT(*) FROM workbench_records').fetchone()[0], 0) + def test_text_tag_uses_real_session_summary_and_record_list_remains_owned(self): + text = self.call('import_text', self.mutation(asset=dict(text='Owned note 山', groups=['owned'])))['record'] + p = self.host._projection + with self.owner.bound(self.host): + expected = dict(project_id=p.session.summary['project_id'], + checkpoint=p.session.summary['checkpoint']) + with patch.object(type(p.session), 'snapshot', new_callable=PropertyMock, + side_effect=AssertionError('Scalar tag copied a full Session')): + self.assertEqual(p.text._tag(dict(value='Exact scalar tag')), + dict(expected, value='Exact scalar tag')) + exact_current = [r['document']['current'] for r in p.session.snapshot['records'] + if r['family'] in ('workbench_text', 'workbench_record') + and r['document']['current']['kind'] == 'text'] + result = p.text.records() + self.assertEqual(result, dict(expected, items=exact_current)) + self.assertEqual([r['id'] for r in result['items']], [text['id']]) + result['items'][0]['text'] = 'Caller mutation' + self.assertEqual(p.text.get(text['id'])['text'], 'Owned note 山') + def test_explicit_rooted_artifact_supply_is_durable_and_never_fetches_or_executes(self): p = self.host._projection raw = b'Unique measured bytes; never regenerate these.\n' diff --git a/tests/test_project_publication_recovery.py b/tests/test_project_publication_recovery.py new file mode 100644 index 00000000..36067a6b --- /dev/null +++ b/tests/test_project_publication_recovery.py @@ -0,0 +1,116 @@ +"""Real committed-stage recovery preserves bytes and never reconstructs sources.""" +import errno +import os +import shutil +import unittest +from unittest.mock import patch + +from project_composition import composition_input +from project_fork import recover_committed_publication +from project_store import ProjectStore, ProjectStoreError +from project_files import file_id, materialize_captured +from test_project_composition_publication import CompositionPublicationTests + + +class PublicationRecoveryTests(unittest.TestCase): + setUp=CompositionPublicationTests.setUp + publish=CompositionPublicationTests.publish + assert_code=CompositionPublicationTests.assert_code + + def stage(self): + with patch('project_fork._rename_noreplace',side_effect=OSError(errno.EIO,'interrupted before rename')): + self.assert_code('unavailable',self.publish) + self.staging=next(self.root.glob('.tuldok-fork-*')) + with ProjectStore.open(self.staging) as store: + self.committed=store.snapshot() + self.request=composition_input(self.snapshots,'combined','Combined','compose-exact') + + def recover(self,**changes): + args=dict(root_path=self.root,expected_project_id=self.committed['project_id'], + expected_checkpoint=self.committed['checkpoint'],expected_request=self.request,quiescent=True) + args.update(changes) + return recover_committed_publication(self.fd,self.staging.name,'combined',**args) + + def test_committed_stage_recovers_without_sources_payload_copies_or_new_identity(self): + self.stage() + before={p.name:(p.read_bytes(),p.stat().st_ino) for p in (self.staging/'artifacts/sha256').iterdir()} + for store,artifacts in zip(self.stores,self.artifacts): + artifacts.close();store.close();shutil.rmtree(store.path) + with patch('project_artifacts.ProjectArtifacts.copy_many_to',side_effect=AssertionError('No recopy')), \ + patch('project_composition.build_candidate',side_effect=AssertionError('No rebuild')): + result=self.recover() + self.assertFalse(self.staging.exists()) + self.assertEqual(result['payloads_copied'],0) + self.assertFalse(result['selected']) + with ProjectStore.open(self.root/'combined') as store: + self.assertEqual(store.snapshot(),self.committed) + after={p.name:(p.read_bytes(),p.stat().st_ino) for p in (self.root/'combined/artifacts/sha256').iterdir()} + self.assertEqual(before,after) + self.assertTrue(self.recover()['reconciled']) + + def test_wrong_request_checkpoint_or_identity_never_publishes(self): + self.stage() + self.assert_code('stale',lambda:self.recover(expected_checkpoint='0'*64)) + self.assert_code('stale',lambda:self.recover(expected_project_id='0'*32)) + self.assert_code('conflict',lambda:self.recover(expected_request={**self.request,'name':'Changed'})) + self.assert_code('busy',lambda:self.recover(quiescent=False)) + self.assertTrue(self.staging.exists());self.assertFalse((self.root/'combined').exists()) + + def test_missing_copied_payload_preserves_stage_and_refuses_publication(self): + self.stage();next((self.staging/'artifacts/sha256').iterdir()).unlink() + with self.assertRaises(ProjectStoreError):self.recover() + self.assertTrue(self.staging.exists());self.assertFalse((self.root/'combined').exists()) + + def test_destination_appearance_never_overwrites_either_tree(self): + self.stage();(self.root/'combined').mkdir();(self.root/'combined/unique').write_bytes(b'keep') + self.assert_code('conflict',self.recover) + self.assertTrue(self.staging.exists());self.assertEqual((self.root/'combined/unique').read_bytes(),b'keep') + + def test_lost_rename_ack_reconciles_exact_published_checkpoint(self): + self.stage();fsync=os.fsync + def fail_after_publish(fd): + if (self.root/'combined').exists():raise OSError(errno.EIO,'lost publication acknowledgement') + fsync(fd) + with patch('project_fork.os.fsync',side_effect=fail_after_publish): + self.assert_code('unknown',self.recover) + self.assertFalse(self.staging.exists());self.assertTrue(self.recover()['reconciled']) + + def test_live_writer_and_non_stage_path_are_refused(self): + self.stage() + with ProjectStore.open(self.staging): + self.assert_code('busy',self.recover) + self.assert_code('invalid',lambda:recover_committed_publication(self.fd,'source0','combined', + root_path=self.root,expected_project_id=self.committed['project_id'],expected_checkpoint=self.committed['checkpoint'], + expected_request=self.request,quiescent=True)) + + def test_initial_empty_stage_cannot_be_mistaken_for_committed_publication(self): + self.stage();empty_name='.tuldok-fork-'+'9'*32 + with ProjectStore.create(self.root/empty_name,'Uncommitted','initial','a'*64) as store: + initial=store.snapshot() + self.assert_code('invalid',lambda:recover_committed_publication(self.fd,empty_name,'combined', + root_path=self.root,expected_project_id=initial['project_id'],expected_checkpoint=initial['checkpoint'], + expected_request=self.request,quiescent=True)) + self.assertTrue((self.root/empty_name).exists());self.assertFalse((self.root/'combined').exists()) + + def test_missing_captured_working_file_refuses_without_changing_committed_history(self): + # Seed a real current note before composing; its cloned editable record + # and materialized bytes then pass through the normal publication path. + from datetime import datetime, timezone + import hashlib + store=self.stores[0];snap=store.snapshot();raw=b'original note\n' + version=dict(revision=1,text=raw.decode(),sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw), + actor_id='1'*32,adopted_at=datetime.now(timezone.utc).isoformat()) + doc=dict(format='project_file_v1',project_id=snap['project_id'],kind='note',path='notes/test.md', + current=version,history=[version],origin=None) + record=dict(family='project_file',id=file_id(snap['project_id'],'note','notes/test.md'),document=doc) + self.snapshots[0]=store.commit([*snap['records'],record],snap['checkpoint'],'note','c'*64) + with store.root_directory() as root: + materialize_captured(self.snapshots[0]['records'],snap['project_id'],root) + self.stage();(self.staging/'notes/test.md').unlink() + self.assert_code('working_files_changed',self.recover) + self.assertFalse((self.root/'combined').exists()) + with ProjectStore.open(self.staging) as retained: + self.assertEqual(retained.snapshot(),self.committed) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_project_receipt_validation.py b/tests/test_project_receipt_validation.py new file mode 100644 index 00000000..bf69926f --- /dev/null +++ b/tests/test_project_receipt_validation.py @@ -0,0 +1,204 @@ +"""One mixed candidate: retain every receipt check without repeated decoding.""" +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import project_files as files +import project_material as material +import project_measurements as measurements +import project_projection as projection +import project_sources as sources +from project_artifacts import ProjectArtifacts, ProjectArtifactError +from project_store import ProjectStore +from test_retained_measurements import fixture +from workbench import WorkbenchError + +PROJECT = 'a' * 32 +ACTOR = 'c' * 32 +INSTANT = '2026-10-25T00:00:00+00:00' + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def operation(name, key, result): + return dict(family='project_operation', id=PROJECT + '.' + sha(key.encode()), document=dict( + format='project_operation_v2', project_id=PROJECT, operation=name, + target=None, request_key=key, input_sha256='d' * 64, result=copy.deepcopy(result))) + + +def candidate(): + packets = [] + for index in range(2): + value = fixture() + value['title'] = f'Packet {index}' + raw = json.dumps(value).encode() + packets.append(dict(raw=raw, sha256=sha(raw), path=f'packet-{index}.json', + reference_id=f'packet-{index}', title=value['title'])) + manifest = dict(schema_version=1, id='b' * 32, name='Receipt source', + data=[dict(id=p['reference_id'], label=p['title'], path=p['path'], sha256=p['sha256']) + for p in packets], recipes=[], evidence=[], requirements=[]) + raw = json.dumps(manifest).encode() + digest = sha(raw) + snapshot = dict(id=manifest['id'], name=manifest['name'], root='original', + root_identity=[1, 2], manifest_path='source.json', manifest_sha256=digest, manifest=manifest) + retained = dict(family='project_snapshot', id=PROJECT + '.' + digest, document=dict( + format=material.FAMILIES['project_snapshot'], origin_catalog_id=PROJECT, value=snapshot, + artifacts=[dict(name='source.json', blob=dict(sha256=digest, bytes=len(raw)), + role='project_manifest', preservation='unique', required_for=['snapshot_verification'])], audit=[])) + records = [retained] + for hashed, size in [(digest, len(raw)), *[(p['sha256'], len(p['raw'])) for p in packets]]: + records.append(dict(family='project_artifact', id=hashed, + document=dict(format='project_artifact_v1', sha256=hashed, bytes=size))) + source_result = dict(origin_catalog_id=PROJECT, project_id=manifest['id'], manifest_sha256=digest, + name=manifest['name'], schema_version=1, actor_id=ACTOR, registered_at=INSTANT) + for index in range(2): + records.append(operation('source.register', f'source-{index}', source_result)) + for index, packet in enumerate(packets): + doc = dict(format=measurements.FORMAT, origin_catalog_id=PROJECT, id=packet['sha256'], + title=packet['title'], blob=dict(sha256=packet['sha256'], bytes=len(packet['raw'])), + source=dict(origin_catalog_id=PROJECT, project_id=manifest['id'], manifest_sha256=digest, + reference_id=packet['reference_id'], path=packet['path'], measured_sha256=packet['sha256']), + actor_id=ACTOR, imported_at=INSTANT) + records.append(dict(family=measurements.FAMILY, id=PROJECT + '.' + packet['sha256'], document=doc)) + records.append(operation('measurements.import', f'packet-{index}', doc)) + for batch in range(2): + items = [] + for index in range(2): + path = f'notes/note-{batch}-{index}.md' + raw = f'Owned note {batch}-{index} 山'.encode() + version = dict(revision=1, sha256=sha(raw), bytes=len(raw), text=raw.decode(), + adopted_at=INSTANT, actor_id=ACTOR) + identity = files.file_id(PROJECT, 'note', path) + doc = dict(format=files.FORMAT, project_id=PROJECT, kind='note', path=path, + current=copy.deepcopy(version), history=[version], origin=None) + records.append(dict(family=files.FAMILY, id=identity, document=doc)) + items.append(dict(id=identity, project_id=PROJECT, kind='note', path=path, changed=True, + **{key: version[key] for key in ('revision', 'sha256', 'bytes', 'adopted_at')})) + records.append(operation('files.adopt_many', f'notes-{batch}', dict(items=items, changed_count=2))) + return records + + +class ReceiptValidationTests(unittest.TestCase): + def test_whole_candidate_decodes_each_receipt_owner_once(self): + records = candidate() + before = copy.deepcopy(records) + with patch.object(files, 'decode_records', wraps=files.decode_records) as file_decode, \ + patch.object(projection, 'decode_file_records', file_decode), \ + patch.object(measurements, 'decode_records', wraps=measurements.decode_records) as measurement_decode, \ + patch.object(projection, 'decode_measurement_records', measurement_decode), \ + patch.object(material, 'decode_records', wraps=material.decode_records) as material_decode, \ + patch.object(measurements, 'material_records', material_decode), \ + patch.object(sources, 'material_records', material_decode): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + self.assertEqual(records, before) + for name, actual, expected in [('files', file_decode.call_count, 1), + ('measurements', measurement_decode.call_count, 1), + ('material-owning readers', material_decode.call_count, 3)]: + with self.subTest(owner=name): + self.assertEqual(actual, expected, 'Receipt validation redecoded the same candidate') + + def test_standalone_file_batch_decodes_inventory_once(self): + records = candidate() + result = next(r['document']['result'] for r in records + if r['family'] == 'project_operation' + and r['document']['operation'] == 'files.adopt_many') + with patch.object(files, 'decode_records', wraps=files.decode_records) as decode: + self.assertTrue(files.validate_operation_result('files.adopt_many', result, records)) + self.assertEqual(decode.call_count, 1, 'Each batch item decoded the full file inventory') + + def test_forged_late_file_or_measurement_receipt_is_rejected(self): + for owner in ('files.adopt_many', 'measurements.import', 'source.register'): + records = candidate() + result = [r['document']['result'] for r in records if r['family'] == 'project_operation' + and r['document']['operation'] == owner][-1] + if owner == 'files.adopt_many': + result['items'][-1]['sha256'] = 'e' * 64 + else: + result['name' if owner == 'source.register' else 'title'] = 'Forged retained value' + with self.subTest(owner=owner), self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_unknown_operation_still_has_no_accepting_owner(self): + records = candidate() + records.append(operation('unknown.capture', 'unknown', {})) + with self.assertRaises(WorkbenchError) as error: + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + self.assertEqual(error.exception.code, 'unsupported') + + def test_owned_validators_and_standalone_calls_do_not_share_candidates(self): + for name, module in [('files.adopt_many', files), ('measurements.import', measurements), + ('source.register', sources)]: + records = candidate() + original_result = copy.deepcopy(next(r['document']['result'] for r in records + if r['family'] == 'project_operation' and r['document']['operation'] == name)) + checker = module.receipt_validator(records) + self.assertTrue(checker(name, original_result)) + self.assertTrue(module.validate_operation_result(name, original_result, records)) + if module is files: + doc = next(r['document'] for r in records if r['family'] == files.FAMILY) + value = doc['history'][0] + value.update(text='Edited candidate', sha256=sha(b'Edited candidate'), bytes=16) + doc['current'] = copy.deepcopy(value) + elif module is measurements: + next(r['document'] for r in records if r['family'] == measurements.FAMILY)['title'] = 'Edited candidate' + else: + value = next(r['document']['value'] for r in records if r['family'] == 'project_snapshot') + value['name'] = value['manifest']['name'] = 'Edited candidate' + with self.subTest(owner=name): + self.assertTrue(checker(name, original_result), 'Owned candidate changed with caller mutation') + with self.assertRaises(WorkbenchError): + module.receipt_validator(records)(name, original_result) + with self.assertRaises(WorkbenchError): + module.validate_operation_result(name, original_result, records) + + def test_standalone_unrecognized_and_malformed_results_do_not_decode(self): + for name, module in [('files.adopt_many', files), ('measurements.import', measurements), + ('source.register', sources)]: + records = candidate() + attribute = 'material_records' if module is sources else 'decode_records' + with self.subTest(owner=name), patch.object(module, attribute, + side_effect=AssertionError('Result shape must precede candidate decoding')) as decode: + self.assertFalse(module.validate_operation_result('unknown.capture', {}, records)) + with self.assertRaises(WorkbenchError): + module.validate_operation_result(name, {}, records) + decode.assert_not_called() + + def test_unused_malformed_file_and_missing_packet_reference_still_reject(self): + records = candidate() + next(r['document']['history'][0] for r in records if r['family'] == files.FAMILY)['text'] = 'Unhashed' + records = [r for r in records if r['family'] != 'project_operation' + or r['document']['operation'] != 'files.adopt_many'] + with self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + records = candidate() + next(r['document'] for r in records if r['family'] == measurements.FAMILY)['source']['reference_id'] = 'absent' + with self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_each_new_measurement_validator_checks_actual_packet_bytes(self): + records = candidate() + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / 'project' + with ProjectStore.create(root, 'Packet custody fixture', 'create', 'a' * 64) as store: + with ProjectArtifacts(store) as artifacts: + docs = [r['document'] for r in records if r['family'] == measurements.FAMILY] + for index, doc in enumerate(docs): + packet = fixture() + packet['title'] = f'Packet {index}' + raw = json.dumps(packet).encode() + self.assertEqual(sha(raw), doc['id']) + artifacts.publish_bytes(raw, measurements.declaration(doc['blob'])) + checker = measurements.receipt_validator(records, artifacts=artifacts) + self.assertTrue(checker('measurements.import', docs[0])) + payload = root / 'artifacts/sha256' / docs[0]['id'] + raw = payload.read_bytes() + payload.chmod(0o600) + payload.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + with self.assertRaises((WorkbenchError, ProjectArtifactError)): + measurements.receipt_validator(records, artifacts=artifacts) diff --git a/tests/test_project_session.py b/tests/test_project_session.py index f9356c44..75ff7795 100644 --- a/tests/test_project_session.py +++ b/tests/test_project_session.py @@ -14,6 +14,7 @@ from unittest.mock import patch import uuid +import project_fork as fork_module import psycopg from psycopg import sql @@ -27,6 +28,24 @@ class OperationCodecTests(unittest.TestCase): + def test_operation_lookup_does_not_copy_unrelated_active_envelopes(self): + key='selected-receipt' + document=dict(format='project_operation_v1',operation='fixture.capture',target=None, + request_key=key,input_sha256='a'*64,result={}) + receipt=dict(family='project_operation',id=hashlib.sha256(key.encode()).hexdigest(),document=document) + unrelated=dict(family='project_artifact',id='b'*64, + document=dict(format='project_artifact_v1',sha256='b'*64,bytes=4)) + copied=[];deepcopy=copy.deepcopy + def observe(value): + if value is unrelated:copied.append((value['family'],value['id'])) + return deepcopy(value) + with patch.object(fork_module,'copy',SimpleNamespace(deepcopy=observe)): + actual=operation_records([unrelated,receipt]) + self.assertEqual(actual,{key:document}) + self.assertEqual(copied,[],'Receipt selection must not clone unrelated active records') + actual[key]['result']['changed']=True + self.assertEqual(document['result'],{}) + def test_namespaced_receipts_preserve_legacy_identity_and_independent_keys(self): key = 'independent-edit' document = dict(format='project_operation_v1', operation='fixture.capture', target=None, @@ -155,6 +174,23 @@ def test_complete_merge_and_original_result_replay_before_stale_cas(self): self.assertIn(('project_artifact', other['sha256']), ids) self.assertEqual(self.artifacts.read_blob({key: declaration[key] for key in ('sha256', 'bytes')}), 'Irreplaceable observation 山'.encode()) + def test_scalar_summary_is_independent_and_tracks_the_bound_publication(self): + before = self.session.summary + arguments, _declaration, _calls = self.mutation() + self.call(arguments) + snapshot = self.session.snapshot + expected = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} + self.assertNotEqual(before['checkpoint'], expected['checkpoint']) + original_copy = copy.deepcopy + def observe(value): + self.assertIsNot(value, self.session._snapshot, 'Summary must not clone the record graph') + return original_copy(value) + with patch('project_session.copy', SimpleNamespace(deepcopy=observe)): + first = self.session.summary + first['name'] = 'Caller mutation' + self.assertEqual(self.session.summary, expected) + self.assertEqual(self.session.snapshot['name'], expected['name']) + def test_key_conflict_stale_and_current_auth_refuse_before_invoke(self): first, _declaration, _calls = self.mutation() self.call(first) diff --git a/tests/test_project_session_freshness.py b/tests/test_project_session_freshness.py new file mode 100644 index 00000000..8ab5634e --- /dev/null +++ b/tests/test_project_session_freshness.py @@ -0,0 +1,136 @@ +"""Freshness enforcement over real canonical files, without a SQL projection. + +Only the session's read guard is exercised here. Publication and projection +binding still require the separate real-PostgreSQL session/workspace tests. +""" +import copy +import hashlib +import json +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from project_session import ProjectSession, ProjectSessionError +from project_store import ProjectStore, ProjectStoreError + + +class ProjectSessionFreshnessTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-freshness-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / 'project' + self.store = ProjectStore.create(self.root, 'Freshness fixture', 'new', 'a' * 64) + self.addCleanup(self.store.close) + self.records = [dict(family='workbench_text', id=name, + document=dict(format='workbench_text_v1', text='Alpha ' + name)) for name in ('one', 'two')] + snapshot = self.store.commit(self.records, self.store.summary()['checkpoint'], 'seed', 'b' * 64) + # Isolate the existing read guard; no fake SQL/projection acceptance. + self.session = object.__new__(ProjectSession) + self.session.store = self.store + self.session._snapshot = copy.deepcopy(snapshot) + self.session.available = True + + def test_freshness_reads_all_records_without_copying_an_unused_graph(self): + before = self.store.snapshot() + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused full snapshot')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: + self.assertIsNone(self.session.check_current()) + self.assertIsNone(self.session.check_current()) + self.assertEqual(reads.call_count, 4) + self.assertEqual(self.store.snapshot(), before) + + def test_changed_canonical_checkpoint_refuses_both_read_contracts(self): + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Independently published content' + self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c' * 64) + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): + with self.assertRaises(ProjectSessionError) as caught: + check() + self.assertEqual(caught.exception.code, 'stale_project') + + def test_unavailable_projection_refuses_before_reading_the_store(self): + self.session.available = False + with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')), \ + patch.object(self.store, 'snapshot', side_effect=AssertionError('Unavailable snapshot read')): + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): + with self.assertRaises(ProjectSessionError) as caught: + check() + self.assertEqual(caught.exception.code, 'projection_unavailable') + + def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self): + self.session.check_current() + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + path = self.root / '.tuldok' / manifest['inventory'][0]['path'] + info, original = path.stat(), path.read_bytes() + changed = original.replace(b'Alpha', b'Omega') + self.assertEqual(len(original), len(changed)) + self.assertNotEqual(hashlib.sha256(original).digest(), hashlib.sha256(changed).digest()) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): + with self.assertRaises(ProjectStoreError) as caught: + check() + self.assertEqual(caught.exception.code, 'stale') + + def test_record_consumers_keep_independent_complete_snapshots(self): + current = self.session.check() + current['records'][0]['document']['text'] = 'Caller mutation' + current['name'] = 'Caller renamed copy' + self.assertEqual(self.session.check(), self.store.snapshot()) + self.assertEqual(self.session._snapshot, self.store.snapshot()) + self.assertNotEqual(current, self.session.check()) + + def add_unrelated_record(self): + unrelated = dict(family='project_artifact', id='d' * 64, + document=dict(format='project_artifact_v1', sha256='d' * 64, bytes=4)) + self.session._snapshot = self.store.commit([*self.records, unrelated], + self.store.summary()['checkpoint'], 'unrelated', 'e' * 64) + + def test_selected_records_are_exact_independent_and_do_not_copy_other_families(self): + self.add_unrelated_record() + expected = {key: value for key, value in self.store.snapshot().items() if key != 'records'} + expected['records'] = self.records + copied = [] + deepcopy = copy.deepcopy + def observe(value): + self.assertEqual([row['family'] for row in value], ['workbench_text', 'workbench_text']) + copied.append(value) + return deepcopy(value) + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unselected graph copy')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ + patch('project_session.copy', SimpleNamespace(deepcopy=observe)): + selected = self.session.check_records('workbench_text') + self.assertEqual(selected, expected) + self.assertEqual(reads.call_count, 3, 'Unselected actual records must still be read and hashed') + self.assertEqual(len(copied), 1) + selected['records'][0]['document']['text'] = 'Caller mutation' + self.assertEqual(self.session.check_records('workbench_text'), expected) + self.assertEqual(self.session.check_records('project_file')['records'], []) + + def test_selected_read_rejects_corruption_in_an_unselected_family(self): + self.add_unrelated_record() + self.session.check_records('workbench_text') + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + entry = next(row for row in manifest['inventory'] if row['family'] == 'project_artifact') + path = self.root / '.tuldok' / entry['path'] + info, original = path.stat(), path.read_bytes() + changed = original.replace(b'"bytes": 4', b'"bytes": 5') + self.assertNotEqual(changed, original) + self.assertEqual(len(changed), len(original)) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + with self.assertRaises(ProjectStoreError) as caught: + self.session.check_records('workbench_text') + self.assertEqual(caught.exception.code, 'stale') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 0f65339d..676ab3bc 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -3,10 +3,12 @@ import errno import hashlib import json +import math import os from pathlib import Path import shutil import subprocess +import struct import sys import tempfile import unittest @@ -71,6 +73,25 @@ def write_manifest(self, value): value['checkpoint'] = sha(canonical(value)) (self.root / '.tuldok/project.json').write_bytes(canonical(value)) + def test_snapshot_preserves_scalar_types_float_bits_and_owned_nested_values(self): + floats = [-0.0, 0.0, 5e-324, -5e-324, 1.7976931348623157e308, + math.nextafter(1.0, 0.0), math.nextafter(1.0, math.inf)] + value = record() + value['document']['scalars'] = [None, False, True, -2**63, 2**63-1, + 'Unicode 漢字 \U0001f30a\x00', *floats] + expected = self.commit([value]) + first, second = self.store.snapshot(), self.store.snapshot() + self.assertEqual(first, expected) + self.assertEqual(second, expected) + actual = first['records'][0]['document']['scalars'] + self.assertEqual([type(v) for v in actual], + [type(v) for v in value['document']['scalars']]) + self.assertEqual([struct.pack('>d', v) for v in actual[6:]], + [struct.pack('>d', v) for v in floats]) + first['records'][0]['document']['history'][0]['reviewed'] = True + first['records'][0]['document']['scalars'].clear() + self.assertEqual(self.store.snapshot(), second) + def test_create_commit_save_cold_reopen_and_independent_snapshots(self): initial = self.store.snapshot() self.assertEqual(initial['records'], []) @@ -86,6 +107,92 @@ def test_create_commit_save_cold_reopen_and_independent_snapshots(self): committed['records'][0]['document']['text'] = 'Unchecked returned mutation' self.assertEqual(self.store.snapshot()['records'][0]['document']['text'], 'Notes preserve \r\n authored values') + def test_appending_one_record_does_not_rewrite_saved_immutable_revisions(self): + values = [record(str(index), 'Retained authored note %d' % index) for index in range(8)] + self.commit(values, key='initial-inventory') + writer = store_module._write_file + with patch.object(store_module, '_write_file', wraps=writer) as writes: + committed = self.commit([*values, record('new')], key='append-one') + record_writes = [call for call in writes.call_args_list if call.args[1].startswith('.record-')] + self.assertEqual(len(record_writes), 1, + 'Appending one record must not write disposable copies of every saved immutable revision') + self.assertEqual(self.reopen(), committed) + + def test_summary_reads_every_current_record_without_cloning_the_record_graph(self): + committed = self.commit([record('a'), record('b')], key='summary-fixture') + expected = {key: committed[key] for key in ('project_id', 'name', 'checkpoint')} + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused record graph copy')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: + first = self.store.summary() + first['name'] = 'Caller mutation' + second = self.store.summary() + self.assertEqual(second, expected) + self.assertEqual(reads.call_count, 4, 'Each summary must still read/hash both actual records') + self.assertEqual(self.store.snapshot(), committed) + + def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_identical_records(self): + rows=[record('a'),record('b','Exact second note\n')] + self.commit(rows,key='repeat-read-fixture') + expected=self.store.snapshot() + canonical_documents={canonical(row['document']) for row in rows} + with patch.object(store_module,'_decode',wraps=store_module._decode) as decode, \ + patch.object(self.store,'_record_file',wraps=self.store._record_file) as read, \ + patch.object(self.store,'_snapshot',wraps=self.store._snapshot) as snapshot: + self.assertEqual(self.store.snapshot(),expected) + with self.store.root_directory(verify_after=True) as descriptor: + self.assertTrue(os.fstat(descriptor).st_ino) + self.assertEqual(self.store.snapshot(),expected) + self.assertEqual(read.call_count,8,'All two records must still be read in each of four checks') + self.assertEqual(snapshot.call_count,2,'Root borrowing must not clone an unused public snapshot') + decoded=[call for call in decode.call_args_list if call.args[0] in canonical_documents] + self.assertEqual(len(decoded),0,'Byte-identical canonical records need no repeat decoding within one Store') + + def test_verified_decode_cache_still_rejects_same_length_corruption_with_restored_mtime(self): + self.commit([record('note','Alpha\n')],key='cache-corruption-fixture') + self.store.snapshot() + entry=self.manifest()['inventory'][0] + path=self.root/'.tuldok'/entry['path'];info=path.stat();raw=path.read_bytes() + changed=raw.replace(b'Alpha',b'Omega') + self.assertEqual(len(changed),len(raw));self.assertNotEqual(changed,raw) + path.chmod(0o600);path.write_bytes(changed) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + self.assert_code('stale',self.store.snapshot) + self.assert_code('stale',self.store.summary) + with self.assertRaises(ProjectStoreError) as caught: + with self.store.root_directory(verify_after=True):pass + self.assertEqual(caught.exception.code,'stale') + + def test_reusing_an_unreferenced_revision_still_checks_bytes_and_syncs_it(self): + first, second = record('first'), record('second') + self.commit([first], key='retain-first') + self.commit([second], key='retain-second') + writer, sync = store_module._write_file, store_module._sync_file + with patch.object(store_module, '_write_file', wraps=writer) as writes: + with patch.object(store_module, '_sync_file', wraps=sync) as syncs: + committed = self.commit([first, second], key='reuse-first') + self.assertFalse(any(call.args[1].startswith('.record-') for call in writes.call_args_list)) + self.assertEqual(syncs.call_count, 2) + self.assertEqual(self.reopen(), committed) + + def test_reusing_a_corrupt_unreferenced_revision_never_publishes_it(self): + first = record('first') + self.commit([first], key='retain-first') + entry = self.manifest()['inventory'][0] + self.commit([], key='exclude-first') + before = self.store.snapshot() + path = self.root / '.tuldok' / entry['path'] + path.unlink() + path.write_bytes(b'{"corrupt":"unreferenced revision"}\n') + self.assert_code('invalid', self.commit, [first], key='reuse-corrupt') + self.assertEqual(self.reopen(), before) + + def test_existing_revision_sync_failure_keeps_the_previous_checkpoint(self): + self.commit([record()], key='initial') + before = self.store.snapshot() + with patch.object(store_module, '_sync_file', side_effect=OSError(errno.EIO, 'existing revision sync failed')): + self.assert_code('unavailable', self.commit, [record()], key='reuse-with-sync-error') + self.assertEqual(self.reopen(), before) + def test_fresh_copy_and_moved_folder_keep_identity_and_values(self): committed = self.commit() self.store.close() @@ -177,12 +284,14 @@ def test_replaced_physical_root_or_metadata_directory_is_rejected(self): self.root.rename(displaced) self.root.mkdir() self.assert_code('stale', self.store.snapshot) + self.assert_code('stale', self.store.summary) self.root.rmdir() displaced.rename(self.root) meta = self.root / '.tuldok' meta.rename(self.root / '.old-tuldok') meta.mkdir() self.assert_code('stale', self.store.snapshot) + self.assert_code('stale', self.store.summary) def test_changed_checkpoint_is_rejected_before_mutation(self): initial = self.store.snapshot() @@ -266,6 +375,33 @@ def test_closed_document_envelopes_and_supported_family_format(self): for value in invalid: with self.subTest(value=repr(value)): self.assert_code('invalid', self.store.commit, [value], checkpoint, 'bad', body('bad')) + + def test_portfolio_above_256_MiB_commits_and_cold_reopens(self): + # Each document fits the 40 MiB owner bound. Aggregate project metadata + # must also support scientific catalogs larger than the old 256 MiB. + text = 'p' * (30 * 1024 * 1024) + values = [record(str(index), text) for index in range(9)] + committed = self.store.commit(values, self.store.snapshot()['checkpoint'], + 'large-portfolio', body('large-portfolio')) + self.assertGreater(sum(row['bytes'] for row in self.manifest()['inventory']), + 256 * 1024 * 1024) + self.assertEqual(self.reopen(), committed) + self.assertEqual(self.store.snapshot()['records'], values) + + def test_aggregate_byte_bound_rejects_commit_and_cold_open(self): + values = [record('a'), record('b')] + total = sum(len(canonical(value['document'])) for value in values) + before = self.bytes_tree() + checkpoint = self.store.snapshot()['checkpoint'] + with patch.object(store_module, 'MAX_TOTAL_BYTES', total - 1): + self.assert_code('invalid', self.store.commit, values, checkpoint, + 'over-budget', body('over-budget')) + self.assertEqual(self.bytes_tree(), before) + committed = self.commit(values) + self.store.close() + with patch.object(store_module, 'MAX_TOTAL_BYTES', total - 1): + self.assert_code('invalid', ProjectStore.open, self.root) + self.assertEqual(self.reopen(), committed) for value in [dict(record(), family='access_credentials'), dict(record(), document={'format': 'workbench_text_v2'}), dict(record(), document={'format': 'unknown_v1'})]: diff --git a/tests/test_project_study_execution.py b/tests/test_project_study_execution.py index 321cae09..7227e00d 100644 --- a/tests/test_project_study_execution.py +++ b/tests/test_project_study_execution.py @@ -95,6 +95,31 @@ def before_write(): class StudyExecutionCodecTests(unittest.TestCase): + def test_canonical_reconstruction_uses_one_selected_material_view(self): + records=[];view=object();calls=[] + class Materials: + def read_view(self,selected): + calls.append(selected);return view + adapter=ProjectStudyExecution.__new__(ProjectStudyExecution) + adapter.session=SimpleNamespace(snapshot={'records':records});adapter.materials=Materials() + def decoded(selected,*,materials): + self.assertIs(selected,records);self.assertIs(materials,view);return 'selected facts' + with patch('project_study_execution.history_records',side_effect=decoded): + self.assertEqual(adapter._canonical(),'selected facts') + self.assertEqual(adapter._canonical(records),'selected facts') + self.assertEqual(len(calls),2) + self.assertTrue(all(selected is records for selected in calls)) + + def test_fixed_record_validation_has_no_live_material_session(self): + import project_study_execution as owner + from project_material import MaterialSnapshot + records=[] + def decoded(selected,*,materials): + self.assertIs(selected,records);self.assertIsInstance(materials,MaterialSnapshot);return 'fixed facts' + with patch('project_study_execution.ProjectMaterial',side_effect=AssertionError('No live material lookups')), \ + patch('project_study_execution.history_records',side_effect=decoded): + self.assertEqual(owner._facts(records),'fixed facts') + def test_admission_has_only_exact_reservation_and_references(self): value=trial('9'*32,'3'*32)['document']['trial'];composition=Materials().recipe(value['material']) charge=composition['material']['resource_envelope'] diff --git a/tests/test_project_text_tag.py b/tests/test_project_text_tag.py new file mode 100644 index 00000000..6a944aa6 --- /dev/null +++ b/tests/test_project_text_tag.py @@ -0,0 +1,52 @@ +"""The scalar text response tag must not copy unrelated project metadata.""" +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import project_fork + +from project_projection import _TextView + + +class TextTagTests(unittest.TestCase): + def test_text_records_does_not_clone_unrelated_active_documents(self): + unrelated = dict(family='project_artifact', id='c' * 64, + document=dict(format='project_artifact_v1', sha256='c' * 64, bytes=4)) + note = dict(family='workbench_text', id='d' * 32, + document=dict(current=dict(kind='text', text='Unique note 山'))) + summary = dict(project_id='a' * 32, checkpoint='b' * 64) + session = SimpleNamespace(summary=summary, + snapshot=dict(summary, records=[unrelated, note]), + check_current=lambda: None) + copied = [] + original = project_fork.copy.deepcopy + + def observe(value): + if value is unrelated: + copied.append(value['id']) + return original(value) + + with patch.object(project_fork, 'copy', SimpleNamespace(deepcopy=observe)): + result = _TextView(SimpleNamespace(session=session)).records() + self.assertEqual(result, dict(summary, items=[dict(kind='text', text='Unique note 山')])) + self.assertEqual(copied, [], 'Text list cloned an unrelated active document') + result['items'][0]['text'] = 'Caller mutation' + self.assertEqual(note['document']['current']['text'], 'Unique note 山') + + def test_text_tag_reads_scalar_summary_without_a_record_snapshot(self): + class Session: + summary = dict(project_id='a' * 32, name='Owned project', checkpoint='b' * 64) + + @property + def snapshot(self): + raise AssertionError('Text response tag copied the full record graph') + + session = Session() + view = _TextView(SimpleNamespace(session=session)) + original = dict(items=[dict(text='Authored note 漢字')]) + result = view._tag(original) + self.assertEqual(result, dict(original, project_id='a' * 32, checkpoint='b' * 64)) + self.assertNotIn('name', result) + self.assertEqual(original, dict(items=[dict(text='Authored note 漢字')])) + result['project_id'] = 'caller edit' + self.assertEqual(session.summary['project_id'], 'a' * 32) diff --git a/tests/test_project_transfer.py b/tests/test_project_transfer.py index efb639b4..5b798960 100644 --- a/tests/test_project_transfer.py +++ b/tests/test_project_transfer.py @@ -433,7 +433,7 @@ def capture(_,roots): from project_material import ProjectMaterial with ProjectStore.open(self.root/'science') as store,ProjectArtifacts(store) as artifacts: snapshot=store.snapshot();records=snapshot['records'] - view=SimpleNamespace(snapshot=snapshot,artifacts=artifacts,check=lambda:None) + view=SimpleNamespace(snapshot=snapshot,artifacts=artifacts,check_current=lambda:None) facts=history(records,materials=ProjectMaterial(view),artifacts=artifacts) self.assertEqual(len(facts.studies),2);self.assertEqual(len(facts.trials),2) self.assertEqual(len(facts.candidates),1);self.assertEqual(len(facts.reservations),1);self.assertEqual(len(facts.runs),1) diff --git a/tests/test_project_workspace.py b/tests/test_project_workspace.py index 0c2dbaee..a78982f4 100644 --- a/tests/test_project_workspace.py +++ b/tests/test_project_workspace.py @@ -72,6 +72,17 @@ def create(self, directory='alpha'): arguments = self.mutation(root='fixture', directory=directory, name='Workspace authored project') return self.call('new', arguments), arguments + def test_status_returns_current_metadata_without_copying_the_record_graph(self): + self.create() + self.import_record('A retained record unrelated to the project summary') + expected = self.state() + with patch.object(self.workspace._store, '_snapshot', + wraps=self.workspace._store._snapshot) as snapshots: + observed = self.state() + self.assertEqual(observed, expected) + self.assertEqual(snapshots.call_count, 0, + 'Status must not copy every retained record to return three metadata fields') + def test_control_project_ancestry_is_rejected_before_destination_or_projection_creation(self): with ProjectStore.create(self.external/'science','Existing portable project','existing-project','b'*64):pass self.start(self.external/'science/control') diff --git a/tests/test_runtime_snapshots.py b/tests/test_runtime_snapshots.py index 8ff9a407..11d58a35 100644 --- a/tests/test_runtime_snapshots.py +++ b/tests/test_runtime_snapshots.py @@ -53,6 +53,21 @@ def test_exact_capture_survives_source_removal_and_reuses_without_overwrite(self self.assertEqual((root/'packages/torch/__init__.py').stat().st_mode&0o777,0o400) self.assertFalse(list(owner.path.glob('.capture-*'))) + def test_capture_seals_intermediate_directories_under_group_writable_umask(self): + db,access,owner,config=self.fixture() + nested=Path(config['site_packages'])/'torch'/'nested'/'data' + nested.mkdir(parents=True);(nested/'fixture.txt').write_text('Recorded nested source') + observed=owner.inspect(config) + previous=os.umask(0o002) + try: + value=owner.create(config,observed['inventory_sha256'],quiesced=True,reserve_bytes=0) + root=Path(value['root']) + for directory in [root,*[path for path in root.rglob('*') if path.is_dir()]]: + self.assertEqual(directory.stat().st_mode & 0o022,0) + self.assertEqual(owner.get(observed['inventory_sha256']),value) + finally: + self.assertEqual(os.umask(previous),0o002) + def test_backup_restore_keeps_cold_runtime_modes_and_refuses_changed_completed_modes(self): from backup_metadata import backup_catalog, restore_catalog, StorageTransferError db, access, owner, config = self.fixture() diff --git a/tests/test_torch_runtime.py b/tests/test_torch_runtime.py index c93d202b..73beb56c 100644 --- a/tests/test_torch_runtime.py +++ b/tests/test_torch_runtime.py @@ -89,6 +89,27 @@ def checkpoint(): with self.assertRaises(Cancelled):runtime.file_record(Path(self.config['python']),checkpoint=checkpoint) self.assertEqual(len(calls),2) + def test_retained_prelink_manifest_keeps_exact_hash_without_current_launch_authority(self): + from pipeline import digest + from project_material import decode_runtime_material + manifest=runtime.inventory(self.config);manifest.pop('file_links') + raw=runtime.encoded(manifest) + descriptor=dict(schema_version=1,profile=runtime.PROFILE,catalog_id='a'*32, + manifest_sha256=runtime.sha(raw),manifest_bytes=len(raw), + file_count=len(manifest['files']),total_bytes=manifest['total_bytes']) + with self.assertRaisesRegex(ValueError,'closed manifest'): + runtime.verify_descriptor(descriptor,manifest) + path=Path(self.temp.name)/'old-manifest.json';path.write_bytes(raw) + with self.assertRaisesRegex(ValueError,'closed manifest'):runtime.load(path) + value=dict(ref=dict(kind='runtime',catalog_id='a'*32,id=digest(descriptor),sha256=digest(descriptor)), + descriptor=descriptor,environment=None,manifest=manifest) + self.assertEqual(decode_runtime_material(value,catalog_id='a'*32),value) + self.assertEqual(runtime.verify_retained_descriptor(descriptor,manifest),manifest) + self.assertNotIn('file_links',manifest) + changed=dict(manifest,unknown_field=[]) + with self.assertRaises(ValueError):runtime.verify_retained_descriptor(descriptor,changed) + with self.assertRaises(ValueError):runtime.verify_retained_descriptor(dict(descriptor,manifest_sha256='f'*64),manifest) + def test_manifest_link_noncanonical_encoding_and_descriptor_mismatch_are_rejected(self): manifest=runtime.inventory(self.config);raw=runtime.encoded(manifest) root=Path(self.temp.name);path=root/'manifest.json';path.write_bytes(raw) @@ -109,6 +130,39 @@ def test_manifest_link_noncanonical_encoding_and_descriptor_mismatch_are_rejecte class TorchRecipe(unittest.TestCase): setUp=fixtures.RecipePipelineTests.setUp + def test_retained_language_policy_keeps_exact_spec_and_cannot_enter_current_admission(self): + InstalledAssets.setUp(self) + manifest=runtime.inventory(self.config);raw=runtime.encoded(manifest) + identity=runtime.sha(raw) + descriptor=dict(schema_version=1,profile=runtime.PROFILE,catalog_id=self.dataset.db.dataset_id, + manifest_sha256=identity,manifest_bytes=len(raw),file_count=len(manifest['files']), + total_bytes=manifest['total_bytes']) + self.value.update(runtime=runtime.PROFILE,runtime_device='cuda',runtime_snapshot_sha256=identity) + self.recipe_raw=json.dumps(self.value).encode();(self.project/'recipe.json').write_bytes(self.recipe_raw) + self.body['expected_sha256']=runtime.sha(self.recipe_raw) + graph=self.dataset.pipelines.create_external(self.body) + with patch('torch_runtimes.TorchRuntimes.get',return_value=(descriptor,raw)), \ + patch('subprocess.Popen',side_effect=AssertionError('Retained validation executes no source')): + preflight=self.dataset.pipelines.preflight_external(graph['id'],1) + captured=self.dataset.pipelines.capture_external(graph['id'],1,preflight['material_sha256'],self.root/'capture') + material=copy.deepcopy(captured['material']);spec=copy.deepcopy(captured['spec']) + material['environment']={'LANG':'C.UTF-8'};spec['env']={'LANG':'C.UTF-8'} + descriptor_in_current_spec=spec.pop('torch_runtime') + original_material=copy.deepcopy(material);original_spec=copy.deepcopy(spec) + with self.assertRaises(WorkbenchError):contract.validate_material(material) + with self.assertRaises(WorkbenchError):contract.spec_for_material(material,self.root/'capture') + with patch('subprocess.Popen',side_effect=AssertionError('Retained spec is inert')): + self.assertEqual(contract.validate_retained_spec(material,spec,self.root/'capture'),spec) + self.assertEqual(material,original_material);self.assertEqual(spec,original_spec) + with self.assertRaises(WorkbenchError):contract.validate_spec(material,spec,self.root/'capture') + for changed_env in ({'LANG':'elsewhere'},{'LANG':'C.UTF-8','PYTHONPATH':'/unsafe'}): + bad=copy.deepcopy(material);bad['environment']=changed_env + with self.assertRaises(WorkbenchError):contract.validate_retained_spec(bad,spec,self.root/'capture') + bad=copy.deepcopy(spec);bad['env']={} + with self.assertRaises(WorkbenchError):contract.validate_retained_spec(material,bad,self.root/'capture') + bad=copy.deepcopy(spec);bad['torch_runtime']=descriptor_in_current_spec + with self.assertRaises(WorkbenchError):contract.validate_retained_spec(material,bad,self.root/'capture') + def test_capture_binds_manifest_bootstrap_and_explicit_device(self): InstalledAssets.setUp(self) manifest=runtime.inventory(self.config) diff --git a/tests/test_workspace_project_files.cjs b/tests/test_workspace_project_files.cjs index 31339b01..86582638 100644 --- a/tests/test_workspace_project_files.cjs +++ b/tests/test_workspace_project_files.cjs @@ -38,6 +38,7 @@ function mount(initial=[reply(state()),reply(records())],capabilityValue=capabil context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); context.ProjectMeasurements=()=>({setContext(){},clear(){}}); + context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,events,discoveries,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, diff --git a/tests/test_workspace_project_files_m4.cjs b/tests/test_workspace_project_files_m4.cjs index dbdb85b7..80e343ca 100644 --- a/tests/test_workspace_project_files_m4.cjs +++ b/tests/test_workspace_project_files_m4.cjs @@ -43,6 +43,7 @@ function mount(capabilityValue=capabilities){ context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); context.ProjectMeasurements=()=>({setContext(){},clear(){}}); + context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, diff --git a/torch_runtime.py b/torch_runtime.py index f664b9fe..e20cb384 100644 --- a/torch_runtime.py +++ b/torch_runtime.py @@ -199,9 +199,20 @@ def validate_descriptor(value): return value -def verify_descriptor(value,manifest): - """Bind a captured manifest to the catalog descriptor before installed asset reads.""" - validate_descriptor(value);validate_manifest(manifest) +def validate_retained_manifest(value): + """Validate original pre-link inventories solely as historical evidence. + + The measured bytes keep their original shape/hash. Current load and launch + still require validate_manifest and its explicit link inventory. + """ + if type(value) is dict and 'file_links' not in value: + validate_manifest(dict(value,file_links=[])) + return value + return validate_manifest(value) + + +def _verify_descriptor(value,manifest,validate): + validate_descriptor(value);validate(manifest) raw=encoded(manifest) require(value['manifest_sha256']==sha(raw) and value['manifest_bytes']==len(raw) and value['file_count']==len(manifest['files']) and value['total_bytes']==manifest['total_bytes'], @@ -209,6 +220,16 @@ def verify_descriptor(value,manifest): return manifest +def verify_descriptor(value,manifest): + """Bind a captured manifest to the catalog descriptor before installed asset reads.""" + return _verify_descriptor(value,manifest,validate_manifest) + + +def verify_retained_descriptor(value,manifest): + """Bind an original current or pre-link manifest without granting execution.""" + return _verify_descriptor(value,manifest,validate_retained_manifest) + + def launch(path,device,entry,arguments): require(device in ('cpu','cuda'),'explicit CPU or CUDA device') value=load(path) diff --git a/tuldok_client.py b/tuldok_client.py index 76d09cc5..fdaef8d8 100644 --- a/tuldok_client.py +++ b/tuldok_client.py @@ -417,6 +417,10 @@ def measurement_subjects(self, **arguments): def measurement_query(self, **arguments): return self._call('measurement_query', arguments) + def measurement_compare(self, **arguments): + """Compare compatible retained sources at one selected checkpoint.""" + return self._call('measurement_compare', arguments) + def measurement_inspect(self, **arguments): return self._call('measurement_inspect', arguments) @@ -702,6 +706,11 @@ def save_as(self, *, root, directory, name, binding, expected_checkpoint, reques return self._call('save_as',dict(root=root,directory=directory,name=name,binding=binding, expected_checkpoint=expected_checkpoint,request_key=request_key)) + def compose(self, *, root, directory, name, sources, binding, expected_checkpoint, request_key): + """Compose explicit retained checkpoints through the negotiated owner.""" + return self._call('compose',dict(root=root,directory=directory,name=name,sources=sources, + binding=binding,expected_checkpoint=expected_checkpoint,request_key=request_key)) + def files_inspect(self, **arguments): return self._call('files_inspect',arguments) @@ -714,6 +723,10 @@ def file_history(self, **arguments): def file_adopt(self, **arguments): return self._call('file_adopt',arguments) + def files_adopt(self, **arguments): + """Atomically adopt a bounded group of reviewed working files.""" + return self._call('files_adopt',arguments) + def file_adopt_deletion(self, **arguments): """Retain an explicitly reviewed missing note as a deletion revision.""" return self._call('file_adopt_deletion',arguments)