diff --git a/agent_api.py b/agent_api.py index 67b99453..90a534c9 100644 --- a/agent_api.py +++ b/agent_api.py @@ -1471,6 +1471,13 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non _LOOKUP = {key: operation for operation in _OPERATIONS for key in (operation['operation_id'], operation['mcp_name'])} # Explicit locally implemented versions. No remotely supplied code or routes. _CONTRACTS = {op['operation_id']: {1: copy.deepcopy(op)} for op in _OPERATIONS} +# Keep the installed v1 bounds/hash exact. Version 2 admits complete larger +# comparisons; each individual packet and every response page retain their caps. +_comparison_capacity=copy.deepcopy(_CONTRACTS['project_files.measurement_compare'][1]) +_comparison_value=_comparison_capacity['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties'] +_comparison_value['observation_count']=integer(1,250000) +_comparison_value['query']['oneOf'][0]['properties']['total']=integer(0,250000) +_CONTRACTS['project_files.measurement_compare'][2]=_comparison_capacity # v1 remains the closed Run-only current custody view. v2 additionally exposes # bounded original local native phases; imported history never populates it. _native_health=copy.deepcopy(_CONTRACTS['project_files.execution_health'][1]) @@ -1974,6 +1981,12 @@ def operation(name, version=None): return copy.deepcopy(versions[max(versions) if version is None else version]) +def measurement_comparison_row_limit(version=None): + """Return the selected immutable public contract's comparison budget.""" + selected=operation('project_files.measurement_compare',version) + return selected['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties']['observation_count']['maximum'] + + def _native_validation_result_ok(value, revision): """Check the finite public truth fields without interpreting native runtime data. @@ -2382,6 +2395,8 @@ def _private_file(path, maximum): class AgentClient: """No catalog access, automatic retries, redirects, proxies or owner identity.""" + MAX_TIMEOUT_SECONDS = 300 + def __init__(self, endpoint, token, expected_catalog_id, timeout=120): try: parsed = urlsplit(endpoint) @@ -2391,7 +2406,7 @@ def __init__(self, endpoint, token, expected_catalog_id, timeout=120): raise ValueError() if not re.fullmatch(r'[A-Za-z0-9_-]{20,4096}', token) or not re.fullmatch('[a-f0-9]{32}', expected_catalog_id): raise ValueError() - if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= 300: + if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= self.MAX_TIMEOUT_SECONDS: raise ValueError() except (ValueError, TypeError): raise AgentAPIError('Invalid fixed loopback client configuration.', 'configuration') from None @@ -3016,6 +3031,10 @@ def call(self, name, arguments, *, observer=None, observation=None): class ProjectFilesOwnerClient(AgentClient): """Explicit owner file-project access; no unrelated owner operations.""" + # Large retained projects may outlast the generic transport's five minutes. + # Only an explicit owner configuration opts into a longer absolute wait. + MAX_TIMEOUT_SECONDS = 7200 + OPERATIONS = frozenset('project_files.' + action for action in ('status', 'new', 'open', 'save', 'close', 'query', 'get', 'import_text', 'annotate')) | PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4 diff --git a/app.py b/app.py index 87c6c9a7..f818cc0b 100644 --- a/app.py +++ b/app.py @@ -919,6 +919,7 @@ def _get(self): '/workspaces.js': ('workspaces.js', 'text/javascript'), '/experiment-graph.js': ('experiment-graph.js', 'text/javascript'), '/project-progression.js': ('project-progression.js', 'text/javascript'), + '/project-model-portfolio.js': ('project-model-portfolio.js', 'text/javascript'), '/project-measurements.js': ('project-measurements.js', 'text/javascript'), '/project-measurements.css': ('project-measurements.css', 'text/css'), '/experiment-lineage.js': ('experiment-lineage.js', 'text/javascript'), diff --git a/docs/contracts/project-files.md b/docs/contracts/project-files.md index e04aa7ac..387c9bc6 100644 --- a/docs/contracts/project-files.md +++ b/docs/contracts/project-files.md @@ -105,6 +105,16 @@ annotations; human review still requires the owner. Every project action is boun to an opaque workspace generation and its expected canonical checkpoint. Use `ProjectFilesOwnerClient` only for the narrow owner project-file operations. +Its explicit `timeout` configuration may be up to 7,200 seconds for large retained +projects; the default remains 120 seconds. The absolute request watchdog still +applies and a transport timeout never authorizes an automatic mutation retry. +Generic clients retain their 300-second maximum. This wait budget is not an +execution deadline or a cancellation request. + +Browser contract tables derive from the local `agent_api` registry. Regenerate +them with `python scripts/generate_project_files_contracts.py`; add `--check` +to verify that the installed v1 pins, all supported version pins and fixed +routes match that registry. Remote schemas remain opaque discovery data. `ProjectFilesClient` wraps a negotiated client. CLI domain `file-*` commands take `--arguments` JSON files; lifecycle commands use explicit flags and select `--owner-project-files` with a diff --git a/docs/contracts/project-model-portfolio.md b/docs/contracts/project-model-portfolio.md new file mode 100644 index 00000000..d7a0a991 --- /dev/null +++ b/docs/contracts/project-model-portfolio.md @@ -0,0 +1,87 @@ +# Authored model organization + +The project workspace can display an adopted `.portfolio.json` note as an +authored guide to model fits/components. This is separate from **Experiment +progression**, which retains genuine Study/Trial ancestry. Group edges mean +authored grouping; they do not establish derivation, admission or scientific +acceptance. Relationship notes have explicit comparison/parameter meanings and +source excerpts. Missing historical Trial ancestry stays unknown. + +## Retained document + +`project_model_portfolio_v1` is a UTF-8 JSON note of at most1MiB with exactly: + +- `format`: the fixed string above; `project_id`: the owning32hex project ID; +- `title` (1–200characters), `description` (1–8,000characters), and + `authorship`: `retrospective`; +- `groups`: at most64records with exact fields `id`, `label`, `parent`, + `description`. IDs are unique, parent is null or another group, and grouping + is acyclic; +- `models`:1–1,000records with exact fields `id`, `label`, `group_id`, `aliases`, + `review`, `card`, `summary`. Each unique ID names an authored model entry, + not necessarily a complete generator, Trial or unique parameter state; +- `relationships`: at most2,000records with exact fields `from`, `to`, `kind`, + `explanation`, `evidence_model`, `excerpt`. All model references exist, endpoints + differ, and each ordered endpoint/kind tuple is unique. + +Group/model IDs contain1–160ASCII letters/digits or `._:/-`, beginning with a +letter/digit. Group labels are at most200characters, model labels/aliases500. +Each model has at most128unique aliases. A card reference has exact fields +`file_id` (64hex), `path` (a relative `notes/` path up to512characters without +empty/dot/dot-dot components), positive integer `revision`, and64hex `sha256`. +These are exact captured note references, never source-reading capabilities. + +Review is `source_reviewed` or `review_pending`. Reviewed summaries have exactly +six nonempty strings, each up to16,384characters: `method`, `purpose`, +`experiment`, `outcome`, `lesson`, `reproduction`. Pending summaries are null. +This status is an author's source-review statement, not numerical qualification. +Relationship kind is `comparison` or `parameter_derivation`; the latter points +from derived model to parameter source and must be acyclic. Explanation is at +most8,000characters and exact excerpt at most16,384. Comparisons are authored +context, not permission to combine incompatible metrics/populations. + +## Ownership and reads + +The ordinary project-file owner captures/revises the note, authenticates reads, +verifies captured UTF-8 bytes/hashes and retains previous revisions. There is no +new canonical family, SQL table, execution grant, recipe or mutation interface. +Existing `files_inspect` and `file_get` contracts are unchanged. Save/load keeps +the original guide and referenced note history. A fork whose new project/note +identities differ requires an explicitly re-authored guide; the reader refuses +foreign identities instead of retargeting them silently. + +`ProjectModelPortfolioDocument` owns the presentation-document decoder and +cross-reference/literal checks. `ProjectModelPortfolio` owns selection, +presentation and its pending reads. Its injected `read` must be the host's +already negotiated, current-authority, byte-verified project-file reader. +The host owns transport/hash validation and context identity. The controller +checks the exact model card ID/path/revision/SHA/project, and that all six +summary strings and that card's relationship excerpts occur literally in its +captured text before displaying a selected summary. Text is inert; no markup, +scripts, parameter loading, scientific scoring or source execution occurs. + +Only explicit actions read notes/cards. Discovery visits40adopted files per +page; no hidden scan/retry/polling occurs. The guide holds at most1,000models, +shows20per graph window with their ancestor groups, and reports exact total, +reviewed and filtered counts. Group/name/alias/review filters, paging and +textual relationship controls select the same entries as graph buttons. +Paragraphs wrap; exact source text remains in a disclosure. + +The host's binding/project/checkpoint/credential/generation context owns every +read. A context change, permission loss or teardown aborts pending work, clears +old content and restores focus to the host status when needed. Late replies +cannot publish into another context. Errors show their declared diagnostic and +clear old detail instead of displaying a previous model as the new selection. +Unsupported format or foreign-project binding is `unsupported`; malformed +document/reference/summary is `invalid`; transport/availability failures retain +their owning diagnostic. No success or empty-data fallback substitutes for a +failed read. + +## Verification scope + +Pure codec tests cover contract decisions using literal fixtures. Controlled +read and real-browser tests cover selection, unavailable/obsolete responses, +repeated invocation, focus, keyboard activation and wrapping on desktop/mobile. +Those fixtures do not establish real project capture or save/load. Native +acceptance additionally requires an actual adopted guide, exact original cards, +current authority, real project reads and retained checkpoint/custody evidence. diff --git a/project_composition.py b/project_composition.py index bcb8a795..53bc04f3 100644 --- a/project_composition.py +++ b/project_composition.py @@ -268,8 +268,23 @@ def allocate(family,identity,revision): records=[indexed[key] for key in sorted(indexed)] validate_source_memberships(records,snapshots,destination_project_id) archived={(r['family'],r['id']) for r in merged['archived']+fork['archived']} + # Every input has its own valid current fork. The combined records receive + # their one new current overlay only in the publication owner. Ask each + # source owner for its preservation policy before combining those roots; + # independent editable forks introduce no new payload identities. + plans={} + inventories=[fork['artifacts']] + inventories.extend(artifact_plan(source['records']) for source in snapshots[1:]) + for inventory in inventories: + for edge in inventory: + sha=edge['declaration']['sha256'];previous=plans.get(sha) + if previous is not None and previous['declaration']!=edge['declaration']: + _fail('Composition source artifact declarations conflict.') + if previous is None or edge['preservation']=='unique': + plans[sha]=copy.deepcopy(edge) return dict(records=records,mapping=copy.deepcopy(fork['mapping']), - archived=[dict(family=f,id=i) for f,i in sorted(archived)],artifacts=artifact_plan(records)) + archived=[dict(family=f,id=i) for f,i in sorted(archived)], + artifacts=[plans[sha] for sha in sorted(plans)]) def publish_composition(primary_store, primary_artifacts, retained_sources, trusted_root_fd, directory, diff --git a/project_custody.py b/project_custody.py index 2e0a97aa..1a188f1d 100644 --- a/project_custody.py +++ b/project_custody.py @@ -197,7 +197,7 @@ def _directory(self, create=False): finally:os.close(parent) def _binding(self, store, projection_id): - snapshot=store.snapshot() + snapshot=store.summary() with store.root_directory() as descriptor: info=os.fstat(descriptor) return dict(host_catalog_id=self.host_catalog_id,project_id=snapshot['project_id'], @@ -247,7 +247,7 @@ def __init__(self, owner, store, binding): persisted=value is not None if value is None: value=dict(format='project_local_custody_v1',**binding, - project_name=store.snapshot()['name'],selected_checkpoint=store.snapshot()['checkpoint'], + project_name=store.summary()['name'],selected_checkpoint=store.summary()['checkpoint'], attempted=None,runs=[],producers=[],runtime_effects=[],package_effects=[]) if any(value[key]!=item for key,item in binding.items()):_fail('Original physical custody binding differs.') self.value=validate(value) diff --git a/project_data.py b/project_data.py index 4a48dd38..ca70bab0 100644 --- a/project_data.py +++ b/project_data.py @@ -1001,7 +1001,7 @@ def _publish(self, raw, kind, role, media_type): declaration = dict(format='project_artifact_v1', sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) self.artifacts.publish_bytes(raw, declaration) self._pending_artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=self.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=self.session.summary['project_id'], sha256=declaration['sha256']), media_type=media_type, kind=kind, role=role, preservation='unique', required_for=['inspect', 'release']) def _require_record(self, record_id, *, extended=False): @@ -1028,7 +1028,7 @@ def _mutate(self, operation, target, body, expected_checkpoint, key, invoke): fail('Data mutation body must be an object.') self._permission(operation, target, body) with self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() if operation in ('data.annotate','data.correct_rights_note','data.save_image_source','data.delete_image_source'): self._require_record(target,extended=operation!='data.annotate') legacy = self._legacy_replay(operation, target, body, key) @@ -1158,10 +1158,10 @@ def freeze_release(self, body, expected_checkpoint, key): def _read(self, action, invoke, target=None): with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() self._sync_sources() result = invoke() - self.session.check() + self.session.check_current() return result def query(self, options=None): @@ -1399,7 +1399,7 @@ def restore(self, snapshot=None): self.db.execute('INSERT INTO saved_searches VALUES (%s,%s,%s,%s,%s,%s)', (record_id,r['name'],r['revision'],encode_retained(r['criteria']),r['created_at'],r['updated_at'])) elif family == 'dataset_exposure': self.db.execute('INSERT INTO workspace_exposures VALUES (%s,%s,%s,%s,%s,%s,%s)', tuple(doc[k] for k in ('id','record_id','source_hash','reason','run_id','release_id','created_at'))) - self.session.check() + self.session.check_current() self._materialize_images_and_releases(owned) return snapshot diff --git a/project_dispatch.py b/project_dispatch.py index 6c9420a2..de519135 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -149,7 +149,7 @@ def _data(projection, action, a): def _definitions(projection, family, a): docs = projection.materials._documents() values = [doc['current'] for (kind, _), doc in docs.items() - if kind == family and doc['origin_catalog_id'] == projection.session.snapshot['project_id']] + if kind == family and doc['origin_catalog_id'] == projection.session.summary['project_id']] return _page(sorted(values, key=lambda item: item['id']), a) @@ -370,7 +370,16 @@ def _m4(projection,action,a): if action=='file_adopt_deletion': return projection.files.adopt_deletion(_body(a),a['expected_checkpoint'],a['request_key']) with session.access.lock,session.lock,session.db.lock: - session.access.require('inspect','project_files.'+action);snapshot=session.check() + session.access.require('inspect','project_files.'+action) + if action=='fork_info': + snapshot=session.check() + from project_fork import decode_fork + overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) + return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], + mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, + copied_count=len(overlay['artifacts']['copied']) if overlay else 0, + missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) + session.check_current() if action=='files_inspect':return _page(projection.files.inspect()['items'],a) if action=='file_get':return projection.files.read(a['file_id'],a.get('revision')) if action=='file_history': @@ -390,13 +399,6 @@ def _m4(projection,action,a): items=[dict(item,requires_reselection=[]) for item in value['editable']] items.extend(dict(family=item['family'],id=item['id'],requires_reselection=item['fields']) for item in value['requires_reselection']) return _page(sorted(items,key=lambda item:(item['family'],item['id'])),a) - if action=='fork_info': - from project_fork import decode_fork - overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) - return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], - mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, - copied_count=len(overlay['artifacts']['copied']) if overlay else 0, - missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) raise AssertionError('Closed M4 dispatch is incomplete.') diff --git a/project_execution.py b/project_execution.py index ca5c36d0..0e4941c6 100644 --- a/project_execution.py +++ b/project_execution.py @@ -52,7 +52,7 @@ def publication_started(self): def confirmed(self): if not self.started:raise RuntimeError('File publication must have a durable start before confirmation.') - execution=self.execution;execution.session.check() + execution=self.execution;execution.session.check_current() if execution.db.in_transaction: raise RuntimeError('File publication confirms only after its canonical and outer SQL acknowledgement.') if execution.custody is not None: @@ -164,7 +164,7 @@ def register_custody(self, identity, handle): def before_launch(self, identity): with self.db.lock: - self.session.check();run=self.runs._load(identity);self._proof(run) + self.session.check_current();run=self.runs._load(identity);self._proof(run) self.runtime_access.require_job('runs',identity,run['admission_id'],run['request_sha256']) if self.custody is not None: with self._journal_guard():self.custody.launch_intent(identity) @@ -176,7 +176,7 @@ def _publication_intent(self, before, records, key, sha): def _local_owner(self, *, recover=True): """Construct execution only after the caller checked exact authority.""" - self.session.check() + self.session.check_current() if self._runs is None: from runs import Runs from pipeline import Pipelines @@ -296,7 +296,7 @@ def _restore_file_writer(self, kind, request_key, input_sha256, content_sha256, authority.require(scope,action,owner_only=True) if self.db.in_transaction: raise RuntimeError('File reconciliation requires no outer SQL transaction.') - if self.session.snapshot['checkpoint']!=expected_checkpoint: + if self.session.summary['checkpoint']!=expected_checkpoint: raise WorkbenchError('Expected current file checkpoint changed.','stale_project',409) if self.custody is None: raise WorkbenchError('No original physical file custody exists.','unavailable',409) @@ -398,7 +398,7 @@ def _producer_authority(self, family, identity, admission_id, request_digest): def register_producer(self, family, identity, admission_id, request_digest): with self.db.lock: - self.session.check();self._producer_authority(family,identity,admission_id,request_digest) + self.session.check_current();self._producer_authority(family,identity,admission_id,request_digest) if self.custody is not None: with self._journal_guard(): if not self.custody.has_pending:self.custody.acknowledge(self.session.snapshot) @@ -406,7 +406,7 @@ def register_producer(self, family, identity, admission_id, request_digest): def producer_effect_intent(self, family, identity): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -416,7 +416,7 @@ def producer_effect_intent(self, family, identity): def update_producer_outcome(self, family, identity, provider_outcome, worker_drained): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -464,7 +464,7 @@ def _authorize_observation(self): # The domain owner has already checked submit/cancel authority; # recheck live host credentials without inventing a worker owner. self.runtime_access.require('inspect', 'runs.observe', identity) - if proof['admission']['catalog_id'] != self.session.snapshot['project_id']: + if proof['admission']['catalog_id'] != self.session.summary['project_id']: raise AccessError('Local custody belongs to another project.') def _capture_run(self, identity): @@ -591,7 +591,7 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt def preflight_external(self, graph_id, revision): self.runtime_access.require('inspect', 'pipeline.preflight_external', graph_id) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() return self.projection.pipelines.preflight_external(graph_id, revision) def native_preflight(self,graph_id,revision,expected_checkpoint,key): @@ -615,7 +615,7 @@ def native_submit(self,body,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.submit',body['graph_id'],owner_only=True) effective=dict(body,actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.submit',body['graph_id'],effective,expected_checkpoint,key) if replay is not None:return replay with self.projection.native_preparation.current_graph(self.projection.pipelines,body['graph_id'], @@ -626,14 +626,14 @@ def invoke(): context.consume_admission() owner=self._local_owner();public=owner.submit(dict(body,request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.submit',body['graph_id'],effective,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.submit',body['graph_id'],effective,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_retry(self,run_id,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.retry',run_id,owner_only=True) body=dict(actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.retry',run_id,body,expected_checkpoint,key) if replay is not None:return replay if self._runs is None:raise WorkbenchError('Retry requires the actual current local Run owner.','unavailable',409) @@ -646,7 +646,7 @@ def invoke(): context.consume_admission() owner=self.runs;public=owner.retry(run_id,dict(request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.retry',run_id,body,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.retry',run_id,body,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_reconcile(self,phase_id,expected_checkpoint): @@ -655,9 +655,9 @@ def native_reconcile(self,phase_id,expected_checkpoint): def submit_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require_external_submission() with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: # A retained request never recreates a local Run/admission or wakes @@ -678,9 +678,9 @@ def invoke(): def approve_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require('approve-execution', 'runs.approve_external', owner_only=True) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: value = prior['result'] diff --git a/project_extraction.py b/project_extraction.py index 82db29c7..461b0d0f 100644 --- a/project_extraction.py +++ b/project_extraction.py @@ -69,7 +69,7 @@ def __init__(self, source): self.payload_gaps = [] self.deleted_intents = set() self.session = SimpleNamespace(snapshot=source.snapshot, db=self.db, - artifacts=source.artifacts, check=lambda: None) + artifacts=source.artifacts, check_current=lambda: None) # Workbench and each metadata-view constructor only assign references; # none runs SQL, creates directories or constructs Dataset/Jobs/Runs. # Never call get/history: those public owners may enroll images. diff --git a/project_files.py b/project_files.py index 08829a6a..ad0f6b3f 100644 --- a/project_files.py +++ b/project_files.py @@ -5,6 +5,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import errno import hashlib import os @@ -128,14 +129,14 @@ def _result(envelope,changed): return result -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): if operation=='files.adopt_many': closed(result,{'items','changed_count'},'batch adoption result') if type(result['items']) is not list or not 1<=len(result['items'])<=MAX_ADOPT_ITEMS: fail('Invalid batch adoption result size.') seen=set();projects=set() for item in result['items']: - validate_operation_result('files.adopt',item,records) + _validate_result('files.adopt',item,lookup) if item['id'] in seen:fail('Batch adoption result repeats a file.') seen.add(item['id']);projects.add(item['project_id']) if (len(projects)!=1 or type(result['changed_count']) is not int @@ -153,14 +154,34 @@ def validate_operation_result(operation,result,records=None): else:sha(result['sha256']) _date(result['adopted_at']) if type(result['revision']) is not int or not 1<=result['revision']<=2**53-1 or type(result['bytes']) is not int or not 0<=result['bytes']<=MAX_FILE_BYTES:fail('Invalid adoption result bounds.') - if records is not None: - owned=decode_records(records) - envelope=next((r for r in owned if r['id']==result['id']),None) + if lookup is not None: + envelope=lookup(result['id']) if envelope is None or not any((value.get('deleted') is True)==deleted and all(value[key]==result[key] for key in ('revision','sha256','bytes','adopted_at')) for value in envelope['document']['history']): fail('Adoption result is not an exact retained working-file revision.') return True +def receipt_validator(records): + """Own one decoded candidate's file histories for its receipt checks. + + The returned validator has no live Session or publication authority. It + owns copied records, so caller mutation cannot change this fixed view. + A different candidate requires a new validator. + """ + owned = {record['id']: record for record in decode_records(records)} + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + # Keep standalone shape/error ordering, and share only this call's lookup + # across batch items. There is no retained cache between invocations. + @cache + def owned(): + return {record['id']: record for record in decode_records(records)} + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class _WorkingError(Exception): def __init__(self,state):self.state=state @@ -231,39 +252,51 @@ def _read(root,parts): os.close(parent) +def inspect_captured(records, project_id, root): + """Observe captured working files using only a locked project capability.""" + return _inspect_validated(decode_records(records), project_id, root) + + +def _inspect_validated(records, project_id, root): + """Consume full-decoder output or fresh values from a validated Session.""" + owned=[r for r in records if r['document']['project_id']==project_id] + items=[] + for envelope in sorted(owned,key=lambda r:r['document']['path']): + doc=envelope['document'];value=doc['current'] + observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], + expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) + try: + parts=_path(doc['kind'],doc['path']) + if value.get('deleted') is True and _missing(root,parts): + observation['state']='deleted' + else: + if value.get('deleted') is True:observation['state']='changed' + _,observed,size=_read(root,parts) + observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') + except _WorkingError as error: + if value.get('deleted') is True: + observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) + else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) + items.append(observation) + return items + + class ProjectFiles: def __init__(self,session): self.session,self.access,self.db,self.store=session,session.access,session.db,session.store def inspect(self): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.inspect');snapshot=self.session.check() - owned=[r for r in decode_records(snapshot['records']) if r['document']['project_id']==snapshot['project_id']];items=[] + self.access.require('inspect','files.inspect');snapshot=self.session.check_records(FAMILY) with self.store.root_directory() as root: - for envelope in sorted(owned,key=lambda r:r['document']['path']): - doc=envelope['document'];value=doc['current'] - observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], - expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) - try: - parts=_path(doc['kind'],doc['path']) - if value.get('deleted') is True and _missing(root,parts): - observation['state']='deleted' - else: - if value.get('deleted') is True:observation['state']='changed' - _,observed,size=_read(root,parts) - observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') - except _WorkingError as error: - if value.get('deleted') is True: - observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) - else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) - items.append(observation) - self.session.check() + items=_inspect_validated(snapshot['records'],snapshot['project_id'],root) + self.session.check_current() return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) def read(self,record_id,revision=None): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.read',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.read',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) values=envelope['document']['history'] if revision is None:value=values[-1] @@ -273,8 +306,8 @@ def read(self,record_id,revision=None): def history(self,record_id): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.history',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.history',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) return copy.deepcopy(envelope['document']['history']) @@ -293,7 +326,7 @@ def materialize_captured(records,project_id,root_fd): def _adoption_intent(self,body): closed(body,{'kind','path','expected_revision','expected_sha256'},'working-file adoption') - parts=_path(body['kind'],body['path']);project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) + parts=_path(body['kind'],body['path']);project_id=self.session.summary['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 0<=body['expected_revision']<=2**53-1:fail('Invalid expected adoption revision.') return dict(body),parts,record_id @@ -309,12 +342,12 @@ def _capture_adoption(self,intent,prior,root): value=dict(revision=revision+1,sha256=observed,bytes=size,text=text,adopted_at=timestamp(),actor_id=actor.actor_id) history=copy.deepcopy(prior['document']['history']) if prior else [];history.append(value) envelope=dict(family=FAMILY,id=record_id,document=dict(format=prior['document']['format'] if prior else FORMAT, - project_id=self.session.snapshot['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), + project_id=self.session.summary['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), history=history,origin=copy.deepcopy(prior['document']['origin']) if prior else None)) return _result(envelope,True),envelope,size def adopt(self,body,expected_checkpoint,key): - intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.snapshot['project_id'] + intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.summary['project_id'] pending=[] with self.access.lock: self.access.require('edit','files.adopt',record_id) @@ -343,7 +376,7 @@ def adopt_many(self,body,expected_checkpoint,key): intents=[self._adoption_intent(item) for item in body['items']] ids=[intent[2] for intent in intents] if len(set(ids))!=len(ids):fail('Batch adoption repeats a working-file path.') - body=dict(items=[intent[0] for intent in intents]);project_id=self.session.snapshot['project_id'] + body=dict(items=[intent[0] for intent in intents]);project_id=self.session.summary['project_id'] target=digest('\0'.join(ids).encode());pending=[] with self.access.lock: for identity in ids:self.access.require('edit','files.adopt',identity) @@ -370,7 +403,7 @@ def adopt_deletion(self,body,expected_checkpoint,key): closed(body,{'path','expected_revision','expected_sha256'},'note deletion adoption') parts=_path('note',body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 1<=body['expected_revision']<=2**53-1:fail('Invalid expected note revision.') - project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] + project_id=self.session.summary['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] with self.access.lock: self.access.require('edit','files.adopt_deletion',record_id) def invoke(): diff --git a/project_fork.py b/project_fork.py index 7aec1a44..f35b9812 100644 --- a/project_fork.py +++ b/project_fork.py @@ -174,10 +174,16 @@ def decode_fork(records, *, project_id=None): return copy.deepcopy(overlays[current]) -def active_records(records, *, project_id=None): +def active_records(records, *, project_id=None, families=None): + """Own selected active copies while validating the complete fork history.""" + if families is not None: + if type(families) is not frozenset: + _fail('Active record families must be an explicit frozenset.') + for family in families:_identifier(family,'Active record family') overlay = decode_fork(records, project_id=project_id) archived = set() if overlay is None else {(item['family'], item['id']) for item in overlay['archived']} - return [copy.deepcopy(item) for item in records if (item['family'], item['id']) not in archived] + return [copy.deepcopy(item) for item in records + if (families is None or item['family'] in families) and (item['family'], item['id']) not in archived] def _inside(candidate_fd, ancestor_fd): @@ -279,6 +285,98 @@ def publish_fork(source_store, source_artifacts, trusted_root_fd, relative, *, r validate_candidate=lambda records,identity,_artifacts:validate_candidate(records,identity)) +def recover_committed_publication(trusted_root_fd, staging, relative, *, root_path, + expected_project_id, expected_checkpoint, + expected_request, quiescent): + """Publish one explicitly selected committed stage without rebuilding it. + + This operator recovery accepts a frozen historical snapshot, not a new read + of its original sources. The caller owns current filesystem authority and + cessation of the former process. The store's exclusive root lock, complete + domain validation, exact original request/commit receipt, payload custody, + and clean captured files are required before a no-overwrite rename. No SQL + projection, source permission, source runtime or worker is restored here. + After an uncertain rename, repeat this exact recovery to inspect the final + destination; never repeat composition or create another stage. + """ + if quiescent is not True: + _fail('Recovery requires proven cessation of the former publication owner.', 'busy', 409) + _uuid(expected_project_id);_sha(expected_checkpoint) + stage_parts=_relative_parts(staging);parts=_relative_parts(relative) + if (stage_parts[:-1]!=parts[:-1] + or not re.fullmatch(r'\.tuldok-fork-[a-f0-9]{32}',stage_parts[-1]) + or stage_parts==parts): + _fail('Recovery requires an explicit fork stage beside its original destination.') + if type(expected_request) is not dict or expected_request.get('directory')!=relative: + _fail('Recovery requires the exact original publication request.') + key=expected_request.get('request_key') + if type(key) is not str or not REQUEST_KEY.fullmatch(key): + _fail('Recovery requires the original publication request key.') + digest=_digest(expected_request) + published=False + try: + with anchored_directory(trusted_root_fd,'/'.join(parts[:-1]),root_path=root_path) as parent: + def exists(name): + try:os.stat(name,dir_fd=parent,follow_symlinks=False);return True + except FileNotFoundError:return False + stage_exists,final_exists=exists(stage_parts[-1]),exists(parts[-1]) + if stage_exists and final_exists: + _fail('Both stage and destination exist; neither was replaced.', 'conflict', 409) + if not stage_exists and not final_exists: + _fail('Neither the selected stage nor its destination is available.', 'unavailable', 409) + selected=relative if final_exists else staging + with ProjectStore.open_at(trusted_root_fd,selected,root_path=root_path) as store: + snapshot=store.snapshot() + if (snapshot['project_id']!=expected_project_id + or snapshot['checkpoint']!=expected_checkpoint): + _fail('Recovery project identity or checkpoint differs.', 'stale', 409) + overlay=decode_fork(snapshot['records'],project_id=expected_project_id) + if overlay is None:_fail('Recovery requires a committed independent publication.') + from project_composition import decode_records, composition_input + composition=decode_records(snapshot['records'],project_id=expected_project_id).get(expected_project_id) + observed=(composition_input(composition['sources'],composition['directory'],composition['name'],composition['request_key']) + if composition is not None else _input(overlay['source'],overlay['directory'],overlay['name'],overlay['request_key'])) + if observed!=expected_request or store.reconcile(key,digest) is None: + _fail('Committed stage does not match the original request and receipt.', 'conflict', 409) + from project_projection import validate_project_records + from project_files import inspect_captured + declarations=_declarations(snapshot['records']) + with ProjectArtifacts(store) as artifacts: + validate_project_records(snapshot['records'],project_id=expected_project_id,artifacts=artifacts) + checks=artifacts.status_many([declarations[identity] for identity in overlay['artifacts']['copied']]) + if any(item['state']!='verified' for item in checks): + _fail('Committed publication payloads are missing or corrupt.', 'unavailable', 409) + with store.root_directory() as project_root: + if any(item['state'] not in ('unchanged','deleted') + for item in inspect_captured(snapshot['records'],expected_project_id,project_root)): + _fail('Committed stage has changed or missing working files.', 'working_files_changed', 409) + store.save(expected_checkpoint) + if not final_exists: + # Recheck the selected directory entry while the store root + # remains locked; an external rename must not substitute it. + with store.root_directory() as project_root: + entry=os.stat(stage_parts[-1],dir_fd=parent,follow_symlinks=False) + actual=os.fstat(project_root) + if (entry.st_dev,entry.st_ino)!=(actual.st_dev,actual.st_ino): + _fail('Selected stage directory changed.', 'stale', 409) + _rename_noreplace(parent,stage_parts[-1],parts[-1]);published=True + os.fsync(parent) + return dict(project_id=expected_project_id,checkpoint=expected_checkpoint, + directory=relative,request_key=key,reconciled=final_exists, + payloads_copied=0,selected=False) + except BaseException as error: + # Anchored directory owners also translate IO failures. Preserve the + # uncertain post-rename outcome through either exception representation. + if published: + raise ProjectForkError('Committed publication was renamed but acknowledgement is uncertain; reconcile this exact recovery.', + 'unknown',503) from error + if isinstance(error,OSError): + code='conflict' if error.errno==errno.EEXIST else 'unavailable' + raise ProjectForkError('Committed publication recovery is unavailable; no stage was rebuilt.', + code,503 if code!='conflict' else 409) from error + raise + + def _publish_project(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, name, expected_checkpoint, request_key, quiescent, build_candidate, validate_candidate, retained_sources=()): diff --git a/project_history.py b/project_history.py index 892ae3c2..b69eccce 100644 --- a/project_history.py +++ b/project_history.py @@ -750,17 +750,29 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): from project_material import ProjectMaterial materials=ProjectMaterial(session) self.materials=materials;self.authorize_disclosure=authorize_disclosure + self._decoded_checkpoint=None + self._decoded_facts=None def _facts(self): - self.session.check() + self.session.check_current() + summary=self.session.summary + checkpoint=(summary['project_id'],summary['checkpoint']) + if self._decoded_checkpoint==checkpoint: + return self._decoded_facts records=self.session.snapshot['records'] - return decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) + facts=decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) + # Current canonical bytes are guarded on every call. Only this owner's + # private decoded metadata is retained; returned values and permissions + # remain the responsibility of each read below. + self._decoded_facts=facts + self._decoded_checkpoint=checkpoint + return facts def _authorize(self,action,target=None): self.access.require('inspect',action,target) def _origin(self,origin): - return self.session.snapshot['project_id'] if origin is None else origin + return self.session.summary['project_id'] if origin is None else origin def _record(self,collection,identity,origin=None): origin=self._origin(origin);identifier(origin,'Origin catalog');identifier(identity) @@ -1163,7 +1175,7 @@ def create_study(self,body,expected_checkpoint,request_key): closed(body,('name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']);policy=validate_retained_policy(body['policy']) def build(facts): - origin=self.session.snapshot['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) + origin=self.session.summary['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) prior=next((key for key,value in facts.policies.items() if key[0]==origin and key[3]==sha),None) policy_id=prior[1] if prior else uuid.uuid4().hex ref={'kind':'selection_policy','catalog_id':origin,'id':policy_id,'revision':1,'sha256':sha} @@ -1180,7 +1192,7 @@ def save_study(self,study_id,body,expected_checkpoint,request_key): closed(body,('revision','name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['policy'],'selection_policy',origin) policy=facts.policies.get((origin,ref['id'],ref['revision'],ref['sha256'])) @@ -1201,7 +1213,7 @@ def prepare_trial_from_material(self,study_id,body,expected_checkpoint,request_k if request_key not in operation_records(self.session.snapshot['records']): self.materials.recipe(body['material'],verify=True) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['material'],'recipe_material',origin);self.materials.recipe(ref) @@ -1224,7 +1236,7 @@ def select_candidate(self,study_id,body,expected_checkpoint,request_key): identifier(body['trial_id']);identifier(body['run_id']) # Expensive semantic proof precedes the write transaction. All exact # associations and selected policy are checked again under publication. - origin=self.session.snapshot['project_id'] + origin=self.session.summary['project_id'] self.access.require('edit','studies.select_candidate',study_id) evidence=(self.evidence(body['run_id'],{},origin_catalog_id=origin) if request_key not in operation_records(self.session.snapshot['records']) else None) @@ -1254,7 +1266,7 @@ def build(facts): def record_lineage(self,body,expected_checkpoint,request_key): closed(body,('trial','intent'));body=checked_json(body,INTENT_LIMIT+4096) def build(facts): - origin=self.session.snapshot['project_id'];trial=facts.trial_ref(body['trial']) + origin=self.session.summary['project_id'];trial=facts.trial_ref(body['trial']) if body['trial']['catalog_id']!=origin:_invalid('Inherited Trials remain immutable historical evidence.','conflict') doc=copy.deepcopy(facts.trials[(origin,trial['id'])]) if doc['lineage'] is not None:_invalid('Trial lineage is already recorded and immutable.','conflict') diff --git a/project_material.py b/project_material.py index 194e91a5..fbfa0189 100644 --- a/project_material.py +++ b/project_material.py @@ -359,7 +359,7 @@ def __init__(self, session): self.owner_access = _MaterialAccess(session) if hasattr(session,'access') else None def _documents(self): - self.session.check() + self.session.check_current() return decode_records(self.session.snapshot['records']) def read_view(self, records): @@ -369,7 +369,7 @@ def read_view(self, records): live publication, verification or execution capability; later reads must select their own checkpoint and construct a new view. """ - self.session.check() + self.session.check_current() return MaterialSnapshot(records) @_boundary @@ -381,7 +381,7 @@ def intents(self): from project_material_fork import is_intent documents = self._documents() active = {(r['family'], r['id']) for r in active_records( - self.session.snapshot['records'], project_id=self.session.snapshot['project_id'])} + self.session.snapshot['records'], project_id=self.session.summary['project_id'])} return [dict(family=family, id=locator, current=copy.deepcopy(doc['current']), fork_origin=copy.deepcopy(doc['fork_origin'])) for (family, locator), doc in sorted(documents.items()) @@ -392,7 +392,7 @@ def intent_capabilities(self): intents = self.intents() unresolved = {(r['family'],r['id']) for r in intents} from project_fork import active_records - active = active_records(self.session.snapshot['records'], project_id=self.session.snapshot['project_id']) + active = active_records(self.session.snapshot['records'], project_id=self.session.summary['project_id']) return dict(requires_reselection=[dict(family=r['family'],id=r['id'],fields=r['current']['required_reselection']) for r in intents], editable=[dict(family=r['family'],id=r['id']) for r in active if r['family'] in EDITABLE and (r['family'],r['id']) not in unresolved]) @@ -461,7 +461,7 @@ def snapshot_ref(self, project_id, manifest_sha256, *, catalog_id=None, verify=F return copy.deepcopy(doc['value']) def capture_definition(self, family, value, before_records, *, action, actor_id=None, audit_facts=None): - catalog_id = self.session.snapshot['project_id'] + catalog_id = self.session.summary['project_id'] value = _definition(family, value, catalog_id) key = catalog_id + '.' + value['id'] prior = decode_records(before_records).get((family, key)) diff --git a/project_measurements.py b/project_measurements.py index 2800b50b..e64a307e 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -6,6 +6,7 @@ """ import copy from datetime import datetime +from functools import cache, partial from project_artifacts import validate_declaration from project_material import decode_records as material_records @@ -101,33 +102,55 @@ def decode_records(records, *, artifacts=None): return result -def validate_operation_result(operation, result, records=None): +def _validate_result(operation, result, lookup=None): if operation != 'measurements.import': return False _document(result) - if records is not None: + if lookup is not None: identity = result['origin_catalog_id'] + '.' + result['id'] - if decode_records(records).get(identity) != result: + if lookup(identity) != result: raise WorkbenchError('Measurement receipt has no exact retained source.') return True +def receipt_validator(records, *, artifacts=None): + """Own one candidate's validated source bindings and packet custody. + + Decode and copy every retained measurement once, including current available + packet bytes when supplied. The returned receipt checker is fixed to those + facts; it carries no live source or publication capability. + """ + owned = decode_records(records, artifacts=artifacts) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return decode_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class ProjectMeasurements: def __init__(self, session): self.session, self.artifacts = session, session.artifacts self.access, self.db = session.access, session.db def _documents(self): - return decode_records(self.session.snapshot['records']) + # Session adoption/publication already validates the complete candidate. + # Verify current canonical bytes, then own only the measurement family. + records = self.session.check_records(FAMILY)['records'] + return {item['id']: item['document'] for item in records} def import_packet(self, raw, source, expected_checkpoint, key): """Called only after SourceAdapter's explicit current rooted capture.""" with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'measurements.import') - self.session.check() + self.session.check_current() table = MeasurementTable(raw) blob = dict(sha256=table.sha256, bytes=len(raw)) - origin = self.session.snapshot['project_id'] + origin = self.session.summary['project_id'] request = dict(blob=blob, source=copy.deepcopy(source)) def invoke(): @@ -152,11 +175,11 @@ def sources(self, *, offset=0, limit=40): raise WorkbenchError('Invalid measurement source page.') with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', 'measurements.sources') - self.session.check() + self.session.check_current() docs = sorted(self._documents().values(), key=lambda d: (d['origin_catalog_id'], d['id'])) items = [dict(doc, availability=self.artifacts.inspect(declaration(doc['blob']))) for doc in docs[offset:offset+limit]] - self.session.check() + self.session.check_current() self.access.require('inspect', 'measurements.sources') return self._bounded(dict(items=items, total=len(docs), offset=offset, limit=limit)) @@ -172,7 +195,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): target = origin_catalog_id + '.' + packet_sha256 with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() doc = self._documents().get(target) if doc is None: raise WorkbenchError('Measurement source is unavailable.', 'not_found', 404) @@ -181,7 +204,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): if table.metadata()['title'] != doc['title']: raise WorkbenchError('Measurement title differs from the exact retained packet.') value = self._bounded(dict(source=doc, value=select(table))) - self.session.check() + self.session.check_current() self.access.require('inspect', action, target) return value @@ -215,7 +238,7 @@ def compare(self, sources, metric_id=None, **selection): with self.access.lock, self.session.lock, self.db.lock: for target in targets: self.access.require('inspect', 'measurements.compare', target) - self.session.check() + self.session.check_current() documents = self._documents() if any(target not in documents for target in targets): raise WorkbenchError('A comparison source is unavailable.', 'not_found', 404) @@ -231,7 +254,7 @@ def compare(self, sources, metric_id=None, **selection): tables.append((locator, table)) value = MeasurementComparison(tables).read(metric_id, **selection) result = self._bounded(dict(sources=chosen, value=value)) - self.session.check() + self.session.check_current() for target in targets: self.access.require('inspect', 'measurements.compare', target) return result diff --git a/project_native_execution.py b/project_native_execution.py index 832a990f..5895e4d7 100644 --- a/project_native_execution.py +++ b/project_native_execution.py @@ -452,7 +452,7 @@ def identity(info): def _publish(self, document, state, *, helper=None, receipt_sha256=None): with self.db.lock,self.db.transaction(): - self.session.check();actor=self._authorize(document['phase_id']) + self.session.check_current();actor=self._authorize(document['phase_id']) document['events'].append(dict(state=state,at=timestamp(),actor_id=actor.actor_id, helper=copy.deepcopy(helper),receipt_sha256=receipt_sha256)) self._dirty[document['phase_id']]=copy.deepcopy(document) @@ -473,7 +473,7 @@ def reconcile(self,phase_id,expected_checkpoint): execution=self.execution;journal=execution.custody if journal is None:_fail('There is no original local native custody.','unavailable') with self.db.lock: - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') local=journal.refresh() selected=next((row for row in local.get('native_phases',[]) if row['phase_id']==phase_id),None) if selected is None:_fail('A retained phase alone cannot select current native custody.','unavailable') @@ -501,7 +501,7 @@ def reconcile(self,phase_id,expected_checkpoint): if self._active:_fail('Current native owner has not finished its bounded observation.','busy') with execution.access.lock,self.session.lock,self.db.lock: self.access.require('cancel','native_execution.reconcile',phase_id,owner_only=True) - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') if journal.refresh()!=local:_fail('Original native custody changed while draining.','stale_project') binding=journal.binding if (binding['project_id']!=self.db.dataset_id or binding['projection_id']!=self.db.projection_id @@ -788,7 +788,7 @@ def _probe_result(recipe, inputs, evidence): def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, run_id=None, job=None, phase_id=None): if self.db.in_transaction:_fail('Native phase cannot launch inside a metadata transaction.','busy') with self.db.lock: - self.session.check() + self.session.check_current() if run_id is None and self._foreground_phase is not None: _fail('Another explicit native preparation phase is active.','busy') if not self.db.claim_runtime('project_native_phases'):_fail('Another current native phase owner is active.','busy') @@ -818,7 +818,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru self._publish(document,'prepared') if self.execution.custody is None:_fail('Native launch requires its original physical custody journal.','unavailable') with self.db.lock,self.execution._journal_guard(): - self._dispatch_authorized(phase_id);self.session.check() + self._dispatch_authorized(phase_id);self.session.check_current() if not self.execution.custody.has_pending:self.execution.custody.acknowledge(self.session.snapshot) self.execution.custody.native_prepared(document) self.execution.custody.native_launch_intent(phase_id) @@ -847,7 +847,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru # Recheck actual authority after both durable ACKs. A # saved GO intent is never permission to retry this send. with self.db.lock: - self.session.check();self._dispatch_authorized(phase_id) + self.session.check_current();self._dispatch_authorized(phase_id) if not self.execution.custody.has_pending:_fail('Native local custody disappeared.','unknown') prepared.check_current_runtime() handle.go(handle.ready) diff --git a/project_native_preparation.py b/project_native_preparation.py index 85a0b1be..24113f63 100644 --- a/project_native_preparation.py +++ b/project_native_preparation.py @@ -188,7 +188,7 @@ def validate_operation_result(operation,result,records=None): _fail('Native intent source or frozen input differs from its graph.','invalid') from types import SimpleNamespace from project_material import ProjectMaterial - retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check=lambda:None)) + retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None)) if result['model'] is not None:retained.model(result['model']) if result['composition'] is not None: composition=retained.recipe(result['composition'])['material'] @@ -230,7 +230,7 @@ def __init__(self,adapter,owner,graph,model,composition,purpose,key): def _check(self,recipe): if not self._active or _CURRENT.get() is not self or recipe!=self.graph['recipe']: _fail('An explicit current native preparation is required.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.probe_native',recipe,owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Native preparation caller changed.','conflict') if (os.environ.get('TULDOK_TRAINING_PYTHON') or sys.executable)!=self.configured_executable: @@ -262,7 +262,7 @@ def probe(self,owner,recipe): if self._probe is None: if self.adapter.db.in_transaction:_fail('Native probe cannot run inside a metadata transaction.','unsupported') self.authorize_probe(recipe) - result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.snapshot['checkpoint'], + result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.summary['checkpoint'], request_key=self.key+'.probe',preparation=self) self._probe=copy.deepcopy(result['runtime']);self.phase=copy.deepcopy(result['phase']) return copy.deepcopy(self._probe) @@ -297,7 +297,7 @@ def __init__(self,adapter,owner,graph): def consume_admission(self): if not self._active or self._consumed:_fail('Current integrity qualification has already been consumed.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.snapshot',self.graph['id'],owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Current integrity caller changed.','conflict') if self.owner.get(self.graph['id'])!=self.graph or self.owner.definition_identity(self.recipe)!=self.definition: @@ -333,8 +333,8 @@ def _scope(self,owner,graph,model,composition,purpose,expected_checkpoint,key): if self.db.in_transaction:_fail('Native preparation must precede the metadata transaction.','unsupported') if _CURRENT.get() is not None:_fail('A native preparation is already active.','busy') self.access.require('edit','native_preparation.'+purpose,graph['id'],owner_only=True) - self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + self.session.check_current() + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') context=NativePreparationContext(self,owner,graph,model,composition,purpose,_key(purpose,key)) result=dict(format='project_native_preparation_intent_v1',origin_catalog_id=self.session.project_id, actor_id=context.actor.actor_id,purpose=purpose,graph=dict(id=graph['id'],revision=graph['revision'], @@ -361,22 +361,22 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): if graph['revision']!=revision:_fail('Graph changed before native validation.','stale') import release_validation if adapter_for_graph(graph) is release_validation: - self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + self.session.check_current() + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') value=owner.validate_graph(graph) return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') with self._scope(owner,graph,None,None,'validation',expected_checkpoint,key) as context: value=owner.validate_graph(graph);context._check(graph['recipe']) - return self.session.mutate('native_preparation.validate',graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate('native_preparation.validate',graph_id,body,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') @contextmanager def current_graph(self,owner,graph_id,revision,expected_checkpoint,key): if self.db.in_transaction:_fail('Current native validation must precede admission.','unsupported') self.access.require('edit','native_preparation.admission',graph_id,owner_only=True) - self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + self.session.check_current() + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') graph=owner.get(graph_id) if graph['revision']!=revision:_fail('Graph changed before native admission.','stale') import release_validation @@ -413,7 +413,7 @@ def capture(value,before): record=self.materials.capture_recipe_material(value,owner.material_path) return dict(records=self.materials._coalesce([record,*self.materials.declarations(record), *self.materials.capture_recipe_dependencies(record,before)]),result=value) - return self.session.mutate(operation,graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate(operation,graph_id,body,self.session.summary['checkpoint'],key, lambda:owner._retain_material(prepared),capture) finally: if prepared['staging'].exists():shutil.rmtree(prepared['staging']) diff --git a/project_producer_execution.py b/project_producer_execution.py index b4456aba..e3729233 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -73,7 +73,7 @@ def _family(self, family): return family def _owner(self, family): - self._family(family);self.session.check() + self._family(family);self.session.check_current() if self._fenced: raise WorkbenchError('Current producer custody is uncertain. Reopen for inert inspection.', 'outcome_unknown', 409) self._refuse_unknown_provider() @@ -264,7 +264,7 @@ def _settle_drained(self): # Every effectful request is preceded by canonical uncertainty; # a terminal job with no such request is known not admitted. outcome='not_admitted' - observation=(outcome,True,self.session.snapshot['checkpoint']) + observation=(outcome,True,self.session.summary['checkpoint']) if self._journal_observations.get(identity_key)!=observation: self.execution.update_producer_outcome(family,producer_id,outcome,True) self._journal_observations[identity_key]=observation @@ -429,7 +429,7 @@ def status(self, *, offset=0, limit=40): raise WorkbenchError('Invalid current producer health page.') with self.session.access.lock,self.session.lock,self.db.lock: self.session.access.require('inspect','producers.health') - self.session.check() + self.session.check_current() self._settle_drained() items=[] for family,owner in self._owners.items(): diff --git a/project_producers.py b/project_producers.py index 79b97f20..8e052b34 100644 --- a/project_producers.py +++ b/project_producers.py @@ -592,9 +592,9 @@ def __init__(self, session): def _read(self, action, invoke): with self.session.access.lock, self.session.lock, self.session.db.lock: self.session.access.require('inspect', action) - self.session.check() + self.session.check_current() result = invoke(self.session.snapshot['records']) - self.session.check() + self.session.check_current() return result def list(self, family, *, origin_catalog_id=None, offset=0, limit=40): @@ -915,7 +915,7 @@ def _publish(owner, raw, role, media='application/json'): fail('Conflicting exact producer payload bytes.') owner._pending_payloads[declaration['sha256']] = raw owner._artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=owner.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=owner.session.summary['project_id'], sha256=declaration['sha256']), media_type=media, kind='producer_evidence', role=role, preservation='unique', required_for=['verify_producer']) diff --git a/project_projection.py b/project_projection.py index fca264be..25b6bf28 100644 --- a/project_projection.py +++ b/project_projection.py @@ -25,14 +25,17 @@ from project_producer_execution import ProjectProducerExecution, validate_operation_result as validate_producer_result from project_files import ProjectFiles, decode_records as decode_file_records from project_files import validate_operation_result as validate_file_result +from project_files import receipt_validator as file_receipt_validator from project_study_execution import ProjectStudyExecution, validate_operation_result as validate_study_execution_result from project_runtime import ProjectRuntime, validate_operation_result as validate_runtime_result from project_native_execution import FAMILY as NATIVE_FAMILY, decode_records as decode_native_records from project_native_preparation import ProjectNativePreparation, validate_operation_result as validate_preparation_result from project_sources import SourceAdapter, validate_operation_result as validate_source_result +from project_sources import receipt_validator as source_receipt_validator from project_measurements import ProjectMeasurements, FAMILY as MEASUREMENT_FAMILY from project_measurements import decode_records as decode_measurement_records from project_measurements import validate_operation_result as validate_measurement_result +from project_measurements import receipt_validator as measurement_receipt_validator from workbench import WorkbenchError from workbench import timestamp @@ -58,12 +61,66 @@ def validate_artifact_resolution(operation, result, records): return True +def validate_project_records(records, *, project_id, artifacts): + """Validate one inert candidate through every owning domain, without SQL reconstruction.""" + claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( + MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) + identities = set() + declarations = {} + for item in records: + identity = (item['family'], item['id']) + if item['family'] not in claimed or identity in identities: + raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) + identities.add(identity) + if item['family'] == 'project_artifact': + declaration = validate_declaration(item['document']) + if declaration['sha256'] != item['id']: + raise WorkbenchError('Artifact inventory identity differs.') + declarations[item['id']] = declaration + decode_data_records(records) + from project_fork import decode_fork + decode_fork(records, project_id=project_id) + from project_composition import decode_records as decode_compositions + decode_compositions(records, project_id=project_id) + from project_transfer import decode_records as decode_extraction_records + decode_extraction_records(records) + operation_records(records, project_id=project_id) + validate_file_receipt = file_receipt_validator(records) + decode_native_records(records,artifacts=artifacts) + materials = MaterialSnapshot(records) + for blob in materials.artifact_blobs(): + if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): + raise WorkbenchError('Material artifact declaration is missing or inconsistent.') + # Resolve every cross-owner reference against THIS candidate, rather + # than a resolver that still sees the preceding committed checkpoint. + decode_producer_records(records, artifacts=artifacts) + decode_history_records(records, materials=materials, artifacts=artifacts) + validate_measurement_receipt = measurement_receipt_validator(records, artifacts=artifacts) + validate_source_receipt = source_receipt_validator(records) + for receipt in operation_documents(records): + operation, result = receipt['operation'], receipt['result'] + if (validate_data_result(operation, result, records) + or validate_material_result(operation, result, records) + or validate_history_result(operation, result, records) + or validate_source_receipt(operation, result) + or validate_measurement_receipt(operation, result) + or validate_execution_result(operation, result, records) + or validate_producer_result(operation, receipt['target'], result, records) + or validate_file_receipt(operation, result) + or validate_study_execution_result(operation, result, records) + or validate_runtime_result(operation, result, records) + or validate_preparation_result(operation, result, records) + or validate_artifact_resolution(operation, result, records)): + continue + raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + + class ProjectProjection: def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None): self.store, self.db, self.access = store, db, access self.custody = custody self.pending_original = [] - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.artifacts = ProjectArtifacts(store) self.sources = None self._disclosures = set() @@ -127,57 +184,9 @@ def execute(self, *args, **kwargs): return self.db.execute(*args, **kwargs) def validate_records(self, records, *, project_id=None, artifacts=None): - artifacts = self.artifacts if artifacts is None else artifacts - project_id = self.project_id if project_id is None else project_id - claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( - MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) - identities = set() - declarations = {} - for item in records: - identity = (item['family'], item['id']) - if item['family'] not in claimed or identity in identities: - raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) - identities.add(identity) - if item['family'] == 'project_artifact': - declaration = validate_declaration(item['document']) - if declaration['sha256'] != item['id']: - raise WorkbenchError('Artifact inventory identity differs.') - declarations[item['id']] = declaration - decode_data_records(records) - from project_fork import decode_fork - decode_fork(records, project_id=project_id) - from project_composition import decode_records as decode_compositions - decode_compositions(records, project_id=project_id) - from project_transfer import decode_records as decode_extraction_records - decode_extraction_records(records) - operation_records(records, project_id=project_id) - decode_file_records(records) - decode_native_records(records,artifacts=artifacts) - materials = MaterialSnapshot(records) - for blob in materials.artifact_blobs(): - if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): - raise WorkbenchError('Material artifact declaration is missing or inconsistent.') - # Resolve every cross-owner reference against THIS candidate, rather - # than a resolver that still sees the preceding committed checkpoint. - decode_producer_records(records, artifacts=artifacts) - decode_history_records(records, materials=materials, artifacts=artifacts) - decode_measurement_records(records, artifacts=artifacts) - for receipt in operation_documents(records): - operation, result = receipt['operation'], receipt['result'] - if (validate_data_result(operation, result, records) - or validate_material_result(operation, result, records) - or validate_history_result(operation, result, records) - or validate_source_result(operation, result, records) - or validate_measurement_result(operation, result, records) - or validate_execution_result(operation, result, records) - or validate_producer_result(operation, receipt['target'], result, records) - or validate_file_result(operation, result, records) - or validate_study_execution_result(operation, result, records) - or validate_runtime_result(operation, result, records) - or validate_preparation_result(operation, result, records) - or validate_artifact_resolution(operation, result, records)): - continue - raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + return validate_project_records(records, + project_id=self.project_id if project_id is None else project_id, + artifacts=self.artifacts if artifacts is None else artifacts) def resolve_artifact(self, body, expected_checkpoint, key): """Explicitly supply exact bytes from a configured current local root.""" @@ -185,7 +194,7 @@ def resolve_artifact(self, body, expected_checkpoint, key): raise WorkbenchError('Choose one declared artifact and a current rooted file.') with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'artifacts.resolve', body['sha256'], owner_only=True) - self.session.check() + self.session.check_current() declaration = next((item['document'] for item in self.session.snapshot['records'] if item['family'] == 'project_artifact' and item['id'] == body['sha256']), None) if declaration is None: @@ -241,7 +250,7 @@ def runtime_configuration(self, request): def runtime_sources(self): with self.access.lock, self.session.lock, self.db.lock: self.access.require('admin', 'runtime.sources', owner_only=True) - self.session.check() + self.session.check_current() return dict(sources=sorted(self._runtime_sources)) def authorize_disclosure(self, run, population): @@ -308,14 +317,15 @@ def available(self): return self.projection.session.available def _tag(self, value): - snapshot = self.projection.session.snapshot - return dict(value, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint']) + summary = self.projection.session.summary + return dict(value, project_id=summary['project_id'], checkpoint=summary['checkpoint']) def records(self): - self.projection.session.check() + self.projection.session.check_current() from project_fork import active_records snapshot = self.projection.session.snapshot - records = active_records(snapshot['records'], project_id=snapshot['project_id']) + records = active_records(snapshot['records'], project_id=snapshot['project_id'], + families=frozenset(('workbench_record', 'workbench_text'))) return self._tag(dict(items=[copy.deepcopy(item['document']['current']) for item in records if item['family'] in ('workbench_record', 'workbench_text') and item['document']['current']['kind'] == 'text'])) diff --git a/project_runtime.py b/project_runtime.py index a94cc6e6..d63a9eb5 100644 --- a/project_runtime.py +++ b/project_runtime.py @@ -156,7 +156,7 @@ def _authorize(self, action): def inspect(self, config_request): """Read an explicitly selected current source; never provision or execute.""" with self.access.lock, self.session.lock, self.db.lock: - self._authorize('runtime_snapshots.inspect'); self.session.check() + self._authorize('runtime_snapshots.inspect'); self.session.check_current() _body(config_request, '0'*64, 0) with self.execution.runtime_inspection(): return self.owner.inspect(self.resolve_configuration(copy.deepcopy(config_request))) @@ -165,7 +165,7 @@ def status(self, key): """Read historical outcome only; captured does not claim local readiness.""" _key(key) with self.access.lock, self.session.lock, self.db.lock: - self.access.require('inspect', 'runtime.capture_status'); self.session.check() + self.access.require('inspect', 'runtime.capture_status'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) intent = receipts.get(_intent_key(key)); complete = receipts.get(key) if intent is not None and intent['operation'] != 'runtime.capture_intent': intent = None @@ -185,7 +185,7 @@ def recover_capture(self, config_request, expected_inventory_sha256, expected_ch body_sha = _intent_input(body) self._authorize('runtime_snapshots.create') self.execution.prepare_runtime_reconciliation(key, body_sha, expected_inventory_sha256, expected_checkpoint) - return self.capture(config_request, expected_inventory_sha256, self.session.snapshot['checkpoint'], key, + return self.capture(config_request, expected_inventory_sha256, self.session.summary['checkpoint'], key, reserve_bytes=reserve_bytes) def abandon(self, capture_key, reason, expected_checkpoint, key): @@ -212,7 +212,7 @@ def forbidden(): _fail('No original local runtime copy exists to abandon.', 'una _fail('Runtime intent differs from its exact original local writer.', 'conflict', 409) if capture_key in receipts: _fail('Capture completed; reconcile its acknowledgement with the exact capture request.', 'conflict', 409) - observed = self.session.snapshot['checkpoint'] + observed = self.session.summary['checkpoint'] def invoke(): current = self._authorize('runtime.capture_abandon') if current.actor_id != actor.actor_id: _fail('Runtime abandonment caller changed.', 'conflict', 409) @@ -248,7 +248,7 @@ def capture(self, config_request, expected_inventory_sha256, expected_checkpoint body_sha = _intent_input(body) intent = None with self.access.lock, self.session.lock, self.db.lock: - actor = self._authorize('runtime_snapshots.create'); self.session.check() + actor = self._authorize('runtime_snapshots.create'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) if any(r['operation']=='runtime.abandon' and r['result']['capture_key']==key for r in receipts.values()): _fail('This capture was abandoned; use a fresh key for a new measurement.', 'conflict', 409) @@ -273,7 +273,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), invalid, lambda *args: invalid(), scope='admin') else: - if expected_checkpoint != self.session.snapshot['checkpoint']: + if expected_checkpoint != self.session.summary['checkpoint']: _fail('Expected project checkpoint changed.', 'stale_project', 409) config = self.resolve_configuration(copy.deepcopy(config_request)) observed = self.owner.inspect(config) @@ -281,7 +281,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 _fail('Runtime inventory changed after inspection.', 'stale', 409) intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key), + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key), lambda result,before:dict(records=[],result=result), scope='admin') guard.publication_started() if os.path.lexists(self.owner.path / expected_inventory_sha256): @@ -294,7 +294,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 from pipeline import digest from pathlib import Path raw = read_regular(Path(descriptor['root'])/'manifest.json', MAX_MANIFEST) - runtime_ref = dict(kind='runtime',catalog_id=self.session.snapshot['project_id'], + runtime_ref = dict(kind='runtime',catalog_id=self.session.summary['project_id'], id=digest(descriptor),sha256=digest(descriptor)) value = dict(ref=runtime_ref,descriptor=descriptor,environment=None,manifest=decode(raw)) observed = dict(inventory_sha256=descriptor['manifest_sha256'],file_count=descriptor['file_count'],total_bytes=descriptor['total_bytes']) @@ -302,8 +302,8 @@ def capture(result, before): record = self.materials.capture_runtime(value, raw) return dict(records=[*self.materials.declarations(record),record],result=result) result = self.session.mutate('runtime.capture', expected_inventory_sha256, body, - self.session.snapshot['checkpoint'], key, - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') + self.session.summary['checkpoint'], key, + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') guard.confirmed() return result except BaseException as error: diff --git a/project_session.py b/project_session.py index 46c106cd..aa471c1e 100644 --- a/project_session.py +++ b/project_session.py @@ -9,6 +9,7 @@ import threading import agent_api +from project_store import copy_validated_json from workbench import WorkbenchError @@ -65,7 +66,8 @@ def operation_records(records, *, project_id=None): if project_id is None and overlay is not None: project_id = overlay['destination_project_id'] result = {} - for value in operation_documents(active_records(records, project_id=project_id)): + for value in operation_documents(active_records(records, project_id=project_id, + families=frozenset({OPERATION_FAMILY}))): if overlay is not None and value['format'] == 'project_operation_v1': raise ProjectSessionError('A fork must retain legacy operation receipts as archived historical facts.') if project_id is not None and value.get('project_id', project_id) != project_id: @@ -85,7 +87,7 @@ def __init__(self, store, db, access, artifacts, *, validate_records): self._validate = validate_records self.available = False self._snapshot = None - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.extra_capture = None self.publication_intent = None self.runtime_access = None @@ -176,10 +178,16 @@ def reconcile_selection(self, snapshot): self._recovery_records = None self._recovery_request = None + @property + def summary(self): + """Scalar metadata for the same bound checkpoint as snapshot.""" + with self.lock: + return {key: self._snapshot[key] for key in ('project_id', 'name', 'checkpoint')} + @property def snapshot(self): with self.lock: - return copy.deepcopy(self._snapshot) + return copy_validated_json(self._snapshot) def _validated(self, records): operation_records(records, project_id=self.project_id) @@ -196,18 +204,45 @@ def adopt(self, snapshot): current = self.store.snapshot() if current != snapshot: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._snapshot = copy.deepcopy(snapshot) + self._snapshot = copy_validated_json(snapshot) self.available = True - def check(self): - """Check freshness while a caller holds host/session/projection guards.""" + def _checked(self, read): if not self.available: raise ProjectSessionError('Reopen this project to reconstruct its projection.', 'projection_unavailable', 503) - current = self.store.snapshot() + current = read() if current['checkpoint'] != self._snapshot['checkpoint']: raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) return current + def check_current(self): + """Verify actual canonical bytes without returning an unused record graph. + + The caller holds the same host/session/projection guards as check(). + Store.summary reads and hashes all current records before returning. + """ + self._checked(self.store.summary) + + def check(self): + """Return independent current records under caller-owned projection guards.""" + return self._checked(self.store.snapshot) + + def check_records(self, family): + """Verify every canonical record, then copy one bound record family. + + The complete snapshot, including cross-family closure, was validated + and isolated by adopt/publication. Returned copies retain that validity + inside the trusted owner boundary; they are not new imported raw input. + Exact current bytes and checkpoint must still match before using it. + Callers hold the same ordered guards as check(); no cached alias escapes. + """ + if type(family) is not str: + raise TypeError('Record-family selection requires a string.') + current = self._checked(self.store.summary) + current['records'] = copy.deepcopy([ + item for item in self._snapshot['records'] if item['family'] == family]) + return current + def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capture, *, scope='edit'): """Invoke existing policy, capture fixed-owner upserts, publish, commit SQL. diff --git a/project_sources.py b/project_sources.py index 08939eba..a17b02a5 100644 --- a/project_sources.py +++ b/project_sources.py @@ -6,6 +6,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import hashlib import os @@ -36,7 +37,7 @@ def _summary(saved, origin): schema_version=saved['manifest']['schema_version']) -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): """Validate the fixed source receipt against its exact retained snapshot.""" if operation!='source.register':return False try: @@ -50,8 +51,8 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Source registration requires an attributed timestamp.') if datetime.fromisoformat(result['registered_at']).utcoffset() is None: raise WorkbenchError('Source registration timestamp requires an offset.') - if records is not None: - document=material_records(records).get(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) + if lookup is not None: + document=lookup(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) expected={key:result[key] for key in ('origin_catalog_id','project_id','manifest_sha256','name','schema_version')} if document is None or _summary(document['value'],document['origin_catalog_id'])!=expected: raise WorkbenchError('Source receipt differs from its exact retained snapshot.') @@ -61,6 +62,20 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Malformed retained source operation result.') from None +def receipt_validator(records): + """Own one candidate's retained material facts for source receipt checks.""" + owned = material_records(records) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return material_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class SourceAdapter: def __init__(self,projection): self.projection=projection @@ -117,9 +132,9 @@ def register(self,body,expected_checkpoint,key): self._mutation_identity(expected_checkpoint,key) with self.access.lock,self.session.lock,self.db.lock: actor=self.access.require('edit','source.register',owner_only=True) - self.session.check() + self.session.check_current() saved,raw=self._read_registration(body) - origin=self.session.snapshot['project_id'];invoked=False + origin=self.session.summary['project_id'];invoked=False def invoke(): nonlocal invoked invoked=True;self._bind(saved,raw) @@ -153,7 +168,7 @@ def references(self,snapshotref,offset=0,limit=40): raise WorkbenchError('Invalid bounded source reference page.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('inspect','source.references',snapshot['project_id']) - self.session.check() + self.session.check_current() historical=self.materials.snapshot_ref(snapshot['project_id'],snapshot['manifest_sha256'], catalog_id=snapshot['origin_catalog_id']) try:current=self.sources._saved(snapshot['project_id'],extensions=True) @@ -171,7 +186,7 @@ def references(self,snapshotref,offset=0,limit=40): selected=current['manifest']['data'][offset:offset+limit];budget=[INSPECTION_LIMIT] items=[self.sources._inspect(current['root'],copy.deepcopy(ref),budget,DATA_LIMIT) for ref in selected] self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) - self.session.check();self.access.require('inspect','source.references',snapshot['project_id']) + self.session.check_current();self.access.require('inspect','source.references',snapshot['project_id']) return dict(items=items,total=len(current['manifest']['data']),offset=offset,limit=limit) def import_scientific(self,body,expected_checkpoint,key): @@ -183,7 +198,7 @@ def import_scientific(self,body,expected_checkpoint,key): if type(body['document']) is not dict:raise WorkbenchError('Scientific import metadata must be an object.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_scientific',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],DATA_LIMIT) # ProjectData owns the one canonical mutation and native parser; # current rooted reads above never become an imported capability. @@ -204,7 +219,7 @@ def _capture_reference(self,snapshot,reference_id,expected_sha256,maximum_bytes) expected_sha256,maximum_bytes) if source['manifest_sha256']!=snapshot['manifest_sha256']: raise WorkbenchError('Current source snapshot changed during capture.','stale',409) - self.session.check() + self.session.check_current() return raw,source def import_measurements(self,body,expected_checkpoint,key): @@ -216,7 +231,7 @@ def import_measurements(self,body,expected_checkpoint,key): from retained_measurements import MAX_BYTES with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_measurements',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],MAX_BYTES) source=dict(source,origin_catalog_id=snapshot['origin_catalog_id']) self.access.require('edit','source.import_measurements',snapshot['project_id']) diff --git a/project_store.py b/project_store.py index bc4b4071..d079ca3f 100644 --- a/project_store.py +++ b/project_store.py @@ -124,6 +124,17 @@ def _encode(value): allow_nan=False) + '\n').encode('utf-8') +def copy_validated_json(value): + """Own another tree of already validated JSON without Python graph copying. + + Canonical Store/Session values contain only JSON scalars, lists and string + keyed dictionaries. This is a copy operation inside that proof boundary, + never an input decoder or a substitute for freshness/codec validation. + """ + return json.loads(json.dumps(value, ensure_ascii=True, allow_nan=False, + separators=(',', ':'))) + + def _manifest_raw(manifest): raw = _encode(manifest) if len(raw) > MAX_MANIFEST_BYTES: @@ -468,6 +479,7 @@ def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path): self._origin_path = Path(origin_path) self._meta_fd = None self._manifest = None + self._read_cache = None self._mutex = threading.RLock() self._unknown = False try: @@ -629,12 +641,12 @@ def root_directory(self, *, verify_after=False): if type(verify_after) is not bool: _fail('Root read verification must be an explicit boolean.') with self._mutex: - before = self.snapshot() + before = self._current()[0]['checkpoint'] descriptor = os.dup(self._root_fd) try: yield descriptor self._binding() - if verify_after and self.snapshot()['checkpoint'] != before['checkpoint']: + if verify_after and self._current()[0]['checkpoint'] != before: _fail('Project checkpoint changed during the bounded root read.', 'stale', 409) finally: os.close(descriptor) @@ -642,6 +654,17 @@ def root_directory(self, *, verify_after=False): def _load(self): self._binding() raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) + cached = self._read_cache + if cached is not None and raw == cached[0]: + manifest, loaded = cached[1:] + # Reuse decoding only after checking every actual canonical file. + # Stat identity is not content identity; corruption still fails. + for entry in manifest['inventory']: + self._verified_record_bytes(entry) + if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: + _fail('Project checkpoint changed while reading.', 'stale', 409) + self._binding() + return manifest, loaded manifest = _decode(raw) if type(manifest) is not dict: _fail('Project checkpoint must be an object.') @@ -683,9 +706,7 @@ def _load(self): total += entry['bytes'] if total > MAX_TOTAL_BYTES: _fail('Project records exceed their total byte limit.') - record_raw = self._record_file(family, record_id, digest) - if len(record_raw) != entry['bytes'] or _sha(record_raw) != digest: - _fail('Canonical record bytes changed or are corrupt.', 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + record_raw = self._verified_record_bytes(entry) document = _decode(record_raw) if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') @@ -709,8 +730,16 @@ def _load(self): if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: _fail('Project checkpoint changed while reading.', 'stale', 409) self._binding() + self._read_cache = (raw, manifest, loaded) return manifest, loaded + def _verified_record_bytes(self, entry): + raw = self._record_file(entry['family'], entry['id'], entry['sha256']) + if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: + _fail('Canonical record bytes changed or are corrupt.', + 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + return raw + def _record_file(self, family, record_id, digest, raw=None, sync=False): fds = [] try: @@ -775,17 +804,27 @@ def _publish(self, manifest, creating=False): def _snapshot(self, manifest, records): return dict(project_id=manifest['project_id'], name=manifest['name'], - checkpoint=manifest['checkpoint'], records=copy.deepcopy(records)) + checkpoint=manifest['checkpoint'], records=copy_validated_json(records)) + + def _current(self): + try: + manifest, records = self._load() + if manifest != self._manifest: + _fail('Project checkpoint changed; close and reopen.', 'stale', 409) + return manifest, records + except OSError as exc: + _io_error(exc, 'read project snapshot') + + def summary(self): + """Return scalar metadata after the same complete freshness check.""" + with self._mutex: + manifest, _records = self._current() + return {key: manifest[key] for key in ('project_id', 'name', 'checkpoint')} def snapshot(self): with self._mutex: - try: - manifest, records = self._load() - if manifest != self._manifest: - _fail('Project checkpoint changed; close and reopen.', 'stale', 409) - return self._snapshot(manifest, records) - except OSError as exc: - _io_error(exc, 'read project snapshot') + manifest, records = self._current() + return self._snapshot(manifest, records) def reconcile(self, request_key, request_sha256): """Inspect retry identity before a domain mutation; never repeat effects.""" @@ -884,6 +923,7 @@ def recover_checkpoint(self): def close(self): with self._mutex: + self._read_cache = None if self._meta_fd is not None: os.close(self._meta_fd) self._meta_fd = None diff --git a/project_study_execution.py b/project_study_execution.py index 561c4779..2133bbce 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -142,7 +142,7 @@ def _validate_preparation(operation,result,records): trial=result['trial'];origin=trial['material']['catalog_id'] candidate=facts.candidates[(origin,result['candidate_id'])]['candidate'] if candidate['study_id']!=trial['study_id']:_invalid('Preparation candidate belongs to another Study.') - materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check=lambda:None)) + materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check_current=lambda:None)) composition=materials.recipe(trial['material'])['material'] if (composition['model']!=result['model'] or composition['recipe_id']!=result['graph']['id'] or composition['recipe_revision']!=result['graph']['revision']): @@ -202,7 +202,7 @@ def validate_operation_result(operation,result,records=None): if saved is None or any(saved[key]!=reservation[key] for key in RESERVATION_FIELDS if key not in ('state','settled')): _invalid('Admission receipt has no exact canonical reservation.') trial=facts.trials[(origin,result['trial_id'])]['trial'];run=facts.runs[(origin,result['run_id'])]['run'] - view=SimpleNamespace(snapshot={'records':records},check=lambda:None) + view=SimpleNamespace(snapshot={'records':records},check_current=lambda:None) validate_retained_reservation(reservation,trial=trial, composition=ProjectMaterial(view).recipe(trial['material']),run=run) elif operation=='study_execution.stop': @@ -267,7 +267,7 @@ def __init__(self,adapter,phase): def __getattr__(self,name):return getattr(self.source,name) def open(self,body): return self.adapter.projection.source_adapter.register(body, - self.adapter.session.snapshot['checkpoint'],self.phase+'-source') + self.adapter.session.summary['checkpoint'],self.phase+'-source') class _PreparationModels: @@ -278,11 +278,11 @@ def _read_blob(self,ref,item,maximum=40*1024**2):return self.adapter.materials.b def create(self,body): adapter=self.adapter return adapter.materials.mutate_definition('model_definition','models.create',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) + adapter.session.summary['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) def publish(self,identity,body): adapter=self.adapter return adapter.materials.publish_model(adapter.projection.models,identity,body, - adapter.session.snapshot['checkpoint'],self.phase+'-publish') + adapter.session.summary['checkpoint'],self.phase+'-publish') class _PreparationPipelines: @@ -295,7 +295,7 @@ def recipe_material(self,ref,*,verify=False):return self.adapter.materials.recip def create_external(self,body): adapter=self.adapter return adapter.materials.mutate_definition('pipeline_graph','graphs.create_external',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) + adapter.session.summary['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) def prepare_external_material(self,*args,**kwargs):return self.owner.prepare_external_material(*args,**kwargs) def _retain_material(self,prepared): adapter=self.adapter;value=self.owner._retain_material(prepared) @@ -310,7 +310,7 @@ def __init__(self,projection,execution): self.projection,self.execution=projection,execution self.session,self.db,self.history=projection.session,projection.db,projection.history self.materials=projection.materials;self.access=execution.runtime_access - self.origin=self.session.snapshot['project_id'] + self.origin=self.session.summary['project_id'] self._dirty=set();self._revisions={};self._epochs={};self._pending_epochs={} self._package_records=[] self.context=copy.copy(projection.context) @@ -329,7 +329,7 @@ def _canonical(self,records=None): def restore_authoring(self,records=None): """Reconstruct intent only, under the current project guard.""" with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check();facts=self._canonical(records) + self.session.check_current();facts=self._canonical(records) with self.db.transaction(): for (origin,identity),doc in sorted(facts.studies.items()): if origin!=self.origin:continue @@ -573,7 +573,7 @@ def approve(self,study_id,body,expected_checkpoint,key): validate_operation_result(prior['operation'],prior['result'],self.session.snapshot['records']) self._current_grant(prior['result'],current.actor_id) else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self._materialize_trial(study_id,body['trial_id'],body['expected_material_sha256']) def invoke(): grant=self.studies.approve(study_id,body) @@ -589,7 +589,7 @@ def admit(self,study_id,body,expected_checkpoint,key): authority=self.access.require_external_submission() with self.session.access.lock,self.session.lock,self.db.lock: builtin=None - self.session.check();self.refresh_authoring() + self.session.check_current();self.refresh_authoring() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: result=prior['result'] @@ -597,7 +597,7 @@ def admit(self,study_id,body,expected_checkpoint,key): 'SELECT run_id FROM model_attempt_reservations WHERE run_id=%s',(result.get('run_id'),)).fetchone() is None): _invalid('A historical admission receipt cannot create a current local attempt.','unavailable') else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self.require_current_epoch(self.studies._study(study_id)) self.reject_historical_request(study_id,authority.actor_id,key,None) trial=self._materialize_trial(study_id,body['trial_id']) @@ -626,7 +626,7 @@ def invoke(): # Qualification may launch the explicit probe. It finishes # before Session opens the finite admission transaction. with preparation.admission(pipelines,ref,expected_checkpoint,key): - return self._mutate('study_execution.admit',study_id,body,self.session.snapshot['checkpoint'],key,invoke,'inspect') + return self._mutate('study_execution.admit',study_id,body,self.session.summary['checkpoint'],key,invoke,'inspect') return self._mutate('study_execution.admit',study_id,body,expected_checkpoint,key,invoke,'inspect') def stop(self,study_id,body,expected_checkpoint,key): @@ -645,7 +645,7 @@ def invoke(): # cancellation phase. Receipt replay does not repeat effects. for run_id in invoked: cancel_key='study-stop-'+hashlib.sha256((key+':'+run_id).encode()).hexdigest() - try:self.execution.cancel_run(run_id,self.session.snapshot['checkpoint'],cancel_key) + try:self.execution.cancel_run(run_id,self.session.summary['checkpoint'],cancel_key) except BaseException as error: error.study_stop_receipt=copy.deepcopy(result) raise @@ -683,7 +683,7 @@ def reconcile_preparation(self,study_id,body,expected_checkpoint): if complete.get('native_receipt'): effective=_preparation_body(intent,body['preparation_key']) effective.pop('request_key');effective['actor_id']=intent['actor_id'] - result=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],body['preparation_key'], + result=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],body['preparation_key'], lambda:copy.deepcopy(complete['result']),'cancel') else:result=copy.deepcopy(complete['result']) guard.confirmed() @@ -703,7 +703,7 @@ def abandon_preparation(self,study_id,body,expected_checkpoint,key): operation,current_intent,complete=_preparation_reference(self.session.snapshot['records'],study_id,body['preparation_key']) if current_intent!=intent:_invalid('Preparation intent changed during guarded reconciliation.','conflict') if complete is not None:_invalid('Preparation is complete; reconcile its acknowledgement without abandoning science.','conflict') - observed_checkpoint=self.session.snapshot['checkpoint'] + observed_checkpoint=self.session.summary['checkpoint'] def invoke(): actor=self.access.require('cancel','studies.abandon_preparation',study_id,owner_only=True) if actor.actor_id!=current.actor_id:_invalid('Abandonment caller changed.','conflict') @@ -734,14 +734,14 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): input_sha=digest(dict(study_id=study_id,body=effective)) intent_key=_intent_key(operation,key);phase=intent_key.removesuffix('-intent') with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check() + self.session.check_current() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: # Current authority is checked above. Historical completion # acknowledges science and cannot repeat filesystem effects. return self._mutate(operation,study_id,effective,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Completed preparation replay invoked effects.')),'edit') - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') receipts=operation_records(self.session.snapshot['records']) if any(_intent_key(other,key) in receipts for other in PREPARATIONS if other!=operation): _invalid('Preparation key already identifies another preparation operation.','conflict') @@ -778,7 +778,7 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): publication=SimpleNamespace(runtime_snapshot=snapshot,before_write=guard.publication_started, read_recipe_blob=self.materials.blob) result=_prepare_operation(studies,study_id,dict(body,request_key=key),kind,publication=publication) - value=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],key, + value=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(result),'edit') guard.confirmed() return value diff --git a/project_text.py b/project_text.py index d9502cb6..fc778528 100644 --- a/project_text.py +++ b/project_text.py @@ -252,7 +252,7 @@ def restore(self, snapshot=None): encode(row['annotation']) if row['annotation'] is not None else None, row['review'], row['revision'], row['created_at'], row['updated_at'])) for state in doc['history']: self._workbench._history(state) - if self.store.snapshot()['checkpoint'] != current['checkpoint']: + if self.store.summary()['checkpoint'] != current['checkpoint']: raise ProjectTextError('Project changed during reconstruction.', 'stale_project', 409) self._snapshot, self.available = current, True return copy.deepcopy(current) diff --git a/project_transfer.py b/project_transfer.py index 10a6b431..91848559 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -720,7 +720,7 @@ def replay(opened): try: store=ProjectStore.create_at(destination_parent_fd,stage_name,name,'extraction-stage.'+digest(request_key.encode())[:32],request_digest,root_path=root_path) with store.root_directory() as owned_fd:stage_identity=_fd_identity(owned_fd) - destination=store.snapshot()['project_id'];created_at=timestamp() + destination=store.summary()['project_id'];created_at=timestamp() from project_data_fork import fork_records as fork_data from project_material_fork import fork_records as fork_material original=copy.deepcopy(records);mappings=[] @@ -755,7 +755,7 @@ def replay(opened): with source.artifacts.open_verified(value,maximum_bytes=value['bytes']) as fd:artifacts.publish_fd(fd,value,maximum_bytes=value['bytes']) with store.root_directory() as fd:materialize_captured(records,destination,fd) _validate_candidate(records,destination,artifacts) - committed=store.commit(records,store.snapshot()['checkpoint'],request_key,request_digest) + committed=store.commit(records,store.summary()['checkpoint'],request_key,request_digest) verification=_cold_verify(store,source) store.save(committed['checkpoint']);_assert_unchanged(source) _rename_noreplace(destination_parent_fd,stage_name,directory);published=True;os.fsync(destination_parent_fd) diff --git a/project_transfer_controller.py b/project_transfer_controller.py index 7362e0ea..3ece224e 100644 --- a/project_transfer_controller.py +++ b/project_transfer_controller.py @@ -252,7 +252,7 @@ def extract(self, binding, report_sha256, destination_root, directory, name, req fail('Recover the selected workspace before reconciling extraction.', 'projection_unavailable', 503) guards.enter_context(projection.session.lock) guards.enter_context(projection.db.lock) - projection.session.check() + projection.session.check_current() opened = self.workspace._store receipt = self._transfer.extract(self._selection, report_sha256, destination_fd, directory, name, request_key, opened_store=opened) except BaseException as error: diff --git a/project_workspace.py b/project_workspace.py index efe93f9a..b006dbe2 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -122,7 +122,7 @@ def _state(self): project = None if self._recovery is None and self._store is not None: try: - snapshot = self._store.snapshot() + snapshot = self._store.summary() except ProjectStoreError as error: if error.code not in ('unknown', 'outcome_unknown', 'stale', 'stale_project', 'invalid', 'unsupported', 'unavailable'): @@ -184,7 +184,7 @@ def _bound(self, arguments, *, may_be_empty=False, check_checkpoint=True, allow_ if arguments.get('expected_checkpoint') is not None: raise WorkbenchError('The empty workspace has no checkpoint.', 'stale_project', 409) return - snapshot = self._store.snapshot() + snapshot = self._store.summary() expected = arguments.get('expected_checkpoint', arguments.get('checkpoint')) if check_checkpoint and expected != snapshot['checkpoint']: raise WorkbenchError('The project checkpoint changed. Refresh before acting.', 'stale_project', 409) @@ -206,7 +206,7 @@ def _root(self, alias, relative): def _projection_for(self, store): identity = uuid.uuid4().hex path = self.path / '.project-projections' / identity - snapshot = store.snapshot() + snapshot = store.summary() projection = Metadata(path, dsn=self._dsn, dataset_id=snapshot['project_id'], projection_id=identity) try: owner = ProjectProjection(store, projection, self.access, @@ -227,7 +227,7 @@ def _retained_projection_for(self, store, projection_id): if type(projection_id) is not str or not re.fullmatch('[a-f0-9]{32}', projection_id): raise WorkbenchError('Invalid original local projection identity.', 'unavailable', 409) db = Metadata(self.path / '.project-projections' / projection_id, dsn=self._dsn, - create=False, dataset_id=store.snapshot()['project_id'], projection_id=projection_id) + create=False, dataset_id=store.summary()['project_id'], projection_id=projection_id) try: return ProjectProjection(store, db, self.access, {alias: value[0] for alias, value in self._roots.items()}, @@ -243,7 +243,7 @@ def _detach(self): self._projection.execution.assert_quiescent() if self._store is not None: # Prove pending project publications and their full closure first. - self._store.save(self._store.snapshot()['checkpoint']) + self._store.save(self._store.summary()['checkpoint']) if self._projection is not None: self._projection.close() if self._store is not None: @@ -272,14 +272,14 @@ def _recover_current(self, arguments): store = projection = None try: store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) - snapshot = store.snapshot() + snapshot = store.summary() if snapshot['project_id'] != self._last_project['project_id']: raise WorkbenchError('Recovery destination has a different project identity.', 'stale_project', 409) # A reread reconciles identity; only a successful durability barrier # resolves a prior checkpoint/directory fsync failure. store.save(snapshot['checkpoint']) projection, text = self._projection_for(store) - snapshot = store.snapshot() + snapshot = store.summary() if self._projection is not None: self._projection.close() self._store, self._projection, self._text = store, projection, text @@ -332,7 +332,7 @@ def _select(self, action, arguments): if store.reconcile(arguments['request_key'], digest) is None: raise error reconciled = True - snapshot = store.snapshot() + snapshot = store.summary() if reconciled: store.save(snapshot['checkpoint']) created = snapshot @@ -342,7 +342,7 @@ def _select(self, action, arguments): self._detach() self._store, self._projection, self._text = store, projection, text self._selection = selection - snapshot = store.snapshot() + snapshot = store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} self._recovery = None self._binding = uuid.uuid4().hex @@ -366,7 +366,7 @@ def _select(self, action, arguments): raise def _record_result(self, record, *, before_revision=None, history_limit=1): - snapshot = self._store.snapshot() + snapshot = self._store.summary() history = [item for item in self._text.history(record['id']) if item['revision'] <= record['revision'] and (before_revision is None or item['revision'] < before_revision)] @@ -402,14 +402,14 @@ def _publish_independent(self, arguments, *, composed=False): if composed: from project_composition import publish_composition from project_artifacts import ProjectArtifacts - others=[];identities={self._store.snapshot()['project_id']} + others=[];identities={self._store.summary()['project_id']} for selected in arguments['sources']: if selected['project_id'] in identities: raise WorkbenchError('Composition source identity repeats.','conflict',409) identities.add(selected['project_id']) directory,descriptor=self._root(selected['root'],selected['directory']) store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory)) - actual=store.snapshot() + actual=store.summary() if actual['project_id']!=selected['project_id'] or actual['checkpoint']!=selected['checkpoint']: raise WorkbenchError('Selected composition source checkpoint differs.','stale_project',409) artifacts=retained.enter_context(ProjectArtifacts(store)) @@ -509,13 +509,17 @@ def dispatch(self, name, arguments, *, selected_version=None): from project_dispatch import dispatch try: value = dispatch(self._projection, name, arguments) + if (name=='project_files.measurement_compare' + and value['value']['observation_count']>agent_api.measurement_comparison_row_limit(selected_version)): + raise WorkbenchError('Comparison exceeds the selected API version observation budget. Select comparison contract version 2 for larger complete populations.', + 'too_large',413) if name == 'project_files.execution_health' and selected_version == 1: value = {key: item for key, item in value.items() if key != 'native_phases'} except BaseException as error: if getattr(error, 'code', None) in ('unknown', 'outcome_unknown', 'projection_unavailable'): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} result = dict(binding=self._binding, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint'], value=value) @@ -625,14 +629,14 @@ def _dispatch_draining(self, name, arguments): raise WorkbenchError('The retained physical recovery owner changed.', 'local_custody_unavailable', 409) recovering = True if incoming is not projection and name == 'project_files.execution_reconcile_current': - value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.snapshot['checkpoint']) + value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.summary['checkpoint']) else: internal = arguments if recovery_operation and recovering and name != 'project_files.execution_reconcile_current': # Public CAS above uses the last acknowledged workspace # view. Only this qualified local owner sees its exact # selected/attempted checkpoint for finite recovery. - internal = dict(arguments, expected_checkpoint=incoming.session.snapshot['checkpoint']) + internal = dict(arguments, expected_checkpoint=incoming.session.summary['checkpoint']) if name == 'project_files.runtime_capture' and recovering: options = {'reserve_bytes': internal['reserve_bytes']} if 'reserve_bytes' in internal else {} value = incoming.runtime.recover_capture({'source': internal['source']}, @@ -656,7 +660,7 @@ def _dispatch_draining(self, name, arguments): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise with self.access.lock, projection.session.lock, projection.db.lock: - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} if name in _RECOVERY_OPERATIONS: self._recovery = None diff --git a/retained_measurements.py b/retained_measurements.py index 796aaa19..4044b017 100644 --- a/retained_measurements.py +++ b/retained_measurements.py @@ -26,6 +26,7 @@ MAX_COMPARISON_SOURCES = 16 MAX_COMPARISON_SUBJECTS = 256 MAX_COMPARISON_BYTES = 128 * 1024 * 1024 +MAX_COMPARISON_ROWS = 250000 STATES = ('measured', 'failed', 'unsupported', 'unavailable') ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') SHA = re.compile(r'^[a-f0-9]{64}$') @@ -411,8 +412,8 @@ def __init__(self, tables): fail('Comparison sources overlap a subject identity.', 'conflict', 409) self._subjects[subject['id']] = dict(copy.deepcopy(subject), source=copy.deepcopy(locator)) observations += metadata['observation_count'] - if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_ROWS: - fail('Comparison exceeds 256 subjects or 100,000 observations.', 'too_large', 413) + if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_COMPARISON_ROWS: + fail('Comparison exceeds 256 subjects or 250,000 observations.', 'too_large', 413) self._tables.append((copy.deepcopy(locator), table)) self._header.update(subjects=list(self._subjects.values()), observation_count=observations) diff --git a/scripts/generate_project_files_contracts.py b/scripts/generate_project_files_contracts.py new file mode 100644 index 00000000..ac373e53 --- /dev/null +++ b/scripts/generate_project_files_contracts.py @@ -0,0 +1,35 @@ +"""Derive installed browser pins/routes from the local canonical API registry.""" +import argparse +import json +from pathlib import Path +import re +import sys + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT)) +import agent_api + + +def render(original): + names=sorted(name for name in agent_api._CONTRACTS if name.startswith('project_files.')) + tables={ + 'CONTRACTS':{name:agent_api.contract(name,1)['sha256'] for name in names}, + 'VERSIONS':{name:{str(v):agent_api.contract(name,v)['sha256'] for v in sorted(agent_api._CONTRACTS[name])} + for name in names if len(agent_api._CONTRACTS[name])>1}, + 'ROUTES':{name:{'method':agent_api.operation(name)['method'],'path':agent_api.operation(name)['path'], + 'mutation':agent_api.operation(name)['side_effects']} + for name in sorted(agent_api.PROJECT_FILES_M2|agent_api.PROJECT_FILES_M3|agent_api.PROJECT_FILES_M4)}} + for label,value in tables.items(): + pattern=r'(/\* PROJECT_FILES_'+label+r'_START \*/\s*)(.*?)(\s*/\* PROJECT_FILES_'+label+r'_END \*/)' + generated=json.dumps(value,sort_keys=True,indent=2) + original,count=re.subn(pattern,lambda m:m[1]+generated+m[3],original,flags=re.S) + if count!=1:raise ValueError('Exactly one installed '+label+' table required') + return original + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--check',action='store_true') + args=parser.parse_args();path=ROOT/'static/project-files.js';original=path.read_text();expected=render(original) + if args.check: + if original!=expected:raise SystemExit('Installed project-file browser contracts are stale') + elif original!=expected:path.write_text(expected) diff --git a/static/project-files.css b/static/project-files.css index 515d9ca5..c18e2e30 100644 --- a/static/project-files.css +++ b/static/project-files.css @@ -10,3 +10,13 @@ #project-runtime-files,#project-runtime-summary,#project-runtime-pending,#project-runtime-layout,#project-runtime-status{overflow-wrap:anywhere;min-width:0}#project-runtime-layout,#project-runtime-status{white-space:pre-wrap;max-height:32rem;overflow:auto} #project-candidate-preparation-result,#project-candidate-preparation-status{overflow-wrap:anywhere;min-width:0}#project-candidate-preparation-result{white-space:pre-wrap;max-height:32rem;overflow:auto} + +#project-portfolio-source,#project-portfolio-detail,#project-portfolio-count{overflow-wrap:anywhere;min-width:0} +#project-portfolio-detail p{white-space:pre-wrap;max-width:85ch} +#project-portfolio-detail pre{white-space:pre-wrap;overflow-wrap:anywhere;max-height:32rem;overflow:auto} +#project-portfolio-detail details{margin:1rem 0} +#project-portfolio-detail h4{margin-bottom:.25rem} +#project-portfolio-detail h3{scroll-margin-top:1rem} +#project-portfolio-filter-form{display:grid;grid-template-columns:1fr 2fr 1fr;gap:.7rem;align-items:end} +#project-portfolio-filter-form button{justify-self:start} +@media(max-width:760px){#project-portfolio-filter-form{grid-template-columns:1fr}} diff --git a/static/project-files.html b/static/project-files.html index de9c6f1a..52f1ccca 100644 --- a/static/project-files.html +++ b/static/project-files.html @@ -14,6 +14,7 @@ + @@ -50,6 +51,7 @@
No unsubmitted edits.
+Choose a current Study, Candidate and captured CPU runtime. Preparation saves an inert Trial and exact sources; approval and Run admission remain separate actions.
diff --git a/static/project-files.js b/static/project-files.js index 65cfa499..3146719e 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -14,6 +14,7 @@ "project_files.candidate": "c519ff04bc06d5e538d36b7f278862d8f282d0e7b97dd7f262444c384a50f854", "project_files.candidate_select": "f043be6923760a208ca50d02a1ff4b60eadf09d48a034acdb18749806a537080", "project_files.close": "95d0099ec2a378a1bbb65676799dd3baae2f36b838301f7aa165b116fba5edf6", + "project_files.compose": "b1dcbbd4ad7ebe17b17e11238f99a93408da24d0d411527e187900cc0f089105", "project_files.data_annotate": "289270f5cb946765c5ced22fbbd18765bb55743145014c81cf54d7a6ba3025ea", "project_files.data_availability": "92332c04973a4fa45d47e384717152ebc4a9ca08e7eb487669688ccd2ed42cbb", "project_files.data_get": "0d19505995cbd8c12e3938cd89cf05955a2a63488a220711d29dd24b27c0c8c8", @@ -167,6 +168,10 @@ "project_files.get": { "1": "6b0f4e16a00aa119b86cbb5493be0bac945ff08a5e63773a8f2c2f9296658d5b", "2": "fdc0858fc03a0866bb743b04251ea5f9a3c1e6be1b211366db7e0652166736ea" + }, + "project_files.measurement_compare": { + "1": "abbb5e406a3c95155e90f08e8a071f330425190b25275f0f6512de08b8afb929", + "2": "d2de9763a9109d0e4599f50e038a3fda0e6bf7c9c60684434825d0108019bc6f" } } /* PROJECT_FILES_VERSIONS_END */; @@ -199,6 +204,11 @@ "mutation": true, "path": "/api/workspace/project-files/studies/{study_id}/candidates" }, + "project_files.compose": { + "method": "POST", + "mutation": true, + "path": "/api/workspace/project-files/compose" + }, "project_files.data_annotate": { "method": "POST", "mutation": true, @@ -892,6 +902,17 @@ onContextFocusLost:()=>$('project-files-status').focus() }); window.addEventListener('pagehide',()=>projectProgression.clear()); + const projectPortfolio=window.ProjectModelPortfolio({ + root:$('project-model-portfolio'), + async read(action,args,signal){ + if(!m2Allowed(action))throw Error('Model organization inspection is unavailable for the selected project.'); + const value=await m2Request(action,args,epoch,{signal}); + if(value===null)throw new DOMException('Project selection changed.','AbortError'); + return value; + }, + onContextFocusLost:()=>$('project-files-status').focus() + }); + window.addEventListener('pagehide',()=>projectPortfolio.clear()); const projectMeasurements=window.ProjectMeasurements({ root:$('project-measurements'), async read(action,args,signal){ @@ -1157,6 +1178,8 @@ } const m2Allowed=action=>!busy&&!blocked&&state?.state==='open'&&selectedContracts.has('project_files.'+action); function m2Controls(){ + projectPortfolio.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), + project_id:state?.project?.project_id,enabled:m2Allowed('files_inspect')&&m2Allowed('file_get')}); projectProgression.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), project_id:state?.project?.project_id,enabled:m2Allowed('progression')&&m2Allowed('progression_origins')}); projectMeasurements.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), diff --git a/static/project-model-portfolio.js b/static/project-model-portfolio.js new file mode 100644 index 00000000..4f1b7fa3 --- /dev/null +++ b/static/project-model-portfolio.js @@ -0,0 +1,188 @@ +'use strict'; +// Authored organization is a retained note, not Trial lineage or qualification. +// This module owns its document contract, finite view and cancellable reads. +(() => { + const fields=['method','purpose','experiment','outcome','lesson','reproduction']; + const exact=(value,keys)=>value&&typeof value==='object'&&!Array.isArray(value)&& + Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); + const text=(value,limit)=>typeof value==='string'&&value.length>0&&value.length<=limit; + const identity=value=>text(value,160)&&/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value); + const sha=value=>typeof value==='string'&&/^[a-f0-9]{64}$/.test(value); + const natural=value=>Number.isSafeInteger(value)&&value>=0; + function fail(message,code='invalid'){throw Object.assign(Error(message),{code});} + function noteRef(value){ + if(!exact(value,['file_id','path','revision','sha256'])||!sha(value.file_id)||!sha(value.sha256)|| + !natural(value.revision)||value.revision<1||!text(value.path,512)||!value.path.startsWith('notes/')|| + value.path.split('/').some(part=>!part||part==='.'||part==='..'))fail('Invalid captured model-note reference.'); + } + function decode(raw){ + if(typeof raw!=='string'||new TextEncoder().encode(raw).length>1048576)fail('Organization note exceeds its retained text limit.'); + let value;try{value=JSON.parse(raw);}catch{fail('Organization note is not valid JSON.');} + if(value?.format!=='project_model_portfolio_v1')fail('Unsupported model-organization document format.','unsupported'); + if(!exact(value,['format','project_id','title','description','authorship','groups','models','relationships'])|| + !/^[a-f0-9]{32}$/.test(value.project_id)||!text(value.title,200)||!text(value.description,8000)|| + value.authorship!=='retrospective'||!Array.isArray(value.groups)||value.groups.length>64|| + !Array.isArray(value.models)||!value.models.length||value.models.length>1000|| + !Array.isArray(value.relationships)||value.relationships.length>2000)fail('Invalid model-organization document.'); + const groups=new Map(),models=new Map(); + for(const group of value.groups){ + if(!exact(group,['id','label','parent','description'])||!identity(group.id)||groups.has(group.id)|| + !text(group.label,200)||!text(group.description,8000)||group.parent!==null&&!identity(group.parent))fail('Invalid or duplicate organization group.'); + groups.set(group.id,group); + } + for(const group of groups.values()){ + const visited=new Set([group.id]);let parent=group.parent; + while(parent!==null){if(!groups.has(parent))fail('Organization group parent is unavailable.'); + if(visited.has(parent))fail('Organization grouping contains a cycle.');visited.add(parent);parent=groups.get(parent).parent;} + } + for(const model of value.models){ + if(!exact(model,['id','label','group_id','aliases','review','card','summary'])||!identity(model.id)||models.has(model.id)|| + !text(model.label,500)||!groups.has(model.group_id)||!Array.isArray(model.aliases)||model.aliases.length>128|| + new Set(model.aliases).size!==model.aliases.length||model.aliases.some(alias=>!text(alias,500))|| + !['source_reviewed','review_pending'].includes(model.review))fail('Invalid or duplicate organized model.'); + noteRef(model.card); + if(model.review==='source_reviewed'){ + if(!exact(model.summary,fields)||fields.some(field=>!text(model.summary[field],16384)))fail('Reviewed model requires all six authored summary fields.'); + }else if(model.summary!==null)fail('Pending review cannot expose a reviewed summary.'); + models.set(model.id,model); + } + const pairs=new Set(); + for(const relation of value.relationships){ + if(!exact(relation,['from','to','kind','explanation','evidence_model','excerpt'])|| + !models.has(relation.from)||!models.has(relation.to)||relation.from===relation.to|| + !['comparison','parameter_derivation'].includes(relation.kind)||!text(relation.explanation,8000)|| + !models.has(relation.evidence_model)||!text(relation.excerpt,16384))fail('Invalid authored model relationship.'); + const key=JSON.stringify([relation.from,relation.to,relation.kind]);if(pairs.has(key))fail('Duplicate authored model relationship.');pairs.add(key); + } + const parents=new Map(),complete=new Set(),visiting=new Set(); + for(const relation of value.relationships)if(relation.kind==='parameter_derivation'){ + if(!parents.has(relation.from))parents.set(relation.from,[]);parents.get(relation.from).push(relation.to); + } + function visit(id){if(complete.has(id))return;if(visiting.has(id))fail('Authored parameter derivation contains a cycle.'); + visiting.add(id);for(const parent of parents.get(id)||[])visit(parent);visiting.delete(id);complete.add(id);} + for(const id of parents.keys())visit(id); + return value; + } + function checkCard(document,model,result){ + const ref=model.card,version=result?.version; + if(result?.project_id!==document.project_id||result.id!==ref.file_id||result.path!==ref.path||result.kind!=='note'|| + version?.deleted===true||version?.revision!==ref.revision||version.sha256!==ref.sha256||typeof version.text!=='string') + fail('Model card differs from its exact retained organization reference.'); + if(model.summary&&fields.some(field=>!version.text.includes(model.summary[field])))fail('Authored summary differs from the captured model card.'); + for(const relation of document.relationships)if(relation.evidence_model===model.id&&!version.text.includes(relation.excerpt)) + fail('Authored relationship excerpt differs from its captured model card.'); + return version.text; + } + window.ProjectModelPortfolioDocument=Object.freeze({decode,checkCard}); + const node=(tag,value)=>{const element=document.createElement(tag);if(value!==undefined)element.textContent=value;return element;}; + window.ProjectModelPortfolio=function({root,read,onContextFocusLost=()=>{}}){ + root.innerHTML='Browse a retained guide to models and their methods. Grouping and retrospective comparison notes are separate from recorded Trial ancestry and scientific qualification.
Solid edges show authored grouping only. Model relationships and their source excerpts are listed with the selected model.