diff --git a/agent_api.py b/agent_api.py index 67b99453..90a534c9 100644 --- a/agent_api.py +++ b/agent_api.py @@ -1471,6 +1471,13 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non _LOOKUP = {key: operation for operation in _OPERATIONS for key in (operation['operation_id'], operation['mcp_name'])} # Explicit locally implemented versions. No remotely supplied code or routes. _CONTRACTS = {op['operation_id']: {1: copy.deepcopy(op)} for op in _OPERATIONS} +# Keep the installed v1 bounds/hash exact. Version 2 admits complete larger +# comparisons; each individual packet and every response page retain their caps. +_comparison_capacity=copy.deepcopy(_CONTRACTS['project_files.measurement_compare'][1]) +_comparison_value=_comparison_capacity['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties'] +_comparison_value['observation_count']=integer(1,250000) +_comparison_value['query']['oneOf'][0]['properties']['total']=integer(0,250000) +_CONTRACTS['project_files.measurement_compare'][2]=_comparison_capacity # v1 remains the closed Run-only current custody view. v2 additionally exposes # bounded original local native phases; imported history never populates it. _native_health=copy.deepcopy(_CONTRACTS['project_files.execution_health'][1]) @@ -1974,6 +1981,12 @@ def operation(name, version=None): return copy.deepcopy(versions[max(versions) if version is None else version]) +def measurement_comparison_row_limit(version=None): + """Return the selected immutable public contract's comparison budget.""" + selected=operation('project_files.measurement_compare',version) + return selected['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties']['observation_count']['maximum'] + + def _native_validation_result_ok(value, revision): """Check the finite public truth fields without interpreting native runtime data. @@ -2382,6 +2395,8 @@ def _private_file(path, maximum): class AgentClient: """No catalog access, automatic retries, redirects, proxies or owner identity.""" + MAX_TIMEOUT_SECONDS = 300 + def __init__(self, endpoint, token, expected_catalog_id, timeout=120): try: parsed = urlsplit(endpoint) @@ -2391,7 +2406,7 @@ def __init__(self, endpoint, token, expected_catalog_id, timeout=120): raise ValueError() if not re.fullmatch(r'[A-Za-z0-9_-]{20,4096}', token) or not re.fullmatch('[a-f0-9]{32}', expected_catalog_id): raise ValueError() - if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= 300: + if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= self.MAX_TIMEOUT_SECONDS: raise ValueError() except (ValueError, TypeError): raise AgentAPIError('Invalid fixed loopback client configuration.', 'configuration') from None @@ -3016,6 +3031,10 @@ def call(self, name, arguments, *, observer=None, observation=None): class ProjectFilesOwnerClient(AgentClient): """Explicit owner file-project access; no unrelated owner operations.""" + # Large retained projects may outlast the generic transport's five minutes. + # Only an explicit owner configuration opts into a longer absolute wait. + MAX_TIMEOUT_SECONDS = 7200 + OPERATIONS = frozenset('project_files.' + action for action in ('status', 'new', 'open', 'save', 'close', 'query', 'get', 'import_text', 'annotate')) | PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4 diff --git a/app.py b/app.py index 87c6c9a7..f818cc0b 100644 --- a/app.py +++ b/app.py @@ -919,6 +919,7 @@ def _get(self): '/workspaces.js': ('workspaces.js', 'text/javascript'), '/experiment-graph.js': ('experiment-graph.js', 'text/javascript'), '/project-progression.js': ('project-progression.js', 'text/javascript'), + '/project-model-portfolio.js': ('project-model-portfolio.js', 'text/javascript'), '/project-measurements.js': ('project-measurements.js', 'text/javascript'), '/project-measurements.css': ('project-measurements.css', 'text/css'), '/experiment-lineage.js': ('experiment-lineage.js', 'text/javascript'), diff --git a/docs/contracts/project-files.md b/docs/contracts/project-files.md index e04aa7ac..387c9bc6 100644 --- a/docs/contracts/project-files.md +++ b/docs/contracts/project-files.md @@ -105,6 +105,16 @@ annotations; human review still requires the owner. Every project action is boun to an opaque workspace generation and its expected canonical checkpoint. Use `ProjectFilesOwnerClient` only for the narrow owner project-file operations. +Its explicit `timeout` configuration may be up to 7,200 seconds for large retained +projects; the default remains 120 seconds. The absolute request watchdog still +applies and a transport timeout never authorizes an automatic mutation retry. +Generic clients retain their 300-second maximum. This wait budget is not an +execution deadline or a cancellation request. + +Browser contract tables derive from the local `agent_api` registry. Regenerate +them with `python scripts/generate_project_files_contracts.py`; add `--check` +to verify that the installed v1 pins, all supported version pins and fixed +routes match that registry. Remote schemas remain opaque discovery data. `ProjectFilesClient` wraps a negotiated client. CLI domain `file-*` commands take `--arguments` JSON files; lifecycle commands use explicit flags and select `--owner-project-files` with a diff --git a/docs/contracts/project-model-portfolio.md b/docs/contracts/project-model-portfolio.md new file mode 100644 index 00000000..d7a0a991 --- /dev/null +++ b/docs/contracts/project-model-portfolio.md @@ -0,0 +1,87 @@ +# Authored model organization + +The project workspace can display an adopted `.portfolio.json` note as an +authored guide to model fits/components. This is separate from **Experiment +progression**, which retains genuine Study/Trial ancestry. Group edges mean +authored grouping; they do not establish derivation, admission or scientific +acceptance. Relationship notes have explicit comparison/parameter meanings and +source excerpts. Missing historical Trial ancestry stays unknown. + +## Retained document + +`project_model_portfolio_v1` is a UTF-8 JSON note of at most1MiB with exactly: + +- `format`: the fixed string above; `project_id`: the owning32hex project ID; +- `title` (1–200characters), `description` (1–8,000characters), and + `authorship`: `retrospective`; +- `groups`: at most64records with exact fields `id`, `label`, `parent`, + `description`. IDs are unique, parent is null or another group, and grouping + is acyclic; +- `models`:1–1,000records with exact fields `id`, `label`, `group_id`, `aliases`, + `review`, `card`, `summary`. Each unique ID names an authored model entry, + not necessarily a complete generator, Trial or unique parameter state; +- `relationships`: at most2,000records with exact fields `from`, `to`, `kind`, + `explanation`, `evidence_model`, `excerpt`. All model references exist, endpoints + differ, and each ordered endpoint/kind tuple is unique. + +Group/model IDs contain1–160ASCII letters/digits or `._:/-`, beginning with a +letter/digit. Group labels are at most200characters, model labels/aliases500. +Each model has at most128unique aliases. A card reference has exact fields +`file_id` (64hex), `path` (a relative `notes/` path up to512characters without +empty/dot/dot-dot components), positive integer `revision`, and64hex `sha256`. +These are exact captured note references, never source-reading capabilities. + +Review is `source_reviewed` or `review_pending`. Reviewed summaries have exactly +six nonempty strings, each up to16,384characters: `method`, `purpose`, +`experiment`, `outcome`, `lesson`, `reproduction`. Pending summaries are null. +This status is an author's source-review statement, not numerical qualification. +Relationship kind is `comparison` or `parameter_derivation`; the latter points +from derived model to parameter source and must be acyclic. Explanation is at +most8,000characters and exact excerpt at most16,384. Comparisons are authored +context, not permission to combine incompatible metrics/populations. + +## Ownership and reads + +The ordinary project-file owner captures/revises the note, authenticates reads, +verifies captured UTF-8 bytes/hashes and retains previous revisions. There is no +new canonical family, SQL table, execution grant, recipe or mutation interface. +Existing `files_inspect` and `file_get` contracts are unchanged. Save/load keeps +the original guide and referenced note history. A fork whose new project/note +identities differ requires an explicitly re-authored guide; the reader refuses +foreign identities instead of retargeting them silently. + +`ProjectModelPortfolioDocument` owns the presentation-document decoder and +cross-reference/literal checks. `ProjectModelPortfolio` owns selection, +presentation and its pending reads. Its injected `read` must be the host's +already negotiated, current-authority, byte-verified project-file reader. +The host owns transport/hash validation and context identity. The controller +checks the exact model card ID/path/revision/SHA/project, and that all six +summary strings and that card's relationship excerpts occur literally in its +captured text before displaying a selected summary. Text is inert; no markup, +scripts, parameter loading, scientific scoring or source execution occurs. + +Only explicit actions read notes/cards. Discovery visits40adopted files per +page; no hidden scan/retry/polling occurs. The guide holds at most1,000models, +shows20per graph window with their ancestor groups, and reports exact total, +reviewed and filtered counts. Group/name/alias/review filters, paging and +textual relationship controls select the same entries as graph buttons. +Paragraphs wrap; exact source text remains in a disclosure. + +The host's binding/project/checkpoint/credential/generation context owns every +read. A context change, permission loss or teardown aborts pending work, clears +old content and restores focus to the host status when needed. Late replies +cannot publish into another context. Errors show their declared diagnostic and +clear old detail instead of displaying a previous model as the new selection. +Unsupported format or foreign-project binding is `unsupported`; malformed +document/reference/summary is `invalid`; transport/availability failures retain +their owning diagnostic. No success or empty-data fallback substitutes for a +failed read. + +## Verification scope + +Pure codec tests cover contract decisions using literal fixtures. Controlled +read and real-browser tests cover selection, unavailable/obsolete responses, +repeated invocation, focus, keyboard activation and wrapping on desktop/mobile. +Those fixtures do not establish real project capture or save/load. Native +acceptance additionally requires an actual adopted guide, exact original cards, +current authority, real project reads and retained checkpoint/custody evidence. diff --git a/project_composition.py b/project_composition.py index bcb8a795..53bc04f3 100644 --- a/project_composition.py +++ b/project_composition.py @@ -268,8 +268,23 @@ def allocate(family,identity,revision): records=[indexed[key] for key in sorted(indexed)] validate_source_memberships(records,snapshots,destination_project_id) archived={(r['family'],r['id']) for r in merged['archived']+fork['archived']} + # Every input has its own valid current fork. The combined records receive + # their one new current overlay only in the publication owner. Ask each + # source owner for its preservation policy before combining those roots; + # independent editable forks introduce no new payload identities. + plans={} + inventories=[fork['artifacts']] + inventories.extend(artifact_plan(source['records']) for source in snapshots[1:]) + for inventory in inventories: + for edge in inventory: + sha=edge['declaration']['sha256'];previous=plans.get(sha) + if previous is not None and previous['declaration']!=edge['declaration']: + _fail('Composition source artifact declarations conflict.') + if previous is None or edge['preservation']=='unique': + plans[sha]=copy.deepcopy(edge) return dict(records=records,mapping=copy.deepcopy(fork['mapping']), - archived=[dict(family=f,id=i) for f,i in sorted(archived)],artifacts=artifact_plan(records)) + archived=[dict(family=f,id=i) for f,i in sorted(archived)], + artifacts=[plans[sha] for sha in sorted(plans)]) def publish_composition(primary_store, primary_artifacts, retained_sources, trusted_root_fd, directory, diff --git a/project_custody.py b/project_custody.py index 2e0a97aa..1a188f1d 100644 --- a/project_custody.py +++ b/project_custody.py @@ -197,7 +197,7 @@ def _directory(self, create=False): finally:os.close(parent) def _binding(self, store, projection_id): - snapshot=store.snapshot() + snapshot=store.summary() with store.root_directory() as descriptor: info=os.fstat(descriptor) return dict(host_catalog_id=self.host_catalog_id,project_id=snapshot['project_id'], @@ -247,7 +247,7 @@ def __init__(self, owner, store, binding): persisted=value is not None if value is None: value=dict(format='project_local_custody_v1',**binding, - project_name=store.snapshot()['name'],selected_checkpoint=store.snapshot()['checkpoint'], + project_name=store.summary()['name'],selected_checkpoint=store.summary()['checkpoint'], attempted=None,runs=[],producers=[],runtime_effects=[],package_effects=[]) if any(value[key]!=item for key,item in binding.items()):_fail('Original physical custody binding differs.') self.value=validate(value) diff --git a/project_data.py b/project_data.py index 4a48dd38..ca70bab0 100644 --- a/project_data.py +++ b/project_data.py @@ -1001,7 +1001,7 @@ def _publish(self, raw, kind, role, media_type): declaration = dict(format='project_artifact_v1', sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) self.artifacts.publish_bytes(raw, declaration) self._pending_artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=self.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=self.session.summary['project_id'], sha256=declaration['sha256']), media_type=media_type, kind=kind, role=role, preservation='unique', required_for=['inspect', 'release']) def _require_record(self, record_id, *, extended=False): @@ -1028,7 +1028,7 @@ def _mutate(self, operation, target, body, expected_checkpoint, key, invoke): fail('Data mutation body must be an object.') self._permission(operation, target, body) with self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() if operation in ('data.annotate','data.correct_rights_note','data.save_image_source','data.delete_image_source'): self._require_record(target,extended=operation!='data.annotate') legacy = self._legacy_replay(operation, target, body, key) @@ -1158,10 +1158,10 @@ def freeze_release(self, body, expected_checkpoint, key): def _read(self, action, invoke, target=None): with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() self._sync_sources() result = invoke() - self.session.check() + self.session.check_current() return result def query(self, options=None): @@ -1399,7 +1399,7 @@ def restore(self, snapshot=None): self.db.execute('INSERT INTO saved_searches VALUES (%s,%s,%s,%s,%s,%s)', (record_id,r['name'],r['revision'],encode_retained(r['criteria']),r['created_at'],r['updated_at'])) elif family == 'dataset_exposure': self.db.execute('INSERT INTO workspace_exposures VALUES (%s,%s,%s,%s,%s,%s,%s)', tuple(doc[k] for k in ('id','record_id','source_hash','reason','run_id','release_id','created_at'))) - self.session.check() + self.session.check_current() self._materialize_images_and_releases(owned) return snapshot diff --git a/project_dispatch.py b/project_dispatch.py index 6c9420a2..de519135 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -149,7 +149,7 @@ def _data(projection, action, a): def _definitions(projection, family, a): docs = projection.materials._documents() values = [doc['current'] for (kind, _), doc in docs.items() - if kind == family and doc['origin_catalog_id'] == projection.session.snapshot['project_id']] + if kind == family and doc['origin_catalog_id'] == projection.session.summary['project_id']] return _page(sorted(values, key=lambda item: item['id']), a) @@ -370,7 +370,16 @@ def _m4(projection,action,a): if action=='file_adopt_deletion': return projection.files.adopt_deletion(_body(a),a['expected_checkpoint'],a['request_key']) with session.access.lock,session.lock,session.db.lock: - session.access.require('inspect','project_files.'+action);snapshot=session.check() + session.access.require('inspect','project_files.'+action) + if action=='fork_info': + snapshot=session.check() + from project_fork import decode_fork + overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) + return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], + mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, + copied_count=len(overlay['artifacts']['copied']) if overlay else 0, + missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) + session.check_current() if action=='files_inspect':return _page(projection.files.inspect()['items'],a) if action=='file_get':return projection.files.read(a['file_id'],a.get('revision')) if action=='file_history': @@ -390,13 +399,6 @@ def _m4(projection,action,a): items=[dict(item,requires_reselection=[]) for item in value['editable']] items.extend(dict(family=item['family'],id=item['id'],requires_reselection=item['fields']) for item in value['requires_reselection']) return _page(sorted(items,key=lambda item:(item['family'],item['id'])),a) - if action=='fork_info': - from project_fork import decode_fork - overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) - return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], - mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, - copied_count=len(overlay['artifacts']['copied']) if overlay else 0, - missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) raise AssertionError('Closed M4 dispatch is incomplete.') diff --git a/project_execution.py b/project_execution.py index ca5c36d0..0e4941c6 100644 --- a/project_execution.py +++ b/project_execution.py @@ -52,7 +52,7 @@ def publication_started(self): def confirmed(self): if not self.started:raise RuntimeError('File publication must have a durable start before confirmation.') - execution=self.execution;execution.session.check() + execution=self.execution;execution.session.check_current() if execution.db.in_transaction: raise RuntimeError('File publication confirms only after its canonical and outer SQL acknowledgement.') if execution.custody is not None: @@ -164,7 +164,7 @@ def register_custody(self, identity, handle): def before_launch(self, identity): with self.db.lock: - self.session.check();run=self.runs._load(identity);self._proof(run) + self.session.check_current();run=self.runs._load(identity);self._proof(run) self.runtime_access.require_job('runs',identity,run['admission_id'],run['request_sha256']) if self.custody is not None: with self._journal_guard():self.custody.launch_intent(identity) @@ -176,7 +176,7 @@ def _publication_intent(self, before, records, key, sha): def _local_owner(self, *, recover=True): """Construct execution only after the caller checked exact authority.""" - self.session.check() + self.session.check_current() if self._runs is None: from runs import Runs from pipeline import Pipelines @@ -296,7 +296,7 @@ def _restore_file_writer(self, kind, request_key, input_sha256, content_sha256, authority.require(scope,action,owner_only=True) if self.db.in_transaction: raise RuntimeError('File reconciliation requires no outer SQL transaction.') - if self.session.snapshot['checkpoint']!=expected_checkpoint: + if self.session.summary['checkpoint']!=expected_checkpoint: raise WorkbenchError('Expected current file checkpoint changed.','stale_project',409) if self.custody is None: raise WorkbenchError('No original physical file custody exists.','unavailable',409) @@ -398,7 +398,7 @@ def _producer_authority(self, family, identity, admission_id, request_digest): def register_producer(self, family, identity, admission_id, request_digest): with self.db.lock: - self.session.check();self._producer_authority(family,identity,admission_id,request_digest) + self.session.check_current();self._producer_authority(family,identity,admission_id,request_digest) if self.custody is not None: with self._journal_guard(): if not self.custody.has_pending:self.custody.acknowledge(self.session.snapshot) @@ -406,7 +406,7 @@ def register_producer(self, family, identity, admission_id, request_digest): def producer_effect_intent(self, family, identity): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -416,7 +416,7 @@ def producer_effect_intent(self, family, identity): def update_producer_outcome(self, family, identity, provider_outcome, worker_drained): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -464,7 +464,7 @@ def _authorize_observation(self): # The domain owner has already checked submit/cancel authority; # recheck live host credentials without inventing a worker owner. self.runtime_access.require('inspect', 'runs.observe', identity) - if proof['admission']['catalog_id'] != self.session.snapshot['project_id']: + if proof['admission']['catalog_id'] != self.session.summary['project_id']: raise AccessError('Local custody belongs to another project.') def _capture_run(self, identity): @@ -591,7 +591,7 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt def preflight_external(self, graph_id, revision): self.runtime_access.require('inspect', 'pipeline.preflight_external', graph_id) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() return self.projection.pipelines.preflight_external(graph_id, revision) def native_preflight(self,graph_id,revision,expected_checkpoint,key): @@ -615,7 +615,7 @@ def native_submit(self,body,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.submit',body['graph_id'],owner_only=True) effective=dict(body,actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.submit',body['graph_id'],effective,expected_checkpoint,key) if replay is not None:return replay with self.projection.native_preparation.current_graph(self.projection.pipelines,body['graph_id'], @@ -626,14 +626,14 @@ def invoke(): context.consume_admission() owner=self._local_owner();public=owner.submit(dict(body,request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.submit',body['graph_id'],effective,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.submit',body['graph_id'],effective,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_retry(self,run_id,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.retry',run_id,owner_only=True) body=dict(actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.retry',run_id,body,expected_checkpoint,key) if replay is not None:return replay if self._runs is None:raise WorkbenchError('Retry requires the actual current local Run owner.','unavailable',409) @@ -646,7 +646,7 @@ def invoke(): context.consume_admission() owner=self.runs;public=owner.retry(run_id,dict(request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.retry',run_id,body,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.retry',run_id,body,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_reconcile(self,phase_id,expected_checkpoint): @@ -655,9 +655,9 @@ def native_reconcile(self,phase_id,expected_checkpoint): def submit_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require_external_submission() with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: # A retained request never recreates a local Run/admission or wakes @@ -678,9 +678,9 @@ def invoke(): def approve_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require('approve-execution', 'runs.approve_external', owner_only=True) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: value = prior['result'] diff --git a/project_extraction.py b/project_extraction.py index 82db29c7..461b0d0f 100644 --- a/project_extraction.py +++ b/project_extraction.py @@ -69,7 +69,7 @@ def __init__(self, source): self.payload_gaps = [] self.deleted_intents = set() self.session = SimpleNamespace(snapshot=source.snapshot, db=self.db, - artifacts=source.artifacts, check=lambda: None) + artifacts=source.artifacts, check_current=lambda: None) # Workbench and each metadata-view constructor only assign references; # none runs SQL, creates directories or constructs Dataset/Jobs/Runs. # Never call get/history: those public owners may enroll images. diff --git a/project_files.py b/project_files.py index 08829a6a..ad0f6b3f 100644 --- a/project_files.py +++ b/project_files.py @@ -5,6 +5,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import errno import hashlib import os @@ -128,14 +129,14 @@ def _result(envelope,changed): return result -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): if operation=='files.adopt_many': closed(result,{'items','changed_count'},'batch adoption result') if type(result['items']) is not list or not 1<=len(result['items'])<=MAX_ADOPT_ITEMS: fail('Invalid batch adoption result size.') seen=set();projects=set() for item in result['items']: - validate_operation_result('files.adopt',item,records) + _validate_result('files.adopt',item,lookup) if item['id'] in seen:fail('Batch adoption result repeats a file.') seen.add(item['id']);projects.add(item['project_id']) if (len(projects)!=1 or type(result['changed_count']) is not int @@ -153,14 +154,34 @@ def validate_operation_result(operation,result,records=None): else:sha(result['sha256']) _date(result['adopted_at']) if type(result['revision']) is not int or not 1<=result['revision']<=2**53-1 or type(result['bytes']) is not int or not 0<=result['bytes']<=MAX_FILE_BYTES:fail('Invalid adoption result bounds.') - if records is not None: - owned=decode_records(records) - envelope=next((r for r in owned if r['id']==result['id']),None) + if lookup is not None: + envelope=lookup(result['id']) if envelope is None or not any((value.get('deleted') is True)==deleted and all(value[key]==result[key] for key in ('revision','sha256','bytes','adopted_at')) for value in envelope['document']['history']): fail('Adoption result is not an exact retained working-file revision.') return True +def receipt_validator(records): + """Own one decoded candidate's file histories for its receipt checks. + + The returned validator has no live Session or publication authority. It + owns copied records, so caller mutation cannot change this fixed view. + A different candidate requires a new validator. + """ + owned = {record['id']: record for record in decode_records(records)} + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + # Keep standalone shape/error ordering, and share only this call's lookup + # across batch items. There is no retained cache between invocations. + @cache + def owned(): + return {record['id']: record for record in decode_records(records)} + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class _WorkingError(Exception): def __init__(self,state):self.state=state @@ -231,39 +252,51 @@ def _read(root,parts): os.close(parent) +def inspect_captured(records, project_id, root): + """Observe captured working files using only a locked project capability.""" + return _inspect_validated(decode_records(records), project_id, root) + + +def _inspect_validated(records, project_id, root): + """Consume full-decoder output or fresh values from a validated Session.""" + owned=[r for r in records if r['document']['project_id']==project_id] + items=[] + for envelope in sorted(owned,key=lambda r:r['document']['path']): + doc=envelope['document'];value=doc['current'] + observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], + expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) + try: + parts=_path(doc['kind'],doc['path']) + if value.get('deleted') is True and _missing(root,parts): + observation['state']='deleted' + else: + if value.get('deleted') is True:observation['state']='changed' + _,observed,size=_read(root,parts) + observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') + except _WorkingError as error: + if value.get('deleted') is True: + observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) + else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) + items.append(observation) + return items + + class ProjectFiles: def __init__(self,session): self.session,self.access,self.db,self.store=session,session.access,session.db,session.store def inspect(self): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.inspect');snapshot=self.session.check() - owned=[r for r in decode_records(snapshot['records']) if r['document']['project_id']==snapshot['project_id']];items=[] + self.access.require('inspect','files.inspect');snapshot=self.session.check_records(FAMILY) with self.store.root_directory() as root: - for envelope in sorted(owned,key=lambda r:r['document']['path']): - doc=envelope['document'];value=doc['current'] - observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], - expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) - try: - parts=_path(doc['kind'],doc['path']) - if value.get('deleted') is True and _missing(root,parts): - observation['state']='deleted' - else: - if value.get('deleted') is True:observation['state']='changed' - _,observed,size=_read(root,parts) - observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') - except _WorkingError as error: - if value.get('deleted') is True: - observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) - else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) - items.append(observation) - self.session.check() + items=_inspect_validated(snapshot['records'],snapshot['project_id'],root) + self.session.check_current() return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) def read(self,record_id,revision=None): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.read',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.read',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) values=envelope['document']['history'] if revision is None:value=values[-1] @@ -273,8 +306,8 @@ def read(self,record_id,revision=None): def history(self,record_id): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.history',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.history',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) return copy.deepcopy(envelope['document']['history']) @@ -293,7 +326,7 @@ def materialize_captured(records,project_id,root_fd): def _adoption_intent(self,body): closed(body,{'kind','path','expected_revision','expected_sha256'},'working-file adoption') - parts=_path(body['kind'],body['path']);project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) + parts=_path(body['kind'],body['path']);project_id=self.session.summary['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 0<=body['expected_revision']<=2**53-1:fail('Invalid expected adoption revision.') return dict(body),parts,record_id @@ -309,12 +342,12 @@ def _capture_adoption(self,intent,prior,root): value=dict(revision=revision+1,sha256=observed,bytes=size,text=text,adopted_at=timestamp(),actor_id=actor.actor_id) history=copy.deepcopy(prior['document']['history']) if prior else [];history.append(value) envelope=dict(family=FAMILY,id=record_id,document=dict(format=prior['document']['format'] if prior else FORMAT, - project_id=self.session.snapshot['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), + project_id=self.session.summary['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), history=history,origin=copy.deepcopy(prior['document']['origin']) if prior else None)) return _result(envelope,True),envelope,size def adopt(self,body,expected_checkpoint,key): - intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.snapshot['project_id'] + intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.summary['project_id'] pending=[] with self.access.lock: self.access.require('edit','files.adopt',record_id) @@ -343,7 +376,7 @@ def adopt_many(self,body,expected_checkpoint,key): intents=[self._adoption_intent(item) for item in body['items']] ids=[intent[2] for intent in intents] if len(set(ids))!=len(ids):fail('Batch adoption repeats a working-file path.') - body=dict(items=[intent[0] for intent in intents]);project_id=self.session.snapshot['project_id'] + body=dict(items=[intent[0] for intent in intents]);project_id=self.session.summary['project_id'] target=digest('\0'.join(ids).encode());pending=[] with self.access.lock: for identity in ids:self.access.require('edit','files.adopt',identity) @@ -370,7 +403,7 @@ def adopt_deletion(self,body,expected_checkpoint,key): closed(body,{'path','expected_revision','expected_sha256'},'note deletion adoption') parts=_path('note',body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 1<=body['expected_revision']<=2**53-1:fail('Invalid expected note revision.') - project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] + project_id=self.session.summary['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] with self.access.lock: self.access.require('edit','files.adopt_deletion',record_id) def invoke(): diff --git a/project_fork.py b/project_fork.py index 7aec1a44..f35b9812 100644 --- a/project_fork.py +++ b/project_fork.py @@ -174,10 +174,16 @@ def decode_fork(records, *, project_id=None): return copy.deepcopy(overlays[current]) -def active_records(records, *, project_id=None): +def active_records(records, *, project_id=None, families=None): + """Own selected active copies while validating the complete fork history.""" + if families is not None: + if type(families) is not frozenset: + _fail('Active record families must be an explicit frozenset.') + for family in families:_identifier(family,'Active record family') overlay = decode_fork(records, project_id=project_id) archived = set() if overlay is None else {(item['family'], item['id']) for item in overlay['archived']} - return [copy.deepcopy(item) for item in records if (item['family'], item['id']) not in archived] + return [copy.deepcopy(item) for item in records + if (families is None or item['family'] in families) and (item['family'], item['id']) not in archived] def _inside(candidate_fd, ancestor_fd): @@ -279,6 +285,98 @@ def publish_fork(source_store, source_artifacts, trusted_root_fd, relative, *, r validate_candidate=lambda records,identity,_artifacts:validate_candidate(records,identity)) +def recover_committed_publication(trusted_root_fd, staging, relative, *, root_path, + expected_project_id, expected_checkpoint, + expected_request, quiescent): + """Publish one explicitly selected committed stage without rebuilding it. + + This operator recovery accepts a frozen historical snapshot, not a new read + of its original sources. The caller owns current filesystem authority and + cessation of the former process. The store's exclusive root lock, complete + domain validation, exact original request/commit receipt, payload custody, + and clean captured files are required before a no-overwrite rename. No SQL + projection, source permission, source runtime or worker is restored here. + After an uncertain rename, repeat this exact recovery to inspect the final + destination; never repeat composition or create another stage. + """ + if quiescent is not True: + _fail('Recovery requires proven cessation of the former publication owner.', 'busy', 409) + _uuid(expected_project_id);_sha(expected_checkpoint) + stage_parts=_relative_parts(staging);parts=_relative_parts(relative) + if (stage_parts[:-1]!=parts[:-1] + or not re.fullmatch(r'\.tuldok-fork-[a-f0-9]{32}',stage_parts[-1]) + or stage_parts==parts): + _fail('Recovery requires an explicit fork stage beside its original destination.') + if type(expected_request) is not dict or expected_request.get('directory')!=relative: + _fail('Recovery requires the exact original publication request.') + key=expected_request.get('request_key') + if type(key) is not str or not REQUEST_KEY.fullmatch(key): + _fail('Recovery requires the original publication request key.') + digest=_digest(expected_request) + published=False + try: + with anchored_directory(trusted_root_fd,'/'.join(parts[:-1]),root_path=root_path) as parent: + def exists(name): + try:os.stat(name,dir_fd=parent,follow_symlinks=False);return True + except FileNotFoundError:return False + stage_exists,final_exists=exists(stage_parts[-1]),exists(parts[-1]) + if stage_exists and final_exists: + _fail('Both stage and destination exist; neither was replaced.', 'conflict', 409) + if not stage_exists and not final_exists: + _fail('Neither the selected stage nor its destination is available.', 'unavailable', 409) + selected=relative if final_exists else staging + with ProjectStore.open_at(trusted_root_fd,selected,root_path=root_path) as store: + snapshot=store.snapshot() + if (snapshot['project_id']!=expected_project_id + or snapshot['checkpoint']!=expected_checkpoint): + _fail('Recovery project identity or checkpoint differs.', 'stale', 409) + overlay=decode_fork(snapshot['records'],project_id=expected_project_id) + if overlay is None:_fail('Recovery requires a committed independent publication.') + from project_composition import decode_records, composition_input + composition=decode_records(snapshot['records'],project_id=expected_project_id).get(expected_project_id) + observed=(composition_input(composition['sources'],composition['directory'],composition['name'],composition['request_key']) + if composition is not None else _input(overlay['source'],overlay['directory'],overlay['name'],overlay['request_key'])) + if observed!=expected_request or store.reconcile(key,digest) is None: + _fail('Committed stage does not match the original request and receipt.', 'conflict', 409) + from project_projection import validate_project_records + from project_files import inspect_captured + declarations=_declarations(snapshot['records']) + with ProjectArtifacts(store) as artifacts: + validate_project_records(snapshot['records'],project_id=expected_project_id,artifacts=artifacts) + checks=artifacts.status_many([declarations[identity] for identity in overlay['artifacts']['copied']]) + if any(item['state']!='verified' for item in checks): + _fail('Committed publication payloads are missing or corrupt.', 'unavailable', 409) + with store.root_directory() as project_root: + if any(item['state'] not in ('unchanged','deleted') + for item in inspect_captured(snapshot['records'],expected_project_id,project_root)): + _fail('Committed stage has changed or missing working files.', 'working_files_changed', 409) + store.save(expected_checkpoint) + if not final_exists: + # Recheck the selected directory entry while the store root + # remains locked; an external rename must not substitute it. + with store.root_directory() as project_root: + entry=os.stat(stage_parts[-1],dir_fd=parent,follow_symlinks=False) + actual=os.fstat(project_root) + if (entry.st_dev,entry.st_ino)!=(actual.st_dev,actual.st_ino): + _fail('Selected stage directory changed.', 'stale', 409) + _rename_noreplace(parent,stage_parts[-1],parts[-1]);published=True + os.fsync(parent) + return dict(project_id=expected_project_id,checkpoint=expected_checkpoint, + directory=relative,request_key=key,reconciled=final_exists, + payloads_copied=0,selected=False) + except BaseException as error: + # Anchored directory owners also translate IO failures. Preserve the + # uncertain post-rename outcome through either exception representation. + if published: + raise ProjectForkError('Committed publication was renamed but acknowledgement is uncertain; reconcile this exact recovery.', + 'unknown',503) from error + if isinstance(error,OSError): + code='conflict' if error.errno==errno.EEXIST else 'unavailable' + raise ProjectForkError('Committed publication recovery is unavailable; no stage was rebuilt.', + code,503 if code!='conflict' else 409) from error + raise + + def _publish_project(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, name, expected_checkpoint, request_key, quiescent, build_candidate, validate_candidate, retained_sources=()): diff --git a/project_history.py b/project_history.py index 892ae3c2..b69eccce 100644 --- a/project_history.py +++ b/project_history.py @@ -750,17 +750,29 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): from project_material import ProjectMaterial materials=ProjectMaterial(session) self.materials=materials;self.authorize_disclosure=authorize_disclosure + self._decoded_checkpoint=None + self._decoded_facts=None def _facts(self): - self.session.check() + self.session.check_current() + summary=self.session.summary + checkpoint=(summary['project_id'],summary['checkpoint']) + if self._decoded_checkpoint==checkpoint: + return self._decoded_facts records=self.session.snapshot['records'] - return decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) + facts=decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) + # Current canonical bytes are guarded on every call. Only this owner's + # private decoded metadata is retained; returned values and permissions + # remain the responsibility of each read below. + self._decoded_facts=facts + self._decoded_checkpoint=checkpoint + return facts def _authorize(self,action,target=None): self.access.require('inspect',action,target) def _origin(self,origin): - return self.session.snapshot['project_id'] if origin is None else origin + return self.session.summary['project_id'] if origin is None else origin def _record(self,collection,identity,origin=None): origin=self._origin(origin);identifier(origin,'Origin catalog');identifier(identity) @@ -1163,7 +1175,7 @@ def create_study(self,body,expected_checkpoint,request_key): closed(body,('name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']);policy=validate_retained_policy(body['policy']) def build(facts): - origin=self.session.snapshot['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) + origin=self.session.summary['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) prior=next((key for key,value in facts.policies.items() if key[0]==origin and key[3]==sha),None) policy_id=prior[1] if prior else uuid.uuid4().hex ref={'kind':'selection_policy','catalog_id':origin,'id':policy_id,'revision':1,'sha256':sha} @@ -1180,7 +1192,7 @@ def save_study(self,study_id,body,expected_checkpoint,request_key): closed(body,('revision','name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['policy'],'selection_policy',origin) policy=facts.policies.get((origin,ref['id'],ref['revision'],ref['sha256'])) @@ -1201,7 +1213,7 @@ def prepare_trial_from_material(self,study_id,body,expected_checkpoint,request_k if request_key not in operation_records(self.session.snapshot['records']): self.materials.recipe(body['material'],verify=True) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['material'],'recipe_material',origin);self.materials.recipe(ref) @@ -1224,7 +1236,7 @@ def select_candidate(self,study_id,body,expected_checkpoint,request_key): identifier(body['trial_id']);identifier(body['run_id']) # Expensive semantic proof precedes the write transaction. All exact # associations and selected policy are checked again under publication. - origin=self.session.snapshot['project_id'] + origin=self.session.summary['project_id'] self.access.require('edit','studies.select_candidate',study_id) evidence=(self.evidence(body['run_id'],{},origin_catalog_id=origin) if request_key not in operation_records(self.session.snapshot['records']) else None) @@ -1254,7 +1266,7 @@ def build(facts): def record_lineage(self,body,expected_checkpoint,request_key): closed(body,('trial','intent'));body=checked_json(body,INTENT_LIMIT+4096) def build(facts): - origin=self.session.snapshot['project_id'];trial=facts.trial_ref(body['trial']) + origin=self.session.summary['project_id'];trial=facts.trial_ref(body['trial']) if body['trial']['catalog_id']!=origin:_invalid('Inherited Trials remain immutable historical evidence.','conflict') doc=copy.deepcopy(facts.trials[(origin,trial['id'])]) if doc['lineage'] is not None:_invalid('Trial lineage is already recorded and immutable.','conflict') diff --git a/project_material.py b/project_material.py index 194e91a5..fbfa0189 100644 --- a/project_material.py +++ b/project_material.py @@ -359,7 +359,7 @@ def __init__(self, session): self.owner_access = _MaterialAccess(session) if hasattr(session,'access') else None def _documents(self): - self.session.check() + self.session.check_current() return decode_records(self.session.snapshot['records']) def read_view(self, records): @@ -369,7 +369,7 @@ def read_view(self, records): live publication, verification or execution capability; later reads must select their own checkpoint and construct a new view. """ - self.session.check() + self.session.check_current() return MaterialSnapshot(records) @_boundary @@ -381,7 +381,7 @@ def intents(self): from project_material_fork import is_intent documents = self._documents() active = {(r['family'], r['id']) for r in active_records( - self.session.snapshot['records'], project_id=self.session.snapshot['project_id'])} + self.session.snapshot['records'], project_id=self.session.summary['project_id'])} return [dict(family=family, id=locator, current=copy.deepcopy(doc['current']), fork_origin=copy.deepcopy(doc['fork_origin'])) for (family, locator), doc in sorted(documents.items()) @@ -392,7 +392,7 @@ def intent_capabilities(self): intents = self.intents() unresolved = {(r['family'],r['id']) for r in intents} from project_fork import active_records - active = active_records(self.session.snapshot['records'], project_id=self.session.snapshot['project_id']) + active = active_records(self.session.snapshot['records'], project_id=self.session.summary['project_id']) return dict(requires_reselection=[dict(family=r['family'],id=r['id'],fields=r['current']['required_reselection']) for r in intents], editable=[dict(family=r['family'],id=r['id']) for r in active if r['family'] in EDITABLE and (r['family'],r['id']) not in unresolved]) @@ -461,7 +461,7 @@ def snapshot_ref(self, project_id, manifest_sha256, *, catalog_id=None, verify=F return copy.deepcopy(doc['value']) def capture_definition(self, family, value, before_records, *, action, actor_id=None, audit_facts=None): - catalog_id = self.session.snapshot['project_id'] + catalog_id = self.session.summary['project_id'] value = _definition(family, value, catalog_id) key = catalog_id + '.' + value['id'] prior = decode_records(before_records).get((family, key)) diff --git a/project_measurements.py b/project_measurements.py index 2800b50b..e64a307e 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -6,6 +6,7 @@ """ import copy from datetime import datetime +from functools import cache, partial from project_artifacts import validate_declaration from project_material import decode_records as material_records @@ -101,33 +102,55 @@ def decode_records(records, *, artifacts=None): return result -def validate_operation_result(operation, result, records=None): +def _validate_result(operation, result, lookup=None): if operation != 'measurements.import': return False _document(result) - if records is not None: + if lookup is not None: identity = result['origin_catalog_id'] + '.' + result['id'] - if decode_records(records).get(identity) != result: + if lookup(identity) != result: raise WorkbenchError('Measurement receipt has no exact retained source.') return True +def receipt_validator(records, *, artifacts=None): + """Own one candidate's validated source bindings and packet custody. + + Decode and copy every retained measurement once, including current available + packet bytes when supplied. The returned receipt checker is fixed to those + facts; it carries no live source or publication capability. + """ + owned = decode_records(records, artifacts=artifacts) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return decode_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class ProjectMeasurements: def __init__(self, session): self.session, self.artifacts = session, session.artifacts self.access, self.db = session.access, session.db def _documents(self): - return decode_records(self.session.snapshot['records']) + # Session adoption/publication already validates the complete candidate. + # Verify current canonical bytes, then own only the measurement family. + records = self.session.check_records(FAMILY)['records'] + return {item['id']: item['document'] for item in records} def import_packet(self, raw, source, expected_checkpoint, key): """Called only after SourceAdapter's explicit current rooted capture.""" with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'measurements.import') - self.session.check() + self.session.check_current() table = MeasurementTable(raw) blob = dict(sha256=table.sha256, bytes=len(raw)) - origin = self.session.snapshot['project_id'] + origin = self.session.summary['project_id'] request = dict(blob=blob, source=copy.deepcopy(source)) def invoke(): @@ -152,11 +175,11 @@ def sources(self, *, offset=0, limit=40): raise WorkbenchError('Invalid measurement source page.') with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', 'measurements.sources') - self.session.check() + self.session.check_current() docs = sorted(self._documents().values(), key=lambda d: (d['origin_catalog_id'], d['id'])) items = [dict(doc, availability=self.artifacts.inspect(declaration(doc['blob']))) for doc in docs[offset:offset+limit]] - self.session.check() + self.session.check_current() self.access.require('inspect', 'measurements.sources') return self._bounded(dict(items=items, total=len(docs), offset=offset, limit=limit)) @@ -172,7 +195,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): target = origin_catalog_id + '.' + packet_sha256 with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() doc = self._documents().get(target) if doc is None: raise WorkbenchError('Measurement source is unavailable.', 'not_found', 404) @@ -181,7 +204,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): if table.metadata()['title'] != doc['title']: raise WorkbenchError('Measurement title differs from the exact retained packet.') value = self._bounded(dict(source=doc, value=select(table))) - self.session.check() + self.session.check_current() self.access.require('inspect', action, target) return value @@ -215,7 +238,7 @@ def compare(self, sources, metric_id=None, **selection): with self.access.lock, self.session.lock, self.db.lock: for target in targets: self.access.require('inspect', 'measurements.compare', target) - self.session.check() + self.session.check_current() documents = self._documents() if any(target not in documents for target in targets): raise WorkbenchError('A comparison source is unavailable.', 'not_found', 404) @@ -231,7 +254,7 @@ def compare(self, sources, metric_id=None, **selection): tables.append((locator, table)) value = MeasurementComparison(tables).read(metric_id, **selection) result = self._bounded(dict(sources=chosen, value=value)) - self.session.check() + self.session.check_current() for target in targets: self.access.require('inspect', 'measurements.compare', target) return result diff --git a/project_native_execution.py b/project_native_execution.py index 832a990f..5895e4d7 100644 --- a/project_native_execution.py +++ b/project_native_execution.py @@ -452,7 +452,7 @@ def identity(info): def _publish(self, document, state, *, helper=None, receipt_sha256=None): with self.db.lock,self.db.transaction(): - self.session.check();actor=self._authorize(document['phase_id']) + self.session.check_current();actor=self._authorize(document['phase_id']) document['events'].append(dict(state=state,at=timestamp(),actor_id=actor.actor_id, helper=copy.deepcopy(helper),receipt_sha256=receipt_sha256)) self._dirty[document['phase_id']]=copy.deepcopy(document) @@ -473,7 +473,7 @@ def reconcile(self,phase_id,expected_checkpoint): execution=self.execution;journal=execution.custody if journal is None:_fail('There is no original local native custody.','unavailable') with self.db.lock: - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') local=journal.refresh() selected=next((row for row in local.get('native_phases',[]) if row['phase_id']==phase_id),None) if selected is None:_fail('A retained phase alone cannot select current native custody.','unavailable') @@ -501,7 +501,7 @@ def reconcile(self,phase_id,expected_checkpoint): if self._active:_fail('Current native owner has not finished its bounded observation.','busy') with execution.access.lock,self.session.lock,self.db.lock: self.access.require('cancel','native_execution.reconcile',phase_id,owner_only=True) - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') if journal.refresh()!=local:_fail('Original native custody changed while draining.','stale_project') binding=journal.binding if (binding['project_id']!=self.db.dataset_id or binding['projection_id']!=self.db.projection_id @@ -788,7 +788,7 @@ def _probe_result(recipe, inputs, evidence): def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, run_id=None, job=None, phase_id=None): if self.db.in_transaction:_fail('Native phase cannot launch inside a metadata transaction.','busy') with self.db.lock: - self.session.check() + self.session.check_current() if run_id is None and self._foreground_phase is not None: _fail('Another explicit native preparation phase is active.','busy') if not self.db.claim_runtime('project_native_phases'):_fail('Another current native phase owner is active.','busy') @@ -818,7 +818,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru self._publish(document,'prepared') if self.execution.custody is None:_fail('Native launch requires its original physical custody journal.','unavailable') with self.db.lock,self.execution._journal_guard(): - self._dispatch_authorized(phase_id);self.session.check() + self._dispatch_authorized(phase_id);self.session.check_current() if not self.execution.custody.has_pending:self.execution.custody.acknowledge(self.session.snapshot) self.execution.custody.native_prepared(document) self.execution.custody.native_launch_intent(phase_id) @@ -847,7 +847,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru # Recheck actual authority after both durable ACKs. A # saved GO intent is never permission to retry this send. with self.db.lock: - self.session.check();self._dispatch_authorized(phase_id) + self.session.check_current();self._dispatch_authorized(phase_id) if not self.execution.custody.has_pending:_fail('Native local custody disappeared.','unknown') prepared.check_current_runtime() handle.go(handle.ready) diff --git a/project_native_preparation.py b/project_native_preparation.py index 85a0b1be..24113f63 100644 --- a/project_native_preparation.py +++ b/project_native_preparation.py @@ -188,7 +188,7 @@ def validate_operation_result(operation,result,records=None): _fail('Native intent source or frozen input differs from its graph.','invalid') from types import SimpleNamespace from project_material import ProjectMaterial - retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check=lambda:None)) + retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None)) if result['model'] is not None:retained.model(result['model']) if result['composition'] is not None: composition=retained.recipe(result['composition'])['material'] @@ -230,7 +230,7 @@ def __init__(self,adapter,owner,graph,model,composition,purpose,key): def _check(self,recipe): if not self._active or _CURRENT.get() is not self or recipe!=self.graph['recipe']: _fail('An explicit current native preparation is required.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.probe_native',recipe,owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Native preparation caller changed.','conflict') if (os.environ.get('TULDOK_TRAINING_PYTHON') or sys.executable)!=self.configured_executable: @@ -262,7 +262,7 @@ def probe(self,owner,recipe): if self._probe is None: if self.adapter.db.in_transaction:_fail('Native probe cannot run inside a metadata transaction.','unsupported') self.authorize_probe(recipe) - result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.snapshot['checkpoint'], + result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.summary['checkpoint'], request_key=self.key+'.probe',preparation=self) self._probe=copy.deepcopy(result['runtime']);self.phase=copy.deepcopy(result['phase']) return copy.deepcopy(self._probe) @@ -297,7 +297,7 @@ def __init__(self,adapter,owner,graph): def consume_admission(self): if not self._active or self._consumed:_fail('Current integrity qualification has already been consumed.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.snapshot',self.graph['id'],owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Current integrity caller changed.','conflict') if self.owner.get(self.graph['id'])!=self.graph or self.owner.definition_identity(self.recipe)!=self.definition: @@ -333,8 +333,8 @@ def _scope(self,owner,graph,model,composition,purpose,expected_checkpoint,key): if self.db.in_transaction:_fail('Native preparation must precede the metadata transaction.','unsupported') if _CURRENT.get() is not None:_fail('A native preparation is already active.','busy') self.access.require('edit','native_preparation.'+purpose,graph['id'],owner_only=True) - self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + self.session.check_current() + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') context=NativePreparationContext(self,owner,graph,model,composition,purpose,_key(purpose,key)) result=dict(format='project_native_preparation_intent_v1',origin_catalog_id=self.session.project_id, actor_id=context.actor.actor_id,purpose=purpose,graph=dict(id=graph['id'],revision=graph['revision'], @@ -361,22 +361,22 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): if graph['revision']!=revision:_fail('Graph changed before native validation.','stale') import release_validation if adapter_for_graph(graph) is release_validation: - self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + self.session.check_current() + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') value=owner.validate_graph(graph) return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') with self._scope(owner,graph,None,None,'validation',expected_checkpoint,key) as context: value=owner.validate_graph(graph);context._check(graph['recipe']) - return self.session.mutate('native_preparation.validate',graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate('native_preparation.validate',graph_id,body,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') @contextmanager def current_graph(self,owner,graph_id,revision,expected_checkpoint,key): if self.db.in_transaction:_fail('Current native validation must precede admission.','unsupported') self.access.require('edit','native_preparation.admission',graph_id,owner_only=True) - self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + self.session.check_current() + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') graph=owner.get(graph_id) if graph['revision']!=revision:_fail('Graph changed before native admission.','stale') import release_validation @@ -413,7 +413,7 @@ def capture(value,before): record=self.materials.capture_recipe_material(value,owner.material_path) return dict(records=self.materials._coalesce([record,*self.materials.declarations(record), *self.materials.capture_recipe_dependencies(record,before)]),result=value) - return self.session.mutate(operation,graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate(operation,graph_id,body,self.session.summary['checkpoint'],key, lambda:owner._retain_material(prepared),capture) finally: if prepared['staging'].exists():shutil.rmtree(prepared['staging']) diff --git a/project_producer_execution.py b/project_producer_execution.py index b4456aba..e3729233 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -73,7 +73,7 @@ def _family(self, family): return family def _owner(self, family): - self._family(family);self.session.check() + self._family(family);self.session.check_current() if self._fenced: raise WorkbenchError('Current producer custody is uncertain. Reopen for inert inspection.', 'outcome_unknown', 409) self._refuse_unknown_provider() @@ -264,7 +264,7 @@ def _settle_drained(self): # Every effectful request is preceded by canonical uncertainty; # a terminal job with no such request is known not admitted. outcome='not_admitted' - observation=(outcome,True,self.session.snapshot['checkpoint']) + observation=(outcome,True,self.session.summary['checkpoint']) if self._journal_observations.get(identity_key)!=observation: self.execution.update_producer_outcome(family,producer_id,outcome,True) self._journal_observations[identity_key]=observation @@ -429,7 +429,7 @@ def status(self, *, offset=0, limit=40): raise WorkbenchError('Invalid current producer health page.') with self.session.access.lock,self.session.lock,self.db.lock: self.session.access.require('inspect','producers.health') - self.session.check() + self.session.check_current() self._settle_drained() items=[] for family,owner in self._owners.items(): diff --git a/project_producers.py b/project_producers.py index 79b97f20..8e052b34 100644 --- a/project_producers.py +++ b/project_producers.py @@ -592,9 +592,9 @@ def __init__(self, session): def _read(self, action, invoke): with self.session.access.lock, self.session.lock, self.session.db.lock: self.session.access.require('inspect', action) - self.session.check() + self.session.check_current() result = invoke(self.session.snapshot['records']) - self.session.check() + self.session.check_current() return result def list(self, family, *, origin_catalog_id=None, offset=0, limit=40): @@ -915,7 +915,7 @@ def _publish(owner, raw, role, media='application/json'): fail('Conflicting exact producer payload bytes.') owner._pending_payloads[declaration['sha256']] = raw owner._artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=owner.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=owner.session.summary['project_id'], sha256=declaration['sha256']), media_type=media, kind='producer_evidence', role=role, preservation='unique', required_for=['verify_producer']) diff --git a/project_projection.py b/project_projection.py index fca264be..25b6bf28 100644 --- a/project_projection.py +++ b/project_projection.py @@ -25,14 +25,17 @@ from project_producer_execution import ProjectProducerExecution, validate_operation_result as validate_producer_result from project_files import ProjectFiles, decode_records as decode_file_records from project_files import validate_operation_result as validate_file_result +from project_files import receipt_validator as file_receipt_validator from project_study_execution import ProjectStudyExecution, validate_operation_result as validate_study_execution_result from project_runtime import ProjectRuntime, validate_operation_result as validate_runtime_result from project_native_execution import FAMILY as NATIVE_FAMILY, decode_records as decode_native_records from project_native_preparation import ProjectNativePreparation, validate_operation_result as validate_preparation_result from project_sources import SourceAdapter, validate_operation_result as validate_source_result +from project_sources import receipt_validator as source_receipt_validator from project_measurements import ProjectMeasurements, FAMILY as MEASUREMENT_FAMILY from project_measurements import decode_records as decode_measurement_records from project_measurements import validate_operation_result as validate_measurement_result +from project_measurements import receipt_validator as measurement_receipt_validator from workbench import WorkbenchError from workbench import timestamp @@ -58,12 +61,66 @@ def validate_artifact_resolution(operation, result, records): return True +def validate_project_records(records, *, project_id, artifacts): + """Validate one inert candidate through every owning domain, without SQL reconstruction.""" + claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( + MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) + identities = set() + declarations = {} + for item in records: + identity = (item['family'], item['id']) + if item['family'] not in claimed or identity in identities: + raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) + identities.add(identity) + if item['family'] == 'project_artifact': + declaration = validate_declaration(item['document']) + if declaration['sha256'] != item['id']: + raise WorkbenchError('Artifact inventory identity differs.') + declarations[item['id']] = declaration + decode_data_records(records) + from project_fork import decode_fork + decode_fork(records, project_id=project_id) + from project_composition import decode_records as decode_compositions + decode_compositions(records, project_id=project_id) + from project_transfer import decode_records as decode_extraction_records + decode_extraction_records(records) + operation_records(records, project_id=project_id) + validate_file_receipt = file_receipt_validator(records) + decode_native_records(records,artifacts=artifacts) + materials = MaterialSnapshot(records) + for blob in materials.artifact_blobs(): + if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): + raise WorkbenchError('Material artifact declaration is missing or inconsistent.') + # Resolve every cross-owner reference against THIS candidate, rather + # than a resolver that still sees the preceding committed checkpoint. + decode_producer_records(records, artifacts=artifacts) + decode_history_records(records, materials=materials, artifacts=artifacts) + validate_measurement_receipt = measurement_receipt_validator(records, artifacts=artifacts) + validate_source_receipt = source_receipt_validator(records) + for receipt in operation_documents(records): + operation, result = receipt['operation'], receipt['result'] + if (validate_data_result(operation, result, records) + or validate_material_result(operation, result, records) + or validate_history_result(operation, result, records) + or validate_source_receipt(operation, result) + or validate_measurement_receipt(operation, result) + or validate_execution_result(operation, result, records) + or validate_producer_result(operation, receipt['target'], result, records) + or validate_file_receipt(operation, result) + or validate_study_execution_result(operation, result, records) + or validate_runtime_result(operation, result, records) + or validate_preparation_result(operation, result, records) + or validate_artifact_resolution(operation, result, records)): + continue + raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + + class ProjectProjection: def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None): self.store, self.db, self.access = store, db, access self.custody = custody self.pending_original = [] - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.artifacts = ProjectArtifacts(store) self.sources = None self._disclosures = set() @@ -127,57 +184,9 @@ def execute(self, *args, **kwargs): return self.db.execute(*args, **kwargs) def validate_records(self, records, *, project_id=None, artifacts=None): - artifacts = self.artifacts if artifacts is None else artifacts - project_id = self.project_id if project_id is None else project_id - claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( - MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) - identities = set() - declarations = {} - for item in records: - identity = (item['family'], item['id']) - if item['family'] not in claimed or identity in identities: - raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) - identities.add(identity) - if item['family'] == 'project_artifact': - declaration = validate_declaration(item['document']) - if declaration['sha256'] != item['id']: - raise WorkbenchError('Artifact inventory identity differs.') - declarations[item['id']] = declaration - decode_data_records(records) - from project_fork import decode_fork - decode_fork(records, project_id=project_id) - from project_composition import decode_records as decode_compositions - decode_compositions(records, project_id=project_id) - from project_transfer import decode_records as decode_extraction_records - decode_extraction_records(records) - operation_records(records, project_id=project_id) - decode_file_records(records) - decode_native_records(records,artifacts=artifacts) - materials = MaterialSnapshot(records) - for blob in materials.artifact_blobs(): - if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): - raise WorkbenchError('Material artifact declaration is missing or inconsistent.') - # Resolve every cross-owner reference against THIS candidate, rather - # than a resolver that still sees the preceding committed checkpoint. - decode_producer_records(records, artifacts=artifacts) - decode_history_records(records, materials=materials, artifacts=artifacts) - decode_measurement_records(records, artifacts=artifacts) - for receipt in operation_documents(records): - operation, result = receipt['operation'], receipt['result'] - if (validate_data_result(operation, result, records) - or validate_material_result(operation, result, records) - or validate_history_result(operation, result, records) - or validate_source_result(operation, result, records) - or validate_measurement_result(operation, result, records) - or validate_execution_result(operation, result, records) - or validate_producer_result(operation, receipt['target'], result, records) - or validate_file_result(operation, result, records) - or validate_study_execution_result(operation, result, records) - or validate_runtime_result(operation, result, records) - or validate_preparation_result(operation, result, records) - or validate_artifact_resolution(operation, result, records)): - continue - raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + return validate_project_records(records, + project_id=self.project_id if project_id is None else project_id, + artifacts=self.artifacts if artifacts is None else artifacts) def resolve_artifact(self, body, expected_checkpoint, key): """Explicitly supply exact bytes from a configured current local root.""" @@ -185,7 +194,7 @@ def resolve_artifact(self, body, expected_checkpoint, key): raise WorkbenchError('Choose one declared artifact and a current rooted file.') with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'artifacts.resolve', body['sha256'], owner_only=True) - self.session.check() + self.session.check_current() declaration = next((item['document'] for item in self.session.snapshot['records'] if item['family'] == 'project_artifact' and item['id'] == body['sha256']), None) if declaration is None: @@ -241,7 +250,7 @@ def runtime_configuration(self, request): def runtime_sources(self): with self.access.lock, self.session.lock, self.db.lock: self.access.require('admin', 'runtime.sources', owner_only=True) - self.session.check() + self.session.check_current() return dict(sources=sorted(self._runtime_sources)) def authorize_disclosure(self, run, population): @@ -308,14 +317,15 @@ def available(self): return self.projection.session.available def _tag(self, value): - snapshot = self.projection.session.snapshot - return dict(value, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint']) + summary = self.projection.session.summary + return dict(value, project_id=summary['project_id'], checkpoint=summary['checkpoint']) def records(self): - self.projection.session.check() + self.projection.session.check_current() from project_fork import active_records snapshot = self.projection.session.snapshot - records = active_records(snapshot['records'], project_id=snapshot['project_id']) + records = active_records(snapshot['records'], project_id=snapshot['project_id'], + families=frozenset(('workbench_record', 'workbench_text'))) return self._tag(dict(items=[copy.deepcopy(item['document']['current']) for item in records if item['family'] in ('workbench_record', 'workbench_text') and item['document']['current']['kind'] == 'text'])) diff --git a/project_runtime.py b/project_runtime.py index a94cc6e6..d63a9eb5 100644 --- a/project_runtime.py +++ b/project_runtime.py @@ -156,7 +156,7 @@ def _authorize(self, action): def inspect(self, config_request): """Read an explicitly selected current source; never provision or execute.""" with self.access.lock, self.session.lock, self.db.lock: - self._authorize('runtime_snapshots.inspect'); self.session.check() + self._authorize('runtime_snapshots.inspect'); self.session.check_current() _body(config_request, '0'*64, 0) with self.execution.runtime_inspection(): return self.owner.inspect(self.resolve_configuration(copy.deepcopy(config_request))) @@ -165,7 +165,7 @@ def status(self, key): """Read historical outcome only; captured does not claim local readiness.""" _key(key) with self.access.lock, self.session.lock, self.db.lock: - self.access.require('inspect', 'runtime.capture_status'); self.session.check() + self.access.require('inspect', 'runtime.capture_status'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) intent = receipts.get(_intent_key(key)); complete = receipts.get(key) if intent is not None and intent['operation'] != 'runtime.capture_intent': intent = None @@ -185,7 +185,7 @@ def recover_capture(self, config_request, expected_inventory_sha256, expected_ch body_sha = _intent_input(body) self._authorize('runtime_snapshots.create') self.execution.prepare_runtime_reconciliation(key, body_sha, expected_inventory_sha256, expected_checkpoint) - return self.capture(config_request, expected_inventory_sha256, self.session.snapshot['checkpoint'], key, + return self.capture(config_request, expected_inventory_sha256, self.session.summary['checkpoint'], key, reserve_bytes=reserve_bytes) def abandon(self, capture_key, reason, expected_checkpoint, key): @@ -212,7 +212,7 @@ def forbidden(): _fail('No original local runtime copy exists to abandon.', 'una _fail('Runtime intent differs from its exact original local writer.', 'conflict', 409) if capture_key in receipts: _fail('Capture completed; reconcile its acknowledgement with the exact capture request.', 'conflict', 409) - observed = self.session.snapshot['checkpoint'] + observed = self.session.summary['checkpoint'] def invoke(): current = self._authorize('runtime.capture_abandon') if current.actor_id != actor.actor_id: _fail('Runtime abandonment caller changed.', 'conflict', 409) @@ -248,7 +248,7 @@ def capture(self, config_request, expected_inventory_sha256, expected_checkpoint body_sha = _intent_input(body) intent = None with self.access.lock, self.session.lock, self.db.lock: - actor = self._authorize('runtime_snapshots.create'); self.session.check() + actor = self._authorize('runtime_snapshots.create'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) if any(r['operation']=='runtime.abandon' and r['result']['capture_key']==key for r in receipts.values()): _fail('This capture was abandoned; use a fresh key for a new measurement.', 'conflict', 409) @@ -273,7 +273,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), invalid, lambda *args: invalid(), scope='admin') else: - if expected_checkpoint != self.session.snapshot['checkpoint']: + if expected_checkpoint != self.session.summary['checkpoint']: _fail('Expected project checkpoint changed.', 'stale_project', 409) config = self.resolve_configuration(copy.deepcopy(config_request)) observed = self.owner.inspect(config) @@ -281,7 +281,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 _fail('Runtime inventory changed after inspection.', 'stale', 409) intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key), + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key), lambda result,before:dict(records=[],result=result), scope='admin') guard.publication_started() if os.path.lexists(self.owner.path / expected_inventory_sha256): @@ -294,7 +294,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 from pipeline import digest from pathlib import Path raw = read_regular(Path(descriptor['root'])/'manifest.json', MAX_MANIFEST) - runtime_ref = dict(kind='runtime',catalog_id=self.session.snapshot['project_id'], + runtime_ref = dict(kind='runtime',catalog_id=self.session.summary['project_id'], id=digest(descriptor),sha256=digest(descriptor)) value = dict(ref=runtime_ref,descriptor=descriptor,environment=None,manifest=decode(raw)) observed = dict(inventory_sha256=descriptor['manifest_sha256'],file_count=descriptor['file_count'],total_bytes=descriptor['total_bytes']) @@ -302,8 +302,8 @@ def capture(result, before): record = self.materials.capture_runtime(value, raw) return dict(records=[*self.materials.declarations(record),record],result=result) result = self.session.mutate('runtime.capture', expected_inventory_sha256, body, - self.session.snapshot['checkpoint'], key, - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') + self.session.summary['checkpoint'], key, + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') guard.confirmed() return result except BaseException as error: diff --git a/project_session.py b/project_session.py index 46c106cd..aa471c1e 100644 --- a/project_session.py +++ b/project_session.py @@ -9,6 +9,7 @@ import threading import agent_api +from project_store import copy_validated_json from workbench import WorkbenchError @@ -65,7 +66,8 @@ def operation_records(records, *, project_id=None): if project_id is None and overlay is not None: project_id = overlay['destination_project_id'] result = {} - for value in operation_documents(active_records(records, project_id=project_id)): + for value in operation_documents(active_records(records, project_id=project_id, + families=frozenset({OPERATION_FAMILY}))): if overlay is not None and value['format'] == 'project_operation_v1': raise ProjectSessionError('A fork must retain legacy operation receipts as archived historical facts.') if project_id is not None and value.get('project_id', project_id) != project_id: @@ -85,7 +87,7 @@ def __init__(self, store, db, access, artifacts, *, validate_records): self._validate = validate_records self.available = False self._snapshot = None - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.extra_capture = None self.publication_intent = None self.runtime_access = None @@ -176,10 +178,16 @@ def reconcile_selection(self, snapshot): self._recovery_records = None self._recovery_request = None + @property + def summary(self): + """Scalar metadata for the same bound checkpoint as snapshot.""" + with self.lock: + return {key: self._snapshot[key] for key in ('project_id', 'name', 'checkpoint')} + @property def snapshot(self): with self.lock: - return copy.deepcopy(self._snapshot) + return copy_validated_json(self._snapshot) def _validated(self, records): operation_records(records, project_id=self.project_id) @@ -196,18 +204,45 @@ def adopt(self, snapshot): current = self.store.snapshot() if current != snapshot: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._snapshot = copy.deepcopy(snapshot) + self._snapshot = copy_validated_json(snapshot) self.available = True - def check(self): - """Check freshness while a caller holds host/session/projection guards.""" + def _checked(self, read): if not self.available: raise ProjectSessionError('Reopen this project to reconstruct its projection.', 'projection_unavailable', 503) - current = self.store.snapshot() + current = read() if current['checkpoint'] != self._snapshot['checkpoint']: raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) return current + def check_current(self): + """Verify actual canonical bytes without returning an unused record graph. + + The caller holds the same host/session/projection guards as check(). + Store.summary reads and hashes all current records before returning. + """ + self._checked(self.store.summary) + + def check(self): + """Return independent current records under caller-owned projection guards.""" + return self._checked(self.store.snapshot) + + def check_records(self, family): + """Verify every canonical record, then copy one bound record family. + + The complete snapshot, including cross-family closure, was validated + and isolated by adopt/publication. Returned copies retain that validity + inside the trusted owner boundary; they are not new imported raw input. + Exact current bytes and checkpoint must still match before using it. + Callers hold the same ordered guards as check(); no cached alias escapes. + """ + if type(family) is not str: + raise TypeError('Record-family selection requires a string.') + current = self._checked(self.store.summary) + current['records'] = copy.deepcopy([ + item for item in self._snapshot['records'] if item['family'] == family]) + return current + def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capture, *, scope='edit'): """Invoke existing policy, capture fixed-owner upserts, publish, commit SQL. diff --git a/project_sources.py b/project_sources.py index 08939eba..a17b02a5 100644 --- a/project_sources.py +++ b/project_sources.py @@ -6,6 +6,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import hashlib import os @@ -36,7 +37,7 @@ def _summary(saved, origin): schema_version=saved['manifest']['schema_version']) -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): """Validate the fixed source receipt against its exact retained snapshot.""" if operation!='source.register':return False try: @@ -50,8 +51,8 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Source registration requires an attributed timestamp.') if datetime.fromisoformat(result['registered_at']).utcoffset() is None: raise WorkbenchError('Source registration timestamp requires an offset.') - if records is not None: - document=material_records(records).get(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) + if lookup is not None: + document=lookup(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) expected={key:result[key] for key in ('origin_catalog_id','project_id','manifest_sha256','name','schema_version')} if document is None or _summary(document['value'],document['origin_catalog_id'])!=expected: raise WorkbenchError('Source receipt differs from its exact retained snapshot.') @@ -61,6 +62,20 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Malformed retained source operation result.') from None +def receipt_validator(records): + """Own one candidate's retained material facts for source receipt checks.""" + owned = material_records(records) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return material_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class SourceAdapter: def __init__(self,projection): self.projection=projection @@ -117,9 +132,9 @@ def register(self,body,expected_checkpoint,key): self._mutation_identity(expected_checkpoint,key) with self.access.lock,self.session.lock,self.db.lock: actor=self.access.require('edit','source.register',owner_only=True) - self.session.check() + self.session.check_current() saved,raw=self._read_registration(body) - origin=self.session.snapshot['project_id'];invoked=False + origin=self.session.summary['project_id'];invoked=False def invoke(): nonlocal invoked invoked=True;self._bind(saved,raw) @@ -153,7 +168,7 @@ def references(self,snapshotref,offset=0,limit=40): raise WorkbenchError('Invalid bounded source reference page.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('inspect','source.references',snapshot['project_id']) - self.session.check() + self.session.check_current() historical=self.materials.snapshot_ref(snapshot['project_id'],snapshot['manifest_sha256'], catalog_id=snapshot['origin_catalog_id']) try:current=self.sources._saved(snapshot['project_id'],extensions=True) @@ -171,7 +186,7 @@ def references(self,snapshotref,offset=0,limit=40): selected=current['manifest']['data'][offset:offset+limit];budget=[INSPECTION_LIMIT] items=[self.sources._inspect(current['root'],copy.deepcopy(ref),budget,DATA_LIMIT) for ref in selected] self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) - self.session.check();self.access.require('inspect','source.references',snapshot['project_id']) + self.session.check_current();self.access.require('inspect','source.references',snapshot['project_id']) return dict(items=items,total=len(current['manifest']['data']),offset=offset,limit=limit) def import_scientific(self,body,expected_checkpoint,key): @@ -183,7 +198,7 @@ def import_scientific(self,body,expected_checkpoint,key): if type(body['document']) is not dict:raise WorkbenchError('Scientific import metadata must be an object.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_scientific',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],DATA_LIMIT) # ProjectData owns the one canonical mutation and native parser; # current rooted reads above never become an imported capability. @@ -204,7 +219,7 @@ def _capture_reference(self,snapshot,reference_id,expected_sha256,maximum_bytes) expected_sha256,maximum_bytes) if source['manifest_sha256']!=snapshot['manifest_sha256']: raise WorkbenchError('Current source snapshot changed during capture.','stale',409) - self.session.check() + self.session.check_current() return raw,source def import_measurements(self,body,expected_checkpoint,key): @@ -216,7 +231,7 @@ def import_measurements(self,body,expected_checkpoint,key): from retained_measurements import MAX_BYTES with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_measurements',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],MAX_BYTES) source=dict(source,origin_catalog_id=snapshot['origin_catalog_id']) self.access.require('edit','source.import_measurements',snapshot['project_id']) diff --git a/project_store.py b/project_store.py index bc4b4071..d079ca3f 100644 --- a/project_store.py +++ b/project_store.py @@ -124,6 +124,17 @@ def _encode(value): allow_nan=False) + '\n').encode('utf-8') +def copy_validated_json(value): + """Own another tree of already validated JSON without Python graph copying. + + Canonical Store/Session values contain only JSON scalars, lists and string + keyed dictionaries. This is a copy operation inside that proof boundary, + never an input decoder or a substitute for freshness/codec validation. + """ + return json.loads(json.dumps(value, ensure_ascii=True, allow_nan=False, + separators=(',', ':'))) + + def _manifest_raw(manifest): raw = _encode(manifest) if len(raw) > MAX_MANIFEST_BYTES: @@ -468,6 +479,7 @@ def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path): self._origin_path = Path(origin_path) self._meta_fd = None self._manifest = None + self._read_cache = None self._mutex = threading.RLock() self._unknown = False try: @@ -629,12 +641,12 @@ def root_directory(self, *, verify_after=False): if type(verify_after) is not bool: _fail('Root read verification must be an explicit boolean.') with self._mutex: - before = self.snapshot() + before = self._current()[0]['checkpoint'] descriptor = os.dup(self._root_fd) try: yield descriptor self._binding() - if verify_after and self.snapshot()['checkpoint'] != before['checkpoint']: + if verify_after and self._current()[0]['checkpoint'] != before: _fail('Project checkpoint changed during the bounded root read.', 'stale', 409) finally: os.close(descriptor) @@ -642,6 +654,17 @@ def root_directory(self, *, verify_after=False): def _load(self): self._binding() raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) + cached = self._read_cache + if cached is not None and raw == cached[0]: + manifest, loaded = cached[1:] + # Reuse decoding only after checking every actual canonical file. + # Stat identity is not content identity; corruption still fails. + for entry in manifest['inventory']: + self._verified_record_bytes(entry) + if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: + _fail('Project checkpoint changed while reading.', 'stale', 409) + self._binding() + return manifest, loaded manifest = _decode(raw) if type(manifest) is not dict: _fail('Project checkpoint must be an object.') @@ -683,9 +706,7 @@ def _load(self): total += entry['bytes'] if total > MAX_TOTAL_BYTES: _fail('Project records exceed their total byte limit.') - record_raw = self._record_file(family, record_id, digest) - if len(record_raw) != entry['bytes'] or _sha(record_raw) != digest: - _fail('Canonical record bytes changed or are corrupt.', 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + record_raw = self._verified_record_bytes(entry) document = _decode(record_raw) if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') @@ -709,8 +730,16 @@ def _load(self): if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: _fail('Project checkpoint changed while reading.', 'stale', 409) self._binding() + self._read_cache = (raw, manifest, loaded) return manifest, loaded + def _verified_record_bytes(self, entry): + raw = self._record_file(entry['family'], entry['id'], entry['sha256']) + if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: + _fail('Canonical record bytes changed or are corrupt.', + 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + return raw + def _record_file(self, family, record_id, digest, raw=None, sync=False): fds = [] try: @@ -775,17 +804,27 @@ def _publish(self, manifest, creating=False): def _snapshot(self, manifest, records): return dict(project_id=manifest['project_id'], name=manifest['name'], - checkpoint=manifest['checkpoint'], records=copy.deepcopy(records)) + checkpoint=manifest['checkpoint'], records=copy_validated_json(records)) + + def _current(self): + try: + manifest, records = self._load() + if manifest != self._manifest: + _fail('Project checkpoint changed; close and reopen.', 'stale', 409) + return manifest, records + except OSError as exc: + _io_error(exc, 'read project snapshot') + + def summary(self): + """Return scalar metadata after the same complete freshness check.""" + with self._mutex: + manifest, _records = self._current() + return {key: manifest[key] for key in ('project_id', 'name', 'checkpoint')} def snapshot(self): with self._mutex: - try: - manifest, records = self._load() - if manifest != self._manifest: - _fail('Project checkpoint changed; close and reopen.', 'stale', 409) - return self._snapshot(manifest, records) - except OSError as exc: - _io_error(exc, 'read project snapshot') + manifest, records = self._current() + return self._snapshot(manifest, records) def reconcile(self, request_key, request_sha256): """Inspect retry identity before a domain mutation; never repeat effects.""" @@ -884,6 +923,7 @@ def recover_checkpoint(self): def close(self): with self._mutex: + self._read_cache = None if self._meta_fd is not None: os.close(self._meta_fd) self._meta_fd = None diff --git a/project_study_execution.py b/project_study_execution.py index 561c4779..2133bbce 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -142,7 +142,7 @@ def _validate_preparation(operation,result,records): trial=result['trial'];origin=trial['material']['catalog_id'] candidate=facts.candidates[(origin,result['candidate_id'])]['candidate'] if candidate['study_id']!=trial['study_id']:_invalid('Preparation candidate belongs to another Study.') - materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check=lambda:None)) + materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check_current=lambda:None)) composition=materials.recipe(trial['material'])['material'] if (composition['model']!=result['model'] or composition['recipe_id']!=result['graph']['id'] or composition['recipe_revision']!=result['graph']['revision']): @@ -202,7 +202,7 @@ def validate_operation_result(operation,result,records=None): if saved is None or any(saved[key]!=reservation[key] for key in RESERVATION_FIELDS if key not in ('state','settled')): _invalid('Admission receipt has no exact canonical reservation.') trial=facts.trials[(origin,result['trial_id'])]['trial'];run=facts.runs[(origin,result['run_id'])]['run'] - view=SimpleNamespace(snapshot={'records':records},check=lambda:None) + view=SimpleNamespace(snapshot={'records':records},check_current=lambda:None) validate_retained_reservation(reservation,trial=trial, composition=ProjectMaterial(view).recipe(trial['material']),run=run) elif operation=='study_execution.stop': @@ -267,7 +267,7 @@ def __init__(self,adapter,phase): def __getattr__(self,name):return getattr(self.source,name) def open(self,body): return self.adapter.projection.source_adapter.register(body, - self.adapter.session.snapshot['checkpoint'],self.phase+'-source') + self.adapter.session.summary['checkpoint'],self.phase+'-source') class _PreparationModels: @@ -278,11 +278,11 @@ def _read_blob(self,ref,item,maximum=40*1024**2):return self.adapter.materials.b def create(self,body): adapter=self.adapter return adapter.materials.mutate_definition('model_definition','models.create',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) + adapter.session.summary['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) def publish(self,identity,body): adapter=self.adapter return adapter.materials.publish_model(adapter.projection.models,identity,body, - adapter.session.snapshot['checkpoint'],self.phase+'-publish') + adapter.session.summary['checkpoint'],self.phase+'-publish') class _PreparationPipelines: @@ -295,7 +295,7 @@ def recipe_material(self,ref,*,verify=False):return self.adapter.materials.recip def create_external(self,body): adapter=self.adapter return adapter.materials.mutate_definition('pipeline_graph','graphs.create_external',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) + adapter.session.summary['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) def prepare_external_material(self,*args,**kwargs):return self.owner.prepare_external_material(*args,**kwargs) def _retain_material(self,prepared): adapter=self.adapter;value=self.owner._retain_material(prepared) @@ -310,7 +310,7 @@ def __init__(self,projection,execution): self.projection,self.execution=projection,execution self.session,self.db,self.history=projection.session,projection.db,projection.history self.materials=projection.materials;self.access=execution.runtime_access - self.origin=self.session.snapshot['project_id'] + self.origin=self.session.summary['project_id'] self._dirty=set();self._revisions={};self._epochs={};self._pending_epochs={} self._package_records=[] self.context=copy.copy(projection.context) @@ -329,7 +329,7 @@ def _canonical(self,records=None): def restore_authoring(self,records=None): """Reconstruct intent only, under the current project guard.""" with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check();facts=self._canonical(records) + self.session.check_current();facts=self._canonical(records) with self.db.transaction(): for (origin,identity),doc in sorted(facts.studies.items()): if origin!=self.origin:continue @@ -573,7 +573,7 @@ def approve(self,study_id,body,expected_checkpoint,key): validate_operation_result(prior['operation'],prior['result'],self.session.snapshot['records']) self._current_grant(prior['result'],current.actor_id) else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self._materialize_trial(study_id,body['trial_id'],body['expected_material_sha256']) def invoke(): grant=self.studies.approve(study_id,body) @@ -589,7 +589,7 @@ def admit(self,study_id,body,expected_checkpoint,key): authority=self.access.require_external_submission() with self.session.access.lock,self.session.lock,self.db.lock: builtin=None - self.session.check();self.refresh_authoring() + self.session.check_current();self.refresh_authoring() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: result=prior['result'] @@ -597,7 +597,7 @@ def admit(self,study_id,body,expected_checkpoint,key): 'SELECT run_id FROM model_attempt_reservations WHERE run_id=%s',(result.get('run_id'),)).fetchone() is None): _invalid('A historical admission receipt cannot create a current local attempt.','unavailable') else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self.require_current_epoch(self.studies._study(study_id)) self.reject_historical_request(study_id,authority.actor_id,key,None) trial=self._materialize_trial(study_id,body['trial_id']) @@ -626,7 +626,7 @@ def invoke(): # Qualification may launch the explicit probe. It finishes # before Session opens the finite admission transaction. with preparation.admission(pipelines,ref,expected_checkpoint,key): - return self._mutate('study_execution.admit',study_id,body,self.session.snapshot['checkpoint'],key,invoke,'inspect') + return self._mutate('study_execution.admit',study_id,body,self.session.summary['checkpoint'],key,invoke,'inspect') return self._mutate('study_execution.admit',study_id,body,expected_checkpoint,key,invoke,'inspect') def stop(self,study_id,body,expected_checkpoint,key): @@ -645,7 +645,7 @@ def invoke(): # cancellation phase. Receipt replay does not repeat effects. for run_id in invoked: cancel_key='study-stop-'+hashlib.sha256((key+':'+run_id).encode()).hexdigest() - try:self.execution.cancel_run(run_id,self.session.snapshot['checkpoint'],cancel_key) + try:self.execution.cancel_run(run_id,self.session.summary['checkpoint'],cancel_key) except BaseException as error: error.study_stop_receipt=copy.deepcopy(result) raise @@ -683,7 +683,7 @@ def reconcile_preparation(self,study_id,body,expected_checkpoint): if complete.get('native_receipt'): effective=_preparation_body(intent,body['preparation_key']) effective.pop('request_key');effective['actor_id']=intent['actor_id'] - result=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],body['preparation_key'], + result=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],body['preparation_key'], lambda:copy.deepcopy(complete['result']),'cancel') else:result=copy.deepcopy(complete['result']) guard.confirmed() @@ -703,7 +703,7 @@ def abandon_preparation(self,study_id,body,expected_checkpoint,key): operation,current_intent,complete=_preparation_reference(self.session.snapshot['records'],study_id,body['preparation_key']) if current_intent!=intent:_invalid('Preparation intent changed during guarded reconciliation.','conflict') if complete is not None:_invalid('Preparation is complete; reconcile its acknowledgement without abandoning science.','conflict') - observed_checkpoint=self.session.snapshot['checkpoint'] + observed_checkpoint=self.session.summary['checkpoint'] def invoke(): actor=self.access.require('cancel','studies.abandon_preparation',study_id,owner_only=True) if actor.actor_id!=current.actor_id:_invalid('Abandonment caller changed.','conflict') @@ -734,14 +734,14 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): input_sha=digest(dict(study_id=study_id,body=effective)) intent_key=_intent_key(operation,key);phase=intent_key.removesuffix('-intent') with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check() + self.session.check_current() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: # Current authority is checked above. Historical completion # acknowledges science and cannot repeat filesystem effects. return self._mutate(operation,study_id,effective,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Completed preparation replay invoked effects.')),'edit') - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') receipts=operation_records(self.session.snapshot['records']) if any(_intent_key(other,key) in receipts for other in PREPARATIONS if other!=operation): _invalid('Preparation key already identifies another preparation operation.','conflict') @@ -778,7 +778,7 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): publication=SimpleNamespace(runtime_snapshot=snapshot,before_write=guard.publication_started, read_recipe_blob=self.materials.blob) result=_prepare_operation(studies,study_id,dict(body,request_key=key),kind,publication=publication) - value=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],key, + value=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(result),'edit') guard.confirmed() return value diff --git a/project_text.py b/project_text.py index d9502cb6..fc778528 100644 --- a/project_text.py +++ b/project_text.py @@ -252,7 +252,7 @@ def restore(self, snapshot=None): encode(row['annotation']) if row['annotation'] is not None else None, row['review'], row['revision'], row['created_at'], row['updated_at'])) for state in doc['history']: self._workbench._history(state) - if self.store.snapshot()['checkpoint'] != current['checkpoint']: + if self.store.summary()['checkpoint'] != current['checkpoint']: raise ProjectTextError('Project changed during reconstruction.', 'stale_project', 409) self._snapshot, self.available = current, True return copy.deepcopy(current) diff --git a/project_transfer.py b/project_transfer.py index 10a6b431..91848559 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -720,7 +720,7 @@ def replay(opened): try: store=ProjectStore.create_at(destination_parent_fd,stage_name,name,'extraction-stage.'+digest(request_key.encode())[:32],request_digest,root_path=root_path) with store.root_directory() as owned_fd:stage_identity=_fd_identity(owned_fd) - destination=store.snapshot()['project_id'];created_at=timestamp() + destination=store.summary()['project_id'];created_at=timestamp() from project_data_fork import fork_records as fork_data from project_material_fork import fork_records as fork_material original=copy.deepcopy(records);mappings=[] @@ -755,7 +755,7 @@ def replay(opened): with source.artifacts.open_verified(value,maximum_bytes=value['bytes']) as fd:artifacts.publish_fd(fd,value,maximum_bytes=value['bytes']) with store.root_directory() as fd:materialize_captured(records,destination,fd) _validate_candidate(records,destination,artifacts) - committed=store.commit(records,store.snapshot()['checkpoint'],request_key,request_digest) + committed=store.commit(records,store.summary()['checkpoint'],request_key,request_digest) verification=_cold_verify(store,source) store.save(committed['checkpoint']);_assert_unchanged(source) _rename_noreplace(destination_parent_fd,stage_name,directory);published=True;os.fsync(destination_parent_fd) diff --git a/project_transfer_controller.py b/project_transfer_controller.py index 7362e0ea..3ece224e 100644 --- a/project_transfer_controller.py +++ b/project_transfer_controller.py @@ -252,7 +252,7 @@ def extract(self, binding, report_sha256, destination_root, directory, name, req fail('Recover the selected workspace before reconciling extraction.', 'projection_unavailable', 503) guards.enter_context(projection.session.lock) guards.enter_context(projection.db.lock) - projection.session.check() + projection.session.check_current() opened = self.workspace._store receipt = self._transfer.extract(self._selection, report_sha256, destination_fd, directory, name, request_key, opened_store=opened) except BaseException as error: diff --git a/project_workspace.py b/project_workspace.py index efe93f9a..b006dbe2 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -122,7 +122,7 @@ def _state(self): project = None if self._recovery is None and self._store is not None: try: - snapshot = self._store.snapshot() + snapshot = self._store.summary() except ProjectStoreError as error: if error.code not in ('unknown', 'outcome_unknown', 'stale', 'stale_project', 'invalid', 'unsupported', 'unavailable'): @@ -184,7 +184,7 @@ def _bound(self, arguments, *, may_be_empty=False, check_checkpoint=True, allow_ if arguments.get('expected_checkpoint') is not None: raise WorkbenchError('The empty workspace has no checkpoint.', 'stale_project', 409) return - snapshot = self._store.snapshot() + snapshot = self._store.summary() expected = arguments.get('expected_checkpoint', arguments.get('checkpoint')) if check_checkpoint and expected != snapshot['checkpoint']: raise WorkbenchError('The project checkpoint changed. Refresh before acting.', 'stale_project', 409) @@ -206,7 +206,7 @@ def _root(self, alias, relative): def _projection_for(self, store): identity = uuid.uuid4().hex path = self.path / '.project-projections' / identity - snapshot = store.snapshot() + snapshot = store.summary() projection = Metadata(path, dsn=self._dsn, dataset_id=snapshot['project_id'], projection_id=identity) try: owner = ProjectProjection(store, projection, self.access, @@ -227,7 +227,7 @@ def _retained_projection_for(self, store, projection_id): if type(projection_id) is not str or not re.fullmatch('[a-f0-9]{32}', projection_id): raise WorkbenchError('Invalid original local projection identity.', 'unavailable', 409) db = Metadata(self.path / '.project-projections' / projection_id, dsn=self._dsn, - create=False, dataset_id=store.snapshot()['project_id'], projection_id=projection_id) + create=False, dataset_id=store.summary()['project_id'], projection_id=projection_id) try: return ProjectProjection(store, db, self.access, {alias: value[0] for alias, value in self._roots.items()}, @@ -243,7 +243,7 @@ def _detach(self): self._projection.execution.assert_quiescent() if self._store is not None: # Prove pending project publications and their full closure first. - self._store.save(self._store.snapshot()['checkpoint']) + self._store.save(self._store.summary()['checkpoint']) if self._projection is not None: self._projection.close() if self._store is not None: @@ -272,14 +272,14 @@ def _recover_current(self, arguments): store = projection = None try: store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) - snapshot = store.snapshot() + snapshot = store.summary() if snapshot['project_id'] != self._last_project['project_id']: raise WorkbenchError('Recovery destination has a different project identity.', 'stale_project', 409) # A reread reconciles identity; only a successful durability barrier # resolves a prior checkpoint/directory fsync failure. store.save(snapshot['checkpoint']) projection, text = self._projection_for(store) - snapshot = store.snapshot() + snapshot = store.summary() if self._projection is not None: self._projection.close() self._store, self._projection, self._text = store, projection, text @@ -332,7 +332,7 @@ def _select(self, action, arguments): if store.reconcile(arguments['request_key'], digest) is None: raise error reconciled = True - snapshot = store.snapshot() + snapshot = store.summary() if reconciled: store.save(snapshot['checkpoint']) created = snapshot @@ -342,7 +342,7 @@ def _select(self, action, arguments): self._detach() self._store, self._projection, self._text = store, projection, text self._selection = selection - snapshot = store.snapshot() + snapshot = store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} self._recovery = None self._binding = uuid.uuid4().hex @@ -366,7 +366,7 @@ def _select(self, action, arguments): raise def _record_result(self, record, *, before_revision=None, history_limit=1): - snapshot = self._store.snapshot() + snapshot = self._store.summary() history = [item for item in self._text.history(record['id']) if item['revision'] <= record['revision'] and (before_revision is None or item['revision'] < before_revision)] @@ -402,14 +402,14 @@ def _publish_independent(self, arguments, *, composed=False): if composed: from project_composition import publish_composition from project_artifacts import ProjectArtifacts - others=[];identities={self._store.snapshot()['project_id']} + others=[];identities={self._store.summary()['project_id']} for selected in arguments['sources']: if selected['project_id'] in identities: raise WorkbenchError('Composition source identity repeats.','conflict',409) identities.add(selected['project_id']) directory,descriptor=self._root(selected['root'],selected['directory']) store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory)) - actual=store.snapshot() + actual=store.summary() if actual['project_id']!=selected['project_id'] or actual['checkpoint']!=selected['checkpoint']: raise WorkbenchError('Selected composition source checkpoint differs.','stale_project',409) artifacts=retained.enter_context(ProjectArtifacts(store)) @@ -509,13 +509,17 @@ def dispatch(self, name, arguments, *, selected_version=None): from project_dispatch import dispatch try: value = dispatch(self._projection, name, arguments) + if (name=='project_files.measurement_compare' + and value['value']['observation_count']>agent_api.measurement_comparison_row_limit(selected_version)): + raise WorkbenchError('Comparison exceeds the selected API version observation budget. Select comparison contract version 2 for larger complete populations.', + 'too_large',413) if name == 'project_files.execution_health' and selected_version == 1: value = {key: item for key, item in value.items() if key != 'native_phases'} except BaseException as error: if getattr(error, 'code', None) in ('unknown', 'outcome_unknown', 'projection_unavailable'): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} result = dict(binding=self._binding, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint'], value=value) @@ -625,14 +629,14 @@ def _dispatch_draining(self, name, arguments): raise WorkbenchError('The retained physical recovery owner changed.', 'local_custody_unavailable', 409) recovering = True if incoming is not projection and name == 'project_files.execution_reconcile_current': - value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.snapshot['checkpoint']) + value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.summary['checkpoint']) else: internal = arguments if recovery_operation and recovering and name != 'project_files.execution_reconcile_current': # Public CAS above uses the last acknowledged workspace # view. Only this qualified local owner sees its exact # selected/attempted checkpoint for finite recovery. - internal = dict(arguments, expected_checkpoint=incoming.session.snapshot['checkpoint']) + internal = dict(arguments, expected_checkpoint=incoming.session.summary['checkpoint']) if name == 'project_files.runtime_capture' and recovering: options = {'reserve_bytes': internal['reserve_bytes']} if 'reserve_bytes' in internal else {} value = incoming.runtime.recover_capture({'source': internal['source']}, @@ -656,7 +660,7 @@ def _dispatch_draining(self, name, arguments): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise with self.access.lock, projection.session.lock, projection.db.lock: - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} if name in _RECOVERY_OPERATIONS: self._recovery = None diff --git a/retained_measurements.py b/retained_measurements.py index 796aaa19..4044b017 100644 --- a/retained_measurements.py +++ b/retained_measurements.py @@ -26,6 +26,7 @@ MAX_COMPARISON_SOURCES = 16 MAX_COMPARISON_SUBJECTS = 256 MAX_COMPARISON_BYTES = 128 * 1024 * 1024 +MAX_COMPARISON_ROWS = 250000 STATES = ('measured', 'failed', 'unsupported', 'unavailable') ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') SHA = re.compile(r'^[a-f0-9]{64}$') @@ -411,8 +412,8 @@ def __init__(self, tables): fail('Comparison sources overlap a subject identity.', 'conflict', 409) self._subjects[subject['id']] = dict(copy.deepcopy(subject), source=copy.deepcopy(locator)) observations += metadata['observation_count'] - if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_ROWS: - fail('Comparison exceeds 256 subjects or 100,000 observations.', 'too_large', 413) + if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_COMPARISON_ROWS: + fail('Comparison exceeds 256 subjects or 250,000 observations.', 'too_large', 413) self._tables.append((copy.deepcopy(locator), table)) self._header.update(subjects=list(self._subjects.values()), observation_count=observations) diff --git a/scripts/generate_project_files_contracts.py b/scripts/generate_project_files_contracts.py new file mode 100644 index 00000000..ac373e53 --- /dev/null +++ b/scripts/generate_project_files_contracts.py @@ -0,0 +1,35 @@ +"""Derive installed browser pins/routes from the local canonical API registry.""" +import argparse +import json +from pathlib import Path +import re +import sys + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT)) +import agent_api + + +def render(original): + names=sorted(name for name in agent_api._CONTRACTS if name.startswith('project_files.')) + tables={ + 'CONTRACTS':{name:agent_api.contract(name,1)['sha256'] for name in names}, + 'VERSIONS':{name:{str(v):agent_api.contract(name,v)['sha256'] for v in sorted(agent_api._CONTRACTS[name])} + for name in names if len(agent_api._CONTRACTS[name])>1}, + 'ROUTES':{name:{'method':agent_api.operation(name)['method'],'path':agent_api.operation(name)['path'], + 'mutation':agent_api.operation(name)['side_effects']} + for name in sorted(agent_api.PROJECT_FILES_M2|agent_api.PROJECT_FILES_M3|agent_api.PROJECT_FILES_M4)}} + for label,value in tables.items(): + pattern=r'(/\* PROJECT_FILES_'+label+r'_START \*/\s*)(.*?)(\s*/\* PROJECT_FILES_'+label+r'_END \*/)' + generated=json.dumps(value,sort_keys=True,indent=2) + original,count=re.subn(pattern,lambda m:m[1]+generated+m[3],original,flags=re.S) + if count!=1:raise ValueError('Exactly one installed '+label+' table required') + return original + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--check',action='store_true') + args=parser.parse_args();path=ROOT/'static/project-files.js';original=path.read_text();expected=render(original) + if args.check: + if original!=expected:raise SystemExit('Installed project-file browser contracts are stale') + elif original!=expected:path.write_text(expected) diff --git a/static/project-files.css b/static/project-files.css index 515d9ca5..c18e2e30 100644 --- a/static/project-files.css +++ b/static/project-files.css @@ -10,3 +10,13 @@ #project-runtime-files,#project-runtime-summary,#project-runtime-pending,#project-runtime-layout,#project-runtime-status{overflow-wrap:anywhere;min-width:0}#project-runtime-layout,#project-runtime-status{white-space:pre-wrap;max-height:32rem;overflow:auto} #project-candidate-preparation-result,#project-candidate-preparation-status{overflow-wrap:anywhere;min-width:0}#project-candidate-preparation-result{white-space:pre-wrap;max-height:32rem;overflow:auto} + +#project-portfolio-source,#project-portfolio-detail,#project-portfolio-count{overflow-wrap:anywhere;min-width:0} +#project-portfolio-detail p{white-space:pre-wrap;max-width:85ch} +#project-portfolio-detail pre{white-space:pre-wrap;overflow-wrap:anywhere;max-height:32rem;overflow:auto} +#project-portfolio-detail details{margin:1rem 0} +#project-portfolio-detail h4{margin-bottom:.25rem} +#project-portfolio-detail h3{scroll-margin-top:1rem} +#project-portfolio-filter-form{display:grid;grid-template-columns:1fr 2fr 1fr;gap:.7rem;align-items:end} +#project-portfolio-filter-form button{justify-self:start} +@media(max-width:760px){#project-portfolio-filter-form{grid-template-columns:1fr}} diff --git a/static/project-files.html b/static/project-files.html index de9c6f1a..52f1ccca 100644 --- a/static/project-files.html +++ b/static/project-files.html @@ -14,6 +14,7 @@ + @@ -50,6 +51,7 @@

Project file

No unsubmitted edits.

+

Prepare a Candidate package or verifier

Choose a current Study, Candidate and captured CPU runtime. Preparation saves an inert Trial and exact sources; approval and Run admission remain separate actions.

diff --git a/static/project-files.js b/static/project-files.js index 65cfa499..3146719e 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -14,6 +14,7 @@ "project_files.candidate": "c519ff04bc06d5e538d36b7f278862d8f282d0e7b97dd7f262444c384a50f854", "project_files.candidate_select": "f043be6923760a208ca50d02a1ff4b60eadf09d48a034acdb18749806a537080", "project_files.close": "95d0099ec2a378a1bbb65676799dd3baae2f36b838301f7aa165b116fba5edf6", + "project_files.compose": "b1dcbbd4ad7ebe17b17e11238f99a93408da24d0d411527e187900cc0f089105", "project_files.data_annotate": "289270f5cb946765c5ced22fbbd18765bb55743145014c81cf54d7a6ba3025ea", "project_files.data_availability": "92332c04973a4fa45d47e384717152ebc4a9ca08e7eb487669688ccd2ed42cbb", "project_files.data_get": "0d19505995cbd8c12e3938cd89cf05955a2a63488a220711d29dd24b27c0c8c8", @@ -167,6 +168,10 @@ "project_files.get": { "1": "6b0f4e16a00aa119b86cbb5493be0bac945ff08a5e63773a8f2c2f9296658d5b", "2": "fdc0858fc03a0866bb743b04251ea5f9a3c1e6be1b211366db7e0652166736ea" + }, + "project_files.measurement_compare": { + "1": "abbb5e406a3c95155e90f08e8a071f330425190b25275f0f6512de08b8afb929", + "2": "d2de9763a9109d0e4599f50e038a3fda0e6bf7c9c60684434825d0108019bc6f" } } /* PROJECT_FILES_VERSIONS_END */; @@ -199,6 +204,11 @@ "mutation": true, "path": "/api/workspace/project-files/studies/{study_id}/candidates" }, + "project_files.compose": { + "method": "POST", + "mutation": true, + "path": "/api/workspace/project-files/compose" + }, "project_files.data_annotate": { "method": "POST", "mutation": true, @@ -892,6 +902,17 @@ onContextFocusLost:()=>$('project-files-status').focus() }); window.addEventListener('pagehide',()=>projectProgression.clear()); + const projectPortfolio=window.ProjectModelPortfolio({ + root:$('project-model-portfolio'), + async read(action,args,signal){ + if(!m2Allowed(action))throw Error('Model organization inspection is unavailable for the selected project.'); + const value=await m2Request(action,args,epoch,{signal}); + if(value===null)throw new DOMException('Project selection changed.','AbortError'); + return value; + }, + onContextFocusLost:()=>$('project-files-status').focus() + }); + window.addEventListener('pagehide',()=>projectPortfolio.clear()); const projectMeasurements=window.ProjectMeasurements({ root:$('project-measurements'), async read(action,args,signal){ @@ -1157,6 +1178,8 @@ } const m2Allowed=action=>!busy&&!blocked&&state?.state==='open'&&selectedContracts.has('project_files.'+action); function m2Controls(){ + projectPortfolio.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), + project_id:state?.project?.project_id,enabled:m2Allowed('files_inspect')&&m2Allowed('file_get')}); projectProgression.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), project_id:state?.project?.project_id,enabled:m2Allowed('progression')&&m2Allowed('progression_origins')}); projectMeasurements.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), diff --git a/static/project-model-portfolio.js b/static/project-model-portfolio.js new file mode 100644 index 00000000..4f1b7fa3 --- /dev/null +++ b/static/project-model-portfolio.js @@ -0,0 +1,188 @@ +'use strict'; +// Authored organization is a retained note, not Trial lineage or qualification. +// This module owns its document contract, finite view and cancellable reads. +(() => { + const fields=['method','purpose','experiment','outcome','lesson','reproduction']; + const exact=(value,keys)=>value&&typeof value==='object'&&!Array.isArray(value)&& + Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); + const text=(value,limit)=>typeof value==='string'&&value.length>0&&value.length<=limit; + const identity=value=>text(value,160)&&/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value); + const sha=value=>typeof value==='string'&&/^[a-f0-9]{64}$/.test(value); + const natural=value=>Number.isSafeInteger(value)&&value>=0; + function fail(message,code='invalid'){throw Object.assign(Error(message),{code});} + function noteRef(value){ + if(!exact(value,['file_id','path','revision','sha256'])||!sha(value.file_id)||!sha(value.sha256)|| + !natural(value.revision)||value.revision<1||!text(value.path,512)||!value.path.startsWith('notes/')|| + value.path.split('/').some(part=>!part||part==='.'||part==='..'))fail('Invalid captured model-note reference.'); + } + function decode(raw){ + if(typeof raw!=='string'||new TextEncoder().encode(raw).length>1048576)fail('Organization note exceeds its retained text limit.'); + let value;try{value=JSON.parse(raw);}catch{fail('Organization note is not valid JSON.');} + if(value?.format!=='project_model_portfolio_v1')fail('Unsupported model-organization document format.','unsupported'); + if(!exact(value,['format','project_id','title','description','authorship','groups','models','relationships'])|| + !/^[a-f0-9]{32}$/.test(value.project_id)||!text(value.title,200)||!text(value.description,8000)|| + value.authorship!=='retrospective'||!Array.isArray(value.groups)||value.groups.length>64|| + !Array.isArray(value.models)||!value.models.length||value.models.length>1000|| + !Array.isArray(value.relationships)||value.relationships.length>2000)fail('Invalid model-organization document.'); + const groups=new Map(),models=new Map(); + for(const group of value.groups){ + if(!exact(group,['id','label','parent','description'])||!identity(group.id)||groups.has(group.id)|| + !text(group.label,200)||!text(group.description,8000)||group.parent!==null&&!identity(group.parent))fail('Invalid or duplicate organization group.'); + groups.set(group.id,group); + } + for(const group of groups.values()){ + const visited=new Set([group.id]);let parent=group.parent; + while(parent!==null){if(!groups.has(parent))fail('Organization group parent is unavailable.'); + if(visited.has(parent))fail('Organization grouping contains a cycle.');visited.add(parent);parent=groups.get(parent).parent;} + } + for(const model of value.models){ + if(!exact(model,['id','label','group_id','aliases','review','card','summary'])||!identity(model.id)||models.has(model.id)|| + !text(model.label,500)||!groups.has(model.group_id)||!Array.isArray(model.aliases)||model.aliases.length>128|| + new Set(model.aliases).size!==model.aliases.length||model.aliases.some(alias=>!text(alias,500))|| + !['source_reviewed','review_pending'].includes(model.review))fail('Invalid or duplicate organized model.'); + noteRef(model.card); + if(model.review==='source_reviewed'){ + if(!exact(model.summary,fields)||fields.some(field=>!text(model.summary[field],16384)))fail('Reviewed model requires all six authored summary fields.'); + }else if(model.summary!==null)fail('Pending review cannot expose a reviewed summary.'); + models.set(model.id,model); + } + const pairs=new Set(); + for(const relation of value.relationships){ + if(!exact(relation,['from','to','kind','explanation','evidence_model','excerpt'])|| + !models.has(relation.from)||!models.has(relation.to)||relation.from===relation.to|| + !['comparison','parameter_derivation'].includes(relation.kind)||!text(relation.explanation,8000)|| + !models.has(relation.evidence_model)||!text(relation.excerpt,16384))fail('Invalid authored model relationship.'); + const key=JSON.stringify([relation.from,relation.to,relation.kind]);if(pairs.has(key))fail('Duplicate authored model relationship.');pairs.add(key); + } + const parents=new Map(),complete=new Set(),visiting=new Set(); + for(const relation of value.relationships)if(relation.kind==='parameter_derivation'){ + if(!parents.has(relation.from))parents.set(relation.from,[]);parents.get(relation.from).push(relation.to); + } + function visit(id){if(complete.has(id))return;if(visiting.has(id))fail('Authored parameter derivation contains a cycle.'); + visiting.add(id);for(const parent of parents.get(id)||[])visit(parent);visiting.delete(id);complete.add(id);} + for(const id of parents.keys())visit(id); + return value; + } + function checkCard(document,model,result){ + const ref=model.card,version=result?.version; + if(result?.project_id!==document.project_id||result.id!==ref.file_id||result.path!==ref.path||result.kind!=='note'|| + version?.deleted===true||version?.revision!==ref.revision||version.sha256!==ref.sha256||typeof version.text!=='string') + fail('Model card differs from its exact retained organization reference.'); + if(model.summary&&fields.some(field=>!version.text.includes(model.summary[field])))fail('Authored summary differs from the captured model card.'); + for(const relation of document.relationships)if(relation.evidence_model===model.id&&!version.text.includes(relation.excerpt)) + fail('Authored relationship excerpt differs from its captured model card.'); + return version.text; + } + window.ProjectModelPortfolioDocument=Object.freeze({decode,checkCard}); + const node=(tag,value)=>{const element=document.createElement(tag);if(value!==undefined)element.textContent=value;return element;}; + window.ProjectModelPortfolio=function({root,read,onContextFocusLost=()=>{}}){ + root.innerHTML='

Authored model organization

Browse a retained guide to models and their methods. Grouping and retrospective comparison notes are separate from recorded Trial ancestry and scientific qualification.

'; + const $=name=>root.querySelector('#project-portfolio-'+name); + let context=null,generation=0,pending=null,busy=false,notes=new Map(),noteOffset=0,noteTotal=0; + let guide=null,groups=new Map(),models=new Map(),applied={group:'',search:'',review:''},offset=0,selected=''; + const pageSize=20; + function controls(){ + const enabled=!!context?.enabled; + for(const name of ['notes','notes-more','note','load','group','search','review','prev','next'])$(name).disabled=!enabled||busy; + $('note').disabled ||= !notes.size;$('load').disabled ||= !notes.has($('note').value); + $('notes-more').hidden=noteOffset>=noteTotal;$('notes-more').disabled ||= noteOffset>=noteTotal; + $('prev').disabled ||= !guide||offset===0;$('next').disabled ||= !guide||offset+pageSize>=filtered().length; + for(const button of root.querySelectorAll('#project-portfolio-filter-form button,[data-lineage-node],#project-portfolio-detail button'))button.disabled=!enabled||busy; + root.setAttribute('aria-busy',String(busy)); + } + function clearGuide(){guide=null;groups.clear();models.clear();selected='';offset=0;applied={group:'',search:'',review:''};$('content').hidden=true;$('detail').replaceChildren();$('graph').replaceChildren();$('search').value='';$('review').value='';$('group').replaceChildren();} + function reset(){++generation;pending?.abort();pending=null;busy=false;notes.clear();noteOffset=noteTotal=0;$('note').replaceChildren();$('note-count').textContent='';$('status').textContent='';clearGuide();controls();} + async function perform(work){ + if(!context?.enabled||busy)return; + const ticket=++generation,key=context.key;pending=new AbortController();const signal=pending.signal;busy=true;controls(); + const active=()=>ticket===generation&&context?.enabled&&context.key===key&&!signal.aborted; + try{await work(active,signal);}catch(error){if(active()){$('detail').replaceChildren();$('status').textContent=(error.code?error.code+': ':'')+error.message;$('status').focus();}} + finally{if(active()){busy=false;pending=null;controls();}} + } + async function findNotes(more=false){ + return perform(async(active,signal)=>{ + const requested=more?noteOffset:0,value=await read('files_inspect',{offset:requested,limit:40},signal);if(!active())return; + if(value?.offset!==requested||value.limit!==40||!natural(value.total)||!Array.isArray(value.items)|| + value.items.length!==Math.min(40,Math.max(0,value.total-requested)))fail('Organization note page differs from its continuation.'); + if(!more){clearGuide();notes.clear();$('note').replaceChildren();} + for(const item of value.items)if(item.kind==='note'&&item.path?.endsWith('.portfolio.json')){ + if(!sha(item.id)||!natural(item.revision)||item.revision<1||!sha(item.expected_sha256)||notes.has(item.id))fail('Invalid organization note selection.'); + notes.set(item.id,structuredClone(item));const option=node('option',item.path+' · revision '+item.revision);option.value=item.id;$('note').append(option); + } + noteOffset=requested+value.items.length;noteTotal=value.total; + $('note-count').textContent=`${notes.size} organization notes found; inspected ${noteOffset} of ${noteTotal} adopted files.`; + $('status').textContent=notes.size?'Choose a retained organization note.':'No organization note on these pages. Adopt a .portfolio.json note or inspect the next page.'; + }); + } + function inGroup(model,wanted){let id=model.group_id;while(id!==null){if(id===wanted)return true;id=groups.get(id).parent;}return false;} + function filtered(){if(!guide)return [];return guide.models.filter(model=>(!applied.group||inGroup(model,applied.group))&& + (!applied.review||model.review===applied.review)&&(!applied.search||[model.label,...model.aliases].some(value=>value.toLocaleLowerCase().includes(applied.search))));} + function render(){ + if(!guide)return; + const focused=$('graph').contains(document.activeElement)?document.activeElement.id:null; + const matching=filtered(),shown=matching.slice(offset,offset+pageSize),included=new Set(); + for(const model of shown){let id=model.group_id;while(id!==null){included.add(id);id=groups.get(id).parent;}} + const graphNodes=[...included].map(id=>{const group=groups.get(id);return {id:'group/'+id,title:group.label,recordedAt:'',parent:group.parent===null?null:'group/'+group.parent,baseline:null,lines:['Authored method group']};}); + graphNodes.push(...shown.map(model=>({id:'model/'+model.id,title:model.label,recordedAt:'',parent:'group/'+model.group_id,baseline:null, + lines:[model.aliases.slice(0,3).join(' · ')||model.id,model.review==='source_reviewed'?'Source reviewed · authored summary':'Source review pending']}))); + $('graph').innerHTML=window.WorkspaceExperimentGraph.render(graphNodes,{selected:selected?'model/'+selected:'',zoom:1}).html; + for(const button of $('graph').querySelectorAll('[data-lineage-node]')){ + const value=button.dataset.lineageNode; + button.onclick=()=>{if(value.startsWith('model/'))void selectModel(value.slice(6));else{$('group').value=value.slice(6);applyFilters();}}; + } + $('count').textContent=`${guide.models.length} models in this guide; ${guide.models.filter(model=>model.review==='source_reviewed').length} source reviewed. ${matching.length} match the applied filters. Showing ${shown.length?offset+1:0}–${offset+shown.length}.`; + controls(); + if(focused){const replacement=$('graph').querySelector('#'+CSS.escape(focused)); + if(replacement)replacement.focus({preventScroll:true});else{$('count').tabIndex=-1;$('count').focus();}} + } + function applyFilters(){if(!guide||busy)return;applied={group:$('group').value,search:$('search').value.trim().toLocaleLowerCase(),review:$('review').value}; + if(applied.group&&!groups.has(applied.group))return;offset=0;selected='';$('detail').replaceChildren();render();$('status').textContent='Applied model filters.';} + async function loadGuide(){ + const choice=notes.get($('note').value);if(!choice)return; + return perform(async(active,signal)=>{ + clearGuide();const value=await read('file_get',{file_id:choice.id,revision:choice.revision},signal);if(!active())return; + if(value?.id!==choice.id||value.project_id!==context.project_id||value.kind!=='note'||value.path!==choice.path|| + value.version?.revision!==choice.revision||value.version.sha256!==choice.expected_sha256||value.version.deleted===true)fail('Organization note differs from the selected captured revision.'); + const parsed=decode(value.version.text); + if(parsed.project_id!==context.project_id)fail('This guide belongs to another project; re-author its captured note bindings.','unsupported'); + guide=parsed;groups=new Map(guide.groups.map(group=>[group.id,group]));models=new Map(guide.models.map(model=>[model.id,model])); + $('title').textContent=guide.title;$('description').textContent=guide.description; + $('source').textContent=`Retrospective authored guide · ${choice.path} · revision ${choice.revision} · SHA256 ${choice.expected_sha256}`; + const all=node('option','All groups');all.value='';$('group').append(all); + for(const group of guide.groups){const option=node('option',group.label);option.value=group.id;$('group').append(option);} + $('content').hidden=false;render();$('status').textContent='Loaded retained authored model organization.'; + }); + } + async function selectModel(id){ + const model=models.get(id);if(!model||busy)return; + return perform(async(active,signal)=>{ + if(!filtered().some(item=>item.id===id)){ + applied={group:'',search:'',review:''};$('group').value='';$('search').value='';$('review').value=''; + } + offset=Math.floor(filtered().findIndex(item=>item.id===id)/pageSize)*pageSize; + selected=id;$('detail').replaceChildren();render(); + const value=await read('file_get',{file_id:model.card.file_id,revision:model.card.revision},signal);if(!active())return; + const captured=checkCard(guide,model,value),heading=node('h3',model.label);heading.id='project-portfolio-model-heading';heading.tabIndex=-1;$('detail').append(heading); + $('detail').append(node('p',(model.aliases.join(' · ')||model.id)+' · '+(model.review==='source_reviewed'?'Source reviewed':'Source review pending'))); + if(model.summary){const labels={method:'Method',purpose:'Purpose',experiment:'Training and experiment',outcome:'Observed outcome',lesson:'Lesson',reproduction:'Reproduction'}; + for(const field of fields){$('detail').append(node('h4',labels[field]),node('p',model.summary[field]));}} + else $('detail').append(node('p','A source-reviewed interpretation has not been recorded for this fit. Its original captured card remains available below.')); + const relations=guide.relationships.filter(relation=>relation.from===id||relation.to===id); + if(relations.length){$('detail').append(node('h4','Authored model relationships')); + for(const relation of relations){const box=node('div'),other=models.get(relation.from===id?relation.to:relation.from),button=node('button', + (relation.kind==='comparison'?'Comparison: ':relation.from===id?'Parameters from: ':'Derived model: ')+other.label);button.type='button';button.onclick=()=>void selectModel(other.id); + box.append(button,node('p',relation.explanation));const evidence=node('details');evidence.append(node('summary','Exact retained relationship excerpt'),node('p',relation.excerpt)); + const source=node('button','Read evidence card: '+models.get(relation.evidence_model).label);source.type='button';source.onclick=()=>void selectModel(relation.evidence_model);evidence.append(source);box.append(evidence);$('detail').append(box);}} + const raw=node('details'),summary=node('summary','Exact captured card and original receipts');raw.append(summary,node('p',`${model.card.path} · revision ${model.card.revision} · SHA256 ${model.card.sha256}`),node('pre',captured));$('detail').append(raw); + $('status').textContent='Exact retained model summary and captured evidence.';heading.focus(); + }); + } + $('notes').onclick=()=>void findNotes();$('notes-more').onclick=()=>void findNotes(true); + $('note').onchange=()=>{clearGuide();controls();}; + $('source-form').onsubmit=event=>{event.preventDefault();void loadGuide();}; + $('filter-form').onsubmit=event=>{event.preventDefault();applyFilters();}; + for(const [name,direction] of [['prev',-1],['next',1]])$(name).onclick=()=>{if(busy||!guide)return;offset=Math.max(0,offset+direction*pageSize);selected='';$('detail').replaceChildren();render();}; + controls(); + return {setContext(value){if(value.key!==context?.key||context?.enabled&&!value.enabled){const owned=root.contains(document.activeElement);context=value;reset();if(owned)onContextFocusLost();}else context=value;root.hidden=!value.enabled;controls();},clear(){context=null;reset();root.hidden=true;}}; + }; +})(); diff --git a/tests/browser_model_portfolio.cjs b/tests/browser_model_portfolio.cjs new file mode 100644 index 00000000..851e650f --- /dev/null +++ b/tests/browser_model_portfolio.cjs @@ -0,0 +1,129 @@ +'use strict'; +// Real Chromium interaction/layout and controlled read-lifecycle evidence. +// This isolated static fixture does not claim native project persistence. +const fs=require('node:fs'),path=require('node:path'),http=require('node:http'),crypto=require('node:crypto'); +const {spawn}=require('node:child_process'); +const {waitForDebugger}=require('./browser_startup.cjs'),{cdpSession}=require('./browser_cdp_session.cjs'); +const output=path.resolve(process.argv[2]||''),engine=process.env.TULDOK_BROWSER_EXECUTABLE; +if(!process.argv[2]||!engine||fs.existsSync(output))throw Error('Explicit fresh output directory and browser executable required'); +fs.mkdirSync(output,{recursive:true,mode:0o700}); +const fixture=structuredClone(require('./model_portfolio_fixture.cjs')); +const hash=value=>crypto.createHash('sha256').update(value).digest('hex'); +const first=fixture.document.models[0]; +const oldMethod=first.summary.method;first.summary.method+=' '+('A physical pressure field with explicit case and work scope. '.repeat(60)); +first.label='First model '; +fixture.cards[0].version.text=fixture.cards[0].version.text.replace(oldMethod,first.summary.method); +fixture.cards[0].version.sha256=first.card.sha256=hash(fixture.cards[0].version.text); +fixture.cards[0].version.bytes=Buffer.byteLength(fixture.cards[0].version.text); +for(let index=0;index<20;index++){ + const model=structuredClone(fixture.document.models[1]);model.id='extra-'+index;model.label='Extra model '+index;model.aliases=[]; + model.card.path='notes/extra-'+index+'.md';model.card.file_id=hash(fixture.project+'\0note\0'+model.card.path); + fixture.document.models.push(model);fixture.cards.push({...structuredClone(fixture.cards[1]),id:model.card.file_id,path:model.card.path}); +} +fixture.guide.version.text=JSON.stringify(fixture.document); +fixture.guide.version.sha256=fixture.inspection.expected_sha256=fixture.inspection.observed_sha256=hash(fixture.guide.version.text); +fixture.guide.version.bytes=fixture.inspection.expected_bytes=fixture.inspection.observed_bytes=Buffer.byteLength(fixture.guide.version.text); +const inline=JSON.stringify(fixture).replaceAll('<','\\u003c'); +const html=`Authored organization fixture
`; +const staticRoot=path.join(__dirname,'../static'),allowed=new Map([ + ['/project-files.css',['project-files.css','text/css']],['/experiment-lineage.css',['experiment-lineage.css','text/css']], + ['/experiment-graph.js',['experiment-graph.js','text/javascript']],['/project-model-portfolio.js',['project-model-portfolio.js','text/javascript']]]); +const server=http.createServer((request,response)=>{ + if(request.url==='/'){response.writeHead(200,{'Content-Type':'text/html; charset=utf-8'});response.end(html);return;} + const asset=allowed.get(request.url);if(!asset){response.writeHead(404);response.end();return;} + response.writeHead(200,{'Content-Type':asset[1]});response.end(fs.readFileSync(path.join(staticRoot,asset[0]))); +}); +let child,session; +const report={status:'starting',engine,verification_only:true,native_persistence_claim:false,graphics:'Headless Chromium with GPU disabled; no model or rendering speed claim.',checks:[],errors:[],browser_stopped:false}; +const sourceFiles=['app.py','static/project-files.css','static/project-files.html','static/project-files.js','static/project-model-portfolio.js', + 'docs/contracts/project-model-portfolio.md','tests/model_portfolio_fixture.cjs','tests/test_project_model_portfolio.cjs', + 'tests/browser_model_portfolio.cjs','tests/test_project_model_portfolio_http.py','tests/test_workspace_project_files.cjs','tests/test_workspace_project_files_m4.cjs']; +report.source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); +const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); +async function main(){ + await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); + const origin='http://127.0.0.1:'+server.address().port; + try{ + const flags=['--headless','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,1000','--user-data-dir='+path.join(output,'profile'),'about:blank']; + report.flags=flags;report.origin=origin; + const log=fs.openSync(path.join(output,'browser.log'),'w');child=spawn(engine,flags,{stdio:['ignore',log,log]});fs.closeSync(log); + const port=await waitForDebugger(child,path.join(output,'profile/DevToolsActivePort')); + const targets=await(await fetch('http://127.0.0.1:'+port+'/json/list')).json(),target=targets.find(target=>target.type==='page'); + const socket=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{socket.addEventListener('open',resolve,{once:true});socket.addEventListener('error',reject,{once:true});}); + session=cdpSession(socket,{onEvent:event=>{if(event.method==='Runtime.exceptionThrown')report.errors.push(event.params.exceptionDetails.text);}}); + const send=session.send; + await send('Page.enable');await send('Runtime.enable'); + async function evaluate(expression){const result=await send('Runtime.evaluate',{expression,returnByValue:true,awaitPromise:true});if(result.exceptionDetails)throw Error(result.exceptionDetails.exception?.description||result.exceptionDetails.text);return result.result.value;} + async function until(expression){const end=Date.now()+15000;while(Date.now(){const element=document.querySelector(${JSON.stringify(selector)});if(!element||element.disabled)throw Error('Unavailable fixture control');element.scrollIntoView({block:'center'});element.focus();})()`); + await send('Input.dispatchKeyEvent',{type:'keyDown',key:'Enter',code:'Enter',windowsVirtualKeyCode:13,text:'\r'});await send('Input.dispatchKeyEvent',{type:'keyUp',key:'Enter',code:'Enter',windowsVirtualKeyCode:13});} + async function fill(selector,value){await evaluate(`(()=>{const element=document.querySelector(${JSON.stringify(selector)});element.value=${JSON.stringify(value)};element.dispatchEvent(new Event('input',{bubbles:true}));element.dispatchEvent(new Event('change',{bubbles:true}));})()`);} + async function screenshot(name){const shot=await send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,name),Buffer.from(shot.data,'base64'));} + async function load(){await keyClick('#project-portfolio-notes');await until("document.querySelector('#project-portfolio-note').options.length===1");await keyClick('#project-portfolio-load');await until("document.querySelector('#project-portfolio-status').textContent==='Loaded retained authored model organization.'");} + async function selectFirst(){await keyClick('[data-lineage-node="model/fit-first"]');await until("document.querySelector('#project-portfolio-status').textContent==='Exact retained model summary and captured evidence.'");} + await send('Page.navigate',{url:origin+'/'});await until("!!window.fixtureUI"); + await load();await check('readCalls.length===2','Guide loading reads no model card or payload'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('22 models in this guide; 1 source reviewed')",'Exact complete and reviewed guide counts'); + const point=await evaluate("(()=>{const element=document.querySelector('[data-lineage-node=\"group/pressure\"]');element.scrollIntoView({block:'center'});const box=element.getBoundingClientRect();return {x:box.x+box.width/2,y:box.y+box.height/2};})()"); + await send('Input.dispatchMouseEvent',{type:'mousePressed',button:'left',clickCount:1,...point});await send('Input.dispatchMouseEvent',{type:'mouseReleased',button:'left',clickCount:1,...point}); + await check("document.querySelector('#project-portfolio-group').value==='pressure'&&document.activeElement.dataset.lineageNode==='group/pressure'",'Pointer group activation applies the group and retains visible keyboard focus'); + await check("!document.querySelector('#root img')&&!window.injected",'Authored graph labels remain inert text'); + await keyClick('#project-portfolio-next');await check("document.querySelector('#project-portfolio-count').textContent.includes('Showing 21–22.')",'Bounded model paging uses the full population'); + await keyClick('#project-portfolio-prev');await selectFirst(); + await check("document.activeElement.id==='project-portfolio-model-heading'",'Keyboard selection moves focus to selected summary'); + await check("document.querySelector('#project-portfolio-detail').innerText.includes(fixtureData.document.models[0].summary.method)",'Complete long method paragraph is readable retained text'); + await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Desktop detail wraps without horizontal overflow'); + await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('desktop-summary.png'); + await send('Emulation.setDeviceMetricsOverride',{width:390,height:844,deviceScaleFactor:1,mobile:true}); + await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Mobile detail wraps without horizontal overflow'); + await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('mobile-summary.png'); + await send('Emulation.clearDeviceMetricsOverride'); + await fill('#project-portfolio-search','C107');await keyClick('#project-portfolio-filter-form button'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('1 match the applied filters')&&document.querySelector('#project-portfolio-next').disabled",'Alias filtering has exact counts and paging'); + await selectFirst();await keyClick('#project-portfolio-detail button');await until("document.querySelector('#project-portfolio-model-heading')?.textContent==='Second model'"); + await check("document.querySelector('#project-portfolio-detail').innerText.includes('A source-reviewed interpretation has not been recorded')",'Pending review does not invent a method summary'); + await check("document.querySelector('[data-lineage-node=\"model/fit-second\"]').getAttribute('aria-pressed')==='true'",'Off-filter relationship navigation brings selected model into the graph window'); + await selectFirst();await evaluate('window.failNext=true');await keyClick('#project-portfolio-detail button'); + await until("document.querySelector('#project-portfolio-status').textContent==='unavailable: Retained card unavailable.'"); + await check("document.querySelector('#project-portfolio-detail').children.length===0",'Unavailable card clears the previous model detail'); + await evaluate('window.corruptNext=true');await keyClick('[data-lineage-node="model/fit-first"]'); + await until("document.querySelector('#project-portfolio-status').textContent==='invalid: Model card differs from its exact retained organization reference.'"); + await check("document.querySelector('#project-portfolio-detail').children.length===0",'Foreign card revision is refused without stale detail'); + await evaluate('window.holdNext=true');await keyClick('[data-lineage-node="model/fit-first"]');await until('!!window.releaseHeld'); + await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:true});window.releaseHeld()"); + await check("window.heldSignal.aborted&&document.querySelector('#project-portfolio-content').hidden",'Context replacement aborts and clears pending view'); + await load();await check("document.querySelector('#project-portfolio-detail').children.length===0",'Reload after supersession starts without old detail'); + await selectFirst();await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:false})"); + await check("document.querySelector('#root').hidden&&document.querySelector('#project-portfolio-detail').children.length===0&&document.activeElement.id==='outside'",'Permission loss clears private view and restores host focus'); + await evaluate("fixtureUI.setContext({key:'three',project_id:fixtureData.project,enabled:true})");await load(); + await fill('#project-portfolio-search','No match');await keyClick('#project-portfolio-filter-form button'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('0 match the applied filters. Showing 0–0.')",'Empty filter result retains exact zero denominator'); + await check('readCalls.every(call=>[\'files_inspect\',\'file_get\'].includes(call.action))','All interactions use read-only project-file actions'); + if(report.errors.length)throw Error('Browser runtime exception');report.status='passed_authored_portfolio_real_browser_component'; + }catch(error){report.status='failed_authored_portfolio_real_browser_component';report.error=String(error); + if(session){try{const state=await session.send('Runtime.evaluate',{expression:'document.body.innerText',returnByValue:true});report.failure_page=state.result?.value; + const shot=await session.send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,'failed-page.png'),Buffer.from(shot.data,'base64'));}catch(captureError){report.failure_capture_error=String(captureError);}} + throw error;} + finally{session?.close();if(child&&child.exitCode===null&&child.signalCode===null){child.kill('SIGTERM');for(let i=0;i<100&&child.exitCode===null&&child.signalCode===null;i++)await pause(50);if(child.exitCode===null&&child.signalCode===null)child.kill('SIGKILL');} + report.browser_stopped=!child||child.exitCode!==null||child.signalCode!==null;await new Promise(resolve=>server.close(resolve)); + report.end_source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); + report.source_stable_through_tests=JSON.stringify(report.source_sha256)===JSON.stringify(report.end_source_sha256); + if(!report.source_stable_through_tests){report.status='failed_authored_portfolio_real_browser_component';report.errors.push('Source changed during browser qualification');process.exitCode=1;} + report.producer_sha256=hash(fs.readFileSync(__filename));fs.writeFileSync(path.join(output,'report.json'),JSON.stringify(report,null,2)+'\n');} + console.log(report.status,report.checks.length,'checks'); +} +main().catch(error=>{console.error(String(error));process.exitCode=1;}); diff --git a/tests/model_portfolio_fixture.cjs b/tests/model_portfolio_fixture.cjs new file mode 100644 index 00000000..cb63135a --- /dev/null +++ b/tests/model_portfolio_fixture.cjs @@ -0,0 +1,16 @@ +'use strict'; +// Literal presentation-document fixtures, independent of the decoder. +const crypto=require('node:crypto'); +const hash=value=>crypto.createHash('sha256').update(value).digest('hex'); +const project='a'.repeat(32),path='notes/00-methods.portfolio.json'; +const summary={method:'A small pressure correction model.',purpose:'Measure one pressure correction.',experiment:'Two retained training scenes.',outcome:'Retained diagnostic result; not a rollout qualification.',lesson:'Compare work and accuracy on the same cases.',reproduction:'Use the retained source recipe; clean retraining remains pending.'}; +const cardText=Object.values(summary).join('\n\n')+'\nOriginal receipt bytes remain here.\n'; +const ref=(path,text)=>({file_id:hash(project+'\0note\0'+path),path,revision:2,sha256:hash(text)}); +const first={id:'fit-first',label:'First model',group_id:'pressure',aliases:['C107'],review:'source_reviewed',card:ref('notes/first.md',cardText),summary}; +const second={id:'fit-second',label:'Second model',group_id:'pressure',aliases:['C106'],review:'review_pending',card:ref('notes/second.md','Original pending card.\n'),summary:null}; +const document={format:'project_model_portfolio_v1',project_id:project,title:'Retained fit guide',description:'Authored retrospective organization; original Trial history is unchanged.',authorship:'retrospective',groups:[{id:'pressure',label:'Pressure methods',parent:null,description:'Pressure requests, not complete simulation steps.'}],models:[first,second],relationships:[{from:first.id,to:second.id,kind:'comparison',explanation:'Authored comparison with the earlier control.',evidence_model:first.id,excerpt:summary.lesson}]}; +const result=(model,text)=>({id:model.card.file_id,project_id:project,kind:'note',path:model.card.path,version:{revision:2,sha256:hash(text),bytes:Buffer.byteLength(text),text,actor_id:'b'.repeat(32),adopted_at:'2026-10-25T00:00:00+00:00'}}); +const cards=[result(first,cardText),result(second,'Original pending card.\n')]; +const raw=JSON.stringify(document),guideRef=ref(path,raw),guide={...result({card:guideRef},raw),version:{...result({card:guideRef},raw).version,revision:1}}; +const inspection={id:guideRef.file_id,kind:'note',path,revision:1,expected_sha256:guideRef.sha256,expected_bytes:Buffer.byteLength(raw),observed_sha256:guideRef.sha256,observed_bytes:Buffer.byteLength(raw),state:'unchanged',reason:null}; +module.exports={project,document,summary,cards,guide,inspection}; diff --git a/tests/test_agent_api.py b/tests/test_agent_api.py index 2c9c1f70..004121b4 100644 --- a/tests/test_agent_api.py +++ b/tests/test_agent_api.py @@ -284,6 +284,14 @@ def test_absolute_header_deadline_includes_trickle(self): with self.assertRaises(api.AgentAPIError): client.call('data.query',{}) self.assertLess(time.monotonic()-start,1) + def test_owner_transport_keeps_the_selected_absolute_wait_and_never_retries(self): + with server('trickle') as (endpoint,calls): + client=api.ProjectFilesOwnerClient(endpoint,TOKEN,CATALOG,timeout=.15) + start=time.monotonic() + with self.assertRaises(api.AgentAPIError):client._request('GET','/api/workbench/records') + self.assertLess(time.monotonic()-start,1) + self.assertEqual(len(calls),1) + class OutputTests(unittest.TestCase): @unittest.skipUnless(HAS_JSONSCHEMA, 'Optional agent jsonschema dependency is not installed.') diff --git a/tests/test_measurement_comparison.py b/tests/test_measurement_comparison.py index b289d751..16418875 100644 --- a/tests/test_measurement_comparison.py +++ b/tests/test_measurement_comparison.py @@ -91,7 +91,7 @@ def test_overlapping_subjects_and_repeated_sources_are_explicit_errors(self): self.assertEqual(error.exception.code, 'conflict') def test_bounds_and_invalid_selections_never_return_partial_results(self): - for constant, maximum in [('MAX_COMPARISON_BYTES', 1), ('MAX_COMPARISON_SUBJECTS', 1), ('MAX_ROWS', 7)]: + for constant, maximum in [('MAX_COMPARISON_BYTES', 1), ('MAX_COMPARISON_SUBJECTS', 1), ('MAX_COMPARISON_ROWS', 7)]: with patch.object(owner, constant, maximum), self.assertRaises(WorkbenchError): MeasurementComparison(self.inputs) with self.assertRaises(WorkbenchError): MeasurementComparison(self.inputs * 9) @@ -101,6 +101,15 @@ def test_bounds_and_invalid_selections_never_return_partial_results(self): self.comparison.read('error', **arguments) with self.assertRaises(WorkbenchError): self.comparison.read(filters={}) + def test_packet_and_combined_observation_budgets_are_separate(self): + # Both real source tables have four observations. A packet limit of + # four still permits their complete eight-observation comparison. + with patch.object(owner,'MAX_ROWS',4),patch.object(owner,'MAX_COMPARISON_ROWS',8,create=True): + self.assertEqual(MeasurementComparison(self.inputs).read()['observation_count'],8) + with self.assertRaises(WorkbenchError):MeasurementTable(encode(fixture())) + with patch.object(owner,'MAX_COMPARISON_ROWS',7,create=True),self.assertRaises(WorkbenchError): + MeasurementComparison(self.inputs) + def test_returned_metadata_and_query_values_have_no_mutable_owner_alias(self): expected = self.comparison.read('error') result = self.comparison.read('error') diff --git a/tests/test_project_artifacts.py b/tests/test_project_artifacts.py index 2a68497d..ad06e375 100644 --- a/tests/test_project_artifacts.py +++ b/tests/test_project_artifacts.py @@ -74,7 +74,7 @@ def mutate_source_after_snapshot(fd, value, **options): original.write_bytes(b'changed after verified capture') return publish(fd, value, **options) with patch.object(destination, 'publish_fd', side_effect=mutate_source_after_snapshot): - with patch.object(self.store, 'snapshot', wraps=self.store.snapshot) as checks: + with patch.object(self.store, '_current', wraps=self.store._current) as checks: self.artifacts.copy_many_to(destination, values) self.assertEqual(checks.call_count, 2) for raw, value in zip(raws, values): @@ -118,7 +118,7 @@ def test_status_group_retains_exact_hash_observations_with_two_closure_checks(se self.payload(values[2]).unlink() self.payload(values[2]).write_bytes(b'changed') expected = [self.artifacts.status(value) for value in values] - with patch.object(self.store, 'snapshot', wraps=self.store.snapshot) as checks: + with patch.object(self.store, '_current', wraps=self.store._current) as checks: actual = self.artifacts.status_many(values) self.assertEqual(actual, expected) self.assertEqual([item['state'] for item in actual], ['verified', 'missing', 'corrupt']) @@ -161,7 +161,7 @@ def test_presence_batch_matches_individual_observations_with_two_closure_checks( self.artifacts.publish_bytes(b'present',values[0]) self.commit(values) expected=[self.artifacts.inspect(value) for value in values] - with patch.object(self.store,'snapshot',wraps=self.store.snapshot) as checks: + with patch.object(self.store,'_current',wraps=self.store._current) as checks: actual=self.artifacts.inspect_many(values) self.assertEqual(actual,expected) self.assertEqual(checks.call_count,2) diff --git a/tests/test_project_composition.py b/tests/test_project_composition.py index cf42c0d3..c20a90df 100644 --- a/tests/test_project_composition.py +++ b/tests/test_project_composition.py @@ -99,6 +99,19 @@ def test_other_family_conflict_and_repeated_source_project_are_rejected(self): self.assertEqual(repeated.exception.code,'conflict') self.assertIn('source project identity repeats',str(repeated.exception)) + def test_source_unique_policy_overrides_another_sources_runtime_dependency(self): + from project_composition import build_candidate + from test_project_fork_composition import runtime_records + import uuid + sha=hashlib.sha256(b'Z').hexdigest() + runtime,_,_=runtime_records('9'*32,shared_sha=sha,shared_bytes=1) + declaration=next(r for r in runtime if r['family']=='project_artifact' and r['id']==sha) + candidate=build_candidate([snapshot('1',runtime),snapshot('2',[copy.deepcopy(declaration)])], + '3'*32,created_at='2026-10-24T12:00:00+00:00',id_factory=lambda *_:uuid.uuid4().hex, + directory='combined',name='Conservative source custody',request_key='shared-policy') + policies={edge['declaration']['sha256']:edge['preservation'] for edge in candidate['artifacts']} + self.assertEqual(policies[sha],'unique') + def test_source_snapshots_can_be_loaded_lazily(self): records=approved_history() def sources(): diff --git a/tests/test_project_composition_publication.py b/tests/test_project_composition_publication.py index 67764875..0bb6d881 100644 --- a/tests/test_project_composition_publication.py +++ b/tests/test_project_composition_publication.py @@ -73,6 +73,23 @@ def test_complete_publication_preserves_both_sources_and_independent_payloads(se self.assertEqual(target.stat().st_nlink,1) self.assertEqual([s.snapshot() for s in self.stores],before) + def test_already_forked_sources_keep_their_independent_histories(self): + for number in range(2): + result=publish_fork(self.stores[number],self.artifacts[number],self.fd,'forked-source-'+str(number), + root_path=self.root,name='Already forked source',expected_checkpoint=self.snapshots[number]['checkpoint'], + request_key='prepare-fork-'+str(number),quiescent=True, + build_candidate=lambda snap,dest:fork_candidate(snap,dest,created_at=NOW,id_factory=lambda *_:uuid.uuid4().hex), + validate_candidate=lambda records,identity:validate(records,identity,None)) + store=ProjectStore.open(self.root/('forked-source-'+str(number))) + artifacts=ProjectArtifacts(store);self.addCleanup(store.close);self.addCleanup(artifacts.close) + self.stores[number]=store;self.artifacts[number]=artifacts;self.snapshots[number]=result['snapshot'] + source_overlays=[next(r for r in s['records'] if r['family']=='project_fork') for s in self.snapshots] + result=self.publish();records=result['snapshot']['records'];identity=result['snapshot']['project_id'] + self.assertTrue(all(old in records for old in source_overlays)) + self.assertTrue(all(dict(family='project_fork',id=old['id']) in result['fork']['archived'] for old in source_overlays)) + self.assertEqual(decode_fork(records,project_id=identity)['destination_project_id'],identity) + self.assertIn(identity,decode_records(records,project_id=identity)) + def test_missing_unique_secondary_bytes_leave_only_retained_staging(self): (self.stores[1].path/'artifacts/sha256'/self.unique[1]).unlink() self.assert_code('unavailable',self.publish) diff --git a/tests/test_project_dispatch.py b/tests/test_project_dispatch.py index f55de4cb..1aee28e5 100644 --- a/tests/test_project_dispatch.py +++ b/tests/test_project_dispatch.py @@ -30,7 +30,8 @@ def require(self, *args, **kwargs): def projection(): access = Authority() - session = SimpleNamespace(access=access, check=Mock(), snapshot={'project_id': PROJECT, 'records': []}) + session = SimpleNamespace(access=access, check=Mock(), check_current=Mock(), snapshot={'project_id': PROJECT, 'records': []}, + summary={'project_id': PROJECT, 'name': 'Dispatch fixture', 'checkpoint': CHECKPOINT}) return SimpleNamespace(session=session, data=Mock(), materials=Mock(), history=Mock(), context=SimpleNamespace(models=Mock(), pipelines=Mock()), artifacts=Mock(), producers=Mock(), source_adapter=Mock(), runtime_manifest=Mock(), disclose=Mock(), resolve_artifact=Mock(), execution=SimpleNamespace(study_execution=Mock(), producer_execution=Mock(), health=Mock(), preflight_external=Mock(), approve_external=Mock(), submit_external=Mock(), cancel_run=Mock(), reconcile_current=Mock())) @@ -38,7 +39,7 @@ def projection(): class Dispatch(unittest.TestCase): def test_supported_operations_match_the_finite_public_contract(self): - self.assertEqual(SUPPORTED_OPERATIONS, agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | (agent_api.PROJECT_FILES_M4 - {'project_files.save_as'})) + self.assertEqual(SUPPORTED_OPERATIONS, agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | (agent_api.PROJECT_FILES_M4 - {'project_files.save_as', 'project_files.compose'})) with self.assertRaises(WorkbenchError) as failure: dispatch(object(), 'project_files.runs_execute', {}) self.assertEqual(failure.exception.code, 'unsupported') diff --git a/tests/test_project_extraction.py b/tests/test_project_extraction.py index 4ef0ff92..34e43176 100644 --- a/tests/test_project_extraction.py +++ b/tests/test_project_extraction.py @@ -20,6 +20,7 @@ from access import Access from metadata import Metadata from model_definitions import ModelDefinitions +from pipeline import digest from project_data import decode_data_records from project_extraction import _Capture, capture_legacy from project_material import decode_records as decode_material @@ -498,7 +499,7 @@ def test_two_study_lineage_reservation_candidate_and_run_science_are_captured(se before = self.files();source = self.view() captured = self.capture(source, {'studies': [child['id']]}) self.assertEqual(captured['blockers'], [], captured['blockers']) - view = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check=lambda: None) + view = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check_current=lambda: None) from project_material import ProjectMaterial facts = decode_history(captured['records'], materials=ProjectMaterial(view)) self.assertEqual(set(facts.studies), {(self.db.dataset_id, parent['id']), (self.db.dataset_id, child['id'])}) @@ -639,7 +640,7 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure captured = self.capture(source, {'runs': [consumer['id']]}) self.assertEqual(captured['blockers'], [], captured['blockers']) facts = decode_records(captured['records'], materials=ProjectMaterial(SimpleNamespace( - snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check=lambda: None))) + snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check_current=lambda: None))) self.assertEqual(set(facts.runs), {(self.db.dataset_id, run['id']) for run in (producer, middle, consumer)}) for downstream, upstream in ((consumer, middle), (middle, producer)): self.assertIn(dict(source=dict(family='run_history', id=self.db.dataset_id + '.' + downstream['id']), @@ -650,7 +651,8 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure # A fresh inert owner reader reconstructs the exact upstream custody # receipt from retained facts/private captured bytes, never source paths. session = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=copy.deepcopy(captured['records'])), - access=self.access, artifacts=source.artifacts, check=lambda: None) + summary=dict(project_id=self.db.dataset_id, checkpoint=digest(captured['records'])), + access=self.access, artifacts=source.artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda run, population: self.access.require('expose-holdout', 'runs.expose_test', run['id'])) with patch.object(source, 'read_catalog', side_effect=AssertionError('No original catalog reread')), \ diff --git a/tests/test_project_files.py b/tests/test_project_files.py index 538501d0..f512d54e 100644 --- a/tests/test_project_files.py +++ b/tests/test_project_files.py @@ -44,6 +44,16 @@ def intent(self,path,text,revision=0,kind='note'): raw=text.encode('utf-8');target=self.root/path;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(raw) return dict(kind=kind,path=path,expected_revision=revision,expected_sha256=hashlib.sha256(raw).hexdigest()) + def test_inspection_never_copies_the_complete_record_graph(self): + self.adopt('notes/inspection.md', 'Exact retained scientific note', kind='note') + expected = self.files.inspect() + with patch.object(self.session.store, '_snapshot', + wraps=self.session.store._snapshot) as snapshots: + observed = self.files.inspect() + self.assertEqual(observed, expected) + self.assertEqual(snapshots.call_count, 0, + 'File inspection needs only independent file records and complete freshness') + def test_batch_captures_exact_files_in_one_canonical_mutation(self): items=[self.intent('notes/a.md','Exact Ω\r\n'),self.intent('notes/empty.md','')] before=self.session.snapshot['checkpoint'] diff --git a/tests/test_project_files_clients.py b/tests/test_project_files_clients.py index e692dc45..2233b259 100644 --- a/tests/test_project_files_clients.py +++ b/tests/test_project_files_clients.py @@ -272,6 +272,9 @@ def test_browser_contract_pins_are_current_registry_output(self): self.assertEqual(pins, {name: api.contract(name)['sha256'] for name in api._CONTRACTS if name.startswith('project_files.')}) for name, digest in pins.items(): self.assertEqual(api.selected_header(name), name + '/1/' + digest) + versions=json.loads(re.search(r'/\* PROJECT_FILES_VERSIONS_START \*/\s*(.*?)\s*/\* PROJECT_FILES_VERSIONS_END \*/',source,re.S)[1]) + self.assertEqual(versions,{name:{str(v):api.contract(name,v)['sha256'] for v in sorted(values)} + for name,values in api._CONTRACTS.items() if name.startswith('project_files.') and len(values)>1}) def test_reads_pin_binding_checkpoint_and_path_record_without_body(self): arguments = {'record_id': RECORD, 'binding': BINDING, 'checkpoint': CHECKPOINT} @@ -352,6 +355,22 @@ def test_closed_record_output_rejects_unknown_and_inconsistent_history(self): class Consumers(unittest.TestCase): + def test_explicit_owner_wait_budget_preserves_generic_bounds_and_private_configuration(self): + endpoint,token='http://127.0.0.1:12345','fixture-agent-token-for-client-tests' + self.assertEqual(api.ProjectFilesOwnerClient(endpoint,token,CATALOG).timeout,120) + self.assertEqual(api.ProjectFilesOwnerClient(endpoint,token,CATALOG,timeout=7200).timeout,7200) + for client_type,invalid in ((api.AgentClient,301),(api.ProjectFilesOwnerClient,7201), + (api.ProjectFilesOwnerClient,float('inf')),(api.ProjectFilesOwnerClient,True)): + with self.subTest(client=client_type.__name__,timeout=invalid),self.assertRaises(api.AgentAPIError): + client_type(endpoint,token,CATALOG,timeout=invalid) + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);credential=root/'token';configuration=root/'config.json' + credential.write_text(token);credential.chmod(0o600) + configuration.write_text(json.dumps(dict(endpoint=endpoint,catalog_id=CATALOG, + token_file=str(credential),timeout=7200)));configuration.chmod(0o600) + self.assertEqual(api.ProjectFilesOwnerClient.from_config(configuration).timeout,7200) + with self.assertRaises(api.AgentAPIError):api.AgentClient.from_config(configuration) + def test_owner_file_client_keeps_unrelated_operations_outside_authority(self): raw = client({'status': 200, 'result': state()}, owner=True) self.assertEqual(ProjectFilesClient(raw).save(**bound()), state()) diff --git a/tests/test_project_files_fork_contracts.py b/tests/test_project_files_fork_contracts.py index 309f8ed2..b731de2c 100644 --- a/tests/test_project_files_fork_contracts.py +++ b/tests/test_project_files_fork_contracts.py @@ -35,8 +35,9 @@ def test_old_pins_stay_exact_and_only_v2_accepts_closed_fork_origins(self): health = 'project_files.execution_health' self.assertEqual(api.contract(health,1)['sha256'], 'c3b1d0017f5a49cf69c3a4c5c33e1d3db9846404d12627563d97ef40de7404a5') + comparison = 'project_files.measurement_compare' self.assertEqual(versions,{name:{str(v):api.contract(name,v)['sha256'] for v in (1,2)} - for name in (*expected, health)}) + for name in (*expected, health, comparison)}) def test_sdk_selects_v2_and_falls_back_to_exact_v1_without_falsifying_provenance(self): value=fork_result();raw=client({'status':200,'result':value});sdk=ProjectFilesClient(raw) diff --git a/tests/test_project_files_m4_clients.py b/tests/test_project_files_m4_clients.py index 5136e6af..90af3962 100644 --- a/tests/test_project_files_m4_clients.py +++ b/tests/test_project_files_m4_clients.py @@ -84,10 +84,12 @@ def test_capability_profiles_fit_unchanged_budget_and_negotiate_only_intersectio with self.assertRaises(api.AgentAPIError): api.capabilities(CATALOG, profile='arbitrary') self.assertEqual(api.REGISTRY_DIGEST, api._LEGACY_DIGEST) - def test_eleven_operations_have_fixed_consumers_and_closed_inputs(self): - self.assertEqual(len(api.PROJECT_FILES_M4), 11) + def test_complete_m4_operations_have_fixed_consumers_and_closed_inputs(self): + self.assertEqual(api.PROJECT_FILES_M4,{'project_files.'+name for name in ( + 'compose','save_as','files_inspect','file_get','file_history','file_adopt','files_adopt', + 'file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')}) self.assertEqual(project_dispatch.SUPPORTED_OPERATIONS & api.PROJECT_FILES_M4, - api.PROJECT_FILES_M4 - {'project_files.save_as'}) + api.PROJECT_FILES_M4 - {'project_files.save_as','project_files.compose'}) client = ProjectFilesClient(Mock()) parser = project_cli.parser() for operation in api.PROJECT_FILES_M4: @@ -229,13 +231,25 @@ def test_dispatch_projects_metadata_pages_without_captured_text_or_definition_bu import threading access=SimpleNamespace(lock=threading.RLock(),require=Mock()) session=SimpleNamespace(access=access,lock=threading.RLock(),db=SimpleNamespace(lock=threading.RLock()), - check=Mock(return_value={'project_id':PROJECT,'records':[]})) + check=Mock(side_effect=AssertionError('Unused full dispatcher snapshot')),check_current=Mock()) files=SimpleNamespace(history=Mock(return_value=[version('large text '+str(i),i+1) for i in range(42)])) projection=SimpleNamespace(session=session,files=files) page=project_dispatch.dispatch(projection,'project_files.file_history',dict(READ,file_id=identity(),offset=40,limit=2)) self.assertEqual((page['total'],[item['revision'] for item in page['items']]),(42,[41,42])) self.assertTrue(all('text' not in item for item in page['items'])) + session.check_current.assert_called_once_with() api.validate_output('project_files.file_history',dict(status=200,result=bound(page))) + def test_dispatch_fork_info_retains_the_complete_snapshot_contract(self): + import threading + access=SimpleNamespace(lock=threading.RLock(),require=Mock()) + session=SimpleNamespace(access=access,lock=threading.RLock(),db=SimpleNamespace(lock=threading.RLock()), + check=Mock(return_value={'project_id':PROJECT,'records':[]}), + check_current=Mock(side_effect=AssertionError('Fork must read its records'))) + value=project_dispatch.dispatch(SimpleNamespace(session=session),'project_files.fork_info',dict(READ,offset=0,limit=40)) + session.check.assert_called_once_with() + self.assertEqual(value,dict(source=None,project_id=PROJECT, + mapping=dict(items=[],offset=0,limit=40,total=0),archived_count=0,copied_count=0,missing_regenerable_count=0)) + if __name__=='__main__':unittest.main() diff --git a/tests/test_project_files_m4_http.py b/tests/test_project_files_m4_http.py index 7280a2b4..8d6b0a6f 100644 --- a/tests/test_project_files_m4_http.py +++ b/tests/test_project_files_m4_http.py @@ -10,6 +10,7 @@ import project_workspace from project_store import ProjectStoreError +from workbench import WorkbenchError import test_project_files_http as m1 import test_project_files_m2_http as m2 @@ -46,6 +47,12 @@ def test_batch_adoption_uses_real_http_and_survives_independent_fork(self): items.append(dict(kind='note',path='notes/'+name,expected_revision=0,expected_sha256=hashlib.sha256(raw).hexdigest())) result=self.call('files_adopt',items=items) self.assertEqual(result['changed_count'],2);self.assertEqual(len(self.call('files_inspect')['items']),2) + with patch.object(self.workspace._store,'_snapshot',side_effect=AssertionError('Unrelated full graph copy')): + self.assertEqual(len(self.call('files_inspect')['items']),2) + selected=self.call('file_get',file_id=result['items'][0]['id']) + self.assertEqual(selected['version']['text'].encode(),b'Exact first note\r\n') + history=self.call('file_history',file_id=result['items'][0]['id']) + self.assertEqual([v['revision'] for v in history['items']],[1]) source=copy.deepcopy(self.workspace._projection.session.snapshot) self.fork() current=self.workspace._projection.session.snapshot;indexed={(r['family'],r['id']):r for r in current['records']} @@ -70,6 +77,18 @@ def test_nonempty_independent_fork_exact_science_new_editable_ids_and_same_reque current=self.workspace._projection.session.snapshot indexed={(r['family'],r['id']):r for r in current['records']} for r in original['records']:self.assertEqual(indexed[(r['family'],r['id'])],r) + forged=copy.deepcopy(current) + inherited=next(r['document'] for r in forged['records'] if r['family']=='project_file' + and r['document']['project_id']==current['project_id']) + forged_version=inherited['history'][0];forged_text='Forged inherited script\n' + forged_version.update(text=forged_text,bytes=len(forged_text.encode()),sha256=hashlib.sha256(forged_text.encode()).hexdigest()) + inherited['current']=copy.deepcopy(inherited['history'][-1]) + from project_files import decode_records + with self.assertRaisesRegex(WorkbenchError,'Working-file fork requires its exact archived history'): + decode_records(forged['records']) + with self.assertRaisesRegex(WorkbenchError,'Working-file fork requires its exact archived history'): + self.workspace._projection.session.adopt(forged) + self.assertEqual(self.workspace._projection.session.snapshot,current) active=self.call('data_query')['items'] self.assertEqual(len(active),1);self.assertNotEqual(active[0]['id'],row['id']) self.assertEqual(self.call('data_get',record_id=active[0]['id'])['text'],row['text']) diff --git a/tests/test_project_fork.py b/tests/test_project_fork.py index 6241c890..0683eef9 100644 --- a/tests/test_project_fork.py +++ b/tests/test_project_fork.py @@ -53,7 +53,7 @@ def setUp(self): capture.mkdir(parents=True) for name, raw in payloads.items(): (capture / name).write_bytes(raw) - session = SimpleNamespace(snapshot=self.source.snapshot(), artifacts=self.artifacts, check=lambda: None) + session = SimpleNamespace(snapshot=self.source.snapshot(), artifacts=self.artifacts, check_current=lambda: None) self.material = ProjectMaterial(session) self.record = self.material.capture_model_material(self.material_value, local) records = [*self.material.declarations(self.record), self.record] @@ -90,7 +90,7 @@ def test_complete_fork_retains_exact_original_material_hashes_and_copies_without self.assertEqual(overlay['source'], dict(project_id=self.origin, checkpoint=self.original['checkpoint'])) self.assertEqual(next(item for item in snapshot['records'] if item['family'] == 'model_material'), self.record) with ProjectStore.open(self.root / 'fork') as destination, ProjectArtifacts(destination) as artifacts: - reader = ProjectMaterial(SimpleNamespace(snapshot=destination.snapshot(), artifacts=artifacts, check=lambda: None)) + reader = ProjectMaterial(SimpleNamespace(snapshot=destination.snapshot(), artifacts=artifacts, check_current=lambda: None)) self.assertEqual(reader.model(self.material_value['ref'], verify=True), self.material_value) source_ref = self.material_value['material']['source_closure'][0]['blob'] self.assertEqual(reader.blob(self.material_value['ref'], source_ref), SOURCE) @@ -371,5 +371,28 @@ def test_current_overlay_filters_receipts_and_original_editables_without_rewriti self.assert_code('invalid', decode_fork, bad) +class ActiveRecordSelectionTests(unittest.TestCase): + def test_family_selection_keeps_independent_values_and_default_membership(self): + records=[dict(family=family,id='a'*32,document=dict(format='fixture_v1',values=[1])) + for family in ('project_operation','workbench_text')] + self.assertEqual(active_records(records),records) + selected=active_records(records,families=frozenset({'project_operation'})) + self.assertEqual(selected,[records[0]]) + selected[0]['document']['values'].append(2) + self.assertEqual(records[0]['document']['values'],[1]) + self.assertEqual(active_records(records,families=frozenset()),[]) + + def test_family_selection_still_validates_excluded_fork_structure(self): + record=dict(family='workbench_text',id='a'*32,document={'format':'workbench_text_v1'}) + with self.assertRaisesRegex(ProjectForkError,'Duplicate'): + active_records([record,copy.deepcopy(record)],families=frozenset({'project_operation'})) + + def test_family_selection_rejects_implicit_or_invalid_selectors(self): + for invalid in ('project_operation',[],frozenset({1}),frozenset({'bad/family'})): + with self.subTest(selector=invalid),self.assertRaises(ProjectStoreError) as caught: + active_records([],families=invalid) + self.assertEqual(caught.exception.code,'invalid') + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_project_fork_composition.py b/tests/test_project_fork_composition.py index b5fcf831..28e48361 100644 --- a/tests/test_project_fork_composition.py +++ b/tests/test_project_fork_composition.py @@ -232,7 +232,7 @@ def setUp(self): def validate(self,records): decode_data_records(records);material_records(records);file_records(records);producer_records(records) - history_records(records,materials=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check=lambda:None))) + history_records(records,materials=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None))) for receipt in operation_documents(records): operation,result=receipt['operation'],receipt['result'] if not (data_result(operation,result,records) or file_result(operation,result,records) diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 700504cd..c42dd135 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -85,6 +85,47 @@ def legacy_run(origin=ORIGIN): class HistoryCodecTests(unittest.TestCase): + def test_progression_reuses_one_decoded_checkpoint_and_keeps_returned_values_independent(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + checks=[] + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER,checkpoint='c'*64), + summary=dict(project_id=OTHER,checkpoint='c'*64),artifacts=None, + access=SimpleNamespace(require=lambda *args,**kwargs:None), + check_current=lambda:checks.append('actual current-state guard')) + history=ProjectHistory(session,materials=Materials()) + with patch('project_history.decode_records',wraps=decode_records) as decoder: + first=history.progression(ORIGIN,offset=0,limit=20) + first['nodes'][0]['study']['name']='Caller mutation' + first['nodes'][0]['trial']['replicate']=99 + second=history.progression(ORIGIN,offset=0,limit=20) + self.assertEqual(decoder.call_count,1) + self.assertEqual(checks,['actual current-state guard']*2) + self.assertEqual(second['nodes'][0]['study']['name'],'Exact intent λ') + self.assertEqual(second['nodes'][0]['trial']['replicate'],0) + + def test_history_cache_refreshes_on_checkpoint_change_and_does_not_cache_decode_failure(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), + summary=dict(project_id=OTHER,checkpoint='c'*64),artifacts=None, + access=SimpleNamespace(require=lambda *args,**kwargs:None),check_current=lambda:None) + history=ProjectHistory(session,materials=Materials()) + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'Exact intent λ') + updated=copy.deepcopy(records) + for value in (updated[0]['document']['study'],updated[0]['document']['revisions'][0]): + value['name']='New checkpoint name' + session.snapshot=dict(records=updated,project_id=OTHER) + session.summary=dict(project_id=OTHER,checkpoint='d'*64) + with patch('project_history.decode_records',side_effect=WorkbenchError('Decode unavailable','unavailable',503)): + with self.assertRaisesRegex(WorkbenchError,'Decode unavailable') as failed: + history.progression(ORIGIN) + self.assertEqual(failed.exception.code,'unavailable') + with patch('project_history.decode_records',wraps=decode_records) as decoder: + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'New checkpoint name') + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'New checkpoint name') + self.assertEqual(decoder.call_count,1) + def test_fork_preserves_archived_disclosure_observation_without_current_permission(self): from project_fork import _digest,_input from project_session import operation_records @@ -137,7 +178,8 @@ def read_view(self, selected): def recipe(self, *args, **kwargs): raise AssertionError('History reread the live material catalog') session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), - artifacts=None,access=SimpleNamespace(),check=lambda:None) + summary=dict(project_id=OTHER,checkpoint='c'*64), + artifacts=None,access=SimpleNamespace(),check_current=lambda:None) facts=ProjectHistory(session,materials=Provider())._facts() self.assertEqual(len(observed),1) self.assertIs(observed[0],records) @@ -312,6 +354,32 @@ def start(self): self.host,self.owner,self.session=host,owner,session self.history=ProjectHistory(session,materials=Materials()) + def test_cached_history_still_reads_actual_bytes_and_rejects_restored_mtime_corruption(self): + from project_store import ProjectStoreError + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + before=self.store.snapshot() + snapshot=self.store.commit(records,before['checkpoint'],'cache-corruption-fixture',digest(records)) + self.session.adopt(snapshot) + with self.owner.bound(self.host): + expected=self.history.progression(ORIGIN) + header=json.loads((self.store.path/'.tuldok/project.json').read_bytes()) + entry=next(item for item in header['inventory'] if item['family']=='study_history') + path=self.store.path/'.tuldok'/entry['path'] + original=path.read_bytes();info=path.stat() + corrupt=original.replace(b'Exact intent',b'Wrong intent',1) + self.assertNotEqual(corrupt,original);self.assertEqual(len(corrupt),len(original)) + try: + path.write_bytes(corrupt) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + with self.assertRaisesRegex(ProjectStoreError,'Canonical record bytes changed or are corrupt') as failed: + self.history.progression(ORIGIN) + self.assertEqual((failed.exception.code,failed.exception.status),('stale',409)) + finally: + path.write_bytes(original) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + self.assertEqual(self.history.progression(ORIGIN),expected) + def test_progression_uses_original_namespaces_exact_edges_and_bounded_pages(self): parent='a'*32;child='b'*32 ref={'catalog_id':ORIGIN,'trial_id':parent,'material_sha256':MATERIAL} @@ -653,7 +721,7 @@ def test_cold_actual_composed_proof_keeps_origin_without_live_authority(self): root=self.path/'portable-history';store=ProjectStore.create(root,'Actual cold history','new','f'*64) self.addCleanup(store.close) artifacts=ProjectArtifacts(store);self.addCleanup(artifacts.close) - view=SimpleNamespace(snapshot={'project_id':store.snapshot()['project_id'],'records':[]},artifacts=artifacts,check=lambda:None) + view=SimpleNamespace(snapshot={'project_id':store.snapshot()['project_id'],'records':[]},artifacts=artifacts,check_current=lambda:None) material=ProjectMaterial(view) model_record=material.capture_model_material(model,self.models.path) recipe_record=material.capture_recipe_material(composition,self.dataset.pipelines.material_path) diff --git a/tests/test_project_material.py b/tests/test_project_material.py index 8fe2d11a..9d69056c 100644 --- a/tests/test_project_material.py +++ b/tests/test_project_material.py @@ -55,7 +55,8 @@ def setUp(self): self.addCleanup(lambda:self.store.close()) self.artifacts=ProjectArtifacts(self.store);self.addCleanup(lambda:self.artifacts.close()) self.catalog=self.store.snapshot()['project_id'] - self.session=SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,check=lambda:None) + self.session=SimpleNamespace(snapshot=self.store.snapshot(),summary=self.store.summary(), + artifacts=self.artifacts,check_current=lambda:None) self.material=ProjectMaterial(self.session) def capture(self): @@ -87,7 +88,7 @@ def test_cold_relocation_uses_canonical_artifacts_after_local_capture_deleted(se self.artifacts.close();self.store.close();shutil.rmtree(root) moved=self.root/'moved';(self.root/'project').rename(moved) with ProjectStore.open(moved) as store,ProjectArtifacts(store) as artifacts: - material=ProjectMaterial(SimpleNamespace(snapshot=store.snapshot(),artifacts=artifacts,check=lambda:None)) + material=ProjectMaterial(SimpleNamespace(snapshot=store.snapshot(),artifacts=artifacts,check_current=lambda:None)) self.assertEqual(store.snapshot(),snapshot) self.assertEqual(material.model(value['ref'],verify=True),value) source=value['material']['source_closure'][0]['blob'] @@ -144,7 +145,7 @@ def test_fixed_material_snapshot_checks_complete_reference_identity(self): with self.assertRaises(WorkbenchError):view.runtime(dict(ref,**change)) def test_invalid_live_material_reference_precedes_snapshot_access(self): - with patch.object(self.session,'check',side_effect=AssertionError('Invalid reference accessed snapshot')): + with patch.object(self.session,'check_current',side_effect=AssertionError('Invalid reference accessed snapshot')): with self.assertRaises(WorkbenchError):self.material.runtime({'catalog_id':'invalid'}) def test_material_read_view_uses_supplied_records_without_resnapshotting(self): @@ -158,7 +159,7 @@ def test_material_read_view_uses_supplied_records_without_resnapshotting(self): self.assertEqual(self.session.snapshot['records'],[]) def test_material_read_view_rechecks_session_availability(self): - with patch.object(self.session,'check',side_effect=WorkbenchError('Stale checkpoint.','stale',409)): + with patch.object(self.session,'check_current',side_effect=WorkbenchError('Stale checkpoint.','stale',409)): with self.assertRaisesRegex(WorkbenchError,'Stale checkpoint'): self.material.read_view([]) @@ -360,7 +361,7 @@ def test_retained_one_fit_readers_match_directory_without_source_paths_or_infere patch('subprocess.Popen',side_effect=AssertionError('No process')): expected=evidence.verify(root,run,metrics) self.assertEqual(evidence.verify_blobs(run,metrics,read_blob=lambda name,maximum:payloads[name]),expected) - material=ProjectMaterial(SimpleNamespace(check=lambda:None)) + material=ProjectMaterial(SimpleNamespace(check_current=lambda:None)) result=material.evidence(run,read_blob=lambda name,maximum:payloads[name]) self.assertEqual(result['state'],'verified') checkpoint=result['checkpoint_index']['entries'][0]['checkpoint']['sha256'] @@ -449,7 +450,7 @@ def test_real_deferred_sql_failure_cannot_delete_canonical_source_material(self) self.assertEqual(error.exception.code,'projection_unavailable') snapshot=self.store.snapshot();retained=next(item for item in snapshot['records'] if item['family']=='model_material')['document']['value'] self.assertFalse((self.models.path/retained['ref']['sha256']).exists(),'Legacy rollback removes only disposable projection cache') - cold=ProjectMaterial(SimpleNamespace(snapshot=snapshot,artifacts=self.artifacts,check=lambda:None)) + cold=ProjectMaterial(SimpleNamespace(snapshot=snapshot,artifacts=self.artifacts,check_current=lambda:None)) self.assertEqual(cold.model(retained['ref'],verify=True),retained) self.assertEqual(cold.blob(retained['ref'],retained['material']['source_closure'][0]['blob']),SOURCE) @@ -460,7 +461,7 @@ def test_restore_is_inert_and_explicit_preparation_rebuilds_only_projection_cach draft=self.create();value=self.material.publish_model(self.models,draft['id'],{'revision':1},self.session.snapshot['checkpoint'],'cold-model') cold_db=Metadata(self.path/'cold-projection',dataset_id=self.db.dataset_id,projection_id=uuid.uuid4().hex) self.resources.append(cold_db);self.schemas.append(cold_db.schema) - cold=ProjectMaterial(SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,db=cold_db,check=lambda:None)) + cold=ProjectMaterial(SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,db=cold_db,check_current=lambda:None)) with patch('model_definitions.ModelDefinitions',side_effect=AssertionError('No mutable owner on Open')), \ patch('pipeline.Pipelines',side_effect=AssertionError('No mutable owner on Open')), \ patch('subprocess.Popen',side_effect=AssertionError('No process')): diff --git a/tests/test_project_material_fork.py b/tests/test_project_material_fork.py index 89681263..fb27a597 100644 --- a/tests/test_project_material_fork.py +++ b/tests/test_project_material_fork.py @@ -52,7 +52,8 @@ def overlay(records, made, source, checkpoint, destination): class MaterialForkCodecs(unittest.TestCase): def setUp(self): self.source=uuid.uuid4().hex;self.checkpoint='a'*64 - self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]),check=lambda:None) + self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]), + summary=dict(project_id=self.source,checkpoint=self.checkpoint),check_current=lambda:None) self.material=ProjectMaterial(self.session) def capture(self,family,current): @@ -108,7 +109,8 @@ def test_bound_drafts_preserve_full_configuration_as_visible_nonexecutable_inten ('pipeline_graph',graph(external=True)),('recipe_definition',recipe(self.source))] captured=[self.capture(f,v) for f,v in drafts] made,combined,destination=self.fork() - session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined),check=lambda:None) + session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined), + summary=dict(project_id=destination),check_current=lambda:None) reader=ProjectMaterial(session) intents=reader.intents() self.assertEqual(len(intents),4) @@ -167,7 +169,8 @@ def test_captured_source_selector_stays_in_inert_intent_and_cannot_publish(self) doc=made['records'][0]['document'] self.assertEqual(doc['current']['required_reselection'],['source_selector']) self.assertEqual(doc['current']['definition']['source_selector'],value['source_selector']) - reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined),check=lambda:None)) + reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined), + summary=dict(project_id=destination),check_current=lambda:None)) with self.assertRaises(WorkbenchError) as caught: reader.publish_model(SimpleNamespace(publish=lambda *args: self.fail('Intent reached owner publication')), doc['current']['id'],{'revision':1},'a'*64,'forbidden-intent') diff --git a/tests/test_project_measurement_comparison_http.py b/tests/test_project_measurement_comparison_http.py index 09063754..df5f9667 100644 --- a/tests/test_project_measurement_comparison_http.py +++ b/tests/test_project_measurement_comparison_http.py @@ -3,6 +3,10 @@ import json import os import unittest +from unittest.mock import patch + +import agent_api +from workbench import WorkbenchError import test_project_files_http as m1 import test_project_files_m2_http as m2 @@ -56,3 +60,14 @@ def test_two_original_sources_compare_and_inspect_through_real_HTTP(self): missing=self.call('measurement_compare',sources=sources,metric_id='latency')['value']['query'] self.assertTrue(all(s['maximum'] is None for s in missing['summary'])) self.assertEqual(self.workspace._projection.session.snapshot,before) + # Exercise version-specific admission at the real workspace/SQL owner + # with small declared fixture budgets; no partial comparison escapes. + arguments=self.arguments('measurement_compare',sources=sources) + with self.owner.bound(self.workspace),patch.object(agent_api,'measurement_comparison_row_limit', + side_effect=lambda version:7 if version==1 else 8): + with self.assertRaises(WorkbenchError) as error: + self.workspace.dispatch('project_files.measurement_compare',arguments,selected_version=1) + self.assertEqual(error.exception.code,'too_large') + value=self.workspace.dispatch('project_files.measurement_compare',arguments,selected_version=2) + self.assertEqual(value['value']['value']['observation_count'],8) + self.assertEqual(self.workspace._projection.session.snapshot,before) diff --git a/tests/test_project_measurement_reads.py b/tests/test_project_measurement_reads.py new file mode 100644 index 00000000..1365d63a --- /dev/null +++ b/tests/test_project_measurement_reads.py @@ -0,0 +1,106 @@ +"""Read the adopted measurement family without cloning unrelated retained facts. + +Real canonical files and real session freshness guards; no SQL reconstruction +or packet import is claimed by this focused fixture. +""" +import copy +import json +import os +from pathlib import Path +import tempfile +import threading +import unittest +from unittest.mock import patch + +import project_measurements +import project_session +from project_projection import validate_project_records +from project_store import ProjectStore, ProjectStoreError +from test_project_receipt_validation import candidate, PROJECT + + +class MeasurementReadTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-measurement-read-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name)/'project' + self.store = ProjectStore.create(self.root, 'Measurement read fixture', 'new', 'a'*64) + self.addCleanup(self.store.close) + self.records = candidate() + self.records.append(dict(family='project_artifact', id='f'*64, + document=dict(format='project_artifact_v1', sha256='f'*64, bytes=7))) + validate_project_records(self.records, project_id=PROJECT, artifacts=None) + snapshot = self.store.commit(self.records, self.store.summary()['checkpoint'], 'seed', 'b'*64) + self.session = object.__new__(project_session.ProjectSession) + self.session.store, self.session.available = self.store, True + self.session._snapshot, self.session.lock = copy.deepcopy(snapshot), threading.RLock() + self.owner = object.__new__(project_measurements.ProjectMeasurements) + self.owner.session = self.session + self.expected = {r['id']: copy.deepcopy(r['document']) for r in self.records + if r['family'] == project_measurements.FAMILY} + + def test_documents_are_literal_independent_and_do_not_copy_unrelated_graph(self): + original = project_session.copy_validated_json + unrelated = [] + def observe(value): + if isinstance(value, dict) and 'records' in value: + unrelated.extend(r['id'] for r in value['records'] if r['id'] == 'f'*64) + return original(value) + with patch.object(project_session, 'copy_validated_json', observe): + result = self.owner._documents() + self.assertEqual(result, self.expected) + self.assertEqual(unrelated, [], 'Measurement read copied an unrelated retained envelope') + next(iter(result.values()))['source']['path'] = 'caller-edited.json' + self.assertEqual(self.owner._documents(), self.expected) + self.assertEqual(self.session._snapshot, self.store.snapshot()) + + def test_lookup_reads_every_canonical_file_without_redecoding_adopted_material(self): + with patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ + patch.object(project_measurements, 'material_records', + side_effect=AssertionError('Read-only lookup redecoded adopted material')): + self.assertEqual(self.owner._documents(), self.expected) + self.assertEqual(reads.call_count, len(self.records), 'All current files must still be read and hashed') + + def test_unavailable_projection_fails_before_any_store_read(self): + self.session.available = False + with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')): + with self.assertRaises(project_session.ProjectSessionError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'projection_unavailable') + + def test_other_family_publication_requires_reopen(self): + changed = copy.deepcopy(self.records) + changed[-1]['document']['bytes'] = 8 + self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c'*64) + with self.assertRaises(project_session.ProjectSessionError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'stale_project') + + def test_same_length_unselected_corruption_with_restored_mtime_is_rejected(self): + inventory = json.loads((self.root/'.tuldok/project.json').read_bytes())['inventory'] + entry = next(r for r in inventory if r['id'] == 'f'*64) + path = self.root/'.tuldok'/entry['path'] + before, info = path.read_bytes(), path.stat() + changed = before.replace(b'"bytes": 7', b'"bytes": 8') + self.assertNotEqual(changed, before) + self.assertEqual(len(changed), len(before)) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + with self.assertRaises(ProjectStoreError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'stale') + + def test_family_can_be_empty_and_returned_snapshot_contract_stays_complete(self): + records = [r for r in self.records if r['family'] not in (project_measurements.FAMILY,'project_operation')] + validate_project_records(records, project_id=PROJECT, artifacts=None) + self.session._snapshot = self.store.commit(records, self.store.summary()['checkpoint'], 'empty', 'd'*64) + self.assertEqual(self.owner._documents(), {}) + snapshot = self.session.snapshot + self.assertEqual(snapshot, self.store.snapshot()) + snapshot['records'].clear() + self.assertEqual(self.session.snapshot, self.store.snapshot()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_measurements_clients.py b/tests/test_project_measurements_clients.py index e05b6b28..079f6ab2 100644 --- a/tests/test_project_measurements_clients.py +++ b/tests/test_project_measurements_clients.py @@ -24,6 +24,16 @@ def sdk(self,value): return ProjectFilesClient(client({'status':200,'result':dict(binding=BINDING,project_id=PROJECT, checkpoint=CHECKPOINT,value=value)})) + def test_comparison_capacity_is_versioned_without_changing_packet_headers(self): + from retained_measurements import MAX_COMPARISON_ROWS,MAX_ROWS + self.assertEqual(api.measurement_comparison_row_limit(1),100000) + self.assertEqual(api.measurement_comparison_row_limit(2),250000) + self.assertEqual(api.measurement_comparison_row_limit(2),MAX_COMPARISON_ROWS) + self.assertEqual(MAX_ROWS,100000) + self.assertEqual(api._MEASUREMENT_HEADER['properties']['observation_count']['maximum'],100000) + self.assertNotEqual(api.contract('project_files.measurement_compare',1)['sha256'], + api.contract('project_files.measurement_compare',2)['sha256']) + def test_group_comparison_header_and_query_correlate_every_original_source(self): locator=dict(origin_catalog_id=PROJECT,packet_sha256=self.table.sha256) comparison=MeasurementComparison([(locator,self.table)]) diff --git a/tests/test_project_model_portfolio.cjs b/tests/test_project_model_portfolio.cjs new file mode 100644 index 00000000..704e4035 --- /dev/null +++ b/tests/test_project_model_portfolio.cjs @@ -0,0 +1,47 @@ +'use strict'; +const assert=require('node:assert/strict'),fs=require('node:fs'),vm=require('node:vm'),path=require('node:path'); +const fixture=require('./model_portfolio_fixture.cjs'),window={}; +vm.runInNewContext(fs.readFileSync(path.join(__dirname,'../static/project-model-portfolio.js'),'utf8'),{window,TextEncoder}); +const {decode,checkCard}=window.ProjectModelPortfolioDocument; +const valid=()=>structuredClone(fixture.document),parse=value=>decode(JSON.stringify(value)); +assert.deepEqual(JSON.parse(JSON.stringify(parse(valid()))),fixture.document); +let checks=1; +function bad(change,message,code='invalid'){ + const value=valid();change(value); + assert.throws(()=>parse(value),error=>error.code===code&&error.message===message);checks++; +} +bad(d=>{d.format='other';},'Unsupported model-organization document format.','unsupported'); +bad(d=>{d.extra=true;},'Invalid model-organization document.'); +bad(d=>{d.models=[];},'Invalid model-organization document.'); +bad(d=>{d.groups.push(structuredClone(d.groups[0]));},'Invalid or duplicate organization group.'); +bad(d=>{d.groups[0].parent='missing';},'Organization group parent is unavailable.'); +bad(d=>{d.groups[0].parent='pressure';},'Organization grouping contains a cycle.'); +bad(d=>{d.models.push(structuredClone(d.models[0]));},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].group_id='missing';},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].aliases.push('C107');},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].review='qualified';},'Invalid or duplicate organized model.'); +bad(d=>{delete d.models[0].summary.method;},'Reviewed model requires all six authored summary fields.'); +bad(d=>{d.models[1].summary={};},'Pending review cannot expose a reviewed summary.'); +bad(d=>{d.models[0].card.path='notes/../private';},'Invalid captured model-note reference.'); +bad(d=>{d.models[0].card.revision=0;},'Invalid captured model-note reference.'); +bad(d=>{d.relationships[0].to='unknown';},'Invalid authored model relationship.'); +bad(d=>{d.relationships[0].to=d.relationships[0].from;},'Invalid authored model relationship.'); +bad(d=>{d.relationships[0].kind='trial_parent';},'Invalid authored model relationship.'); +bad(d=>{d.relationships.push(structuredClone(d.relationships[0]));},'Duplicate authored model relationship.'); +bad(d=>{const a=d.relationships[0];a.kind='parameter_derivation';d.relationships.push({...a,from:a.to,to:a.from});},'Authored parameter derivation contains a cycle.'); +assert.throws(()=>decode('not JSON'),error=>error.code==='invalid'&&error.message==='Organization note is not valid JSON.');checks++; +assert.throws(()=>decode(' '.repeat(1048577)),error=>error.code==='invalid'&&error.message==='Organization note exceeds its retained text limit.');checks++; +assert.equal(checkCard(valid(),fixture.document.models[0],fixture.cards[0]),fixture.cards[0].version.text);checks++; +for(const change of [r=>{r.project_id='c'.repeat(32);},r=>{r.id='d'.repeat(64);},r=>{r.path='notes/other.md';},r=>{r.version.revision=3;},r=>{r.version.sha256='e'.repeat(64);},r=>{r.version.deleted=true;}]){ + const result=structuredClone(fixture.cards[0]);change(result); + assert.throws(()=>checkCard(valid(),fixture.document.models[0],result),error=>error.code==='invalid'&&error.message==='Model card differs from its exact retained organization reference.');checks++; +} +{ + const value=valid();value.models[0].summary.method='Not present in the captured card.'; + assert.throws(()=>checkCard(value,value.models[0],fixture.cards[0]),error=>error.message==='Authored summary differs from the captured model card.');checks++; +} +{ + const value=valid();value.relationships[0].excerpt='Not present in the captured card.'; + assert.throws(()=>checkCard(value,value.models[0],fixture.cards[0]),error=>error.message==='Authored relationship excerpt differs from its captured model card.');checks++; +} +console.log('PASS authored portfolio document:',checks,'literal contract/reference decisions; no native capture or browser claim.'); diff --git a/tests/test_project_model_portfolio_http.py b/tests/test_project_model_portfolio_http.py new file mode 100644 index 00000000..8fbda68f --- /dev/null +++ b/tests/test_project_model_portfolio_http.py @@ -0,0 +1,84 @@ +"""Real authenticated note transport/cold reload; no model or recipe execution.""" +import hashlib +import json +import os +from pathlib import Path +import unittest +from unittest.mock import patch + +import project_workspace +import test_project_files_http as m1 +import test_project_files_m2_http as m2 + +ROOT = Path(__file__).resolve().parents[1] + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class ModelPortfolioHTTP(unittest.TestCase): + cleanup_owned = m1.ProjectFilesHTTP.cleanup_owned + select_host = m1.ProjectFilesHTTP.select_host + request = m1.ProjectFilesHTTP.request + state = m1.ProjectFilesHTTP.state + envelope = m1.ProjectFilesHTTP.envelope + action = m1.ProjectFilesHTTP.action + create = m1.ProjectFilesHTTP.create + arguments = m2.ProjectFilesM2HTTP.arguments + transport = m2.ProjectFilesM2HTTP.transport + call = m2.ProjectFilesM2HTTP.call + + def setUp(self): + self.operations = [] + m1.ProjectFilesHTTP.setUp(self) + gate = patch.object(project_workspace, 'M4_ENABLED', True) + gate.start() + self.addCleanup(gate.stop) + + def test_native_static_module_exact_note_revision_and_cold_reload(self): + status, html = self.request('/project-files') + self.assertEqual(status, 200) + self.assertIn(b'/project-model-portfolio.js', html) + self.assertIn(b'id="project-model-portfolio"', html) + status, script = self.request('/project-model-portfolio.js') + self.assertEqual(status, 200) + self.assertEqual(script, (ROOT/'static/project-model-portfolio.js').read_bytes()) + opened = self.create() + root = self.external/'alpha' + (root/'notes').mkdir() + paragraphs = {key: 'Literal retained '+key+' paragraph.' for key in + ('method','purpose','experiment','outcome','lesson','reproduction')} + card_text = '\n\n'.join(paragraphs.values())+'\nOriginal evidence remains.\n' + (root/'notes/model.md').write_text(card_text) + card = self.call('file_adopt',kind='note',path='notes/model.md',expected_revision=0, + expected_sha256=hashlib.sha256(card_text.encode()).hexdigest()) + guide = dict(format='project_model_portfolio_v1',project_id=opened['project']['project_id'], + title='Authored retained fixture',description='Synthetic presentation fixture; no scientific quality claim.', + authorship='retrospective',groups=[dict(id='pressure',label='Pressure methods',parent=None,description='An authored group.')], + models=[dict(id='fixture-fit',label='One retained fit',group_id='pressure',aliases=['Fixture'],review='source_reviewed', + card=dict(file_id=card['id'],path='notes/model.md',revision=1,sha256=card['sha256']),summary=paragraphs)],relationships=[]) + raw=(json.dumps(guide,ensure_ascii=False)+'\n').encode() + (root/'notes/00-fixture.portfolio.json').write_bytes(raw) + saved=self.call('file_adopt',kind='note',path='notes/00-fixture.portfolio.json',expected_revision=0, + expected_sha256=hashlib.sha256(raw).hexdigest()) + observed=self.call('file_get',file_id=saved['id'],revision=1) + self.assertEqual(observed['version']['text'].encode(),raw) + self.assertEqual(json.loads(observed['version']['text']),guide) + changed='A later authored card revision.\n' + (root/'notes/model.md').write_text(changed) + self.call('file_adopt',kind='note',path='notes/model.md',expected_revision=1, + expected_sha256=hashlib.sha256(changed.encode()).hexdigest()) + self.assertEqual(self.call('file_get',file_id=card['id'],revision=1)['version']['text'],card_text) + self.assertEqual(self.call('file_get',file_id=card['id'])['version']['text'],changed) + self.action('save') + expected=(root/'.tuldok/project.json').read_bytes() + self.action('close') + cold=os.environ.get('TULDOK_PROJECT_FILES_COLD_DATABASE_URL') + self.assertTrue(cold,'Independent cold role required for retained-note acceptance') + self.select_host(self.path/'cold-control',{'fixture':self.external},cold) + self.action('open',root='fixture',directory='alpha') + self.assertEqual((root/'.tuldok/project.json').read_bytes(),expected) + self.assertEqual(self.call('file_get',file_id=saved['id'],revision=1)['version']['text'].encode(),raw) + retained=self.call('file_get',file_id=card['id'],revision=1) + self.assertEqual(retained['version']['text'],card_text) + self.assertEqual(retained['project_id'],guide['project_id']) + self.assertEqual(self.call('file_get',file_id=card['id'])['version']['text'],changed) + self.action('close') diff --git a/tests/test_project_native_extraction.py b/tests/test_project_native_extraction.py index c2c3372b..291e2247 100644 --- a/tests/test_project_native_extraction.py +++ b/tests/test_project_native_extraction.py @@ -86,7 +86,7 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact captured = self.capture_without_host(run, source) self.assertEqual(captured['blockers'], [], captured['blockers']) materials = ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, - records=captured['records']), check=lambda: None)) + records=captured['records']), check_current=lambda: None)) facts = decode_records(captured['records'], materials=materials) saved = facts.runs[(self.db.dataset_id, run['id'])] self.assertEqual(saved['run']['runtime'], run['runtime']) @@ -115,7 +115,10 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact for name in originals: (stage / name).unlink() cold = ProjectStore.open(destination); self.addCleanup(cold.close) cold_artifacts = ProjectArtifacts(cold); self.addCleanup(cold_artifacts.close) - session = SimpleNamespace(snapshot=cold.snapshot(), access=self.access, artifacts=cold_artifacts, check=lambda: None) + snapshot = cold.snapshot() + session = SimpleNamespace(snapshot=snapshot, + summary={key:snapshot[key] for key in ('project_id','checkpoint')}, + access=self.access, artifacts=cold_artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda *args: None) for name, raw in originals.items(): self.assertEqual(history.artifact(run['id'], 'source/' + name, origin_catalog_id=self.db.dataset_id), raw) diff --git a/tests/test_project_native_process.py b/tests/test_project_native_process.py index 287de66a..5df959c4 100644 --- a/tests/test_project_native_process.py +++ b/tests/test_project_native_process.py @@ -5,6 +5,7 @@ import os from pathlib import Path import select +import shutil import signal import sys import tempfile @@ -311,8 +312,11 @@ def test_current_interpreter_alias_drift_before_launch_cannot_select_another_int stage=self.root/'venv-alias-drift' prepared=native.prepare(stage,phase='probe',attempt_id=uuid.uuid4().hex,request_sha256='d'*64, runtime=runtime,worker='text_training.py',sources={'text_training.py':WORKER},wall_seconds=8) - Path(executable).unlink();Path(executable).symlink_to(Path('/usr/bin/python3').resolve()) - with self.assertRaises(custody.ProcessError) as error:prepared.launch(stage/'custody') + replacement=self.root/'replacement-python' + shutil.copy2(runtime['executable_source'],replacement) + self.assertNotEqual(replacement.resolve(),Path(executable).resolve()) + Path(executable).unlink();Path(executable).symlink_to(replacement) + with self.assertRaises(custody.ProcessError) as error:self.launch(prepared) self.assertEqual(error.exception.code,'stale');self.assertFalse((stage/'custody').exists()) def test_spawn_callback_is_not_reached_for_current_config_alias_or_capture_refusal(self): diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index d08aaadc..3f1e3e98 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -6,7 +6,7 @@ from pathlib import Path import tempfile import unittest -from unittest.mock import patch +from unittest.mock import PropertyMock, patch import uuid import psycopg @@ -120,6 +120,25 @@ def test_unknown_receipt_is_rejected_before_invoke_and_sql_or_checkpoint_changes self.assertEqual(p.store.snapshot(), before) self.assertEqual(p.db.execute('SELECT COUNT(*) FROM workbench_records').fetchone()[0], 0) + def test_text_tag_uses_real_session_summary_and_record_list_remains_owned(self): + text = self.call('import_text', self.mutation(asset=dict(text='Owned note 山', groups=['owned'])))['record'] + p = self.host._projection + with self.owner.bound(self.host): + expected = dict(project_id=p.session.summary['project_id'], + checkpoint=p.session.summary['checkpoint']) + with patch.object(type(p.session), 'snapshot', new_callable=PropertyMock, + side_effect=AssertionError('Scalar tag copied a full Session')): + self.assertEqual(p.text._tag(dict(value='Exact scalar tag')), + dict(expected, value='Exact scalar tag')) + exact_current = [r['document']['current'] for r in p.session.snapshot['records'] + if r['family'] in ('workbench_text', 'workbench_record') + and r['document']['current']['kind'] == 'text'] + result = p.text.records() + self.assertEqual(result, dict(expected, items=exact_current)) + self.assertEqual([r['id'] for r in result['items']], [text['id']]) + result['items'][0]['text'] = 'Caller mutation' + self.assertEqual(p.text.get(text['id'])['text'], 'Owned note 山') + def test_explicit_rooted_artifact_supply_is_durable_and_never_fetches_or_executes(self): p = self.host._projection raw = b'Unique measured bytes; never regenerate these.\n' diff --git a/tests/test_project_publication_recovery.py b/tests/test_project_publication_recovery.py new file mode 100644 index 00000000..36067a6b --- /dev/null +++ b/tests/test_project_publication_recovery.py @@ -0,0 +1,116 @@ +"""Real committed-stage recovery preserves bytes and never reconstructs sources.""" +import errno +import os +import shutil +import unittest +from unittest.mock import patch + +from project_composition import composition_input +from project_fork import recover_committed_publication +from project_store import ProjectStore, ProjectStoreError +from project_files import file_id, materialize_captured +from test_project_composition_publication import CompositionPublicationTests + + +class PublicationRecoveryTests(unittest.TestCase): + setUp=CompositionPublicationTests.setUp + publish=CompositionPublicationTests.publish + assert_code=CompositionPublicationTests.assert_code + + def stage(self): + with patch('project_fork._rename_noreplace',side_effect=OSError(errno.EIO,'interrupted before rename')): + self.assert_code('unavailable',self.publish) + self.staging=next(self.root.glob('.tuldok-fork-*')) + with ProjectStore.open(self.staging) as store: + self.committed=store.snapshot() + self.request=composition_input(self.snapshots,'combined','Combined','compose-exact') + + def recover(self,**changes): + args=dict(root_path=self.root,expected_project_id=self.committed['project_id'], + expected_checkpoint=self.committed['checkpoint'],expected_request=self.request,quiescent=True) + args.update(changes) + return recover_committed_publication(self.fd,self.staging.name,'combined',**args) + + def test_committed_stage_recovers_without_sources_payload_copies_or_new_identity(self): + self.stage() + before={p.name:(p.read_bytes(),p.stat().st_ino) for p in (self.staging/'artifacts/sha256').iterdir()} + for store,artifacts in zip(self.stores,self.artifacts): + artifacts.close();store.close();shutil.rmtree(store.path) + with patch('project_artifacts.ProjectArtifacts.copy_many_to',side_effect=AssertionError('No recopy')), \ + patch('project_composition.build_candidate',side_effect=AssertionError('No rebuild')): + result=self.recover() + self.assertFalse(self.staging.exists()) + self.assertEqual(result['payloads_copied'],0) + self.assertFalse(result['selected']) + with ProjectStore.open(self.root/'combined') as store: + self.assertEqual(store.snapshot(),self.committed) + after={p.name:(p.read_bytes(),p.stat().st_ino) for p in (self.root/'combined/artifacts/sha256').iterdir()} + self.assertEqual(before,after) + self.assertTrue(self.recover()['reconciled']) + + def test_wrong_request_checkpoint_or_identity_never_publishes(self): + self.stage() + self.assert_code('stale',lambda:self.recover(expected_checkpoint='0'*64)) + self.assert_code('stale',lambda:self.recover(expected_project_id='0'*32)) + self.assert_code('conflict',lambda:self.recover(expected_request={**self.request,'name':'Changed'})) + self.assert_code('busy',lambda:self.recover(quiescent=False)) + self.assertTrue(self.staging.exists());self.assertFalse((self.root/'combined').exists()) + + def test_missing_copied_payload_preserves_stage_and_refuses_publication(self): + self.stage();next((self.staging/'artifacts/sha256').iterdir()).unlink() + with self.assertRaises(ProjectStoreError):self.recover() + self.assertTrue(self.staging.exists());self.assertFalse((self.root/'combined').exists()) + + def test_destination_appearance_never_overwrites_either_tree(self): + self.stage();(self.root/'combined').mkdir();(self.root/'combined/unique').write_bytes(b'keep') + self.assert_code('conflict',self.recover) + self.assertTrue(self.staging.exists());self.assertEqual((self.root/'combined/unique').read_bytes(),b'keep') + + def test_lost_rename_ack_reconciles_exact_published_checkpoint(self): + self.stage();fsync=os.fsync + def fail_after_publish(fd): + if (self.root/'combined').exists():raise OSError(errno.EIO,'lost publication acknowledgement') + fsync(fd) + with patch('project_fork.os.fsync',side_effect=fail_after_publish): + self.assert_code('unknown',self.recover) + self.assertFalse(self.staging.exists());self.assertTrue(self.recover()['reconciled']) + + def test_live_writer_and_non_stage_path_are_refused(self): + self.stage() + with ProjectStore.open(self.staging): + self.assert_code('busy',self.recover) + self.assert_code('invalid',lambda:recover_committed_publication(self.fd,'source0','combined', + root_path=self.root,expected_project_id=self.committed['project_id'],expected_checkpoint=self.committed['checkpoint'], + expected_request=self.request,quiescent=True)) + + def test_initial_empty_stage_cannot_be_mistaken_for_committed_publication(self): + self.stage();empty_name='.tuldok-fork-'+'9'*32 + with ProjectStore.create(self.root/empty_name,'Uncommitted','initial','a'*64) as store: + initial=store.snapshot() + self.assert_code('invalid',lambda:recover_committed_publication(self.fd,empty_name,'combined', + root_path=self.root,expected_project_id=initial['project_id'],expected_checkpoint=initial['checkpoint'], + expected_request=self.request,quiescent=True)) + self.assertTrue((self.root/empty_name).exists());self.assertFalse((self.root/'combined').exists()) + + def test_missing_captured_working_file_refuses_without_changing_committed_history(self): + # Seed a real current note before composing; its cloned editable record + # and materialized bytes then pass through the normal publication path. + from datetime import datetime, timezone + import hashlib + store=self.stores[0];snap=store.snapshot();raw=b'original note\n' + version=dict(revision=1,text=raw.decode(),sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw), + actor_id='1'*32,adopted_at=datetime.now(timezone.utc).isoformat()) + doc=dict(format='project_file_v1',project_id=snap['project_id'],kind='note',path='notes/test.md', + current=version,history=[version],origin=None) + record=dict(family='project_file',id=file_id(snap['project_id'],'note','notes/test.md'),document=doc) + self.snapshots[0]=store.commit([*snap['records'],record],snap['checkpoint'],'note','c'*64) + with store.root_directory() as root: + materialize_captured(self.snapshots[0]['records'],snap['project_id'],root) + self.stage();(self.staging/'notes/test.md').unlink() + self.assert_code('working_files_changed',self.recover) + self.assertFalse((self.root/'combined').exists()) + with ProjectStore.open(self.staging) as retained: + self.assertEqual(retained.snapshot(),self.committed) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_project_receipt_validation.py b/tests/test_project_receipt_validation.py new file mode 100644 index 00000000..bf69926f --- /dev/null +++ b/tests/test_project_receipt_validation.py @@ -0,0 +1,204 @@ +"""One mixed candidate: retain every receipt check without repeated decoding.""" +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import project_files as files +import project_material as material +import project_measurements as measurements +import project_projection as projection +import project_sources as sources +from project_artifacts import ProjectArtifacts, ProjectArtifactError +from project_store import ProjectStore +from test_retained_measurements import fixture +from workbench import WorkbenchError + +PROJECT = 'a' * 32 +ACTOR = 'c' * 32 +INSTANT = '2026-10-25T00:00:00+00:00' + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def operation(name, key, result): + return dict(family='project_operation', id=PROJECT + '.' + sha(key.encode()), document=dict( + format='project_operation_v2', project_id=PROJECT, operation=name, + target=None, request_key=key, input_sha256='d' * 64, result=copy.deepcopy(result))) + + +def candidate(): + packets = [] + for index in range(2): + value = fixture() + value['title'] = f'Packet {index}' + raw = json.dumps(value).encode() + packets.append(dict(raw=raw, sha256=sha(raw), path=f'packet-{index}.json', + reference_id=f'packet-{index}', title=value['title'])) + manifest = dict(schema_version=1, id='b' * 32, name='Receipt source', + data=[dict(id=p['reference_id'], label=p['title'], path=p['path'], sha256=p['sha256']) + for p in packets], recipes=[], evidence=[], requirements=[]) + raw = json.dumps(manifest).encode() + digest = sha(raw) + snapshot = dict(id=manifest['id'], name=manifest['name'], root='original', + root_identity=[1, 2], manifest_path='source.json', manifest_sha256=digest, manifest=manifest) + retained = dict(family='project_snapshot', id=PROJECT + '.' + digest, document=dict( + format=material.FAMILIES['project_snapshot'], origin_catalog_id=PROJECT, value=snapshot, + artifacts=[dict(name='source.json', blob=dict(sha256=digest, bytes=len(raw)), + role='project_manifest', preservation='unique', required_for=['snapshot_verification'])], audit=[])) + records = [retained] + for hashed, size in [(digest, len(raw)), *[(p['sha256'], len(p['raw'])) for p in packets]]: + records.append(dict(family='project_artifact', id=hashed, + document=dict(format='project_artifact_v1', sha256=hashed, bytes=size))) + source_result = dict(origin_catalog_id=PROJECT, project_id=manifest['id'], manifest_sha256=digest, + name=manifest['name'], schema_version=1, actor_id=ACTOR, registered_at=INSTANT) + for index in range(2): + records.append(operation('source.register', f'source-{index}', source_result)) + for index, packet in enumerate(packets): + doc = dict(format=measurements.FORMAT, origin_catalog_id=PROJECT, id=packet['sha256'], + title=packet['title'], blob=dict(sha256=packet['sha256'], bytes=len(packet['raw'])), + source=dict(origin_catalog_id=PROJECT, project_id=manifest['id'], manifest_sha256=digest, + reference_id=packet['reference_id'], path=packet['path'], measured_sha256=packet['sha256']), + actor_id=ACTOR, imported_at=INSTANT) + records.append(dict(family=measurements.FAMILY, id=PROJECT + '.' + packet['sha256'], document=doc)) + records.append(operation('measurements.import', f'packet-{index}', doc)) + for batch in range(2): + items = [] + for index in range(2): + path = f'notes/note-{batch}-{index}.md' + raw = f'Owned note {batch}-{index} 山'.encode() + version = dict(revision=1, sha256=sha(raw), bytes=len(raw), text=raw.decode(), + adopted_at=INSTANT, actor_id=ACTOR) + identity = files.file_id(PROJECT, 'note', path) + doc = dict(format=files.FORMAT, project_id=PROJECT, kind='note', path=path, + current=copy.deepcopy(version), history=[version], origin=None) + records.append(dict(family=files.FAMILY, id=identity, document=doc)) + items.append(dict(id=identity, project_id=PROJECT, kind='note', path=path, changed=True, + **{key: version[key] for key in ('revision', 'sha256', 'bytes', 'adopted_at')})) + records.append(operation('files.adopt_many', f'notes-{batch}', dict(items=items, changed_count=2))) + return records + + +class ReceiptValidationTests(unittest.TestCase): + def test_whole_candidate_decodes_each_receipt_owner_once(self): + records = candidate() + before = copy.deepcopy(records) + with patch.object(files, 'decode_records', wraps=files.decode_records) as file_decode, \ + patch.object(projection, 'decode_file_records', file_decode), \ + patch.object(measurements, 'decode_records', wraps=measurements.decode_records) as measurement_decode, \ + patch.object(projection, 'decode_measurement_records', measurement_decode), \ + patch.object(material, 'decode_records', wraps=material.decode_records) as material_decode, \ + patch.object(measurements, 'material_records', material_decode), \ + patch.object(sources, 'material_records', material_decode): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + self.assertEqual(records, before) + for name, actual, expected in [('files', file_decode.call_count, 1), + ('measurements', measurement_decode.call_count, 1), + ('material-owning readers', material_decode.call_count, 3)]: + with self.subTest(owner=name): + self.assertEqual(actual, expected, 'Receipt validation redecoded the same candidate') + + def test_standalone_file_batch_decodes_inventory_once(self): + records = candidate() + result = next(r['document']['result'] for r in records + if r['family'] == 'project_operation' + and r['document']['operation'] == 'files.adopt_many') + with patch.object(files, 'decode_records', wraps=files.decode_records) as decode: + self.assertTrue(files.validate_operation_result('files.adopt_many', result, records)) + self.assertEqual(decode.call_count, 1, 'Each batch item decoded the full file inventory') + + def test_forged_late_file_or_measurement_receipt_is_rejected(self): + for owner in ('files.adopt_many', 'measurements.import', 'source.register'): + records = candidate() + result = [r['document']['result'] for r in records if r['family'] == 'project_operation' + and r['document']['operation'] == owner][-1] + if owner == 'files.adopt_many': + result['items'][-1]['sha256'] = 'e' * 64 + else: + result['name' if owner == 'source.register' else 'title'] = 'Forged retained value' + with self.subTest(owner=owner), self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_unknown_operation_still_has_no_accepting_owner(self): + records = candidate() + records.append(operation('unknown.capture', 'unknown', {})) + with self.assertRaises(WorkbenchError) as error: + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + self.assertEqual(error.exception.code, 'unsupported') + + def test_owned_validators_and_standalone_calls_do_not_share_candidates(self): + for name, module in [('files.adopt_many', files), ('measurements.import', measurements), + ('source.register', sources)]: + records = candidate() + original_result = copy.deepcopy(next(r['document']['result'] for r in records + if r['family'] == 'project_operation' and r['document']['operation'] == name)) + checker = module.receipt_validator(records) + self.assertTrue(checker(name, original_result)) + self.assertTrue(module.validate_operation_result(name, original_result, records)) + if module is files: + doc = next(r['document'] for r in records if r['family'] == files.FAMILY) + value = doc['history'][0] + value.update(text='Edited candidate', sha256=sha(b'Edited candidate'), bytes=16) + doc['current'] = copy.deepcopy(value) + elif module is measurements: + next(r['document'] for r in records if r['family'] == measurements.FAMILY)['title'] = 'Edited candidate' + else: + value = next(r['document']['value'] for r in records if r['family'] == 'project_snapshot') + value['name'] = value['manifest']['name'] = 'Edited candidate' + with self.subTest(owner=name): + self.assertTrue(checker(name, original_result), 'Owned candidate changed with caller mutation') + with self.assertRaises(WorkbenchError): + module.receipt_validator(records)(name, original_result) + with self.assertRaises(WorkbenchError): + module.validate_operation_result(name, original_result, records) + + def test_standalone_unrecognized_and_malformed_results_do_not_decode(self): + for name, module in [('files.adopt_many', files), ('measurements.import', measurements), + ('source.register', sources)]: + records = candidate() + attribute = 'material_records' if module is sources else 'decode_records' + with self.subTest(owner=name), patch.object(module, attribute, + side_effect=AssertionError('Result shape must precede candidate decoding')) as decode: + self.assertFalse(module.validate_operation_result('unknown.capture', {}, records)) + with self.assertRaises(WorkbenchError): + module.validate_operation_result(name, {}, records) + decode.assert_not_called() + + def test_unused_malformed_file_and_missing_packet_reference_still_reject(self): + records = candidate() + next(r['document']['history'][0] for r in records if r['family'] == files.FAMILY)['text'] = 'Unhashed' + records = [r for r in records if r['family'] != 'project_operation' + or r['document']['operation'] != 'files.adopt_many'] + with self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + records = candidate() + next(r['document'] for r in records if r['family'] == measurements.FAMILY)['source']['reference_id'] = 'absent' + with self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_each_new_measurement_validator_checks_actual_packet_bytes(self): + records = candidate() + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / 'project' + with ProjectStore.create(root, 'Packet custody fixture', 'create', 'a' * 64) as store: + with ProjectArtifacts(store) as artifacts: + docs = [r['document'] for r in records if r['family'] == measurements.FAMILY] + for index, doc in enumerate(docs): + packet = fixture() + packet['title'] = f'Packet {index}' + raw = json.dumps(packet).encode() + self.assertEqual(sha(raw), doc['id']) + artifacts.publish_bytes(raw, measurements.declaration(doc['blob'])) + checker = measurements.receipt_validator(records, artifacts=artifacts) + self.assertTrue(checker('measurements.import', docs[0])) + payload = root / 'artifacts/sha256' / docs[0]['id'] + raw = payload.read_bytes() + payload.chmod(0o600) + payload.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + with self.assertRaises((WorkbenchError, ProjectArtifactError)): + measurements.receipt_validator(records, artifacts=artifacts) diff --git a/tests/test_project_session.py b/tests/test_project_session.py index f9356c44..75ff7795 100644 --- a/tests/test_project_session.py +++ b/tests/test_project_session.py @@ -14,6 +14,7 @@ from unittest.mock import patch import uuid +import project_fork as fork_module import psycopg from psycopg import sql @@ -27,6 +28,24 @@ class OperationCodecTests(unittest.TestCase): + def test_operation_lookup_does_not_copy_unrelated_active_envelopes(self): + key='selected-receipt' + document=dict(format='project_operation_v1',operation='fixture.capture',target=None, + request_key=key,input_sha256='a'*64,result={}) + receipt=dict(family='project_operation',id=hashlib.sha256(key.encode()).hexdigest(),document=document) + unrelated=dict(family='project_artifact',id='b'*64, + document=dict(format='project_artifact_v1',sha256='b'*64,bytes=4)) + copied=[];deepcopy=copy.deepcopy + def observe(value): + if value is unrelated:copied.append((value['family'],value['id'])) + return deepcopy(value) + with patch.object(fork_module,'copy',SimpleNamespace(deepcopy=observe)): + actual=operation_records([unrelated,receipt]) + self.assertEqual(actual,{key:document}) + self.assertEqual(copied,[],'Receipt selection must not clone unrelated active records') + actual[key]['result']['changed']=True + self.assertEqual(document['result'],{}) + def test_namespaced_receipts_preserve_legacy_identity_and_independent_keys(self): key = 'independent-edit' document = dict(format='project_operation_v1', operation='fixture.capture', target=None, @@ -155,6 +174,23 @@ def test_complete_merge_and_original_result_replay_before_stale_cas(self): self.assertIn(('project_artifact', other['sha256']), ids) self.assertEqual(self.artifacts.read_blob({key: declaration[key] for key in ('sha256', 'bytes')}), 'Irreplaceable observation 山'.encode()) + def test_scalar_summary_is_independent_and_tracks_the_bound_publication(self): + before = self.session.summary + arguments, _declaration, _calls = self.mutation() + self.call(arguments) + snapshot = self.session.snapshot + expected = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} + self.assertNotEqual(before['checkpoint'], expected['checkpoint']) + original_copy = copy.deepcopy + def observe(value): + self.assertIsNot(value, self.session._snapshot, 'Summary must not clone the record graph') + return original_copy(value) + with patch('project_session.copy', SimpleNamespace(deepcopy=observe)): + first = self.session.summary + first['name'] = 'Caller mutation' + self.assertEqual(self.session.summary, expected) + self.assertEqual(self.session.snapshot['name'], expected['name']) + def test_key_conflict_stale_and_current_auth_refuse_before_invoke(self): first, _declaration, _calls = self.mutation() self.call(first) diff --git a/tests/test_project_session_freshness.py b/tests/test_project_session_freshness.py new file mode 100644 index 00000000..8ab5634e --- /dev/null +++ b/tests/test_project_session_freshness.py @@ -0,0 +1,136 @@ +"""Freshness enforcement over real canonical files, without a SQL projection. + +Only the session's read guard is exercised here. Publication and projection +binding still require the separate real-PostgreSQL session/workspace tests. +""" +import copy +import hashlib +import json +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from project_session import ProjectSession, ProjectSessionError +from project_store import ProjectStore, ProjectStoreError + + +class ProjectSessionFreshnessTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-freshness-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / 'project' + self.store = ProjectStore.create(self.root, 'Freshness fixture', 'new', 'a' * 64) + self.addCleanup(self.store.close) + self.records = [dict(family='workbench_text', id=name, + document=dict(format='workbench_text_v1', text='Alpha ' + name)) for name in ('one', 'two')] + snapshot = self.store.commit(self.records, self.store.summary()['checkpoint'], 'seed', 'b' * 64) + # Isolate the existing read guard; no fake SQL/projection acceptance. + self.session = object.__new__(ProjectSession) + self.session.store = self.store + self.session._snapshot = copy.deepcopy(snapshot) + self.session.available = True + + def test_freshness_reads_all_records_without_copying_an_unused_graph(self): + before = self.store.snapshot() + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused full snapshot')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: + self.assertIsNone(self.session.check_current()) + self.assertIsNone(self.session.check_current()) + self.assertEqual(reads.call_count, 4) + self.assertEqual(self.store.snapshot(), before) + + def test_changed_canonical_checkpoint_refuses_both_read_contracts(self): + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Independently published content' + self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c' * 64) + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): + with self.assertRaises(ProjectSessionError) as caught: + check() + self.assertEqual(caught.exception.code, 'stale_project') + + def test_unavailable_projection_refuses_before_reading_the_store(self): + self.session.available = False + with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')), \ + patch.object(self.store, 'snapshot', side_effect=AssertionError('Unavailable snapshot read')): + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): + with self.assertRaises(ProjectSessionError) as caught: + check() + self.assertEqual(caught.exception.code, 'projection_unavailable') + + def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self): + self.session.check_current() + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + path = self.root / '.tuldok' / manifest['inventory'][0]['path'] + info, original = path.stat(), path.read_bytes() + changed = original.replace(b'Alpha', b'Omega') + self.assertEqual(len(original), len(changed)) + self.assertNotEqual(hashlib.sha256(original).digest(), hashlib.sha256(changed).digest()) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): + with self.assertRaises(ProjectStoreError) as caught: + check() + self.assertEqual(caught.exception.code, 'stale') + + def test_record_consumers_keep_independent_complete_snapshots(self): + current = self.session.check() + current['records'][0]['document']['text'] = 'Caller mutation' + current['name'] = 'Caller renamed copy' + self.assertEqual(self.session.check(), self.store.snapshot()) + self.assertEqual(self.session._snapshot, self.store.snapshot()) + self.assertNotEqual(current, self.session.check()) + + def add_unrelated_record(self): + unrelated = dict(family='project_artifact', id='d' * 64, + document=dict(format='project_artifact_v1', sha256='d' * 64, bytes=4)) + self.session._snapshot = self.store.commit([*self.records, unrelated], + self.store.summary()['checkpoint'], 'unrelated', 'e' * 64) + + def test_selected_records_are_exact_independent_and_do_not_copy_other_families(self): + self.add_unrelated_record() + expected = {key: value for key, value in self.store.snapshot().items() if key != 'records'} + expected['records'] = self.records + copied = [] + deepcopy = copy.deepcopy + def observe(value): + self.assertEqual([row['family'] for row in value], ['workbench_text', 'workbench_text']) + copied.append(value) + return deepcopy(value) + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unselected graph copy')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ + patch('project_session.copy', SimpleNamespace(deepcopy=observe)): + selected = self.session.check_records('workbench_text') + self.assertEqual(selected, expected) + self.assertEqual(reads.call_count, 3, 'Unselected actual records must still be read and hashed') + self.assertEqual(len(copied), 1) + selected['records'][0]['document']['text'] = 'Caller mutation' + self.assertEqual(self.session.check_records('workbench_text'), expected) + self.assertEqual(self.session.check_records('project_file')['records'], []) + + def test_selected_read_rejects_corruption_in_an_unselected_family(self): + self.add_unrelated_record() + self.session.check_records('workbench_text') + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + entry = next(row for row in manifest['inventory'] if row['family'] == 'project_artifact') + path = self.root / '.tuldok' / entry['path'] + info, original = path.stat(), path.read_bytes() + changed = original.replace(b'"bytes": 4', b'"bytes": 5') + self.assertNotEqual(changed, original) + self.assertEqual(len(changed), len(original)) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + with self.assertRaises(ProjectStoreError) as caught: + self.session.check_records('workbench_text') + self.assertEqual(caught.exception.code, 'stale') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 9f321617..676ab3bc 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -3,10 +3,12 @@ import errno import hashlib import json +import math import os from pathlib import Path import shutil import subprocess +import struct import sys import tempfile import unittest @@ -71,6 +73,25 @@ def write_manifest(self, value): value['checkpoint'] = sha(canonical(value)) (self.root / '.tuldok/project.json').write_bytes(canonical(value)) + def test_snapshot_preserves_scalar_types_float_bits_and_owned_nested_values(self): + floats = [-0.0, 0.0, 5e-324, -5e-324, 1.7976931348623157e308, + math.nextafter(1.0, 0.0), math.nextafter(1.0, math.inf)] + value = record() + value['document']['scalars'] = [None, False, True, -2**63, 2**63-1, + 'Unicode 漢字 \U0001f30a\x00', *floats] + expected = self.commit([value]) + first, second = self.store.snapshot(), self.store.snapshot() + self.assertEqual(first, expected) + self.assertEqual(second, expected) + actual = first['records'][0]['document']['scalars'] + self.assertEqual([type(v) for v in actual], + [type(v) for v in value['document']['scalars']]) + self.assertEqual([struct.pack('>d', v) for v in actual[6:]], + [struct.pack('>d', v) for v in floats]) + first['records'][0]['document']['history'][0]['reviewed'] = True + first['records'][0]['document']['scalars'].clear() + self.assertEqual(self.store.snapshot(), second) + def test_create_commit_save_cold_reopen_and_independent_snapshots(self): initial = self.store.snapshot() self.assertEqual(initial['records'], []) @@ -97,6 +118,50 @@ def test_appending_one_record_does_not_rewrite_saved_immutable_revisions(self): 'Appending one record must not write disposable copies of every saved immutable revision') self.assertEqual(self.reopen(), committed) + def test_summary_reads_every_current_record_without_cloning_the_record_graph(self): + committed = self.commit([record('a'), record('b')], key='summary-fixture') + expected = {key: committed[key] for key in ('project_id', 'name', 'checkpoint')} + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused record graph copy')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: + first = self.store.summary() + first['name'] = 'Caller mutation' + second = self.store.summary() + self.assertEqual(second, expected) + self.assertEqual(reads.call_count, 4, 'Each summary must still read/hash both actual records') + self.assertEqual(self.store.snapshot(), committed) + + def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_identical_records(self): + rows=[record('a'),record('b','Exact second note\n')] + self.commit(rows,key='repeat-read-fixture') + expected=self.store.snapshot() + canonical_documents={canonical(row['document']) for row in rows} + with patch.object(store_module,'_decode',wraps=store_module._decode) as decode, \ + patch.object(self.store,'_record_file',wraps=self.store._record_file) as read, \ + patch.object(self.store,'_snapshot',wraps=self.store._snapshot) as snapshot: + self.assertEqual(self.store.snapshot(),expected) + with self.store.root_directory(verify_after=True) as descriptor: + self.assertTrue(os.fstat(descriptor).st_ino) + self.assertEqual(self.store.snapshot(),expected) + self.assertEqual(read.call_count,8,'All two records must still be read in each of four checks') + self.assertEqual(snapshot.call_count,2,'Root borrowing must not clone an unused public snapshot') + decoded=[call for call in decode.call_args_list if call.args[0] in canonical_documents] + self.assertEqual(len(decoded),0,'Byte-identical canonical records need no repeat decoding within one Store') + + def test_verified_decode_cache_still_rejects_same_length_corruption_with_restored_mtime(self): + self.commit([record('note','Alpha\n')],key='cache-corruption-fixture') + self.store.snapshot() + entry=self.manifest()['inventory'][0] + path=self.root/'.tuldok'/entry['path'];info=path.stat();raw=path.read_bytes() + changed=raw.replace(b'Alpha',b'Omega') + self.assertEqual(len(changed),len(raw));self.assertNotEqual(changed,raw) + path.chmod(0o600);path.write_bytes(changed) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + self.assert_code('stale',self.store.snapshot) + self.assert_code('stale',self.store.summary) + with self.assertRaises(ProjectStoreError) as caught: + with self.store.root_directory(verify_after=True):pass + self.assertEqual(caught.exception.code,'stale') + def test_reusing_an_unreferenced_revision_still_checks_bytes_and_syncs_it(self): first, second = record('first'), record('second') self.commit([first], key='retain-first') @@ -219,12 +284,14 @@ def test_replaced_physical_root_or_metadata_directory_is_rejected(self): self.root.rename(displaced) self.root.mkdir() self.assert_code('stale', self.store.snapshot) + self.assert_code('stale', self.store.summary) self.root.rmdir() displaced.rename(self.root) meta = self.root / '.tuldok' meta.rename(self.root / '.old-tuldok') meta.mkdir() self.assert_code('stale', self.store.snapshot) + self.assert_code('stale', self.store.summary) def test_changed_checkpoint_is_rejected_before_mutation(self): initial = self.store.snapshot() diff --git a/tests/test_project_text_tag.py b/tests/test_project_text_tag.py new file mode 100644 index 00000000..6a944aa6 --- /dev/null +++ b/tests/test_project_text_tag.py @@ -0,0 +1,52 @@ +"""The scalar text response tag must not copy unrelated project metadata.""" +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import project_fork + +from project_projection import _TextView + + +class TextTagTests(unittest.TestCase): + def test_text_records_does_not_clone_unrelated_active_documents(self): + unrelated = dict(family='project_artifact', id='c' * 64, + document=dict(format='project_artifact_v1', sha256='c' * 64, bytes=4)) + note = dict(family='workbench_text', id='d' * 32, + document=dict(current=dict(kind='text', text='Unique note 山'))) + summary = dict(project_id='a' * 32, checkpoint='b' * 64) + session = SimpleNamespace(summary=summary, + snapshot=dict(summary, records=[unrelated, note]), + check_current=lambda: None) + copied = [] + original = project_fork.copy.deepcopy + + def observe(value): + if value is unrelated: + copied.append(value['id']) + return original(value) + + with patch.object(project_fork, 'copy', SimpleNamespace(deepcopy=observe)): + result = _TextView(SimpleNamespace(session=session)).records() + self.assertEqual(result, dict(summary, items=[dict(kind='text', text='Unique note 山')])) + self.assertEqual(copied, [], 'Text list cloned an unrelated active document') + result['items'][0]['text'] = 'Caller mutation' + self.assertEqual(note['document']['current']['text'], 'Unique note 山') + + def test_text_tag_reads_scalar_summary_without_a_record_snapshot(self): + class Session: + summary = dict(project_id='a' * 32, name='Owned project', checkpoint='b' * 64) + + @property + def snapshot(self): + raise AssertionError('Text response tag copied the full record graph') + + session = Session() + view = _TextView(SimpleNamespace(session=session)) + original = dict(items=[dict(text='Authored note 漢字')]) + result = view._tag(original) + self.assertEqual(result, dict(original, project_id='a' * 32, checkpoint='b' * 64)) + self.assertNotIn('name', result) + self.assertEqual(original, dict(items=[dict(text='Authored note 漢字')])) + result['project_id'] = 'caller edit' + self.assertEqual(session.summary['project_id'], 'a' * 32) diff --git a/tests/test_project_transfer.py b/tests/test_project_transfer.py index efb639b4..5b798960 100644 --- a/tests/test_project_transfer.py +++ b/tests/test_project_transfer.py @@ -433,7 +433,7 @@ def capture(_,roots): from project_material import ProjectMaterial with ProjectStore.open(self.root/'science') as store,ProjectArtifacts(store) as artifacts: snapshot=store.snapshot();records=snapshot['records'] - view=SimpleNamespace(snapshot=snapshot,artifacts=artifacts,check=lambda:None) + view=SimpleNamespace(snapshot=snapshot,artifacts=artifacts,check_current=lambda:None) facts=history(records,materials=ProjectMaterial(view),artifacts=artifacts) self.assertEqual(len(facts.studies),2);self.assertEqual(len(facts.trials),2) self.assertEqual(len(facts.candidates),1);self.assertEqual(len(facts.reservations),1);self.assertEqual(len(facts.runs),1) diff --git a/tests/test_project_workspace.py b/tests/test_project_workspace.py index 0c2dbaee..a78982f4 100644 --- a/tests/test_project_workspace.py +++ b/tests/test_project_workspace.py @@ -72,6 +72,17 @@ def create(self, directory='alpha'): arguments = self.mutation(root='fixture', directory=directory, name='Workspace authored project') return self.call('new', arguments), arguments + def test_status_returns_current_metadata_without_copying_the_record_graph(self): + self.create() + self.import_record('A retained record unrelated to the project summary') + expected = self.state() + with patch.object(self.workspace._store, '_snapshot', + wraps=self.workspace._store._snapshot) as snapshots: + observed = self.state() + self.assertEqual(observed, expected) + self.assertEqual(snapshots.call_count, 0, + 'Status must not copy every retained record to return three metadata fields') + def test_control_project_ancestry_is_rejected_before_destination_or_projection_creation(self): with ProjectStore.create(self.external/'science','Existing portable project','existing-project','b'*64):pass self.start(self.external/'science/control') diff --git a/tests/test_workspace_project_files.cjs b/tests/test_workspace_project_files.cjs index 31339b01..86582638 100644 --- a/tests/test_workspace_project_files.cjs +++ b/tests/test_workspace_project_files.cjs @@ -38,6 +38,7 @@ function mount(initial=[reply(state()),reply(records())],capabilityValue=capabil context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); context.ProjectMeasurements=()=>({setContext(){},clear(){}}); + context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,events,discoveries,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, diff --git a/tests/test_workspace_project_files_m4.cjs b/tests/test_workspace_project_files_m4.cjs index dbdb85b7..80e343ca 100644 --- a/tests/test_workspace_project_files_m4.cjs +++ b/tests/test_workspace_project_files_m4.cjs @@ -43,6 +43,7 @@ function mount(capabilityValue=capabilities){ context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); context.ProjectMeasurements=()=>({setContext(){},clear(){}}); + context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, diff --git a/tuldok_client.py b/tuldok_client.py index be0daf7f..fdaef8d8 100644 --- a/tuldok_client.py +++ b/tuldok_client.py @@ -706,6 +706,11 @@ def save_as(self, *, root, directory, name, binding, expected_checkpoint, reques return self._call('save_as',dict(root=root,directory=directory,name=name,binding=binding, expected_checkpoint=expected_checkpoint,request_key=request_key)) + def compose(self, *, root, directory, name, sources, binding, expected_checkpoint, request_key): + """Compose explicit retained checkpoints through the negotiated owner.""" + return self._call('compose',dict(root=root,directory=directory,name=name,sources=sources, + binding=binding,expected_checkpoint=expected_checkpoint,request_key=request_key)) + def files_inspect(self, **arguments): return self._call('files_inspect',arguments)