diff --git a/docs/plans/database-working-store.md b/docs/plans/database-working-store.md new file mode 100644 index 00000000..2def6cce --- /dev/null +++ b/docs/plans/database-working-store.md @@ -0,0 +1,94 @@ +# Database working store for external projects + +Status: stages 1–2 implemented; stage 3 revised (2026-11-02). Owner decision recorded with the user. + +## Decision + +The local PostgreSQL database is the persistent **working copy** of an opened +project and is kept across sessions. The external project folder +(`.tuldok/project.json` plus content-addressed `records/`, tracked in Git) is +only: + +- an **import source**, read when its files changed since Tuldok last imported + or exported them, or when the project is not yet in this workspace; and +- an **export target**, written only by an explicit Save. + +Edits during a session are committed to the database and stay there. Closing a +project does not write the external folder. Complete domain validation runs +only when the record set changes (import of a changed folder, or a commit). + +There is exactly one durable copy of the record set (the database) and no +complete in-memory copy: owners read the record families or records they need. + +## Why + +Opening the 54k-record, 1.77 GB Rheon portfolio rebuilt a fresh SQL projection +from every JSON file on each Open (9+ minutes, 17–21 GB RAM: the Store, the +Session and owner copies each held the whole record set). JSON exists only so +the project can be tracked in Git. + +## Identity and integrity + +- Checkpoint identity is unchanged: the working copy keeps the same manifest + (inventory of `family/id/format/bytes/sha256`, roots, receipts) and the same + checkpoint digest as the file format, so Save writes byte-identical files and + existing receipts, validation markers and custody records stay meaningful. +- Each record is stored once as its exact canonical bytes (not `jsonb`, which + would change number spelling such as `-0.0`) with its SHA256. +- Import verifies `project.json`, then only reads, SHA256-checks and + canonical-checks records whose digest the working copy does not already hold. +- Save re-checks that the external `project.json` is still the one last + imported/exported (refusing to overwrite external changes), writes only + missing record files, then publishes `project.json` atomically. + +## Open + +1. Read external `project.json` (one file) and its SHA256. +2. Working copy exists and its last imported/exported manifest hash matches: + attach. No record file is read; the SQL projection is reused. +3. External folder changed, working copy has no unsaved changes: incremental + import, validate (cached per checkpoint), rebuild projection tables. +4. External folder changed **and** working copy has unsaved changes: refuse with + an explicit conflict (keep working copy and overwrite on Save, or discard and + re-import). Never merge silently. + +## Stages + +1. **Working store — done.** `WorkingProjectStore` (project_working_store.py) + with the ProjectStore interface, incremental import, commits joined to the + projection's open SQL transaction, Save as the only folder write. Workspace + keeps a private folder→working-copy registry. +2. **No whole-project memory copy — done.** Sessions over a working copy bind + checkpoint metadata only; family/record/identity reads come from + PostgreSQL, commit hooks never materialize the record set, Study authoring + restores from Study/Trial documents and Run history loads per Run. +3. **Projection reuse — revised.** Reusing the SQL projection across sessions + would carry live execution state (runs, admissions, execution grants) into + a later session, which a cold open deliberately starts without. Instead the + projection is still reconstructed fresh, from cheap family-scoped reads. +4. **Edit-scoped validation — done.** `project_validation_delta` declares each + validation component's input families (from a per-component audit); an + edit re-runs only components whose inputs it changed and only the old + receipts bound to changed records or sharing a key space. Identity, claim, + fork and receipt-identity checks always run; unclassified operations are + always validated. `TULDOK_VALIDATION_CROSSCHECK=1` runs complete validation + alongside and fails on disagreement (the project suite passes in that mode). + A text import on the portfolio: 161 s → ~31 s; validation itself 1–3 s. The + remainder is the receipt digest over every record's bytes (format-defined) + and the full record read for owner captures, both dominated by stage 5. +5. **Smaller project files — open.** Store each captured runtime file listing + once and reference it from Runs (format migration with exact round trip). + +## Measured on the Rheon portfolio (54k records, 1.77 GB metadata) + +| | before | reopen now | +|---|---|---| +| Open | ~9 min (CPU) | ~11 s | +| Peak memory | 21.7 GB | 0.76 GB | +| First history/progression read | ~100 s | 0.5 s | + +First import into a new workspace takes ~50 s plus one complete validation +(~2.5 min) per checkpoint and validator version. + +Payload blobs (`artifacts/sha256/*`, 16 GB for the portfolio) remain +content-addressed files under the project folder in stages 1–4. diff --git a/model_definitions.py b/model_definitions.py index c69c0281..ab861b52 100644 --- a/model_definitions.py +++ b/model_definitions.py @@ -33,9 +33,13 @@ def natural(value, name, minimum=0, maximum=MAX_INTEGER): return value +# Same code points as any(ord(c)<32 or ord(c)==127 or 0xD800<=ord(c)<=0xDFFF), scanned in C. +_TEXT_FORBIDDEN=re.compile('[\x00-\x1f\x7f\ud800-\udfff]') + + def text(value, name, maximum=120): if (type(value) is not str or not value.strip() or len(value)>maximum or - any(ord(c)<32 or ord(c)==127 or 0xD800<=ord(c)<=0xDFFF for c in value)): + _TEXT_FORBIDDEN.search(value)): raise WorkbenchError(name+' must be bounded nonempty text without controls.') return value diff --git a/project_composition.py b/project_composition.py index 53bc04f3..ee667167 100644 --- a/project_composition.py +++ b/project_composition.py @@ -7,6 +7,8 @@ The lifecycle owner must validate each source and the complete final candidate, clone primary edits and publish all required bytes through the native owners. """ +import contextvars +from contextlib import contextmanager import hashlib from project_fork import ARCHIVE_FAMILIES, _sha, _uuid, decode_fork @@ -23,7 +25,30 @@ def _fail(message, code='invalid', status=400): raise ProjectCompositionError(message, code, status) +_KNOWN_ENVELOPES = contextvars.ContextVar('known_envelope_digests', default=None) + + +@contextmanager +def known_envelope_digests(known): + """Supply verified envelope digests for exact record objects during validation. + + known maps id(record) -> (record, digest) for envelopes whose document is the + stored canonical revision. Only that same object uses its digest; any other + envelope (for example a constructed observation variant) is encoded. + """ + token = _KNOWN_ENVELOPES.set(known) + try: + yield + finally: + _KNOWN_ENVELOPES.reset(token) + + def _record_sha(record): + known = _KNOWN_ENVELOPES.get() + if known is not None: + entry = known.get(id(record)) + if entry is not None and entry[0] is record: + return entry[1] return hashlib.sha256(_encode(record)).hexdigest() diff --git a/project_data.py b/project_data.py index ca70bab0..1b58eae6 100644 --- a/project_data.py +++ b/project_data.py @@ -20,6 +20,7 @@ from saved_searches import SavedSearches, criteria from saved_selections import SavedSelections from dataset_releases import Releases +from project_session import session_active, session_records, session_records_for FAMILIES = { 'workbench_record': 'workbench_record_v1', @@ -971,11 +972,10 @@ def capture_runtime_changes(self, before_records, *, operation, key): return captured def _sync_sources(self): - records = self.session.snapshot['records'] - from project_fork import active_records - active = active_records(records, project_id=self.session.project_id) - self.context.project_sources = {r['id']: copy.deepcopy(r['document']['source']) for r in active if r['family'] == 'workbench_record'} - self._declarations = {r['id']: r['document'] for r in records if r['family'] == 'project_artifact'} + # Only the selected families are copied; fork history is still checked. + active = session_active(self.session, 'workbench_record') + self.context.project_sources = {r['id']: r['document']['source'] for r in active if r['family'] == 'workbench_record'} + self._declarations = {r['id']: r['document'] for r in session_records(self.session, 'project_artifact')} def _available(self, record_id): source = self.context.project_sources.get(record_id) @@ -1005,9 +1005,8 @@ def _publish(self, raw, kind, role, media_type): media_type=media_type, kind=kind, role=role, preservation='unique', required_for=['inspect', 'release']) def _require_record(self, record_id, *, extended=False): - from project_fork import active_records - snapshot = self.session.snapshot - for item in active_records(snapshot['records'], project_id=snapshot['project_id']): + # Only these families are inspected; fork history is still checked. + for item in session_active(self.session, 'workbench_record', 'workbench_text'): if item['id'] == record_id and item['family'] in ('workbench_record', 'workbench_text'): if extended and item['family'] == 'workbench_text': fail('This operation requires an explicitly converted data record.', 'unsupported', 409) @@ -1047,9 +1046,8 @@ def _mutate(self, operation, target, body, expected_checkpoint, key, invoke): def _legacy_replay(self, operation, target, body, key): """Retain M1 request meaning after the shared M2 session cutover.""" - records = self.session.snapshot['records'] - from project_fork import active_records - records = active_records(records, project_id=self.session.project_id) + # Only receipts and legacy text are inspected below. + records = session_active(self.session, 'project_operation', 'workbench_text') if any(r['family']=='project_operation' and r['document']['request_key']==key for r in records): return None if operation not in ('data.import_text','data.annotate'): @@ -1291,7 +1289,8 @@ def _capture(self, result, before_records, operation, key): for family, table, owner in (('saved_selection', 'saved_selections', self.selections), ('saved_search', 'saved_searches', self.searches)): live = {r[0] for r in self.db.execute('SELECT id FROM ' + table)} from project_fork import active_records - all_ids = live | {r['id'] for r in active_records(before_records, project_id=self.session.project_id) if r['family'] == family} + all_ids = live | {r['id'] for r in active_records(before_records, project_id=self.session.project_id, + families=frozenset({family})) if r['family'] == family} for record_id in all_ids: old = before.get((family, record_id)) current = owner._get(record_id) if record_id in live else copy.deepcopy(old['document']['current']) @@ -1333,16 +1332,18 @@ def _capture(self, result, before_records, operation, key): return {'records': list(unique.values()), 'result': copy.deepcopy(result)} def restore(self, snapshot=None): - snapshot = snapshot or self.session.snapshot - if snapshot != self.session.snapshot: + if snapshot is not None and snapshot != self.session.snapshot: fail('Data reconstruction snapshot is stale.', 'stale_project', 409) - owned = decode_data_records(snapshot['records']) + # The decoder reads its own families and project_fork overlays, and its + # fork validation needs every record identity; unrelated documents are + # shared read-only instead of copying unrelated Run history. + families = (*FAMILIES, 'workbench_text', 'project_artifact', 'project_fork') + owned = decode_data_records(session_records_for(self.session, *families)) self._sync_sources() # Validate present source bytes through the existing semantic owners # before touching any selected SQL row. Missing bytes retain metadata. self._verify_sources(owned) - from project_fork import active_records - active = active_records(snapshot['records'], project_id=self.session.project_id) + active = session_active(self.session, *families) identities = {(r['family'], r['id']) for r in active} owned = [r for r in owned if (r['family'], r['id']) in identities] with self.db.lock, self.db.transaction(): diff --git a/project_dispatch.py b/project_dispatch.py index de519135..e517c2e6 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -9,6 +9,7 @@ import copy from workbench import WorkbenchError +from project_session import session_records _BOUND_FIELDS = frozenset(('binding', 'checkpoint', 'expected_checkpoint', 'request_key')) @@ -61,8 +62,7 @@ def _origin_items(items, arguments): def _declarations(projection): - return {item['id']: item['document'] for item in projection.session.snapshot['records'] - if item['family'] == 'project_artifact'} + return {item['id']: item['document'] for item in session_records(projection.session, 'project_artifact')} def _artifact(projection, sha): diff --git a/project_execution.py b/project_execution.py index 0e4941c6..9f7a5878 100644 --- a/project_execution.py +++ b/project_execution.py @@ -21,6 +21,7 @@ from project_history import capture_run, declarations from project_session import operation_records from workbench import WorkbenchError +from project_session import session_operations OPERATIONS = frozenset(('runs.submit_external', 'runs.submit', 'runs.retry', 'runs.cancel', 'runs.approve_external')) @@ -527,11 +528,26 @@ def capture_dirty(self, before_records): merged[identity] = record return list(merged.values()) + def _capture_pending(self): + """Whether capture_dirty could return any record (see its owners).""" + if self._dirty_runs or self._pending_studies or any(self._pending_producers.values()) or self.native.has_dirty: + return True + study = self.study_execution + if study is not None and (study._dirty or study._package_records): + return True + producers = self.producer_execution + if producers is not None and (producers._runtime_data is not None or any(producers._dirty.values())): + return True + return False + def before_commit(self): if self._reconstructing or self.session.foreground_publication: return if not self._dirty_runs and not self._pending_studies and not any(self._pending_producers.values()) and not self.native.has_dirty and self.study_execution is None and self.producer_execution is None: return + if not self._capture_pending(): + # Nothing can be captured; never materialize the record set. + return snapshot = self.session.snapshot records = self.capture_dirty(snapshot['records']) if records: @@ -600,7 +616,7 @@ def native_preflight(self,graph_id,revision,expected_checkpoint,key): graph_id,revision,expected_checkpoint,key) def _native_replay(self,operation,target,body,expected_checkpoint,key): - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is None:return None if (prior['operation']!=operation or self.db.execute('SELECT id FROM workspace_runs WHERE id=%s', (prior['result'].get('run_id'),)).fetchone() is None): @@ -656,7 +672,7 @@ def submit_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require_external_submission() with self.access.lock, self.session.lock, self.db.lock: self.session.check_current() - prior = operation_records(self.session.snapshot['records']).get(key) + prior = session_operations(self.session).get(key) if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: @@ -679,7 +695,7 @@ def approve_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require('approve-execution', 'runs.approve_external', owner_only=True) with self.access.lock, self.session.lock, self.db.lock: self.session.check_current() - prior = operation_records(self.session.snapshot['records']).get(key) + prior = session_operations(self.session).get(key) if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: diff --git a/project_history.py b/project_history.py index b69eccce..e3b1c7c7 100644 --- a/project_history.py +++ b/project_history.py @@ -5,6 +5,7 @@ SQL connection is only a query engine for bounded supplied immutable JSON. """ import base64 +from collections.abc import Mapping import copy import hashlib import re @@ -25,6 +26,7 @@ from runs import (ACTIVE, TERMINAL, TEST_FILES, MAX_ARTIFACT_BYTES, validate_retained_run, project_run_projection, public_run_projection, artifact_capture_source, validate_external_artifact_inventory) from workbench import WorkbenchError, timestamp +from project_session import session_derived, session_operations, session_record, session_records, session_records_for FAMILY_FORMATS = { 'study_history': 'project_study_history_v1', @@ -562,6 +564,100 @@ def visit_run(key,path): return facts +def assemble_validated_facts(records, *, owned=False): + """Collect HistoryFacts from records whose complete validation is on record. + + decode_records validates and collects exactly these values: each history + document (as a checked JSON copy), Run observations, and the policy and + reservation indexes taken from Study documents. Only for a checkpoint that + already passed complete domain validation (which runs decode_records over + these same records) may the collection be repeated without re-running the + validators. tests/test_project_history.py and the portfolio equivalence + check compare this against decode_records field by field. owned=True means + the caller supplies independent copies the facts may keep without recopying. + """ + facts=HistoryFacts() + for record in records: + family=record.get('family') + if family not in FAMILY_FORMATS:continue + if family=='run_observation': + doc=_observation_document(record) + facts.observations[(doc['origin_catalog_id'],doc['run_id'],doc['sha256'])]=doc + continue + doc=record['document'] if owned else checked_json(record['document'],MAX_HISTORY_BYTES) + origin=doc['origin_catalog_id'] + if family=='study_history':target,identity=facts.studies,doc['study']['id'] + elif family=='trial_history':target,identity=facts.trials,doc['trial']['id'] + elif family=='run_history':target,identity=facts.runs,doc['run']['id'] + else:target,identity=facts.candidates,doc['candidate']['id'] + target[(origin,identity)]=doc + for (origin,_identity),doc in facts.studies.items(): + for item in doc['policies']: + ref=item['ref'];facts.policies[(origin,ref['id'],ref['revision'],ref['sha256'])]=item['policy'] + for reservation in doc['reservations']: + facts.reservations[(origin,reservation.get('run_id'))]=reservation + return facts + + +class _LazyRuns(Mapping): + """Validated Run history documents by (origin, run_id), loaded on first use. + + Keys come from verified record identities (origin + '.' + run_id); each + document is read from the working copy once per checkpoint and shared like + every other fact (callers copy what they return). + """ + def __init__(self, identities, load): + self._identities = identities + self._load = load + self._documents = {} + + def __getitem__(self, key): + if key not in self._identities: + raise KeyError(key) + if key not in self._documents: + envelope = self._load(self._identities[key]) + if envelope is None: + _invalid('Historical Run is unavailable.', 'unavailable') + self._documents[key] = envelope['document'] + return self._documents[key] + + def __iter__(self): + return iter(self._identities) + + def __len__(self): + return len(self._identities) + + +def checkpoint_facts(session, materials): + """Decoded history of the session's bound checkpoint, shared by its owners. + + Decoded once per checkpoint; callers copy any part they return. When the + session records that this exact checkpoint passed complete validation, + the facts are collected without repeating the validators. + """ + def compute(): + validated=getattr(session,'validated_checkpoint',None) + if validated is not None and validated==session.summary['checkpoint'] and getattr(session,'_lazy',False): + # A working copy serves Run documents on demand: decode everything + # except Run history now, and keep only Run identities. + selected=('study_history','trial_history','run_observation','candidate_history') + view=session_records_for(session,*selected) + facts=assemble_validated_facts([r for r in view if r['family'] in selected],owned=True) + identities={} + for envelope in view: + if envelope['family']=='run_history': + origin,run_id=envelope['id'].split('.',1) + identities[(origin,run_id)]=envelope['id'] + facts.runs=_LazyRuns(identities,lambda record_id:session_record(session,'run_history',record_id)) + return facts + if validated is not None and validated==session.summary['checkpoint']: + # records_for returns independent copies of exactly these families. + return assemble_validated_facts(session_records_for(session,*FAMILY_FORMATS),owned=True) + records=session.snapshot['records'] + return decode_records(records,materials=materials.read_view(records)) + return session_derived(session, 'history_facts',compute) + + def decode_records(records, *, materials=None, artifacts=None): try: return _decode_records(records,materials=materials,artifacts=artifacts) @@ -755,15 +851,18 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): def _facts(self): self.session.check_current() + # Current canonical bytes are guarded on every call. A ProjectSession + # shares decoded metadata per checkpoint with the Study owner; other + # snapshot views keep this owner's private per-checkpoint cache. + # Returned values and permissions remain each read's responsibility. + if callable(getattr(self.session,'derived',None)): + return checkpoint_facts(self.session,self.materials) summary=self.session.summary checkpoint=(summary['project_id'],summary['checkpoint']) if self._decoded_checkpoint==checkpoint: return self._decoded_facts records=self.session.snapshot['records'] facts=decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) - # Current canonical bytes are guarded on every call. Only this owner's - # private decoded metadata is retained; returned values and permissions - # remain the responsibility of each read below. self._decoded_facts=facts self._decoded_checkpoint=checkpoint return facts @@ -827,10 +926,12 @@ def progression_origins(self,*,offset=0,limit=40): """Discover original namespaces without inventing current project ancestry.""" self._authorize('studies.lineage') natural(offset,'Origin offset');natural(limit,'Origin page limit',1,40) - facts=self._facts();origins={origin for origin,_ in facts.studies}|{origin for origin,_ in facts.trials} - items=[{'origin_catalog_id':origin, - 'study_count':sum(namespace==origin for namespace,_ in facts.studies), - 'trial_count':sum(namespace==origin for namespace,_ in facts.trials)} for origin in sorted(origins)] + facts=self._facts();counts={} + # One pass over Studies and Trials instead of origins x (studies+trials). + for namespace,_ in facts.studies:counts.setdefault(namespace,[0,0])[0]+=1 + for namespace,_ in facts.trials:counts.setdefault(namespace,[0,0])[1]+=1 + items=[{'origin_catalog_id':origin,'study_count':counts[origin][0],'trial_count':counts[origin][1]} + for origin in sorted(counts)] self._authorize('studies.lineage') return {'items':items[offset:offset+limit],'offset':offset,'limit':limit,'total':len(items)} @@ -935,7 +1036,7 @@ def payload_set_sha256(self,run_id,*,origin_catalog_id=None): def disclosure_facts(self,run_id,*,origin_catalog_id=None): self._authorize('runs.get',run_id);origin=self._origin(origin_catalog_id) result=[] - for operation in operation_documents(self.session.snapshot['records']): + for operation in operation_documents(session_records(self.session, 'project_operation')): if operation['operation']!='history.disclose':continue value=validate_disclosure_fact(operation['result']) if value['origin_catalog_id']==origin and value['run_id']==run_id:result.append(value) @@ -965,8 +1066,7 @@ def capture(result,before_records):return {'records':captured,'result':result} def dependencies(self,run_id,*,origin_catalog_id=None,offset=0,limit=100): self._authorize('runs.get',run_id);natural(offset,'Dependency offset');natural(limit,'Dependency limit',1,100) doc=self._record(self._facts().runs,run_id,origin_catalog_id);run=doc['run'] - declarations_by_sha={item['document']['sha256']:item['document'] for item in self.session.snapshot['records'] - if item['family']=='project_artifact'} + declarations_by_sha={item['document']['sha256']:item['document'] for item in session_records(self.session, 'project_artifact')} required=[] for edge in doc['payloads']: required.append({'name':edge['name'],'sha256':edge['blob']['sha256'],'bytes':edge['blob']['bytes'], @@ -1210,7 +1310,7 @@ def prepare_trial_from_material(self,study_id,body,expected_checkpoint,request_k closed(body,('revision','material','replicate'));body=checked_json(body) natural(body['replicate'],'Replicate') self.access.require('edit','studies.prepare_trial',study_id) - if request_key not in operation_records(self.session.snapshot['records']): + if request_key not in session_operations(self.session): self.materials.recipe(body['material'],verify=True) def build(facts): origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) @@ -1239,7 +1339,7 @@ def select_candidate(self,study_id,body,expected_checkpoint,request_key): origin=self.session.summary['project_id'] self.access.require('edit','studies.select_candidate',study_id) evidence=(self.evidence(body['run_id'],{},origin_catalog_id=origin) - if request_key not in operation_records(self.session.snapshot['records']) else None) + if request_key not in session_operations(self.session) else None) def build(facts): doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') diff --git a/project_material.py b/project_material.py index fbfa0189..75af5b96 100644 --- a/project_material.py +++ b/project_material.py @@ -19,6 +19,7 @@ reference, natural, retained_json, MAX_SOURCE_BYTES) from pipeline import decode_graph, decode_recipe_draft, decode_recipe_material, verify_recipe_closure, digest, execution_digest from project_store import ProjectStoreError +from project_session import session_active, session_records_for FAMILIES = { 'model_definition': 'model_definition_history_v1', @@ -325,7 +326,9 @@ def _retained_value(get_documents, family, ref): raise WorkbenchError('Retained material is unavailable.', 'unavailable', 409) if document['value']['ref'] != ref: raise WorkbenchError('Retained material reference differs.', 'invalid', 400) - return copy.deepcopy(document['value']) + # Decoded retained values are plain JSON: an exact independent copy. + from project_store import copy_validated_json + return copy_validated_json(document['value']) class MaterialSnapshot: @@ -360,7 +363,7 @@ def __init__(self, session): def _documents(self): self.session.check_current() - return decode_records(self.session.snapshot['records']) + return decode_records(session_records_for(self.session, *FAMILIES, 'project_fork')) def read_view(self, records): """Own validated material facts from one guarded checkpoint read. @@ -380,8 +383,8 @@ def intents(self): from project_fork import active_records from project_material_fork import is_intent documents = self._documents() - active = {(r['family'], r['id']) for r in active_records( - self.session.snapshot['records'], project_id=self.session.summary['project_id'])} + active = {(r['family'], r['id']) for r in session_active(self.session, + *FAMILIES, project_id=self.session.summary['project_id'])} return [dict(family=family, id=locator, current=copy.deepcopy(doc['current']), fork_origin=copy.deepcopy(doc['fork_origin'])) for (family, locator), doc in sorted(documents.items()) @@ -392,7 +395,7 @@ def intent_capabilities(self): intents = self.intents() unresolved = {(r['family'],r['id']) for r in intents} from project_fork import active_records - active = active_records(self.session.snapshot['records'], project_id=self.session.summary['project_id']) + active = session_active(self.session, *EDITABLE, project_id=self.session.summary['project_id']) return dict(requires_reselection=[dict(family=r['family'],id=r['id'],fields=r['current']['required_reselection']) for r in intents], editable=[dict(family=r['family'],id=r['id']) for r in active if r['family'] in EDITABLE and (r['family'],r['id']) not in unresolved]) @@ -510,7 +513,7 @@ def _capture_material(self, family, value, root): audit=copy.deepcopy([fact for fact in self.owner_access.facts if fact['action']==('models.publish' if family=='model_material' else 'recipes.material') and fact['target'] in (value['ref'].get('id'),value['ref']['sha256'])]) if self.owner_access else [])) - prior=decode_records(self.session.snapshot['records']).get((family,record['id'])) + prior=decode_records(session_records_for(self.session, *FAMILIES, 'project_fork')).get((family,record['id'])) if prior is not None: if prior['value']!=record['document']['value'] or prior['artifacts']!=record['document']['artifacts']: raise WorkbenchError('Retained immutable material changed.','invalid',400) @@ -594,7 +597,7 @@ def capture_snapshot(self, value, raw, *, catalog_id): if type(raw) is not bytes or len(raw)>64*1024 or hashlib.sha256(raw).hexdigest()!=value['manifest_sha256'] or decode_retained_manifest(decode(raw))!=value['manifest']: raise WorkbenchError('Exact original project manifest bytes differ.') key=catalog_id+'.'+value['manifest_sha256'] - prior=decode_records(self.session.snapshot['records']).get(('project_snapshot',key)) + prior=decode_records(session_records_for(self.session, *FAMILIES, 'project_fork')).get(('project_snapshot',key)) if prior is not None: original=prior['value'] if any(original[field]!=value[field] for field in ('id','name','manifest_sha256','manifest')): @@ -628,7 +631,7 @@ def restore(self): documents=self._documents();catalog_id=self.session.db.dataset_id from project_fork import active_records from project_material_fork import is_intent - active={(r['family'],r['id']) for r in active_records(self.session.snapshot['records'],project_id=catalog_id)} + active={(r['family'],r['id']) for r in session_active(self.session, *FAMILIES,project_id=catalog_id)} with self.session.db.transaction(): for table in ('model_definition_drafts','workspace_graphs','workspace_recipe_drafts','model_materials','workspace_recipe_materials'): self.session.db.execute('DELETE FROM '+table) @@ -656,8 +659,7 @@ def reconcile_projection(self): catalog_id = self.session.db.dataset_id from project_fork import active_records from project_material_fork import is_intent - active = {(r['family'], r['id']) for r in active_records( - self.session.snapshot['records'], project_id=catalog_id)} + active = {(r['family'], r['id']) for r in session_active(self.session, *FAMILIES, project_id=catalog_id)} with self.session.db.transaction(): for (family, locator), doc in documents.items(): if (family, locator) not in active or doc['origin_catalog_id'] != catalog_id: diff --git a/project_native_execution.py b/project_native_execution.py index 5895e4d7..b2ff98ea 100644 --- a/project_native_execution.py +++ b/project_native_execution.py @@ -20,6 +20,7 @@ from project_native_process import ENVIRONMENT, PHASES, WORKERS, prepare, validate_prepared_spec, validate_job_evidence from project_session import REQUEST_KEY from workbench import WorkbenchError, timestamp +from project_session import session_record, session_records FAMILY = 'native_execution_history' FORMAT = 'native_execution_history_v1' @@ -404,7 +405,7 @@ def _retain_outputs(self,document,*,run=None,output=None): from project_store import DIR_FLAGS, FILE_FLAGS, ProjectStoreError, _open_directory_path, anchored_directory stage_id=document['phase_id'] if document['phase']=='verify': - matches=[row['document'] for row in self.session.snapshot['records'] + matches=[row['document'] for row in session_records(self.session, FAMILY) if row['family']==FAMILY and row['document']['phase']=='train' and row['document']['origin_catalog_id']==self.db.dataset_id and row['document']['run_id']==document['run_id']] @@ -634,8 +635,8 @@ def probe_current_run(self,owner,recipe): def capture_run_payloads(self,run,artifacts): """Copy a newly admitted frozen native input, then reuse exact custody.""" from projects import Projects - prior=next((record['document'] for record in self.session.snapshot['records'] - if record['family']=='run_history' and record['id']==run['dataset_id']+'.'+run['id']),None) + prior=session_record(self.session, 'run_history',run['dataset_id']+'.'+run['id']) + prior=None if prior is None else prior['document'] if prior is not None: retained=[copy.deepcopy(edge) for edge in prior['payloads'] if edge['name']=='release.zip' or edge['name'].startswith('source/')] @@ -881,8 +882,8 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru try:handle.close_control() except (OSError,recipe_process.ProcessError):pass if phase_id not in self._never_launched: - retained=next((record['document'] for record in self.session.snapshot['records'] - if record['family']==FAMILY and record['id']==self.session.project_id+'.'+phase_id),None) + retained=session_record(self.session, FAMILY,self.session.project_id+'.'+phase_id) + retained=None if retained is None else retained['document'] acknowledged=(self.session.available and retained is not None and retained['events'][-1]['state'] in ('completed','failed') and retained['events'][-1]['receipt_sha256'] is not None) diff --git a/project_native_preparation.py b/project_native_preparation.py index 24113f63..ee4195a8 100644 --- a/project_native_preparation.py +++ b/project_native_preparation.py @@ -19,6 +19,7 @@ identifier, model_aware_graph, closed, decode_probe_observation) from project_session import operation_records from workbench import WorkbenchError, timestamp +from project_session import session_operations OPERATIONS=frozenset(('native_preparation.intent','native_preparation.validate')) @@ -352,7 +353,7 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): self.access.require('edit','native_preparation.validation',graph_id,owner_only=True) self.access.require('execute','pipeline.probe_native',graph_id,owner_only=True) body=dict(graph_id=graph_id,revision=revision) - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is not None: return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Native validation replay launched a probe.')), @@ -398,7 +399,7 @@ def prepare_recipe(self,owner,graph_id,revision,model_ref,expected_checkpoint,ke self.access.require('execute','pipeline.probe_native',graph_id,owner_only=True) body=dict(graph_id=graph_id,revision=revision,model=model_ref,reconcile_capture=False) operation='recipes.prepare_builtin_material' - if key in operation_records(self.session.snapshot['records']): + if key in session_operations(self.session): return self.session.mutate(operation,graph_id,body,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Material replay launched a native probe.')), lambda value,before:dict(records=[],result=value)) diff --git a/project_producer_execution.py b/project_producer_execution.py index e3729233..c8833909 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -15,6 +15,7 @@ _CapturedEvidence, _composed, _declarations, _referenced_artifacts, decode_records, identity, shape) from workbench import WorkbenchError +from project_session import session_record, session_records KINDS = {'generation_history':'generation', 'grounded_history':'grounded', 'caption_proposal_history':'caption', 'text_proposal_history':'text_proposal'} @@ -81,7 +82,7 @@ def _owner(self, family): # Imported science retains its physical ordering. A fresh local # authority table must allocate beyond those observed ordinals. table=TABLES[family] - observed=max((r['document']['ordinal'] for r in self.session.snapshot['records'] + observed=max((r['document']['ordinal'] for r in session_records(self.session, family) if r['family']==family and r['document']['origin_catalog_id']==self.db.dataset_id),default=0) current=self.db.execute('SELECT COALESCE(max(ordinal),0) FROM '+table).fetchone()[0] maximum=max(observed,current) @@ -210,7 +211,8 @@ def outcome(self, outcome): # Unknown acknowledgement fences authority. It never authorizes an # unacknowledged cancellation signal to a current worker. if outcome == 'committed': - records = self.session.snapshot['records'] + from project_session import session_records + records = session_records(self.session, *FAMILIES, 'project_artifact') self.archive._records = copy.deepcopy([r for r in records if r['family'] in FAMILIES]) declarations = _declarations(records) self.archive._artifacts = {sha:declarations[sha] for sha in _referenced_artifacts(self.archive._records)} @@ -279,7 +281,7 @@ def start(self, family, body, expected_checkpoint, key): prepared = [] def invoke(): requested = body.get('request_id') if type(body) is dict else None - if requested is not None and any(r['family']==family and r['id']==self.db.dataset_id+'.'+requested for r in self.session.snapshot['records']): + if requested is not None and session_record(self.session, family,self.db.dataset_id+'.'+requested) is not None: if self.db.execute('SELECT id FROM '+TABLES[family]+' WHERE id=%s',(requested,)).fetchone() is None: raise WorkbenchError('A retained producer identity cannot acquire current admission. Start with a new identity.','unavailable',409) owner = self._owner(family) diff --git a/project_projection.py b/project_projection.py index 25b6bf28..e7aa8366 100644 --- a/project_projection.py +++ b/project_projection.py @@ -5,6 +5,7 @@ checkpoint before reconstruction and before each canonical publication. """ import copy +import hashlib from datetime import datetime import re from types import SimpleNamespace @@ -19,6 +20,7 @@ from project_material import validate_operation_result as validate_material_result from project_producers import ProjectProducers, FAMILIES as PRODUCER_FAMILIES from project_producers import decode_records as decode_producer_records +from project_session import session_records_for from project_session import ProjectSession, operation_documents, operation_records from project_locks import ProjectLock from project_execution import ProjectExecution, validate_operation_result as validate_execution_result @@ -38,6 +40,7 @@ from project_measurements import receipt_validator as measurement_receipt_validator from workbench import WorkbenchError from workbench import timestamp +from project_session import session_record def validate_artifact_resolution(operation, result, records): @@ -62,7 +65,52 @@ def validate_artifact_resolution(operation, result, records): def validate_project_records(records, *, project_id, artifacts): - """Validate one inert candidate through every owning domain, without SQL reconstruction.""" + """Validate one inert candidate through every owning domain, without SQL reconstruction. + + Under an edit delta (project_validation_delta.edit_delta) the candidate is + a validated base checkpoint plus changed records; components and receipts + that cannot be affected are skipped. TULDOK_VALIDATION_CROSSCHECK=1 also + runs complete validation and requires the same outcome. + """ + from project_validation_delta import current_delta + delta = current_delta() + if delta is None: + return _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=None) + import os + if os.environ.get('TULDOK_VALIDATION_CROSSCHECK') == '1': + full_error = scoped_error = None + try: + _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=None) + except Exception as error: + full_error = error + try: + _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=delta) + except Exception as error: + scoped_error = error + if (full_error is None) != (scoped_error is None): + raise AssertionError(f'Edit-scoped validation disagrees with complete validation: ' + f'complete={full_error!r} scoped={scoped_error!r}') + if full_error is not None: + raise full_error + return None + return _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=delta) + + +def _composition_members(records): + members = set() + for item in records: + if item['family'] != 'project_composition': + continue + for source in item['document'].get('sources', ()): + for family, entries in source.get('records', {}).items(): + members.update((family, identity) for identity in entries) + members.update(('run_observation', sha) for sha in source.get('run_observations', {}).values()) + return members + + +def _validate_project_records(records, *, project_id, artifacts, delta): + def needed(component): + return delta is None or delta.touches(component) claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) identities = set() @@ -77,28 +125,62 @@ def validate_project_records(records, *, project_id, artifacts): if declaration['sha256'] != item['id']: raise WorkbenchError('Artifact inventory identity differs.') declarations[item['id']] = declaration - decode_data_records(records) + if needed('data'): + decode_data_records(records) from project_fork import decode_fork decode_fork(records, project_id=project_id) - from project_composition import decode_records as decode_compositions - decode_compositions(records, project_id=project_id) - from project_transfer import decode_records as decode_extraction_records - decode_extraction_records(records) + if needed('composition') or (delta is not None and delta.changed & _composition_members(records)): + from project_composition import decode_records as decode_compositions + decode_compositions(records, project_id=project_id) + if needed('extraction'): + from project_transfer import decode_records as decode_extraction_records + decode_extraction_records(records) operation_records(records, project_id=project_id) - validate_file_receipt = file_receipt_validator(records) - decode_native_records(records,artifacts=artifacts) - materials = MaterialSnapshot(records) - for blob in materials.artifact_blobs(): - if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): - raise WorkbenchError('Material artifact declaration is missing or inconsistent.') + if needed('native'): + decode_native_records(records,artifacts=artifacts) + material_cache = [] + def materials(): + if not material_cache: + material_cache.append(MaterialSnapshot(records)) + return material_cache[0] + if needed('material'): + for blob in materials().artifact_blobs(): + if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): + raise WorkbenchError('Material artifact declaration is missing or inconsistent.') # Resolve every cross-owner reference against THIS candidate, rather # than a resolver that still sees the preceding committed checkpoint. - decode_producer_records(records, artifacts=artifacts) - decode_history_records(records, materials=materials, artifacts=artifacts) - validate_measurement_receipt = measurement_receipt_validator(records, artifacts=artifacts) - validate_source_receipt = source_receipt_validator(records) + if needed('producers'): + decode_producer_records(records, artifacts=artifacts) + if needed('history') or (delta is not None and delta.history_receipt_added()): + decode_history_records(records, materials=materials(), artifacts=artifacts) + validators = {} + def lazy(name, component, claims, build): + # claims(operation) is exactly what the built validator can accept; + # for any other operation it would return False, so it is not built. + def validate(operation, result): + if not claims(operation): + return False + if name not in validators: + validators[name] = build() + return validators[name](operation, result) + if delta is None or needed(component): + validators[name] = build() + return validate + validate_file_receipt = lazy('file', 'files', lambda op: op.startswith('files.'), + lambda: file_receipt_validator(records)) + validate_measurement_receipt = lazy('measurement', 'measurements', lambda op: op == 'measurements.import', + lambda: measurement_receipt_validator(records, artifacts=artifacts)) + validate_source_receipt = lazy('source', 'sources', lambda op: op == 'source.register', + lambda: source_receipt_validator(records)) + from project_session import OPERATION_FAMILY for receipt in operation_documents(records): operation, result = receipt['operation'], receipt['result'] + if delta is not None: + identity = (OPERATION_FAMILY, receipt['project_id'] + '.' + hashlib.sha256(receipt['request_key'].encode('utf-8')).hexdigest() + if receipt.get('format') == 'project_operation_v2' else + hashlib.sha256(receipt['request_key'].encode('utf-8')).hexdigest()) + if not delta.receipt_needed(operation, identity): + continue if (validate_data_result(operation, result, records) or validate_material_result(operation, result, records) or validate_history_result(operation, result, records) @@ -116,7 +198,8 @@ def validate_project_records(records, *, project_id, artifacts): class ProjectProjection: - def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None): + def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None, + validation_cache=None): self.store, self.db, self.access = store, db, access self.custody = custody self.pending_original = [] @@ -129,7 +212,8 @@ def __init__(self, store, db, access, roots, *, custody=None, restore=True, runt self._runtime_sources = copy.deepcopy(runtime_sources or {}) try: self.session = ProjectSession(store, db, access, self.artifacts, - validate_records=self.validate_records) + validate_records=self.validate_records, + validation_cache=validation_cache) # Install before domain constructors retain their lock references. db.lock = ProjectLock(access.lock, self.session.lock, db.lock) self.producers = ProjectProducers(self.session) @@ -139,7 +223,9 @@ def __init__(self, store, db, access, roots, *, custody=None, restore=True, runt self.history = ProjectHistory(self.session, materials=self.materials, authorize_disclosure=self.authorize_disclosure) if restore: - self.producers.restore(self.session.snapshot) + # Producer restore reads its families, declarations and forks. + self.producers.restore(dict(records=session_records_for( + self.session, *PRODUCER_FAMILIES, 'project_artifact', 'project_fork'))) self.data.restore() self.materials.restore() # These owners create only disposable local directories. Their @@ -195,8 +281,8 @@ def resolve_artifact(self, body, expected_checkpoint, key): with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'artifacts.resolve', body['sha256'], owner_only=True) self.session.check_current() - declaration = next((item['document'] for item in self.session.snapshot['records'] - if item['family'] == 'project_artifact' and item['id'] == body['sha256']), None) + declaration = session_record(self.session, 'project_artifact', body['sha256']) + declaration = None if declaration is None else declaration['document'] if declaration is None: raise WorkbenchError('Artifact is not declared by this project.', 'not_found', 404) def invoke(): diff --git a/project_record_dedupe.py b/project_record_dedupe.py new file mode 100644 index 00000000..aff3948d --- /dev/null +++ b/project_record_dedupe.py @@ -0,0 +1,124 @@ +"""Physical deduplication of retained runtime closures in Run history records. + +Retained external-recipe Runs each embed the same captured runtime closure three +times: run.spec.runtime_files, run.material.runtime and a contiguous block of +'historical_runtime' payload edges (about 2.5 MB per Run). The store keeps one +content-addressed `run_runtime_closure` record and a compact Run document +(`project_run_history_v2`) that references it. Stores expand compact Runs back +to the exact logical v1 document on read, so domain owners and validators see +unchanged documents. A Run is compacted only when expansion reproduces its +canonical bytes exactly. +""" +import copy +import hashlib + +RUN_FAMILY = 'run_history' +RUN_V1 = 'project_run_history_v1' +RUN_V2 = 'project_run_history_v2' +CLOSURE_FAMILY = 'run_runtime_closure' +CLOSURE_FORMAT = 'project_run_runtime_closure_v1' +RUNTIME_ROLE = 'historical_runtime' + + +def _encode(value): + from project_store import _encode as canonical + return canonical(value) + + +def closure_id(closure): + """Content address of a closure document.""" + return hashlib.sha256(_encode(closure)).hexdigest() + + +def compact(document): + """(compact v2 document, closure document) or None when not exactly reversible.""" + if type(document) is not dict or document.get('format') != RUN_V1: + return None + run = document.get('run') + payloads = document.get('payloads') + if type(run) is not dict or type(payloads) is not list: + return None + spec, material = run.get('spec'), run.get('material') + if (type(spec) is not dict or type(material) is not dict or 'runtime_files' not in spec + or 'runtime' not in material or 'runtime_closure' in document): + return None + positions = [index for index, edge in enumerate(payloads) + if type(edge) is dict and edge.get('role') == RUNTIME_ROLE] + if not positions or positions != list(range(positions[0], positions[-1] + 1)): + return None + start, end = positions[0], positions[-1] + 1 + closure = dict(format=CLOSURE_FORMAT, runtime_files=copy.deepcopy(spec['runtime_files']), + runtime=copy.deepcopy(material['runtime']), payload_block=copy.deepcopy(payloads[start:end])) + reduced = copy.deepcopy(document) + del reduced['run']['spec']['runtime_files'] + del reduced['run']['material']['runtime'] + reduced['payloads'] = reduced['payloads'][:start] + reduced['payloads'][end:] + reduced['format'] = RUN_V2 + reduced['runtime_closure'] = dict(id=closure_id(closure), payload_index=start) + if _encode(expand(reduced, closure)) != _encode(document): + return None + return reduced, closure + + +def expand(document, closure): + """The exact logical v1 Run document of a compact v2 document.""" + reference = document['runtime_closure'] + if closure.get('format') != CLOSURE_FORMAT or closure_id(closure) != reference['id']: + from project_store import _fail + _fail('Run runtime closure differs from its reference.') + result = copy.deepcopy(document) + del result['runtime_closure'] + result['format'] = RUN_V1 + result['run']['spec']['runtime_files'] = copy.deepcopy(closure['runtime_files']) + result['run']['material']['runtime'] = copy.deepcopy(closure['runtime']) + index = reference['payload_index'] + if type(index) is not int or not 0 <= index <= len(result['payloads']): + from project_store import _fail + _fail('Invalid Run runtime closure position.') + result['payloads'] = result['payloads'][:index] + copy.deepcopy(closure['payload_block']) + result['payloads'][index:] + return result + + +def is_compact(family, document): + return family == RUN_FAMILY and type(document) is dict and document.get('format') == RUN_V2 + + +def logical_records(physical): + """Logical envelopes of physical records: closures hidden, compact Runs expanded.""" + closures = {item['id']: item['document'] for item in physical if item['family'] == CLOSURE_FAMILY} + result = [] + for item in physical: + if item['family'] == CLOSURE_FAMILY: + continue + document = item['document'] + if is_compact(item['family'], document): + closure = closures.get(document['runtime_closure']['id']) + if closure is None: + from project_store import _fail + _fail('Compact Run references a missing runtime closure.') + document = expand(document, closure) + result.append(dict(family=item['family'], id=item['id'], document=document)) + return result + + +def physical_records(logical, *, existing_closures=None): + """Physical envelopes for logical records, compacting Runs where exact. + + existing_closures maps closure id -> document already stored; compacted + Runs reference them and every referenced closure is included once. + """ + closures = dict(existing_closures or {}) + used = set() + result = [] + for item in logical: + document = item['document'] + if item['family'] == RUN_FAMILY: + compacted = compact(document) + if compacted is not None: + document, closure = compacted + identity = document['runtime_closure']['id'] + closures.setdefault(identity, closure) + used.add(identity) + result.append(dict(family=item['family'], id=item['id'], document=document)) + result.extend(dict(family=CLOSURE_FAMILY, id=identity, document=closures[identity]) for identity in sorted(used)) + return sorted(result, key=lambda item: (item['family'], item['id'])) diff --git a/project_runtime.py b/project_runtime.py index d63a9eb5..d14c59d4 100644 --- a/project_runtime.py +++ b/project_runtime.py @@ -15,6 +15,7 @@ from project_session import operation_records, REQUEST_KEY, _digest as operation_digest from runtime_snapshots import RuntimeSnapshots, DEFAULT_RESERVE, MAX_FILES, MAX_BYTES, MAX_MANIFEST from workbench import WorkbenchError, timestamp +from project_session import session_operations OPERATIONS = frozenset(('runtime.capture_intent', 'runtime.capture', 'runtime.abandon')) RESULT_FIELDS = frozenset(('format','project_id','manifest_sha256','file_count','total_bytes', @@ -166,7 +167,7 @@ def status(self, key): _key(key) with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', 'runtime.capture_status'); self.session.check_current() - receipts = operation_records(self.session.snapshot['records']) + receipts = session_operations(self.session) intent = receipts.get(_intent_key(key)); complete = receipts.get(key) if intent is not None and intent['operation'] != 'runtime.capture_intent': intent = None if complete is not None and complete['operation'] != 'runtime.capture': complete = None @@ -203,7 +204,7 @@ def forbidden(): _fail('No original local runtime copy exists to abandon.', 'una forbidden, lambda *args: forbidden(), scope='admin') with self.execution.runtime_reconciliation(capture_key, marker['input_sha256'], marker['manifest_sha256'], expected_checkpoint) as guard: - receipts = operation_records(self.session.snapshot['records']) + receipts = session_operations(self.session) intent = receipts.get(_intent_key(capture_key)) if (intent is None or intent['operation']!='runtime.capture_intent' or intent['input_sha256']!=marker['input_sha256'] @@ -249,7 +250,7 @@ def capture(self, config_request, expected_inventory_sha256, expected_checkpoint intent = None with self.access.lock, self.session.lock, self.db.lock: actor = self._authorize('runtime_snapshots.create'); self.session.check_current() - receipts = operation_records(self.session.snapshot['records']) + receipts = session_operations(self.session) if any(r['operation']=='runtime.abandon' and r['result']['capture_key']==key for r in receipts.values()): _fail('This capture was abandoned; use a fresh key for a new measurement.', 'conflict', 409) if (_intent_key(key) in receipts and key not in receipts @@ -257,7 +258,7 @@ def capture(self, config_request, expected_inventory_sha256, expected_checkpoint _fail('This incomplete capture intent is inert history; use a fresh key for a new capture.', 'unavailable', 409) try: with self.execution.runtime_effect(key, body_sha, expected_inventory_sha256) as guard: - receipts = operation_records(self.session.snapshot['records']) + receipts = session_operations(self.session) if key in receipts: result = self._replay(key, body, expected_checkpoint) descriptor = self.owner.get(expected_inventory_sha256) diff --git a/project_session.py b/project_session.py index aa471c1e..cfd60555 100644 --- a/project_session.py +++ b/project_session.py @@ -79,12 +79,18 @@ def operation_records(records, *, project_id=None): class ProjectSession: - def __init__(self, store, db, access, artifacts, *, validate_records): + # Sessions over a database working copy hold no record set (see __init__). + _lazy = False + + def __init__(self, store, db, access, artifacts, *, validate_records, validation_cache=None): if not callable(validate_records): raise TypeError('A complete owner-codec validator is required.') self.store, self.db, self.access, self.artifacts = store, db, access, artifacts self.lock = threading.RLock() self._validate = validate_records + self.validation_cache = validation_cache + # Checkpoint whose exact records passed complete domain validation. + self.validated_checkpoint = None self.available = False self._snapshot = None self.project_id = store.summary()['project_id'] @@ -95,15 +101,88 @@ def __init__(self, store, db, access, artifacts, *, validate_records): self._pending_observation = None self._recovery_records = None self._recovery_request = None - self.adopt(store.snapshot()) + # A database working copy serves record reads directly; the session + # then binds checkpoint metadata only and holds no record set. + self._lazy = callable(getattr(store, 'records_view', None)) + if self._lazy: + self._adopt_working_copy() + else: + self.adopt(store.snapshot(), _owned=True) + + def _adopt_working_copy(self): + """Bind the working copy's checkpoint; validate its records only if unknown.""" + with self.access.lock, self.lock, self.db.lock: + summary = self.store.summary() + cache = getattr(self, 'validation_cache', None) + if cache is not None and cache.known(self.project_id, summary['checkpoint']): + self.validated_checkpoint = summary['checkpoint'] + else: + snapshot = self.store.snapshot() + if snapshot['checkpoint'] != summary['checkpoint']: + raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) + with self._known_envelopes(snapshot['records']): + self._validated(snapshot['records'], snapshot['checkpoint']) + del snapshot + if self.store.summary()['checkpoint'] != summary['checkpoint']: + raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) + self._bind(summary) + self.available = True + + def _known_envelopes(self, records, changed=frozenset()): + """Validation context with stored envelope digests for unchanged records.""" + from contextlib import nullcontext + if not self._lazy or not callable(getattr(self.store, 'envelope_digests', None)): + return nullcontext() + from project_composition import known_envelope_digests + digests = self.store.envelope_digests() + known = {} + for record in records: + identity = (record['family'], record['id']) + if identity not in changed and identity in digests and digests[identity] is not None: + known[id(record)] = (record, digests[identity]) + return known_envelope_digests(known) + + def _edit_scope(self, base_checkpoint, changed, operations=(), base_records=None): + """Validate only what this edit can affect when its base is validated. + + The candidate is the base checkpoint's records plus changed identities + (never deletions); see project_validation_delta. + """ + from contextlib import nullcontext + if getattr(self, 'validated_checkpoint', None) != base_checkpoint: + return nullcontext() + from project_validation_delta import ValidationDelta, edit_delta + base = set() if base_records is None else {(item['family'], item['id']) for item in base_records} + added = frozenset(identity for identity in changed if base_records is not None and identity not in base) + return edit_delta(ValidationDelta(frozenset(changed), frozenset(operations), added)) + + def _bind(self, snapshot): + """Bind a committed checkpoint; a lazy session keeps only its metadata.""" + if self._lazy: + self._snapshot = dict(project_id=snapshot['project_id'], name=snapshot['name'], + checkpoint=snapshot['checkpoint'], records=None) + else: + self._snapshot = snapshot + + def _fresh_store(self): + """The working copy must still be at the bound checkpoint.""" + if self.store.summary()['checkpoint'] != self._snapshot['checkpoint']: + raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) + return self.store @property def foreground_publication(self): return self._foreground_publication @staticmethod - def _merge(records, upserts): - merged = {(item['family'], item['id']): copy.deepcopy(item) for item in records} + def _merge(records, upserts, *, owned=False): + """Candidate record map: base records plus copied upserts. + + owned=True means the caller owns these base records (a fresh working + copy read), so they are used without another whole-project copy. + """ + merged = ({(item['family'], item['id']): item for item in records} if owned else + {(item['family'], item['id']): copy.deepcopy(item) for item in records}) seen = {} for item in upserts: if type(item) is not dict or set(item) != {'family', 'id', 'document'}: @@ -132,32 +211,55 @@ def _publish_observation(self, records, key, digest, *, authorize): raise ProjectSessionError('Invalid background publication identity.') authorize() current = self.check() - merged = self._merge(current['records'], records) + merged = self._merge(current['records'], records, owned=self._lazy) candidate = [merged[identity] for identity in sorted(merged)] - self._validated(candidate) + changed = {(item['family'], item['id']) for item in records} + with self._known_envelopes(candidate, changed), \ + self._edit_scope(current['checkpoint'], changed, base_records=current['records']): + self._validated(candidate) if self.publication_intent is not None: self.publication_intent(copy.deepcopy(current), copy.deepcopy(candidate), key, digest) try: - self._pending_observation = self.store.commit(candidate, current['checkpoint'], key, digest) + if self._lazy: + changed = {(item['family'], item['id']) for item in records} + self._pending_observation = self.store.commit(candidate, current['checkpoint'], key, digest, + changed=changed) + else: + self._pending_observation = self.store.commit(candidate, current['checkpoint'], key, digest) except BaseException as error: if getattr(error, 'code', None) in ('unknown', 'outcome_unknown'): self.available = False self._recovery_records = copy.deepcopy(candidate) self._recovery_request = (key, digest) raise + self._remember_validated(self._pending_observation['checkpoint']) return copy.deepcopy(self._pending_observation) def _observation_outcome(self, outcome): if outcome not in ('committed', 'rolled_back', 'unknown'): raise ValueError('Invalid project SQL outcome.') pending, self._pending_observation = self._pending_observation, None + if pending is not None and self._atomic_store() and outcome == 'rolled_back': + # The record commit rolled back with the SQL transaction, so the + # bound checkpoint is still exactly current. Background local + # effects (for example a live phase) may have advanced, so their + # custody is reconciled explicitly before further use. A foreground + # mutate reports its own definite rollback and stays available. + if not self._foreground_publication: + self.available = False + return if pending is not None: - self._snapshot = pending + self._bind(pending) if outcome != 'committed': self.available = False if outcome == 'unknown': self.available = False + def _atomic_store(self): + """Whether record commits share this session's SQL transactions.""" + bound = getattr(self.store, '_db', None) + return bound is not None and bound is self.db + def reconcile_selection(self, snapshot): """Bind only this owner's selected or uncertain publication, never a checkout. @@ -165,7 +267,10 @@ def reconcile_selection(self, snapshot): all ordered guards until fixed inert reconstruction succeeds or fences. """ with self.access.lock, self.lock, self.db.lock: - known = snapshot == self._snapshot + if self._lazy: + known = all(snapshot[key] == self._snapshot[key] for key in ('project_id', 'name', 'checkpoint')) + else: + known = snapshot == self._snapshot attempted = (self._recovery_records is not None and snapshot['project_id'] == self._snapshot['project_id'] and snapshot['name'] == self._snapshot['name'] @@ -187,24 +292,168 @@ def summary(self): @property def snapshot(self): with self.lock: + if self._lazy: + # Complete independent record set from the working copy (costly). + current = self._fresh_store().snapshot() + if current['checkpoint'] != self._snapshot['checkpoint']: + raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) + return current return copy_validated_json(self._snapshot) - def _validated(self, records): + def records(self, *families): + """Independent copies of only the selected families at the bound checkpoint. + + Equivalent to filtering a complete snapshot copy, without copying the + unselected families (most bytes are usually retained Run history). + """ + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + selected = frozenset(families) + with self.lock: + if self._lazy: + return self._fresh_store().records(*families) + return copy_validated_json([item for item in self._snapshot['records'] + if item['family'] in selected]) + + def records_for(self, *families): + """Every bound record, in order, for one owner's pure decoder. + + Envelopes of the selected families are independent copies the owner + may keep or mutate. Every other envelope is a fresh envelope sharing its + verified document read-only, so fork/membership validators still see + the complete record set without copying unrelated Run history. Owners + must not mutate documents outside their selected families. + """ + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + selected = frozenset(families) + with self.lock: + if self._lazy: + # Unselected envelopes carry document None (identity only). + return self._fresh_store().records_view(*families) + return [copy_validated_json(item) if item['family'] in selected else + dict(family=item['family'], id=item['id'], document=item['document']) + for item in self._snapshot['records']] + + def active(self, *families, project_id=None): + """active_records for selected families over the bound checkpoint. + + Validates the complete fork history against every bound identity, but + copies only the selected active envelopes. + """ + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + from project_fork import active_records + with self.lock: + if self._lazy: + return active_records(self._fresh_store().records_view(*families, 'project_fork'), + project_id=self.project_id if project_id is None else project_id, + families=frozenset(families)) + return active_records(self._snapshot['records'], + project_id=self.project_id if project_id is None else project_id, + families=frozenset(families)) + + def derived(self, name, compute): + """Memoize one owner-derived read-only fact set per bound checkpoint. + + compute() must be a pure function of the bound records. Owners share + the result; they must copy any part they return or mutate. + """ + if type(name) is not str or not callable(compute): + raise TypeError('Derived facts need a name and a compute function.') + with self.lock: + checkpoint = self._snapshot['checkpoint'] + cache = getattr(self, '_derived_cache', None) + if cache is None or cache[0] != checkpoint: + cache = (checkpoint, {}) + self._derived_cache = cache + if name not in cache[1]: + value = compute() + if self._snapshot['checkpoint'] != checkpoint: + raise ProjectSessionError('Project changed while deriving facts.', 'stale_project', 409) + cache[1][name] = value + return cache[1][name] + + def record(self, family, record_id): + """An independent copy of one bound record envelope, or None.""" + if type(family) is not str or type(record_id) is not str: + raise TypeError('Select a record by family and ID.') + with self.lock: + if self._lazy: + return self._fresh_store().record(family, record_id) + index = getattr(self, '_record_index', None) + if index is None or index[0] != self._snapshot['checkpoint']: + index = (self._snapshot['checkpoint'], + {(item['family'], item['id']): item for item in self._snapshot['records']}) + self._record_index = index + found = index[1].get((family, record_id)) + return None if found is None else copy_validated_json(found) + + def operations(self, *, project_id=None): + """operation_records over the bound checkpoint, computed once per checkpoint. + + The bound records are immutable for a checkpoint, so the result is a + pure function of (checkpoint, project_id). Callers receive independent + copies, exactly as from operation_records(self.snapshot['records']). + """ + with self.lock: + key = (self._snapshot['checkpoint'], project_id) + cached = getattr(self, '_operations_cache', None) + if cached is None or cached[0] != key: + # operation_records copies what it returns and never mutates input. + records = (self._fresh_store().records_view(OPERATION_FAMILY, 'project_fork') if self._lazy + else self._snapshot['records']) + cached = (key, operation_records(records, project_id=project_id)) + self._operations_cache = cached + return copy.deepcopy(cached[1]) + + def _validated(self, records, checkpoint=None): + """Completely validate records; reuse an exact earlier success. + + checkpoint, when given, must be the Store checkpoint naming exactly + these records. Only then can a remembered validation be reused. + """ + cache = getattr(self, 'validation_cache', None) + if checkpoint is not None and cache is not None and cache.known(self.project_id, checkpoint): + self.validated_checkpoint = checkpoint + return operation_records(records, project_id=self.project_id) self._validate(records) - - def adopt(self, snapshot): + if checkpoint is not None: + self.validated_checkpoint = checkpoint + if cache is not None: + cache.remember(self.project_id, checkpoint) + + def _remember_validated(self, checkpoint): + # The caller completely validated exactly the records this names. + self.validated_checkpoint = checkpoint + cache = getattr(self, 'validation_cache', None) + if cache is not None: + cache.remember(self.project_id, checkpoint) + + def adopt(self, snapshot, *, _owned=False): """Bind a fully validated checkpoint after inert reconstruction/cutover. The caller owns SQL reconstruction. This method never repairs or writes canonical bytes, and is not authorization to trust an imported receipt. + _owned marks a snapshot this session just read from its own Store: its + checkpoint then names its records, so freshness is a checkpoint check + and the independent snapshot needs no second copy or full comparison. """ with self.access.lock, self.lock, self.db.lock: + if _owned: + self._validated(snapshot['records'], snapshot['checkpoint']) + if self.store.summary()['checkpoint'] != snapshot['checkpoint']: + raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) + self._bind(snapshot) + self.available = True + return self._validated(snapshot['records']) current = self.store.snapshot() if current != snapshot: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._snapshot = copy_validated_json(snapshot) + self._remember_validated(snapshot['checkpoint']) + self._bind(copy_validated_json(snapshot) if not self._lazy else snapshot) self.available = True def _checked(self, read): @@ -239,6 +488,9 @@ def check_records(self, family): if type(family) is not str: raise TypeError('Record-family selection requires a string.') current = self._checked(self.store.summary) + if self._lazy: + current['records'] = self.store.records(family) + return current current['records'] = copy.deepcopy([ item for item in self._snapshot['records'] if item['family'] == family]) return current @@ -272,7 +524,7 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt raise ProjectSessionError('Request key already has different input.', 'conflict', 409) # The current composed codec validates the exact historical # result too; receipt facts can never authorize execution. - self._validated(current['records']) + self._validated(current['records'], current['checkpoint']) return copy.deepcopy(prior['result']) if self.store.reconcile(key, digest) is not None: raise ProjectSessionError('Request key belongs to another canonical operation.', 'conflict', 409) @@ -282,38 +534,66 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt self._foreground_publication = True with self.db.transaction(): result = invoke() - captured = capture(result, copy.deepcopy(current['records'])) + # A working-copy read is already this mutation's own fresh + # copy; owner captures read it and copy what they change. + before = current['records'] if self._lazy else copy.deepcopy(current['records']) + captured = capture(result, before) if type(captured) is not dict or set(captured) != {'records', 'result'} or type(captured['records']) is not list: raise ProjectSessionError('Invalid owned project capture.') if len(agent_api.encoded(captured['result'])) > agent_api.RESPONSE_LIMIT: raise ProjectSessionError('Project result exceeds its transport bound.', 'too_large', 413) upserts = captured['records'] if self.extra_capture is not None: - extra = self.extra_capture(copy.deepcopy(current['records'])) + extra = self.extra_capture(current['records'] if self._lazy else copy.deepcopy(current['records'])) if type(extra) is not list: raise ProjectSessionError('Invalid fixed runtime capture.') upserts = upserts + extra - merged = self._merge(current['records'], upserts) + merged = self._merge(current['records'], upserts, owned=self._lazy) document = dict(format=OPERATION_FORMAT, project_id=self.project_id, operation=operation, target=target, request_key=key, input_sha256=digest, result=copy.deepcopy(captured['result'])) identity = (OPERATION_FAMILY, self.project_id + '.' + hashlib.sha256(key.encode('utf-8')).hexdigest()) merged[identity] = dict(family=identity[0], id=identity[1], document=document) records = [merged[identity] for identity in sorted(merged)] - self._validated(records) + changed = {(item['family'], item['id']) for item in upserts} | {identity} + with self._known_envelopes(records, changed), \ + self._edit_scope(current['checkpoint'], changed, (operation,), + base_records=current['records']): + self._validated(records) # Unique payloads have already reached artifact custody; # SQL callbacks can delete only disposable projection files. attempted_records = records if self.publication_intent is not None: self.publication_intent(copy.deepcopy(current), copy.deepcopy(records), key, digest) - committed = self.store.commit(records, current['checkpoint'], key, digest) + if self._lazy: + # Only the captured upserts and this receipt can differ. + changed = {(item['family'], item['id']) for item in upserts} | {identity} + committed = self.store.commit(records, current['checkpoint'], key, digest, changed=changed) + else: + committed = self.store.commit(records, current['checkpoint'], key, digest) # Outcome hooks see canonical selection before SQL outcome # and before any deferred dispatch/wake can run. self._pending_observation = committed - self._snapshot = committed + self._bind(committed) + # These exact records were completely validated as the candidate. + self._remember_validated(committed['checkpoint']) return copy.deepcopy(captured['result']) except BaseException as error: + if committed is not None and self._atomic_store(): + from metadata import CommitOutcomeUnknown + if isinstance(error, CommitOutcomeUnknown): + # PostgreSQL may or may not have committed both the + # records and the projection; reopening reads the outcome. + self.available = False + raise ProjectSessionError('PostgreSQL did not confirm this edit. Reopen the project to see whether it was saved.', + 'unknown', 503) from error + # Records and projection rolled back together; nothing changed. + if isinstance(error, ProjectSessionError): + raise + raise ProjectSessionError('PostgreSQL rejected this edit; nothing was saved.', + 'rejected', 409) from error if committed is not None: - self._snapshot, self.available = committed, False + self._bind(committed) + self.available = False failure = ProjectSessionError('Canonical operation committed; reopen to reconstruct the projection.', 'projection_unavailable', 503) failure.receipt = dict(committed=True, selected=True, project_id=committed['project_id'], @@ -328,3 +608,49 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt finally: self._foreground_publication = False self._pending_observation = None + + +# Owners also accept lightweight snapshot views (for example inert candidate or +# retained-material views that expose only snapshot/check_current). These +# helpers use a ProjectSession's copy-avoiding bound reads when available and +# otherwise compute exactly the same values from the view's snapshot. + +def session_records(session, *families): + if hasattr(session, 'records') and callable(session.records): + return session.records(*families) + return [item for item in session.snapshot['records'] if item['family'] in families] + + +def session_record(session, family, record_id): + if hasattr(session, 'record') and callable(session.record): + return session.record(family, record_id) + return next((item for item in session.snapshot['records'] + if item['family'] == family and item['id'] == record_id), None) + + +def session_records_for(session, *families): + if hasattr(session, 'records_for') and callable(session.records_for): + return session.records_for(*families) + return session.snapshot['records'] + + +def session_active(session, *families, project_id=None): + if hasattr(session, 'active') and callable(session.active): + return session.active(*families, project_id=project_id) + from project_fork import active_records + snapshot = session.snapshot + return active_records(snapshot['records'], + project_id=session.project_id if project_id is None else project_id, + families=frozenset(families)) + + +def session_operations(session, *, project_id=None): + if hasattr(session, 'operations') and callable(session.operations): + return session.operations(project_id=project_id) + return operation_records(session.snapshot['records'], project_id=project_id) + + +def session_derived(session, name, compute): + if hasattr(session, 'derived') and callable(session.derived): + return session.derived(name, compute) + return compute() diff --git a/project_store.py b/project_store.py index d079ca3f..826a5ab7 100644 --- a/project_store.py +++ b/project_store.py @@ -61,6 +61,9 @@ 'grounded_history': 'project_grounded_history_v1', 'caption_proposal_history': 'project_caption_proposal_history_v1', 'text_proposal_history': 'project_text_proposal_history_v1', + # Store-only: one shared retained runtime closure for compact Runs + # (project_record_dedupe); never visible to domain owners. + 'run_runtime_closure': 'project_run_runtime_closure_v1', } MAX_RECORDS = 100_000 # Scientific history can retain original runtime/file observations in hundreds @@ -185,13 +188,41 @@ def _domain_format(value, family): _fail('Domain document requires a versioned format.') if value != SUPPORTED_FORMATS[family] and not ( family == 'project_operation' and value == 'project_operation_v2' - or family == 'project_file' and value == 'project_file_v2'): + or family == 'project_file' and value == 'project_file_v2' + or family == 'run_history' and value == 'project_run_history_v2'): _fail('Unsupported domain document format.', 'unsupported', 422) -def _records(records): +def _compact(value): + # CPython's C encoder is used only without indent; it distinguishes 1, 1.0, + # True and 0.0/-0.0, which Python == equality does not. + return json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False, + separators=(',', ':')) + + +def _same_json(candidate, verified): + """Exact JSON value equality: same types, float bits and key sets.""" + if candidate is verified: + return True + if candidate != verified: + return False + try: + return _compact(candidate) == _compact(verified) + except (TypeError, ValueError, RecursionError): + return False + + +def _prepared_records(records, unchanged=None): + """Validate and canonically encode a complete candidate record set. + + unchanged maps (family, id) -> (verified current document, inventory entry). + A candidate document equal to its verified current revision reuses that + revision's known canonical byte count and digest instead of re-encoding it; + its raw slot is None and the caller must not rewrite it. + """ if type(records) is not list or len(records) > MAX_RECORDS: _fail('Project records must be a bounded list.') + unchanged = unchanged or {} result, seen, total = [], set(), 0 for record in records: _keys(record, ('family', 'id', 'document'), 'Project record') @@ -206,21 +237,207 @@ def _records(records): if type(document) is not dict: _fail('Domain document must be an object.') _domain_format(document.get('format'), family) + current = unchanged.get((family, identity)) + if current is not None and _same_json(document, current[0]) \ + and current[1]['format'] == document['format']: + # Exactly equal JSON values have identical canonical bytes; share the + # already verified current document (never handed out by alias). + total += current[1]['bytes'] + if total > MAX_TOTAL_BYTES: + _fail('Project records exceed the synchronous byte limit.') + result.append((family, identity, current[0], None, current[1])) + continue document = copy.deepcopy(document) raw = _encode(document) total += len(raw) if len(raw) > MAX_RECORD_BYTES or total > MAX_TOTAL_BYTES: _fail('Project records exceed the synchronous byte limit.') - result.append((family, identity, document, raw)) + result.append((family, identity, document, raw, None)) return sorted(result, key=lambda item: (item[0], item[1])) +_NESTED_DOCUMENT = b'\n' + b' ' * 4 + + +class _RecordsDigest: + """Incremental SHA256 of _encode([dict(family, id, document), ...]). + + add() takes each record's own canonical _encode(document) bytes, in record + order. Canonical JSON never contains a literal newline inside a string, so + a document nested two levels deep differs from its standalone encoding only + by four extra spaces after each newline. The result is byte-identical to + hashing the complete re-encoded list, without re-encoding any document. + """ + def __init__(self): + self._digest = hashlib.sha256() + self._first = True + + def add(self, family, identity, raw): + if not raw.endswith(b'\n'): + _fail('Canonical record bytes must end with LF.') + self._digest.update(b'[\n {\n "document": ' if self._first else b',\n {\n "document": ') + self._first = False + self._digest.update(raw[:-1].replace(b'\n', _NESTED_DOCUMENT)) + self._digest.update(b',\n "family": ') + self._digest.update(json.dumps(family, ensure_ascii=False).encode('utf-8')) + self._digest.update(b',\n "id": ') + self._digest.update(json.dumps(identity, ensure_ascii=False).encode('utf-8')) + self._digest.update(b'\n }') + + def hexdigest(self): + final = self._digest.copy() + final.update(b'[]\n' if self._first else b'\n]\n') + return final.hexdigest() + + +def envelope_digest(family, identity, raw): + """SHA256 of _encode(dict(family, id, document)) from the document's canonical bytes. + + Canonical JSON has no literal newline inside strings, so the document nested + one level deep differs from its standalone encoding only by two spaces after + each newline. + """ + if not raw.endswith(b'\n'): + _fail('Canonical record bytes must end with LF.') + digest = hashlib.sha256(b'{\n "document": ') + digest.update(raw[:-1].replace(b'\n', b'\n ')) + digest.update(b',\n "family": ' + json.dumps(family, ensure_ascii=False).encode('utf-8')) + digest.update(b',\n "id": ' + json.dumps(identity, ensure_ascii=False).encode('utf-8') + b'\n}\n') + return digest.hexdigest() + + +def _records_digest(rows): + """Digest of (family, id, canonical raw) rows; see _RecordsDigest.""" + digest = _RecordsDigest() + for family, identity, raw in rows: + digest.add(family, identity, raw) + return digest.hexdigest() + + +def _records(records): + """Complete canonical encoding as (family, id, document, raw) tuples.""" + return [(family, identity, document, raw) + for family, identity, document, raw, _entry in _prepared_records(records)] + + +def _physical_candidates(records, logical, physical): + """Physical envelopes for a logical candidate record set. + + A candidate equal to its current logical document reuses the current + physical document (so its verified revision is kept). Changed Runs are + compacted to a shared runtime closure when exactly reversible; every + referenced closure is included once. Returns (physical candidates, logical + result in record order) or (records, None) for malformed input, which + _prepared_records then rejects with its usual errors. + """ + from project_record_dedupe import CLOSURE_FAMILY, RUN_FAMILY, compact, is_compact + if type(records) is not list or any(type(item) is not dict or set(item) != {'family', 'id', 'document'} + or type(item['family']) is not str or type(item['id']) is not str + for item in records): + return records, None + if any(item['family'] == CLOSURE_FAMILY for item in records): + _fail('Runtime closures are store-owned records.', 'unsupported', 422) + current_logical = {(item['family'], item['id']): item['document'] for item in logical} + current_physical = {(item['family'], item['id']): item['document'] for item in physical} + closures = {item['id']: item['document'] for item in physical if item['family'] == CLOSURE_FAMILY} + candidates, logical_out, referenced = [], [], set() + for item in records: + key = (item['family'], item['id']) + document = item['document'] + if key in current_logical and _same_json(document, current_logical[key]): + stored = current_physical[key] + candidates.append(dict(family=key[0], id=key[1], document=stored)) + logical_out.append(dict(family=key[0], id=key[1], document=current_logical[key])) + if is_compact(key[0], stored): + referenced.add(stored['runtime_closure']['id']) + continue + owned = copy.deepcopy(document) + logical_out.append(dict(family=key[0], id=key[1], document=owned)) + compacted = compact(owned) if key[0] == RUN_FAMILY else None + if compacted is not None: + reduced, closure = compacted + identity = reduced['runtime_closure']['id'] + closures.setdefault(identity, closure) + referenced.add(identity) + candidates.append(dict(family=key[0], id=key[1], document=reduced)) + else: + candidates.append(dict(family=key[0], id=key[1], document=owned)) + candidates.extend(dict(family=CLOSURE_FAMILY, id=identity, document=closures[identity]) for identity in sorted(referenced)) + logical_out.sort(key=lambda item: (item['family'], item['id'])) + return candidates, logical_out + + def _request(key, digest): if type(key) is not str or not REQUEST_KEY.fullmatch(key): _fail('Request key must contain 1–128 supported characters.') _digest(digest, 'Request body digest') +def checked_manifest(raw): + """Decode and check a project checkpoint without reading any record file. + + Verifies format, identity, the checkpoint digest, every inventory entry's + shape, ordering, rooted path and byte bounds, membership roots and receipt + structure. Record bytes and the latest receipt's records digest are checked + by whoever reads the records. + """ + manifest = _decode(raw) + if type(manifest) is not dict: + _fail('Project checkpoint must be an object.') + if manifest.get('format') != FORMAT: + if type(manifest.get('format')) is str and re.fullmatch(r'tuldok_project_v[1-9][0-9]*', manifest['format']): + _fail('Unsupported project format.', 'unsupported', 422) + _fail('Invalid project format.') + _keys(manifest, ('format', 'project_id', 'name', 'checkpoint', 'roots', 'inventory', 'receipts'), 'Project checkpoint') + identity = manifest['project_id'] + if type(identity) is not str or not re.fullmatch('[a-f0-9]{32}', identity): + _fail('Invalid project UUID.') + if type(manifest['name']) is not str or not manifest['name'].strip() or len(manifest['name']) > 200: + _fail('Invalid project display name.') + _digest(manifest['checkpoint'], 'Checkpoint') + core = {k: v for k, v in manifest.items() if k != 'checkpoint'} + if _sha(_encode(core)) != manifest['checkpoint']: + _fail('Project checkpoint digest does not match its contents.') + inventory = manifest['inventory'] + if type(inventory) is not list or len(inventory) > MAX_RECORDS: + _fail('Invalid project inventory.') + roots, total, previous = [], 0, None + for entry in inventory: + _keys(entry, ('family', 'id', 'format', 'namespace', 'path', 'bytes', 'sha256'), 'Inventory entry') + family, record_id, digest = entry['family'], entry['id'], entry['sha256'] + _identifier(family, 'Record family') + if family not in SUPPORTED_FORMATS: + _fail('Unsupported project record family.', 'unsupported', 422) + _identifier(record_id, 'Record ID') + _digest(digest, 'Record digest') + _domain_format(entry['format'], family) + key = (family, record_id) + if previous is not None and key <= previous: + _fail('Inventory must have unique sorted record identities.') + previous = key + if entry['namespace'] != identity or entry['path'] != f'records/{family}/{record_id}/{digest}.json': + _fail('Invalid inventory namespace or rooted record path.') + if type(entry['bytes']) is not int or not 0 < entry['bytes'] <= MAX_RECORD_BYTES: + _fail('Invalid inventory record byte count.') + total += entry['bytes'] + if total > MAX_TOTAL_BYTES: + _fail('Project records exceed their total byte limit.') + roots.append(dict(family=family, id=record_id)) + if manifest['roots'] != roots: + _fail('Project membership roots disagree with the inventory.') + if type(manifest['receipts']) is not list or not manifest['receipts']: + _fail('Project requires operation receipts.') + seen = set() + for receipt in manifest['receipts']: + _keys(receipt, ('request_key', 'request_sha256', 'records_sha256'), 'Operation receipt') + _request(receipt['request_key'], receipt['request_sha256']) + _digest(receipt['records_sha256'], 'Receipt records digest') + if receipt['request_key'] in seen: + _fail('Duplicate operation receipt key.') + seen.add(receipt['request_key']) + return manifest + + def _path_string(path): try: value = os.fspath(path) @@ -471,8 +688,10 @@ class ProjectStore: revisions are never pruned. Callers validate domain semantics before commit and after Open; this owner validates only the generic JSON/inventory contract. """ - def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path): + def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path, canonical_digests=None): self.path = Path(path) + # Optional private set of digests already proven canonical; see _load. + self._canonical_digests = canonical_digests self._root_fd = root_fd self._origin_fd = origin_fd self._relative_parts = relative_parts @@ -482,6 +701,7 @@ def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path): self._read_cache = None self._mutex = threading.RLock() self._unknown = False + self._folder_mode = False try: fcntl.flock(root_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError as exc: @@ -578,7 +798,7 @@ def create_at(cls, trusted_root_fd, relative, name, request_key, request_sha256, os.close(origin_fd) @classmethod - def open(cls, path): + def open(cls, path, *, canonical_digests=None): raw = _path_string(path) if type(raw) is not str or '\x00' in raw or '..' in raw.split('/'): _fail('Unsafe project root path.') @@ -586,12 +806,12 @@ def open(cls, path): parent, leaf = os.path.split(absolute) _, parent_fd = _open_directory_path(parent) try: - return cls.open_at(parent_fd, leaf, root_path=parent) + return cls.open_at(parent_fd, leaf, root_path=parent, canonical_digests=canonical_digests) finally: os.close(parent_fd) @classmethod - def open_at(cls, trusted_root_fd, relative, *, root_path): + def open_at(cls, trusted_root_fd, relative, *, root_path, canonical_digests=None): """Open a descendant without following any configured-root or child alias.""" parts = _relative_parts(relative) absolute = _verify_origin(trusted_root_fd, root_path) @@ -600,7 +820,8 @@ def open_at(cls, trusted_root_fd, relative, *, root_path): try: fd = _walk(origin_fd, parts) owned_origin, origin_fd = origin_fd, None - store = cls(Path(absolute).joinpath(*parts), fd, owned_origin, parts, absolute) + store = cls(Path(absolute).joinpath(*parts), fd, owned_origin, parts, absolute, + canonical_digests=canonical_digests) store._meta_fd = _directory(fd, '.tuldok') store._manifest, _ = store._load() return store @@ -616,6 +837,115 @@ def open_at(cls, trusted_root_fd, relative, *, root_path): if origin_fd is not None: os.close(origin_fd) + # ---- Folder mode: the external project as an import source/export target. + # A folder-mode Store holds the writer lock and root binding but never + # loads the record set; the database working copy owns current records. + + @classmethod + def open_folder_at(cls, trusted_root_fd, relative, *, root_path): + """Bind an existing external project folder without reading its records.""" + parts = _relative_parts(relative) + absolute = _verify_origin(trusted_root_fd, root_path) + origin_fd = os.dup(trusted_root_fd) + store = None + try: + fd = _walk(origin_fd, parts) + owned_origin, origin_fd = origin_fd, None + store = cls(Path(absolute).joinpath(*parts), fd, owned_origin, parts, absolute) + store._meta_fd = _directory(fd, '.tuldok') + store._folder_mode = True + return store + except OSError as exc: + if store is not None: + store.close() + _io_error(exc, 'open project folder') + except BaseException: + if store is not None: + store.close() + raise + finally: + if origin_fd is not None: + os.close(origin_fd) + + def folder_checkpoint(self): + """(raw bytes, checked manifest) of the folder's current project.json.""" + with self._mutex: + self._binding() + try: + raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) + except OSError as exc: + _io_error(exc, 'read project checkpoint') + return raw, checked_manifest(raw) + + def folder_record(self, entry): + """One inventory entry's exact bytes after length/SHA256 verification.""" + with self._mutex: + try: + raw = self._record_file(entry['family'], entry['id'], entry['sha256']) + except OSError as exc: + _io_error(exc, 'read project record') + if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: + _fail('Canonical record bytes changed or are corrupt.') + return raw + + def folder_has_record(self, entry): + """Whether the content-addressed revision file exists with its byte length.""" + with self._mutex: + fds = [] + try: + parent = self._meta_fd + for part in ('records', entry['family'], entry['id']): + parent = _directory(parent, part) + fds.append(parent) + info = os.stat(entry['sha256'] + '.json', dir_fd=parent, follow_symlinks=False) + return stat.S_ISREG(info.st_mode) and info.st_size == entry['bytes'] + except FileNotFoundError: + return False + except OSError as exc: + _io_error(exc, 'inspect project record') + finally: + for fd in reversed(fds): + os.close(fd) + + def folder_write_record(self, family, record_id, digest, raw): + """Durably publish one immutable revision; existing bytes must be identical.""" + with self._mutex: + try: + written = self._record_file(family, record_id, digest, raw) + except OSError as exc: + _io_error(exc, 'write project record') + if written != raw: + _fail('Immutable project revision already has different bytes.') + + def folder_publish(self, raw, expected_raw): + """Atomically replace project.json only if it still has expected_raw. + + expected_raw None means the folder has no checkpoint yet. Refuses to + overwrite a checkpoint someone else published since import/export. + """ + if len(raw) > MAX_MANIFEST_BYTES: + _fail('Project checkpoint exceeds its byte limit.') + with self._mutex: + temporary = '.checkpoint-' + uuid.uuid4().hex + '.tmp' + replaced = False + try: + _write_file(self._meta_fd, temporary, raw) + self._binding() + try: + current = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) + except FileNotFoundError: + current = None + if current != expected_raw: + os.unlink(temporary, dir_fd=self._meta_fd) + _fail('The external project changed since Tuldok last imported or saved it.', 'conflict', 409) + os.replace(temporary, 'project.json', src_dir_fd=self._meta_fd, dst_dir_fd=self._meta_fd) + replaced = True + os.fsync(self._meta_fd) + os.fsync(self._root_fd) + except OSError as exc: + self._unknown = replaced + _io_error(exc, 'publish checkpoint', unknown=replaced) + def _binding(self): if self._root_fd is None: _fail('Project store is closed.', 'unavailable', 409) @@ -656,82 +986,49 @@ def _load(self): raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) cached = self._read_cache if cached is not None and raw == cached[0]: - manifest, loaded = cached[1:] - # Reuse decoding only after checking every actual canonical file. - # Stat identity is not content identity; corruption still fails. - for entry in manifest['inventory']: - self._verified_record_bytes(entry) - if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: - _fail('Project checkpoint changed while reading.', 'stale', 409) + # Record files are content-addressed by their inventory digest and + # are never rewritten, and every one was length/SHA256 verified when + # this Store decoded it. Byte-identical checkpoint bytes therefore + # name the same verified record set; re-hashing every record on each + # freshness check is redundant. On-disk corruption after that point + # is still refused by Save (verify_records) and by every reopen. self._binding() - return manifest, loaded - manifest = _decode(raw) - if type(manifest) is not dict: - _fail('Project checkpoint must be an object.') - if manifest.get('format') != FORMAT: - if type(manifest.get('format')) is str and re.fullmatch(r'tuldok_project_v[1-9][0-9]*', manifest['format']): - _fail('Unsupported project format.', 'unsupported', 422) - _fail('Invalid project format.') - _keys(manifest, ('format', 'project_id', 'name', 'checkpoint', 'roots', 'inventory', 'receipts'), 'Project checkpoint') - identity = manifest['project_id'] - if type(identity) is not str or not re.fullmatch('[a-f0-9]{32}', identity): - _fail('Invalid project UUID.') - if type(manifest['name']) is not str or not manifest['name'].strip() or len(manifest['name']) > 200: - _fail('Invalid project display name.') - _digest(manifest['checkpoint'], 'Checkpoint') - core = {k: v for k, v in manifest.items() if k != 'checkpoint'} - if _sha(_encode(core)) != manifest['checkpoint']: - _fail('Project checkpoint digest does not match its contents.') - inventory = manifest['inventory'] - if type(inventory) is not list or len(inventory) > MAX_RECORDS: - _fail('Invalid project inventory.') - loaded, roots, total, previous = [], [], 0, None - for entry in inventory: - _keys(entry, ('family', 'id', 'format', 'namespace', 'path', 'bytes', 'sha256'), 'Inventory entry') + return cached[1], cached[2] + manifest = checked_manifest(raw) + loaded = [] + records_digest = _RecordsDigest() + known = self._canonical_digests + proven = [] + for entry in manifest['inventory']: family, record_id, digest = entry['family'], entry['id'], entry['sha256'] - _identifier(family, 'Record family') - if family not in SUPPORTED_FORMATS: - _fail('Unsupported project record family.', 'unsupported', 422) - _identifier(record_id, 'Record ID') - _digest(digest, 'Record digest') - _domain_format(entry['format'], family) - key = (family, record_id) - if previous is not None and key <= previous: - _fail('Inventory must have unique sorted record identities.') - previous = key - if entry['namespace'] != identity or entry['path'] != f'records/{family}/{record_id}/{digest}.json': - _fail('Invalid inventory namespace or rooted record path.') - if type(entry['bytes']) is not int or not 0 < entry['bytes'] <= MAX_RECORD_BYTES: - _fail('Invalid inventory record byte count.') - total += entry['bytes'] - if total > MAX_TOTAL_BYTES: - _fail('Project records exceed their total byte limit.') record_raw = self._verified_record_bytes(entry) - document = _decode(record_raw) + if known is not None and digest in known: + # These exact bytes (named by their verified SHA256) were already + # proven to be their value's canonical encoding, which excludes + # duplicate keys and non-finite numbers; a plain parse is exact. + document = json.loads(record_raw) + else: + document = _decode(record_raw) + proven.append(digest) if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') + records_digest.add(family, record_id, record_raw) loaded.append(dict(family=family, id=record_id, document=document)) - roots.append(dict(family=family, id=record_id)) - if manifest['roots'] != roots: - _fail('Project membership roots disagree with the inventory.') - if type(manifest['receipts']) is not list or not manifest['receipts']: - _fail('Project requires operation receipts.') - seen = set() - for receipt in manifest['receipts']: - _keys(receipt, ('request_key', 'request_sha256', 'records_sha256'), 'Operation receipt') - _request(receipt['request_key'], receipt['request_sha256']) - _digest(receipt['records_sha256'], 'Receipt records digest') - if receipt['request_key'] in seen: - _fail('Duplicate operation receipt key.') - seen.add(receipt['request_key']) - if manifest['receipts'][-1]['records_sha256'] != _sha(_encode(loaded)): + # _decode proved each raw revision is its document's canonical encoding. + if manifest['receipts'][-1]['records_sha256'] != records_digest.hexdigest(): _fail('Latest operation receipt disagrees with the current record set.') # A second pointer read detects checkout/replacement during closure validation. if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: _fail('Project checkpoint changed while reading.', 'stale', 409) self._binding() - self._read_cache = (raw, manifest, loaded) - return manifest, loaded + if known is not None and proven: + known.add_many(proven) + # Domain owners see logical records: shared runtime closures hidden and + # compact Runs expanded to their exact original documents. + from project_record_dedupe import logical_records + logical = logical_records(loaded) + self._read_cache = (raw, manifest, logical, loaded) + return manifest, logical def _verified_record_bytes(self, entry): raw = self._record_file(entry['family'], entry['id'], entry['sha256']) @@ -800,6 +1097,7 @@ def _publish(self, manifest, creating=False): except OSError as exc: self._unknown = replaced _io_error(exc, 'publish checkpoint', unknown=replaced) + return raw # Temporary/orphan files remain inert after failures and process interruption. def _snapshot(self, manifest, records): @@ -815,8 +1113,24 @@ def _current(self): except OSError as exc: _io_error(exc, 'read project snapshot') + def verify_records(self): + """Re-read and SHA256-check every referenced canonical revision on disk. + + Ordinary freshness checks compare checkpoint bytes only; this explicit + pass detects later same-length corruption of already-decoded revisions. + """ + with self._mutex: + manifest, _records = self._current() + try: + for entry in manifest['inventory']: + self._verified_record_bytes(entry) + except OSError as exc: + _io_error(exc, 'verify project records') + if self._current()[0] != manifest: + _fail('Project checkpoint changed during verification.', 'stale', 409) + def summary(self): - """Return scalar metadata after the same complete freshness check.""" + """Return scalar metadata after the checkpoint freshness check.""" with self._mutex: manifest, _records = self._current() return {key: manifest[key] for key in ('project_id', 'name', 'checkpoint')} @@ -841,14 +1155,27 @@ def reconcile(self, request_key, request_sha256): def commit(self, records, expected_checkpoint, request_key, request_sha256): _request(request_key, request_sha256) _digest(expected_checkpoint, 'Expected checkpoint') - prepared = _records(records) - normalized = [dict(family=f, id=i, document=d) for f, i, d, _ in prepared] - records_digest = _sha(_encode(normalized)) with self._mutex: try: current, loaded = self._load() if current != self._manifest: _fail('Project checkpoint changed; close and reopen.', 'stale', 409) + physical = self._read_cache[3] + # Unchanged records keep their verified canonical revision: no + # re-encoding, rewrite or read-back of identical immutable bytes. + entries = {(entry['family'], entry['id']): entry for entry in current['inventory']} + unchanged = {(item['family'], item['id']): (item['document'], entries[(item['family'], item['id'])]) + for item in physical} + candidates, logical_out = _physical_candidates(records, loaded, physical) + prepared = _prepared_records(candidates, unchanged) + if logical_out is None: + logical_out = [dict(family=f, id=i, document=d) for f, i, d, _, _ in prepared] + digest_rows = _RecordsDigest() + for family, identity, _document, raw, entry in prepared: + if raw is None: + raw = self._verified_record_bytes(entry) + digest_rows.add(family, identity, raw) + records_digest = digest_rows.hexdigest() for receipt in current['receipts']: if receipt['request_key'] == request_key: if receipt['request_sha256'] != request_sha256 or receipt['records_sha256'] != records_digest: @@ -858,13 +1185,16 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256): _fail('Expected project checkpoint is stale.', 'stale', 409) manifest = copy.deepcopy(current) inventory = [] - for family, identity, document, raw in prepared: + for family, identity, document, raw, entry in prepared: + if raw is None: + inventory.append(copy.deepcopy(entry)) + continue digest = _sha(raw) inventory.append(dict(family=family, id=identity, format=document['format'], namespace=current['project_id'], path=f'records/{family}/{identity}/{digest}.json', bytes=len(raw), sha256=digest)) manifest['inventory'] = inventory - manifest['roots'] = [dict(family=f, id=i) for f, i, _, _ in prepared] + manifest['roots'] = [dict(family=f, id=i) for f, i, _, _, _ in prepared] manifest['receipts'].append(dict(request_key=request_key, request_sha256=request_sha256, records_sha256=records_digest)) del manifest['checkpoint'] @@ -872,11 +1202,20 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256): # Refuse before record staging. The final publication also # rechecks this bound after the immutable files are durable. _manifest_raw(manifest) - for family, identity, document, raw in prepared: - self._record_file(family, identity, _sha(raw), raw) - self._publish(manifest) + for family, identity, document, raw, _entry in prepared: + if raw is not None: + self._record_file(family, identity, _sha(raw), raw) + published = self._publish(manifest) self._manifest = manifest - return self._snapshot(manifest, normalized) + if self._canonical_digests is not None: + # Freshly written revisions are canonical by construction. + self._canonical_digests.add_many( + [_sha(raw) for _f, _i, _d, raw, _e in prepared if raw is not None]) + # Every committed revision is either freshly written and read + # back byte-for-byte or an unchanged verified revision. + self._read_cache = (published, manifest, logical_out, + [dict(family=f, id=i, document=d) for f, i, d, _, _ in prepared]) + return self._snapshot(manifest, logical_out) except OSError as exc: _io_error(exc, 'commit project records') @@ -887,8 +1226,12 @@ def save(self, expected_checkpoint): if expected_checkpoint != snapshot['checkpoint']: _fail('Expected project checkpoint is stale.', 'stale', 409) try: + # Save is the explicit durability and integrity boundary: sync and + # re-verify every referenced canonical revision's actual bytes. for entry in self._manifest['inventory']: - self._record_file(entry['family'], entry['id'], entry['sha256'], sync=True) + raw = self._record_file(entry['family'], entry['id'], entry['sha256'], sync=True) + if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: + _fail('Canonical record bytes changed or are corrupt.', 'stale', 409) _sync_file(self._meta_fd, 'project.json') os.fsync(self._meta_fd) os.fsync(self._root_fd) diff --git a/project_study_execution.py b/project_study_execution.py index 2133bbce..af7ed930 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -19,6 +19,7 @@ from project_material import ProjectMaterial, MaterialSnapshot from project_session import operation_records from workbench import WorkbenchError, encode, timestamp +from project_session import session_operations, session_records PREPARATIONS={'study_execution.prepare_package':'studies.prepare_package', 'study_execution.prepare_verifier':'studies.prepare_verifier'} @@ -323,13 +324,30 @@ def __init__(self,projection,execution): self.history.study_execution=self def _canonical(self,records=None): + if records is None and callable(getattr(self.session,'derived',None)): + from project_history import checkpoint_facts + return checkpoint_facts(self.session,self.materials) records=self.session.snapshot['records'] if records is None else records return history_records(records,materials=self.materials.read_view(records)) + def _authoring_facts(self,records=None): + """Studies and Trials of the bound checkpoint for SQL authoring. + + For a checkpoint whose complete validation is on record, only the Study + and Trial documents are needed and collected (no Run history); this is + exactly the studies/trials part of the complete decode. + """ + if records is None: + validated=getattr(self.session,'validated_checkpoint',None) + if validated is not None and validated==self.session.summary['checkpoint']: + from project_history import assemble_validated_facts + return assemble_validated_facts(session_records(self.session,'study_history','trial_history'),owned=True) + return self._canonical(records) + def restore_authoring(self,records=None): """Reconstruct intent only, under the current project guard.""" with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check_current();facts=self._canonical(records) + self.session.check_current();facts=self._authoring_facts(records) with self.db.transaction(): for (origin,identity),doc in sorted(facts.studies.items()): if origin!=self.origin:continue @@ -521,7 +539,7 @@ def continue_study(self,study_id,body,expected_checkpoint,key): self.access.require('execute','studies.continue',study_id,owner_only=True) with self.session.access.lock,self.session.lock,self.db.lock: self.refresh_authoring() - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is not None and not self.epoch_current(self.studies._study(study_id)): _invalid('A historical Continue receipt cannot activate this runtime; use a fresh key.','unavailable') def invoke(): @@ -567,7 +585,7 @@ def approve(self,study_id,body,expected_checkpoint,key): current=self.access.require('approve-execution','studies.approve',study_id,owner_only=True) effective=dict(body,approver_actor_id=current.actor_id) with self.session.access.lock,self.session.lock,self.db.lock: - self.refresh_authoring();prior=operation_records(self.session.snapshot['records']).get(key) + self.refresh_authoring();prior=session_operations(self.session).get(key) if prior is not None: if prior['operation']!='study_execution.approve':_invalid('Request key has another operation.','conflict') validate_operation_result(prior['operation'],prior['result'],self.session.snapshot['records']) @@ -590,7 +608,7 @@ def admit(self,study_id,body,expected_checkpoint,key): with self.session.access.lock,self.session.lock,self.db.lock: builtin=None self.session.check_current();self.refresh_authoring() - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is not None: result=prior['result'] if (prior['operation']!='study_execution.admit' or self.db.execute( @@ -735,14 +753,14 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): intent_key=_intent_key(operation,key);phase=intent_key.removesuffix('-intent') with self.session.access.lock,self.session.lock,self.db.lock: self.session.check_current() - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is not None: # Current authority is checked above. Historical completion # acknowledges science and cannot repeat filesystem effects. return self._mutate(operation,study_id,effective,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Completed preparation replay invoked effects.')),'edit') if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') - receipts=operation_records(self.session.snapshot['records']) + receipts=session_operations(self.session) if any(_intent_key(other,key) in receipts for other in PREPARATIONS if other!=operation): _invalid('Preparation key already identifies another preparation operation.','conflict') previous_intent=receipts.get(intent_key) diff --git a/project_transfer.py b/project_transfer.py index 91848559..2fb1d3ec 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -703,7 +703,8 @@ def replay(opened): if opened_store is not None: # Trusted controller-only capability. Its caller pins workspace, # authority, selected session and SQL ownership through this replay. - if not isinstance(opened_store,ProjectStore) or existing is None: + from project_working_store import WorkingProjectStore + if not isinstance(opened_store,(ProjectStore,WorkingProjectStore)) or existing is None: fail('Current selected destination is unavailable.','stale',409) with opened_store.root_directory() as owned_fd: try:target_fd=os.open(directory,DIR_FLAGS,dir_fd=destination_parent_fd) diff --git a/project_validation_cache.py b/project_validation_cache.py new file mode 100644 index 00000000..4d06760e --- /dev/null +++ b/project_validation_cache.py @@ -0,0 +1,155 @@ +"""Remember complete domain validation of exact project checkpoints. + +A project checkpoint digest covers its manifest, which names every canonical +record revision by length and SHA256. Records are verified against those +digests before use, so (project ID, checkpoint) identifies the exact record set. +Complete cross-domain validation of that set is a pure function of those +records and the validating source code, so an earlier success can be reused +when both are identical. + +Markers live only in the private local workspace control directory; a portable +project folder never carries validation authority. The validator fingerprint +covers every Tuldok Python module and extension schema plus the optional +validator package versions, so any code or dependency change revalidates. +""" +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path +import re +import sys +import threading +import uuid + +FORMAT = 'project_validation_marker_v1' +_HEX32 = re.compile(r'^[a-f0-9]{32}$') +_HEX64 = re.compile(r'^[a-f0-9]{64}$') +_SOURCE = Path(__file__).resolve().parent +_fingerprint = None +_fingerprint_lock = threading.Lock() + + +def _package_version(name): + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def validator_fingerprint(): + """SHA256 over validating source bytes, schemas and optional validators.""" + global _fingerprint + with _fingerprint_lock: + if _fingerprint is None: + digest = hashlib.sha256(b'tuldok-validator-fingerprint-v1\0') + paths = sorted(path for path in _SOURCE.glob('*.py')) + schemas = _SOURCE / 'extension_schemas' + if schemas.is_dir(): + paths += sorted(path for path in schemas.rglob('*') if path.is_file()) + for path in paths: + digest.update(str(path.relative_to(_SOURCE)).encode('utf-8') + b'\0') + digest.update(hashlib.sha256(path.read_bytes()).digest()) + environment = dict(python=sys.version, jsonschema=_package_version('jsonschema'), + referencing=_package_version('referencing')) + digest.update(json.dumps(environment, sort_keys=True).encode('utf-8')) + _fingerprint = digest.hexdigest() + return _fingerprint + + +class ProjectValidationCache: + """Private per-workspace markers of fully validated project checkpoints.""" + def __init__(self, control_path, *, fingerprint=None): + self.path = Path(control_path) / '.project-validation' + self.fingerprint = fingerprint or validator_fingerprint() + + def _marker(self, project_id, checkpoint): + if (type(project_id) is not str or not _HEX32.fullmatch(project_id) + or type(checkpoint) is not str or not _HEX64.fullmatch(checkpoint)): + raise ValueError('Invalid project validation identity.') + name = hashlib.sha256(f'{project_id}:{checkpoint}:{self.fingerprint}'.encode()).hexdigest() + return self.path / f'{name}.json', dict(format=FORMAT, project_id=project_id, + checkpoint=checkpoint, fingerprint=self.fingerprint) + + def known(self, project_id, checkpoint): + path, expected = self._marker(project_id, checkpoint) + try: + with open(path, 'rb') as stream: + return json.loads(stream.read(4096)) == expected + except (OSError, ValueError): + return False + + def remember(self, project_id, checkpoint): + """Best effort: a missing marker only means the next Open revalidates.""" + path, marker = self._marker(project_id, checkpoint) + try: + self.path.mkdir(mode=0o700, exist_ok=True) + temporary = self.path / f'.{uuid.uuid4().hex}.tmp' + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + os.write(descriptor, json.dumps(marker, sort_keys=True).encode('utf-8')) + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, path) + except OSError: + pass + + +class CanonicalDigestCache: + """Private set of record digests whose bytes were proven canonical JSON. + + Canonicality is a property of the bytes alone, and record files are named + by their SHA256, so a proof for one digest holds for every project that + contains those exact bytes. The set is versioned by the canonical codec's + own source, so any change to the encoding rules starts a fresh set. + """ + def __init__(self, control_path): + import inspect + import project_store + codec = ''.join(inspect.getsource(function) for function in ( + project_store._json_value, project_store._encode, project_store._pairs, project_store._decode)) + version = hashlib.sha256((codec + json.__version__).encode('utf-8')).hexdigest()[:32] + self.path = Path(control_path) / '.canonical-digests' / f'{version}.bin' + self._known = None + self._lock = threading.Lock() + + def _load(self): + if self._known is None: + known = set() + try: + with open(self.path, 'rb') as stream: + raw = stream.read() + if len(raw) % 32 == 0: + known = {raw[index:index + 32] for index in range(0, len(raw), 32)} + except OSError: + pass + self._known = known + return self._known + + def __contains__(self, digest): + with self._lock: + return bytes.fromhex(digest) in self._load() + + def add_many(self, digests): + """Best effort: a lost update only means those records are re-proven.""" + with self._lock: + known = self._load() + fresh = {bytes.fromhex(digest) for digest in digests} - known + if not fresh: + return + known |= fresh + try: + self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = self.path.parent / f'.{uuid.uuid4().hex}.tmp' + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + remaining = memoryview(b''.join(sorted(known))) + while remaining: + remaining = remaining[os.write(descriptor, remaining):] + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, self.path) + except OSError: + pass diff --git a/project_validation_delta.py b/project_validation_delta.py new file mode 100644 index 00000000..d63814c5 --- /dev/null +++ b/project_validation_delta.py @@ -0,0 +1,119 @@ +"""Edit-scoped validation: re-check only what a mutation can affect. + +A mutation's candidate is the previous, completely validated checkpoint plus +changed/added records (no deletions) and its new operation receipt. Each +validation component of validate_project_records is a pure function of the +documents and identities of a known set of record families (see the audit in +docs/plans/database-working-store.md, stage 4). A component whose input +families contain no changed record has the same outcome as for the previous +checkpoint, so it is skipped. Old operation receipts are re-checked only when +records they bind to changed or a new receipt shares their key space. + +Anything not classified here is always validated. TULDOK_VALIDATION_CROSSCHECK=1 +additionally runs complete validation and requires the same outcome. +""" +import contextvars +from contextlib import contextmanager +from dataclasses import dataclass + +from project_data import FAMILIES as _DATA +from project_history import FAMILY_FORMATS as _HISTORY +from project_material import FAMILIES as _MATERIAL +from project_producers import FAMILIES as _PRODUCERS + +DATA = frozenset(_DATA) +HISTORY = frozenset(_HISTORY) +MATERIAL = frozenset(_MATERIAL) +PRODUCERS = frozenset(_PRODUCERS) + +# Families whose documents or identities each component reads. +COMPONENTS = { + 'data': DATA | {'workbench_text', 'project_artifact', 'project_fork'} | PRODUCERS, + 'fork': frozenset({'project_fork'}), + 'composition': frozenset({'project_composition', 'project_fork', 'run_observation'}), + 'extraction': frozenset({'project_extraction', 'project_fork', 'project_snapshot', 'project_file', 'project_artifact'}), + 'files': frozenset({'project_file', 'project_fork'}), + 'native': frozenset({'native_execution_history', 'project_artifact', 'run_history'}), + 'material': MATERIAL | {'project_fork', 'project_artifact'}, + 'producers': PRODUCERS | {'project_artifact', 'workbench_record', 'workbench_text'}, + 'history': HISTORY | MATERIAL | {'project_artifact', 'dataset_exposure', 'pipeline_graph'}, + 'measurements': frozenset({'measurement_source', 'project_artifact', 'project_snapshot'}), + 'sources': frozenset({'project_snapshot'}), +} + +# Operation-receipt prefixes -> families their result binds to. A prefix +# mapped to an empty set binds only immutable or append-only facts. +RECEIPT_BINDINGS = { + 'data.': DATA | {'workbench_text'}, + 'models.': frozenset(), 'graphs.': frozenset(), 'recipes.': frozenset(), + 'studies.': HISTORY | {'pipeline_graph', 'project_artifact'}, + 'history.disclose': HISTORY | {'pipeline_graph', 'project_artifact'}, + 'source.': frozenset(), + 'measurements.': frozenset({'measurement_source'}), + 'runs.': frozenset({'run_history'}), + 'producers.': frozenset(), + 'files.': frozenset(), + 'study_execution.': HISTORY | MATERIAL, + 'runtime.': frozenset({'runtime_material', 'project_artifact'}), + 'native_preparation.': MATERIAL | {'pipeline_graph', 'dataset_release'}, + 'artifacts.resolve': frozenset(), +} +# A new receipt in these key spaces can change the meaning of older ones. +SHARED_KEY_SPACES = ('study_execution.', 'runtime.') +# Receipts these prefixes claim are also validated inside history decoding. +HISTORY_RECEIPTS = ('studies.', 'history.disclose') + + +def _prefix(operation): + return next((prefix for prefix in RECEIPT_BINDINGS if operation.startswith(prefix)), None) + + +@dataclass(frozen=True) +class ValidationDelta: + """Identities a mutation changed relative to its validated base checkpoint.""" + changed: frozenset + operations: frozenset = frozenset() # operation names of new/changed receipts + added: frozenset = frozenset() # changed identities absent from the base + + @property + def families(self): + # A newly added artifact declaration cannot change any component: an + # unchanged record referencing an undeclared blob already failed base + # validation. New declarations are still checked by the claim loop. + return frozenset(family for family, identity in self.changed + if not (family == 'project_artifact' and (family, identity) in self.added)) + + def touches(self, component): + return bool(self.families & COMPONENTS[component]) + + def history_receipt_added(self): + return any(operation.startswith(HISTORY_RECEIPTS) for operation in self.operations) + + def receipt_needed(self, operation, identity): + if identity in self.changed: + return True + prefix = _prefix(operation) + if prefix is None: + return True + if self.families & RECEIPT_BINDINGS[prefix]: + return True + if prefix in SHARED_KEY_SPACES and any(op.startswith(prefix) for op in self.operations): + return True + return False + + +_DELTA = contextvars.ContextVar('validation_delta', default=None) + + +@contextmanager +def edit_delta(delta): + """Validate the enclosed candidate relative to its validated base checkpoint.""" + token = _DELTA.set(delta) + try: + yield + finally: + _DELTA.reset(token) + + +def current_delta(): + return _DELTA.get() diff --git a/project_working_store.py b/project_working_store.py new file mode 100644 index 00000000..03d35f5e --- /dev/null +++ b/project_working_store.py @@ -0,0 +1,634 @@ +"""Persistent PostgreSQL working copy of one external project folder. + +The database holds the current record set of an opened project and is kept +across sessions. The external folder (`.tuldok/project.json` plus +content-addressed `records/`, tracked in Git) is an import source, read only +when its checkpoint changed since Tuldok last imported or saved it, and an +export target written only by save(). See docs/plans/database-working-store.md. + +Records are kept exactly as their canonical bytes with their SHA256, so the +working copy has the same manifest, checkpoint digest and receipts as the file +format and save() writes byte-identical files. This store keeps no record state +in memory: inventory, receipts and bodies are read from PostgreSQL, so an outer +transaction rollback can never leave a stale in-process copy. + +Interface parity with ProjectStore (summary, snapshot, commit, save, +reconcile, root_directory, verify_records, recover_checkpoint, close, path) +lets Session, owners, artifacts and custody use either store. +""" +from contextlib import contextmanager, nullcontext +import json +import os +from pathlib import Path +import re +import threading +import uuid + +import psycopg +from psycopg import sql + +from metadata import _key, connection_info +from project_record_dedupe import CLOSURE_FAMILY, RUN_FAMILY, compact, expand, is_compact +from project_store import (FORMAT, MAX_RECORD_BYTES, MAX_RECORDS, MAX_TOTAL_BYTES, SUPPORTED_FORMATS, + ProjectStoreError, _RecordsDigest, _decode, _digest, _domain_format, _encode, + _fail, _identifier, _keys, _manifest_raw, _request, _sha, envelope_digest) + +WORKING_ID = re.compile(r'^[a-f0-9]{32}$') +_TABLES = ( + '''CREATE TABLE IF NOT EXISTS {schema}.state ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + project_id text NOT NULL, name text NOT NULL, checkpoint text NOT NULL, + receipts bytea NOT NULL, external_sha256 text, dirty boolean NOT NULL)''', + '''CREATE TABLE IF NOT EXISTS {schema}.records ( + family text COLLATE "C" NOT NULL, id text COLLATE "C" NOT NULL, + format text NOT NULL, bytes integer NOT NULL, sha256 text NOT NULL, + raw bytea NOT NULL, PRIMARY KEY (family, id))''', +) +_MIGRATIONS = ( + # SHA256 of the canonical {family, id, document} envelope (see envelope_digest). + 'ALTER TABLE {schema}.records ADD COLUMN IF NOT EXISTS envelope_sha256 text', + # Shared runtime closure referenced by a compact Run row (project_record_dedupe). + 'ALTER TABLE {schema}.records ADD COLUMN IF NOT EXISTS closure_id text', +) +_ORDER = sql.SQL('ORDER BY family COLLATE "C", id COLLATE "C"') + + +class WorkingProjectStore: + """One exclusive owner of a project's database working copy.""" + + def __init__(self, folder, working_id, *, dsn=None, canonical_digests=None): + if type(working_id) is not str or not WORKING_ID.fullmatch(working_id): + raise ValueError('Invalid working copy identity.') + self._folder = folder + self.working_id = working_id + self.schema = 'working_' + working_id + self._canonical_digests = canonical_digests + self._mutex = threading.RLock() + self._db = None + self._closed = False + try: + self._own = psycopg.connect(connection_info(dsn), autocommit=True) + except psycopg.Error: + raise ProjectStoreError('Local PostgreSQL is unavailable.', 'unavailable', 503) from None + try: + # One Tuldok process owns a working copy at a time. + if not self._own.execute('SELECT pg_try_advisory_lock(%s)', (_key(self.schema),)).fetchone()[0]: + raise ProjectStoreError('This project is already open in another Tuldok session.', 'busy', 409) + with self._own.transaction(): + self._own.execute(sql.SQL('CREATE SCHEMA IF NOT EXISTS {}').format(sql.Identifier(self.schema))) + for statement in (*_TABLES, *_MIGRATIONS): + self._own.execute(sql.SQL(statement).format(schema=sql.Identifier(self.schema))) + except BaseException: + self._own.close() + raise + + # ---- Connection: own until bound to the projection, then shared so + # record commits are atomic with the projection's SQL writes. + + def bind(self, db): + """Use the projection database's connection and transactions.""" + with self._locked(): + self._db = db + + @contextmanager + def _locked(self): + """Store ownership in one global order: projection lock, then this mutex. + + A bound projection connection is shared with Session/owner threads that + hold the projection lock across their transactions, so store work never + takes its own mutex first and then waits for that lock. + """ + if self._db is not None: + with self._catalog_lock(), self._mutex: + yield + else: + with self._mutex: + yield + + def _catalog_lock(self): + # The projection may install a composite host -> session -> catalog + # lock; the store is innermost, so it takes only the catalog lock. + lock = self._db.lock + return getattr(lock, 'projection', lock) + + def _joined(self): + # Join the projection only inside its open transaction (Session.mutate, + # background publication), so record and projection rows commit + # atomically. Other commits use this store's own connection and never + # fire the projection's commit hooks. + return self._db is not None and self._db.in_transaction + + def _connection(self): + return self._db.connection if self._joined() else self._own + + def _guard(self): + # A bound projection connection is shared; its catalog lock serializes + # every command and transaction on it across threads. + return self._catalog_lock() if self._joined() else nullcontext() + + @contextmanager + def _transaction(self): + if self._joined(): + with self._catalog_lock(), self._db.transaction(): + yield + else: + with self._own.transaction(): + yield + + def _execute(self, statement, parameters=None): + if self._closed: + _fail('Project store is closed.', 'unavailable', 409) + with self._guard(): + return self._connection().execute(statement, parameters) + + def _q(self, text): + return sql.SQL(text).format(state=sql.Identifier(self.schema, 'state'), + records=sql.Identifier(self.schema, 'records'), order=_ORDER) + + # ---- Open: attach, import or refuse. + + @classmethod + def open(cls, folder, working_id, *, dsn=None, canonical_digests=None): + """Attach the working copy, importing the folder only if it changed.""" + store = cls(folder, working_id, dsn=dsn, canonical_digests=canonical_digests) + try: + raw, manifest = folder.folder_checkpoint() + external = _sha(raw) + state = store._state(required=False) + if state is not None and state['project_id'] != manifest['project_id']: + _fail('This working copy belongs to a different project.', 'conflict', 409) + if state is None or state['external_sha256'] != external: + if state is not None and state['dirty']: + _fail('The external project changed while this workspace has unsaved changes. ' + 'Save over it or discard the working copy and import again.', 'conflict', 409) + store._import(raw, manifest) + return store + except BaseException: + store.close() + raise + + def _state(self, *, required=True): + row = self._execute(self._q('SELECT project_id, name, checkpoint, receipts, external_sha256, dirty FROM {state}')).fetchone() + if row is None: + if required: + _fail('Project working copy is missing.', 'unavailable', 409) + return None + return dict(project_id=row[0], name=row[1], checkpoint=row[2], receipts=json.loads(bytes(row[3])), + external_sha256=row[4], dirty=row[5]) + + def _import(self, raw, manifest): + """Bring the working copy to the folder's checkpoint, reading only new revisions.""" + with self._locked(), self._transaction(): + existing = {(row[0], row[1]): row[2] for row in + self._execute(self._q('SELECT family, id, sha256 FROM {records}'))} + wanted = {} + proven = [] + known = self._canonical_digests + for entry in manifest['inventory']: + identity = (entry['family'], entry['id']) + wanted[identity] = entry + if existing.get(identity) == entry['sha256']: + continue + body = self._folder.folder_record(entry) + if known is not None and entry['sha256'] in known: + document = json.loads(body) + else: + document = _decode(body) + proven.append(entry['sha256']) + if type(document) is not dict or document.get('format') != entry['format']: + _fail('Domain document format disagrees with its inventory.') + compact_run = is_compact(entry['family'], document) + self._execute(self._q( + 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256, closure_id) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ' + 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' + 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256, ' + 'closure_id=EXCLUDED.closure_id'), + (entry['family'], entry['id'], entry['format'], entry['bytes'], entry['sha256'], body, + # A compact Run's logical envelope digest is filled from its expansion. + None if compact_run or entry['family'] == CLOSURE_FAMILY else envelope_digest(entry['family'], entry['id'], body), + document['runtime_closure']['id'] if compact_run else None)) + for family, record_id in set(existing) - set(wanted): + self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, record_id)) + if self._records_digest() != manifest['receipts'][-1]['records_sha256']: + _fail('Latest operation receipt disagrees with the current record set.') + self._write_state(manifest, external_sha256=_sha(raw), dirty=False) + if known is not None and proven: + known.add_many(proven) + + def _write_state(self, manifest, *, external_sha256, dirty): + receipts = _encode(manifest['receipts']) + self._execute(self._q('DELETE FROM {state}')) + self._execute(self._q('INSERT INTO {state} (project_id, name, checkpoint, receipts, external_sha256, dirty) ' + 'VALUES (%s,%s,%s,%s,%s,%s)'), + (manifest['project_id'], manifest['name'], manifest['checkpoint'], receipts, + external_sha256, dirty)) + + def _rows(self, statement, parameters=None): + """Stream (family, id, raw) rows through a server-side cursor. + + Bodies are fetched in small batches, never the whole record set at once. + """ + if self._closed: + _fail('Project store is closed.', 'unavailable', 409) + with self._guard(): + connection = self._connection() + idle = connection.info.transaction_status == psycopg.pq.TransactionStatus.IDLE + with (connection.transaction() if idle else nullcontext()), \ + connection.cursor(name='working_' + uuid.uuid4().hex) as cursor: + cursor.itersize = 64 + cursor.execute(statement, parameters) + for row in cursor: + yield row[0], row[1], bytes(row[2]) + + def _records_digest(self, replacements=None, identities=None): + """Streamed receipt digest over the stored (or a candidate) record set.""" + digest = _RecordsDigest() + replacements = replacements or {} + if identities is None: + for family, record_id, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}')): + digest.add(family, record_id, raw) + return digest.hexdigest() + stored = self._rows(self._q('SELECT family, id, raw FROM {records} {order}')) + pending = next(stored, None) + for identity in identities: + while pending is not None and (pending[0], pending[1]) < identity: + pending = next(stored, None) + if identity in replacements: + digest.add(identity[0], identity[1], replacements[identity]) + elif pending is not None and (pending[0], pending[1]) == identity: + digest.add(identity[0], identity[1], pending[2]) + else: + _fail('Unchanged candidate record is missing from the working copy.') + return digest.hexdigest() + + # ---- Logical documents: closures hidden, compact Runs expanded. + + def _closure(self, identity): + cache = self.__dict__.setdefault('_closures', {}) + if identity not in cache: + row = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), + (CLOSURE_FAMILY, identity)).fetchone() + if row is None: + _fail('Compact Run references a missing runtime closure.') + cache[identity] = json.loads(bytes(row[0])) + return cache[identity] + + def _logical(self, family, raw): + document = json.loads(raw) + if is_compact(family, document): + document = expand(document, self._closure(document['runtime_closure']['id'])) + return document + + # ---- Manifest identity, identical to the file format. + + def _inventory(self): + return [(row[0], row[1], row[2], row[3], row[4]) for row in + self._execute(self._q('SELECT family, id, format, bytes, sha256 FROM {records} {order}'))] + + @staticmethod + def _manifest(state, inventory): + manifest = dict(format=FORMAT, project_id=state['project_id'], name=state['name'], + roots=[dict(family=f, id=i) for f, i, _fmt, _b, _s in inventory], + inventory=[dict(family=f, id=i, format=fmt, namespace=state['project_id'], + path=f'records/{f}/{i}/{sha}.json', bytes=size, sha256=sha) + for f, i, fmt, size, sha in inventory], + receipts=state['receipts']) + manifest['checkpoint'] = _sha(_encode(manifest)) + return manifest + + # ---- ProjectStore interface. + + @property + def path(self): + return self._folder.path + + @property + def _root_fd(self): + """The external folder's locked physical root (read-only).""" + return self._folder._root_fd + + def summary(self): + with self._locked(): + state = self._state() + return {key: state[key] for key in ('project_id', 'name', 'checkpoint')} + + def manifest_raw(self): + """Canonical project.json bytes of the working copy's current checkpoint.""" + with self._locked(): + return _manifest_raw(self._manifest(self._state(), self._inventory())) + + def dirty(self): + """Whether the working copy has changes not yet saved to the folder.""" + with self._locked(): + return self._state()['dirty'] + + def snapshot(self): + """Complete independent record set (costly for large projects).""" + with self._locked(): + state = self._state() + records = [dict(family=f, id=i, document=self._logical(f, raw)) for f, i, raw in self._rows( + self._q('SELECT family, id, raw FROM {records} WHERE family <> %s {order}'), (CLOSURE_FAMILY,))] + return dict(project_id=state['project_id'], name=state['name'], + checkpoint=state['checkpoint'], records=records) + + def records(self, *families): + """Independent decoded envelopes of the selected families, in record order.""" + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + with self._locked(): + selected = [family for family in families if family != CLOSURE_FAMILY] + return [dict(family=f, id=i, document=self._logical(f, raw)) for f, i, raw in self._rows( + self._q('SELECT family, id, raw FROM {records} WHERE family = ANY(%s) {order}'), (selected,))] + + def envelope_digests(self): + """(family, id) -> canonical envelope SHA256 for every stored revision. + + Rows imported before this column existed are filled once from their + stored canonical bytes. + """ + with self._locked(): + missing = self._execute(self._q('SELECT count(*) FROM {records} WHERE envelope_sha256 IS NULL AND family <> %s'), + (CLOSURE_FAMILY,)).fetchone()[0] + if missing: + with self._transaction(): + # Logical envelopes: a compact Run hashes its exact expansion. + digests = [(envelope_digest(family, record_id, _encode(self._logical(family, raw)) + if family == RUN_FAMILY else raw), family, record_id) + for family, record_id, raw in self._rows(self._q( + 'SELECT family, id, raw FROM {records} WHERE envelope_sha256 IS NULL AND family <> %s {order}'), + (CLOSURE_FAMILY,))] + with self._guard(): + with self._connection().cursor() as cursor: + cursor.executemany(self._q('UPDATE {records} SET envelope_sha256=%s WHERE family=%s AND id=%s'), + digests) + return {(row[0], row[1]): row[2] for row in + self._execute(self._q('SELECT family, id, envelope_sha256 FROM {records} WHERE family <> %s'), + (CLOSURE_FAMILY,))} + + def records_view(self, *families): + """Every record identity in order; documents decoded only for selected families. + + Unselected envelopes carry document None. Fork and membership + validators need every identity but only read their own families. + """ + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + with self._locked(): + connection = self._connection() + idle = connection.info.transaction_status == psycopg.pq.TransactionStatus.IDLE + result = [] + with (connection.transaction() if idle else nullcontext()), \ + connection.cursor(name='working_' + uuid.uuid4().hex) as cursor: + cursor.itersize = 256 + cursor.execute(self._q('SELECT family, id, CASE WHEN family = ANY(%s) THEN raw END ' + 'FROM {records} WHERE family <> %s {order}'), (list(families), CLOSURE_FAMILY)) + rows = [(family, record_id, None if raw is None else bytes(raw)) for family, record_id, raw in cursor] + for family, record_id, raw in rows: + result.append(dict(family=family, id=record_id, + document=None if raw is None else self._logical(family, raw))) + return result + + def record(self, family, record_id): + if family == CLOSURE_FAMILY: + return None + with self._locked(): + row = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), + (family, record_id)).fetchone() + return None if row is None else dict(family=family, id=record_id, document=self._logical(family, bytes(row[0]))) + + def reconcile(self, request_key, request_sha256): + _request(request_key, request_sha256) + with self._locked(): + for receipt in self._state()['receipts']: + if receipt['request_key'] == request_key: + if receipt['request_sha256'] != request_sha256: + _fail('Request key was already used with different input.', 'conflict', 409) + return self.snapshot() + return None + + def commit(self, records, expected_checkpoint, request_key, request_sha256, *, changed=None): + """Replace the complete current record set in the working copy. + + changed, when given, names every (family, id) whose document may differ + from the stored revision; all other candidate records are the caller's + unmodified copies of stored revisions (Session.mutate constructs them + so). Without it every candidate is compared exactly with its revision. + Returns the committed snapshot, sharing the caller's record list. + """ + _request(request_key, request_sha256) + _digest(expected_checkpoint, 'Expected checkpoint') + if type(records) is not list or len(records) > MAX_RECORDS: + _fail('Project records must be a bounded list.') + with self._locked(), self._transaction(): + state = self._state() + inventory = {(f, i): (fmt, size, sha) for f, i, fmt, size, sha in self._inventory()} + candidates = {} + for record in records: + _keys(record, ('family', 'id', 'document'), 'Project record') + family, identity, document = record['family'], record['id'], record['document'] + _identifier(family, 'Record family') + if family not in SUPPORTED_FORMATS: + _fail('Unsupported project record family.', 'unsupported', 422) + if family == CLOSURE_FAMILY: + _fail('Runtime closures are store-owned records.', 'unsupported', 422) + _identifier(identity, 'Record ID') + if (family, identity) in candidates: + _fail('Duplicate project record identity.') + if type(document) is not dict: + _fail('Domain document must be an object.') + _domain_format(document.get('format'), family) + candidates[(family, identity)] = document + replacements, entries, total = {}, {}, 0 + closure_rows = {row[1]: row[2] for row in self._execute(self._q( + 'SELECT family, id, closure_id FROM {records} WHERE closure_id IS NOT NULL'))} + new_closures, row_closures, logical_raw, referenced = {}, {}, {}, set() + for identity in sorted(candidates): + document = candidates[identity] + current = inventory.get(identity) + if current is not None and changed is not None and identity not in changed \ + and current[0] in (document['format'], 'project_run_history_v2'): + entries[identity] = current + total += current[1] + if identity[1] in closure_rows and identity[0] == RUN_FAMILY: + referenced.add(closure_rows[identity[1]]) + continue + physical = document + if (identity[0] == RUN_FAMILY and changed is None and current is not None + and current[0] == document['format'] and _sha(_encode(document)) == current[2]): + # An unchanged uncompacted Run keeps its verified revision (as in + # ProjectStore); compact_run_closures migrates explicitly. + entries[identity] = current + total += current[1] + continue + if identity[0] == RUN_FAMILY: + compacted = compact(document) + if compacted is not None: + physical, closure = compacted + closure_id = physical['runtime_closure']['id'] + new_closures.setdefault(closure_id, closure) + referenced.add(closure_id) + row_closures[identity] = closure_id + logical_raw[identity] = _encode(document) + raw = _encode(physical) + if len(raw) > MAX_RECORD_BYTES: + _fail('Project records exceed the synchronous byte limit.') + digest = _sha(raw) + if current is None or current[2] != digest: + replacements[identity] = raw + entries[identity] = (physical['format'], len(raw), digest) + total += len(raw) + if total > MAX_TOTAL_BYTES: + _fail('Project records exceed the synchronous byte limit.') + # Exactly the closures referenced by stored compact Runs are kept. + for closure_id in sorted(referenced): + identity = (CLOSURE_FAMILY, closure_id) + if identity in inventory: + entries[identity] = inventory[identity] + total += inventory[identity][1] + continue + raw = _encode(new_closures[closure_id]) + replacements[identity] = raw + entries[identity] = (new_closures[closure_id]['format'], len(raw), _sha(raw)) + total += len(raw) + if total > MAX_TOTAL_BYTES: + _fail('Project records exceed the synchronous byte limit.') + records_digest = self._records_digest(replacements, sorted(entries)) + for receipt in state['receipts']: + if receipt['request_key'] == request_key: + if receipt['request_sha256'] != request_sha256 or receipt['records_sha256'] != records_digest: + _fail('Request key was already used with different input.', 'conflict', 409) + return self.snapshot() + if expected_checkpoint != state['checkpoint']: + _fail('Expected project checkpoint is stale.', 'stale', 409) + receipts = state['receipts'] + [dict(request_key=request_key, request_sha256=request_sha256, + records_sha256=records_digest)] + manifest = self._manifest(dict(state, receipts=receipts), + [(f, i, *entries[(f, i)]) for f, i in sorted(entries)]) + _manifest_raw(manifest) + for (family, identity), raw in replacements.items(): + fmt, size, digest = entries[(family, identity)] + key = (family, identity) + envelope = (None if family == CLOSURE_FAMILY else + envelope_digest(family, identity, logical_raw.get(key, raw))) + self._execute(self._q( + 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256, closure_id) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ' + 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' + 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256, ' + 'closure_id=EXCLUDED.closure_id'), + (family, identity, fmt, size, digest, raw, envelope, row_closures.get(key))) + for family, identity in set(inventory) - set(entries): + self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, identity)) + self._write_state(manifest, external_sha256=state['external_sha256'], dirty=True) + if self._canonical_digests is not None and replacements: + self._canonical_digests.add_many([_sha(raw) for raw in replacements.values()]) + return dict(project_id=state['project_id'], name=state['name'], checkpoint=manifest['checkpoint'], + records=[dict(family=f, id=i, document=candidates[(f, i)]) for f, i in sorted(candidates)]) + + def compact_run_closures(self, request_key, request_sha256): + """Format migration: store every compactable Run against a shared closure. + + The logical record set is unchanged (each compaction is checked to expand + to the exact original bytes); only the physical records and checkpoint + change. Returns (previous checkpoint, committed snapshot summary). + """ + with self._locked(): + state = self._state() + uncompacted = {(row[0], row[1]) for row in self._execute(self._q( + "SELECT family, id FROM {records} WHERE family=%s AND format='project_run_history_v1'"), (RUN_FAMILY,))} + snapshot = self.snapshot() + committed = self.commit(snapshot['records'], state['checkpoint'], request_key, request_sha256, + changed=uncompacted) + return state['checkpoint'], {key: committed[key] for key in ('project_id', 'name', 'checkpoint')} + + def save(self, expected_checkpoint): + """Export the working copy to the external folder (the only folder write).""" + _digest(expected_checkpoint, 'Expected checkpoint') + with self._locked(): + state = self._state() + if expected_checkpoint != state['checkpoint']: + _fail('Expected project checkpoint is stale.', 'stale', 409) + try: + external_raw, _external = self._folder.folder_checkpoint() + except ProjectStoreError: + external_raw = None + if (state['external_sha256'] is not None and + (external_raw is None or _sha(external_raw) != state['external_sha256'])): + _fail('The external project changed since Tuldok last imported or saved it.', 'conflict', 409) + manifest = self._manifest(state, self._inventory()) + if manifest['checkpoint'] != state['checkpoint']: + _fail('Working copy checkpoint disagrees with its records.', 'invalid', 409) + raw = _manifest_raw(manifest) + if state['external_sha256'] != _sha(raw): + for entry in manifest['inventory']: + if not self._folder.folder_has_record(entry): + body = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), + (entry['family'], entry['id'])).fetchone() + body = bytes(body[0]) + if len(body) != entry['bytes'] or _sha(body) != entry['sha256']: + _fail('Working copy record bytes are corrupt.', 'invalid', 409) + self._folder.folder_write_record(entry['family'], entry['id'], entry['sha256'], body) + self._folder.folder_publish(raw, external_raw) + with self._transaction(): + self._write_state(manifest, external_sha256=_sha(raw), dirty=False) + return {key: state[key] for key in ('project_id', 'name', 'checkpoint')} + + def verify_records(self): + """Re-check every stored revision's bytes against its SHA256 and length.""" + with self._locked(): + inventory = {(f, i): (size, sha) for f, i, _fmt, size, sha in self._inventory()} + for family, record_id, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}')): + size, digest = inventory[(family, record_id)] + if len(raw) != size or _sha(raw) != digest: + _fail('Canonical record bytes changed or are corrupt.', 'stale', 409) + + def recover_checkpoint(self): + """Database commits are atomic; return the committed record set.""" + return self.snapshot() + + @contextmanager + def root_directory(self, *, verify_after=False): + """Borrow the external folder's physical root (payload artifacts live there).""" + if type(verify_after) is not bool: + _fail('Root read verification must be an explicit boolean.') + with self._locked(): + before = self._state()['checkpoint'] + descriptor = os.dup(self._folder._root_fd) + try: + yield descriptor + self._folder._binding() + if verify_after and self._state()['checkpoint'] != before: + _fail('Project checkpoint changed during the bounded root read.', 'stale', 409) + finally: + os.close(descriptor) + + def close(self): + # The projection (and its lock's connection) may already be closed. + with self._mutex: + if self._closed: + return + self._closed = True + try: + self._folder.close() + finally: + self._own.close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + +def registered_schemas(control_path): + """Working-copy schemas registered by a workspace control directory.""" + schemas = [] + for path in sorted(Path(control_path).glob('.project-working/*.json')): + try: + identity = json.loads(path.read_bytes()).get('working_id') + except (OSError, ValueError, AttributeError): + continue + if type(identity) is str and WORKING_ID.fullmatch(identity): + schemas.append('working_' + identity) + return schemas diff --git a/project_workspace.py b/project_workspace.py index b006dbe2..b6f61256 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -20,6 +20,8 @@ from project_store import ProjectStore, ProjectStoreError from project_projection import ProjectProjection from project_custody import ProjectCustody +from project_validation_cache import CanonicalDigestCache, ProjectValidationCache +from project_working_store import WORKING_ID, WorkingProjectStore from workbench import WorkbenchError @@ -83,6 +85,8 @@ def __init__(self, control_path, roots, *, dsn=None, runtime_sources=None): self.lock = self.db.lock self.access = Access(self.db, self.path) self.custody = ProjectCustody(self.path, self.db.dataset_id) + self.validation_cache = ProjectValidationCache(self.path) + self.canonical_digests = CanonicalDigestCache(self.path) self._retained_recovery = None self._dsn = dsn self._runtime_sources = configured @@ -203,16 +207,53 @@ def _root(self, alias, relative): assert_separate_project_control(root_fd, relative, root_path=root_path, control_path=self.path) return self._roots[alias] + def _working_identity(self, alias, directory, project_id): + """The persistent working copy for one project folder (private registry).""" + key = hashlib.sha256(f'{alias}\0{directory}'.encode('utf-8')).hexdigest() + registry = self.path / '.project-working' + path = registry / f'{key}.json' + try: + value = json.loads(path.read_bytes()) + if (value.get('project_id') == project_id and type(value.get('working_id')) is str + and WORKING_ID.fullmatch(value['working_id'])): + return value['working_id'] + except (OSError, ValueError, AttributeError): + pass + working_id = uuid.uuid4().hex + registry.mkdir(mode=0o700, exist_ok=True) + temporary = registry / f'.{uuid.uuid4().hex}.tmp' + temporary.write_bytes(json.dumps(dict(format='project_working_registration_v1', root=alias, + directory=directory, project_id=project_id, working_id=working_id), sort_keys=True).encode('utf-8')) + os.chmod(temporary, 0o600) + os.replace(temporary, path) + return working_id + + def _open_working(self, root_fd, root_path, alias, directory): + """Attach the folder's working copy, importing only if the folder changed.""" + folder = ProjectStore.open_folder_at(root_fd, directory, root_path=root_path) + try: + _raw, manifest = folder.folder_checkpoint() + working_id = self._working_identity(alias, directory, manifest['project_id']) + return WorkingProjectStore.open(folder, working_id, dsn=self._dsn, + canonical_digests=self.canonical_digests) + except BaseException: + folder.close() + raise + def _projection_for(self, store): identity = uuid.uuid4().hex path = self.path / '.project-projections' / identity snapshot = store.summary() projection = Metadata(path, dsn=self._dsn, dataset_id=snapshot['project_id'], projection_id=identity) + if hasattr(store, 'bind'): + # Record commits share the projection's transactions. + store.bind(projection) try: owner = ProjectProjection(store, projection, self.access, {alias: value[0] for alias, value in self._roots.items()}, custody=self.custody.for_projection(store, identity), - runtime_sources=self._runtime_sources) + runtime_sources=self._runtime_sources, + validation_cache=self.validation_cache) owner.pending_original = self.custody.pending(store) if owner.pending_original: owner.session.available = False @@ -228,11 +269,13 @@ def _retained_projection_for(self, store, projection_id): raise WorkbenchError('Invalid original local projection identity.', 'unavailable', 409) db = Metadata(self.path / '.project-projections' / projection_id, dsn=self._dsn, create=False, dataset_id=store.summary()['project_id'], projection_id=projection_id) + if hasattr(store, 'bind'): + store.bind(db) try: return ProjectProjection(store, db, self.access, {alias: value[0] for alias, value in self._roots.items()}, custody=self.custody.for_projection(store, projection_id), restore=False, - runtime_sources=self._runtime_sources) + runtime_sources=self._runtime_sources, validation_cache=self.validation_cache) except BaseException: db.close() raise @@ -241,9 +284,8 @@ def _detach(self): """Release only a project whose current local effects have drained.""" if self._projection is not None: self._projection.execution.assert_quiescent() - if self._store is not None: - # Prove pending project publications and their full closure first. - self._store.save(self._store.summary()['checkpoint']) + # Closing keeps unsaved work in the database working copy; only an + # explicit Save writes the external folder. if self._projection is not None: self._projection.close() if self._store is not None: @@ -271,13 +313,12 @@ def _recover_current(self, arguments): self._store = None store = projection = None try: - store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) + store = self._open_working(root_fd, root_path, *self._selection) snapshot = store.summary() if snapshot['project_id'] != self._last_project['project_id']: raise WorkbenchError('Recovery destination has a different project identity.', 'stale_project', 409) - # A reread reconciles identity; only a successful durability barrier - # resolves a prior checkpoint/directory fsync failure. - store.save(snapshot['checkpoint']) + # Database commits are atomic: reattaching the working copy + # reconciles the outcome without writing the external folder. projection, text = self._projection_for(store) snapshot = store.summary() if self._projection is not None: @@ -326,18 +367,22 @@ def _select(self, action, arguments): # unselected destination. Only its exact receipt authorizes # treating it as this New; unrelated folders remain conflicts. try: - store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) + store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path, canonical_digests=self.canonical_digests) except ProjectStoreError: raise error if store.reconcile(arguments['request_key'], digest) is None: + store.close() raise error reconciled = True snapshot = store.summary() if reconciled: store.save(snapshot['checkpoint']) created = snapshot + # The new folder becomes this workspace's working copy. + store.close() + store = self._open_working(root_fd, root_path, arguments['root'], arguments['directory']) else: - store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) + store = self._open_working(root_fd, root_path, arguments['root'], arguments['directory']) projection, text = self._projection_for(store) self._detach() self._store, self._projection, self._text = store, projection, text @@ -394,7 +439,6 @@ def _publish_independent(self, arguments, *, composed=False): source = self._projection source.execution.assert_quiescent() source.files.require_clean() - self._store.save(arguments['expected_checkpoint']) published = None incoming = projection = None retained = ExitStack() @@ -408,7 +452,7 @@ def _publish_independent(self, arguments, *, composed=False): raise WorkbenchError('Composition source identity repeats.','conflict',409) identities.add(selected['project_id']) directory,descriptor=self._root(selected['root'],selected['directory']) - store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory)) + store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory, canonical_digests=self.canonical_digests)) actual=store.summary() if actual['project_id']!=selected['project_id'] or actual['checkpoint']!=selected['checkpoint']: raise WorkbenchError('Selected composition source checkpoint differs.','stale_project',409) @@ -429,7 +473,7 @@ def _publish_independent(self, arguments, *, composed=False): # A filesystem editor can act outside the workspace lock. Leave the # complete fork unselected if its source working files changed. source.files.require_clean() - incoming = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) + incoming = self._open_working(root_fd, root_path, arguments['root'], arguments['directory']) projection, text = self._projection_for(incoming) snapshot, overlay = published['snapshot'], published['fork'] self._detach() diff --git a/projects.py b/projects.py index 5860d846..3898ba0b 100644 --- a/projects.py +++ b/projects.py @@ -28,9 +28,13 @@ def closed(value, required, optional=()): raise WorkbenchError('Invalid project fields.') +# Same code points as any(ord(c) < 32 or 0xD800 <= ord(c) <= 0xDFFF), scanned in C. +_TEXT_FORBIDDEN = re.compile('[\x00-\x1f\ud800-\udfff]') + + def text(value, label, limit=160): if (not isinstance(value, str) or not value.strip() or len(value) > limit - or any(ord(c) < 32 or 0xD800 <= ord(c) <= 0xDFFF for c in value)): + or _TEXT_FORBIDDEN.search(value)): raise WorkbenchError(label + ' must be bounded, nonempty text without control characters.') return value diff --git a/retained_measurements.py b/retained_measurements.py index 4044b017..4eefa6d5 100644 --- a/retained_measurements.py +++ b/retained_measurements.py @@ -309,16 +309,31 @@ def query(self, metric_id, *, subject_ids=None, filters=None, offset=0, limit=50 fail('Filters must use declared dimensions and typed scalar values.') if type(offset) is not int or not 0<=offset<=2**53-1 or type(limit) is not int or not 1<=limit<=100: fail('Page bounds must be explicit finite integers.') - rows = [v for v in self._doc['observations'] if v['metric_id']==metric_id - and v['subject_id'] in selected - and all(self._row_dimensions(v)[k]==wanted for k,wanted in filters.items())] + # One pass over observations; set membership and per-subject grouping + # replace the former subjects x rows rescans. Output is unchanged: + # rows keep packet order and ties keep the first maximal observation. + chosen = set(selected) + rows = [] + for v in self._doc['observations']: + if v['metric_id']!=metric_id or v['subject_id'] not in chosen: + continue + if filters: + dimensions = self._row_dimensions(v) + if not all(dimensions[k]==wanted for k,wanted in filters.items()): + continue + rows.append(v) + grouped = {identity:{'count':0,'states':{state:0 for state in STATES},'maximum':None} for identity in selected} + for v in rows: + own = grouped[v['subject_id']] + own['count'] += 1 + if v['state'] in own['states']: + own['states'][v['state']] += 1 + if v['state']=='measured' and (own['maximum'] is None or v['value'] > own['maximum']['value']): + own['maximum'] = v summaries = [] for identity in selected: - own = [v for v in rows if v['subject_id']==identity] - counts = {state:sum(v['state']==state for v in own) for state in STATES} - measured = [v for v in own if v['state']=='measured'] - maximum = max(measured, key=lambda v:v['value']) if measured else None - summaries.append({'subject_id':identity,'count':len(own),'states':counts, + own = grouped[identity];maximum = own['maximum'] + summaries.append({'subject_id':identity,'count':own['count'],'states':own['states'], 'maximum':None if maximum is None else {k:maximum[k] for k in ('id','case_id','value')}}) result = {'format':'retained_measurement_query_v1','packet_sha256':self.sha256, 'scope':copy.deepcopy(self._doc['scope']),'metric':copy.deepcopy(self._metrics[metric_id]), diff --git a/runs.py b/runs.py index c4cfcfc7..8e17e652 100644 --- a/runs.py +++ b/runs.py @@ -150,7 +150,8 @@ def project_run_projection(run, *, imported=False): item['name'] in seen or type(item['sha256']) is not str or not re.fullmatch('[a-f0-9]{64}',item['sha256'])): raise WorkbenchError('Historical Run artifact identity differs.') seen.add(item['name']);natural(item['bytes'],'Artifact bytes');text(item['media_type'],'Artifact media type',120) - return {key:copy.deepcopy(item) for key,item in run.items() if key in RUN_FIELDS} + # checked_json above already returned an independent copy owned here. + return {key:item for key,item in run.items() if key in RUN_FIELDS} def validate_approved_composition(value, sha, graph, execution_material_sha256, *, catalog_id): diff --git a/tests/browser_project_files_m3.cjs b/tests/browser_project_files_m3.cjs index 3f12ed69..5fc1a735 100644 --- a/tests/browser_project_files_m3.cjs +++ b/tests/browser_project_files_m3.cjs @@ -103,6 +103,8 @@ for name in ('torch','numpy'): await activate('#project-runtime-status-form button');await settled();const historical=JSON.parse(await evaluate('document.getElementById("project-runtime-status").textContent'));assert.equal(historical.state,'captured');assert.deepEqual(historical.result,captured); assert.ok(!fs.existsSync(sentinel));assert.ok(!fs.existsSync(audit),'Application never attempted subprocess/exec/install'); check('Fresh explicit confirmed capture copies offline bytes and retains intent/result; no child process or package import executes'); + // Edits reach the project folder on Save; Close keeps them only in the working copy. + await activate('#project-files-save');await settled(); await activate('#project-files-close');await until(async()=>(await state()).state==='no_project_open','close captured project');await stop(server); const source=path.join(external,'alpha'),coldRoot=path.join(temporary,'cold-external');fs.mkdirSync(coldRoot);const clone=path.join(coldRoot,'clone'),canonical=snapshot(source); const git=(...args)=>execFileSync('git',args,{env:gitEnv,stdio:['ignore','pipe','pipe']});git('init','-q','--template=',source);git('-C',source,'add','.');git('-C',source,'-c','user.name=Disposable M3 QA','-c','user.email=qa@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','Inert canonical runtime facts');git('-c','core.hooksPath=/dev/null','clone','-q','--no-hardlinks','--template=',source,clone);if(fs.existsSync(path.join(source,'artifacts')))fs.cpSync(path.join(source,'artifacts'),path.join(clone,'artifacts'),{recursive:true});assert.deepEqual(snapshot(clone),canonical); diff --git a/tests/browser_project_files_native.cjs b/tests/browser_project_files_native.cjs index a4297d5b..b9a978af 100644 --- a/tests/browser_project_files_native.cjs +++ b/tests/browser_project_files_native.cjs @@ -85,6 +85,8 @@ function check(value){checks.push(value);console.log('PASS '+value);} facts=await fixture.command('facts');assert.equal(facts.go_events.length,1);assert.equal(facts.pending,false);assert.equal(facts.available,true);assert.equal(facts.workspace_runs,0); await activate('#project-files-discard');await activate('#project-files-refresh');await settled();sourceFacts=await fixture.command('facts'); check('Named retry refuses absent current Run; real deferrable SQL fault after canonical GO intent sends zero GO, then keyboard health phase selection and explicit reconciliation restore availability without execution'); + // Edits reach the project folder on Save; Close keeps them only in the working copy. + await activate('#project-files-save');await settled(); const source=path.join(sourceHost.ready.external,'alpha'),canonical=snapshot(source),saved=(await state()).project; await activate('#project-files-close');await until(async()=>(await state()).state==='no_project_open','close before cold clone');await sourceHost.stop(); const coldHostRoot=path.join(temporary,'cold-host'),external=path.join(coldHostRoot,'external');fs.mkdirSync(external,{recursive:true});const clone=path.join(external,'clone'); diff --git a/tests/test_project_artifacts.py b/tests/test_project_artifacts.py index ad06e375..87caa214 100644 --- a/tests/test_project_artifacts.py +++ b/tests/test_project_artifacts.py @@ -83,7 +83,7 @@ def mutate_source_after_snapshot(fd, value, **options): self.assertEqual(copied.stat().st_nlink, 1) self.assertNotEqual(copied.stat().st_ino, self.payload(value).stat().st_ino) - def test_copy_group_rejects_source_canonical_corruption_before_success(self): + def test_copy_group_uses_verified_records_and_verification_rejects_later_corruption(self): value = declaration(b'present') self.artifacts.publish_bytes(b'present', value) self.commit([value]) @@ -96,7 +96,9 @@ def corrupt_after_copy(*args, **options): record.write_bytes(b'{"corrupt":true}\n') return result with patch.object(destination, 'publish_fd', side_effect=corrupt_after_copy): - self.assert_code('stale', self.artifacts.copy_many_to, destination, [value]) + # The copy uses the verified declaration, never the corrupt bytes. + self.artifacts.copy_many_to(destination, [value]) + self.assert_code('stale', self.store.verify_records) def test_copy_group_prevalidates_inventory_and_preserves_missing_payload_failure(self): value, missing = declaration(b'present'), declaration(b'missing') @@ -126,7 +128,7 @@ def test_status_group_retains_exact_hash_observations_with_two_closure_checks(se self.assertEqual(actual[0]['observed_sha256'], values[0]['sha256']) self.assertEqual(actual[2]['observed_sha256'], hashlib.sha256(b'changed').hexdigest()) - def test_status_group_rejects_canonical_corruption_during_observation(self): + def test_status_group_uses_verified_records_and_verification_rejects_later_corruption(self): value = declaration(b'present') self.artifacts.publish_bytes(b'present', value) self.commit([value]) @@ -139,7 +141,8 @@ def corrupt_after_observation(*args): yield source record.write_bytes(b'{"corrupt":true}\n') with patch.object(self.artifacts, '_source', side_effect=corrupt_after_observation): - self.assert_code('stale', self.artifacts.status_many, [value]) + self.artifacts.status_many([value]) + self.assert_code('stale', self.store.verify_records) def test_status_group_rejects_checkpoint_change_and_unsafe_aliases(self): value = declaration(b'present') @@ -167,7 +170,7 @@ def test_presence_batch_matches_individual_observations_with_two_closure_checks( self.assertEqual(checks.call_count,2) self.assertTrue(all(v['observed_sha256'] is None for v in actual)) - def test_presence_batch_detects_canonical_corruption_before_returning(self): + def test_presence_batch_uses_verified_records_and_verification_rejects_later_corruption(self): value=declaration(b'present') self.artifacts.publish_bytes(b'present',value) self.commit([value]) @@ -180,7 +183,8 @@ def corrupt_after_observation(*args): yield source record.write_bytes(b'{"corrupt":"outside canonical membership"}\n') with patch.object(self.artifacts,'_source',side_effect=corrupt_after_observation): - self.assert_code('stale',self.artifacts.inspect_many,[value]) + self.artifacts.inspect_many([value]) + self.assert_code('stale',self.store.verify_records) def test_presence_batch_detects_checkpoint_change_and_rejects_unsafe_aliases(self): value=declaration(b'present') diff --git a/tests/test_project_files_http.py b/tests/test_project_files_http.py index 39a271f3..edef4f68 100644 --- a/tests/test_project_files_http.py +++ b/tests/test_project_files_http.py @@ -73,7 +73,11 @@ def cleanup_owned(self): with psycopg.connect(connection_info(dsn), autocommit=True) as connection: connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(schema))) # Projection manifests are physical identities, distinct from logical project IDs. + from project_working_store import registered_schemas for dsn, control in self.controls: + for schema in registered_schemas(control): + with psycopg.connect(connection_info(dsn), autocommit=True) as connection: + connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(schema))) for manifest in control.rglob('.tuldok.json'): value = json.loads(manifest.read_bytes()) schema = ('projection_' + value['projection_id'] if value['version'] == 2 @@ -282,57 +286,39 @@ def test_legacy_mutations_are_unavailable_before_and_after_open(self): self.assertEqual(canonical_bytes(self.external), before) self.assertEqual(list((self.path / 'control').rglob('*.png')), []) - def test_real_deferred_sql_failure_returns_committed_receipt_and_explicit_recovery(self): + def test_real_deferred_sql_failure_rolls_back_the_edit_and_a_retry_succeeds(self): self.create() selected = self.state() projection_schema = self.workspace._projection.schema - # A real deferred PostgreSQL trigger fails the outer SQL COMMIT, after - # canonical publication. It exists only in this disposable projection. + # A real deferred PostgreSQL trigger fails the outer SQL COMMIT. Records + # and projection rows share that transaction, so both roll back. with psycopg.connect(connection_info(), autocommit=True) as connection: function = sql.Identifier(projection_schema, 'qa_reject_projection_commit') table = sql.Identifier(projection_schema, 'workbench_records') connection.execute(sql.SQL('''CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql - AS $$ BEGIN RAISE EXCEPTION 'Disposable QA deferred commit failure'; END; $$''').format(function)) + AS $$ BEGIN RAISE EXCEPTION 'Disposable QA deferred commit failure' USING ERRCODE='23514'; END; $$''').format(function)) connection.execute(sql.SQL('''CREATE CONSTRAINT TRIGGER qa_reject_projection_commit AFTER INSERT ON {} DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION {}()''').format(table, function)) - asset = {'name': 'Committed despite unavailable SQL 山', 'text': TEXT, + asset = {'name': 'Rejected with its SQL commit 山', 'text': TEXT, 'groups': ['recovery'], 'parents': [], 'rights': 'Owned synthetic recovery fixture'} original = self.envelope(selected, asset=asset) + before = canonical_bytes(self.external / 'alpha') status, error = self.request(API + '/import-text', original) - self.assertEqual(status, 503, error) - self.assertEqual(error['code'], 'projection_unavailable') - receipt = error['receipt'] - self.assertEqual(receipt['committed'], True) - self.assertEqual(receipt['selected'], True) - self.assertEqual(receipt['project_id'], selected['project']['project_id']) - self.assertEqual(receipt['request_key'], original['request_key']) - self.assertNotEqual(receipt['checkpoint'], selected['project']['checkpoint']) - before_recovery = canonical_bytes(self.external / 'alpha') - manifest = json.loads(before_recovery['.tuldok/project.json']) - self.assertEqual(manifest['checkpoint'], receipt['checkpoint']) - recovery = self.state() - self.assertEqual(recovery['state'], 'recovery_required') - self.assertEqual(recovery['available_actions'], ['open']) - self.assertEqual(recovery['recovery']['reason'], 'projection_unavailable') - status, denied = self.request(API + '/save', self.envelope(recovery)) - self.assertEqual(status, 503, denied) - self.assertEqual(canonical_bytes(self.external / 'alpha'), before_recovery) - self.action('open', recovery, root='fixture', directory='alpha') - recovered = self.state() - self.assertEqual(recovered['state'], 'open') - self.assertEqual(recovered['project']['checkpoint'], receipt['checkpoint']) - self.assertNotEqual(self.workspace._projection.schema, projection_schema) - records = self.records(recovered) + self.assertEqual(status, 409, error) + self.assertEqual(error['code'], 'rejected') + self.assertEqual(self.state(), selected, 'Nothing changed; the project stays open and current') + self.assertEqual(self.records(selected), []) + self.assertEqual(canonical_bytes(self.external / 'alpha'), before) + with psycopg.connect(connection_info(), autocommit=True) as connection: + connection.execute(sql.SQL('DROP TRIGGER qa_reject_projection_commit ON {}').format( + sql.Identifier(projection_schema, 'workbench_records'))) + status, imported = self.request(API + '/import-text', original) + self.assertEqual(status, 200, imported) + records = self.records() self.assertEqual(len(records), 1) self.assertEqual(self.detail(records[0]['id'])['record']['text'], TEXT) - # Reconcile the exact durable request with its old checkpoint and the - # new process binding. A committed operation cannot be duplicated. - status, replayed = self.request(API + '/import-text', dict(original, binding=recovered['binding'])) - self.assertEqual(status, 200, replayed) - self.assertEqual(replayed['record']['id'], records[0]['id']) - self.assertEqual(len(self.records()), 1) - self.assertEqual(canonical_bytes(self.external / 'alpha'), before_recovery) + self.assertEqual(canonical_bytes(self.external / 'alpha'), before, 'Edits are not written until Save') @unittest.skipUnless(importlib.util.find_spec('mcp') and importlib.util.find_spec('anyio'), 'Requires the pinned optional MCP environment for real adapter acceptance') diff --git a/tests/test_project_files_m2_http.py b/tests/test_project_files_m2_http.py index 82492847..9d66176f 100644 --- a/tests/test_project_files_m2_http.py +++ b/tests/test_project_files_m2_http.py @@ -404,6 +404,8 @@ def test_retained_measurement_import_query_permissions_and_independent_cold_copy status,admitted=self.transport('measurement_import',import_args) self.assertEqual(status,200,admitted) origin=admitted['project_id'];selection=dict(origin_catalog_id=origin,packet_sha256=digest) + # Edits stay in the database working copy until an explicit Save. + self.action('save') canonical=m1.canonical_bytes(self.external/'alpha') self.assertEqual(self.transport('measurement_import',import_args)[1],admitted) self.assertEqual(m1.canonical_bytes(self.external/'alpha'),canonical) diff --git a/tests/test_project_files_m4_http.py b/tests/test_project_files_m4_http.py index 8d6b0a6f..bd7d48ca 100644 --- a/tests/test_project_files_m4_http.py +++ b/tests/test_project_files_m4_http.py @@ -47,7 +47,7 @@ def test_batch_adoption_uses_real_http_and_survives_independent_fork(self): items.append(dict(kind='note',path='notes/'+name,expected_revision=0,expected_sha256=hashlib.sha256(raw).hexdigest())) result=self.call('files_adopt',items=items) self.assertEqual(result['changed_count'],2);self.assertEqual(len(self.call('files_inspect')['items']),2) - with patch.object(self.workspace._store,'_snapshot',side_effect=AssertionError('Unrelated full graph copy')): + with patch.object(self.workspace._store,'snapshot',side_effect=AssertionError('Unrelated full graph copy')): self.assertEqual(len(self.call('files_inspect')['items']),2) selected=self.call('file_get',file_id=result['items'][0]['id']) self.assertEqual(selected['version']['text'].encode(),b'Exact first note\r\n') @@ -133,16 +133,23 @@ def test_published_unselected_fork_exact_retry_and_changed_file_guard(self): def test_git_checkout_stale_projection_and_moved_folder_inert_reconstruction(self): self.create();root=self.external/'alpha' first=self.call('data_import_text',text='Git observation one',name='One',groups=['fixture'],rights='Owned') + self.action('save') def git(*args):return subprocess.check_output(['git','-C',str(root),*args],stderr=subprocess.STDOUT) git('init','-q');git('config','user.name','Disposable Fixture');git('config','user.email','fixture@example.invalid') git('add','.');git('commit','-qm','First authored checkpoint');old=git('rev-parse','HEAD').decode().strip() self.call('data_import_text',text='Git observation two',name='Two',groups=['fixture'],rights='Owned') + self.action('save') git('add','.');git('commit','-qm','Second authored checkpoint') - stale=self.state() git('checkout','-q',old) - state=self.state();self.assertEqual(state['state'],'recovery_required') - status,failure=self.transport('data_import_text',dict(self.envelope(stale),text='Must not overwrite',name='Blocked',groups=['fixture'],rights='Owned')) - self.assertEqual(status,503,failure) + # The database working copy stays the session's authority; Save refuses + # to overwrite the externally checked-out folder. + self.assertEqual(self.state()['state'],'open') + status,failure=self.transport('save',self.arguments('save')) + self.assertEqual(status,409,failure);self.assertEqual(failure['code'],'conflict') + self.assertEqual(git('rev-parse','HEAD').decode().strip(),old) + self.assertEqual(git('status','--porcelain').decode(),'') + # Reopening a clean working copy imports the checked-out revision. + self.action('close') self.action('open',root='fixture',directory='alpha') self.assertEqual([r['id'] for r in self.call('data_query')['items']],[first['id']]) self.action('close');shutil.move(root,self.external/'moved') diff --git a/tests/test_project_files_owner_mcp.py b/tests/test_project_files_owner_mcp.py index 1e3d74ca..95094583 100644 --- a/tests/test_project_files_owner_mcp.py +++ b/tests/test_project_files_owner_mcp.py @@ -178,9 +178,10 @@ async def exercise(): imported=await self.tool(session,'import_text',f.envelope(opened,asset=dict( text=http_fixture.TEXT,name='Actual owner MCP authored observation',groups=['owner-mcp'],parents=[],rights='Owned synthetic fixture'))) state=await self.tool(session,'status',{});source_id=state['project']['project_id'] - source_before=http_fixture.canonical_bytes(f.external/'owner-source') saved=await self.tool(session,'save',f.envelope(state)) self.assertEqual(saved['project'],state['project']) + # Edits reach the folder on Save; Save As must leave the saved source exact. + source_before=http_fixture.canonical_bytes(f.external/'owner-source') fork=await self.tool(session,'save_as',f.envelope(saved,root='fixture',directory='owner-fork',name='Independent MCP Ω')) current=fork['state'];self.assertNotEqual(current['project']['project_id'],source_id) self.assertEqual(http_fixture.canonical_bytes(f.external/'owner-source'),source_before) diff --git a/tests/test_project_history.py b/tests/test_project_history.py index c42dd135..ba632ad8 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -167,6 +167,33 @@ def test_cross_study_parent_baseline_and_origin_namespace_remain_exact(self): changed=copy.deepcopy(records);changed[-1]['document']['lineage']['intent']['baseline']['catalog_id']=OTHER with self.assertRaises(WorkbenchError):decode_records(changed,materials=Materials()) + def test_validated_fact_assembly_equals_complete_decode(self): + from project_history import assemble_validated_facts + parent_id='9'*32;child_id='a'*32 + parent_ref={'catalog_id':ORIGIN,'trial_id':parent_id,'material_sha256':MATERIAL} + saved=capture_run(legacy_run(),payloads=[]) + fixtures=( + [study('3'*32,trials=[parent_id]),study('b'*32,trials=[child_id]), + trial(parent_id,'3'*32),trial(child_id,'b'*32,parent=parent_ref,baseline=parent_ref)], + [saved,*declarations(saved)], + ) + for records in fixtures: + with self.subTest(families=sorted({r['family'] for r in records})): + decoded=decode_records(copy.deepcopy(records),materials=Materials()) + for owned in (False,True): + before=copy.deepcopy(records) + with patch('project_history._validate_run_document',side_effect=AssertionError('Validator rerun')): + assembled=assemble_validated_facts(copy.deepcopy(records) if owned else records,owned=owned) + self.assertEqual(records,before) + for field in ('studies','trials','runs','candidates','policies','reservations','observations'): + self.assertEqual(getattr(assembled,field),getattr(decoded,field),field) + self.assertEqual(list(getattr(assembled,field)),list(getattr(decoded,field)),field) + # Study authoring restore collects only Study and Trial documents. + authoring=assemble_validated_facts([r for r in copy.deepcopy(records) + if r['family'] in ('study_history','trial_history')],owned=True) + for field in ('studies','trials','policies','reservations'): + self.assertEqual(getattr(authoring,field),getattr(decoded,field),field) + def test_history_read_uses_one_material_view_of_the_selected_records(self): identity='9'*32 records=[study(trials=[identity]),trial(identity,'3'*32)] @@ -354,7 +381,7 @@ def start(self): self.host,self.owner,self.session=host,owner,session self.history=ProjectHistory(session,materials=Materials()) - def test_cached_history_still_reads_actual_bytes_and_rejects_restored_mtime_corruption(self): + def test_cached_history_serves_verified_values_and_verification_rejects_corruption(self): from project_store import ProjectStoreError identity='9'*32 records=[study(trials=[identity]),trial(identity,'3'*32)] @@ -372,8 +399,11 @@ def test_cached_history_still_reads_actual_bytes_and_rejects_restored_mtime_corr try: path.write_bytes(corrupt) os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + # Cached history keeps serving the verified value, never the + # corrupt bytes; explicit verification refuses them. + self.assertEqual(self.history.progression(ORIGIN),expected) with self.assertRaisesRegex(ProjectStoreError,'Canonical record bytes changed or are corrupt') as failed: - self.history.progression(ORIGIN) + self.store.verify_records() self.assertEqual((failed.exception.code,failed.exception.status),('stale',409)) finally: path.write_bytes(original) diff --git a/tests/test_project_measurement_reads.py b/tests/test_project_measurement_reads.py index 1365d63a..1ef28124 100644 --- a/tests/test_project_measurement_reads.py +++ b/tests/test_project_measurement_reads.py @@ -54,12 +54,12 @@ def observe(value): self.assertEqual(self.owner._documents(), self.expected) self.assertEqual(self.session._snapshot, self.store.snapshot()) - def test_lookup_reads_every_canonical_file_without_redecoding_adopted_material(self): + def test_lookup_reuses_verified_records_without_redecoding_adopted_material(self): with patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ patch.object(project_measurements, 'material_records', side_effect=AssertionError('Read-only lookup redecoded adopted material')): self.assertEqual(self.owner._documents(), self.expected) - self.assertEqual(reads.call_count, len(self.records), 'All current files must still be read and hashed') + self.assertEqual(reads.call_count, 0, 'Verified immutable records are not re-read per lookup') def test_unavailable_projection_fails_before_any_store_read(self): self.session.available = False @@ -76,7 +76,7 @@ def test_other_family_publication_requires_reopen(self): self.owner._documents() self.assertEqual(caught.exception.code, 'stale_project') - def test_same_length_unselected_corruption_with_restored_mtime_is_rejected(self): + def test_same_length_unselected_corruption_is_never_served_and_verification_rejects_it(self): inventory = json.loads((self.root/'.tuldok/project.json').read_bytes())['inventory'] entry = next(r for r in inventory if r['id'] == 'f'*64) path = self.root/'.tuldok'/entry['path'] @@ -87,8 +87,10 @@ def test_same_length_unselected_corruption_with_restored_mtime_is_rejected(self) path.chmod(0o600) path.write_bytes(changed) os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + # Reads keep serving verified values; explicit verification refuses. + self.assertEqual(self.owner._documents(), self.expected) with self.assertRaises(ProjectStoreError) as caught: - self.owner._documents() + self.store.verify_records() self.assertEqual(caught.exception.code, 'stale') def test_family_can_be_empty_and_returned_snapshot_contract_stays_complete(self): diff --git a/tests/test_project_native_study.py b/tests/test_project_native_study.py index 282837ab..969c6e3b 100644 --- a/tests/test_project_native_study.py +++ b/tests/test_project_native_study.py @@ -85,7 +85,9 @@ def retain_project_evidence(projection, label): store=projection.session.store with projection.session.access.lock,projection.session.lock,projection.db.lock: snapshot=store.snapshot() - manifest=(store.path/'.tuldok/project.json').read_bytes() + # A database working copy's checkpoint reaches the folder only on Save. + manifest=(store.manifest_raw() if callable(getattr(store,'manifest_raw',None)) + else (store.path/'.tuldok/project.json').read_bytes()) if json.loads(manifest)['checkpoint']!=snapshot['checkpoint']: raise AssertionError('Native fixture snapshot and manifest checkpoints differ') output=output/(label+'-'+snapshot['project_id']+'-'+snapshot['checkpoint']) diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index 3f1e3e98..377b2767 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -76,6 +76,8 @@ def test_mixed_cold_projection_preserves_notes_and_definitions_without_worker_co preprocessing_policy={'kind': 'identity'}, output_policy={'kind': 'bytes'}) model = p.materials.mutate_definition('model_definition', 'models.create', None, body, p.session.snapshot['checkpoint'], 'model-create', lambda: p.models.create(body)) + # Another workspace sees edits only after they are saved to the folder. + self.call('save', self.mutation()) snapshot = self.host._store.snapshot() canonical = (self.external / 'project' / '.tuldok' / 'project.json').read_bytes() sentinel = self.external / 'project' / 'must-not-run' diff --git a/tests/test_project_record_dedupe.py b/tests/test_project_record_dedupe.py new file mode 100644 index 00000000..3c8d1760 --- /dev/null +++ b/tests/test_project_record_dedupe.py @@ -0,0 +1,150 @@ +"""Shared runtime closures: compact storage, exact logical Run documents.""" +import copy +import json +from pathlib import Path +import tempfile +import unittest + +import project_record_dedupe as dedupe +import project_store as store_module +from project_store import ProjectStore + + +def runtime(): + files = {f'/opt/py/lib/module{i}.py': dict(bytes=10 + i, sha256=f'{i:064x}') for i in range(40)} + return dict(assumption='fixture', closure_sha256='c' * 64, executable='/opt/py/bin/python', + executable_sha256='e' * 64, files=files, flags=[], profile='fixture', trusted_install_owner_uid=0) + + +def run(identity, *, unique=3): + edges = [dict(blob=dict(bytes=len(path), sha256=item['sha256']), name=f'runtime-files/{n}', population='unknown', + preservation='regenerable', required_for=['scientific_inspection'], role='historical_runtime') + for n, (path, item) in enumerate(sorted(runtime()['files'].items()))] + before = [dict(blob=dict(bytes=n, sha256=f'{n+100:064x}'), name=f'input/{identity}/{n}', population='unknown', + preservation='unique', required_for=['execution_proof'], role='execution_source') for n in range(unique)] + after = [dict(blob=dict(bytes=7, sha256='a' * 64), name='output/model', population='unknown', + preservation='unique', required_for=['inspect'], role='output')] + spec = dict(argv=['python', 'train.py', identity], runtime_files=[dict(path=p, **v) for p, v in sorted(runtime()['files'].items())]) + document = dict(format='project_run_history_v1', origin_catalog_id='a' * 32, admission_proof=None, publication=None, + run=dict(id=identity, spec=spec, material=dict(recipe=dict(id='r'), runtime=runtime()), events=[1.5, -0.0, 'é']), + payloads=before + edges + after) + return dict(family='run_history', id='a' * 32 + '.' + identity, document=document) + + +class CompactionTests(unittest.TestCase): + def test_exact_roundtrip_and_refusals(self): + record = run('one') + compacted, closure = dedupe.compact(record['document']) + self.assertEqual(compacted['format'], dedupe.RUN_V2) + self.assertNotIn('runtime_files', compacted['run']['spec']) + self.assertEqual(store_module._encode(dedupe.expand(compacted, closure)), store_module._encode(record['document'])) + split = copy.deepcopy(record['document']) + split['payloads'].insert(10, split['payloads'].pop(0)) + self.assertIsNone(dedupe.compact(split), 'A non-contiguous runtime block stays uncompacted') + tampered = copy.deepcopy(closure); tampered['runtime']['profile'] = 'other' + with self.assertRaises(store_module.ProjectStoreError): + dedupe.expand(compacted, tampered) + + +class StoreTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-dedupe-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / 'project' + self.store = ProjectStore.create(self.root, 'Dedupe fixture', 'new', 'a' * 64) + self.addCleanup(lambda: self.store.close()) + + def files(self, family): + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + return [json.loads((self.root / '.tuldok' / e['path']).read_bytes()) for e in manifest['inventory'] if e['family'] == family] + + def test_runs_share_one_stored_closure_and_read_back_exactly(self): + records = [run('one'), run('two', unique=5)] + committed = self.store.commit(copy.deepcopy(records), self.store.summary()['checkpoint'], 'runs', 'b' * 64) + self.assertEqual([r['document'] for r in committed['records']], [r['document'] for r in records]) + self.assertEqual([d['format'] for d in self.files('run_history')], [dedupe.RUN_V2, dedupe.RUN_V2]) + self.assertEqual(len(self.files(dedupe.CLOSURE_FAMILY)), 1) + self.assertNotIn(dedupe.CLOSURE_FAMILY, {r['family'] for r in self.store.snapshot()['records']}) + self.store.close() + self.store = ProjectStore.open(self.root) + self.assertEqual(self.store.snapshot(), committed) + # Editing one Run keeps the other's verified revision and the closure. + changed = copy.deepcopy(records); changed[0]['document']['run']['events'].append('edited') + again = self.store.commit(changed, committed['checkpoint'], 'edit', 'c' * 64) + self.assertEqual(again['records'][0]['document'], changed[0]['document']) + self.store.close() + self.store = ProjectStore.open(self.root) + self.assertEqual(self.store.snapshot(), again) + self.assertEqual(len(self.files(dedupe.CLOSURE_FAMILY)), 1) + + def test_closures_are_store_owned(self): + with self.assertRaises(store_module.ProjectStoreError): + self.store.commit([dict(family=dedupe.CLOSURE_FAMILY, id='x', document=dict(format=dedupe.CLOSURE_FORMAT))], + self.store.summary()['checkpoint'], 'forged', 'd' * 64) + + +if __name__ == '__main__': + unittest.main() + + +import os +import uuid + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class WorkingStoreTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-dedupe-working-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / 'project' + with ProjectStore.create(self.root, 'Legacy runs', 'new', 'a' * 64) as store: + records = [run('one'), run('two', unique=4)] + self.legacy = store.commit(copy.deepcopy(records), store.summary()['checkpoint'], 'runs', 'b' * 64) + self.working_id = uuid.uuid4().hex + self.addCleanup(self.drop) + + def drop(self): + import psycopg + from psycopg import sql + from metadata import connection_info + with psycopg.connect(connection_info(None), autocommit=True) as connection: + connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier('working_' + self.working_id))) + + def working(self): + from project_working_store import WorkingProjectStore + parent, leaf = os.path.split(str(self.root)) + descriptor = os.open(parent, store_module.DIR_FLAGS) + try: + folder = ProjectStore.open_folder_at(descriptor, leaf, root_path=parent) + finally: + os.close(descriptor) + return WorkingProjectStore.open(folder, self.working_id) + + def test_migration_keeps_logical_records_and_saves_compact_files(self): + from project_composition import _record_sha + store = self.working() + try: + before = store.snapshot()['records'] + previous, after = store.compact_run_closures('compact-runs', 'c' * 64) + self.assertNotEqual(previous, after['checkpoint']) + self.assertEqual(store.snapshot()['records'], before, 'Logical records are unchanged') + self.assertEqual(store.records('run_history'), [r for r in before if r['family'] == 'run_history']) + self.assertEqual(store.record('run_history', before[0]['id']), before[0]) + view = store.records_view('run_history') + self.assertNotIn(dedupe.CLOSURE_FAMILY, {r['family'] for r in view}) + digests = store.envelope_digests() + for record in before: + self.assertEqual(digests[(record['family'], record['id'])], _record_sha(record)) + # An edit through the session path keeps the shared closure. + changed = copy.deepcopy(before); changed[1]['document']['run']['events'].append('edited') + edited = store.commit(changed, after['checkpoint'], 'edit', 'd' * 64, changed={(changed[1]['family'], changed[1]['id']), }) + store.save(edited['checkpoint']) + saved = store.snapshot() + finally: + store.close() + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + families = [e['family'] for e in manifest['inventory']] + self.assertEqual(families.count(dedupe.CLOSURE_FAMILY), 1) + self.assertEqual({e['format'] for e in manifest['inventory'] if e['family'] == 'run_history'}, {dedupe.RUN_V2}) + with ProjectStore.open(self.root) as reread: + self.assertEqual(reread.snapshot(), saved, 'The file store reads the same logical records') diff --git a/tests/test_project_runtime_http.py b/tests/test_project_runtime_http.py index 844ba5da..eba12786 100644 --- a/tests/test_project_runtime_http.py +++ b/tests/test_project_runtime_http.py @@ -65,6 +65,8 @@ def test_configured_alias_actual_capture_and_cold_historical_status_never_execut self.assertTrue(any(r['operation']=='runtime.capture' for r in receipts)) status,replay=self.transport('runtime_capture',arguments);self.assertEqual(status,200,replay) self.assertEqual(replay['value'],captured) + # Another workspace sees the capture only after it is saved. + self.action('save') self.action('close') self.select_host(self.path/'cold-control',{'fixture':self.external}) self.action('open',root='fixture',directory='alpha') diff --git a/tests/test_project_runtime_recovery_http.py b/tests/test_project_runtime_recovery_http.py index f22b0b8e..6817d148 100644 --- a/tests/test_project_runtime_recovery_http.py +++ b/tests/test_project_runtime_recovery_http.py @@ -158,7 +158,7 @@ def test_actual_partial_copy_abandon_needs_no_remaining_source_and_retains_bindi self.assertEqual(self.call('runtime_status',capture_key=key)['state'],'abandoned') self.action('save');self.action('close') - def test_final_sql_failure_retry_requires_exact_public_view_and_never_recopies(self): + def test_final_sql_failure_rolls_back_and_an_exact_retry_completes_without_recopying(self): projection=self.workspace._projection;db=projection.db;key='lost-final-sql' db.execute('CREATE TABLE runtime_ack_parent(id INTEGER PRIMARY KEY)') db.execute('CREATE TABLE runtime_ack_child(id INTEGER REFERENCES runtime_ack_parent(id) DEFERRABLE INITIALLY DEFERRED)') @@ -170,20 +170,23 @@ def bad(): return mutate(operation,target,body,checkpoint,request_key,bad,capture,**kwargs) with patch.object(projection.session,'mutate',side_effect=fault): status,error=self.transport('runtime_capture',self.capture_arguments(key)) - self.assertEqual(status,503,error);self.assertFalse(projection.session.available) - state=self.state();self.assertEqual(state['state'],'recovery_required') - arguments=self.capture_arguments(key);before=projection.session.snapshot - for change in (dict(binding='f'*32),dict(expected_checkpoint='f'*64)): - with patch.object(projection.runtime,'recover_capture',side_effect=AssertionError('Rejected public view reached recovery')): - status,error=self.transport('runtime_capture',dict(arguments,**change)) - self.assertEqual(status,409,error);self.assertEqual(error['code'],'stale_project') - self.assertEqual(projection.session.snapshot,before);self.assertTrue(projection.custody.has_pending) - with patch.object(projection.runtime.owner,'create',side_effect=AssertionError('Do not recopy committed runtime')): + # The final records and projection rows rolled back together; only the + # previously committed intent remains (its receipt is reported). + self.assertEqual(status,409,error);self.assertEqual(error['code'],'rejected') + self.assertTrue(error['receipt']['committed']) + self.assertTrue(projection.session.available) + self.assertEqual(self.state()['state'],'open') + self.assertNotIn(key,projection.session.operations()) + arguments=self.capture_arguments(key) + status,error=self.transport('runtime_capture',dict(arguments,binding='f'*32)) + self.assertEqual(status,409,error);self.assertEqual(error['code'],'stale_project') + with patch.object(projection.runtime.owner,'create',side_effect=AssertionError('Do not recopy the copied runtime')): status,recovered=self.transport('runtime_capture',arguments) - self.assertEqual(status,200,recovered);self.assertEqual(recovered['binding'],state['binding']) + self.assertEqual(status,200,recovered) self.assertEqual(self.state()['state'],'open');self.assertTrue(projection.session.available) - self.assertFalse(projection.custody.has_pending) self.assertEqual(self.call('runtime_status',capture_key=key)['state'],'captured') + receipts=[r for r in projection.session.operations().values() if r['operation']=='runtime.capture'] + self.assertEqual(len(receipts),1) self.action('close') def test_original_physical_cache_recovery_and_foreign_intent_cannot_regenerate(self): @@ -195,6 +198,9 @@ def after_copy(*args,**kwargs): status,error=self.transport('runtime_capture',self.capture_arguments(key)) self.assertEqual(status,503,error);physical=projection.db.projection_id self.assertTrue((projection.runtime.owner.path/self.inventory['inventory_sha256']).is_dir()) + # The committed intent was saved to the folder before it was copied + # (the workspace itself refuses Save while recovery is required). + original._store.save(original._store.summary()['checkpoint']) shutil.copytree(self.external/'alpha',self.external/'clone',ignore=shutil.ignore_patterns('.tuldok-local')) self.start_application(self.path/'foreign-control',sources={'inert-fixture':self.config}) self.action('open',root='fixture',directory='clone') diff --git a/tests/test_project_session_freshness.py b/tests/test_project_session_freshness.py index 8ab5634e..f2ebe64c 100644 --- a/tests/test_project_session_freshness.py +++ b/tests/test_project_session_freshness.py @@ -33,13 +33,13 @@ def setUp(self): self.session._snapshot = copy.deepcopy(snapshot) self.session.available = True - def test_freshness_reads_all_records_without_copying_an_unused_graph(self): + def test_freshness_compares_checkpoint_without_rereading_or_copying_records(self): before = self.store.snapshot() with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused full snapshot')), \ patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: self.assertIsNone(self.session.check_current()) self.assertIsNone(self.session.check_current()) - self.assertEqual(reads.call_count, 4) + self.assertEqual(reads.call_count, 0) self.assertEqual(self.store.snapshot(), before) def test_changed_canonical_checkpoint_refuses_both_read_contracts(self): @@ -62,7 +62,7 @@ def test_unavailable_projection_refuses_before_reading_the_store(self): check() self.assertEqual(caught.exception.code, 'projection_unavailable') - def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self): + def test_same_length_corruption_is_never_served_and_explicit_verification_refuses_it(self): self.session.check_current() manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) path = self.root / '.tuldok' / manifest['inventory'][0]['path'] @@ -73,11 +73,13 @@ def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self) path.chmod(0o600) path.write_bytes(changed) os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) - for check in (self.session.check_current, self.session.check, - lambda: self.session.check_records('workbench_text')): - with self.assertRaises(ProjectStoreError) as caught: - check() - self.assertEqual(caught.exception.code, 'stale') + # Reads serve the verified in-memory value, never the corrupt bytes. + self.assertIsNone(self.session.check_current()) + self.assertEqual(self.session.check()['records'][0]['document']['text'], 'Alpha one') + self.assertEqual(self.session.check_records('workbench_text')['records'], self.records) + with self.assertRaises(ProjectStoreError) as caught: + self.store.verify_records() + self.assertEqual(caught.exception.code, 'stale') def test_record_consumers_keep_independent_complete_snapshots(self): current = self.session.check() @@ -108,13 +110,13 @@ def observe(value): patch('project_session.copy', SimpleNamespace(deepcopy=observe)): selected = self.session.check_records('workbench_text') self.assertEqual(selected, expected) - self.assertEqual(reads.call_count, 3, 'Unselected actual records must still be read and hashed') + self.assertEqual(reads.call_count, 0, 'Verified immutable records are not re-read per request') self.assertEqual(len(copied), 1) selected['records'][0]['document']['text'] = 'Caller mutation' self.assertEqual(self.session.check_records('workbench_text'), expected) self.assertEqual(self.session.check_records('project_file')['records'], []) - def test_selected_read_rejects_corruption_in_an_unselected_family(self): + def test_explicit_verification_rejects_corruption_in_an_unselected_family(self): self.add_unrelated_record() self.session.check_records('workbench_text') manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) @@ -127,8 +129,9 @@ def test_selected_read_rejects_corruption_in_an_unselected_family(self): path.chmod(0o600) path.write_bytes(changed) os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + self.assertEqual(self.session.check_records('workbench_text')['records'], self.records) with self.assertRaises(ProjectStoreError) as caught: - self.session.check_records('workbench_text') + self.store.verify_records() self.assertEqual(caught.exception.code, 'stale') diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 676ab3bc..ac2fa578 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -107,6 +107,41 @@ def test_create_commit_save_cold_reopen_and_independent_snapshots(self): committed['records'][0]['document']['text'] = 'Unchecked returned mutation' self.assertEqual(self.store.snapshot()['records'][0]['document']['text'], 'Notes preserve \r\n authored values') + def test_streamed_records_digest_equals_complete_canonical_encoding(self): + rows = [record('a'), record('b', 'Line\none "quoted" \\ ☃ 
'), + dict(family='workbench_text', id='c', document=dict(format='workbench_text_v1', text='', + history=[], nested={'z': [1, 1.0, -0.0, None, True, {}], 'ü': {'k\n': []}}))] + expected = store_module._sha(store_module._encode(rows)) + streamed = store_module._records_digest( + (row['family'], row['id'], store_module._encode(row['document'])) for row in rows) + self.assertEqual(streamed, expected) + self.assertEqual(store_module._records_digest([]), store_module._sha(store_module._encode([]))) + + def test_commit_reuses_unchanged_revisions_without_reencoding_them(self): + values = [record(str(index), 'Retained authored note %d' % index) for index in range(6)] + self.commit(values, key='initial-six') + changed = copy.deepcopy(values) + changed[2]['document']['text'] = 'Only this revision changes' + encode = store_module._encode + with patch.object(store_module, '_encode', wraps=encode) as encodes: + committed = self.commit(changed, key='change-one') + documents = [call.args[0] for call in encodes.call_args_list + if type(call.args[0]) is dict and call.args[0].get('format') == 'workbench_text_v1'] + self.assertEqual(documents, [changed[2]['document']]) + self.assertEqual(self.reopen(), committed) + + def test_python_equal_but_differently_typed_values_are_new_revisions(self): + for before, after in ((1, 1.0), (0.0, -0.0), (True, 1), (1.0, 1)): + with self.subTest(before=before, after=after): + original = dict(family='workbench_text', id='typed', document=dict(format='workbench_text_v1', value=before)) + self.commit([original], key=f'typed-{before!r}-{after!r}-a'.replace('.', '_').replace('-', '_')) + updated = copy.deepcopy(original) + updated['document']['value'] = after + self.commit([updated], key=f'typed-{before!r}-{after!r}-b'.replace('.', '_').replace('-', '_')) + value = self.reopen()['records'][0]['document']['value'] + self.assertIs(type(value), type(after)) + self.assertEqual(repr(value), repr(after)) + def test_appending_one_record_does_not_rewrite_saved_immutable_revisions(self): values = [record(str(index), 'Retained authored note %d' % index) for index in range(8)] self.commit(values, key='initial-inventory') @@ -127,7 +162,8 @@ def test_summary_reads_every_current_record_without_cloning_the_record_graph(sel first['name'] = 'Caller mutation' second = self.store.summary() self.assertEqual(second, expected) - self.assertEqual(reads.call_count, 4, 'Each summary must still read/hash both actual records') + self.assertEqual(reads.call_count, 0, + 'Freshness compares checkpoint bytes; verified immutable revisions are not re-read') self.assertEqual(self.store.snapshot(), committed) def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_identical_records(self): @@ -142,7 +178,7 @@ def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_ident with self.store.root_directory(verify_after=True) as descriptor: self.assertTrue(os.fstat(descriptor).st_ino) self.assertEqual(self.store.snapshot(),expected) - self.assertEqual(read.call_count,8,'All two records must still be read in each of four checks') + self.assertEqual(read.call_count,0,'Byte-identical checkpoints reuse the verified immutable records') self.assertEqual(snapshot.call_count,2,'Root borrowing must not clone an unused public snapshot') decoded=[call for call in decode.call_args_list if call.args[0] in canonical_documents] self.assertEqual(len(decoded),0,'Byte-identical canonical records need no repeat decoding within one Store') @@ -156,11 +192,13 @@ def test_verified_decode_cache_still_rejects_same_length_corruption_with_restore self.assertEqual(len(changed),len(raw));self.assertNotEqual(changed,raw) path.chmod(0o600);path.write_bytes(changed) os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) - self.assert_code('stale',self.store.snapshot) - self.assert_code('stale',self.store.summary) - with self.assertRaises(ProjectStoreError) as caught: - with self.store.root_directory(verify_after=True):pass - self.assertEqual(caught.exception.code,'stale') + # Ordinary reads keep serving the already verified value, never the + # corrupt bytes; explicit verification, Save and reopen refuse them. + self.assertEqual(self.store.snapshot()['records'][0]['document']['text'],'Alpha\n') + self.assert_code('stale',self.store.verify_records) + self.assert_code('stale',self.store.save,self.store.summary()['checkpoint']) + self.store.close() + self.assert_code('invalid',ProjectStore.open,self.root) def test_reusing_an_unreferenced_revision_still_checks_bytes_and_syncs_it(self): first, second = record('first'), record('second') @@ -171,7 +209,9 @@ def test_reusing_an_unreferenced_revision_still_checks_bytes_and_syncs_it(self): with patch.object(store_module, '_sync_file', wraps=sync) as syncs: committed = self.commit([first, second], key='reuse-first') self.assertFalse(any(call.args[1].startswith('.record-') for call in writes.call_args_list)) - self.assertEqual(syncs.call_count, 2) + # Only the newly referenced unreferenced revision is checked and synced; + # the unchanged current revision is reused without rewrite or re-sync. + self.assertEqual(syncs.call_count, 1) self.assertEqual(self.reopen(), committed) def test_reusing_a_corrupt_unreferenced_revision_never_publishes_it(self): @@ -190,7 +230,7 @@ def test_existing_revision_sync_failure_keeps_the_previous_checkpoint(self): self.commit([record()], key='initial') before = self.store.snapshot() with patch.object(store_module, '_sync_file', side_effect=OSError(errno.EIO, 'existing revision sync failed')): - self.assert_code('unavailable', self.commit, [record()], key='reuse-with-sync-error') + self.assert_code('unavailable', self.commit, [record(text='Changed revision')], key='new-revision-sync-error') self.assertEqual(self.reopen(), before) def test_fresh_copy_and_moved_folder_keep_identity_and_values(self): @@ -308,7 +348,8 @@ def test_missing_or_changed_canonical_record_never_becomes_empty_project(self): path = self.root / '.tuldok' / entry['path'] raw = path.read_bytes() path.write_bytes(raw.replace(b'Unique', b'Broken')) - self.assert_code('stale', self.store.snapshot) + self.assertEqual(self.store.snapshot()['records'][0]['document']['history'][0]['note'], 'Unique annotation') + self.assert_code('stale', self.store.verify_records) self.store.close() self.assert_code('invalid', ProjectStore.open, self.root) path.unlink() diff --git a/tests/test_project_validation_cache.py b/tests/test_project_validation_cache.py new file mode 100644 index 00000000..034d50dd --- /dev/null +++ b/tests/test_project_validation_cache.py @@ -0,0 +1,105 @@ +"""Private workspace caches that make reopening an unchanged project fast.""" +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import project_store as store_module +from project_store import ProjectStore +from project_validation_cache import CanonicalDigestCache, ProjectValidationCache + +PROJECT = 'a' * 32 +CHECKPOINT = 'b' * 64 + + +def note(identity, text): + return dict(family='workbench_text', id=identity, + document=dict(format='workbench_text_v1', text=text, history=[])) + + +class ValidationCacheTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-validation-cache-') + self.addCleanup(temporary.cleanup) + self.control = Path(temporary.name) + + def test_marker_binds_project_checkpoint_and_validator_fingerprint(self): + cache = ProjectValidationCache(self.control, fingerprint='1' * 64) + self.assertFalse(cache.known(PROJECT, CHECKPOINT)) + cache.remember(PROJECT, CHECKPOINT) + self.assertTrue(cache.known(PROJECT, CHECKPOINT)) + self.assertTrue(ProjectValidationCache(self.control, fingerprint='1' * 64).known(PROJECT, CHECKPOINT)) + self.assertFalse(cache.known(PROJECT, 'c' * 64)) + self.assertFalse(cache.known('d' * 32, CHECKPOINT)) + self.assertFalse(ProjectValidationCache(self.control, fingerprint='2' * 64).known(PROJECT, CHECKPOINT), + 'Changed validator source must revalidate') + + def test_tampered_or_invalid_markers_are_not_trusted(self): + cache = ProjectValidationCache(self.control, fingerprint='1' * 64) + cache.remember(PROJECT, CHECKPOINT) + marker = next((self.control / '.project-validation').glob('*.json')) + marker.write_text('{"format":"project_validation_marker_v1","checkpoint":"other"}') + self.assertFalse(cache.known(PROJECT, CHECKPOINT)) + with self.assertRaises(ValueError): + cache.known('not-a-project', CHECKPOINT) + + def test_default_fingerprint_covers_validating_sources(self): + self.assertRegex(ProjectValidationCache(self.control).fingerprint, '^[a-f0-9]{64}$') + + +class CanonicalDigestCacheTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-canonical-cache-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.control = self.root / 'control' + self.control.mkdir() + store = ProjectStore.create(self.root / 'project', 'Canonical fixture', 'new', 'a' * 64) + self.records = [note('one', 'Alpha ☃'), note('two', 'Beta\nwith "quotes"')] + self.committed = store.commit(self.records, store.summary()['checkpoint'], 'seed', 'b' * 64) + store.close() + + def open(self, cache): + store = ProjectStore.open(self.root / 'project', canonical_digests=cache) + self.addCleanup(store.close) + return store + + def test_reopen_skips_only_repeated_canonical_proofs_and_keeps_exact_values(self): + cache = CanonicalDigestCache(self.control) + with patch.object(store_module, '_decode', wraps=store_module._decode) as decode: + store = self.open(cache) + first = store.snapshot() + store.close() + self.assertEqual(first, self.committed) + self.assertEqual(decode.call_count, 1 + len(self.records), 'Checkpoint plus each record proven once') + reopened = CanonicalDigestCache(self.control) + with patch.object(store_module, '_decode', wraps=store_module._decode) as decode: + second = ProjectStore.open(self.root / 'project', canonical_digests=reopened) + try: + self.assertEqual(second.snapshot(), self.committed) + finally: + second.close() + self.assertEqual(decode.call_count, 1, 'Only the checkpoint itself is decoded canonically') + + def test_corruption_still_fails_with_a_warm_canonical_cache(self): + cache = CanonicalDigestCache(self.control) + self.open(cache).close() + manifest = store_module._decode((self.root / 'project/.tuldok/project.json').read_bytes()) + path = self.root / 'project/.tuldok' / manifest['inventory'][0]['path'] + path.write_bytes(path.read_bytes().replace(b'Alpha', b'Omega')) + with self.assertRaises(store_module.ProjectStoreError): + ProjectStore.open(self.root / 'project', canonical_digests=CanonicalDigestCache(self.control)) + + def test_commit_registers_new_revisions_and_unreadable_cache_is_ignored(self): + cache = CanonicalDigestCache(self.control) + store = self.open(cache) + store.commit([*self.records, note('three', 'Gamma')], store.summary()['checkpoint'], 'more', 'c' * 64) + manifest = store_module._decode((self.root / 'project/.tuldok/project.json').read_bytes()) + added = next(entry for entry in manifest['inventory'] if entry['id'] == 'three') + self.assertIn(added['sha256'], CanonicalDigestCache(self.control)) + cache.path.write_bytes(b'truncated') + self.assertNotIn(added['sha256'], CanonicalDigestCache(self.control)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_validation_delta.py b/tests/test_project_validation_delta.py new file mode 100644 index 00000000..73dfea16 --- /dev/null +++ b/tests/test_project_validation_delta.py @@ -0,0 +1,67 @@ +"""Edit-scoped validation re-checks exactly what an edit can affect.""" +import os +import unittest +from unittest.mock import patch + +from project_projection import validate_project_records +from project_validation_delta import COMPONENTS, ValidationDelta, edit_delta +from workbench import WorkbenchError + +PROJECT = 'a' * 32 +BROKEN_TEXT = dict(family='workbench_text', id='broken', document=dict(format='workbench_text_v1', text=17)) + + +class DeltaRules(unittest.TestCase): + def test_new_artifact_declarations_touch_no_component(self): + delta = ValidationDelta(frozenset({('project_artifact', 'b' * 64)}), added=frozenset({('project_artifact', 'b' * 64)})) + self.assertEqual(delta.families, frozenset()) + self.assertFalse(any(delta.touches(component) for component in COMPONENTS)) + changed = ValidationDelta(frozenset({('project_artifact', 'b' * 64)})) + self.assertTrue(changed.touches('history'), 'A changed existing declaration is re-checked') + + def test_receipt_rules(self): + delta = ValidationDelta(frozenset({('workbench_record', 'r'), ('project_operation', 'new')}), + frozenset({'data.import_text'})) + self.assertTrue(delta.receipt_needed('data.annotate', ('project_operation', 'old')), 'Data receipts bind to changed data records') + self.assertFalse(delta.receipt_needed('files.adopt', ('project_operation', 'old'))) + self.assertTrue(delta.receipt_needed('files.adopt', ('project_operation', 'new')), 'The new receipt is always checked') + self.assertTrue(delta.receipt_needed('unclassified.operation', ('project_operation', 'old')), 'Unknown receipts are always checked') + runtime = ValidationDelta(frozenset({('project_operation', 'new')}), frozenset({'runtime.capture'})) + self.assertTrue(runtime.receipt_needed('runtime.abandon', ('project_operation', 'old')), 'Shared key spaces are re-checked') + self.assertFalse(runtime.receipt_needed('studies.save', ('project_operation', 'old'))) + self.assertTrue(ValidationDelta(frozenset(), frozenset({'studies.save'})).history_receipt_added()) + + +class ScopedValidation(unittest.TestCase): + def validate(self, records, delta): + with edit_delta(delta): + validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_touched_component_still_rejects_an_invalid_change(self): + with self.assertRaises(WorkbenchError): + self.validate([BROKEN_TEXT], ValidationDelta(frozenset({('workbench_text', 'broken')}))) + with self.assertRaises(WorkbenchError): + validate_project_records([BROKEN_TEXT], project_id=PROJECT, artifacts=None) + + def test_untouched_component_is_not_revalidated(self): + # The base checkpoint is assumed validated; an unrelated edit skips data. + with patch.dict(os.environ, {'TULDOK_VALIDATION_CROSSCHECK': '0'}): + self.validate([BROKEN_TEXT], ValidationDelta(frozenset({('project_file', 'other')}))) + + def test_crosscheck_reports_a_skipped_failure(self): + # Only possible when the 'validated base' assumption is violated, as here. + with patch.dict(os.environ, {'TULDOK_VALIDATION_CROSSCHECK': '1'}): + with self.assertRaisesRegex(AssertionError, 'disagrees'): + self.validate([BROKEN_TEXT], ValidationDelta(frozenset({('project_file', 'other')}))) + + def test_identity_and_claim_checks_always_run(self): + duplicate = [BROKEN_TEXT, dict(BROKEN_TEXT)] + with self.assertRaises(WorkbenchError): + self.validate(duplicate, ValidationDelta(frozenset({('project_file', 'other')}))) + unknown = [dict(family='unclaimed_family', id='x', document={})] + with self.assertRaises(WorkbenchError): + self.validate(unknown, ValidationDelta(frozenset({('project_file', 'other')}))) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_working_store.py b/tests/test_project_working_store.py new file mode 100644 index 00000000..d943af12 --- /dev/null +++ b/tests/test_project_working_store.py @@ -0,0 +1,231 @@ +"""Database working copy of an external project: import, commit, save.""" +import copy +import os +from pathlib import Path +import shutil +import tempfile +import unittest +import uuid +from unittest.mock import patch + +import project_store as store_module +from project_store import ProjectStore, ProjectStoreError +from project_working_store import WorkingProjectStore + + +def note(identity, text): + return dict(family='workbench_text', id=identity, + document=dict(format='workbench_text_v1', text=text, history=[])) + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class WorkingProjectStoreTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-working-store-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.folder_path = self.root / 'project' + store = ProjectStore.create(self.folder_path, 'Working fixture', 'new', 'a' * 64) + self.records = [note('one', 'Alpha ☃'), note('two', 'Beta "quoted"\nline'), note('three', 'Gamma')] + self.initial = store.commit(self.records, store.summary()['checkpoint'], 'seed', 'b' * 64) + store.close() + self.working_id = uuid.uuid4().hex + self.addCleanup(self.drop_schema) + self.opened = [] + self.addCleanup(lambda: [store.close() for store in self.opened]) + + def drop_schema(self): + import psycopg + from psycopg import sql + from metadata import connection_info + with psycopg.connect(connection_info(None), autocommit=True) as connection: + connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format( + sql.Identifier('working_' + self.working_id))) + + def folder(self): + parent, leaf = os.path.split(str(self.folder_path)) + descriptor = os.open(parent, store_module.DIR_FLAGS) + try: + return ProjectStore.open_folder_at(descriptor, leaf, root_path=parent) + finally: + os.close(descriptor) + + def open(self): + store = WorkingProjectStore.open(self.folder(), self.working_id) + self.opened.append(store) + return store + + def reopen(self, store): + store.close() + return self.open() + + def file_snapshot(self): + with ProjectStore.open(self.folder_path) as store: + return store.snapshot() + + def test_import_matches_the_file_store_exactly(self): + store = self.open() + self.assertEqual(store.snapshot(), self.initial) + self.assertEqual(store.summary(), {k: self.initial[k] for k in ('project_id', 'name', 'checkpoint')}) + self.assertFalse(store.dirty()) + self.assertEqual(store.records('workbench_text'), self.initial['records']) + self.assertEqual(store.record('workbench_text', 'two'), self.initial['records'][2]) + self.assertIsNone(store.record('workbench_text', 'absent')) + + def test_unchanged_folder_reattaches_without_reading_records(self): + store = self.open() + store.close() + with patch.object(ProjectStore, 'folder_record', side_effect=AssertionError('Record file read')): + again = self.open() + self.assertEqual(again.snapshot(), self.initial) + + def test_commit_stays_in_database_and_matches_file_store_identity(self): + store = self.open() + before_files = sorted(p.relative_to(self.folder_path) for p in self.folder_path.rglob('*')) + before_checkpoint = (self.folder_path / '.tuldok/project.json').read_bytes() + changed = copy.deepcopy(self.records) + changed[1]['document']['text'] = 'Edited in Tuldok' + changed.append(note('four', 'Delta')) + committed = store.commit(changed, self.initial['checkpoint'], 'edit', 'c' * 64) + self.assertTrue(store.dirty()) + self.assertEqual(sorted(p.relative_to(self.folder_path) for p in self.folder_path.rglob('*')), before_files) + self.assertEqual((self.folder_path / '.tuldok/project.json').read_bytes(), before_checkpoint) + # The same commit on a file store yields the identical checkpoint. + mirror = self.root / 'mirror' + shutil.copytree(self.folder_path, mirror) + with ProjectStore.open(mirror) as file_store: + expected = file_store.commit(copy.deepcopy(changed), self.initial['checkpoint'], 'edit', 'c' * 64) + self.assertEqual(committed['checkpoint'], expected['checkpoint']) + self.assertEqual(store.snapshot(), expected) + reopened = self.reopen(store) + self.assertEqual(reopened.snapshot(), expected, 'Unsaved work persists across sessions') + self.assertTrue(reopened.dirty()) + + def test_changed_hint_and_exact_comparison_agree(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Hinted change' + hinted = store.commit(changed, self.initial['checkpoint'], 'hinted', 'd' * 64, + changed={('workbench_text', 'one')}) + store.close() + other = WorkingProjectStore.open(self.folder(), uuid.uuid4().hex) + self.addCleanup(lambda: self._drop(other)) + exact = other.commit(copy.deepcopy(changed), self.initial['checkpoint'], 'hinted', 'd' * 64) + self.assertEqual(hinted['checkpoint'], exact['checkpoint']) + + def _drop(self, store): + schema = store.schema + store.close() + import psycopg + from psycopg import sql + from metadata import connection_info + with psycopg.connect(connection_info(None), autocommit=True) as connection: + connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(schema))) + + def test_save_exports_byte_identical_files(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[2]['document']['text'] = 'Saved change' + committed = store.commit(changed, self.initial['checkpoint'], 'edit', 'e' * 64) + store.save(committed['checkpoint']) + self.assertFalse(store.dirty()) + saved = store.snapshot() + store.close() + self.assertEqual(self.file_snapshot(), saved) + with patch.object(ProjectStore, 'folder_record', side_effect=AssertionError('Re-import after own save')): + self.assertEqual(self.open().snapshot(), saved) + + def test_clean_working_copy_imports_only_changed_external_records(self): + store = self.open() + store.close() + with ProjectStore.open(self.folder_path) as file_store: + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Changed outside Tuldok' + external = file_store.commit(changed, self.initial['checkpoint'], 'outside', 'f' * 64) + reads = [] + original = ProjectStore.folder_record + def observe(folder, entry): + reads.append(entry['id']) + return original(folder, entry) + with patch.object(ProjectStore, 'folder_record', observe): + store = self.open() + self.assertEqual(reads, ['one']) + self.assertEqual(store.snapshot(), external) + + def test_external_change_with_unsaved_work_is_an_explicit_conflict(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Unsaved' + committed = store.commit(changed, self.initial['checkpoint'], 'unsaved', '1' * 64) + store.close() + with ProjectStore.open(self.folder_path) as file_store: + other = copy.deepcopy(self.records) + other[1]['document']['text'] = 'Outside' + file_store.commit(other, self.initial['checkpoint'], 'outside', '2' * 64) + with self.assertRaises(ProjectStoreError) as caught: + self.open() + self.assertEqual(caught.exception.code, 'conflict') + + def test_save_refuses_to_overwrite_an_external_change(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Unsaved' + committed = store.commit(changed, self.initial['checkpoint'], 'unsaved', '3' * 64) + external = (self.folder_path / '.tuldok/project.json') + before = external.read_bytes() + # Simulate a checkout replacing project.json while Tuldok holds the folder. + raw = before.replace(b'"Working fixture"', b'"Renamed outside"') + external.chmod(0o600) + external.write_bytes(raw) + with self.assertRaises(ProjectStoreError) as caught: + store.save(committed['checkpoint']) + self.assertEqual(caught.exception.code, 'conflict') + self.assertEqual(external.read_bytes(), raw) + + def test_second_session_on_the_same_working_copy_is_busy(self): + self.open() + with self.assertRaises(ProjectStoreError) as caught: + WorkingProjectStore(self.folder(), self.working_id) + self.assertEqual(caught.exception.code, 'busy') + + def test_receipt_replay_and_stale_checkpoint(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Once' + first = store.commit(changed, self.initial['checkpoint'], 'once', '4' * 64) + self.assertEqual(store.commit(copy.deepcopy(changed), first['checkpoint'], 'once', '4' * 64)['checkpoint'], + first['checkpoint']) + self.assertIsNotNone(store.reconcile('once', '4' * 64)) + with self.assertRaises(ProjectStoreError) as caught: + store.commit(changed, self.initial['checkpoint'], 'later', '5' * 64) + self.assertEqual(caught.exception.code, 'stale') + + def test_envelope_digests_equal_canonical_envelope_hashes(self): + from project_composition import _record_sha, known_envelope_digests + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Committed after import' + store.commit(changed, self.initial['checkpoint'], 'edit', '6' * 64) + digests = store.envelope_digests() + snapshot = store.snapshot() + self.assertEqual(set(digests), {(r['family'], r['id']) for r in snapshot['records']}) + for record in snapshot['records']: + self.assertEqual(digests[(record['family'], record['id'])], _record_sha(record)) + # A supplied digest applies only to that exact object. + record = snapshot['records'][0] + with known_envelope_digests({id(record): (record, 'f' * 64)}): + self.assertEqual(_record_sha(record), 'f' * 64) + self.assertNotEqual(_record_sha(copy.deepcopy(record)), 'f' * 64) + store._execute(store._q('UPDATE {records} SET envelope_sha256=NULL')) + self.assertEqual(store.envelope_digests(), digests, 'Missing digests are recomputed from stored bytes') + + def test_verify_records_detects_working_copy_corruption(self): + store = self.open() + store.verify_records() + store._execute(store._q("UPDATE {records} SET raw = convert_to(replace(convert_from(raw, 'UTF8'), 'Gamma', 'Omega'), 'UTF8') WHERE id='three'")) + with self.assertRaises(ProjectStoreError): + store.verify_records() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_workspace.py b/tests/test_project_workspace.py index a78982f4..aa0b5c4d 100644 --- a/tests/test_project_workspace.py +++ b/tests/test_project_workspace.py @@ -49,6 +49,8 @@ def cleanup_owned(self): value = json.loads(manifest.read_bytes()) schemas.add('projection_' + value['projection_id'] if value['version'] == 2 else 'catalog_' + value['dataset_id']) + from project_working_store import registered_schemas + schemas.update(registered_schemas(control)) with psycopg.connect(connection_info(), autocommit=True) as connection: for schema in schemas: connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(schema))) @@ -76,8 +78,8 @@ def test_status_returns_current_metadata_without_copying_the_record_graph(self): self.create() self.import_record('A retained record unrelated to the project summary') expected = self.state() - with patch.object(self.workspace._store, '_snapshot', - wraps=self.workspace._store._snapshot) as snapshots: + with patch.object(self.workspace._store, 'snapshot', + wraps=self.workspace._store.snapshot) as snapshots: observed = self.state() self.assertEqual(observed, expected) self.assertEqual(snapshots.call_count, 0, @@ -126,20 +128,25 @@ def assert_code(self, code, action, arguments): return caught.exception def unknown_import(self): + """Commit an edit whose PostgreSQL COMMIT acknowledgement is lost. + + Records and projection rows commit together in the database working + copy, so the only uncertain outcome is a lost commit acknowledgement. + """ + from contextlib import contextmanager + from metadata import CommitOutcomeUnknown before = self.state() arguments = self.mutation(before, asset=dict(text='Uncertain publication', groups=['owned'])) - replace, fsync = project_store.os.replace, project_store.os.fsync - published = False - def after_replace(*args, **kwargs): - nonlocal published - replace(*args, **kwargs) - published = True - def fail_directory(fd): - if published and fd == self.workspace._store._meta_fd: - raise OSError(errno.EIO, 'actual checkpoint directory fsync failure') - return fsync(fd) - with patch.object(project_store.os, 'replace', side_effect=after_replace), patch.object( - project_store.os, 'fsync', side_effect=fail_directory): + db = self.workspace._projection.db + real = db.transaction + @contextmanager + def lost_acknowledgement(*args, **kwargs): + outermost = not db._rollback + with real(*args, **kwargs) as value: + yield value + if outermost: + raise CommitOutcomeUnknown('fixture lost COMMIT acknowledgement') + with patch.object(db, 'transaction', lost_acknowledgement): self.assert_code('unknown', 'import_text', arguments) return before, arguments @@ -167,20 +174,15 @@ def test_unknown_fsync_status_and_explicit_current_open_reconcile_canonical_edit with self.owner.bound(self.workspace): self.assertEqual(len(self.workspace._text.records()['items']), 1) - def test_unknown_recovery_does_not_succeed_until_directory_sync_failure_clears(self): + def test_unknown_recovery_does_not_succeed_until_the_working_copy_is_available(self): self.create() _before, edit = self.unknown_import() recovery = self.state() old_projection = self.workspace._projection - metadata = (self.external / 'alpha/.tuldok').stat() - fsync = project_store.os.fsync - def fail_metadata_sync(fd): - info = os.fstat(fd) - if (info.st_dev, info.st_ino) == (metadata.st_dev, metadata.st_ino): - raise OSError(errno.EIO, 'persistent checkpoint directory fsync failure') - return fsync(fd) reopen = self.mutation(recovery, root='fixture', directory='alpha') - with patch.object(project_store.os, 'fsync', side_effect=fail_metadata_sync): + from project_working_store import WorkingProjectStore + with patch.object(WorkingProjectStore, 'open', + side_effect=project_store.ProjectStoreError('Local PostgreSQL is unavailable.', 'unavailable', 503)): self.assert_code('unavailable', 'open', reopen) self.assertEqual(self.state(), recovery) self.assertIs(self.workspace._projection, old_projection) @@ -208,7 +210,7 @@ def fail_published_parent(fd): self.assertEqual(selected['project']['project_id'], expected['project_id']) self.assertEqual(selected['project']['checkpoint'], expected['checkpoint']) - def test_real_sql_commit_failure_reports_durable_receipt_and_explicit_open_restores(self): + def test_rejected_sql_commit_rolls_back_records_and_projection_together(self): self.create() before = self.state() projection = self.workspace._projection @@ -216,22 +218,17 @@ def test_real_sql_commit_failure_reports_durable_receipt_and_explicit_open_resto BEGIN RAISE EXCEPTION 'fixture deferred rejection' USING ERRCODE='23514'; END $$""") projection.execute("""CREATE CONSTRAINT TRIGGER fixture_workspace_commit_failure AFTER INSERT ON workbench_records DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION fixture_workspace_refuse_commit()""") - arguments = self.mutation(before, asset=dict(text='Durable despite rejected SQL commit', groups=['owned'])) - failure = self.assert_code('projection_unavailable', 'import_text', arguments) - canonical = self.workspace._store.snapshot() - self.assertEqual(failure.receipt, dict(committed=True, selected=True, project_id=canonical['project_id'], - checkpoint=canonical['checkpoint'], request_key=arguments['request_key'])) + arguments = self.mutation(before, asset=dict(text='Rejected with its SQL commit', groups=['owned'])) + with self.assertRaises(Exception): + self.call('import_text', arguments) + # Nothing was committed: same checkpoint, no records, still open. + self.assertEqual(self.state(), before) + self.assertEqual(self.workspace._store.summary()['checkpoint'], before['project']['checkpoint']) self.assertEqual(projection.execute('SELECT count(*) FROM workbench_records').fetchone()[0], 0) - recovery = self.state() - self.assertEqual(recovery['state'], 'recovery_required') - self.assertEqual(recovery['project'], before['project']) - self.assertEqual(recovery['recovery']['reason'], 'projection_unavailable') - opened = self.call('open', self.mutation(recovery, root='fixture', directory='alpha')) - self.assertEqual(opened['project']['checkpoint'], canonical['checkpoint']) - result = self.call('import_text', dict(arguments, binding=opened['binding'])) - self.assertEqual(result['record']['text'], 'Durable despite rejected SQL commit') - self.assertEqual(self.workspace._projection.execute('SELECT count(*) FROM workbench_records').fetchone()[0], 1) - self.assert_code('stale_project', 'import_text', arguments) + projection.execute('DROP TRIGGER fixture_workspace_commit_failure ON workbench_records') + result = self.call('import_text', arguments) + self.assertEqual(result['record']['text'], 'Rejected with its SQL commit') + self.assertEqual(projection.execute('SELECT count(*) FROM workbench_records').fetchone()[0], 1) def test_failed_reconstruction_keeps_recovery_generation_and_old_projection(self): self.create() diff --git a/torch_runtime.py b/torch_runtime.py index e20cb384..80c1b8a9 100644 --- a/torch_runtime.py +++ b/torch_runtime.py @@ -110,6 +110,15 @@ def inventory(configuration): return validate_manifest(value) +def _contained_path(path, roots): + """Canonical absolute path under one of roots (one Path construction).""" + if not (type(path) is str and 0