From 076e4565a8aa9cba762460a9424d822ca905f15c Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 08:21:02 -0700 Subject: [PATCH 01/13] perf(projects): open large projects without repeated verification and copies Opening the 1.77 GB / 54k-record Rheon portfolio project took ~9 minutes and 21.7 GB of memory; each later request re-hashed every record file. Store - Freshness compares checkpoint bytes and root binding only. Record files are content-addressed and immutable and are verified when first decoded, so a byte-identical checkpoint names the same verified records. Save and the new verify_records() still re-read and SHA256-check every referenced revision; reopen re-verifies everything. - Commit reuses unchanged revisions (exact JSON equality incl. types and float sign) instead of re-encoding, rewriting and re-reading all records, and primes the read cache with the committed records. - Receipt digests stream from canonical record bytes (byte-identical to hashing the re-encoded list), removing a full re-encode on load and commit. Session / owners - Remember complete domain validation per (project, checkpoint, validator fingerprint) in the private workspace control directory; the fingerprint covers all Tuldok sources, extension schemas and validator package versions. - Bound-checkpoint reads: records(*families), record(family, id), records_for(*families), active(*families), memoized operations() and derived() facts, with view-tolerant helpers for snapshot-only views. - Decode retained history once per checkpoint, shared by history reads and Study authoring restore. - Data/material restore and source sync copy only their families. - One-pass MeasurementTable.query and progression_origins (CodeRabbit #120). Tests updated for the new integrity contract: ordinary reads serve verified values and never corrupt bytes; explicit verification, Save and reopen refuse later on-disk corruption. Co-Authored-By: Claude Opus 5.5 --- project_data.py | 22 +-- project_dispatch.py | 4 +- project_execution.py | 7 +- project_history.py | 40 ++++-- project_material.py | 18 +-- project_native_execution.py | 11 +- project_native_preparation.py | 5 +- project_producer_execution.py | 5 +- project_projection.py | 11 +- project_runtime.py | 9 +- project_session.py | 180 +++++++++++++++++++++++- project_store.py | 167 +++++++++++++++++++--- project_study_execution.py | 14 +- project_validation_cache.py | 96 +++++++++++++ project_workspace.py | 7 +- retained_measurements.py | 31 ++-- tests/test_project_artifacts.py | 16 ++- tests/test_project_history.py | 7 +- tests/test_project_measurement_reads.py | 10 +- tests/test_project_session_freshness.py | 25 ++-- tests/test_project_store.py | 61 ++++++-- 21 files changed, 619 insertions(+), 127 deletions(-) create mode 100644 project_validation_cache.py diff --git a/project_data.py b/project_data.py index ca70bab0..c51bf64c 100644 --- a/project_data.py +++ b/project_data.py @@ -20,6 +20,7 @@ from saved_searches import SavedSearches, criteria from saved_selections import SavedSelections from dataset_releases import Releases +from project_session import session_active, session_records, session_records_for FAMILIES = { 'workbench_record': 'workbench_record_v1', @@ -971,11 +972,10 @@ def capture_runtime_changes(self, before_records, *, operation, key): return captured def _sync_sources(self): - records = self.session.snapshot['records'] - from project_fork import active_records - active = active_records(records, project_id=self.session.project_id) - self.context.project_sources = {r['id']: copy.deepcopy(r['document']['source']) for r in active if r['family'] == 'workbench_record'} - self._declarations = {r['id']: r['document'] for r in records if r['family'] == 'project_artifact'} + # Only the selected families are copied; fork history is still checked. + active = session_active(self.session, 'workbench_record') + self.context.project_sources = {r['id']: r['document']['source'] for r in active if r['family'] == 'workbench_record'} + self._declarations = {r['id']: r['document'] for r in session_records(self.session, 'project_artifact')} def _available(self, record_id): source = self.context.project_sources.get(record_id) @@ -1333,16 +1333,18 @@ def _capture(self, result, before_records, operation, key): return {'records': list(unique.values()), 'result': copy.deepcopy(result)} def restore(self, snapshot=None): - snapshot = snapshot or self.session.snapshot - if snapshot != self.session.snapshot: + if snapshot is not None and snapshot != self.session.snapshot: fail('Data reconstruction snapshot is stale.', 'stale_project', 409) - owned = decode_data_records(snapshot['records']) + # The decoder reads its own families and project_fork overlays, and its + # fork validation needs every record identity; unrelated documents are + # shared read-only instead of copying unrelated Run history. + families = (*FAMILIES, 'workbench_text', 'project_artifact', 'project_fork') + owned = decode_data_records(session_records_for(self.session, *families)) self._sync_sources() # Validate present source bytes through the existing semantic owners # before touching any selected SQL row. Missing bytes retain metadata. self._verify_sources(owned) - from project_fork import active_records - active = active_records(snapshot['records'], project_id=self.session.project_id) + active = session_active(self.session, *families) identities = {(r['family'], r['id']) for r in active} owned = [r for r in owned if (r['family'], r['id']) in identities] with self.db.lock, self.db.transaction(): diff --git a/project_dispatch.py b/project_dispatch.py index de519135..e517c2e6 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -9,6 +9,7 @@ import copy from workbench import WorkbenchError +from project_session import session_records _BOUND_FIELDS = frozenset(('binding', 'checkpoint', 'expected_checkpoint', 'request_key')) @@ -61,8 +62,7 @@ def _origin_items(items, arguments): def _declarations(projection): - return {item['id']: item['document'] for item in projection.session.snapshot['records'] - if item['family'] == 'project_artifact'} + return {item['id']: item['document'] for item in session_records(projection.session, 'project_artifact')} def _artifact(projection, sha): diff --git a/project_execution.py b/project_execution.py index 0e4941c6..478915d2 100644 --- a/project_execution.py +++ b/project_execution.py @@ -21,6 +21,7 @@ from project_history import capture_run, declarations from project_session import operation_records from workbench import WorkbenchError +from project_session import session_operations OPERATIONS = frozenset(('runs.submit_external', 'runs.submit', 'runs.retry', 'runs.cancel', 'runs.approve_external')) @@ -600,7 +601,7 @@ def native_preflight(self,graph_id,revision,expected_checkpoint,key): graph_id,revision,expected_checkpoint,key) def _native_replay(self,operation,target,body,expected_checkpoint,key): - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is None:return None if (prior['operation']!=operation or self.db.execute('SELECT id FROM workspace_runs WHERE id=%s', (prior['result'].get('run_id'),)).fetchone() is None): @@ -656,7 +657,7 @@ def submit_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require_external_submission() with self.access.lock, self.session.lock, self.db.lock: self.session.check_current() - prior = operation_records(self.session.snapshot['records']).get(key) + prior = session_operations(self.session).get(key) if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: @@ -679,7 +680,7 @@ def approve_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require('approve-execution', 'runs.approve_external', owner_only=True) with self.access.lock, self.session.lock, self.db.lock: self.session.check_current() - prior = operation_records(self.session.snapshot['records']).get(key) + prior = session_operations(self.session).get(key) if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: diff --git a/project_history.py b/project_history.py index b69eccce..45d8ee86 100644 --- a/project_history.py +++ b/project_history.py @@ -25,6 +25,7 @@ from runs import (ACTIVE, TERMINAL, TEST_FILES, MAX_ARTIFACT_BYTES, validate_retained_run, project_run_projection, public_run_projection, artifact_capture_source, validate_external_artifact_inventory) from workbench import WorkbenchError, timestamp +from project_session import session_derived, session_operations, session_records FAMILY_FORMATS = { 'study_history': 'project_study_history_v1', @@ -562,6 +563,17 @@ def visit_run(key,path): return facts +def checkpoint_facts(session, materials): + """Decoded history of the session's bound checkpoint, shared by its owners. + + Decoded once per checkpoint; callers copy any part they return. + """ + def compute(): + records=session.snapshot['records'] + return decode_records(records,materials=materials.read_view(records)) + return session_derived(session, 'history_facts',compute) + + def decode_records(records, *, materials=None, artifacts=None): try: return _decode_records(records,materials=materials,artifacts=artifacts) @@ -755,15 +767,18 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): def _facts(self): self.session.check_current() + # Current canonical bytes are guarded on every call. A ProjectSession + # shares decoded metadata per checkpoint with the Study owner; other + # snapshot views keep this owner's private per-checkpoint cache. + # Returned values and permissions remain each read's responsibility. + if callable(getattr(self.session,'derived',None)): + return checkpoint_facts(self.session,self.materials) summary=self.session.summary checkpoint=(summary['project_id'],summary['checkpoint']) if self._decoded_checkpoint==checkpoint: return self._decoded_facts records=self.session.snapshot['records'] facts=decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) - # Current canonical bytes are guarded on every call. Only this owner's - # private decoded metadata is retained; returned values and permissions - # remain the responsibility of each read below. self._decoded_facts=facts self._decoded_checkpoint=checkpoint return facts @@ -827,10 +842,12 @@ def progression_origins(self,*,offset=0,limit=40): """Discover original namespaces without inventing current project ancestry.""" self._authorize('studies.lineage') natural(offset,'Origin offset');natural(limit,'Origin page limit',1,40) - facts=self._facts();origins={origin for origin,_ in facts.studies}|{origin for origin,_ in facts.trials} - items=[{'origin_catalog_id':origin, - 'study_count':sum(namespace==origin for namespace,_ in facts.studies), - 'trial_count':sum(namespace==origin for namespace,_ in facts.trials)} for origin in sorted(origins)] + facts=self._facts();counts={} + # One pass over Studies and Trials instead of origins x (studies+trials). + for namespace,_ in facts.studies:counts.setdefault(namespace,[0,0])[0]+=1 + for namespace,_ in facts.trials:counts.setdefault(namespace,[0,0])[1]+=1 + items=[{'origin_catalog_id':origin,'study_count':counts[origin][0],'trial_count':counts[origin][1]} + for origin in sorted(counts)] self._authorize('studies.lineage') return {'items':items[offset:offset+limit],'offset':offset,'limit':limit,'total':len(items)} @@ -935,7 +952,7 @@ def payload_set_sha256(self,run_id,*,origin_catalog_id=None): def disclosure_facts(self,run_id,*,origin_catalog_id=None): self._authorize('runs.get',run_id);origin=self._origin(origin_catalog_id) result=[] - for operation in operation_documents(self.session.snapshot['records']): + for operation in operation_documents(session_records(self.session, 'project_operation')): if operation['operation']!='history.disclose':continue value=validate_disclosure_fact(operation['result']) if value['origin_catalog_id']==origin and value['run_id']==run_id:result.append(value) @@ -965,8 +982,7 @@ def capture(result,before_records):return {'records':captured,'result':result} def dependencies(self,run_id,*,origin_catalog_id=None,offset=0,limit=100): self._authorize('runs.get',run_id);natural(offset,'Dependency offset');natural(limit,'Dependency limit',1,100) doc=self._record(self._facts().runs,run_id,origin_catalog_id);run=doc['run'] - declarations_by_sha={item['document']['sha256']:item['document'] for item in self.session.snapshot['records'] - if item['family']=='project_artifact'} + declarations_by_sha={item['document']['sha256']:item['document'] for item in session_records(self.session, 'project_artifact')} required=[] for edge in doc['payloads']: required.append({'name':edge['name'],'sha256':edge['blob']['sha256'],'bytes':edge['blob']['bytes'], @@ -1210,7 +1226,7 @@ def prepare_trial_from_material(self,study_id,body,expected_checkpoint,request_k closed(body,('revision','material','replicate'));body=checked_json(body) natural(body['replicate'],'Replicate') self.access.require('edit','studies.prepare_trial',study_id) - if request_key not in operation_records(self.session.snapshot['records']): + if request_key not in session_operations(self.session): self.materials.recipe(body['material'],verify=True) def build(facts): origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) @@ -1239,7 +1255,7 @@ def select_candidate(self,study_id,body,expected_checkpoint,request_key): origin=self.session.summary['project_id'] self.access.require('edit','studies.select_candidate',study_id) evidence=(self.evidence(body['run_id'],{},origin_catalog_id=origin) - if request_key not in operation_records(self.session.snapshot['records']) else None) + if request_key not in session_operations(self.session) else None) def build(facts): doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') diff --git a/project_material.py b/project_material.py index fbfa0189..aee2aee0 100644 --- a/project_material.py +++ b/project_material.py @@ -19,6 +19,7 @@ reference, natural, retained_json, MAX_SOURCE_BYTES) from pipeline import decode_graph, decode_recipe_draft, decode_recipe_material, verify_recipe_closure, digest, execution_digest from project_store import ProjectStoreError +from project_session import session_active, session_records_for FAMILIES = { 'model_definition': 'model_definition_history_v1', @@ -360,7 +361,7 @@ def __init__(self, session): def _documents(self): self.session.check_current() - return decode_records(self.session.snapshot['records']) + return decode_records(session_records_for(self.session, *FAMILIES, 'project_fork')) def read_view(self, records): """Own validated material facts from one guarded checkpoint read. @@ -380,8 +381,8 @@ def intents(self): from project_fork import active_records from project_material_fork import is_intent documents = self._documents() - active = {(r['family'], r['id']) for r in active_records( - self.session.snapshot['records'], project_id=self.session.summary['project_id'])} + active = {(r['family'], r['id']) for r in session_active(self.session, + *FAMILIES, project_id=self.session.summary['project_id'])} return [dict(family=family, id=locator, current=copy.deepcopy(doc['current']), fork_origin=copy.deepcopy(doc['fork_origin'])) for (family, locator), doc in sorted(documents.items()) @@ -392,7 +393,7 @@ def intent_capabilities(self): intents = self.intents() unresolved = {(r['family'],r['id']) for r in intents} from project_fork import active_records - active = active_records(self.session.snapshot['records'], project_id=self.session.summary['project_id']) + active = session_active(self.session, *EDITABLE, project_id=self.session.summary['project_id']) return dict(requires_reselection=[dict(family=r['family'],id=r['id'],fields=r['current']['required_reselection']) for r in intents], editable=[dict(family=r['family'],id=r['id']) for r in active if r['family'] in EDITABLE and (r['family'],r['id']) not in unresolved]) @@ -510,7 +511,7 @@ def _capture_material(self, family, value, root): audit=copy.deepcopy([fact for fact in self.owner_access.facts if fact['action']==('models.publish' if family=='model_material' else 'recipes.material') and fact['target'] in (value['ref'].get('id'),value['ref']['sha256'])]) if self.owner_access else [])) - prior=decode_records(self.session.snapshot['records']).get((family,record['id'])) + prior=decode_records(session_records_for(self.session, *FAMILIES, 'project_fork')).get((family,record['id'])) if prior is not None: if prior['value']!=record['document']['value'] or prior['artifacts']!=record['document']['artifacts']: raise WorkbenchError('Retained immutable material changed.','invalid',400) @@ -594,7 +595,7 @@ def capture_snapshot(self, value, raw, *, catalog_id): if type(raw) is not bytes or len(raw)>64*1024 or hashlib.sha256(raw).hexdigest()!=value['manifest_sha256'] or decode_retained_manifest(decode(raw))!=value['manifest']: raise WorkbenchError('Exact original project manifest bytes differ.') key=catalog_id+'.'+value['manifest_sha256'] - prior=decode_records(self.session.snapshot['records']).get(('project_snapshot',key)) + prior=decode_records(session_records_for(self.session, *FAMILIES, 'project_fork')).get(('project_snapshot',key)) if prior is not None: original=prior['value'] if any(original[field]!=value[field] for field in ('id','name','manifest_sha256','manifest')): @@ -628,7 +629,7 @@ def restore(self): documents=self._documents();catalog_id=self.session.db.dataset_id from project_fork import active_records from project_material_fork import is_intent - active={(r['family'],r['id']) for r in active_records(self.session.snapshot['records'],project_id=catalog_id)} + active={(r['family'],r['id']) for r in session_active(self.session, *FAMILIES,project_id=catalog_id)} with self.session.db.transaction(): for table in ('model_definition_drafts','workspace_graphs','workspace_recipe_drafts','model_materials','workspace_recipe_materials'): self.session.db.execute('DELETE FROM '+table) @@ -656,8 +657,7 @@ def reconcile_projection(self): catalog_id = self.session.db.dataset_id from project_fork import active_records from project_material_fork import is_intent - active = {(r['family'], r['id']) for r in active_records( - self.session.snapshot['records'], project_id=catalog_id)} + active = {(r['family'], r['id']) for r in session_active(self.session, *FAMILIES, project_id=catalog_id)} with self.session.db.transaction(): for (family, locator), doc in documents.items(): if (family, locator) not in active or doc['origin_catalog_id'] != catalog_id: diff --git a/project_native_execution.py b/project_native_execution.py index 5895e4d7..b2ff98ea 100644 --- a/project_native_execution.py +++ b/project_native_execution.py @@ -20,6 +20,7 @@ from project_native_process import ENVIRONMENT, PHASES, WORKERS, prepare, validate_prepared_spec, validate_job_evidence from project_session import REQUEST_KEY from workbench import WorkbenchError, timestamp +from project_session import session_record, session_records FAMILY = 'native_execution_history' FORMAT = 'native_execution_history_v1' @@ -404,7 +405,7 @@ def _retain_outputs(self,document,*,run=None,output=None): from project_store import DIR_FLAGS, FILE_FLAGS, ProjectStoreError, _open_directory_path, anchored_directory stage_id=document['phase_id'] if document['phase']=='verify': - matches=[row['document'] for row in self.session.snapshot['records'] + matches=[row['document'] for row in session_records(self.session, FAMILY) if row['family']==FAMILY and row['document']['phase']=='train' and row['document']['origin_catalog_id']==self.db.dataset_id and row['document']['run_id']==document['run_id']] @@ -634,8 +635,8 @@ def probe_current_run(self,owner,recipe): def capture_run_payloads(self,run,artifacts): """Copy a newly admitted frozen native input, then reuse exact custody.""" from projects import Projects - prior=next((record['document'] for record in self.session.snapshot['records'] - if record['family']=='run_history' and record['id']==run['dataset_id']+'.'+run['id']),None) + prior=session_record(self.session, 'run_history',run['dataset_id']+'.'+run['id']) + prior=None if prior is None else prior['document'] if prior is not None: retained=[copy.deepcopy(edge) for edge in prior['payloads'] if edge['name']=='release.zip' or edge['name'].startswith('source/')] @@ -881,8 +882,8 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru try:handle.close_control() except (OSError,recipe_process.ProcessError):pass if phase_id not in self._never_launched: - retained=next((record['document'] for record in self.session.snapshot['records'] - if record['family']==FAMILY and record['id']==self.session.project_id+'.'+phase_id),None) + retained=session_record(self.session, FAMILY,self.session.project_id+'.'+phase_id) + retained=None if retained is None else retained['document'] acknowledged=(self.session.available and retained is not None and retained['events'][-1]['state'] in ('completed','failed') and retained['events'][-1]['receipt_sha256'] is not None) diff --git a/project_native_preparation.py b/project_native_preparation.py index 24113f63..ee4195a8 100644 --- a/project_native_preparation.py +++ b/project_native_preparation.py @@ -19,6 +19,7 @@ identifier, model_aware_graph, closed, decode_probe_observation) from project_session import operation_records from workbench import WorkbenchError, timestamp +from project_session import session_operations OPERATIONS=frozenset(('native_preparation.intent','native_preparation.validate')) @@ -352,7 +353,7 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): self.access.require('edit','native_preparation.validation',graph_id,owner_only=True) self.access.require('execute','pipeline.probe_native',graph_id,owner_only=True) body=dict(graph_id=graph_id,revision=revision) - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is not None: return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Native validation replay launched a probe.')), @@ -398,7 +399,7 @@ def prepare_recipe(self,owner,graph_id,revision,model_ref,expected_checkpoint,ke self.access.require('execute','pipeline.probe_native',graph_id,owner_only=True) body=dict(graph_id=graph_id,revision=revision,model=model_ref,reconcile_capture=False) operation='recipes.prepare_builtin_material' - if key in operation_records(self.session.snapshot['records']): + if key in session_operations(self.session): return self.session.mutate(operation,graph_id,body,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Material replay launched a native probe.')), lambda value,before:dict(records=[],result=value)) diff --git a/project_producer_execution.py b/project_producer_execution.py index e3729233..8b37faf4 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -15,6 +15,7 @@ _CapturedEvidence, _composed, _declarations, _referenced_artifacts, decode_records, identity, shape) from workbench import WorkbenchError +from project_session import session_record, session_records KINDS = {'generation_history':'generation', 'grounded_history':'grounded', 'caption_proposal_history':'caption', 'text_proposal_history':'text_proposal'} @@ -81,7 +82,7 @@ def _owner(self, family): # Imported science retains its physical ordering. A fresh local # authority table must allocate beyond those observed ordinals. table=TABLES[family] - observed=max((r['document']['ordinal'] for r in self.session.snapshot['records'] + observed=max((r['document']['ordinal'] for r in session_records(self.session, family) if r['family']==family and r['document']['origin_catalog_id']==self.db.dataset_id),default=0) current=self.db.execute('SELECT COALESCE(max(ordinal),0) FROM '+table).fetchone()[0] maximum=max(observed,current) @@ -279,7 +280,7 @@ def start(self, family, body, expected_checkpoint, key): prepared = [] def invoke(): requested = body.get('request_id') if type(body) is dict else None - if requested is not None and any(r['family']==family and r['id']==self.db.dataset_id+'.'+requested for r in self.session.snapshot['records']): + if requested is not None and session_record(self.session, family,self.db.dataset_id+'.'+requested) is not None: if self.db.execute('SELECT id FROM '+TABLES[family]+' WHERE id=%s',(requested,)).fetchone() is None: raise WorkbenchError('A retained producer identity cannot acquire current admission. Start with a new identity.','unavailable',409) owner = self._owner(family) diff --git a/project_projection.py b/project_projection.py index 25b6bf28..611fff49 100644 --- a/project_projection.py +++ b/project_projection.py @@ -38,6 +38,7 @@ from project_measurements import receipt_validator as measurement_receipt_validator from workbench import WorkbenchError from workbench import timestamp +from project_session import session_record def validate_artifact_resolution(operation, result, records): @@ -116,7 +117,8 @@ def validate_project_records(records, *, project_id, artifacts): class ProjectProjection: - def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None): + def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None, + validation_cache=None): self.store, self.db, self.access = store, db, access self.custody = custody self.pending_original = [] @@ -129,7 +131,8 @@ def __init__(self, store, db, access, roots, *, custody=None, restore=True, runt self._runtime_sources = copy.deepcopy(runtime_sources or {}) try: self.session = ProjectSession(store, db, access, self.artifacts, - validate_records=self.validate_records) + validate_records=self.validate_records, + validation_cache=validation_cache) # Install before domain constructors retain their lock references. db.lock = ProjectLock(access.lock, self.session.lock, db.lock) self.producers = ProjectProducers(self.session) @@ -195,8 +198,8 @@ def resolve_artifact(self, body, expected_checkpoint, key): with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'artifacts.resolve', body['sha256'], owner_only=True) self.session.check_current() - declaration = next((item['document'] for item in self.session.snapshot['records'] - if item['family'] == 'project_artifact' and item['id'] == body['sha256']), None) + declaration = session_record(self.session, 'project_artifact', body['sha256']) + declaration = None if declaration is None else declaration['document'] if declaration is None: raise WorkbenchError('Artifact is not declared by this project.', 'not_found', 404) def invoke(): diff --git a/project_runtime.py b/project_runtime.py index d63a9eb5..d14c59d4 100644 --- a/project_runtime.py +++ b/project_runtime.py @@ -15,6 +15,7 @@ from project_session import operation_records, REQUEST_KEY, _digest as operation_digest from runtime_snapshots import RuntimeSnapshots, DEFAULT_RESERVE, MAX_FILES, MAX_BYTES, MAX_MANIFEST from workbench import WorkbenchError, timestamp +from project_session import session_operations OPERATIONS = frozenset(('runtime.capture_intent', 'runtime.capture', 'runtime.abandon')) RESULT_FIELDS = frozenset(('format','project_id','manifest_sha256','file_count','total_bytes', @@ -166,7 +167,7 @@ def status(self, key): _key(key) with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', 'runtime.capture_status'); self.session.check_current() - receipts = operation_records(self.session.snapshot['records']) + receipts = session_operations(self.session) intent = receipts.get(_intent_key(key)); complete = receipts.get(key) if intent is not None and intent['operation'] != 'runtime.capture_intent': intent = None if complete is not None and complete['operation'] != 'runtime.capture': complete = None @@ -203,7 +204,7 @@ def forbidden(): _fail('No original local runtime copy exists to abandon.', 'una forbidden, lambda *args: forbidden(), scope='admin') with self.execution.runtime_reconciliation(capture_key, marker['input_sha256'], marker['manifest_sha256'], expected_checkpoint) as guard: - receipts = operation_records(self.session.snapshot['records']) + receipts = session_operations(self.session) intent = receipts.get(_intent_key(capture_key)) if (intent is None or intent['operation']!='runtime.capture_intent' or intent['input_sha256']!=marker['input_sha256'] @@ -249,7 +250,7 @@ def capture(self, config_request, expected_inventory_sha256, expected_checkpoint intent = None with self.access.lock, self.session.lock, self.db.lock: actor = self._authorize('runtime_snapshots.create'); self.session.check_current() - receipts = operation_records(self.session.snapshot['records']) + receipts = session_operations(self.session) if any(r['operation']=='runtime.abandon' and r['result']['capture_key']==key for r in receipts.values()): _fail('This capture was abandoned; use a fresh key for a new measurement.', 'conflict', 409) if (_intent_key(key) in receipts and key not in receipts @@ -257,7 +258,7 @@ def capture(self, config_request, expected_inventory_sha256, expected_checkpoint _fail('This incomplete capture intent is inert history; use a fresh key for a new capture.', 'unavailable', 409) try: with self.execution.runtime_effect(key, body_sha, expected_inventory_sha256) as guard: - receipts = operation_records(self.session.snapshot['records']) + receipts = session_operations(self.session) if key in receipts: result = self._replay(key, body, expected_checkpoint) descriptor = self.owner.get(expected_inventory_sha256) diff --git a/project_session.py b/project_session.py index aa471c1e..56f07a48 100644 --- a/project_session.py +++ b/project_session.py @@ -79,12 +79,13 @@ def operation_records(records, *, project_id=None): class ProjectSession: - def __init__(self, store, db, access, artifacts, *, validate_records): + def __init__(self, store, db, access, artifacts, *, validate_records, validation_cache=None): if not callable(validate_records): raise TypeError('A complete owner-codec validator is required.') self.store, self.db, self.access, self.artifacts = store, db, access, artifacts self.lock = threading.RLock() self._validate = validate_records + self.validation_cache = validation_cache self.available = False self._snapshot = None self.project_id = store.summary()['project_id'] @@ -95,7 +96,7 @@ def __init__(self, store, db, access, artifacts, *, validate_records): self._pending_observation = None self._recovery_records = None self._recovery_request = None - self.adopt(store.snapshot()) + self.adopt(store.snapshot(), _owned=True) @property def foreground_publication(self): @@ -145,6 +146,7 @@ def _publish_observation(self, records, key, digest, *, authorize): self._recovery_records = copy.deepcopy(candidate) self._recovery_request = (key, digest) raise + self._remember_validated(self._pending_observation['checkpoint']) return copy.deepcopy(self._pending_observation) def _observation_outcome(self, outcome): @@ -189,21 +191,141 @@ def snapshot(self): with self.lock: return copy_validated_json(self._snapshot) - def _validated(self, records): + def records(self, *families): + """Independent copies of only the selected families at the bound checkpoint. + + Equivalent to filtering a complete snapshot copy, without copying the + unselected families (most bytes are usually retained Run history). + """ + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + selected = frozenset(families) + with self.lock: + return copy_validated_json([item for item in self._snapshot['records'] + if item['family'] in selected]) + + def records_for(self, *families): + """Every bound record, in order, for one owner's pure decoder. + + Envelopes of the selected families are independent copies the owner + may keep or mutate. Every other envelope is a fresh envelope sharing its + verified document read-only, so fork/membership validators still see + the complete record set without copying unrelated Run history. Owners + must not mutate documents outside their selected families. + """ + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + selected = frozenset(families) + with self.lock: + return [copy_validated_json(item) if item['family'] in selected else + dict(family=item['family'], id=item['id'], document=item['document']) + for item in self._snapshot['records']] + + def active(self, *families, project_id=None): + """active_records for selected families over the bound checkpoint. + + Validates the complete fork history against every bound identity, but + copies only the selected active envelopes. + """ + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + from project_fork import active_records + with self.lock: + return active_records(self._snapshot['records'], + project_id=self.project_id if project_id is None else project_id, + families=frozenset(families)) + + def derived(self, name, compute): + """Memoize one owner-derived read-only fact set per bound checkpoint. + + compute() must be a pure function of the bound records. Owners share + the result; they must copy any part they return or mutate. + """ + if type(name) is not str or not callable(compute): + raise TypeError('Derived facts need a name and a compute function.') + with self.lock: + checkpoint = self._snapshot['checkpoint'] + cache = getattr(self, '_derived_cache', None) + if cache is None or cache[0] != checkpoint: + cache = (checkpoint, {}) + self._derived_cache = cache + if name not in cache[1]: + value = compute() + if self._snapshot['checkpoint'] != checkpoint: + raise ProjectSessionError('Project changed while deriving facts.', 'stale_project', 409) + cache[1][name] = value + return cache[1][name] + + def record(self, family, record_id): + """An independent copy of one bound record envelope, or None.""" + if type(family) is not str or type(record_id) is not str: + raise TypeError('Select a record by family and ID.') + with self.lock: + index = getattr(self, '_record_index', None) + if index is None or index[0] != self._snapshot['checkpoint']: + index = (self._snapshot['checkpoint'], + {(item['family'], item['id']): item for item in self._snapshot['records']}) + self._record_index = index + found = index[1].get((family, record_id)) + return None if found is None else copy_validated_json(found) + + def operations(self, *, project_id=None): + """operation_records over the bound checkpoint, computed once per checkpoint. + + The bound records are immutable for a checkpoint, so the result is a + pure function of (checkpoint, project_id). Callers receive independent + copies, exactly as from operation_records(self.snapshot['records']). + """ + with self.lock: + key = (self._snapshot['checkpoint'], project_id) + cached = getattr(self, '_operations_cache', None) + if cached is None or cached[0] != key: + # operation_records copies what it returns and never mutates input. + cached = (key, operation_records(self._snapshot['records'], project_id=project_id)) + self._operations_cache = cached + return copy.deepcopy(cached[1]) + + def _validated(self, records, checkpoint=None): + """Completely validate records; reuse an exact earlier success. + + checkpoint, when given, must be the Store checkpoint naming exactly + these records. Only then can a remembered validation be reused. + """ + cache = getattr(self, 'validation_cache', None) + if checkpoint is not None and cache is not None and cache.known(self.project_id, checkpoint): + return operation_records(records, project_id=self.project_id) self._validate(records) + if checkpoint is not None and cache is not None: + cache.remember(self.project_id, checkpoint) - def adopt(self, snapshot): + def _remember_validated(self, checkpoint): + cache = getattr(self, 'validation_cache', None) + if cache is not None: + cache.remember(self.project_id, checkpoint) + + def adopt(self, snapshot, *, _owned=False): """Bind a fully validated checkpoint after inert reconstruction/cutover. The caller owns SQL reconstruction. This method never repairs or writes canonical bytes, and is not authorization to trust an imported receipt. + _owned marks a snapshot this session just read from its own Store: its + checkpoint then names its records, so freshness is a checkpoint check + and the independent snapshot needs no second copy or full comparison. """ with self.access.lock, self.lock, self.db.lock: + if _owned: + self._validated(snapshot['records'], snapshot['checkpoint']) + if self.store.summary()['checkpoint'] != snapshot['checkpoint']: + raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) + self._snapshot = snapshot + self.available = True + return self._validated(snapshot['records']) current = self.store.snapshot() if current != snapshot: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) + self._remember_validated(snapshot['checkpoint']) self._snapshot = copy_validated_json(snapshot) self.available = True @@ -272,7 +394,7 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt raise ProjectSessionError('Request key already has different input.', 'conflict', 409) # The current composed codec validates the exact historical # result too; receipt facts can never authorize execution. - self._validated(current['records']) + self._validated(current['records'], current['checkpoint']) return copy.deepcopy(prior['result']) if self.store.reconcile(key, digest) is not None: raise ProjectSessionError('Request key belongs to another canonical operation.', 'conflict', 409) @@ -310,6 +432,8 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt # and before any deferred dispatch/wake can run. self._pending_observation = committed self._snapshot = committed + # These exact records were completely validated as the candidate. + self._remember_validated(committed['checkpoint']) return copy.deepcopy(captured['result']) except BaseException as error: if committed is not None: @@ -328,3 +452,49 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt finally: self._foreground_publication = False self._pending_observation = None + + +# Owners also accept lightweight snapshot views (for example inert candidate or +# retained-material views that expose only snapshot/check_current). These +# helpers use a ProjectSession's copy-avoiding bound reads when available and +# otherwise compute exactly the same values from the view's snapshot. + +def session_records(session, *families): + if hasattr(session, 'records') and callable(session.records): + return session.records(*families) + return [item for item in session.snapshot['records'] if item['family'] in families] + + +def session_record(session, family, record_id): + if hasattr(session, 'record') and callable(session.record): + return session.record(family, record_id) + return next((item for item in session.snapshot['records'] + if item['family'] == family and item['id'] == record_id), None) + + +def session_records_for(session, *families): + if hasattr(session, 'records_for') and callable(session.records_for): + return session.records_for(*families) + return session.snapshot['records'] + + +def session_active(session, *families, project_id=None): + if hasattr(session, 'active') and callable(session.active): + return session.active(*families, project_id=project_id) + from project_fork import active_records + snapshot = session.snapshot + return active_records(snapshot['records'], + project_id=session.project_id if project_id is None else project_id, + families=frozenset(families)) + + +def session_operations(session, *, project_id=None): + if hasattr(session, 'operations') and callable(session.operations): + return session.operations(project_id=project_id) + return operation_records(session.snapshot['records'], project_id=project_id) + + +def session_derived(session, name, compute): + if hasattr(session, 'derived') and callable(session.derived): + return session.derived(name, compute) + return compute() diff --git a/project_store.py b/project_store.py index d079ca3f..06ac8d84 100644 --- a/project_store.py +++ b/project_store.py @@ -189,9 +189,36 @@ def _domain_format(value, family): _fail('Unsupported domain document format.', 'unsupported', 422) -def _records(records): +def _compact(value): + # CPython's C encoder is used only without indent; it distinguishes 1, 1.0, + # True and 0.0/-0.0, which Python == equality does not. + return json.dumps(value, ensure_ascii=False, sort_keys=True, allow_nan=False, + separators=(',', ':')) + + +def _same_json(candidate, verified): + """Exact JSON value equality: same types, float bits and key sets.""" + if candidate is verified: + return True + if candidate != verified: + return False + try: + return _compact(candidate) == _compact(verified) + except (TypeError, ValueError, RecursionError): + return False + + +def _prepared_records(records, unchanged=None): + """Validate and canonically encode a complete candidate record set. + + unchanged maps (family, id) -> (verified current document, inventory entry). + A candidate document equal to its verified current revision reuses that + revision's known canonical byte count and digest instead of re-encoding it; + its raw slot is None and the caller must not rewrite it. + """ if type(records) is not list or len(records) > MAX_RECORDS: _fail('Project records must be a bounded list.') + unchanged = unchanged or {} result, seen, total = [], set(), 0 for record in records: _keys(record, ('family', 'id', 'document'), 'Project record') @@ -206,15 +233,73 @@ def _records(records): if type(document) is not dict: _fail('Domain document must be an object.') _domain_format(document.get('format'), family) + current = unchanged.get((family, identity)) + if current is not None and _same_json(document, current[0]) \ + and current[1]['format'] == document['format']: + # Exactly equal JSON values have identical canonical bytes; share the + # already verified current document (never handed out by alias). + total += current[1]['bytes'] + if total > MAX_TOTAL_BYTES: + _fail('Project records exceed the synchronous byte limit.') + result.append((family, identity, current[0], None, current[1])) + continue document = copy.deepcopy(document) raw = _encode(document) total += len(raw) if len(raw) > MAX_RECORD_BYTES or total > MAX_TOTAL_BYTES: _fail('Project records exceed the synchronous byte limit.') - result.append((family, identity, document, raw)) + result.append((family, identity, document, raw, None)) return sorted(result, key=lambda item: (item[0], item[1])) +_NESTED_DOCUMENT = b'\n' + b' ' * 4 + + +class _RecordsDigest: + """Incremental SHA256 of _encode([dict(family, id, document), ...]). + + add() takes each record's own canonical _encode(document) bytes, in record + order. Canonical JSON never contains a literal newline inside a string, so + a document nested two levels deep differs from its standalone encoding only + by four extra spaces after each newline. The result is byte-identical to + hashing the complete re-encoded list, without re-encoding any document. + """ + def __init__(self): + self._digest = hashlib.sha256() + self._first = True + + def add(self, family, identity, raw): + if not raw.endswith(b'\n'): + _fail('Canonical record bytes must end with LF.') + self._digest.update(b'[\n {\n "document": ' if self._first else b',\n {\n "document": ') + self._first = False + self._digest.update(raw[:-1].replace(b'\n', _NESTED_DOCUMENT)) + self._digest.update(b',\n "family": ') + self._digest.update(json.dumps(family, ensure_ascii=False).encode('utf-8')) + self._digest.update(b',\n "id": ') + self._digest.update(json.dumps(identity, ensure_ascii=False).encode('utf-8')) + self._digest.update(b'\n }') + + def hexdigest(self): + final = self._digest.copy() + final.update(b'[]\n' if self._first else b'\n]\n') + return final.hexdigest() + + +def _records_digest(rows): + """Digest of (family, id, canonical raw) rows; see _RecordsDigest.""" + digest = _RecordsDigest() + for family, identity, raw in rows: + digest.add(family, identity, raw) + return digest.hexdigest() + + +def _records(records): + """Complete canonical encoding as (family, id, document, raw) tuples.""" + return [(family, identity, document, raw) + for family, identity, document, raw, _entry in _prepared_records(records)] + + def _request(key, digest): if type(key) is not str or not REQUEST_KEY.fullmatch(key): _fail('Request key must contain 1–128 supported characters.') @@ -656,15 +741,14 @@ def _load(self): raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) cached = self._read_cache if cached is not None and raw == cached[0]: - manifest, loaded = cached[1:] - # Reuse decoding only after checking every actual canonical file. - # Stat identity is not content identity; corruption still fails. - for entry in manifest['inventory']: - self._verified_record_bytes(entry) - if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: - _fail('Project checkpoint changed while reading.', 'stale', 409) + # Record files are content-addressed by their inventory digest and + # are never rewritten, and every one was length/SHA256 verified when + # this Store decoded it. Byte-identical checkpoint bytes therefore + # name the same verified record set; re-hashing every record on each + # freshness check is redundant. On-disk corruption after that point + # is still refused by Save (verify_records) and by every reopen. self._binding() - return manifest, loaded + return cached[1:] manifest = _decode(raw) if type(manifest) is not dict: _fail('Project checkpoint must be an object.') @@ -686,6 +770,7 @@ def _load(self): if type(inventory) is not list or len(inventory) > MAX_RECORDS: _fail('Invalid project inventory.') loaded, roots, total, previous = [], [], 0, None + records_digest = _RecordsDigest() for entry in inventory: _keys(entry, ('family', 'id', 'format', 'namespace', 'path', 'bytes', 'sha256'), 'Inventory entry') family, record_id, digest = entry['family'], entry['id'], entry['sha256'] @@ -710,6 +795,7 @@ def _load(self): document = _decode(record_raw) if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') + records_digest.add(family, record_id, record_raw) loaded.append(dict(family=family, id=record_id, document=document)) roots.append(dict(family=family, id=record_id)) if manifest['roots'] != roots: @@ -724,7 +810,8 @@ def _load(self): if receipt['request_key'] in seen: _fail('Duplicate operation receipt key.') seen.add(receipt['request_key']) - if manifest['receipts'][-1]['records_sha256'] != _sha(_encode(loaded)): + # _decode proved each raw revision is its document's canonical encoding. + if manifest['receipts'][-1]['records_sha256'] != records_digest.hexdigest(): _fail('Latest operation receipt disagrees with the current record set.') # A second pointer read detects checkout/replacement during closure validation. if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: @@ -800,6 +887,7 @@ def _publish(self, manifest, creating=False): except OSError as exc: self._unknown = replaced _io_error(exc, 'publish checkpoint', unknown=replaced) + return raw # Temporary/orphan files remain inert after failures and process interruption. def _snapshot(self, manifest, records): @@ -815,8 +903,24 @@ def _current(self): except OSError as exc: _io_error(exc, 'read project snapshot') + def verify_records(self): + """Re-read and SHA256-check every referenced canonical revision on disk. + + Ordinary freshness checks compare checkpoint bytes only; this explicit + pass detects later same-length corruption of already-decoded revisions. + """ + with self._mutex: + manifest, _records = self._current() + try: + for entry in manifest['inventory']: + self._verified_record_bytes(entry) + except OSError as exc: + _io_error(exc, 'verify project records') + if self._current()[0] != manifest: + _fail('Project checkpoint changed during verification.', 'stale', 409) + def summary(self): - """Return scalar metadata after the same complete freshness check.""" + """Return scalar metadata after the checkpoint freshness check.""" with self._mutex: manifest, _records = self._current() return {key: manifest[key] for key in ('project_id', 'name', 'checkpoint')} @@ -841,14 +945,24 @@ def reconcile(self, request_key, request_sha256): def commit(self, records, expected_checkpoint, request_key, request_sha256): _request(request_key, request_sha256) _digest(expected_checkpoint, 'Expected checkpoint') - prepared = _records(records) - normalized = [dict(family=f, id=i, document=d) for f, i, d, _ in prepared] - records_digest = _sha(_encode(normalized)) with self._mutex: try: current, loaded = self._load() if current != self._manifest: _fail('Project checkpoint changed; close and reopen.', 'stale', 409) + # Unchanged records keep their verified canonical revision: no + # re-encoding, rewrite or read-back of identical immutable bytes. + entries = {(entry['family'], entry['id']): entry for entry in current['inventory']} + unchanged = {(item['family'], item['id']): (item['document'], entries[(item['family'], item['id'])]) + for item in loaded} + prepared = _prepared_records(records, unchanged) + normalized = [dict(family=f, id=i, document=d) for f, i, d, _, _ in prepared] + digest_rows = _RecordsDigest() + for family, identity, _document, raw, entry in prepared: + if raw is None: + raw = self._verified_record_bytes(entry) + digest_rows.add(family, identity, raw) + records_digest = digest_rows.hexdigest() for receipt in current['receipts']: if receipt['request_key'] == request_key: if receipt['request_sha256'] != request_sha256 or receipt['records_sha256'] != records_digest: @@ -858,13 +972,16 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256): _fail('Expected project checkpoint is stale.', 'stale', 409) manifest = copy.deepcopy(current) inventory = [] - for family, identity, document, raw in prepared: + for family, identity, document, raw, entry in prepared: + if raw is None: + inventory.append(copy.deepcopy(entry)) + continue digest = _sha(raw) inventory.append(dict(family=family, id=identity, format=document['format'], namespace=current['project_id'], path=f'records/{family}/{identity}/{digest}.json', bytes=len(raw), sha256=digest)) manifest['inventory'] = inventory - manifest['roots'] = [dict(family=f, id=i) for f, i, _, _ in prepared] + manifest['roots'] = [dict(family=f, id=i) for f, i, _, _, _ in prepared] manifest['receipts'].append(dict(request_key=request_key, request_sha256=request_sha256, records_sha256=records_digest)) del manifest['checkpoint'] @@ -872,10 +989,14 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256): # Refuse before record staging. The final publication also # rechecks this bound after the immutable files are durable. _manifest_raw(manifest) - for family, identity, document, raw in prepared: - self._record_file(family, identity, _sha(raw), raw) - self._publish(manifest) + for family, identity, document, raw, _entry in prepared: + if raw is not None: + self._record_file(family, identity, _sha(raw), raw) + published = self._publish(manifest) self._manifest = manifest + # Every committed revision is either freshly written and read + # back byte-for-byte or an unchanged verified revision. + self._read_cache = (published, manifest, normalized) return self._snapshot(manifest, normalized) except OSError as exc: _io_error(exc, 'commit project records') @@ -887,8 +1008,12 @@ def save(self, expected_checkpoint): if expected_checkpoint != snapshot['checkpoint']: _fail('Expected project checkpoint is stale.', 'stale', 409) try: + # Save is the explicit durability and integrity boundary: sync and + # re-verify every referenced canonical revision's actual bytes. for entry in self._manifest['inventory']: - self._record_file(entry['family'], entry['id'], entry['sha256'], sync=True) + raw = self._record_file(entry['family'], entry['id'], entry['sha256'], sync=True) + if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: + _fail('Canonical record bytes changed or are corrupt.', 'stale', 409) _sync_file(self._meta_fd, 'project.json') os.fsync(self._meta_fd) os.fsync(self._root_fd) diff --git a/project_study_execution.py b/project_study_execution.py index 2133bbce..c6a3494e 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -19,6 +19,7 @@ from project_material import ProjectMaterial, MaterialSnapshot from project_session import operation_records from workbench import WorkbenchError, encode, timestamp +from project_session import session_operations PREPARATIONS={'study_execution.prepare_package':'studies.prepare_package', 'study_execution.prepare_verifier':'studies.prepare_verifier'} @@ -323,6 +324,9 @@ def __init__(self,projection,execution): self.history.study_execution=self def _canonical(self,records=None): + if records is None and callable(getattr(self.session,'derived',None)): + from project_history import checkpoint_facts + return checkpoint_facts(self.session,self.materials) records=self.session.snapshot['records'] if records is None else records return history_records(records,materials=self.materials.read_view(records)) @@ -521,7 +525,7 @@ def continue_study(self,study_id,body,expected_checkpoint,key): self.access.require('execute','studies.continue',study_id,owner_only=True) with self.session.access.lock,self.session.lock,self.db.lock: self.refresh_authoring() - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is not None and not self.epoch_current(self.studies._study(study_id)): _invalid('A historical Continue receipt cannot activate this runtime; use a fresh key.','unavailable') def invoke(): @@ -567,7 +571,7 @@ def approve(self,study_id,body,expected_checkpoint,key): current=self.access.require('approve-execution','studies.approve',study_id,owner_only=True) effective=dict(body,approver_actor_id=current.actor_id) with self.session.access.lock,self.session.lock,self.db.lock: - self.refresh_authoring();prior=operation_records(self.session.snapshot['records']).get(key) + self.refresh_authoring();prior=session_operations(self.session).get(key) if prior is not None: if prior['operation']!='study_execution.approve':_invalid('Request key has another operation.','conflict') validate_operation_result(prior['operation'],prior['result'],self.session.snapshot['records']) @@ -590,7 +594,7 @@ def admit(self,study_id,body,expected_checkpoint,key): with self.session.access.lock,self.session.lock,self.db.lock: builtin=None self.session.check_current();self.refresh_authoring() - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is not None: result=prior['result'] if (prior['operation']!='study_execution.admit' or self.db.execute( @@ -735,14 +739,14 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): intent_key=_intent_key(operation,key);phase=intent_key.removesuffix('-intent') with self.session.access.lock,self.session.lock,self.db.lock: self.session.check_current() - prior=operation_records(self.session.snapshot['records']).get(key) + prior=session_operations(self.session).get(key) if prior is not None: # Current authority is checked above. Historical completion # acknowledges science and cannot repeat filesystem effects. return self._mutate(operation,study_id,effective,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Completed preparation replay invoked effects.')),'edit') if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') - receipts=operation_records(self.session.snapshot['records']) + receipts=session_operations(self.session) if any(_intent_key(other,key) in receipts for other in PREPARATIONS if other!=operation): _invalid('Preparation key already identifies another preparation operation.','conflict') previous_intent=receipts.get(intent_key) diff --git a/project_validation_cache.py b/project_validation_cache.py new file mode 100644 index 00000000..00f70621 --- /dev/null +++ b/project_validation_cache.py @@ -0,0 +1,96 @@ +"""Remember complete domain validation of exact project checkpoints. + +A project checkpoint digest covers its manifest, which names every canonical +record revision by length and SHA256. Records are verified against those +digests before use, so (project ID, checkpoint) identifies the exact record set. +Complete cross-domain validation of that set is a pure function of those +records and the validating source code, so an earlier success can be reused +when both are identical. + +Markers live only in the private local workspace control directory; a portable +project folder never carries validation authority. The validator fingerprint +covers every Tuldok Python module and extension schema plus the optional +validator package versions, so any code or dependency change revalidates. +""" +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path +import re +import sys +import threading +import uuid + +FORMAT = 'project_validation_marker_v1' +_HEX32 = re.compile(r'^[a-f0-9]{32}$') +_HEX64 = re.compile(r'^[a-f0-9]{64}$') +_SOURCE = Path(__file__).resolve().parent +_fingerprint = None +_fingerprint_lock = threading.Lock() + + +def _package_version(name): + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def validator_fingerprint(): + """SHA256 over validating source bytes, schemas and optional validators.""" + global _fingerprint + with _fingerprint_lock: + if _fingerprint is None: + digest = hashlib.sha256(b'tuldok-validator-fingerprint-v1\0') + paths = sorted(path for path in _SOURCE.glob('*.py')) + schemas = _SOURCE / 'extension_schemas' + if schemas.is_dir(): + paths += sorted(path for path in schemas.rglob('*') if path.is_file()) + for path in paths: + digest.update(str(path.relative_to(_SOURCE)).encode('utf-8') + b'\0') + digest.update(hashlib.sha256(path.read_bytes()).digest()) + environment = dict(python=sys.version, jsonschema=_package_version('jsonschema'), + referencing=_package_version('referencing')) + digest.update(json.dumps(environment, sort_keys=True).encode('utf-8')) + _fingerprint = digest.hexdigest() + return _fingerprint + + +class ProjectValidationCache: + """Private per-workspace markers of fully validated project checkpoints.""" + def __init__(self, control_path, *, fingerprint=None): + self.path = Path(control_path) / '.project-validation' + self.fingerprint = fingerprint or validator_fingerprint() + + def _marker(self, project_id, checkpoint): + if (type(project_id) is not str or not _HEX32.fullmatch(project_id) + or type(checkpoint) is not str or not _HEX64.fullmatch(checkpoint)): + raise ValueError('Invalid project validation identity.') + name = hashlib.sha256(f'{project_id}:{checkpoint}:{self.fingerprint}'.encode()).hexdigest() + return self.path / f'{name}.json', dict(format=FORMAT, project_id=project_id, + checkpoint=checkpoint, fingerprint=self.fingerprint) + + def known(self, project_id, checkpoint): + path, expected = self._marker(project_id, checkpoint) + try: + with open(path, 'rb') as stream: + return json.loads(stream.read(4096)) == expected + except (OSError, ValueError): + return False + + def remember(self, project_id, checkpoint): + """Best effort: a missing marker only means the next Open revalidates.""" + path, marker = self._marker(project_id, checkpoint) + try: + self.path.mkdir(mode=0o700, exist_ok=True) + temporary = self.path / f'.{uuid.uuid4().hex}.tmp' + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + os.write(descriptor, json.dumps(marker, sort_keys=True).encode('utf-8')) + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, path) + except OSError: + pass diff --git a/project_workspace.py b/project_workspace.py index b006dbe2..eed1788f 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -20,6 +20,7 @@ from project_store import ProjectStore, ProjectStoreError from project_projection import ProjectProjection from project_custody import ProjectCustody +from project_validation_cache import ProjectValidationCache from workbench import WorkbenchError @@ -83,6 +84,7 @@ def __init__(self, control_path, roots, *, dsn=None, runtime_sources=None): self.lock = self.db.lock self.access = Access(self.db, self.path) self.custody = ProjectCustody(self.path, self.db.dataset_id) + self.validation_cache = ProjectValidationCache(self.path) self._retained_recovery = None self._dsn = dsn self._runtime_sources = configured @@ -212,7 +214,8 @@ def _projection_for(self, store): owner = ProjectProjection(store, projection, self.access, {alias: value[0] for alias, value in self._roots.items()}, custody=self.custody.for_projection(store, identity), - runtime_sources=self._runtime_sources) + runtime_sources=self._runtime_sources, + validation_cache=self.validation_cache) owner.pending_original = self.custody.pending(store) if owner.pending_original: owner.session.available = False @@ -232,7 +235,7 @@ def _retained_projection_for(self, store, projection_id): return ProjectProjection(store, db, self.access, {alias: value[0] for alias, value in self._roots.items()}, custody=self.custody.for_projection(store, projection_id), restore=False, - runtime_sources=self._runtime_sources) + runtime_sources=self._runtime_sources, validation_cache=self.validation_cache) except BaseException: db.close() raise diff --git a/retained_measurements.py b/retained_measurements.py index 4044b017..4eefa6d5 100644 --- a/retained_measurements.py +++ b/retained_measurements.py @@ -309,16 +309,31 @@ def query(self, metric_id, *, subject_ids=None, filters=None, offset=0, limit=50 fail('Filters must use declared dimensions and typed scalar values.') if type(offset) is not int or not 0<=offset<=2**53-1 or type(limit) is not int or not 1<=limit<=100: fail('Page bounds must be explicit finite integers.') - rows = [v for v in self._doc['observations'] if v['metric_id']==metric_id - and v['subject_id'] in selected - and all(self._row_dimensions(v)[k]==wanted for k,wanted in filters.items())] + # One pass over observations; set membership and per-subject grouping + # replace the former subjects x rows rescans. Output is unchanged: + # rows keep packet order and ties keep the first maximal observation. + chosen = set(selected) + rows = [] + for v in self._doc['observations']: + if v['metric_id']!=metric_id or v['subject_id'] not in chosen: + continue + if filters: + dimensions = self._row_dimensions(v) + if not all(dimensions[k]==wanted for k,wanted in filters.items()): + continue + rows.append(v) + grouped = {identity:{'count':0,'states':{state:0 for state in STATES},'maximum':None} for identity in selected} + for v in rows: + own = grouped[v['subject_id']] + own['count'] += 1 + if v['state'] in own['states']: + own['states'][v['state']] += 1 + if v['state']=='measured' and (own['maximum'] is None or v['value'] > own['maximum']['value']): + own['maximum'] = v summaries = [] for identity in selected: - own = [v for v in rows if v['subject_id']==identity] - counts = {state:sum(v['state']==state for v in own) for state in STATES} - measured = [v for v in own if v['state']=='measured'] - maximum = max(measured, key=lambda v:v['value']) if measured else None - summaries.append({'subject_id':identity,'count':len(own),'states':counts, + own = grouped[identity];maximum = own['maximum'] + summaries.append({'subject_id':identity,'count':own['count'],'states':own['states'], 'maximum':None if maximum is None else {k:maximum[k] for k in ('id','case_id','value')}}) result = {'format':'retained_measurement_query_v1','packet_sha256':self.sha256, 'scope':copy.deepcopy(self._doc['scope']),'metric':copy.deepcopy(self._metrics[metric_id]), diff --git a/tests/test_project_artifacts.py b/tests/test_project_artifacts.py index ad06e375..87caa214 100644 --- a/tests/test_project_artifacts.py +++ b/tests/test_project_artifacts.py @@ -83,7 +83,7 @@ def mutate_source_after_snapshot(fd, value, **options): self.assertEqual(copied.stat().st_nlink, 1) self.assertNotEqual(copied.stat().st_ino, self.payload(value).stat().st_ino) - def test_copy_group_rejects_source_canonical_corruption_before_success(self): + def test_copy_group_uses_verified_records_and_verification_rejects_later_corruption(self): value = declaration(b'present') self.artifacts.publish_bytes(b'present', value) self.commit([value]) @@ -96,7 +96,9 @@ def corrupt_after_copy(*args, **options): record.write_bytes(b'{"corrupt":true}\n') return result with patch.object(destination, 'publish_fd', side_effect=corrupt_after_copy): - self.assert_code('stale', self.artifacts.copy_many_to, destination, [value]) + # The copy uses the verified declaration, never the corrupt bytes. + self.artifacts.copy_many_to(destination, [value]) + self.assert_code('stale', self.store.verify_records) def test_copy_group_prevalidates_inventory_and_preserves_missing_payload_failure(self): value, missing = declaration(b'present'), declaration(b'missing') @@ -126,7 +128,7 @@ def test_status_group_retains_exact_hash_observations_with_two_closure_checks(se self.assertEqual(actual[0]['observed_sha256'], values[0]['sha256']) self.assertEqual(actual[2]['observed_sha256'], hashlib.sha256(b'changed').hexdigest()) - def test_status_group_rejects_canonical_corruption_during_observation(self): + def test_status_group_uses_verified_records_and_verification_rejects_later_corruption(self): value = declaration(b'present') self.artifacts.publish_bytes(b'present', value) self.commit([value]) @@ -139,7 +141,8 @@ def corrupt_after_observation(*args): yield source record.write_bytes(b'{"corrupt":true}\n') with patch.object(self.artifacts, '_source', side_effect=corrupt_after_observation): - self.assert_code('stale', self.artifacts.status_many, [value]) + self.artifacts.status_many([value]) + self.assert_code('stale', self.store.verify_records) def test_status_group_rejects_checkpoint_change_and_unsafe_aliases(self): value = declaration(b'present') @@ -167,7 +170,7 @@ def test_presence_batch_matches_individual_observations_with_two_closure_checks( self.assertEqual(checks.call_count,2) self.assertTrue(all(v['observed_sha256'] is None for v in actual)) - def test_presence_batch_detects_canonical_corruption_before_returning(self): + def test_presence_batch_uses_verified_records_and_verification_rejects_later_corruption(self): value=declaration(b'present') self.artifacts.publish_bytes(b'present',value) self.commit([value]) @@ -180,7 +183,8 @@ def corrupt_after_observation(*args): yield source record.write_bytes(b'{"corrupt":"outside canonical membership"}\n') with patch.object(self.artifacts,'_source',side_effect=corrupt_after_observation): - self.assert_code('stale',self.artifacts.inspect_many,[value]) + self.artifacts.inspect_many([value]) + self.assert_code('stale',self.store.verify_records) def test_presence_batch_detects_checkpoint_change_and_rejects_unsafe_aliases(self): value=declaration(b'present') diff --git a/tests/test_project_history.py b/tests/test_project_history.py index c42dd135..301bd0ee 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -354,7 +354,7 @@ def start(self): self.host,self.owner,self.session=host,owner,session self.history=ProjectHistory(session,materials=Materials()) - def test_cached_history_still_reads_actual_bytes_and_rejects_restored_mtime_corruption(self): + def test_cached_history_serves_verified_values_and_verification_rejects_corruption(self): from project_store import ProjectStoreError identity='9'*32 records=[study(trials=[identity]),trial(identity,'3'*32)] @@ -372,8 +372,11 @@ def test_cached_history_still_reads_actual_bytes_and_rejects_restored_mtime_corr try: path.write_bytes(corrupt) os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + # Cached history keeps serving the verified value, never the + # corrupt bytes; explicit verification refuses them. + self.assertEqual(self.history.progression(ORIGIN),expected) with self.assertRaisesRegex(ProjectStoreError,'Canonical record bytes changed or are corrupt') as failed: - self.history.progression(ORIGIN) + self.store.verify_records() self.assertEqual((failed.exception.code,failed.exception.status),('stale',409)) finally: path.write_bytes(original) diff --git a/tests/test_project_measurement_reads.py b/tests/test_project_measurement_reads.py index 1365d63a..1ef28124 100644 --- a/tests/test_project_measurement_reads.py +++ b/tests/test_project_measurement_reads.py @@ -54,12 +54,12 @@ def observe(value): self.assertEqual(self.owner._documents(), self.expected) self.assertEqual(self.session._snapshot, self.store.snapshot()) - def test_lookup_reads_every_canonical_file_without_redecoding_adopted_material(self): + def test_lookup_reuses_verified_records_without_redecoding_adopted_material(self): with patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ patch.object(project_measurements, 'material_records', side_effect=AssertionError('Read-only lookup redecoded adopted material')): self.assertEqual(self.owner._documents(), self.expected) - self.assertEqual(reads.call_count, len(self.records), 'All current files must still be read and hashed') + self.assertEqual(reads.call_count, 0, 'Verified immutable records are not re-read per lookup') def test_unavailable_projection_fails_before_any_store_read(self): self.session.available = False @@ -76,7 +76,7 @@ def test_other_family_publication_requires_reopen(self): self.owner._documents() self.assertEqual(caught.exception.code, 'stale_project') - def test_same_length_unselected_corruption_with_restored_mtime_is_rejected(self): + def test_same_length_unselected_corruption_is_never_served_and_verification_rejects_it(self): inventory = json.loads((self.root/'.tuldok/project.json').read_bytes())['inventory'] entry = next(r for r in inventory if r['id'] == 'f'*64) path = self.root/'.tuldok'/entry['path'] @@ -87,8 +87,10 @@ def test_same_length_unselected_corruption_with_restored_mtime_is_rejected(self) path.chmod(0o600) path.write_bytes(changed) os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + # Reads keep serving verified values; explicit verification refuses. + self.assertEqual(self.owner._documents(), self.expected) with self.assertRaises(ProjectStoreError) as caught: - self.owner._documents() + self.store.verify_records() self.assertEqual(caught.exception.code, 'stale') def test_family_can_be_empty_and_returned_snapshot_contract_stays_complete(self): diff --git a/tests/test_project_session_freshness.py b/tests/test_project_session_freshness.py index 8ab5634e..f2ebe64c 100644 --- a/tests/test_project_session_freshness.py +++ b/tests/test_project_session_freshness.py @@ -33,13 +33,13 @@ def setUp(self): self.session._snapshot = copy.deepcopy(snapshot) self.session.available = True - def test_freshness_reads_all_records_without_copying_an_unused_graph(self): + def test_freshness_compares_checkpoint_without_rereading_or_copying_records(self): before = self.store.snapshot() with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused full snapshot')), \ patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: self.assertIsNone(self.session.check_current()) self.assertIsNone(self.session.check_current()) - self.assertEqual(reads.call_count, 4) + self.assertEqual(reads.call_count, 0) self.assertEqual(self.store.snapshot(), before) def test_changed_canonical_checkpoint_refuses_both_read_contracts(self): @@ -62,7 +62,7 @@ def test_unavailable_projection_refuses_before_reading_the_store(self): check() self.assertEqual(caught.exception.code, 'projection_unavailable') - def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self): + def test_same_length_corruption_is_never_served_and_explicit_verification_refuses_it(self): self.session.check_current() manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) path = self.root / '.tuldok' / manifest['inventory'][0]['path'] @@ -73,11 +73,13 @@ def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self) path.chmod(0o600) path.write_bytes(changed) os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) - for check in (self.session.check_current, self.session.check, - lambda: self.session.check_records('workbench_text')): - with self.assertRaises(ProjectStoreError) as caught: - check() - self.assertEqual(caught.exception.code, 'stale') + # Reads serve the verified in-memory value, never the corrupt bytes. + self.assertIsNone(self.session.check_current()) + self.assertEqual(self.session.check()['records'][0]['document']['text'], 'Alpha one') + self.assertEqual(self.session.check_records('workbench_text')['records'], self.records) + with self.assertRaises(ProjectStoreError) as caught: + self.store.verify_records() + self.assertEqual(caught.exception.code, 'stale') def test_record_consumers_keep_independent_complete_snapshots(self): current = self.session.check() @@ -108,13 +110,13 @@ def observe(value): patch('project_session.copy', SimpleNamespace(deepcopy=observe)): selected = self.session.check_records('workbench_text') self.assertEqual(selected, expected) - self.assertEqual(reads.call_count, 3, 'Unselected actual records must still be read and hashed') + self.assertEqual(reads.call_count, 0, 'Verified immutable records are not re-read per request') self.assertEqual(len(copied), 1) selected['records'][0]['document']['text'] = 'Caller mutation' self.assertEqual(self.session.check_records('workbench_text'), expected) self.assertEqual(self.session.check_records('project_file')['records'], []) - def test_selected_read_rejects_corruption_in_an_unselected_family(self): + def test_explicit_verification_rejects_corruption_in_an_unselected_family(self): self.add_unrelated_record() self.session.check_records('workbench_text') manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) @@ -127,8 +129,9 @@ def test_selected_read_rejects_corruption_in_an_unselected_family(self): path.chmod(0o600) path.write_bytes(changed) os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + self.assertEqual(self.session.check_records('workbench_text')['records'], self.records) with self.assertRaises(ProjectStoreError) as caught: - self.session.check_records('workbench_text') + self.store.verify_records() self.assertEqual(caught.exception.code, 'stale') diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 676ab3bc..ac2fa578 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -107,6 +107,41 @@ def test_create_commit_save_cold_reopen_and_independent_snapshots(self): committed['records'][0]['document']['text'] = 'Unchecked returned mutation' self.assertEqual(self.store.snapshot()['records'][0]['document']['text'], 'Notes preserve \r\n authored values') + def test_streamed_records_digest_equals_complete_canonical_encoding(self): + rows = [record('a'), record('b', 'Line\none "quoted" \\ ☃ 
'), + dict(family='workbench_text', id='c', document=dict(format='workbench_text_v1', text='', + history=[], nested={'z': [1, 1.0, -0.0, None, True, {}], 'ü': {'k\n': []}}))] + expected = store_module._sha(store_module._encode(rows)) + streamed = store_module._records_digest( + (row['family'], row['id'], store_module._encode(row['document'])) for row in rows) + self.assertEqual(streamed, expected) + self.assertEqual(store_module._records_digest([]), store_module._sha(store_module._encode([]))) + + def test_commit_reuses_unchanged_revisions_without_reencoding_them(self): + values = [record(str(index), 'Retained authored note %d' % index) for index in range(6)] + self.commit(values, key='initial-six') + changed = copy.deepcopy(values) + changed[2]['document']['text'] = 'Only this revision changes' + encode = store_module._encode + with patch.object(store_module, '_encode', wraps=encode) as encodes: + committed = self.commit(changed, key='change-one') + documents = [call.args[0] for call in encodes.call_args_list + if type(call.args[0]) is dict and call.args[0].get('format') == 'workbench_text_v1'] + self.assertEqual(documents, [changed[2]['document']]) + self.assertEqual(self.reopen(), committed) + + def test_python_equal_but_differently_typed_values_are_new_revisions(self): + for before, after in ((1, 1.0), (0.0, -0.0), (True, 1), (1.0, 1)): + with self.subTest(before=before, after=after): + original = dict(family='workbench_text', id='typed', document=dict(format='workbench_text_v1', value=before)) + self.commit([original], key=f'typed-{before!r}-{after!r}-a'.replace('.', '_').replace('-', '_')) + updated = copy.deepcopy(original) + updated['document']['value'] = after + self.commit([updated], key=f'typed-{before!r}-{after!r}-b'.replace('.', '_').replace('-', '_')) + value = self.reopen()['records'][0]['document']['value'] + self.assertIs(type(value), type(after)) + self.assertEqual(repr(value), repr(after)) + def test_appending_one_record_does_not_rewrite_saved_immutable_revisions(self): values = [record(str(index), 'Retained authored note %d' % index) for index in range(8)] self.commit(values, key='initial-inventory') @@ -127,7 +162,8 @@ def test_summary_reads_every_current_record_without_cloning_the_record_graph(sel first['name'] = 'Caller mutation' second = self.store.summary() self.assertEqual(second, expected) - self.assertEqual(reads.call_count, 4, 'Each summary must still read/hash both actual records') + self.assertEqual(reads.call_count, 0, + 'Freshness compares checkpoint bytes; verified immutable revisions are not re-read') self.assertEqual(self.store.snapshot(), committed) def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_identical_records(self): @@ -142,7 +178,7 @@ def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_ident with self.store.root_directory(verify_after=True) as descriptor: self.assertTrue(os.fstat(descriptor).st_ino) self.assertEqual(self.store.snapshot(),expected) - self.assertEqual(read.call_count,8,'All two records must still be read in each of four checks') + self.assertEqual(read.call_count,0,'Byte-identical checkpoints reuse the verified immutable records') self.assertEqual(snapshot.call_count,2,'Root borrowing must not clone an unused public snapshot') decoded=[call for call in decode.call_args_list if call.args[0] in canonical_documents] self.assertEqual(len(decoded),0,'Byte-identical canonical records need no repeat decoding within one Store') @@ -156,11 +192,13 @@ def test_verified_decode_cache_still_rejects_same_length_corruption_with_restore self.assertEqual(len(changed),len(raw));self.assertNotEqual(changed,raw) path.chmod(0o600);path.write_bytes(changed) os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) - self.assert_code('stale',self.store.snapshot) - self.assert_code('stale',self.store.summary) - with self.assertRaises(ProjectStoreError) as caught: - with self.store.root_directory(verify_after=True):pass - self.assertEqual(caught.exception.code,'stale') + # Ordinary reads keep serving the already verified value, never the + # corrupt bytes; explicit verification, Save and reopen refuse them. + self.assertEqual(self.store.snapshot()['records'][0]['document']['text'],'Alpha\n') + self.assert_code('stale',self.store.verify_records) + self.assert_code('stale',self.store.save,self.store.summary()['checkpoint']) + self.store.close() + self.assert_code('invalid',ProjectStore.open,self.root) def test_reusing_an_unreferenced_revision_still_checks_bytes_and_syncs_it(self): first, second = record('first'), record('second') @@ -171,7 +209,9 @@ def test_reusing_an_unreferenced_revision_still_checks_bytes_and_syncs_it(self): with patch.object(store_module, '_sync_file', wraps=sync) as syncs: committed = self.commit([first, second], key='reuse-first') self.assertFalse(any(call.args[1].startswith('.record-') for call in writes.call_args_list)) - self.assertEqual(syncs.call_count, 2) + # Only the newly referenced unreferenced revision is checked and synced; + # the unchanged current revision is reused without rewrite or re-sync. + self.assertEqual(syncs.call_count, 1) self.assertEqual(self.reopen(), committed) def test_reusing_a_corrupt_unreferenced_revision_never_publishes_it(self): @@ -190,7 +230,7 @@ def test_existing_revision_sync_failure_keeps_the_previous_checkpoint(self): self.commit([record()], key='initial') before = self.store.snapshot() with patch.object(store_module, '_sync_file', side_effect=OSError(errno.EIO, 'existing revision sync failed')): - self.assert_code('unavailable', self.commit, [record()], key='reuse-with-sync-error') + self.assert_code('unavailable', self.commit, [record(text='Changed revision')], key='new-revision-sync-error') self.assertEqual(self.reopen(), before) def test_fresh_copy_and_moved_folder_keep_identity_and_values(self): @@ -308,7 +348,8 @@ def test_missing_or_changed_canonical_record_never_becomes_empty_project(self): path = self.root / '.tuldok' / entry['path'] raw = path.read_bytes() path.write_bytes(raw.replace(b'Unique', b'Broken')) - self.assert_code('stale', self.store.snapshot) + self.assertEqual(self.store.snapshot()['records'][0]['document']['history'][0]['note'], 'Unique annotation') + self.assert_code('stale', self.store.verify_records) self.store.close() self.assert_code('invalid', ProjectStore.open, self.root) path.unlink() From 92814b2c46ebd4a68a6b8e03a6510726697d7986 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 08:49:17 -0700 Subject: [PATCH 02/13] perf(projects): reopen validated projects without repeating proofs - Collect history facts directly from a checkpoint whose complete domain validation is on record, instead of re-running every Run validator to restore Study authoring. Equivalence with decode_records is tested on fixtures and was checked field by field on the 460-Run portfolio project. - Remember record digests already proven to be canonical JSON in the private workspace control directory (versioned by the codec source), so reopening parses those bytes directly; every record is still length/SHA256 verified. Co-Authored-By: Claude Opus 5.5 --- project_history.py | 45 ++++++++++- project_session.py | 11 ++- project_store.py | 30 +++++-- project_validation_cache.py | 59 ++++++++++++++ project_workspace.py | 13 +-- tests/test_project_history.py | 22 ++++++ tests/test_project_validation_cache.py | 105 +++++++++++++++++++++++++ 7 files changed, 269 insertions(+), 16 deletions(-) create mode 100644 tests/test_project_validation_cache.py diff --git a/project_history.py b/project_history.py index 45d8ee86..521bc826 100644 --- a/project_history.py +++ b/project_history.py @@ -25,7 +25,7 @@ from runs import (ACTIVE, TERMINAL, TEST_FILES, MAX_ARTIFACT_BYTES, validate_retained_run, project_run_projection, public_run_projection, artifact_capture_source, validate_external_artifact_inventory) from workbench import WorkbenchError, timestamp -from project_session import session_derived, session_operations, session_records +from project_session import session_derived, session_operations, session_records, session_records_for FAMILY_FORMATS = { 'study_history': 'project_study_history_v1', @@ -563,12 +563,53 @@ def visit_run(key,path): return facts +def assemble_validated_facts(records, *, owned=False): + """Collect HistoryFacts from records whose complete validation is on record. + + decode_records validates and collects exactly these values: each history + document (as a checked JSON copy), Run observations, and the policy and + reservation indexes taken from Study documents. Only for a checkpoint that + already passed complete domain validation (which runs decode_records over + these same records) may the collection be repeated without re-running the + validators. tests/test_project_history.py and the portfolio equivalence + check compare this against decode_records field by field. owned=True means + the caller supplies independent copies the facts may keep without recopying. + """ + facts=HistoryFacts() + for record in records: + family=record.get('family') + if family not in FAMILY_FORMATS:continue + if family=='run_observation': + doc=_observation_document(record) + facts.observations[(doc['origin_catalog_id'],doc['run_id'],doc['sha256'])]=doc + continue + doc=record['document'] if owned else checked_json(record['document'],MAX_HISTORY_BYTES) + origin=doc['origin_catalog_id'] + if family=='study_history':target,identity=facts.studies,doc['study']['id'] + elif family=='trial_history':target,identity=facts.trials,doc['trial']['id'] + elif family=='run_history':target,identity=facts.runs,doc['run']['id'] + else:target,identity=facts.candidates,doc['candidate']['id'] + target[(origin,identity)]=doc + for (origin,_identity),doc in facts.studies.items(): + for item in doc['policies']: + ref=item['ref'];facts.policies[(origin,ref['id'],ref['revision'],ref['sha256'])]=item['policy'] + for reservation in doc['reservations']: + facts.reservations[(origin,reservation.get('run_id'))]=reservation + return facts + + def checkpoint_facts(session, materials): """Decoded history of the session's bound checkpoint, shared by its owners. - Decoded once per checkpoint; callers copy any part they return. + Decoded once per checkpoint; callers copy any part they return. When the + session records that this exact checkpoint passed complete validation, + the facts are collected without repeating the validators. """ def compute(): + validated=getattr(session,'validated_checkpoint',None) + if validated is not None and validated==session.summary['checkpoint']: + # records_for returns independent copies of exactly these families. + return assemble_validated_facts(session_records_for(session,*FAMILY_FORMATS),owned=True) records=session.snapshot['records'] return decode_records(records,materials=materials.read_view(records)) return session_derived(session, 'history_facts',compute) diff --git a/project_session.py b/project_session.py index 56f07a48..be5509ff 100644 --- a/project_session.py +++ b/project_session.py @@ -86,6 +86,8 @@ def __init__(self, store, db, access, artifacts, *, validate_records, validation self.lock = threading.RLock() self._validate = validate_records self.validation_cache = validation_cache + # Checkpoint whose exact records passed complete domain validation. + self.validated_checkpoint = None self.available = False self._snapshot = None self.project_id = store.summary()['project_id'] @@ -293,13 +295,18 @@ def _validated(self, records, checkpoint=None): """ cache = getattr(self, 'validation_cache', None) if checkpoint is not None and cache is not None and cache.known(self.project_id, checkpoint): + self.validated_checkpoint = checkpoint return operation_records(records, project_id=self.project_id) self._validate(records) - if checkpoint is not None and cache is not None: - cache.remember(self.project_id, checkpoint) + if checkpoint is not None: + self.validated_checkpoint = checkpoint + if cache is not None: + cache.remember(self.project_id, checkpoint) def _remember_validated(self, checkpoint): + # The caller completely validated exactly the records this names. + self.validated_checkpoint = checkpoint cache = getattr(self, 'validation_cache', None) if cache is not None: cache.remember(self.project_id, checkpoint) diff --git a/project_store.py b/project_store.py index 06ac8d84..b36360dd 100644 --- a/project_store.py +++ b/project_store.py @@ -556,8 +556,10 @@ class ProjectStore: revisions are never pruned. Callers validate domain semantics before commit and after Open; this owner validates only the generic JSON/inventory contract. """ - def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path): + def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path, canonical_digests=None): self.path = Path(path) + # Optional private set of digests already proven canonical; see _load. + self._canonical_digests = canonical_digests self._root_fd = root_fd self._origin_fd = origin_fd self._relative_parts = relative_parts @@ -663,7 +665,7 @@ def create_at(cls, trusted_root_fd, relative, name, request_key, request_sha256, os.close(origin_fd) @classmethod - def open(cls, path): + def open(cls, path, *, canonical_digests=None): raw = _path_string(path) if type(raw) is not str or '\x00' in raw or '..' in raw.split('/'): _fail('Unsafe project root path.') @@ -671,12 +673,12 @@ def open(cls, path): parent, leaf = os.path.split(absolute) _, parent_fd = _open_directory_path(parent) try: - return cls.open_at(parent_fd, leaf, root_path=parent) + return cls.open_at(parent_fd, leaf, root_path=parent, canonical_digests=canonical_digests) finally: os.close(parent_fd) @classmethod - def open_at(cls, trusted_root_fd, relative, *, root_path): + def open_at(cls, trusted_root_fd, relative, *, root_path, canonical_digests=None): """Open a descendant without following any configured-root or child alias.""" parts = _relative_parts(relative) absolute = _verify_origin(trusted_root_fd, root_path) @@ -685,7 +687,8 @@ def open_at(cls, trusted_root_fd, relative, *, root_path): try: fd = _walk(origin_fd, parts) owned_origin, origin_fd = origin_fd, None - store = cls(Path(absolute).joinpath(*parts), fd, owned_origin, parts, absolute) + store = cls(Path(absolute).joinpath(*parts), fd, owned_origin, parts, absolute, + canonical_digests=canonical_digests) store._meta_fd = _directory(fd, '.tuldok') store._manifest, _ = store._load() return store @@ -771,6 +774,8 @@ def _load(self): _fail('Invalid project inventory.') loaded, roots, total, previous = [], [], 0, None records_digest = _RecordsDigest() + known = self._canonical_digests + proven = [] for entry in inventory: _keys(entry, ('family', 'id', 'format', 'namespace', 'path', 'bytes', 'sha256'), 'Inventory entry') family, record_id, digest = entry['family'], entry['id'], entry['sha256'] @@ -792,7 +797,14 @@ def _load(self): if total > MAX_TOTAL_BYTES: _fail('Project records exceed their total byte limit.') record_raw = self._verified_record_bytes(entry) - document = _decode(record_raw) + if known is not None and digest in known: + # These exact bytes (named by their verified SHA256) were already + # proven to be their value's canonical encoding, which excludes + # duplicate keys and non-finite numbers; a plain parse is exact. + document = json.loads(record_raw) + else: + document = _decode(record_raw) + proven.append(digest) if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') records_digest.add(family, record_id, record_raw) @@ -817,6 +829,8 @@ def _load(self): if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: _fail('Project checkpoint changed while reading.', 'stale', 409) self._binding() + if known is not None and proven: + known.add_many(proven) self._read_cache = (raw, manifest, loaded) return manifest, loaded @@ -994,6 +1008,10 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256): self._record_file(family, identity, _sha(raw), raw) published = self._publish(manifest) self._manifest = manifest + if self._canonical_digests is not None: + # Freshly written revisions are canonical by construction. + self._canonical_digests.add_many( + [_sha(raw) for _f, _i, _d, raw, _e in prepared if raw is not None]) # Every committed revision is either freshly written and read # back byte-for-byte or an unchanged verified revision. self._read_cache = (published, manifest, normalized) diff --git a/project_validation_cache.py b/project_validation_cache.py index 00f70621..4d06760e 100644 --- a/project_validation_cache.py +++ b/project_validation_cache.py @@ -94,3 +94,62 @@ def remember(self, project_id, checkpoint): os.replace(temporary, path) except OSError: pass + + +class CanonicalDigestCache: + """Private set of record digests whose bytes were proven canonical JSON. + + Canonicality is a property of the bytes alone, and record files are named + by their SHA256, so a proof for one digest holds for every project that + contains those exact bytes. The set is versioned by the canonical codec's + own source, so any change to the encoding rules starts a fresh set. + """ + def __init__(self, control_path): + import inspect + import project_store + codec = ''.join(inspect.getsource(function) for function in ( + project_store._json_value, project_store._encode, project_store._pairs, project_store._decode)) + version = hashlib.sha256((codec + json.__version__).encode('utf-8')).hexdigest()[:32] + self.path = Path(control_path) / '.canonical-digests' / f'{version}.bin' + self._known = None + self._lock = threading.Lock() + + def _load(self): + if self._known is None: + known = set() + try: + with open(self.path, 'rb') as stream: + raw = stream.read() + if len(raw) % 32 == 0: + known = {raw[index:index + 32] for index in range(0, len(raw), 32)} + except OSError: + pass + self._known = known + return self._known + + def __contains__(self, digest): + with self._lock: + return bytes.fromhex(digest) in self._load() + + def add_many(self, digests): + """Best effort: a lost update only means those records are re-proven.""" + with self._lock: + known = self._load() + fresh = {bytes.fromhex(digest) for digest in digests} - known + if not fresh: + return + known |= fresh + try: + self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = self.path.parent / f'.{uuid.uuid4().hex}.tmp' + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + remaining = memoryview(b''.join(sorted(known))) + while remaining: + remaining = remaining[os.write(descriptor, remaining):] + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, self.path) + except OSError: + pass diff --git a/project_workspace.py b/project_workspace.py index eed1788f..047a7c82 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -20,7 +20,7 @@ from project_store import ProjectStore, ProjectStoreError from project_projection import ProjectProjection from project_custody import ProjectCustody -from project_validation_cache import ProjectValidationCache +from project_validation_cache import CanonicalDigestCache, ProjectValidationCache from workbench import WorkbenchError @@ -85,6 +85,7 @@ def __init__(self, control_path, roots, *, dsn=None, runtime_sources=None): self.access = Access(self.db, self.path) self.custody = ProjectCustody(self.path, self.db.dataset_id) self.validation_cache = ProjectValidationCache(self.path) + self.canonical_digests = CanonicalDigestCache(self.path) self._retained_recovery = None self._dsn = dsn self._runtime_sources = configured @@ -274,7 +275,7 @@ def _recover_current(self, arguments): self._store = None store = projection = None try: - store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) + store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path, canonical_digests=self.canonical_digests) snapshot = store.summary() if snapshot['project_id'] != self._last_project['project_id']: raise WorkbenchError('Recovery destination has a different project identity.', 'stale_project', 409) @@ -329,7 +330,7 @@ def _select(self, action, arguments): # unselected destination. Only its exact receipt authorizes # treating it as this New; unrelated folders remain conflicts. try: - store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) + store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path, canonical_digests=self.canonical_digests) except ProjectStoreError: raise error if store.reconcile(arguments['request_key'], digest) is None: @@ -340,7 +341,7 @@ def _select(self, action, arguments): store.save(snapshot['checkpoint']) created = snapshot else: - store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) + store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path, canonical_digests=self.canonical_digests) projection, text = self._projection_for(store) self._detach() self._store, self._projection, self._text = store, projection, text @@ -411,7 +412,7 @@ def _publish_independent(self, arguments, *, composed=False): raise WorkbenchError('Composition source identity repeats.','conflict',409) identities.add(selected['project_id']) directory,descriptor=self._root(selected['root'],selected['directory']) - store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory)) + store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory, canonical_digests=self.canonical_digests)) actual=store.summary() if actual['project_id']!=selected['project_id'] or actual['checkpoint']!=selected['checkpoint']: raise WorkbenchError('Selected composition source checkpoint differs.','stale_project',409) @@ -432,7 +433,7 @@ def _publish_independent(self, arguments, *, composed=False): # A filesystem editor can act outside the workspace lock. Leave the # complete fork unselected if its source working files changed. source.files.require_clean() - incoming = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) + incoming = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path, canonical_digests=self.canonical_digests) projection, text = self._projection_for(incoming) snapshot, overlay = published['snapshot'], published['fork'] self._detach() diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 301bd0ee..8051eb75 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -167,6 +167,28 @@ def test_cross_study_parent_baseline_and_origin_namespace_remain_exact(self): changed=copy.deepcopy(records);changed[-1]['document']['lineage']['intent']['baseline']['catalog_id']=OTHER with self.assertRaises(WorkbenchError):decode_records(changed,materials=Materials()) + def test_validated_fact_assembly_equals_complete_decode(self): + from project_history import assemble_validated_facts + parent_id='9'*32;child_id='a'*32 + parent_ref={'catalog_id':ORIGIN,'trial_id':parent_id,'material_sha256':MATERIAL} + saved=capture_run(legacy_run(),payloads=[]) + fixtures=( + [study('3'*32,trials=[parent_id]),study('b'*32,trials=[child_id]), + trial(parent_id,'3'*32),trial(child_id,'b'*32,parent=parent_ref,baseline=parent_ref)], + [saved,*declarations(saved)], + ) + for records in fixtures: + with self.subTest(families=sorted({r['family'] for r in records})): + decoded=decode_records(copy.deepcopy(records),materials=Materials()) + for owned in (False,True): + before=copy.deepcopy(records) + with patch('project_history._validate_run_document',side_effect=AssertionError('Validator rerun')): + assembled=assemble_validated_facts(copy.deepcopy(records) if owned else records,owned=owned) + self.assertEqual(records,before) + for field in ('studies','trials','runs','candidates','policies','reservations','observations'): + self.assertEqual(getattr(assembled,field),getattr(decoded,field),field) + self.assertEqual(list(getattr(assembled,field)),list(getattr(decoded,field)),field) + def test_history_read_uses_one_material_view_of_the_selected_records(self): identity='9'*32 records=[study(trials=[identity]),trial(identity,'3'*32)] diff --git a/tests/test_project_validation_cache.py b/tests/test_project_validation_cache.py new file mode 100644 index 00000000..034d50dd --- /dev/null +++ b/tests/test_project_validation_cache.py @@ -0,0 +1,105 @@ +"""Private workspace caches that make reopening an unchanged project fast.""" +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import project_store as store_module +from project_store import ProjectStore +from project_validation_cache import CanonicalDigestCache, ProjectValidationCache + +PROJECT = 'a' * 32 +CHECKPOINT = 'b' * 64 + + +def note(identity, text): + return dict(family='workbench_text', id=identity, + document=dict(format='workbench_text_v1', text=text, history=[])) + + +class ValidationCacheTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-validation-cache-') + self.addCleanup(temporary.cleanup) + self.control = Path(temporary.name) + + def test_marker_binds_project_checkpoint_and_validator_fingerprint(self): + cache = ProjectValidationCache(self.control, fingerprint='1' * 64) + self.assertFalse(cache.known(PROJECT, CHECKPOINT)) + cache.remember(PROJECT, CHECKPOINT) + self.assertTrue(cache.known(PROJECT, CHECKPOINT)) + self.assertTrue(ProjectValidationCache(self.control, fingerprint='1' * 64).known(PROJECT, CHECKPOINT)) + self.assertFalse(cache.known(PROJECT, 'c' * 64)) + self.assertFalse(cache.known('d' * 32, CHECKPOINT)) + self.assertFalse(ProjectValidationCache(self.control, fingerprint='2' * 64).known(PROJECT, CHECKPOINT), + 'Changed validator source must revalidate') + + def test_tampered_or_invalid_markers_are_not_trusted(self): + cache = ProjectValidationCache(self.control, fingerprint='1' * 64) + cache.remember(PROJECT, CHECKPOINT) + marker = next((self.control / '.project-validation').glob('*.json')) + marker.write_text('{"format":"project_validation_marker_v1","checkpoint":"other"}') + self.assertFalse(cache.known(PROJECT, CHECKPOINT)) + with self.assertRaises(ValueError): + cache.known('not-a-project', CHECKPOINT) + + def test_default_fingerprint_covers_validating_sources(self): + self.assertRegex(ProjectValidationCache(self.control).fingerprint, '^[a-f0-9]{64}$') + + +class CanonicalDigestCacheTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-canonical-cache-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.control = self.root / 'control' + self.control.mkdir() + store = ProjectStore.create(self.root / 'project', 'Canonical fixture', 'new', 'a' * 64) + self.records = [note('one', 'Alpha ☃'), note('two', 'Beta\nwith "quotes"')] + self.committed = store.commit(self.records, store.summary()['checkpoint'], 'seed', 'b' * 64) + store.close() + + def open(self, cache): + store = ProjectStore.open(self.root / 'project', canonical_digests=cache) + self.addCleanup(store.close) + return store + + def test_reopen_skips_only_repeated_canonical_proofs_and_keeps_exact_values(self): + cache = CanonicalDigestCache(self.control) + with patch.object(store_module, '_decode', wraps=store_module._decode) as decode: + store = self.open(cache) + first = store.snapshot() + store.close() + self.assertEqual(first, self.committed) + self.assertEqual(decode.call_count, 1 + len(self.records), 'Checkpoint plus each record proven once') + reopened = CanonicalDigestCache(self.control) + with patch.object(store_module, '_decode', wraps=store_module._decode) as decode: + second = ProjectStore.open(self.root / 'project', canonical_digests=reopened) + try: + self.assertEqual(second.snapshot(), self.committed) + finally: + second.close() + self.assertEqual(decode.call_count, 1, 'Only the checkpoint itself is decoded canonically') + + def test_corruption_still_fails_with_a_warm_canonical_cache(self): + cache = CanonicalDigestCache(self.control) + self.open(cache).close() + manifest = store_module._decode((self.root / 'project/.tuldok/project.json').read_bytes()) + path = self.root / 'project/.tuldok' / manifest['inventory'][0]['path'] + path.write_bytes(path.read_bytes().replace(b'Alpha', b'Omega')) + with self.assertRaises(store_module.ProjectStoreError): + ProjectStore.open(self.root / 'project', canonical_digests=CanonicalDigestCache(self.control)) + + def test_commit_registers_new_revisions_and_unreadable_cache_is_ignored(self): + cache = CanonicalDigestCache(self.control) + store = self.open(cache) + store.commit([*self.records, note('three', 'Gamma')], store.summary()['checkpoint'], 'more', 'c' * 64) + manifest = store_module._decode((self.root / 'project/.tuldok/project.json').read_bytes()) + added = next(entry for entry in manifest['inventory'] if entry['id'] == 'three') + self.assertIn(added['sha256'], CanonicalDigestCache(self.control)) + cache.path.write_bytes(b'truncated') + self.assertNotIn(added['sha256'], CanonicalDigestCache(self.control)) + + +if __name__ == '__main__': + unittest.main() From e9f853d617fed6324abc095eaf51ba3aebb989fb Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 10:04:39 -0700 Subject: [PATCH 03/13] feat(projects): keep a persistent database working copy of each project The local PostgreSQL database is now the working copy of an opened project and is kept across sessions. The external folder (project.json plus content-addressed records, tracked in Git) is an import source, read only when its checkpoint changed since Tuldok last imported or saved it, and an export target written only by Save. See docs/plans/database-working-store.md. - WorkingProjectStore: ProjectStore interface over a per-folder schema of exact canonical record bytes; identical manifest/checkpoint/receipt identity to the file format, so Save writes byte-identical files. Incremental import reads only revisions the database lacks; reattaching an unchanged folder reads no record file. No record state is cached in memory. - ProjectStore folder mode (lock + binding without loading records), checked_manifest() and import/export primitives. - Record commits share the projection's SQL transaction: a rejected COMMIT rolls back records and projection together ('rejected', session stays current); a lost acknowledgement is 'unknown' and reopen reconciles it. Background publications that roll back still require custody reconciliation because local effects may have advanced. - Workspace: private folder->working-copy registry; Open/New/recovery/Save As attach working copies; Close keeps unsaved work in the database; Save is the only folder write and refuses to overwrite external changes; an external change with unsaved work is an explicit conflict at Open. Tests: new working-store suite (file-store checkpoint parity, byte-identical export, incremental import, conflicts, busy, corruption); workspace/HTTP tests updated where they assumed every edit or Close wrote the folder. Co-Authored-By: Claude Opus 5.5 --- docs/plans/database-working-store.md | 74 ++++ project_session.py | 27 ++ project_store.py | 231 +++++++--- project_transfer.py | 3 +- project_working_store.py | 449 ++++++++++++++++++++ project_workspace.py | 60 ++- tests/test_project_files_http.py | 54 +-- tests/test_project_files_m2_http.py | 2 + tests/test_project_files_m4_http.py | 17 +- tests/test_project_runtime_http.py | 2 + tests/test_project_runtime_recovery_http.py | 30 +- tests/test_project_working_store.py | 212 +++++++++ tests/test_project_workspace.py | 73 ++-- 13 files changed, 1078 insertions(+), 156 deletions(-) create mode 100644 docs/plans/database-working-store.md create mode 100644 project_working_store.py create mode 100644 tests/test_project_working_store.py diff --git a/docs/plans/database-working-store.md b/docs/plans/database-working-store.md new file mode 100644 index 00000000..69482e62 --- /dev/null +++ b/docs/plans/database-working-store.md @@ -0,0 +1,74 @@ +# Database working store for external projects + +Status: in progress (2026-11-02). Owner decision recorded with the user. + +## Decision + +The local PostgreSQL database is the persistent **working copy** of an opened +project and is kept across sessions. The external project folder +(`.tuldok/project.json` plus content-addressed `records/`, tracked in Git) is +only: + +- an **import source**, read when its files changed since Tuldok last imported + or exported them, or when the project is not yet in this workspace; and +- an **export target**, written only by an explicit Save. + +Edits during a session are committed to the database and stay there. Closing a +project does not write the external folder. Complete domain validation runs +only when the record set changes (import of a changed folder, or a commit). + +There is exactly one durable copy of the record set (the database) and no +complete in-memory copy: owners read the record families or records they need. + +## Why + +Opening the 54k-record, 1.77 GB Rheon portfolio rebuilt a fresh SQL projection +from every JSON file on each Open (9+ minutes, 17–21 GB RAM: the Store, the +Session and owner copies each held the whole record set). JSON exists only so +the project can be tracked in Git. + +## Identity and integrity + +- Checkpoint identity is unchanged: the working copy keeps the same manifest + (inventory of `family/id/format/bytes/sha256`, roots, receipts) and the same + checkpoint digest as the file format, so Save writes byte-identical files and + existing receipts, validation markers and custody records stay meaningful. +- Each record is stored once as its exact canonical bytes (not `jsonb`, which + would change number spelling such as `-0.0`) with its SHA256. +- Import verifies `project.json`, then only reads, SHA256-checks and + canonical-checks records whose digest the working copy does not already hold. +- Save re-checks that the external `project.json` is still the one last + imported/exported (refusing to overwrite external changes), writes only + missing record files, then publishes `project.json` atomically. + +## Open + +1. Read external `project.json` (one file) and its SHA256. +2. Working copy exists and its last imported/exported manifest hash matches: + attach. No record file is read; the SQL projection is reused. +3. External folder changed, working copy has no unsaved changes: incremental + import, validate (cached per checkpoint), rebuild projection tables. +4. External folder changed **and** working copy has unsaved changes: refuse with + an explicit conflict (keep working copy and overwrite on Save, or discard and + re-import). Never merge silently. + +## Stages + +1. **Working store.** `WorkingProjectStore` with the ProjectStore interface + (`summary`, `snapshot`, `commit`, `save`, `reconcile`, `root_directory`, + `verify_records`, `recover_checkpoint`, `close`) plus family/record reads, + backed by tables in the project's persistent projection schema. Incremental + import; commit to the database; Save exports. Workspace keeps one persistent + projection per project folder and re-attaches it. +2. **No whole-project memory copy.** Session binds checkpoint metadata and reads + through the store; convert remaining `snapshot['records']` consumers to + family/record reads. Target: hundreds of MB, not GB. +3. **Projection reuse.** Skip SQL restore when the projection already reflects + the bound checkpoint. +4. **Incremental validation for commits.** Re-run only the owner validators + whose families a mutation touches. +5. **Smaller project files.** Store each captured runtime file listing once and + reference it from Runs (format migration with exact round trip). + +Payload blobs (`artifacts/sha256/*`, 16 GB for the portfolio) remain +content-addressed files under the project folder in stages 1–4. diff --git a/project_session.py b/project_session.py index be5509ff..2d339289 100644 --- a/project_session.py +++ b/project_session.py @@ -155,6 +155,15 @@ def _observation_outcome(self, outcome): if outcome not in ('committed', 'rolled_back', 'unknown'): raise ValueError('Invalid project SQL outcome.') pending, self._pending_observation = self._pending_observation, None + if pending is not None and self._atomic_store() and outcome == 'rolled_back': + # The record commit rolled back with the SQL transaction, so the + # bound checkpoint is still exactly current. Background local + # effects (for example a live phase) may have advanced, so their + # custody is reconciled explicitly before further use. A foreground + # mutate reports its own definite rollback and stays available. + if not self._foreground_publication: + self.available = False + return if pending is not None: self._snapshot = pending if outcome != 'committed': @@ -162,6 +171,11 @@ def _observation_outcome(self, outcome): if outcome == 'unknown': self.available = False + def _atomic_store(self): + """Whether record commits share this session's SQL transactions.""" + bound = getattr(self.store, '_db', None) + return bound is not None and bound is self.db + def reconcile_selection(self, snapshot): """Bind only this owner's selected or uncertain publication, never a checkout. @@ -443,6 +457,19 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt self._remember_validated(committed['checkpoint']) return copy.deepcopy(captured['result']) except BaseException as error: + if committed is not None and self._atomic_store(): + from metadata import CommitOutcomeUnknown + if isinstance(error, CommitOutcomeUnknown): + # PostgreSQL may or may not have committed both the + # records and the projection; reopening reads the outcome. + self.available = False + raise ProjectSessionError('PostgreSQL did not confirm this edit. Reopen the project to see whether it was saved.', + 'unknown', 503) from error + # Records and projection rolled back together; nothing changed. + if isinstance(error, ProjectSessionError): + raise + raise ProjectSessionError('PostgreSQL rejected this edit; nothing was saved.', + 'rejected', 409) from error if committed is not None: self._snapshot, self.available = committed, False failure = ProjectSessionError('Canonical operation committed; reopen to reconstruct the projection.', diff --git a/project_store.py b/project_store.py index b36360dd..508fe989 100644 --- a/project_store.py +++ b/project_store.py @@ -306,6 +306,71 @@ def _request(key, digest): _digest(digest, 'Request body digest') +def checked_manifest(raw): + """Decode and check a project checkpoint without reading any record file. + + Verifies format, identity, the checkpoint digest, every inventory entry's + shape, ordering, rooted path and byte bounds, membership roots and receipt + structure. Record bytes and the latest receipt's records digest are checked + by whoever reads the records. + """ + manifest = _decode(raw) + if type(manifest) is not dict: + _fail('Project checkpoint must be an object.') + if manifest.get('format') != FORMAT: + if type(manifest.get('format')) is str and re.fullmatch(r'tuldok_project_v[1-9][0-9]*', manifest['format']): + _fail('Unsupported project format.', 'unsupported', 422) + _fail('Invalid project format.') + _keys(manifest, ('format', 'project_id', 'name', 'checkpoint', 'roots', 'inventory', 'receipts'), 'Project checkpoint') + identity = manifest['project_id'] + if type(identity) is not str or not re.fullmatch('[a-f0-9]{32}', identity): + _fail('Invalid project UUID.') + if type(manifest['name']) is not str or not manifest['name'].strip() or len(manifest['name']) > 200: + _fail('Invalid project display name.') + _digest(manifest['checkpoint'], 'Checkpoint') + core = {k: v for k, v in manifest.items() if k != 'checkpoint'} + if _sha(_encode(core)) != manifest['checkpoint']: + _fail('Project checkpoint digest does not match its contents.') + inventory = manifest['inventory'] + if type(inventory) is not list or len(inventory) > MAX_RECORDS: + _fail('Invalid project inventory.') + roots, total, previous = [], 0, None + for entry in inventory: + _keys(entry, ('family', 'id', 'format', 'namespace', 'path', 'bytes', 'sha256'), 'Inventory entry') + family, record_id, digest = entry['family'], entry['id'], entry['sha256'] + _identifier(family, 'Record family') + if family not in SUPPORTED_FORMATS: + _fail('Unsupported project record family.', 'unsupported', 422) + _identifier(record_id, 'Record ID') + _digest(digest, 'Record digest') + _domain_format(entry['format'], family) + key = (family, record_id) + if previous is not None and key <= previous: + _fail('Inventory must have unique sorted record identities.') + previous = key + if entry['namespace'] != identity or entry['path'] != f'records/{family}/{record_id}/{digest}.json': + _fail('Invalid inventory namespace or rooted record path.') + if type(entry['bytes']) is not int or not 0 < entry['bytes'] <= MAX_RECORD_BYTES: + _fail('Invalid inventory record byte count.') + total += entry['bytes'] + if total > MAX_TOTAL_BYTES: + _fail('Project records exceed their total byte limit.') + roots.append(dict(family=family, id=record_id)) + if manifest['roots'] != roots: + _fail('Project membership roots disagree with the inventory.') + if type(manifest['receipts']) is not list or not manifest['receipts']: + _fail('Project requires operation receipts.') + seen = set() + for receipt in manifest['receipts']: + _keys(receipt, ('request_key', 'request_sha256', 'records_sha256'), 'Operation receipt') + _request(receipt['request_key'], receipt['request_sha256']) + _digest(receipt['records_sha256'], 'Receipt records digest') + if receipt['request_key'] in seen: + _fail('Duplicate operation receipt key.') + seen.add(receipt['request_key']) + return manifest + + def _path_string(path): try: value = os.fspath(path) @@ -569,6 +634,7 @@ def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path, canoni self._read_cache = None self._mutex = threading.RLock() self._unknown = False + self._folder_mode = False try: fcntl.flock(root_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError as exc: @@ -704,6 +770,115 @@ def open_at(cls, trusted_root_fd, relative, *, root_path, canonical_digests=None if origin_fd is not None: os.close(origin_fd) + # ---- Folder mode: the external project as an import source/export target. + # A folder-mode Store holds the writer lock and root binding but never + # loads the record set; the database working copy owns current records. + + @classmethod + def open_folder_at(cls, trusted_root_fd, relative, *, root_path): + """Bind an existing external project folder without reading its records.""" + parts = _relative_parts(relative) + absolute = _verify_origin(trusted_root_fd, root_path) + origin_fd = os.dup(trusted_root_fd) + store = None + try: + fd = _walk(origin_fd, parts) + owned_origin, origin_fd = origin_fd, None + store = cls(Path(absolute).joinpath(*parts), fd, owned_origin, parts, absolute) + store._meta_fd = _directory(fd, '.tuldok') + store._folder_mode = True + return store + except OSError as exc: + if store is not None: + store.close() + _io_error(exc, 'open project folder') + except BaseException: + if store is not None: + store.close() + raise + finally: + if origin_fd is not None: + os.close(origin_fd) + + def folder_checkpoint(self): + """(raw bytes, checked manifest) of the folder's current project.json.""" + with self._mutex: + self._binding() + try: + raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) + except OSError as exc: + _io_error(exc, 'read project checkpoint') + return raw, checked_manifest(raw) + + def folder_record(self, entry): + """One inventory entry's exact bytes after length/SHA256 verification.""" + with self._mutex: + try: + raw = self._record_file(entry['family'], entry['id'], entry['sha256']) + except OSError as exc: + _io_error(exc, 'read project record') + if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: + _fail('Canonical record bytes changed or are corrupt.') + return raw + + def folder_has_record(self, entry): + """Whether the content-addressed revision file exists with its byte length.""" + with self._mutex: + fds = [] + try: + parent = self._meta_fd + for part in ('records', entry['family'], entry['id']): + parent = _directory(parent, part) + fds.append(parent) + info = os.stat(entry['sha256'] + '.json', dir_fd=parent, follow_symlinks=False) + return stat.S_ISREG(info.st_mode) and info.st_size == entry['bytes'] + except FileNotFoundError: + return False + except OSError as exc: + _io_error(exc, 'inspect project record') + finally: + for fd in reversed(fds): + os.close(fd) + + def folder_write_record(self, family, record_id, digest, raw): + """Durably publish one immutable revision; existing bytes must be identical.""" + with self._mutex: + try: + written = self._record_file(family, record_id, digest, raw) + except OSError as exc: + _io_error(exc, 'write project record') + if written != raw: + _fail('Immutable project revision already has different bytes.') + + def folder_publish(self, raw, expected_raw): + """Atomically replace project.json only if it still has expected_raw. + + expected_raw None means the folder has no checkpoint yet. Refuses to + overwrite a checkpoint someone else published since import/export. + """ + if len(raw) > MAX_MANIFEST_BYTES: + _fail('Project checkpoint exceeds its byte limit.') + with self._mutex: + temporary = '.checkpoint-' + uuid.uuid4().hex + '.tmp' + replaced = False + try: + _write_file(self._meta_fd, temporary, raw) + self._binding() + try: + current = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) + except FileNotFoundError: + current = None + if current != expected_raw: + os.unlink(temporary, dir_fd=self._meta_fd) + _fail('The external project changed since Tuldok last imported or saved it.', 'conflict', 409) + os.replace(temporary, 'project.json', src_dir_fd=self._meta_fd, dst_dir_fd=self._meta_fd) + replaced = True + os.fsync(self._meta_fd) + os.fsync(self._root_fd) + except OSError as exc: + self._unknown = replaced + _io_error(exc, 'publish checkpoint', unknown=replaced) + def _binding(self): if self._root_fd is None: _fail('Project store is closed.', 'unavailable', 409) @@ -752,50 +927,13 @@ def _load(self): # is still refused by Save (verify_records) and by every reopen. self._binding() return cached[1:] - manifest = _decode(raw) - if type(manifest) is not dict: - _fail('Project checkpoint must be an object.') - if manifest.get('format') != FORMAT: - if type(manifest.get('format')) is str and re.fullmatch(r'tuldok_project_v[1-9][0-9]*', manifest['format']): - _fail('Unsupported project format.', 'unsupported', 422) - _fail('Invalid project format.') - _keys(manifest, ('format', 'project_id', 'name', 'checkpoint', 'roots', 'inventory', 'receipts'), 'Project checkpoint') - identity = manifest['project_id'] - if type(identity) is not str or not re.fullmatch('[a-f0-9]{32}', identity): - _fail('Invalid project UUID.') - if type(manifest['name']) is not str or not manifest['name'].strip() or len(manifest['name']) > 200: - _fail('Invalid project display name.') - _digest(manifest['checkpoint'], 'Checkpoint') - core = {k: v for k, v in manifest.items() if k != 'checkpoint'} - if _sha(_encode(core)) != manifest['checkpoint']: - _fail('Project checkpoint digest does not match its contents.') - inventory = manifest['inventory'] - if type(inventory) is not list or len(inventory) > MAX_RECORDS: - _fail('Invalid project inventory.') - loaded, roots, total, previous = [], [], 0, None + manifest = checked_manifest(raw) + loaded = [] records_digest = _RecordsDigest() known = self._canonical_digests proven = [] - for entry in inventory: - _keys(entry, ('family', 'id', 'format', 'namespace', 'path', 'bytes', 'sha256'), 'Inventory entry') + for entry in manifest['inventory']: family, record_id, digest = entry['family'], entry['id'], entry['sha256'] - _identifier(family, 'Record family') - if family not in SUPPORTED_FORMATS: - _fail('Unsupported project record family.', 'unsupported', 422) - _identifier(record_id, 'Record ID') - _digest(digest, 'Record digest') - _domain_format(entry['format'], family) - key = (family, record_id) - if previous is not None and key <= previous: - _fail('Inventory must have unique sorted record identities.') - previous = key - if entry['namespace'] != identity or entry['path'] != f'records/{family}/{record_id}/{digest}.json': - _fail('Invalid inventory namespace or rooted record path.') - if type(entry['bytes']) is not int or not 0 < entry['bytes'] <= MAX_RECORD_BYTES: - _fail('Invalid inventory record byte count.') - total += entry['bytes'] - if total > MAX_TOTAL_BYTES: - _fail('Project records exceed their total byte limit.') record_raw = self._verified_record_bytes(entry) if known is not None and digest in known: # These exact bytes (named by their verified SHA256) were already @@ -809,19 +947,6 @@ def _load(self): _fail('Domain document format disagrees with its inventory.') records_digest.add(family, record_id, record_raw) loaded.append(dict(family=family, id=record_id, document=document)) - roots.append(dict(family=family, id=record_id)) - if manifest['roots'] != roots: - _fail('Project membership roots disagree with the inventory.') - if type(manifest['receipts']) is not list or not manifest['receipts']: - _fail('Project requires operation receipts.') - seen = set() - for receipt in manifest['receipts']: - _keys(receipt, ('request_key', 'request_sha256', 'records_sha256'), 'Operation receipt') - _request(receipt['request_key'], receipt['request_sha256']) - _digest(receipt['records_sha256'], 'Receipt records digest') - if receipt['request_key'] in seen: - _fail('Duplicate operation receipt key.') - seen.add(receipt['request_key']) # _decode proved each raw revision is its document's canonical encoding. if manifest['receipts'][-1]['records_sha256'] != records_digest.hexdigest(): _fail('Latest operation receipt disagrees with the current record set.') diff --git a/project_transfer.py b/project_transfer.py index 91848559..2fb1d3ec 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -703,7 +703,8 @@ def replay(opened): if opened_store is not None: # Trusted controller-only capability. Its caller pins workspace, # authority, selected session and SQL ownership through this replay. - if not isinstance(opened_store,ProjectStore) or existing is None: + from project_working_store import WorkingProjectStore + if not isinstance(opened_store,(ProjectStore,WorkingProjectStore)) or existing is None: fail('Current selected destination is unavailable.','stale',409) with opened_store.root_directory() as owned_fd: try:target_fd=os.open(directory,DIR_FLAGS,dir_fd=destination_parent_fd) diff --git a/project_working_store.py b/project_working_store.py new file mode 100644 index 00000000..636050fa --- /dev/null +++ b/project_working_store.py @@ -0,0 +1,449 @@ +"""Persistent PostgreSQL working copy of one external project folder. + +The database holds the current record set of an opened project and is kept +across sessions. The external folder (`.tuldok/project.json` plus +content-addressed `records/`, tracked in Git) is an import source, read only +when its checkpoint changed since Tuldok last imported or saved it, and an +export target written only by save(). See docs/plans/database-working-store.md. + +Records are kept exactly as their canonical bytes with their SHA256, so the +working copy has the same manifest, checkpoint digest and receipts as the file +format and save() writes byte-identical files. This store keeps no record state +in memory: inventory, receipts and bodies are read from PostgreSQL, so an outer +transaction rollback can never leave a stale in-process copy. + +Interface parity with ProjectStore (summary, snapshot, commit, save, +reconcile, root_directory, verify_records, recover_checkpoint, close, path) +lets Session, owners, artifacts and custody use either store. +""" +from contextlib import contextmanager, nullcontext +import json +import os +import re +import threading +import uuid + +import psycopg +from psycopg import sql + +from metadata import _key, connection_info +from project_store import (FORMAT, MAX_RECORD_BYTES, MAX_RECORDS, MAX_TOTAL_BYTES, SUPPORTED_FORMATS, + ProjectStoreError, _RecordsDigest, _decode, _digest, _domain_format, _encode, + _fail, _identifier, _keys, _manifest_raw, _request, _sha) + +WORKING_ID = re.compile(r'^[a-f0-9]{32}$') +_TABLES = ( + '''CREATE TABLE IF NOT EXISTS {schema}.state ( + singleton boolean PRIMARY KEY DEFAULT true CHECK (singleton), + project_id text NOT NULL, name text NOT NULL, checkpoint text NOT NULL, + receipts bytea NOT NULL, external_sha256 text, dirty boolean NOT NULL)''', + '''CREATE TABLE IF NOT EXISTS {schema}.records ( + family text COLLATE "C" NOT NULL, id text COLLATE "C" NOT NULL, + format text NOT NULL, bytes integer NOT NULL, sha256 text NOT NULL, + raw bytea NOT NULL, PRIMARY KEY (family, id))''', +) +_ORDER = sql.SQL('ORDER BY family COLLATE "C", id COLLATE "C"') + + +class WorkingProjectStore: + """One exclusive owner of a project's database working copy.""" + + def __init__(self, folder, working_id, *, dsn=None, canonical_digests=None): + if type(working_id) is not str or not WORKING_ID.fullmatch(working_id): + raise ValueError('Invalid working copy identity.') + self._folder = folder + self.working_id = working_id + self.schema = 'working_' + working_id + self._canonical_digests = canonical_digests + self._mutex = threading.RLock() + self._db = None + self._closed = False + try: + self._own = psycopg.connect(connection_info(dsn), autocommit=True) + except psycopg.Error: + raise ProjectStoreError('Local PostgreSQL is unavailable.', 'unavailable', 503) from None + try: + # One Tuldok process owns a working copy at a time. + if not self._own.execute('SELECT pg_try_advisory_lock(%s)', (_key(self.schema),)).fetchone()[0]: + raise ProjectStoreError('This project is already open in another Tuldok session.', 'busy', 409) + with self._own.transaction(): + self._own.execute(sql.SQL('CREATE SCHEMA IF NOT EXISTS {}').format(sql.Identifier(self.schema))) + for statement in _TABLES: + self._own.execute(sql.SQL(statement).format(schema=sql.Identifier(self.schema))) + except BaseException: + self._own.close() + raise + + # ---- Connection: own until bound to the projection, then shared so + # record commits are atomic with the projection's SQL writes. + + def bind(self, db): + """Use the projection database's connection and transactions.""" + with self._mutex: + self._db = db + + def _connection(self): + return self._db.connection if self._db is not None else self._own + + def _guard(self): + # A bound projection connection is shared; its catalog lock serializes + # every command and transaction on it across threads. + return self._db.lock if self._db is not None else nullcontext() + + @contextmanager + def _transaction(self): + with self._guard(): + if self._db is not None: + with self._db.transaction(): + yield + else: + with self._own.transaction(): + yield + + def _execute(self, statement, parameters=None): + if self._closed: + _fail('Project store is closed.', 'unavailable', 409) + with self._guard(): + return self._connection().execute(statement, parameters) + + def _q(self, text): + return sql.SQL(text).format(state=sql.Identifier(self.schema, 'state'), + records=sql.Identifier(self.schema, 'records'), order=_ORDER) + + # ---- Open: attach, import or refuse. + + @classmethod + def open(cls, folder, working_id, *, dsn=None, canonical_digests=None): + """Attach the working copy, importing the folder only if it changed.""" + store = cls(folder, working_id, dsn=dsn, canonical_digests=canonical_digests) + try: + raw, manifest = folder.folder_checkpoint() + external = _sha(raw) + state = store._state(required=False) + if state is not None and state['project_id'] != manifest['project_id']: + _fail('This working copy belongs to a different project.', 'conflict', 409) + if state is None or state['external_sha256'] != external: + if state is not None and state['dirty']: + _fail('The external project changed while this workspace has unsaved changes. ' + 'Save over it or discard the working copy and import again.', 'conflict', 409) + store._import(raw, manifest) + return store + except BaseException: + store.close() + raise + + def _state(self, *, required=True): + row = self._execute(self._q('SELECT project_id, name, checkpoint, receipts, external_sha256, dirty FROM {state}')).fetchone() + if row is None: + if required: + _fail('Project working copy is missing.', 'unavailable', 409) + return None + return dict(project_id=row[0], name=row[1], checkpoint=row[2], receipts=json.loads(bytes(row[3])), + external_sha256=row[4], dirty=row[5]) + + def _import(self, raw, manifest): + """Bring the working copy to the folder's checkpoint, reading only new revisions.""" + with self._mutex, self._transaction(): + existing = {(row[0], row[1]): row[2] for row in + self._execute(self._q('SELECT family, id, sha256 FROM {records}'))} + wanted = {} + proven = [] + known = self._canonical_digests + for entry in manifest['inventory']: + identity = (entry['family'], entry['id']) + wanted[identity] = entry + if existing.get(identity) == entry['sha256']: + continue + body = self._folder.folder_record(entry) + if known is not None and entry['sha256'] in known: + document = json.loads(body) + else: + document = _decode(body) + proven.append(entry['sha256']) + if type(document) is not dict or document.get('format') != entry['format']: + _fail('Domain document format disagrees with its inventory.') + self._execute(self._q( + 'INSERT INTO {records} (family, id, format, bytes, sha256, raw) VALUES (%s,%s,%s,%s,%s,%s) ' + 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' + 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw'), + (entry['family'], entry['id'], entry['format'], entry['bytes'], entry['sha256'], body)) + for family, record_id in set(existing) - set(wanted): + self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, record_id)) + if self._records_digest() != manifest['receipts'][-1]['records_sha256']: + _fail('Latest operation receipt disagrees with the current record set.') + self._write_state(manifest, external_sha256=_sha(raw), dirty=False) + if known is not None and proven: + known.add_many(proven) + + def _write_state(self, manifest, *, external_sha256, dirty): + receipts = _encode(manifest['receipts']) + self._execute(self._q('DELETE FROM {state}')) + self._execute(self._q('INSERT INTO {state} (project_id, name, checkpoint, receipts, external_sha256, dirty) ' + 'VALUES (%s,%s,%s,%s,%s,%s)'), + (manifest['project_id'], manifest['name'], manifest['checkpoint'], receipts, + external_sha256, dirty)) + + def _rows(self, statement, parameters=None): + """Stream (family, id, raw) rows through a server-side cursor. + + Bodies are fetched in small batches, never the whole record set at once. + """ + if self._closed: + _fail('Project store is closed.', 'unavailable', 409) + with self._guard(): + connection = self._connection() + idle = connection.info.transaction_status == psycopg.pq.TransactionStatus.IDLE + with (connection.transaction() if idle else nullcontext()), \ + connection.cursor(name='working_' + uuid.uuid4().hex) as cursor: + cursor.itersize = 64 + cursor.execute(statement, parameters) + for row in cursor: + yield row[0], row[1], bytes(row[2]) + + def _records_digest(self, replacements=None, identities=None): + """Streamed receipt digest over the stored (or a candidate) record set.""" + digest = _RecordsDigest() + replacements = replacements or {} + if identities is None: + for family, record_id, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}')): + digest.add(family, record_id, raw) + return digest.hexdigest() + stored = self._rows(self._q('SELECT family, id, raw FROM {records} {order}')) + pending = next(stored, None) + for identity in identities: + while pending is not None and (pending[0], pending[1]) < identity: + pending = next(stored, None) + if identity in replacements: + digest.add(identity[0], identity[1], replacements[identity]) + elif pending is not None and (pending[0], pending[1]) == identity: + digest.add(identity[0], identity[1], pending[2]) + else: + _fail('Unchanged candidate record is missing from the working copy.') + return digest.hexdigest() + + # ---- Manifest identity, identical to the file format. + + def _inventory(self): + return [(row[0], row[1], row[2], row[3], row[4]) for row in + self._execute(self._q('SELECT family, id, format, bytes, sha256 FROM {records} {order}'))] + + @staticmethod + def _manifest(state, inventory): + manifest = dict(format=FORMAT, project_id=state['project_id'], name=state['name'], + roots=[dict(family=f, id=i) for f, i, _fmt, _b, _s in inventory], + inventory=[dict(family=f, id=i, format=fmt, namespace=state['project_id'], + path=f'records/{f}/{i}/{sha}.json', bytes=size, sha256=sha) + for f, i, fmt, size, sha in inventory], + receipts=state['receipts']) + manifest['checkpoint'] = _sha(_encode(manifest)) + return manifest + + # ---- ProjectStore interface. + + @property + def path(self): + return self._folder.path + + @property + def _root_fd(self): + """The external folder's locked physical root (read-only).""" + return self._folder._root_fd + + def summary(self): + with self._mutex: + state = self._state() + return {key: state[key] for key in ('project_id', 'name', 'checkpoint')} + + def dirty(self): + """Whether the working copy has changes not yet saved to the folder.""" + with self._mutex: + return self._state()['dirty'] + + def snapshot(self): + """Complete independent record set (costly for large projects).""" + with self._mutex: + state = self._state() + records = [dict(family=f, id=i, document=json.loads(raw)) + for f, i, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}'))] + return dict(project_id=state['project_id'], name=state['name'], + checkpoint=state['checkpoint'], records=records) + + def records(self, *families): + """Independent decoded envelopes of the selected families, in record order.""" + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + with self._mutex: + return [dict(family=f, id=i, document=json.loads(raw)) for f, i, raw in self._rows( + self._q('SELECT family, id, raw FROM {records} WHERE family = ANY(%s) {order}'), (list(families),))] + + def record(self, family, record_id): + with self._mutex: + row = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), + (family, record_id)).fetchone() + return None if row is None else dict(family=family, id=record_id, document=json.loads(bytes(row[0]))) + + def reconcile(self, request_key, request_sha256): + _request(request_key, request_sha256) + with self._mutex: + for receipt in self._state()['receipts']: + if receipt['request_key'] == request_key: + if receipt['request_sha256'] != request_sha256: + _fail('Request key was already used with different input.', 'conflict', 409) + return self.snapshot() + return None + + def commit(self, records, expected_checkpoint, request_key, request_sha256, *, changed=None): + """Replace the complete current record set in the working copy. + + changed, when given, names every (family, id) whose document may differ + from the stored revision; all other candidate records are the caller's + unmodified copies of stored revisions (Session.mutate constructs them + so). Without it every candidate is compared exactly with its revision. + Returns the committed snapshot, sharing the caller's record list. + """ + _request(request_key, request_sha256) + _digest(expected_checkpoint, 'Expected checkpoint') + if type(records) is not list or len(records) > MAX_RECORDS: + _fail('Project records must be a bounded list.') + with self._mutex, self._transaction(): + state = self._state() + inventory = {(f, i): (fmt, size, sha) for f, i, fmt, size, sha in self._inventory()} + candidates = {} + for record in records: + _keys(record, ('family', 'id', 'document'), 'Project record') + family, identity, document = record['family'], record['id'], record['document'] + _identifier(family, 'Record family') + if family not in SUPPORTED_FORMATS: + _fail('Unsupported project record family.', 'unsupported', 422) + _identifier(identity, 'Record ID') + if (family, identity) in candidates: + _fail('Duplicate project record identity.') + if type(document) is not dict: + _fail('Domain document must be an object.') + _domain_format(document.get('format'), family) + candidates[(family, identity)] = document + replacements, entries, total = {}, {}, 0 + for identity in sorted(candidates): + document = candidates[identity] + current = inventory.get(identity) + if current is not None and changed is not None and identity not in changed \ + and current[0] == document['format']: + entries[identity] = current + total += current[1] + continue + raw = _encode(document) + if len(raw) > MAX_RECORD_BYTES: + _fail('Project records exceed the synchronous byte limit.') + digest = _sha(raw) + if current is None or current[2] != digest: + replacements[identity] = raw + entries[identity] = (document['format'], len(raw), digest) + total += len(raw) + if total > MAX_TOTAL_BYTES: + _fail('Project records exceed the synchronous byte limit.') + if total > MAX_TOTAL_BYTES: + _fail('Project records exceed the synchronous byte limit.') + records_digest = self._records_digest(replacements, sorted(candidates)) + for receipt in state['receipts']: + if receipt['request_key'] == request_key: + if receipt['request_sha256'] != request_sha256 or receipt['records_sha256'] != records_digest: + _fail('Request key was already used with different input.', 'conflict', 409) + return self.snapshot() + if expected_checkpoint != state['checkpoint']: + _fail('Expected project checkpoint is stale.', 'stale', 409) + receipts = state['receipts'] + [dict(request_key=request_key, request_sha256=request_sha256, + records_sha256=records_digest)] + manifest = self._manifest(dict(state, receipts=receipts), + [(f, i, *entries[(f, i)]) for f, i in sorted(entries)]) + _manifest_raw(manifest) + for (family, identity), raw in replacements.items(): + fmt, size, digest = entries[(family, identity)] + self._execute(self._q( + 'INSERT INTO {records} (family, id, format, bytes, sha256, raw) VALUES (%s,%s,%s,%s,%s,%s) ' + 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' + 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw'), (family, identity, fmt, size, digest, raw)) + for family, identity in set(inventory) - set(candidates): + self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, identity)) + self._write_state(manifest, external_sha256=state['external_sha256'], dirty=True) + if self._canonical_digests is not None and replacements: + self._canonical_digests.add_many([_sha(raw) for raw in replacements.values()]) + return dict(project_id=state['project_id'], name=state['name'], checkpoint=manifest['checkpoint'], + records=[dict(family=f, id=i, document=candidates[(f, i)]) for f, i in sorted(candidates)]) + + def save(self, expected_checkpoint): + """Export the working copy to the external folder (the only folder write).""" + _digest(expected_checkpoint, 'Expected checkpoint') + with self._mutex: + state = self._state() + if expected_checkpoint != state['checkpoint']: + _fail('Expected project checkpoint is stale.', 'stale', 409) + try: + external_raw, _external = self._folder.folder_checkpoint() + except ProjectStoreError: + external_raw = None + if (state['external_sha256'] is not None and + (external_raw is None or _sha(external_raw) != state['external_sha256'])): + _fail('The external project changed since Tuldok last imported or saved it.', 'conflict', 409) + manifest = self._manifest(state, self._inventory()) + if manifest['checkpoint'] != state['checkpoint']: + _fail('Working copy checkpoint disagrees with its records.', 'invalid', 409) + raw = _manifest_raw(manifest) + if state['external_sha256'] != _sha(raw): + for entry in manifest['inventory']: + if not self._folder.folder_has_record(entry): + body = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), + (entry['family'], entry['id'])).fetchone() + body = bytes(body[0]) + if len(body) != entry['bytes'] or _sha(body) != entry['sha256']: + _fail('Working copy record bytes are corrupt.', 'invalid', 409) + self._folder.folder_write_record(entry['family'], entry['id'], entry['sha256'], body) + self._folder.folder_publish(raw, external_raw) + with self._transaction(): + self._write_state(manifest, external_sha256=_sha(raw), dirty=False) + return {key: state[key] for key in ('project_id', 'name', 'checkpoint')} + + def verify_records(self): + """Re-check every stored revision's bytes against its SHA256 and length.""" + with self._mutex: + inventory = {(f, i): (size, sha) for f, i, _fmt, size, sha in self._inventory()} + for family, record_id, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}')): + size, digest = inventory[(family, record_id)] + if len(raw) != size or _sha(raw) != digest: + _fail('Canonical record bytes changed or are corrupt.', 'stale', 409) + + def recover_checkpoint(self): + """Database commits are atomic; return the committed record set.""" + return self.snapshot() + + @contextmanager + def root_directory(self, *, verify_after=False): + """Borrow the external folder's physical root (payload artifacts live there).""" + if type(verify_after) is not bool: + _fail('Root read verification must be an explicit boolean.') + with self._mutex: + before = self._state()['checkpoint'] + descriptor = os.dup(self._folder._root_fd) + try: + yield descriptor + self._folder._binding() + if verify_after and self._state()['checkpoint'] != before: + _fail('Project checkpoint changed during the bounded root read.', 'stale', 409) + finally: + os.close(descriptor) + + def close(self): + with self._mutex: + if self._closed: + return + self._closed = True + try: + self._folder.close() + finally: + self._own.close() + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + diff --git a/project_workspace.py b/project_workspace.py index 047a7c82..b6f61256 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -21,6 +21,7 @@ from project_projection import ProjectProjection from project_custody import ProjectCustody from project_validation_cache import CanonicalDigestCache, ProjectValidationCache +from project_working_store import WORKING_ID, WorkingProjectStore from workbench import WorkbenchError @@ -206,11 +207,47 @@ def _root(self, alias, relative): assert_separate_project_control(root_fd, relative, root_path=root_path, control_path=self.path) return self._roots[alias] + def _working_identity(self, alias, directory, project_id): + """The persistent working copy for one project folder (private registry).""" + key = hashlib.sha256(f'{alias}\0{directory}'.encode('utf-8')).hexdigest() + registry = self.path / '.project-working' + path = registry / f'{key}.json' + try: + value = json.loads(path.read_bytes()) + if (value.get('project_id') == project_id and type(value.get('working_id')) is str + and WORKING_ID.fullmatch(value['working_id'])): + return value['working_id'] + except (OSError, ValueError, AttributeError): + pass + working_id = uuid.uuid4().hex + registry.mkdir(mode=0o700, exist_ok=True) + temporary = registry / f'.{uuid.uuid4().hex}.tmp' + temporary.write_bytes(json.dumps(dict(format='project_working_registration_v1', root=alias, + directory=directory, project_id=project_id, working_id=working_id), sort_keys=True).encode('utf-8')) + os.chmod(temporary, 0o600) + os.replace(temporary, path) + return working_id + + def _open_working(self, root_fd, root_path, alias, directory): + """Attach the folder's working copy, importing only if the folder changed.""" + folder = ProjectStore.open_folder_at(root_fd, directory, root_path=root_path) + try: + _raw, manifest = folder.folder_checkpoint() + working_id = self._working_identity(alias, directory, manifest['project_id']) + return WorkingProjectStore.open(folder, working_id, dsn=self._dsn, + canonical_digests=self.canonical_digests) + except BaseException: + folder.close() + raise + def _projection_for(self, store): identity = uuid.uuid4().hex path = self.path / '.project-projections' / identity snapshot = store.summary() projection = Metadata(path, dsn=self._dsn, dataset_id=snapshot['project_id'], projection_id=identity) + if hasattr(store, 'bind'): + # Record commits share the projection's transactions. + store.bind(projection) try: owner = ProjectProjection(store, projection, self.access, {alias: value[0] for alias, value in self._roots.items()}, @@ -232,6 +269,8 @@ def _retained_projection_for(self, store, projection_id): raise WorkbenchError('Invalid original local projection identity.', 'unavailable', 409) db = Metadata(self.path / '.project-projections' / projection_id, dsn=self._dsn, create=False, dataset_id=store.summary()['project_id'], projection_id=projection_id) + if hasattr(store, 'bind'): + store.bind(db) try: return ProjectProjection(store, db, self.access, {alias: value[0] for alias, value in self._roots.items()}, @@ -245,9 +284,8 @@ def _detach(self): """Release only a project whose current local effects have drained.""" if self._projection is not None: self._projection.execution.assert_quiescent() - if self._store is not None: - # Prove pending project publications and their full closure first. - self._store.save(self._store.summary()['checkpoint']) + # Closing keeps unsaved work in the database working copy; only an + # explicit Save writes the external folder. if self._projection is not None: self._projection.close() if self._store is not None: @@ -275,13 +313,12 @@ def _recover_current(self, arguments): self._store = None store = projection = None try: - store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path, canonical_digests=self.canonical_digests) + store = self._open_working(root_fd, root_path, *self._selection) snapshot = store.summary() if snapshot['project_id'] != self._last_project['project_id']: raise WorkbenchError('Recovery destination has a different project identity.', 'stale_project', 409) - # A reread reconciles identity; only a successful durability barrier - # resolves a prior checkpoint/directory fsync failure. - store.save(snapshot['checkpoint']) + # Database commits are atomic: reattaching the working copy + # reconciles the outcome without writing the external folder. projection, text = self._projection_for(store) snapshot = store.summary() if self._projection is not None: @@ -334,14 +371,18 @@ def _select(self, action, arguments): except ProjectStoreError: raise error if store.reconcile(arguments['request_key'], digest) is None: + store.close() raise error reconciled = True snapshot = store.summary() if reconciled: store.save(snapshot['checkpoint']) created = snapshot + # The new folder becomes this workspace's working copy. + store.close() + store = self._open_working(root_fd, root_path, arguments['root'], arguments['directory']) else: - store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path, canonical_digests=self.canonical_digests) + store = self._open_working(root_fd, root_path, arguments['root'], arguments['directory']) projection, text = self._projection_for(store) self._detach() self._store, self._projection, self._text = store, projection, text @@ -398,7 +439,6 @@ def _publish_independent(self, arguments, *, composed=False): source = self._projection source.execution.assert_quiescent() source.files.require_clean() - self._store.save(arguments['expected_checkpoint']) published = None incoming = projection = None retained = ExitStack() @@ -433,7 +473,7 @@ def _publish_independent(self, arguments, *, composed=False): # A filesystem editor can act outside the workspace lock. Leave the # complete fork unselected if its source working files changed. source.files.require_clean() - incoming = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path, canonical_digests=self.canonical_digests) + incoming = self._open_working(root_fd, root_path, arguments['root'], arguments['directory']) projection, text = self._projection_for(incoming) snapshot, overlay = published['snapshot'], published['fork'] self._detach() diff --git a/tests/test_project_files_http.py b/tests/test_project_files_http.py index 39a271f3..b592014a 100644 --- a/tests/test_project_files_http.py +++ b/tests/test_project_files_http.py @@ -282,57 +282,39 @@ def test_legacy_mutations_are_unavailable_before_and_after_open(self): self.assertEqual(canonical_bytes(self.external), before) self.assertEqual(list((self.path / 'control').rglob('*.png')), []) - def test_real_deferred_sql_failure_returns_committed_receipt_and_explicit_recovery(self): + def test_real_deferred_sql_failure_rolls_back_the_edit_and_a_retry_succeeds(self): self.create() selected = self.state() projection_schema = self.workspace._projection.schema - # A real deferred PostgreSQL trigger fails the outer SQL COMMIT, after - # canonical publication. It exists only in this disposable projection. + # A real deferred PostgreSQL trigger fails the outer SQL COMMIT. Records + # and projection rows share that transaction, so both roll back. with psycopg.connect(connection_info(), autocommit=True) as connection: function = sql.Identifier(projection_schema, 'qa_reject_projection_commit') table = sql.Identifier(projection_schema, 'workbench_records') connection.execute(sql.SQL('''CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql - AS $$ BEGIN RAISE EXCEPTION 'Disposable QA deferred commit failure'; END; $$''').format(function)) + AS $$ BEGIN RAISE EXCEPTION 'Disposable QA deferred commit failure' USING ERRCODE='23514'; END; $$''').format(function)) connection.execute(sql.SQL('''CREATE CONSTRAINT TRIGGER qa_reject_projection_commit AFTER INSERT ON {} DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION {}()''').format(table, function)) - asset = {'name': 'Committed despite unavailable SQL 山', 'text': TEXT, + asset = {'name': 'Rejected with its SQL commit 山', 'text': TEXT, 'groups': ['recovery'], 'parents': [], 'rights': 'Owned synthetic recovery fixture'} original = self.envelope(selected, asset=asset) + before = canonical_bytes(self.external / 'alpha') status, error = self.request(API + '/import-text', original) - self.assertEqual(status, 503, error) - self.assertEqual(error['code'], 'projection_unavailable') - receipt = error['receipt'] - self.assertEqual(receipt['committed'], True) - self.assertEqual(receipt['selected'], True) - self.assertEqual(receipt['project_id'], selected['project']['project_id']) - self.assertEqual(receipt['request_key'], original['request_key']) - self.assertNotEqual(receipt['checkpoint'], selected['project']['checkpoint']) - before_recovery = canonical_bytes(self.external / 'alpha') - manifest = json.loads(before_recovery['.tuldok/project.json']) - self.assertEqual(manifest['checkpoint'], receipt['checkpoint']) - recovery = self.state() - self.assertEqual(recovery['state'], 'recovery_required') - self.assertEqual(recovery['available_actions'], ['open']) - self.assertEqual(recovery['recovery']['reason'], 'projection_unavailable') - status, denied = self.request(API + '/save', self.envelope(recovery)) - self.assertEqual(status, 503, denied) - self.assertEqual(canonical_bytes(self.external / 'alpha'), before_recovery) - self.action('open', recovery, root='fixture', directory='alpha') - recovered = self.state() - self.assertEqual(recovered['state'], 'open') - self.assertEqual(recovered['project']['checkpoint'], receipt['checkpoint']) - self.assertNotEqual(self.workspace._projection.schema, projection_schema) - records = self.records(recovered) + self.assertEqual(status, 409, error) + self.assertEqual(error['code'], 'rejected') + self.assertEqual(self.state(), selected, 'Nothing changed; the project stays open and current') + self.assertEqual(self.records(selected), []) + self.assertEqual(canonical_bytes(self.external / 'alpha'), before) + with psycopg.connect(connection_info(), autocommit=True) as connection: + connection.execute(sql.SQL('DROP TRIGGER qa_reject_projection_commit ON {}').format( + sql.Identifier(projection_schema, 'workbench_records'))) + status, imported = self.request(API + '/import-text', original) + self.assertEqual(status, 200, imported) + records = self.records() self.assertEqual(len(records), 1) self.assertEqual(self.detail(records[0]['id'])['record']['text'], TEXT) - # Reconcile the exact durable request with its old checkpoint and the - # new process binding. A committed operation cannot be duplicated. - status, replayed = self.request(API + '/import-text', dict(original, binding=recovered['binding'])) - self.assertEqual(status, 200, replayed) - self.assertEqual(replayed['record']['id'], records[0]['id']) - self.assertEqual(len(self.records()), 1) - self.assertEqual(canonical_bytes(self.external / 'alpha'), before_recovery) + self.assertEqual(canonical_bytes(self.external / 'alpha'), before, 'Edits are not written until Save') @unittest.skipUnless(importlib.util.find_spec('mcp') and importlib.util.find_spec('anyio'), 'Requires the pinned optional MCP environment for real adapter acceptance') diff --git a/tests/test_project_files_m2_http.py b/tests/test_project_files_m2_http.py index 82492847..9d66176f 100644 --- a/tests/test_project_files_m2_http.py +++ b/tests/test_project_files_m2_http.py @@ -404,6 +404,8 @@ def test_retained_measurement_import_query_permissions_and_independent_cold_copy status,admitted=self.transport('measurement_import',import_args) self.assertEqual(status,200,admitted) origin=admitted['project_id'];selection=dict(origin_catalog_id=origin,packet_sha256=digest) + # Edits stay in the database working copy until an explicit Save. + self.action('save') canonical=m1.canonical_bytes(self.external/'alpha') self.assertEqual(self.transport('measurement_import',import_args)[1],admitted) self.assertEqual(m1.canonical_bytes(self.external/'alpha'),canonical) diff --git a/tests/test_project_files_m4_http.py b/tests/test_project_files_m4_http.py index 8d6b0a6f..bd7d48ca 100644 --- a/tests/test_project_files_m4_http.py +++ b/tests/test_project_files_m4_http.py @@ -47,7 +47,7 @@ def test_batch_adoption_uses_real_http_and_survives_independent_fork(self): items.append(dict(kind='note',path='notes/'+name,expected_revision=0,expected_sha256=hashlib.sha256(raw).hexdigest())) result=self.call('files_adopt',items=items) self.assertEqual(result['changed_count'],2);self.assertEqual(len(self.call('files_inspect')['items']),2) - with patch.object(self.workspace._store,'_snapshot',side_effect=AssertionError('Unrelated full graph copy')): + with patch.object(self.workspace._store,'snapshot',side_effect=AssertionError('Unrelated full graph copy')): self.assertEqual(len(self.call('files_inspect')['items']),2) selected=self.call('file_get',file_id=result['items'][0]['id']) self.assertEqual(selected['version']['text'].encode(),b'Exact first note\r\n') @@ -133,16 +133,23 @@ def test_published_unselected_fork_exact_retry_and_changed_file_guard(self): def test_git_checkout_stale_projection_and_moved_folder_inert_reconstruction(self): self.create();root=self.external/'alpha' first=self.call('data_import_text',text='Git observation one',name='One',groups=['fixture'],rights='Owned') + self.action('save') def git(*args):return subprocess.check_output(['git','-C',str(root),*args],stderr=subprocess.STDOUT) git('init','-q');git('config','user.name','Disposable Fixture');git('config','user.email','fixture@example.invalid') git('add','.');git('commit','-qm','First authored checkpoint');old=git('rev-parse','HEAD').decode().strip() self.call('data_import_text',text='Git observation two',name='Two',groups=['fixture'],rights='Owned') + self.action('save') git('add','.');git('commit','-qm','Second authored checkpoint') - stale=self.state() git('checkout','-q',old) - state=self.state();self.assertEqual(state['state'],'recovery_required') - status,failure=self.transport('data_import_text',dict(self.envelope(stale),text='Must not overwrite',name='Blocked',groups=['fixture'],rights='Owned')) - self.assertEqual(status,503,failure) + # The database working copy stays the session's authority; Save refuses + # to overwrite the externally checked-out folder. + self.assertEqual(self.state()['state'],'open') + status,failure=self.transport('save',self.arguments('save')) + self.assertEqual(status,409,failure);self.assertEqual(failure['code'],'conflict') + self.assertEqual(git('rev-parse','HEAD').decode().strip(),old) + self.assertEqual(git('status','--porcelain').decode(),'') + # Reopening a clean working copy imports the checked-out revision. + self.action('close') self.action('open',root='fixture',directory='alpha') self.assertEqual([r['id'] for r in self.call('data_query')['items']],[first['id']]) self.action('close');shutil.move(root,self.external/'moved') diff --git a/tests/test_project_runtime_http.py b/tests/test_project_runtime_http.py index 844ba5da..eba12786 100644 --- a/tests/test_project_runtime_http.py +++ b/tests/test_project_runtime_http.py @@ -65,6 +65,8 @@ def test_configured_alias_actual_capture_and_cold_historical_status_never_execut self.assertTrue(any(r['operation']=='runtime.capture' for r in receipts)) status,replay=self.transport('runtime_capture',arguments);self.assertEqual(status,200,replay) self.assertEqual(replay['value'],captured) + # Another workspace sees the capture only after it is saved. + self.action('save') self.action('close') self.select_host(self.path/'cold-control',{'fixture':self.external}) self.action('open',root='fixture',directory='alpha') diff --git a/tests/test_project_runtime_recovery_http.py b/tests/test_project_runtime_recovery_http.py index f22b0b8e..6817d148 100644 --- a/tests/test_project_runtime_recovery_http.py +++ b/tests/test_project_runtime_recovery_http.py @@ -158,7 +158,7 @@ def test_actual_partial_copy_abandon_needs_no_remaining_source_and_retains_bindi self.assertEqual(self.call('runtime_status',capture_key=key)['state'],'abandoned') self.action('save');self.action('close') - def test_final_sql_failure_retry_requires_exact_public_view_and_never_recopies(self): + def test_final_sql_failure_rolls_back_and_an_exact_retry_completes_without_recopying(self): projection=self.workspace._projection;db=projection.db;key='lost-final-sql' db.execute('CREATE TABLE runtime_ack_parent(id INTEGER PRIMARY KEY)') db.execute('CREATE TABLE runtime_ack_child(id INTEGER REFERENCES runtime_ack_parent(id) DEFERRABLE INITIALLY DEFERRED)') @@ -170,20 +170,23 @@ def bad(): return mutate(operation,target,body,checkpoint,request_key,bad,capture,**kwargs) with patch.object(projection.session,'mutate',side_effect=fault): status,error=self.transport('runtime_capture',self.capture_arguments(key)) - self.assertEqual(status,503,error);self.assertFalse(projection.session.available) - state=self.state();self.assertEqual(state['state'],'recovery_required') - arguments=self.capture_arguments(key);before=projection.session.snapshot - for change in (dict(binding='f'*32),dict(expected_checkpoint='f'*64)): - with patch.object(projection.runtime,'recover_capture',side_effect=AssertionError('Rejected public view reached recovery')): - status,error=self.transport('runtime_capture',dict(arguments,**change)) - self.assertEqual(status,409,error);self.assertEqual(error['code'],'stale_project') - self.assertEqual(projection.session.snapshot,before);self.assertTrue(projection.custody.has_pending) - with patch.object(projection.runtime.owner,'create',side_effect=AssertionError('Do not recopy committed runtime')): + # The final records and projection rows rolled back together; only the + # previously committed intent remains (its receipt is reported). + self.assertEqual(status,409,error);self.assertEqual(error['code'],'rejected') + self.assertTrue(error['receipt']['committed']) + self.assertTrue(projection.session.available) + self.assertEqual(self.state()['state'],'open') + self.assertNotIn(key,projection.session.operations()) + arguments=self.capture_arguments(key) + status,error=self.transport('runtime_capture',dict(arguments,binding='f'*32)) + self.assertEqual(status,409,error);self.assertEqual(error['code'],'stale_project') + with patch.object(projection.runtime.owner,'create',side_effect=AssertionError('Do not recopy the copied runtime')): status,recovered=self.transport('runtime_capture',arguments) - self.assertEqual(status,200,recovered);self.assertEqual(recovered['binding'],state['binding']) + self.assertEqual(status,200,recovered) self.assertEqual(self.state()['state'],'open');self.assertTrue(projection.session.available) - self.assertFalse(projection.custody.has_pending) self.assertEqual(self.call('runtime_status',capture_key=key)['state'],'captured') + receipts=[r for r in projection.session.operations().values() if r['operation']=='runtime.capture'] + self.assertEqual(len(receipts),1) self.action('close') def test_original_physical_cache_recovery_and_foreign_intent_cannot_regenerate(self): @@ -195,6 +198,9 @@ def after_copy(*args,**kwargs): status,error=self.transport('runtime_capture',self.capture_arguments(key)) self.assertEqual(status,503,error);physical=projection.db.projection_id self.assertTrue((projection.runtime.owner.path/self.inventory['inventory_sha256']).is_dir()) + # The committed intent was saved to the folder before it was copied + # (the workspace itself refuses Save while recovery is required). + original._store.save(original._store.summary()['checkpoint']) shutil.copytree(self.external/'alpha',self.external/'clone',ignore=shutil.ignore_patterns('.tuldok-local')) self.start_application(self.path/'foreign-control',sources={'inert-fixture':self.config}) self.action('open',root='fixture',directory='clone') diff --git a/tests/test_project_working_store.py b/tests/test_project_working_store.py new file mode 100644 index 00000000..3c2fe2f4 --- /dev/null +++ b/tests/test_project_working_store.py @@ -0,0 +1,212 @@ +"""Database working copy of an external project: import, commit, save.""" +import copy +import os +from pathlib import Path +import shutil +import tempfile +import unittest +import uuid +from unittest.mock import patch + +import project_store as store_module +from project_store import ProjectStore, ProjectStoreError +from project_working_store import WorkingProjectStore + + +def note(identity, text): + return dict(family='workbench_text', id=identity, + document=dict(format='workbench_text_v1', text=text, history=[])) + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class WorkingProjectStoreTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-working-store-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.folder_path = self.root / 'project' + store = ProjectStore.create(self.folder_path, 'Working fixture', 'new', 'a' * 64) + self.records = [note('one', 'Alpha ☃'), note('two', 'Beta "quoted"\nline'), note('three', 'Gamma')] + self.initial = store.commit(self.records, store.summary()['checkpoint'], 'seed', 'b' * 64) + store.close() + self.working_id = uuid.uuid4().hex + self.addCleanup(self.drop_schema) + self.opened = [] + self.addCleanup(lambda: [store.close() for store in self.opened]) + + def drop_schema(self): + import psycopg + from psycopg import sql + from metadata import connection_info + with psycopg.connect(connection_info(None), autocommit=True) as connection: + connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format( + sql.Identifier('working_' + self.working_id))) + + def folder(self): + parent, leaf = os.path.split(str(self.folder_path)) + descriptor = os.open(parent, store_module.DIR_FLAGS) + try: + return ProjectStore.open_folder_at(descriptor, leaf, root_path=parent) + finally: + os.close(descriptor) + + def open(self): + store = WorkingProjectStore.open(self.folder(), self.working_id) + self.opened.append(store) + return store + + def reopen(self, store): + store.close() + return self.open() + + def file_snapshot(self): + with ProjectStore.open(self.folder_path) as store: + return store.snapshot() + + def test_import_matches_the_file_store_exactly(self): + store = self.open() + self.assertEqual(store.snapshot(), self.initial) + self.assertEqual(store.summary(), {k: self.initial[k] for k in ('project_id', 'name', 'checkpoint')}) + self.assertFalse(store.dirty()) + self.assertEqual(store.records('workbench_text'), self.initial['records']) + self.assertEqual(store.record('workbench_text', 'two'), self.initial['records'][2]) + self.assertIsNone(store.record('workbench_text', 'absent')) + + def test_unchanged_folder_reattaches_without_reading_records(self): + store = self.open() + store.close() + with patch.object(ProjectStore, 'folder_record', side_effect=AssertionError('Record file read')): + again = self.open() + self.assertEqual(again.snapshot(), self.initial) + + def test_commit_stays_in_database_and_matches_file_store_identity(self): + store = self.open() + before_files = sorted(p.relative_to(self.folder_path) for p in self.folder_path.rglob('*')) + before_checkpoint = (self.folder_path / '.tuldok/project.json').read_bytes() + changed = copy.deepcopy(self.records) + changed[1]['document']['text'] = 'Edited in Tuldok' + changed.append(note('four', 'Delta')) + committed = store.commit(changed, self.initial['checkpoint'], 'edit', 'c' * 64) + self.assertTrue(store.dirty()) + self.assertEqual(sorted(p.relative_to(self.folder_path) for p in self.folder_path.rglob('*')), before_files) + self.assertEqual((self.folder_path / '.tuldok/project.json').read_bytes(), before_checkpoint) + # The same commit on a file store yields the identical checkpoint. + mirror = self.root / 'mirror' + shutil.copytree(self.folder_path, mirror) + with ProjectStore.open(mirror) as file_store: + expected = file_store.commit(copy.deepcopy(changed), self.initial['checkpoint'], 'edit', 'c' * 64) + self.assertEqual(committed['checkpoint'], expected['checkpoint']) + self.assertEqual(store.snapshot(), expected) + reopened = self.reopen(store) + self.assertEqual(reopened.snapshot(), expected, 'Unsaved work persists across sessions') + self.assertTrue(reopened.dirty()) + + def test_changed_hint_and_exact_comparison_agree(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Hinted change' + hinted = store.commit(changed, self.initial['checkpoint'], 'hinted', 'd' * 64, + changed={('workbench_text', 'one')}) + store.close() + other = WorkingProjectStore.open(self.folder(), uuid.uuid4().hex) + self.addCleanup(lambda: self._drop(other)) + exact = other.commit(copy.deepcopy(changed), self.initial['checkpoint'], 'hinted', 'd' * 64) + self.assertEqual(hinted['checkpoint'], exact['checkpoint']) + + def _drop(self, store): + schema = store.schema + store.close() + import psycopg + from psycopg import sql + from metadata import connection_info + with psycopg.connect(connection_info(None), autocommit=True) as connection: + connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(schema))) + + def test_save_exports_byte_identical_files(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[2]['document']['text'] = 'Saved change' + committed = store.commit(changed, self.initial['checkpoint'], 'edit', 'e' * 64) + store.save(committed['checkpoint']) + self.assertFalse(store.dirty()) + saved = store.snapshot() + store.close() + self.assertEqual(self.file_snapshot(), saved) + with patch.object(ProjectStore, 'folder_record', side_effect=AssertionError('Re-import after own save')): + self.assertEqual(self.open().snapshot(), saved) + + def test_clean_working_copy_imports_only_changed_external_records(self): + store = self.open() + store.close() + with ProjectStore.open(self.folder_path) as file_store: + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Changed outside Tuldok' + external = file_store.commit(changed, self.initial['checkpoint'], 'outside', 'f' * 64) + reads = [] + original = ProjectStore.folder_record + def observe(folder, entry): + reads.append(entry['id']) + return original(folder, entry) + with patch.object(ProjectStore, 'folder_record', observe): + store = self.open() + self.assertEqual(reads, ['one']) + self.assertEqual(store.snapshot(), external) + + def test_external_change_with_unsaved_work_is_an_explicit_conflict(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Unsaved' + committed = store.commit(changed, self.initial['checkpoint'], 'unsaved', '1' * 64) + store.close() + with ProjectStore.open(self.folder_path) as file_store: + other = copy.deepcopy(self.records) + other[1]['document']['text'] = 'Outside' + file_store.commit(other, self.initial['checkpoint'], 'outside', '2' * 64) + with self.assertRaises(ProjectStoreError) as caught: + self.open() + self.assertEqual(caught.exception.code, 'conflict') + + def test_save_refuses_to_overwrite_an_external_change(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Unsaved' + committed = store.commit(changed, self.initial['checkpoint'], 'unsaved', '3' * 64) + external = (self.folder_path / '.tuldok/project.json') + before = external.read_bytes() + # Simulate a checkout replacing project.json while Tuldok holds the folder. + raw = before.replace(b'"Working fixture"', b'"Renamed outside"') + external.chmod(0o600) + external.write_bytes(raw) + with self.assertRaises(ProjectStoreError) as caught: + store.save(committed['checkpoint']) + self.assertEqual(caught.exception.code, 'conflict') + self.assertEqual(external.read_bytes(), raw) + + def test_second_session_on_the_same_working_copy_is_busy(self): + self.open() + with self.assertRaises(ProjectStoreError) as caught: + WorkingProjectStore(self.folder(), self.working_id) + self.assertEqual(caught.exception.code, 'busy') + + def test_receipt_replay_and_stale_checkpoint(self): + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Once' + first = store.commit(changed, self.initial['checkpoint'], 'once', '4' * 64) + self.assertEqual(store.commit(copy.deepcopy(changed), first['checkpoint'], 'once', '4' * 64)['checkpoint'], + first['checkpoint']) + self.assertIsNotNone(store.reconcile('once', '4' * 64)) + with self.assertRaises(ProjectStoreError) as caught: + store.commit(changed, self.initial['checkpoint'], 'later', '5' * 64) + self.assertEqual(caught.exception.code, 'stale') + + def test_verify_records_detects_working_copy_corruption(self): + store = self.open() + store.verify_records() + store._execute(store._q("UPDATE {records} SET raw = convert_to(replace(convert_from(raw, 'UTF8'), 'Gamma', 'Omega'), 'UTF8') WHERE id='three'")) + with self.assertRaises(ProjectStoreError): + store.verify_records() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_workspace.py b/tests/test_project_workspace.py index a78982f4..26d3f662 100644 --- a/tests/test_project_workspace.py +++ b/tests/test_project_workspace.py @@ -76,8 +76,8 @@ def test_status_returns_current_metadata_without_copying_the_record_graph(self): self.create() self.import_record('A retained record unrelated to the project summary') expected = self.state() - with patch.object(self.workspace._store, '_snapshot', - wraps=self.workspace._store._snapshot) as snapshots: + with patch.object(self.workspace._store, 'snapshot', + wraps=self.workspace._store.snapshot) as snapshots: observed = self.state() self.assertEqual(observed, expected) self.assertEqual(snapshots.call_count, 0, @@ -126,20 +126,25 @@ def assert_code(self, code, action, arguments): return caught.exception def unknown_import(self): + """Commit an edit whose PostgreSQL COMMIT acknowledgement is lost. + + Records and projection rows commit together in the database working + copy, so the only uncertain outcome is a lost commit acknowledgement. + """ + from contextlib import contextmanager + from metadata import CommitOutcomeUnknown before = self.state() arguments = self.mutation(before, asset=dict(text='Uncertain publication', groups=['owned'])) - replace, fsync = project_store.os.replace, project_store.os.fsync - published = False - def after_replace(*args, **kwargs): - nonlocal published - replace(*args, **kwargs) - published = True - def fail_directory(fd): - if published and fd == self.workspace._store._meta_fd: - raise OSError(errno.EIO, 'actual checkpoint directory fsync failure') - return fsync(fd) - with patch.object(project_store.os, 'replace', side_effect=after_replace), patch.object( - project_store.os, 'fsync', side_effect=fail_directory): + db = self.workspace._projection.db + real = db.transaction + @contextmanager + def lost_acknowledgement(*args, **kwargs): + outermost = not db._rollback + with real(*args, **kwargs) as value: + yield value + if outermost: + raise CommitOutcomeUnknown('fixture lost COMMIT acknowledgement') + with patch.object(db, 'transaction', lost_acknowledgement): self.assert_code('unknown', 'import_text', arguments) return before, arguments @@ -167,20 +172,15 @@ def test_unknown_fsync_status_and_explicit_current_open_reconcile_canonical_edit with self.owner.bound(self.workspace): self.assertEqual(len(self.workspace._text.records()['items']), 1) - def test_unknown_recovery_does_not_succeed_until_directory_sync_failure_clears(self): + def test_unknown_recovery_does_not_succeed_until_the_working_copy_is_available(self): self.create() _before, edit = self.unknown_import() recovery = self.state() old_projection = self.workspace._projection - metadata = (self.external / 'alpha/.tuldok').stat() - fsync = project_store.os.fsync - def fail_metadata_sync(fd): - info = os.fstat(fd) - if (info.st_dev, info.st_ino) == (metadata.st_dev, metadata.st_ino): - raise OSError(errno.EIO, 'persistent checkpoint directory fsync failure') - return fsync(fd) reopen = self.mutation(recovery, root='fixture', directory='alpha') - with patch.object(project_store.os, 'fsync', side_effect=fail_metadata_sync): + from project_working_store import WorkingProjectStore + with patch.object(WorkingProjectStore, 'open', + side_effect=project_store.ProjectStoreError('Local PostgreSQL is unavailable.', 'unavailable', 503)): self.assert_code('unavailable', 'open', reopen) self.assertEqual(self.state(), recovery) self.assertIs(self.workspace._projection, old_projection) @@ -208,7 +208,7 @@ def fail_published_parent(fd): self.assertEqual(selected['project']['project_id'], expected['project_id']) self.assertEqual(selected['project']['checkpoint'], expected['checkpoint']) - def test_real_sql_commit_failure_reports_durable_receipt_and_explicit_open_restores(self): + def test_rejected_sql_commit_rolls_back_records_and_projection_together(self): self.create() before = self.state() projection = self.workspace._projection @@ -216,22 +216,17 @@ def test_real_sql_commit_failure_reports_durable_receipt_and_explicit_open_resto BEGIN RAISE EXCEPTION 'fixture deferred rejection' USING ERRCODE='23514'; END $$""") projection.execute("""CREATE CONSTRAINT TRIGGER fixture_workspace_commit_failure AFTER INSERT ON workbench_records DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION fixture_workspace_refuse_commit()""") - arguments = self.mutation(before, asset=dict(text='Durable despite rejected SQL commit', groups=['owned'])) - failure = self.assert_code('projection_unavailable', 'import_text', arguments) - canonical = self.workspace._store.snapshot() - self.assertEqual(failure.receipt, dict(committed=True, selected=True, project_id=canonical['project_id'], - checkpoint=canonical['checkpoint'], request_key=arguments['request_key'])) + arguments = self.mutation(before, asset=dict(text='Rejected with its SQL commit', groups=['owned'])) + with self.assertRaises(Exception): + self.call('import_text', arguments) + # Nothing was committed: same checkpoint, no records, still open. + self.assertEqual(self.state(), before) + self.assertEqual(self.workspace._store.summary()['checkpoint'], before['project']['checkpoint']) self.assertEqual(projection.execute('SELECT count(*) FROM workbench_records').fetchone()[0], 0) - recovery = self.state() - self.assertEqual(recovery['state'], 'recovery_required') - self.assertEqual(recovery['project'], before['project']) - self.assertEqual(recovery['recovery']['reason'], 'projection_unavailable') - opened = self.call('open', self.mutation(recovery, root='fixture', directory='alpha')) - self.assertEqual(opened['project']['checkpoint'], canonical['checkpoint']) - result = self.call('import_text', dict(arguments, binding=opened['binding'])) - self.assertEqual(result['record']['text'], 'Durable despite rejected SQL commit') - self.assertEqual(self.workspace._projection.execute('SELECT count(*) FROM workbench_records').fetchone()[0], 1) - self.assert_code('stale_project', 'import_text', arguments) + projection.execute('DROP TRIGGER fixture_workspace_commit_failure ON workbench_records') + result = self.call('import_text', arguments) + self.assertEqual(result['record']['text'], 'Rejected with its SQL commit') + self.assertEqual(projection.execute('SELECT count(*) FROM workbench_records').fetchone()[0], 1) def test_failed_reconstruction_keeps_recovery_generation_and_old_projection(self): self.create() From fec4a4ddfccc86d226b2b1aab111b464478312e2 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 10:38:46 -0700 Subject: [PATCH 04/13] perf(projects): read records from the working copy instead of holding them - Sessions over a database working copy bind checkpoint metadata only and hold no record set; records/record/records_for/active/operations read the selected families from PostgreSQL (records_view returns every identity with documents only for the selected families, as fork validators require). Validation is skipped entirely when the checkpoint is already validated. - Mutations pass the working store the exact changed identities, so a commit encodes only the captured upserts and the operation receipt. - The working store joins the projection connection only inside its open transaction; other commits use its own connection and never fire projection commit hooks. - Study authoring restore collects only Study/Trial documents for a validated checkpoint (equivalence covered by the history assembly test). Co-Authored-By: Claude Opus 5.5 --- project_session.py | 97 +++++++++++++++++++++++++++++++---- project_study_execution.py | 18 ++++++- project_working_store.py | 46 +++++++++++++---- tests/test_project_history.py | 5 ++ 4 files changed, 145 insertions(+), 21 deletions(-) diff --git a/project_session.py b/project_session.py index 2d339289..a610d5d2 100644 --- a/project_session.py +++ b/project_session.py @@ -79,6 +79,9 @@ def operation_records(records, *, project_id=None): class ProjectSession: + # Sessions over a database working copy hold no record set (see __init__). + _lazy = False + def __init__(self, store, db, access, artifacts, *, validate_records, validation_cache=None): if not callable(validate_records): raise TypeError('A complete owner-codec validator is required.') @@ -98,7 +101,45 @@ def __init__(self, store, db, access, artifacts, *, validate_records, validation self._pending_observation = None self._recovery_records = None self._recovery_request = None - self.adopt(store.snapshot(), _owned=True) + # A database working copy serves record reads directly; the session + # then binds checkpoint metadata only and holds no record set. + self._lazy = callable(getattr(store, 'records_view', None)) + if self._lazy: + self._adopt_working_copy() + else: + self.adopt(store.snapshot(), _owned=True) + + def _adopt_working_copy(self): + """Bind the working copy's checkpoint; validate its records only if unknown.""" + with self.access.lock, self.lock, self.db.lock: + summary = self.store.summary() + cache = getattr(self, 'validation_cache', None) + if cache is not None and cache.known(self.project_id, summary['checkpoint']): + self.validated_checkpoint = summary['checkpoint'] + else: + snapshot = self.store.snapshot() + if snapshot['checkpoint'] != summary['checkpoint']: + raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) + self._validated(snapshot['records'], snapshot['checkpoint']) + del snapshot + if self.store.summary()['checkpoint'] != summary['checkpoint']: + raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) + self._bind(summary) + self.available = True + + def _bind(self, snapshot): + """Bind a committed checkpoint; a lazy session keeps only its metadata.""" + if self._lazy: + self._snapshot = dict(project_id=snapshot['project_id'], name=snapshot['name'], + checkpoint=snapshot['checkpoint'], records=None) + else: + self._snapshot = snapshot + + def _fresh_store(self): + """The working copy must still be at the bound checkpoint.""" + if self.store.summary()['checkpoint'] != self._snapshot['checkpoint']: + raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) + return self.store @property def foreground_publication(self): @@ -141,7 +182,12 @@ def _publish_observation(self, records, key, digest, *, authorize): if self.publication_intent is not None: self.publication_intent(copy.deepcopy(current), copy.deepcopy(candidate), key, digest) try: - self._pending_observation = self.store.commit(candidate, current['checkpoint'], key, digest) + if self._lazy: + changed = {(item['family'], item['id']) for item in records} + self._pending_observation = self.store.commit(candidate, current['checkpoint'], key, digest, + changed=changed) + else: + self._pending_observation = self.store.commit(candidate, current['checkpoint'], key, digest) except BaseException as error: if getattr(error, 'code', None) in ('unknown', 'outcome_unknown'): self.available = False @@ -165,7 +211,7 @@ def _observation_outcome(self, outcome): self.available = False return if pending is not None: - self._snapshot = pending + self._bind(pending) if outcome != 'committed': self.available = False if outcome == 'unknown': @@ -183,7 +229,10 @@ def reconcile_selection(self, snapshot): all ordered guards until fixed inert reconstruction succeeds or fences. """ with self.access.lock, self.lock, self.db.lock: - known = snapshot == self._snapshot + if self._lazy: + known = all(snapshot[key] == self._snapshot[key] for key in ('project_id', 'name', 'checkpoint')) + else: + known = snapshot == self._snapshot attempted = (self._recovery_records is not None and snapshot['project_id'] == self._snapshot['project_id'] and snapshot['name'] == self._snapshot['name'] @@ -205,6 +254,12 @@ def summary(self): @property def snapshot(self): with self.lock: + if self._lazy: + # Complete independent record set from the working copy (costly). + current = self._fresh_store().snapshot() + if current['checkpoint'] != self._snapshot['checkpoint']: + raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) + return current return copy_validated_json(self._snapshot) def records(self, *families): @@ -217,6 +272,8 @@ def records(self, *families): raise TypeError('Select one or more record families by name.') selected = frozenset(families) with self.lock: + if self._lazy: + return self._fresh_store().records(*families) return copy_validated_json([item for item in self._snapshot['records'] if item['family'] in selected]) @@ -233,6 +290,9 @@ def records_for(self, *families): raise TypeError('Select one or more record families by name.') selected = frozenset(families) with self.lock: + if self._lazy: + # Unselected envelopes carry document None (identity only). + return self._fresh_store().records_view(*families) return [copy_validated_json(item) if item['family'] in selected else dict(family=item['family'], id=item['id'], document=item['document']) for item in self._snapshot['records']] @@ -247,6 +307,10 @@ def active(self, *families, project_id=None): raise TypeError('Select one or more record families by name.') from project_fork import active_records with self.lock: + if self._lazy: + return active_records(self._fresh_store().records_view(*families, 'project_fork'), + project_id=self.project_id if project_id is None else project_id, + families=frozenset(families)) return active_records(self._snapshot['records'], project_id=self.project_id if project_id is None else project_id, families=frozenset(families)) @@ -277,6 +341,8 @@ def record(self, family, record_id): if type(family) is not str or type(record_id) is not str: raise TypeError('Select a record by family and ID.') with self.lock: + if self._lazy: + return self._fresh_store().record(family, record_id) index = getattr(self, '_record_index', None) if index is None or index[0] != self._snapshot['checkpoint']: index = (self._snapshot['checkpoint'], @@ -297,7 +363,9 @@ def operations(self, *, project_id=None): cached = getattr(self, '_operations_cache', None) if cached is None or cached[0] != key: # operation_records copies what it returns and never mutates input. - cached = (key, operation_records(self._snapshot['records'], project_id=project_id)) + records = (self._fresh_store().records_view(OPERATION_FAMILY, 'project_fork') if self._lazy + else self._snapshot['records']) + cached = (key, operation_records(records, project_id=project_id)) self._operations_cache = cached return copy.deepcopy(cached[1]) @@ -339,7 +407,7 @@ def adopt(self, snapshot, *, _owned=False): self._validated(snapshot['records'], snapshot['checkpoint']) if self.store.summary()['checkpoint'] != snapshot['checkpoint']: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._snapshot = snapshot + self._bind(snapshot) self.available = True return self._validated(snapshot['records']) @@ -347,7 +415,7 @@ def adopt(self, snapshot, *, _owned=False): if current != snapshot: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) self._remember_validated(snapshot['checkpoint']) - self._snapshot = copy_validated_json(snapshot) + self._bind(copy_validated_json(snapshot) if not self._lazy else snapshot) self.available = True def _checked(self, read): @@ -382,6 +450,9 @@ def check_records(self, family): if type(family) is not str: raise TypeError('Record-family selection requires a string.') current = self._checked(self.store.summary) + if self._lazy: + current['records'] = self.store.records(family) + return current current['records'] = copy.deepcopy([ item for item in self._snapshot['records'] if item['family'] == family]) return current @@ -448,11 +519,16 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt attempted_records = records if self.publication_intent is not None: self.publication_intent(copy.deepcopy(current), copy.deepcopy(records), key, digest) - committed = self.store.commit(records, current['checkpoint'], key, digest) + if self._lazy: + # Only the captured upserts and this receipt can differ. + changed = {(item['family'], item['id']) for item in upserts} | {identity} + committed = self.store.commit(records, current['checkpoint'], key, digest, changed=changed) + else: + committed = self.store.commit(records, current['checkpoint'], key, digest) # Outcome hooks see canonical selection before SQL outcome # and before any deferred dispatch/wake can run. self._pending_observation = committed - self._snapshot = committed + self._bind(committed) # These exact records were completely validated as the candidate. self._remember_validated(committed['checkpoint']) return copy.deepcopy(captured['result']) @@ -471,7 +547,8 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt raise ProjectSessionError('PostgreSQL rejected this edit; nothing was saved.', 'rejected', 409) from error if committed is not None: - self._snapshot, self.available = committed, False + self._bind(committed) + self.available = False failure = ProjectSessionError('Canonical operation committed; reopen to reconstruct the projection.', 'projection_unavailable', 503) failure.receipt = dict(committed=True, selected=True, project_id=committed['project_id'], diff --git a/project_study_execution.py b/project_study_execution.py index c6a3494e..af7ed930 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -19,7 +19,7 @@ from project_material import ProjectMaterial, MaterialSnapshot from project_session import operation_records from workbench import WorkbenchError, encode, timestamp -from project_session import session_operations +from project_session import session_operations, session_records PREPARATIONS={'study_execution.prepare_package':'studies.prepare_package', 'study_execution.prepare_verifier':'studies.prepare_verifier'} @@ -330,10 +330,24 @@ def _canonical(self,records=None): records=self.session.snapshot['records'] if records is None else records return history_records(records,materials=self.materials.read_view(records)) + def _authoring_facts(self,records=None): + """Studies and Trials of the bound checkpoint for SQL authoring. + + For a checkpoint whose complete validation is on record, only the Study + and Trial documents are needed and collected (no Run history); this is + exactly the studies/trials part of the complete decode. + """ + if records is None: + validated=getattr(self.session,'validated_checkpoint',None) + if validated is not None and validated==self.session.summary['checkpoint']: + from project_history import assemble_validated_facts + return assemble_validated_facts(session_records(self.session,'study_history','trial_history'),owned=True) + return self._canonical(records) + def restore_authoring(self,records=None): """Reconstruct intent only, under the current project guard.""" with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check_current();facts=self._canonical(records) + self.session.check_current();facts=self._authoring_facts(records) with self.db.transaction(): for (origin,identity),doc in sorted(facts.studies.items()): if origin!=self.origin:continue diff --git a/project_working_store.py b/project_working_store.py index 636050fa..4e36488b 100644 --- a/project_working_store.py +++ b/project_working_store.py @@ -82,23 +82,29 @@ def bind(self, db): with self._mutex: self._db = db + def _joined(self): + # Join the projection only inside its open transaction (Session.mutate, + # background publication), so record and projection rows commit + # atomically. Other commits use this store's own connection and never + # fire the projection's commit hooks. + return self._db is not None and self._db.in_transaction + def _connection(self): - return self._db.connection if self._db is not None else self._own + return self._db.connection if self._joined() else self._own def _guard(self): # A bound projection connection is shared; its catalog lock serializes # every command and transaction on it across threads. - return self._db.lock if self._db is not None else nullcontext() + return self._db.lock if self._joined() else nullcontext() @contextmanager def _transaction(self): - with self._guard(): - if self._db is not None: - with self._db.transaction(): - yield - else: - with self._own.transaction(): - yield + if self._joined(): + with self._db.lock, self._db.transaction(): + yield + else: + with self._own.transaction(): + yield def _execute(self, statement, parameters=None): if self._closed: @@ -276,6 +282,28 @@ def records(self, *families): return [dict(family=f, id=i, document=json.loads(raw)) for f, i, raw in self._rows( self._q('SELECT family, id, raw FROM {records} WHERE family = ANY(%s) {order}'), (list(families),))] + def records_view(self, *families): + """Every record identity in order; documents decoded only for selected families. + + Unselected envelopes carry document None. Fork and membership + validators need every identity but only read their own families. + """ + if not families or any(type(family) is not str for family in families): + raise TypeError('Select one or more record families by name.') + with self._mutex, self._guard(): + connection = self._connection() + idle = connection.info.transaction_status == psycopg.pq.TransactionStatus.IDLE + result = [] + with (connection.transaction() if idle else nullcontext()), \ + connection.cursor(name='working_' + uuid.uuid4().hex) as cursor: + cursor.itersize = 256 + cursor.execute(self._q('SELECT family, id, CASE WHEN family = ANY(%s) THEN raw END ' + 'FROM {records} {order}'), (list(families),)) + for family, record_id, raw in cursor: + result.append(dict(family=family, id=record_id, + document=None if raw is None else json.loads(bytes(raw)))) + return result + def record(self, family, record_id): with self._mutex: row = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 8051eb75..ba632ad8 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -188,6 +188,11 @@ def test_validated_fact_assembly_equals_complete_decode(self): for field in ('studies','trials','runs','candidates','policies','reservations','observations'): self.assertEqual(getattr(assembled,field),getattr(decoded,field),field) self.assertEqual(list(getattr(assembled,field)),list(getattr(decoded,field)),field) + # Study authoring restore collects only Study and Trial documents. + authoring=assemble_validated_facts([r for r in copy.deepcopy(records) + if r['family'] in ('study_history','trial_history')],owned=True) + for field in ('studies','trials','policies','reservations'): + self.assertEqual(getattr(authoring,field),getattr(decoded,field),field) def test_history_read_uses_one_material_view_of_the_selected_records(self): identity='9'*32 From 82dcba1ec7bc73674f8d7aa5763ea261e0a1ff96 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 10:50:53 -0700 Subject: [PATCH 05/13] perf(projects): never materialize the record set for no-op commit hooks Execution's pre-commit capture loads records only when a capture owner has dirty state; producer outcome and producer restore read their own families, declarations and forks instead of a complete snapshot. Co-Authored-By: Claude Opus 5.5 --- project_execution.py | 15 +++++++++++++++ project_producer_execution.py | 3 ++- project_projection.py | 5 ++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/project_execution.py b/project_execution.py index 478915d2..9f7a5878 100644 --- a/project_execution.py +++ b/project_execution.py @@ -528,11 +528,26 @@ def capture_dirty(self, before_records): merged[identity] = record return list(merged.values()) + def _capture_pending(self): + """Whether capture_dirty could return any record (see its owners).""" + if self._dirty_runs or self._pending_studies or any(self._pending_producers.values()) or self.native.has_dirty: + return True + study = self.study_execution + if study is not None and (study._dirty or study._package_records): + return True + producers = self.producer_execution + if producers is not None and (producers._runtime_data is not None or any(producers._dirty.values())): + return True + return False + def before_commit(self): if self._reconstructing or self.session.foreground_publication: return if not self._dirty_runs and not self._pending_studies and not any(self._pending_producers.values()) and not self.native.has_dirty and self.study_execution is None and self.producer_execution is None: return + if not self._capture_pending(): + # Nothing can be captured; never materialize the record set. + return snapshot = self.session.snapshot records = self.capture_dirty(snapshot['records']) if records: diff --git a/project_producer_execution.py b/project_producer_execution.py index 8b37faf4..c8833909 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -211,7 +211,8 @@ def outcome(self, outcome): # Unknown acknowledgement fences authority. It never authorizes an # unacknowledged cancellation signal to a current worker. if outcome == 'committed': - records = self.session.snapshot['records'] + from project_session import session_records + records = session_records(self.session, *FAMILIES, 'project_artifact') self.archive._records = copy.deepcopy([r for r in records if r['family'] in FAMILIES]) declarations = _declarations(records) self.archive._artifacts = {sha:declarations[sha] for sha in _referenced_artifacts(self.archive._records)} diff --git a/project_projection.py b/project_projection.py index 611fff49..f121cb26 100644 --- a/project_projection.py +++ b/project_projection.py @@ -19,6 +19,7 @@ from project_material import validate_operation_result as validate_material_result from project_producers import ProjectProducers, FAMILIES as PRODUCER_FAMILIES from project_producers import decode_records as decode_producer_records +from project_session import session_records_for from project_session import ProjectSession, operation_documents, operation_records from project_locks import ProjectLock from project_execution import ProjectExecution, validate_operation_result as validate_execution_result @@ -142,7 +143,9 @@ def __init__(self, store, db, access, roots, *, custody=None, restore=True, runt self.history = ProjectHistory(self.session, materials=self.materials, authorize_disclosure=self.authorize_disclosure) if restore: - self.producers.restore(self.session.snapshot) + # Producer restore reads its families, declarations and forks. + self.producers.restore(dict(records=session_records_for( + self.session, *PRODUCER_FAMILIES, 'project_artifact', 'project_fork'))) self.data.restore() self.materials.restore() # These owners create only disposable local directories. Their From 410f35ba5452b86d8e479bf9902289844a9e7397 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 11:07:24 -0700 Subject: [PATCH 06/13] perf(history): load Run history documents on demand from the working copy For a validated checkpoint in a database working copy, history facts decode Studies, Trials, observations and candidates, and keep Run documents as identities loaded on first access (checked equal to the complete decode on the 460-Run portfolio project). Co-Authored-By: Claude Opus 5.5 --- project_history.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/project_history.py b/project_history.py index 521bc826..e3b1c7c7 100644 --- a/project_history.py +++ b/project_history.py @@ -5,6 +5,7 @@ SQL connection is only a query engine for bounded supplied immutable JSON. """ import base64 +from collections.abc import Mapping import copy import hashlib import re @@ -25,7 +26,7 @@ from runs import (ACTIVE, TERMINAL, TEST_FILES, MAX_ARTIFACT_BYTES, validate_retained_run, project_run_projection, public_run_projection, artifact_capture_source, validate_external_artifact_inventory) from workbench import WorkbenchError, timestamp -from project_session import session_derived, session_operations, session_records, session_records_for +from project_session import session_derived, session_operations, session_record, session_records, session_records_for FAMILY_FORMATS = { 'study_history': 'project_study_history_v1', @@ -598,6 +599,35 @@ def assemble_validated_facts(records, *, owned=False): return facts +class _LazyRuns(Mapping): + """Validated Run history documents by (origin, run_id), loaded on first use. + + Keys come from verified record identities (origin + '.' + run_id); each + document is read from the working copy once per checkpoint and shared like + every other fact (callers copy what they return). + """ + def __init__(self, identities, load): + self._identities = identities + self._load = load + self._documents = {} + + def __getitem__(self, key): + if key not in self._identities: + raise KeyError(key) + if key not in self._documents: + envelope = self._load(self._identities[key]) + if envelope is None: + _invalid('Historical Run is unavailable.', 'unavailable') + self._documents[key] = envelope['document'] + return self._documents[key] + + def __iter__(self): + return iter(self._identities) + + def __len__(self): + return len(self._identities) + + def checkpoint_facts(session, materials): """Decoded history of the session's bound checkpoint, shared by its owners. @@ -607,6 +637,19 @@ def checkpoint_facts(session, materials): """ def compute(): validated=getattr(session,'validated_checkpoint',None) + if validated is not None and validated==session.summary['checkpoint'] and getattr(session,'_lazy',False): + # A working copy serves Run documents on demand: decode everything + # except Run history now, and keep only Run identities. + selected=('study_history','trial_history','run_observation','candidate_history') + view=session_records_for(session,*selected) + facts=assemble_validated_facts([r for r in view if r['family'] in selected],owned=True) + identities={} + for envelope in view: + if envelope['family']=='run_history': + origin,run_id=envelope['id'].split('.',1) + identities[(origin,run_id)]=envelope['id'] + facts.runs=_LazyRuns(identities,lambda record_id:session_record(session,'run_history',record_id)) + return facts if validated is not None and validated==session.summary['checkpoint']: # records_for returns independent copies of exactly these families. return assemble_validated_facts(session_records_for(session,*FAMILY_FORMATS),owned=True) From 66c7d80517797aac215f57af6b8fdb39f156e24f Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 11:08:23 -0700 Subject: [PATCH 07/13] docs(plan): record working-store results and revised projection decision Co-Authored-By: Claude Opus 5.5 --- docs/plans/database-working-store.md | 44 ++++++++++++++++++---------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/docs/plans/database-working-store.md b/docs/plans/database-working-store.md index 69482e62..1ba0e80f 100644 --- a/docs/plans/database-working-store.md +++ b/docs/plans/database-working-store.md @@ -1,6 +1,6 @@ # Database working store for external projects -Status: in progress (2026-11-02). Owner decision recorded with the user. +Status: stages 1–2 implemented; stage 3 revised (2026-11-02). Owner decision recorded with the user. ## Decision @@ -54,21 +54,33 @@ the project can be tracked in Git. ## Stages -1. **Working store.** `WorkingProjectStore` with the ProjectStore interface - (`summary`, `snapshot`, `commit`, `save`, `reconcile`, `root_directory`, - `verify_records`, `recover_checkpoint`, `close`) plus family/record reads, - backed by tables in the project's persistent projection schema. Incremental - import; commit to the database; Save exports. Workspace keeps one persistent - projection per project folder and re-attaches it. -2. **No whole-project memory copy.** Session binds checkpoint metadata and reads - through the store; convert remaining `snapshot['records']` consumers to - family/record reads. Target: hundreds of MB, not GB. -3. **Projection reuse.** Skip SQL restore when the projection already reflects - the bound checkpoint. -4. **Incremental validation for commits.** Re-run only the owner validators - whose families a mutation touches. -5. **Smaller project files.** Store each captured runtime file listing once and - reference it from Runs (format migration with exact round trip). +1. **Working store — done.** `WorkingProjectStore` (project_working_store.py) + with the ProjectStore interface, incremental import, commits joined to the + projection's open SQL transaction, Save as the only folder write. Workspace + keeps a private folder→working-copy registry. +2. **No whole-project memory copy — done.** Sessions over a working copy bind + checkpoint metadata only; family/record/identity reads come from + PostgreSQL, commit hooks never materialize the record set, Study authoring + restores from Study/Trial documents and Run history loads per Run. +3. **Projection reuse — revised.** Reusing the SQL projection across sessions + would carry live execution state (runs, admissions, execution grants) into + a later session, which a cold open deliberately starts without. Instead the + projection is still reconstructed fresh, from cheap family-scoped reads. +4. **Incremental validation for commits — open.** Each commit still validates + the complete candidate record set (about 2.5 minutes on the portfolio). +5. **Smaller project files — open.** Store each captured runtime file listing + once and reference it from Runs (format migration with exact round trip). + +## Measured on the Rheon portfolio (54k records, 1.77 GB metadata) + +| | before | reopen now | +|---|---|---| +| Open | ~9 min (CPU) | ~11 s | +| Peak memory | 21.7 GB | 0.76 GB | +| First history/progression read | ~100 s | 0.5 s | + +First import into a new workspace takes ~50 s plus one complete validation +(~2.5 min) per checkpoint and validator version. Payload blobs (`artifacts/sha256/*`, 16 GB for the portfolio) remain content-addressed files under the project folder in stages 1–4. From 63e36285f13b7c8799f6f09d78fda397ef79086c Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 11:59:53 -0700 Subject: [PATCH 08/13] perf(validation): remove redundant encoding and copying from full validation Complete domain validation of the 1.77 GB portfolio took ~145 s per open or edit; it is now ~one third faster with byte-identical results: - Composition membership uses each stored revision's canonical envelope digest (recorded by the working copy, derived from its canonical bytes) for exactly those unchanged record objects, instead of re-encoding tens of thousands of envelopes; any other envelope is still encoded. - Plain-JSON copies use the C JSON round trip instead of deepcopy; the Run projection no longer copies values checked_json already copied. - Text validators scan for forbidden code points with a regex (fuzzed equal to the per-character checks); runtime manifests build each path once. - The working store takes only the projection's innermost catalog lock, then its own mutex, so it never inverts host/session lock order. Co-Authored-By: Claude Opus 5.5 --- model_definitions.py | 6 +- project_composition.py | 25 ++++++++ project_material.py | 4 +- project_session.py | 23 +++++++- project_store.py | 16 +++++ project_working_store.py | 91 ++++++++++++++++++++++------- projects.py | 6 +- runs.py | 3 +- tests/test_project_working_store.py | 19 ++++++ torch_runtime.py | 18 ++++-- 10 files changed, 176 insertions(+), 35 deletions(-) diff --git a/model_definitions.py b/model_definitions.py index c69c0281..ab861b52 100644 --- a/model_definitions.py +++ b/model_definitions.py @@ -33,9 +33,13 @@ def natural(value, name, minimum=0, maximum=MAX_INTEGER): return value +# Same code points as any(ord(c)<32 or ord(c)==127 or 0xD800<=ord(c)<=0xDFFF), scanned in C. +_TEXT_FORBIDDEN=re.compile('[\x00-\x1f\x7f\ud800-\udfff]') + + def text(value, name, maximum=120): if (type(value) is not str or not value.strip() or len(value)>maximum or - any(ord(c)<32 or ord(c)==127 or 0xD800<=ord(c)<=0xDFFF for c in value)): + _TEXT_FORBIDDEN.search(value)): raise WorkbenchError(name+' must be bounded nonempty text without controls.') return value diff --git a/project_composition.py b/project_composition.py index 53bc04f3..ee667167 100644 --- a/project_composition.py +++ b/project_composition.py @@ -7,6 +7,8 @@ The lifecycle owner must validate each source and the complete final candidate, clone primary edits and publish all required bytes through the native owners. """ +import contextvars +from contextlib import contextmanager import hashlib from project_fork import ARCHIVE_FAMILIES, _sha, _uuid, decode_fork @@ -23,7 +25,30 @@ def _fail(message, code='invalid', status=400): raise ProjectCompositionError(message, code, status) +_KNOWN_ENVELOPES = contextvars.ContextVar('known_envelope_digests', default=None) + + +@contextmanager +def known_envelope_digests(known): + """Supply verified envelope digests for exact record objects during validation. + + known maps id(record) -> (record, digest) for envelopes whose document is the + stored canonical revision. Only that same object uses its digest; any other + envelope (for example a constructed observation variant) is encoded. + """ + token = _KNOWN_ENVELOPES.set(known) + try: + yield + finally: + _KNOWN_ENVELOPES.reset(token) + + def _record_sha(record): + known = _KNOWN_ENVELOPES.get() + if known is not None: + entry = known.get(id(record)) + if entry is not None and entry[0] is record: + return entry[1] return hashlib.sha256(_encode(record)).hexdigest() diff --git a/project_material.py b/project_material.py index aee2aee0..75af5b96 100644 --- a/project_material.py +++ b/project_material.py @@ -326,7 +326,9 @@ def _retained_value(get_documents, family, ref): raise WorkbenchError('Retained material is unavailable.', 'unavailable', 409) if document['value']['ref'] != ref: raise WorkbenchError('Retained material reference differs.', 'invalid', 400) - return copy.deepcopy(document['value']) + # Decoded retained values are plain JSON: an exact independent copy. + from project_store import copy_validated_json + return copy_validated_json(document['value']) class MaterialSnapshot: diff --git a/project_session.py b/project_session.py index a610d5d2..2cea853f 100644 --- a/project_session.py +++ b/project_session.py @@ -120,13 +120,28 @@ def _adopt_working_copy(self): snapshot = self.store.snapshot() if snapshot['checkpoint'] != summary['checkpoint']: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._validated(snapshot['records'], snapshot['checkpoint']) + with self._known_envelopes(snapshot['records']): + self._validated(snapshot['records'], snapshot['checkpoint']) del snapshot if self.store.summary()['checkpoint'] != summary['checkpoint']: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) self._bind(summary) self.available = True + def _known_envelopes(self, records, changed=frozenset()): + """Validation context with stored envelope digests for unchanged records.""" + from contextlib import nullcontext + if not self._lazy or not callable(getattr(self.store, 'envelope_digests', None)): + return nullcontext() + from project_composition import known_envelope_digests + digests = self.store.envelope_digests() + known = {} + for record in records: + identity = (record['family'], record['id']) + if identity not in changed and identity in digests and digests[identity] is not None: + known[id(record)] = (record, digests[identity]) + return known_envelope_digests(known) + def _bind(self, snapshot): """Bind a committed checkpoint; a lazy session keeps only its metadata.""" if self._lazy: @@ -178,7 +193,8 @@ def _publish_observation(self, records, key, digest, *, authorize): current = self.check() merged = self._merge(current['records'], records) candidate = [merged[identity] for identity in sorted(merged)] - self._validated(candidate) + with self._known_envelopes(candidate, {(item['family'], item['id']) for item in records}): + self._validated(candidate) if self.publication_intent is not None: self.publication_intent(copy.deepcopy(current), copy.deepcopy(candidate), key, digest) try: @@ -513,7 +529,8 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt identity = (OPERATION_FAMILY, self.project_id + '.' + hashlib.sha256(key.encode('utf-8')).hexdigest()) merged[identity] = dict(family=identity[0], id=identity[1], document=document) records = [merged[identity] for identity in sorted(merged)] - self._validated(records) + with self._known_envelopes(records, {(item['family'], item['id']) for item in upserts} | {identity}): + self._validated(records) # Unique payloads have already reached artifact custody; # SQL callbacks can delete only disposable projection files. attempted_records = records diff --git a/project_store.py b/project_store.py index 508fe989..899c569a 100644 --- a/project_store.py +++ b/project_store.py @@ -286,6 +286,22 @@ def hexdigest(self): return final.hexdigest() +def envelope_digest(family, identity, raw): + """SHA256 of _encode(dict(family, id, document)) from the document's canonical bytes. + + Canonical JSON has no literal newline inside strings, so the document nested + one level deep differs from its standalone encoding only by two spaces after + each newline. + """ + if not raw.endswith(b'\n'): + _fail('Canonical record bytes must end with LF.') + digest = hashlib.sha256(b'{\n "document": ') + digest.update(raw[:-1].replace(b'\n', b'\n ')) + digest.update(b',\n "family": ' + json.dumps(family, ensure_ascii=False).encode('utf-8')) + digest.update(b',\n "id": ' + json.dumps(identity, ensure_ascii=False).encode('utf-8') + b'\n}\n') + return digest.hexdigest() + + def _records_digest(rows): """Digest of (family, id, canonical raw) rows; see _RecordsDigest.""" digest = _RecordsDigest() diff --git a/project_working_store.py b/project_working_store.py index 4e36488b..569067e6 100644 --- a/project_working_store.py +++ b/project_working_store.py @@ -29,7 +29,7 @@ from metadata import _key, connection_info from project_store import (FORMAT, MAX_RECORD_BYTES, MAX_RECORDS, MAX_TOTAL_BYTES, SUPPORTED_FORMATS, ProjectStoreError, _RecordsDigest, _decode, _digest, _domain_format, _encode, - _fail, _identifier, _keys, _manifest_raw, _request, _sha) + _fail, _identifier, _keys, _manifest_raw, _request, _sha, envelope_digest) WORKING_ID = re.compile(r'^[a-f0-9]{32}$') _TABLES = ( @@ -42,6 +42,10 @@ format text NOT NULL, bytes integer NOT NULL, sha256 text NOT NULL, raw bytea NOT NULL, PRIMARY KEY (family, id))''', ) +_MIGRATIONS = ( + # SHA256 of the canonical {family, id, document} envelope (see envelope_digest). + 'ALTER TABLE {schema}.records ADD COLUMN IF NOT EXISTS envelope_sha256 text', +) _ORDER = sql.SQL('ORDER BY family COLLATE "C", id COLLATE "C"') @@ -68,7 +72,7 @@ def __init__(self, folder, working_id, *, dsn=None, canonical_digests=None): raise ProjectStoreError('This project is already open in another Tuldok session.', 'busy', 409) with self._own.transaction(): self._own.execute(sql.SQL('CREATE SCHEMA IF NOT EXISTS {}').format(sql.Identifier(self.schema))) - for statement in _TABLES: + for statement in (*_TABLES, *_MIGRATIONS): self._own.execute(sql.SQL(statement).format(schema=sql.Identifier(self.schema))) except BaseException: self._own.close() @@ -79,9 +83,30 @@ def __init__(self, folder, working_id, *, dsn=None, canonical_digests=None): def bind(self, db): """Use the projection database's connection and transactions.""" - with self._mutex: + with self._locked(): self._db = db + @contextmanager + def _locked(self): + """Store ownership in one global order: projection lock, then this mutex. + + A bound projection connection is shared with Session/owner threads that + hold the projection lock across their transactions, so store work never + takes its own mutex first and then waits for that lock. + """ + if self._db is not None: + with self._catalog_lock(), self._mutex: + yield + else: + with self._mutex: + yield + + def _catalog_lock(self): + # The projection may install a composite host -> session -> catalog + # lock; the store is innermost, so it takes only the catalog lock. + lock = self._db.lock + return getattr(lock, 'projection', lock) + def _joined(self): # Join the projection only inside its open transaction (Session.mutate, # background publication), so record and projection rows commit @@ -95,12 +120,12 @@ def _connection(self): def _guard(self): # A bound projection connection is shared; its catalog lock serializes # every command and transaction on it across threads. - return self._db.lock if self._joined() else nullcontext() + return self._catalog_lock() if self._joined() else nullcontext() @contextmanager def _transaction(self): if self._joined(): - with self._db.lock, self._db.transaction(): + with self._catalog_lock(), self._db.transaction(): yield else: with self._own.transaction(): @@ -149,7 +174,7 @@ def _state(self, *, required=True): def _import(self, raw, manifest): """Bring the working copy to the folder's checkpoint, reading only new revisions.""" - with self._mutex, self._transaction(): + with self._locked(), self._transaction(): existing = {(row[0], row[1]): row[2] for row in self._execute(self._q('SELECT family, id, sha256 FROM {records}'))} wanted = {} @@ -169,10 +194,11 @@ def _import(self, raw, manifest): if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') self._execute(self._q( - 'INSERT INTO {records} (family, id, format, bytes, sha256, raw) VALUES (%s,%s,%s,%s,%s,%s) ' + 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256) VALUES (%s,%s,%s,%s,%s,%s,%s) ' 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' - 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw'), - (entry['family'], entry['id'], entry['format'], entry['bytes'], entry['sha256'], body)) + 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256'), + (entry['family'], entry['id'], entry['format'], entry['bytes'], entry['sha256'], body, + envelope_digest(entry['family'], entry['id'], body))) for family, record_id in set(existing) - set(wanted): self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, record_id)) if self._records_digest() != manifest['receipts'][-1]['records_sha256']: @@ -256,18 +282,18 @@ def _root_fd(self): return self._folder._root_fd def summary(self): - with self._mutex: + with self._locked(): state = self._state() return {key: state[key] for key in ('project_id', 'name', 'checkpoint')} def dirty(self): """Whether the working copy has changes not yet saved to the folder.""" - with self._mutex: + with self._locked(): return self._state()['dirty'] def snapshot(self): """Complete independent record set (costly for large projects).""" - with self._mutex: + with self._locked(): state = self._state() records = [dict(family=f, id=i, document=json.loads(raw)) for f, i, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}'))] @@ -278,10 +304,29 @@ def records(self, *families): """Independent decoded envelopes of the selected families, in record order.""" if not families or any(type(family) is not str for family in families): raise TypeError('Select one or more record families by name.') - with self._mutex: + with self._locked(): return [dict(family=f, id=i, document=json.loads(raw)) for f, i, raw in self._rows( self._q('SELECT family, id, raw FROM {records} WHERE family = ANY(%s) {order}'), (list(families),))] + def envelope_digests(self): + """(family, id) -> canonical envelope SHA256 for every stored revision. + + Rows imported before this column existed are filled once from their + stored canonical bytes. + """ + with self._locked(): + missing = self._execute(self._q('SELECT count(*) FROM {records} WHERE envelope_sha256 IS NULL')).fetchone()[0] + if missing: + with self._transaction(): + digests = [(envelope_digest(family, record_id, raw), family, record_id) for family, record_id, raw + in self._rows(self._q('SELECT family, id, raw FROM {records} WHERE envelope_sha256 IS NULL {order}'))] + with self._guard(): + with self._connection().cursor() as cursor: + cursor.executemany(self._q('UPDATE {records} SET envelope_sha256=%s WHERE family=%s AND id=%s'), + digests) + return {(row[0], row[1]): row[2] for row in + self._execute(self._q('SELECT family, id, envelope_sha256 FROM {records}'))} + def records_view(self, *families): """Every record identity in order; documents decoded only for selected families. @@ -290,7 +335,7 @@ def records_view(self, *families): """ if not families or any(type(family) is not str for family in families): raise TypeError('Select one or more record families by name.') - with self._mutex, self._guard(): + with self._locked(): connection = self._connection() idle = connection.info.transaction_status == psycopg.pq.TransactionStatus.IDLE result = [] @@ -305,14 +350,14 @@ def records_view(self, *families): return result def record(self, family, record_id): - with self._mutex: + with self._locked(): row = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), (family, record_id)).fetchone() return None if row is None else dict(family=family, id=record_id, document=json.loads(bytes(row[0]))) def reconcile(self, request_key, request_sha256): _request(request_key, request_sha256) - with self._mutex: + with self._locked(): for receipt in self._state()['receipts']: if receipt['request_key'] == request_key: if receipt['request_sha256'] != request_sha256: @@ -333,7 +378,7 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256, *, c _digest(expected_checkpoint, 'Expected checkpoint') if type(records) is not list or len(records) > MAX_RECORDS: _fail('Project records must be a bounded list.') - with self._mutex, self._transaction(): + with self._locked(), self._transaction(): state = self._state() inventory = {(f, i): (fmt, size, sha) for f, i, fmt, size, sha in self._inventory()} candidates = {} @@ -387,9 +432,10 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256, *, c for (family, identity), raw in replacements.items(): fmt, size, digest = entries[(family, identity)] self._execute(self._q( - 'INSERT INTO {records} (family, id, format, bytes, sha256, raw) VALUES (%s,%s,%s,%s,%s,%s) ' + 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256) VALUES (%s,%s,%s,%s,%s,%s,%s) ' 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' - 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw'), (family, identity, fmt, size, digest, raw)) + 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256'), + (family, identity, fmt, size, digest, raw, envelope_digest(family, identity, raw))) for family, identity in set(inventory) - set(candidates): self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, identity)) self._write_state(manifest, external_sha256=state['external_sha256'], dirty=True) @@ -401,7 +447,7 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256, *, c def save(self, expected_checkpoint): """Export the working copy to the external folder (the only folder write).""" _digest(expected_checkpoint, 'Expected checkpoint') - with self._mutex: + with self._locked(): state = self._state() if expected_checkpoint != state['checkpoint']: _fail('Expected project checkpoint is stale.', 'stale', 409) @@ -432,7 +478,7 @@ def save(self, expected_checkpoint): def verify_records(self): """Re-check every stored revision's bytes against its SHA256 and length.""" - with self._mutex: + with self._locked(): inventory = {(f, i): (size, sha) for f, i, _fmt, size, sha in self._inventory()} for family, record_id, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}')): size, digest = inventory[(family, record_id)] @@ -448,7 +494,7 @@ def root_directory(self, *, verify_after=False): """Borrow the external folder's physical root (payload artifacts live there).""" if type(verify_after) is not bool: _fail('Root read verification must be an explicit boolean.') - with self._mutex: + with self._locked(): before = self._state()['checkpoint'] descriptor = os.dup(self._folder._root_fd) try: @@ -460,6 +506,7 @@ def root_directory(self, *, verify_after=False): os.close(descriptor) def close(self): + # The projection (and its lock's connection) may already be closed. with self._mutex: if self._closed: return diff --git a/projects.py b/projects.py index 5860d846..3898ba0b 100644 --- a/projects.py +++ b/projects.py @@ -28,9 +28,13 @@ def closed(value, required, optional=()): raise WorkbenchError('Invalid project fields.') +# Same code points as any(ord(c) < 32 or 0xD800 <= ord(c) <= 0xDFFF), scanned in C. +_TEXT_FORBIDDEN = re.compile('[\x00-\x1f\ud800-\udfff]') + + def text(value, label, limit=160): if (not isinstance(value, str) or not value.strip() or len(value) > limit - or any(ord(c) < 32 or 0xD800 <= ord(c) <= 0xDFFF for c in value)): + or _TEXT_FORBIDDEN.search(value)): raise WorkbenchError(label + ' must be bounded, nonempty text without control characters.') return value diff --git a/runs.py b/runs.py index c4cfcfc7..8e17e652 100644 --- a/runs.py +++ b/runs.py @@ -150,7 +150,8 @@ def project_run_projection(run, *, imported=False): item['name'] in seen or type(item['sha256']) is not str or not re.fullmatch('[a-f0-9]{64}',item['sha256'])): raise WorkbenchError('Historical Run artifact identity differs.') seen.add(item['name']);natural(item['bytes'],'Artifact bytes');text(item['media_type'],'Artifact media type',120) - return {key:copy.deepcopy(item) for key,item in run.items() if key in RUN_FIELDS} + # checked_json above already returned an independent copy owned here. + return {key:item for key,item in run.items() if key in RUN_FIELDS} def validate_approved_composition(value, sha, graph, execution_material_sha256, *, catalog_id): diff --git a/tests/test_project_working_store.py b/tests/test_project_working_store.py index 3c2fe2f4..d943af12 100644 --- a/tests/test_project_working_store.py +++ b/tests/test_project_working_store.py @@ -200,6 +200,25 @@ def test_receipt_replay_and_stale_checkpoint(self): store.commit(changed, self.initial['checkpoint'], 'later', '5' * 64) self.assertEqual(caught.exception.code, 'stale') + def test_envelope_digests_equal_canonical_envelope_hashes(self): + from project_composition import _record_sha, known_envelope_digests + store = self.open() + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Committed after import' + store.commit(changed, self.initial['checkpoint'], 'edit', '6' * 64) + digests = store.envelope_digests() + snapshot = store.snapshot() + self.assertEqual(set(digests), {(r['family'], r['id']) for r in snapshot['records']}) + for record in snapshot['records']: + self.assertEqual(digests[(record['family'], record['id'])], _record_sha(record)) + # A supplied digest applies only to that exact object. + record = snapshot['records'][0] + with known_envelope_digests({id(record): (record, 'f' * 64)}): + self.assertEqual(_record_sha(record), 'f' * 64) + self.assertNotEqual(_record_sha(copy.deepcopy(record)), 'f' * 64) + store._execute(store._q('UPDATE {records} SET envelope_sha256=NULL')) + self.assertEqual(store.envelope_digests(), digests, 'Missing digests are recomputed from stored bytes') + def test_verify_records_detects_working_copy_corruption(self): store = self.open() store.verify_records() diff --git a/torch_runtime.py b/torch_runtime.py index e20cb384..80c1b8a9 100644 --- a/torch_runtime.py +++ b/torch_runtime.py @@ -110,6 +110,15 @@ def inventory(configuration): return validate_manifest(value) +def _contained_path(path, roots): + """Canonical absolute path under one of roots (one Path construction).""" + if not (type(path) is str and 0 Date: Mon, 2 Nov 2026 14:03:09 -0700 Subject: [PATCH 09/13] perf(validation): validate only what an edit can affect Each edit validated the complete project (~145 s on the portfolio). A mutation's candidate is a validated base checkpoint plus changed records and its receipt, so validate_project_records now runs only the components whose input families (audited per component in project_validation_delta) contain a changed record, and re-checks old receipts only when records they bind to changed or a new receipt shares their key space. Newly added artifact declarations are checked themselves but touch no component. Identity/claim checks, fork decoding and receipt identities always run; anything unclassified is always validated. TULDOK_VALIDATION_CROSSCHECK=1 runs complete validation alongside and fails on any disagreement; the full project suite passes in that mode. Mutations over a working copy also stop deep-copying the whole record set (captures read the mutation's own fresh read; data capture/replay select their families). A text import on the portfolio project: 161 s -> ~31 s, validation 1-3 s. Co-Authored-By: Claude Opus 5.5 --- project_data.py | 13 ++- project_projection.py | 112 +++++++++++++++++++---- project_session.py | 44 +++++++-- project_validation_delta.py | 119 +++++++++++++++++++++++++ tests/test_project_validation_delta.py | 58 ++++++++++++ 5 files changed, 315 insertions(+), 31 deletions(-) create mode 100644 project_validation_delta.py create mode 100644 tests/test_project_validation_delta.py diff --git a/project_data.py b/project_data.py index c51bf64c..1b58eae6 100644 --- a/project_data.py +++ b/project_data.py @@ -1005,9 +1005,8 @@ def _publish(self, raw, kind, role, media_type): media_type=media_type, kind=kind, role=role, preservation='unique', required_for=['inspect', 'release']) def _require_record(self, record_id, *, extended=False): - from project_fork import active_records - snapshot = self.session.snapshot - for item in active_records(snapshot['records'], project_id=snapshot['project_id']): + # Only these families are inspected; fork history is still checked. + for item in session_active(self.session, 'workbench_record', 'workbench_text'): if item['id'] == record_id and item['family'] in ('workbench_record', 'workbench_text'): if extended and item['family'] == 'workbench_text': fail('This operation requires an explicitly converted data record.', 'unsupported', 409) @@ -1047,9 +1046,8 @@ def _mutate(self, operation, target, body, expected_checkpoint, key, invoke): def _legacy_replay(self, operation, target, body, key): """Retain M1 request meaning after the shared M2 session cutover.""" - records = self.session.snapshot['records'] - from project_fork import active_records - records = active_records(records, project_id=self.session.project_id) + # Only receipts and legacy text are inspected below. + records = session_active(self.session, 'project_operation', 'workbench_text') if any(r['family']=='project_operation' and r['document']['request_key']==key for r in records): return None if operation not in ('data.import_text','data.annotate'): @@ -1291,7 +1289,8 @@ def _capture(self, result, before_records, operation, key): for family, table, owner in (('saved_selection', 'saved_selections', self.selections), ('saved_search', 'saved_searches', self.searches)): live = {r[0] for r in self.db.execute('SELECT id FROM ' + table)} from project_fork import active_records - all_ids = live | {r['id'] for r in active_records(before_records, project_id=self.session.project_id) if r['family'] == family} + all_ids = live | {r['id'] for r in active_records(before_records, project_id=self.session.project_id, + families=frozenset({family})) if r['family'] == family} for record_id in all_ids: old = before.get((family, record_id)) current = owner._get(record_id) if record_id in live else copy.deepcopy(old['document']['current']) diff --git a/project_projection.py b/project_projection.py index f121cb26..e7aa8366 100644 --- a/project_projection.py +++ b/project_projection.py @@ -5,6 +5,7 @@ checkpoint before reconstruction and before each canonical publication. """ import copy +import hashlib from datetime import datetime import re from types import SimpleNamespace @@ -64,7 +65,52 @@ def validate_artifact_resolution(operation, result, records): def validate_project_records(records, *, project_id, artifacts): - """Validate one inert candidate through every owning domain, without SQL reconstruction.""" + """Validate one inert candidate through every owning domain, without SQL reconstruction. + + Under an edit delta (project_validation_delta.edit_delta) the candidate is + a validated base checkpoint plus changed records; components and receipts + that cannot be affected are skipped. TULDOK_VALIDATION_CROSSCHECK=1 also + runs complete validation and requires the same outcome. + """ + from project_validation_delta import current_delta + delta = current_delta() + if delta is None: + return _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=None) + import os + if os.environ.get('TULDOK_VALIDATION_CROSSCHECK') == '1': + full_error = scoped_error = None + try: + _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=None) + except Exception as error: + full_error = error + try: + _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=delta) + except Exception as error: + scoped_error = error + if (full_error is None) != (scoped_error is None): + raise AssertionError(f'Edit-scoped validation disagrees with complete validation: ' + f'complete={full_error!r} scoped={scoped_error!r}') + if full_error is not None: + raise full_error + return None + return _validate_project_records(records, project_id=project_id, artifacts=artifacts, delta=delta) + + +def _composition_members(records): + members = set() + for item in records: + if item['family'] != 'project_composition': + continue + for source in item['document'].get('sources', ()): + for family, entries in source.get('records', {}).items(): + members.update((family, identity) for identity in entries) + members.update(('run_observation', sha) for sha in source.get('run_observations', {}).values()) + return members + + +def _validate_project_records(records, *, project_id, artifacts, delta): + def needed(component): + return delta is None or delta.touches(component) claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) identities = set() @@ -79,28 +125,62 @@ def validate_project_records(records, *, project_id, artifacts): if declaration['sha256'] != item['id']: raise WorkbenchError('Artifact inventory identity differs.') declarations[item['id']] = declaration - decode_data_records(records) + if needed('data'): + decode_data_records(records) from project_fork import decode_fork decode_fork(records, project_id=project_id) - from project_composition import decode_records as decode_compositions - decode_compositions(records, project_id=project_id) - from project_transfer import decode_records as decode_extraction_records - decode_extraction_records(records) + if needed('composition') or (delta is not None and delta.changed & _composition_members(records)): + from project_composition import decode_records as decode_compositions + decode_compositions(records, project_id=project_id) + if needed('extraction'): + from project_transfer import decode_records as decode_extraction_records + decode_extraction_records(records) operation_records(records, project_id=project_id) - validate_file_receipt = file_receipt_validator(records) - decode_native_records(records,artifacts=artifacts) - materials = MaterialSnapshot(records) - for blob in materials.artifact_blobs(): - if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): - raise WorkbenchError('Material artifact declaration is missing or inconsistent.') + if needed('native'): + decode_native_records(records,artifacts=artifacts) + material_cache = [] + def materials(): + if not material_cache: + material_cache.append(MaterialSnapshot(records)) + return material_cache[0] + if needed('material'): + for blob in materials().artifact_blobs(): + if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): + raise WorkbenchError('Material artifact declaration is missing or inconsistent.') # Resolve every cross-owner reference against THIS candidate, rather # than a resolver that still sees the preceding committed checkpoint. - decode_producer_records(records, artifacts=artifacts) - decode_history_records(records, materials=materials, artifacts=artifacts) - validate_measurement_receipt = measurement_receipt_validator(records, artifacts=artifacts) - validate_source_receipt = source_receipt_validator(records) + if needed('producers'): + decode_producer_records(records, artifacts=artifacts) + if needed('history') or (delta is not None and delta.history_receipt_added()): + decode_history_records(records, materials=materials(), artifacts=artifacts) + validators = {} + def lazy(name, component, claims, build): + # claims(operation) is exactly what the built validator can accept; + # for any other operation it would return False, so it is not built. + def validate(operation, result): + if not claims(operation): + return False + if name not in validators: + validators[name] = build() + return validators[name](operation, result) + if delta is None or needed(component): + validators[name] = build() + return validate + validate_file_receipt = lazy('file', 'files', lambda op: op.startswith('files.'), + lambda: file_receipt_validator(records)) + validate_measurement_receipt = lazy('measurement', 'measurements', lambda op: op == 'measurements.import', + lambda: measurement_receipt_validator(records, artifacts=artifacts)) + validate_source_receipt = lazy('source', 'sources', lambda op: op == 'source.register', + lambda: source_receipt_validator(records)) + from project_session import OPERATION_FAMILY for receipt in operation_documents(records): operation, result = receipt['operation'], receipt['result'] + if delta is not None: + identity = (OPERATION_FAMILY, receipt['project_id'] + '.' + hashlib.sha256(receipt['request_key'].encode('utf-8')).hexdigest() + if receipt.get('format') == 'project_operation_v2' else + hashlib.sha256(receipt['request_key'].encode('utf-8')).hexdigest()) + if not delta.receipt_needed(operation, identity): + continue if (validate_data_result(operation, result, records) or validate_material_result(operation, result, records) or validate_history_result(operation, result, records) diff --git a/project_session.py b/project_session.py index 2cea853f..cfd60555 100644 --- a/project_session.py +++ b/project_session.py @@ -142,6 +142,20 @@ def _known_envelopes(self, records, changed=frozenset()): known[id(record)] = (record, digests[identity]) return known_envelope_digests(known) + def _edit_scope(self, base_checkpoint, changed, operations=(), base_records=None): + """Validate only what this edit can affect when its base is validated. + + The candidate is the base checkpoint's records plus changed identities + (never deletions); see project_validation_delta. + """ + from contextlib import nullcontext + if getattr(self, 'validated_checkpoint', None) != base_checkpoint: + return nullcontext() + from project_validation_delta import ValidationDelta, edit_delta + base = set() if base_records is None else {(item['family'], item['id']) for item in base_records} + added = frozenset(identity for identity in changed if base_records is not None and identity not in base) + return edit_delta(ValidationDelta(frozenset(changed), frozenset(operations), added)) + def _bind(self, snapshot): """Bind a committed checkpoint; a lazy session keeps only its metadata.""" if self._lazy: @@ -161,8 +175,14 @@ def foreground_publication(self): return self._foreground_publication @staticmethod - def _merge(records, upserts): - merged = {(item['family'], item['id']): copy.deepcopy(item) for item in records} + def _merge(records, upserts, *, owned=False): + """Candidate record map: base records plus copied upserts. + + owned=True means the caller owns these base records (a fresh working + copy read), so they are used without another whole-project copy. + """ + merged = ({(item['family'], item['id']): item for item in records} if owned else + {(item['family'], item['id']): copy.deepcopy(item) for item in records}) seen = {} for item in upserts: if type(item) is not dict or set(item) != {'family', 'id', 'document'}: @@ -191,9 +211,11 @@ def _publish_observation(self, records, key, digest, *, authorize): raise ProjectSessionError('Invalid background publication identity.') authorize() current = self.check() - merged = self._merge(current['records'], records) + merged = self._merge(current['records'], records, owned=self._lazy) candidate = [merged[identity] for identity in sorted(merged)] - with self._known_envelopes(candidate, {(item['family'], item['id']) for item in records}): + changed = {(item['family'], item['id']) for item in records} + with self._known_envelopes(candidate, changed), \ + self._edit_scope(current['checkpoint'], changed, base_records=current['records']): self._validated(candidate) if self.publication_intent is not None: self.publication_intent(copy.deepcopy(current), copy.deepcopy(candidate), key, digest) @@ -512,24 +534,30 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt self._foreground_publication = True with self.db.transaction(): result = invoke() - captured = capture(result, copy.deepcopy(current['records'])) + # A working-copy read is already this mutation's own fresh + # copy; owner captures read it and copy what they change. + before = current['records'] if self._lazy else copy.deepcopy(current['records']) + captured = capture(result, before) if type(captured) is not dict or set(captured) != {'records', 'result'} or type(captured['records']) is not list: raise ProjectSessionError('Invalid owned project capture.') if len(agent_api.encoded(captured['result'])) > agent_api.RESPONSE_LIMIT: raise ProjectSessionError('Project result exceeds its transport bound.', 'too_large', 413) upserts = captured['records'] if self.extra_capture is not None: - extra = self.extra_capture(copy.deepcopy(current['records'])) + extra = self.extra_capture(current['records'] if self._lazy else copy.deepcopy(current['records'])) if type(extra) is not list: raise ProjectSessionError('Invalid fixed runtime capture.') upserts = upserts + extra - merged = self._merge(current['records'], upserts) + merged = self._merge(current['records'], upserts, owned=self._lazy) document = dict(format=OPERATION_FORMAT, project_id=self.project_id, operation=operation, target=target, request_key=key, input_sha256=digest, result=copy.deepcopy(captured['result'])) identity = (OPERATION_FAMILY, self.project_id + '.' + hashlib.sha256(key.encode('utf-8')).hexdigest()) merged[identity] = dict(family=identity[0], id=identity[1], document=document) records = [merged[identity] for identity in sorted(merged)] - with self._known_envelopes(records, {(item['family'], item['id']) for item in upserts} | {identity}): + changed = {(item['family'], item['id']) for item in upserts} | {identity} + with self._known_envelopes(records, changed), \ + self._edit_scope(current['checkpoint'], changed, (operation,), + base_records=current['records']): self._validated(records) # Unique payloads have already reached artifact custody; # SQL callbacks can delete only disposable projection files. diff --git a/project_validation_delta.py b/project_validation_delta.py new file mode 100644 index 00000000..d63814c5 --- /dev/null +++ b/project_validation_delta.py @@ -0,0 +1,119 @@ +"""Edit-scoped validation: re-check only what a mutation can affect. + +A mutation's candidate is the previous, completely validated checkpoint plus +changed/added records (no deletions) and its new operation receipt. Each +validation component of validate_project_records is a pure function of the +documents and identities of a known set of record families (see the audit in +docs/plans/database-working-store.md, stage 4). A component whose input +families contain no changed record has the same outcome as for the previous +checkpoint, so it is skipped. Old operation receipts are re-checked only when +records they bind to changed or a new receipt shares their key space. + +Anything not classified here is always validated. TULDOK_VALIDATION_CROSSCHECK=1 +additionally runs complete validation and requires the same outcome. +""" +import contextvars +from contextlib import contextmanager +from dataclasses import dataclass + +from project_data import FAMILIES as _DATA +from project_history import FAMILY_FORMATS as _HISTORY +from project_material import FAMILIES as _MATERIAL +from project_producers import FAMILIES as _PRODUCERS + +DATA = frozenset(_DATA) +HISTORY = frozenset(_HISTORY) +MATERIAL = frozenset(_MATERIAL) +PRODUCERS = frozenset(_PRODUCERS) + +# Families whose documents or identities each component reads. +COMPONENTS = { + 'data': DATA | {'workbench_text', 'project_artifact', 'project_fork'} | PRODUCERS, + 'fork': frozenset({'project_fork'}), + 'composition': frozenset({'project_composition', 'project_fork', 'run_observation'}), + 'extraction': frozenset({'project_extraction', 'project_fork', 'project_snapshot', 'project_file', 'project_artifact'}), + 'files': frozenset({'project_file', 'project_fork'}), + 'native': frozenset({'native_execution_history', 'project_artifact', 'run_history'}), + 'material': MATERIAL | {'project_fork', 'project_artifact'}, + 'producers': PRODUCERS | {'project_artifact', 'workbench_record', 'workbench_text'}, + 'history': HISTORY | MATERIAL | {'project_artifact', 'dataset_exposure', 'pipeline_graph'}, + 'measurements': frozenset({'measurement_source', 'project_artifact', 'project_snapshot'}), + 'sources': frozenset({'project_snapshot'}), +} + +# Operation-receipt prefixes -> families their result binds to. A prefix +# mapped to an empty set binds only immutable or append-only facts. +RECEIPT_BINDINGS = { + 'data.': DATA | {'workbench_text'}, + 'models.': frozenset(), 'graphs.': frozenset(), 'recipes.': frozenset(), + 'studies.': HISTORY | {'pipeline_graph', 'project_artifact'}, + 'history.disclose': HISTORY | {'pipeline_graph', 'project_artifact'}, + 'source.': frozenset(), + 'measurements.': frozenset({'measurement_source'}), + 'runs.': frozenset({'run_history'}), + 'producers.': frozenset(), + 'files.': frozenset(), + 'study_execution.': HISTORY | MATERIAL, + 'runtime.': frozenset({'runtime_material', 'project_artifact'}), + 'native_preparation.': MATERIAL | {'pipeline_graph', 'dataset_release'}, + 'artifacts.resolve': frozenset(), +} +# A new receipt in these key spaces can change the meaning of older ones. +SHARED_KEY_SPACES = ('study_execution.', 'runtime.') +# Receipts these prefixes claim are also validated inside history decoding. +HISTORY_RECEIPTS = ('studies.', 'history.disclose') + + +def _prefix(operation): + return next((prefix for prefix in RECEIPT_BINDINGS if operation.startswith(prefix)), None) + + +@dataclass(frozen=True) +class ValidationDelta: + """Identities a mutation changed relative to its validated base checkpoint.""" + changed: frozenset + operations: frozenset = frozenset() # operation names of new/changed receipts + added: frozenset = frozenset() # changed identities absent from the base + + @property + def families(self): + # A newly added artifact declaration cannot change any component: an + # unchanged record referencing an undeclared blob already failed base + # validation. New declarations are still checked by the claim loop. + return frozenset(family for family, identity in self.changed + if not (family == 'project_artifact' and (family, identity) in self.added)) + + def touches(self, component): + return bool(self.families & COMPONENTS[component]) + + def history_receipt_added(self): + return any(operation.startswith(HISTORY_RECEIPTS) for operation in self.operations) + + def receipt_needed(self, operation, identity): + if identity in self.changed: + return True + prefix = _prefix(operation) + if prefix is None: + return True + if self.families & RECEIPT_BINDINGS[prefix]: + return True + if prefix in SHARED_KEY_SPACES and any(op.startswith(prefix) for op in self.operations): + return True + return False + + +_DELTA = contextvars.ContextVar('validation_delta', default=None) + + +@contextmanager +def edit_delta(delta): + """Validate the enclosed candidate relative to its validated base checkpoint.""" + token = _DELTA.set(delta) + try: + yield + finally: + _DELTA.reset(token) + + +def current_delta(): + return _DELTA.get() diff --git a/tests/test_project_validation_delta.py b/tests/test_project_validation_delta.py new file mode 100644 index 00000000..38eed517 --- /dev/null +++ b/tests/test_project_validation_delta.py @@ -0,0 +1,58 @@ +"""Edit-scoped validation re-checks exactly what an edit can affect.""" +import unittest + +from project_projection import validate_project_records +from project_validation_delta import COMPONENTS, ValidationDelta, edit_delta +from workbench import WorkbenchError + +PROJECT = 'a' * 32 +BROKEN_TEXT = dict(family='workbench_text', id='broken', document=dict(format='workbench_text_v1', text=17)) + + +class DeltaRules(unittest.TestCase): + def test_new_artifact_declarations_touch_no_component(self): + delta = ValidationDelta(frozenset({('project_artifact', 'b' * 64)}), added=frozenset({('project_artifact', 'b' * 64)})) + self.assertEqual(delta.families, frozenset()) + self.assertFalse(any(delta.touches(component) for component in COMPONENTS)) + changed = ValidationDelta(frozenset({('project_artifact', 'b' * 64)})) + self.assertTrue(changed.touches('history'), 'A changed existing declaration is re-checked') + + def test_receipt_rules(self): + delta = ValidationDelta(frozenset({('workbench_record', 'r'), ('project_operation', 'new')}), + frozenset({'data.import_text'})) + self.assertTrue(delta.receipt_needed('data.annotate', ('project_operation', 'old')), 'Data receipts bind to changed data records') + self.assertFalse(delta.receipt_needed('files.adopt', ('project_operation', 'old'))) + self.assertTrue(delta.receipt_needed('files.adopt', ('project_operation', 'new')), 'The new receipt is always checked') + self.assertTrue(delta.receipt_needed('unclassified.operation', ('project_operation', 'old')), 'Unknown receipts are always checked') + runtime = ValidationDelta(frozenset({('project_operation', 'new')}), frozenset({'runtime.capture'})) + self.assertTrue(runtime.receipt_needed('runtime.abandon', ('project_operation', 'old')), 'Shared key spaces are re-checked') + self.assertFalse(runtime.receipt_needed('studies.save', ('project_operation', 'old'))) + self.assertTrue(ValidationDelta(frozenset(), frozenset({'studies.save'})).history_receipt_added()) + + +class ScopedValidation(unittest.TestCase): + def validate(self, records, delta): + with edit_delta(delta): + validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_touched_component_still_rejects_an_invalid_change(self): + with self.assertRaises(WorkbenchError): + self.validate([BROKEN_TEXT], ValidationDelta(frozenset({('workbench_text', 'broken')}))) + with self.assertRaises(WorkbenchError): + validate_project_records([BROKEN_TEXT], project_id=PROJECT, artifacts=None) + + def test_untouched_component_is_not_revalidated(self): + # The base checkpoint is assumed validated; an unrelated edit skips data. + self.validate([BROKEN_TEXT], ValidationDelta(frozenset({('project_file', 'other')}))) + + def test_identity_and_claim_checks_always_run(self): + duplicate = [BROKEN_TEXT, dict(BROKEN_TEXT)] + with self.assertRaises(WorkbenchError): + self.validate(duplicate, ValidationDelta(frozenset({('project_file', 'other')}))) + unknown = [dict(family='unclaimed_family', id='x', document={})] + with self.assertRaises(WorkbenchError): + self.validate(unknown, ValidationDelta(frozenset({('project_file', 'other')}))) + + +if __name__ == '__main__': + unittest.main() From a30e4c8993beedfa509fb972e4d81945f1a11bf3 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 14:03:24 -0700 Subject: [PATCH 10/13] docs(plan): record edit-scoped validation results Co-Authored-By: Claude Opus 5.5 --- docs/plans/database-working-store.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/plans/database-working-store.md b/docs/plans/database-working-store.md index 1ba0e80f..2def6cce 100644 --- a/docs/plans/database-working-store.md +++ b/docs/plans/database-working-store.md @@ -66,8 +66,16 @@ the project can be tracked in Git. would carry live execution state (runs, admissions, execution grants) into a later session, which a cold open deliberately starts without. Instead the projection is still reconstructed fresh, from cheap family-scoped reads. -4. **Incremental validation for commits — open.** Each commit still validates - the complete candidate record set (about 2.5 minutes on the portfolio). +4. **Edit-scoped validation — done.** `project_validation_delta` declares each + validation component's input families (from a per-component audit); an + edit re-runs only components whose inputs it changed and only the old + receipts bound to changed records or sharing a key space. Identity, claim, + fork and receipt-identity checks always run; unclassified operations are + always validated. `TULDOK_VALIDATION_CROSSCHECK=1` runs complete validation + alongside and fails on disagreement (the project suite passes in that mode). + A text import on the portfolio: 161 s → ~31 s; validation itself 1–3 s. The + remainder is the receipt digest over every record's bytes (format-defined) + and the full record read for owner captures, both dominated by stage 5. 5. **Smaller project files — open.** Store each captured runtime file listing once and reference it from Runs (format migration with exact round trip). From d196bc3dbefc69c22b75b5df45d7ae3d43c5cc64 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 14:28:08 -0700 Subject: [PATCH 11/13] feat(projects): store retained Run runtime closures once Every retained external-recipe Run embedded the same captured runtime closure three times (spec.runtime_files, material.runtime and a contiguous block of historical_runtime payload edges, ~3.3 MB per Run). On the Rheon portfolio this was 1.51 GB of the 1.79 GB of canonical metadata. Stores now keep one content-addressed, store-owned `run_runtime_closure` record and compact Run documents (project_run_history_v2) that reference it. Compaction happens only when expansion reproduces the original canonical bytes exactly, and both stores expand compact Runs on every read, so domain owners and validators see unchanged logical documents. Receipts cover the physical records; logical envelope digests are kept for composition checks. WorkingProjectStore.compact_run_closures() migrates a working copy (logical records proven unchanged); Save then writes the compact files. Ordinary commits compact only changed Runs. Portfolio: 460 Runs compact exactly, 1569 MB -> 55 MB + one 3.1 MB closure; metadata 1790 MB -> 279 MB. Co-Authored-By: Claude Opus 5.5 --- project_record_dedupe.py | 124 ++++++++++++++++++++ project_store.py | 77 +++++++++++-- project_working_store.py | 140 +++++++++++++++++++---- tests/test_project_record_dedupe.py | 150 +++++++++++++++++++++++++ tests/test_project_validation_delta.py | 11 +- 5 files changed, 468 insertions(+), 34 deletions(-) create mode 100644 project_record_dedupe.py create mode 100644 tests/test_project_record_dedupe.py diff --git a/project_record_dedupe.py b/project_record_dedupe.py new file mode 100644 index 00000000..aff3948d --- /dev/null +++ b/project_record_dedupe.py @@ -0,0 +1,124 @@ +"""Physical deduplication of retained runtime closures in Run history records. + +Retained external-recipe Runs each embed the same captured runtime closure three +times: run.spec.runtime_files, run.material.runtime and a contiguous block of +'historical_runtime' payload edges (about 2.5 MB per Run). The store keeps one +content-addressed `run_runtime_closure` record and a compact Run document +(`project_run_history_v2`) that references it. Stores expand compact Runs back +to the exact logical v1 document on read, so domain owners and validators see +unchanged documents. A Run is compacted only when expansion reproduces its +canonical bytes exactly. +""" +import copy +import hashlib + +RUN_FAMILY = 'run_history' +RUN_V1 = 'project_run_history_v1' +RUN_V2 = 'project_run_history_v2' +CLOSURE_FAMILY = 'run_runtime_closure' +CLOSURE_FORMAT = 'project_run_runtime_closure_v1' +RUNTIME_ROLE = 'historical_runtime' + + +def _encode(value): + from project_store import _encode as canonical + return canonical(value) + + +def closure_id(closure): + """Content address of a closure document.""" + return hashlib.sha256(_encode(closure)).hexdigest() + + +def compact(document): + """(compact v2 document, closure document) or None when not exactly reversible.""" + if type(document) is not dict or document.get('format') != RUN_V1: + return None + run = document.get('run') + payloads = document.get('payloads') + if type(run) is not dict or type(payloads) is not list: + return None + spec, material = run.get('spec'), run.get('material') + if (type(spec) is not dict or type(material) is not dict or 'runtime_files' not in spec + or 'runtime' not in material or 'runtime_closure' in document): + return None + positions = [index for index, edge in enumerate(payloads) + if type(edge) is dict and edge.get('role') == RUNTIME_ROLE] + if not positions or positions != list(range(positions[0], positions[-1] + 1)): + return None + start, end = positions[0], positions[-1] + 1 + closure = dict(format=CLOSURE_FORMAT, runtime_files=copy.deepcopy(spec['runtime_files']), + runtime=copy.deepcopy(material['runtime']), payload_block=copy.deepcopy(payloads[start:end])) + reduced = copy.deepcopy(document) + del reduced['run']['spec']['runtime_files'] + del reduced['run']['material']['runtime'] + reduced['payloads'] = reduced['payloads'][:start] + reduced['payloads'][end:] + reduced['format'] = RUN_V2 + reduced['runtime_closure'] = dict(id=closure_id(closure), payload_index=start) + if _encode(expand(reduced, closure)) != _encode(document): + return None + return reduced, closure + + +def expand(document, closure): + """The exact logical v1 Run document of a compact v2 document.""" + reference = document['runtime_closure'] + if closure.get('format') != CLOSURE_FORMAT or closure_id(closure) != reference['id']: + from project_store import _fail + _fail('Run runtime closure differs from its reference.') + result = copy.deepcopy(document) + del result['runtime_closure'] + result['format'] = RUN_V1 + result['run']['spec']['runtime_files'] = copy.deepcopy(closure['runtime_files']) + result['run']['material']['runtime'] = copy.deepcopy(closure['runtime']) + index = reference['payload_index'] + if type(index) is not int or not 0 <= index <= len(result['payloads']): + from project_store import _fail + _fail('Invalid Run runtime closure position.') + result['payloads'] = result['payloads'][:index] + copy.deepcopy(closure['payload_block']) + result['payloads'][index:] + return result + + +def is_compact(family, document): + return family == RUN_FAMILY and type(document) is dict and document.get('format') == RUN_V2 + + +def logical_records(physical): + """Logical envelopes of physical records: closures hidden, compact Runs expanded.""" + closures = {item['id']: item['document'] for item in physical if item['family'] == CLOSURE_FAMILY} + result = [] + for item in physical: + if item['family'] == CLOSURE_FAMILY: + continue + document = item['document'] + if is_compact(item['family'], document): + closure = closures.get(document['runtime_closure']['id']) + if closure is None: + from project_store import _fail + _fail('Compact Run references a missing runtime closure.') + document = expand(document, closure) + result.append(dict(family=item['family'], id=item['id'], document=document)) + return result + + +def physical_records(logical, *, existing_closures=None): + """Physical envelopes for logical records, compacting Runs where exact. + + existing_closures maps closure id -> document already stored; compacted + Runs reference them and every referenced closure is included once. + """ + closures = dict(existing_closures or {}) + used = set() + result = [] + for item in logical: + document = item['document'] + if item['family'] == RUN_FAMILY: + compacted = compact(document) + if compacted is not None: + document, closure = compacted + identity = document['runtime_closure']['id'] + closures.setdefault(identity, closure) + used.add(identity) + result.append(dict(family=item['family'], id=item['id'], document=document)) + result.extend(dict(family=CLOSURE_FAMILY, id=identity, document=closures[identity]) for identity in sorted(used)) + return sorted(result, key=lambda item: (item['family'], item['id'])) diff --git a/project_store.py b/project_store.py index 899c569a..826a5ab7 100644 --- a/project_store.py +++ b/project_store.py @@ -61,6 +61,9 @@ 'grounded_history': 'project_grounded_history_v1', 'caption_proposal_history': 'project_caption_proposal_history_v1', 'text_proposal_history': 'project_text_proposal_history_v1', + # Store-only: one shared retained runtime closure for compact Runs + # (project_record_dedupe); never visible to domain owners. + 'run_runtime_closure': 'project_run_runtime_closure_v1', } MAX_RECORDS = 100_000 # Scientific history can retain original runtime/file observations in hundreds @@ -185,7 +188,8 @@ def _domain_format(value, family): _fail('Domain document requires a versioned format.') if value != SUPPORTED_FORMATS[family] and not ( family == 'project_operation' and value == 'project_operation_v2' - or family == 'project_file' and value == 'project_file_v2'): + or family == 'project_file' and value == 'project_file_v2' + or family == 'run_history' and value == 'project_run_history_v2'): _fail('Unsupported domain document format.', 'unsupported', 422) @@ -316,6 +320,53 @@ def _records(records): for family, identity, document, raw, _entry in _prepared_records(records)] +def _physical_candidates(records, logical, physical): + """Physical envelopes for a logical candidate record set. + + A candidate equal to its current logical document reuses the current + physical document (so its verified revision is kept). Changed Runs are + compacted to a shared runtime closure when exactly reversible; every + referenced closure is included once. Returns (physical candidates, logical + result in record order) or (records, None) for malformed input, which + _prepared_records then rejects with its usual errors. + """ + from project_record_dedupe import CLOSURE_FAMILY, RUN_FAMILY, compact, is_compact + if type(records) is not list or any(type(item) is not dict or set(item) != {'family', 'id', 'document'} + or type(item['family']) is not str or type(item['id']) is not str + for item in records): + return records, None + if any(item['family'] == CLOSURE_FAMILY for item in records): + _fail('Runtime closures are store-owned records.', 'unsupported', 422) + current_logical = {(item['family'], item['id']): item['document'] for item in logical} + current_physical = {(item['family'], item['id']): item['document'] for item in physical} + closures = {item['id']: item['document'] for item in physical if item['family'] == CLOSURE_FAMILY} + candidates, logical_out, referenced = [], [], set() + for item in records: + key = (item['family'], item['id']) + document = item['document'] + if key in current_logical and _same_json(document, current_logical[key]): + stored = current_physical[key] + candidates.append(dict(family=key[0], id=key[1], document=stored)) + logical_out.append(dict(family=key[0], id=key[1], document=current_logical[key])) + if is_compact(key[0], stored): + referenced.add(stored['runtime_closure']['id']) + continue + owned = copy.deepcopy(document) + logical_out.append(dict(family=key[0], id=key[1], document=owned)) + compacted = compact(owned) if key[0] == RUN_FAMILY else None + if compacted is not None: + reduced, closure = compacted + identity = reduced['runtime_closure']['id'] + closures.setdefault(identity, closure) + referenced.add(identity) + candidates.append(dict(family=key[0], id=key[1], document=reduced)) + else: + candidates.append(dict(family=key[0], id=key[1], document=owned)) + candidates.extend(dict(family=CLOSURE_FAMILY, id=identity, document=closures[identity]) for identity in sorted(referenced)) + logical_out.sort(key=lambda item: (item['family'], item['id'])) + return candidates, logical_out + + def _request(key, digest): if type(key) is not str or not REQUEST_KEY.fullmatch(key): _fail('Request key must contain 1–128 supported characters.') @@ -942,7 +993,7 @@ def _load(self): # freshness check is redundant. On-disk corruption after that point # is still refused by Save (verify_records) and by every reopen. self._binding() - return cached[1:] + return cached[1], cached[2] manifest = checked_manifest(raw) loaded = [] records_digest = _RecordsDigest() @@ -972,8 +1023,12 @@ def _load(self): self._binding() if known is not None and proven: known.add_many(proven) - self._read_cache = (raw, manifest, loaded) - return manifest, loaded + # Domain owners see logical records: shared runtime closures hidden and + # compact Runs expanded to their exact original documents. + from project_record_dedupe import logical_records + logical = logical_records(loaded) + self._read_cache = (raw, manifest, logical, loaded) + return manifest, logical def _verified_record_bytes(self, entry): raw = self._record_file(entry['family'], entry['id'], entry['sha256']) @@ -1105,13 +1160,16 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256): current, loaded = self._load() if current != self._manifest: _fail('Project checkpoint changed; close and reopen.', 'stale', 409) + physical = self._read_cache[3] # Unchanged records keep their verified canonical revision: no # re-encoding, rewrite or read-back of identical immutable bytes. entries = {(entry['family'], entry['id']): entry for entry in current['inventory']} unchanged = {(item['family'], item['id']): (item['document'], entries[(item['family'], item['id'])]) - for item in loaded} - prepared = _prepared_records(records, unchanged) - normalized = [dict(family=f, id=i, document=d) for f, i, d, _, _ in prepared] + for item in physical} + candidates, logical_out = _physical_candidates(records, loaded, physical) + prepared = _prepared_records(candidates, unchanged) + if logical_out is None: + logical_out = [dict(family=f, id=i, document=d) for f, i, d, _, _ in prepared] digest_rows = _RecordsDigest() for family, identity, _document, raw, entry in prepared: if raw is None: @@ -1155,8 +1213,9 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256): [_sha(raw) for _f, _i, _d, raw, _e in prepared if raw is not None]) # Every committed revision is either freshly written and read # back byte-for-byte or an unchanged verified revision. - self._read_cache = (published, manifest, normalized) - return self._snapshot(manifest, normalized) + self._read_cache = (published, manifest, logical_out, + [dict(family=f, id=i, document=d) for f, i, d, _, _ in prepared]) + return self._snapshot(manifest, logical_out) except OSError as exc: _io_error(exc, 'commit project records') diff --git a/project_working_store.py b/project_working_store.py index 569067e6..10140d74 100644 --- a/project_working_store.py +++ b/project_working_store.py @@ -27,6 +27,7 @@ from psycopg import sql from metadata import _key, connection_info +from project_record_dedupe import CLOSURE_FAMILY, RUN_FAMILY, compact, expand, is_compact from project_store import (FORMAT, MAX_RECORD_BYTES, MAX_RECORDS, MAX_TOTAL_BYTES, SUPPORTED_FORMATS, ProjectStoreError, _RecordsDigest, _decode, _digest, _domain_format, _encode, _fail, _identifier, _keys, _manifest_raw, _request, _sha, envelope_digest) @@ -45,6 +46,8 @@ _MIGRATIONS = ( # SHA256 of the canonical {family, id, document} envelope (see envelope_digest). 'ALTER TABLE {schema}.records ADD COLUMN IF NOT EXISTS envelope_sha256 text', + # Shared runtime closure referenced by a compact Run row (project_record_dedupe). + 'ALTER TABLE {schema}.records ADD COLUMN IF NOT EXISTS closure_id text', ) _ORDER = sql.SQL('ORDER BY family COLLATE "C", id COLLATE "C"') @@ -193,12 +196,17 @@ def _import(self, raw, manifest): proven.append(entry['sha256']) if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') + compact_run = is_compact(entry['family'], document) self._execute(self._q( - 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256) VALUES (%s,%s,%s,%s,%s,%s,%s) ' + 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256, closure_id) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ' 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' - 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256'), + 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256, ' + 'closure_id=EXCLUDED.closure_id'), (entry['family'], entry['id'], entry['format'], entry['bytes'], entry['sha256'], body, - envelope_digest(entry['family'], entry['id'], body))) + # A compact Run's logical envelope digest is filled from its expansion. + None if compact_run or entry['family'] == CLOSURE_FAMILY else envelope_digest(entry['family'], entry['id'], body), + document['runtime_closure']['id'] if compact_run else None)) for family, record_id in set(existing) - set(wanted): self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, record_id)) if self._records_digest() != manifest['receipts'][-1]['records_sha256']: @@ -253,6 +261,24 @@ def _records_digest(self, replacements=None, identities=None): _fail('Unchanged candidate record is missing from the working copy.') return digest.hexdigest() + # ---- Logical documents: closures hidden, compact Runs expanded. + + def _closure(self, identity): + cache = self.__dict__.setdefault('_closures', {}) + if identity not in cache: + row = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), + (CLOSURE_FAMILY, identity)).fetchone() + if row is None: + _fail('Compact Run references a missing runtime closure.') + cache[identity] = json.loads(bytes(row[0])) + return cache[identity] + + def _logical(self, family, raw): + document = json.loads(raw) + if is_compact(family, document): + document = expand(document, self._closure(document['runtime_closure']['id'])) + return document + # ---- Manifest identity, identical to the file format. def _inventory(self): @@ -295,8 +321,8 @@ def snapshot(self): """Complete independent record set (costly for large projects).""" with self._locked(): state = self._state() - records = [dict(family=f, id=i, document=json.loads(raw)) - for f, i, raw in self._rows(self._q('SELECT family, id, raw FROM {records} {order}'))] + records = [dict(family=f, id=i, document=self._logical(f, raw)) for f, i, raw in self._rows( + self._q('SELECT family, id, raw FROM {records} WHERE family <> %s {order}'), (CLOSURE_FAMILY,))] return dict(project_id=state['project_id'], name=state['name'], checkpoint=state['checkpoint'], records=records) @@ -305,8 +331,9 @@ def records(self, *families): if not families or any(type(family) is not str for family in families): raise TypeError('Select one or more record families by name.') with self._locked(): - return [dict(family=f, id=i, document=json.loads(raw)) for f, i, raw in self._rows( - self._q('SELECT family, id, raw FROM {records} WHERE family = ANY(%s) {order}'), (list(families),))] + selected = [family for family in families if family != CLOSURE_FAMILY] + return [dict(family=f, id=i, document=self._logical(f, raw)) for f, i, raw in self._rows( + self._q('SELECT family, id, raw FROM {records} WHERE family = ANY(%s) {order}'), (selected,))] def envelope_digests(self): """(family, id) -> canonical envelope SHA256 for every stored revision. @@ -315,17 +342,23 @@ def envelope_digests(self): stored canonical bytes. """ with self._locked(): - missing = self._execute(self._q('SELECT count(*) FROM {records} WHERE envelope_sha256 IS NULL')).fetchone()[0] + missing = self._execute(self._q('SELECT count(*) FROM {records} WHERE envelope_sha256 IS NULL AND family <> %s'), + (CLOSURE_FAMILY,)).fetchone()[0] if missing: with self._transaction(): - digests = [(envelope_digest(family, record_id, raw), family, record_id) for family, record_id, raw - in self._rows(self._q('SELECT family, id, raw FROM {records} WHERE envelope_sha256 IS NULL {order}'))] + # Logical envelopes: a compact Run hashes its exact expansion. + digests = [(envelope_digest(family, record_id, _encode(self._logical(family, raw)) + if family == RUN_FAMILY else raw), family, record_id) + for family, record_id, raw in self._rows(self._q( + 'SELECT family, id, raw FROM {records} WHERE envelope_sha256 IS NULL AND family <> %s {order}'), + (CLOSURE_FAMILY,))] with self._guard(): with self._connection().cursor() as cursor: cursor.executemany(self._q('UPDATE {records} SET envelope_sha256=%s WHERE family=%s AND id=%s'), digests) return {(row[0], row[1]): row[2] for row in - self._execute(self._q('SELECT family, id, envelope_sha256 FROM {records}'))} + self._execute(self._q('SELECT family, id, envelope_sha256 FROM {records} WHERE family <> %s'), + (CLOSURE_FAMILY,))} def records_view(self, *families): """Every record identity in order; documents decoded only for selected families. @@ -343,17 +376,20 @@ def records_view(self, *families): connection.cursor(name='working_' + uuid.uuid4().hex) as cursor: cursor.itersize = 256 cursor.execute(self._q('SELECT family, id, CASE WHEN family = ANY(%s) THEN raw END ' - 'FROM {records} {order}'), (list(families),)) - for family, record_id, raw in cursor: - result.append(dict(family=family, id=record_id, - document=None if raw is None else json.loads(bytes(raw)))) + 'FROM {records} WHERE family <> %s {order}'), (list(families), CLOSURE_FAMILY)) + rows = [(family, record_id, None if raw is None else bytes(raw)) for family, record_id, raw in cursor] + for family, record_id, raw in rows: + result.append(dict(family=family, id=record_id, + document=None if raw is None else self._logical(family, raw))) return result def record(self, family, record_id): + if family == CLOSURE_FAMILY: + return None with self._locked(): row = self._execute(self._q('SELECT raw FROM {records} WHERE family=%s AND id=%s'), (family, record_id)).fetchone() - return None if row is None else dict(family=family, id=record_id, document=json.loads(bytes(row[0]))) + return None if row is None else dict(family=family, id=record_id, document=self._logical(family, bytes(row[0]))) def reconcile(self, request_key, request_sha256): _request(request_key, request_sha256) @@ -388,6 +424,8 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256, *, c _identifier(family, 'Record family') if family not in SUPPORTED_FORMATS: _fail('Unsupported project record family.', 'unsupported', 422) + if family == CLOSURE_FAMILY: + _fail('Runtime closures are store-owned records.', 'unsupported', 422) _identifier(identity, 'Record ID') if (family, identity) in candidates: _fail('Duplicate project record identity.') @@ -396,27 +434,60 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256, *, c _domain_format(document.get('format'), family) candidates[(family, identity)] = document replacements, entries, total = {}, {}, 0 + closure_rows = {row[1]: row[2] for row in self._execute(self._q( + 'SELECT family, id, closure_id FROM {records} WHERE closure_id IS NOT NULL'))} + new_closures, row_closures, logical_raw, referenced = {}, {}, {}, set() for identity in sorted(candidates): document = candidates[identity] current = inventory.get(identity) if current is not None and changed is not None and identity not in changed \ - and current[0] == document['format']: + and current[0] in (document['format'], 'project_run_history_v2'): + entries[identity] = current + total += current[1] + if identity[1] in closure_rows and identity[0] == RUN_FAMILY: + referenced.add(closure_rows[identity[1]]) + continue + physical = document + if (identity[0] == RUN_FAMILY and changed is None and current is not None + and current[0] == document['format'] and _sha(_encode(document)) == current[2]): + # An unchanged uncompacted Run keeps its verified revision (as in + # ProjectStore); compact_run_closures migrates explicitly. entries[identity] = current total += current[1] continue - raw = _encode(document) + if identity[0] == RUN_FAMILY: + compacted = compact(document) + if compacted is not None: + physical, closure = compacted + closure_id = physical['runtime_closure']['id'] + new_closures.setdefault(closure_id, closure) + referenced.add(closure_id) + row_closures[identity] = closure_id + logical_raw[identity] = _encode(document) + raw = _encode(physical) if len(raw) > MAX_RECORD_BYTES: _fail('Project records exceed the synchronous byte limit.') digest = _sha(raw) if current is None or current[2] != digest: replacements[identity] = raw - entries[identity] = (document['format'], len(raw), digest) + entries[identity] = (physical['format'], len(raw), digest) total += len(raw) if total > MAX_TOTAL_BYTES: _fail('Project records exceed the synchronous byte limit.') + # Exactly the closures referenced by stored compact Runs are kept. + for closure_id in sorted(referenced): + identity = (CLOSURE_FAMILY, closure_id) + if identity in inventory: + entries[identity] = inventory[identity] + total += inventory[identity][1] + continue + raw = _encode(new_closures[closure_id]) + replacements[identity] = raw + entries[identity] = (new_closures[closure_id]['format'], len(raw), _sha(raw)) + total += len(raw) if total > MAX_TOTAL_BYTES: _fail('Project records exceed the synchronous byte limit.') - records_digest = self._records_digest(replacements, sorted(candidates)) + records_digest = self._records_digest(replacements, sorted(entries)) for receipt in state['receipts']: if receipt['request_key'] == request_key: if receipt['request_sha256'] != request_sha256 or receipt['records_sha256'] != records_digest: @@ -431,12 +502,17 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256, *, c _manifest_raw(manifest) for (family, identity), raw in replacements.items(): fmt, size, digest = entries[(family, identity)] + key = (family, identity) + envelope = (None if family == CLOSURE_FAMILY else + envelope_digest(family, identity, logical_raw.get(key, raw))) self._execute(self._q( - 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256) VALUES (%s,%s,%s,%s,%s,%s,%s) ' + 'INSERT INTO {records} (family, id, format, bytes, sha256, raw, envelope_sha256, closure_id) ' + 'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) ' 'ON CONFLICT (family, id) DO UPDATE SET format=EXCLUDED.format, bytes=EXCLUDED.bytes, ' - 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256'), - (family, identity, fmt, size, digest, raw, envelope_digest(family, identity, raw))) - for family, identity in set(inventory) - set(candidates): + 'sha256=EXCLUDED.sha256, raw=EXCLUDED.raw, envelope_sha256=EXCLUDED.envelope_sha256, ' + 'closure_id=EXCLUDED.closure_id'), + (family, identity, fmt, size, digest, raw, envelope, row_closures.get(key))) + for family, identity in set(inventory) - set(entries): self._execute(self._q('DELETE FROM {records} WHERE family=%s AND id=%s'), (family, identity)) self._write_state(manifest, external_sha256=state['external_sha256'], dirty=True) if self._canonical_digests is not None and replacements: @@ -444,6 +520,22 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256, *, c return dict(project_id=state['project_id'], name=state['name'], checkpoint=manifest['checkpoint'], records=[dict(family=f, id=i, document=candidates[(f, i)]) for f, i in sorted(candidates)]) + def compact_run_closures(self, request_key, request_sha256): + """Format migration: store every compactable Run against a shared closure. + + The logical record set is unchanged (each compaction is checked to expand + to the exact original bytes); only the physical records and checkpoint + change. Returns (previous checkpoint, committed snapshot summary). + """ + with self._locked(): + state = self._state() + uncompacted = {(row[0], row[1]) for row in self._execute(self._q( + "SELECT family, id FROM {records} WHERE family=%s AND format='project_run_history_v1'"), (RUN_FAMILY,))} + snapshot = self.snapshot() + committed = self.commit(snapshot['records'], state['checkpoint'], request_key, request_sha256, + changed=uncompacted) + return state['checkpoint'], {key: committed[key] for key in ('project_id', 'name', 'checkpoint')} + def save(self, expected_checkpoint): """Export the working copy to the external folder (the only folder write).""" _digest(expected_checkpoint, 'Expected checkpoint') diff --git a/tests/test_project_record_dedupe.py b/tests/test_project_record_dedupe.py new file mode 100644 index 00000000..3c8d1760 --- /dev/null +++ b/tests/test_project_record_dedupe.py @@ -0,0 +1,150 @@ +"""Shared runtime closures: compact storage, exact logical Run documents.""" +import copy +import json +from pathlib import Path +import tempfile +import unittest + +import project_record_dedupe as dedupe +import project_store as store_module +from project_store import ProjectStore + + +def runtime(): + files = {f'/opt/py/lib/module{i}.py': dict(bytes=10 + i, sha256=f'{i:064x}') for i in range(40)} + return dict(assumption='fixture', closure_sha256='c' * 64, executable='/opt/py/bin/python', + executable_sha256='e' * 64, files=files, flags=[], profile='fixture', trusted_install_owner_uid=0) + + +def run(identity, *, unique=3): + edges = [dict(blob=dict(bytes=len(path), sha256=item['sha256']), name=f'runtime-files/{n}', population='unknown', + preservation='regenerable', required_for=['scientific_inspection'], role='historical_runtime') + for n, (path, item) in enumerate(sorted(runtime()['files'].items()))] + before = [dict(blob=dict(bytes=n, sha256=f'{n+100:064x}'), name=f'input/{identity}/{n}', population='unknown', + preservation='unique', required_for=['execution_proof'], role='execution_source') for n in range(unique)] + after = [dict(blob=dict(bytes=7, sha256='a' * 64), name='output/model', population='unknown', + preservation='unique', required_for=['inspect'], role='output')] + spec = dict(argv=['python', 'train.py', identity], runtime_files=[dict(path=p, **v) for p, v in sorted(runtime()['files'].items())]) + document = dict(format='project_run_history_v1', origin_catalog_id='a' * 32, admission_proof=None, publication=None, + run=dict(id=identity, spec=spec, material=dict(recipe=dict(id='r'), runtime=runtime()), events=[1.5, -0.0, 'é']), + payloads=before + edges + after) + return dict(family='run_history', id='a' * 32 + '.' + identity, document=document) + + +class CompactionTests(unittest.TestCase): + def test_exact_roundtrip_and_refusals(self): + record = run('one') + compacted, closure = dedupe.compact(record['document']) + self.assertEqual(compacted['format'], dedupe.RUN_V2) + self.assertNotIn('runtime_files', compacted['run']['spec']) + self.assertEqual(store_module._encode(dedupe.expand(compacted, closure)), store_module._encode(record['document'])) + split = copy.deepcopy(record['document']) + split['payloads'].insert(10, split['payloads'].pop(0)) + self.assertIsNone(dedupe.compact(split), 'A non-contiguous runtime block stays uncompacted') + tampered = copy.deepcopy(closure); tampered['runtime']['profile'] = 'other' + with self.assertRaises(store_module.ProjectStoreError): + dedupe.expand(compacted, tampered) + + +class StoreTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-dedupe-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / 'project' + self.store = ProjectStore.create(self.root, 'Dedupe fixture', 'new', 'a' * 64) + self.addCleanup(lambda: self.store.close()) + + def files(self, family): + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + return [json.loads((self.root / '.tuldok' / e['path']).read_bytes()) for e in manifest['inventory'] if e['family'] == family] + + def test_runs_share_one_stored_closure_and_read_back_exactly(self): + records = [run('one'), run('two', unique=5)] + committed = self.store.commit(copy.deepcopy(records), self.store.summary()['checkpoint'], 'runs', 'b' * 64) + self.assertEqual([r['document'] for r in committed['records']], [r['document'] for r in records]) + self.assertEqual([d['format'] for d in self.files('run_history')], [dedupe.RUN_V2, dedupe.RUN_V2]) + self.assertEqual(len(self.files(dedupe.CLOSURE_FAMILY)), 1) + self.assertNotIn(dedupe.CLOSURE_FAMILY, {r['family'] for r in self.store.snapshot()['records']}) + self.store.close() + self.store = ProjectStore.open(self.root) + self.assertEqual(self.store.snapshot(), committed) + # Editing one Run keeps the other's verified revision and the closure. + changed = copy.deepcopy(records); changed[0]['document']['run']['events'].append('edited') + again = self.store.commit(changed, committed['checkpoint'], 'edit', 'c' * 64) + self.assertEqual(again['records'][0]['document'], changed[0]['document']) + self.store.close() + self.store = ProjectStore.open(self.root) + self.assertEqual(self.store.snapshot(), again) + self.assertEqual(len(self.files(dedupe.CLOSURE_FAMILY)), 1) + + def test_closures_are_store_owned(self): + with self.assertRaises(store_module.ProjectStoreError): + self.store.commit([dict(family=dedupe.CLOSURE_FAMILY, id='x', document=dict(format=dedupe.CLOSURE_FORMAT))], + self.store.summary()['checkpoint'], 'forged', 'd' * 64) + + +if __name__ == '__main__': + unittest.main() + + +import os +import uuid + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class WorkingStoreTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-dedupe-working-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / 'project' + with ProjectStore.create(self.root, 'Legacy runs', 'new', 'a' * 64) as store: + records = [run('one'), run('two', unique=4)] + self.legacy = store.commit(copy.deepcopy(records), store.summary()['checkpoint'], 'runs', 'b' * 64) + self.working_id = uuid.uuid4().hex + self.addCleanup(self.drop) + + def drop(self): + import psycopg + from psycopg import sql + from metadata import connection_info + with psycopg.connect(connection_info(None), autocommit=True) as connection: + connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier('working_' + self.working_id))) + + def working(self): + from project_working_store import WorkingProjectStore + parent, leaf = os.path.split(str(self.root)) + descriptor = os.open(parent, store_module.DIR_FLAGS) + try: + folder = ProjectStore.open_folder_at(descriptor, leaf, root_path=parent) + finally: + os.close(descriptor) + return WorkingProjectStore.open(folder, self.working_id) + + def test_migration_keeps_logical_records_and_saves_compact_files(self): + from project_composition import _record_sha + store = self.working() + try: + before = store.snapshot()['records'] + previous, after = store.compact_run_closures('compact-runs', 'c' * 64) + self.assertNotEqual(previous, after['checkpoint']) + self.assertEqual(store.snapshot()['records'], before, 'Logical records are unchanged') + self.assertEqual(store.records('run_history'), [r for r in before if r['family'] == 'run_history']) + self.assertEqual(store.record('run_history', before[0]['id']), before[0]) + view = store.records_view('run_history') + self.assertNotIn(dedupe.CLOSURE_FAMILY, {r['family'] for r in view}) + digests = store.envelope_digests() + for record in before: + self.assertEqual(digests[(record['family'], record['id'])], _record_sha(record)) + # An edit through the session path keeps the shared closure. + changed = copy.deepcopy(before); changed[1]['document']['run']['events'].append('edited') + edited = store.commit(changed, after['checkpoint'], 'edit', 'd' * 64, changed={(changed[1]['family'], changed[1]['id']), }) + store.save(edited['checkpoint']) + saved = store.snapshot() + finally: + store.close() + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + families = [e['family'] for e in manifest['inventory']] + self.assertEqual(families.count(dedupe.CLOSURE_FAMILY), 1) + self.assertEqual({e['format'] for e in manifest['inventory'] if e['family'] == 'run_history'}, {dedupe.RUN_V2}) + with ProjectStore.open(self.root) as reread: + self.assertEqual(reread.snapshot(), saved, 'The file store reads the same logical records') diff --git a/tests/test_project_validation_delta.py b/tests/test_project_validation_delta.py index 38eed517..73dfea16 100644 --- a/tests/test_project_validation_delta.py +++ b/tests/test_project_validation_delta.py @@ -1,5 +1,7 @@ """Edit-scoped validation re-checks exactly what an edit can affect.""" +import os import unittest +from unittest.mock import patch from project_projection import validate_project_records from project_validation_delta import COMPONENTS, ValidationDelta, edit_delta @@ -43,7 +45,14 @@ def test_touched_component_still_rejects_an_invalid_change(self): def test_untouched_component_is_not_revalidated(self): # The base checkpoint is assumed validated; an unrelated edit skips data. - self.validate([BROKEN_TEXT], ValidationDelta(frozenset({('project_file', 'other')}))) + with patch.dict(os.environ, {'TULDOK_VALIDATION_CROSSCHECK': '0'}): + self.validate([BROKEN_TEXT], ValidationDelta(frozenset({('project_file', 'other')}))) + + def test_crosscheck_reports_a_skipped_failure(self): + # Only possible when the 'validated base' assumption is violated, as here. + with patch.dict(os.environ, {'TULDOK_VALIDATION_CROSSCHECK': '1'}): + with self.assertRaisesRegex(AssertionError, 'disagrees'): + self.validate([BROKEN_TEXT], ValidationDelta(frozenset({('project_file', 'other')}))) def test_identity_and_claim_checks_always_run(self): duplicate = [BROKEN_TEXT, dict(BROKEN_TEXT)] From 670abbbc2d26743b88c790ae012e783dc17ad9a9 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 19:19:57 -0700 Subject: [PATCH 12/13] fix(projects): expose working-copy manifest and clean test schemas Native study evidence reads the working copy's checkpoint manifest instead of the folder file, which only changes on Save. Workspace tests drop every registered working schema, and the cold projection test saves before another workspace reads the folder. Co-Authored-By: Claude Opus 5.5 --- project_working_store.py | 18 ++++++++++++++++++ tests/test_project_files_http.py | 4 ++++ tests/test_project_native_study.py | 4 +++- tests/test_project_projection.py | 2 ++ tests/test_project_workspace.py | 2 ++ 5 files changed, 29 insertions(+), 1 deletion(-) diff --git a/project_working_store.py b/project_working_store.py index 10140d74..03d35f5e 100644 --- a/project_working_store.py +++ b/project_working_store.py @@ -19,6 +19,7 @@ from contextlib import contextmanager, nullcontext import json import os +from pathlib import Path import re import threading import uuid @@ -312,6 +313,11 @@ def summary(self): state = self._state() return {key: state[key] for key in ('project_id', 'name', 'checkpoint')} + def manifest_raw(self): + """Canonical project.json bytes of the working copy's current checkpoint.""" + with self._locked(): + return _manifest_raw(self._manifest(self._state(), self._inventory())) + def dirty(self): """Whether the working copy has changes not yet saved to the folder.""" with self._locked(): @@ -614,3 +620,15 @@ def __enter__(self): def __exit__(self, *exc): self.close() + +def registered_schemas(control_path): + """Working-copy schemas registered by a workspace control directory.""" + schemas = [] + for path in sorted(Path(control_path).glob('.project-working/*.json')): + try: + identity = json.loads(path.read_bytes()).get('working_id') + except (OSError, ValueError, AttributeError): + continue + if type(identity) is str and WORKING_ID.fullmatch(identity): + schemas.append('working_' + identity) + return schemas diff --git a/tests/test_project_files_http.py b/tests/test_project_files_http.py index b592014a..edef4f68 100644 --- a/tests/test_project_files_http.py +++ b/tests/test_project_files_http.py @@ -73,7 +73,11 @@ def cleanup_owned(self): with psycopg.connect(connection_info(dsn), autocommit=True) as connection: connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(schema))) # Projection manifests are physical identities, distinct from logical project IDs. + from project_working_store import registered_schemas for dsn, control in self.controls: + for schema in registered_schemas(control): + with psycopg.connect(connection_info(dsn), autocommit=True) as connection: + connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(schema))) for manifest in control.rglob('.tuldok.json'): value = json.loads(manifest.read_bytes()) schema = ('projection_' + value['projection_id'] if value['version'] == 2 diff --git a/tests/test_project_native_study.py b/tests/test_project_native_study.py index 282837ab..969c6e3b 100644 --- a/tests/test_project_native_study.py +++ b/tests/test_project_native_study.py @@ -85,7 +85,9 @@ def retain_project_evidence(projection, label): store=projection.session.store with projection.session.access.lock,projection.session.lock,projection.db.lock: snapshot=store.snapshot() - manifest=(store.path/'.tuldok/project.json').read_bytes() + # A database working copy's checkpoint reaches the folder only on Save. + manifest=(store.manifest_raw() if callable(getattr(store,'manifest_raw',None)) + else (store.path/'.tuldok/project.json').read_bytes()) if json.loads(manifest)['checkpoint']!=snapshot['checkpoint']: raise AssertionError('Native fixture snapshot and manifest checkpoints differ') output=output/(label+'-'+snapshot['project_id']+'-'+snapshot['checkpoint']) diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index 3f1e3e98..377b2767 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -76,6 +76,8 @@ def test_mixed_cold_projection_preserves_notes_and_definitions_without_worker_co preprocessing_policy={'kind': 'identity'}, output_policy={'kind': 'bytes'}) model = p.materials.mutate_definition('model_definition', 'models.create', None, body, p.session.snapshot['checkpoint'], 'model-create', lambda: p.models.create(body)) + # Another workspace sees edits only after they are saved to the folder. + self.call('save', self.mutation()) snapshot = self.host._store.snapshot() canonical = (self.external / 'project' / '.tuldok' / 'project.json').read_bytes() sentinel = self.external / 'project' / 'must-not-run' diff --git a/tests/test_project_workspace.py b/tests/test_project_workspace.py index 26d3f662..aa0b5c4d 100644 --- a/tests/test_project_workspace.py +++ b/tests/test_project_workspace.py @@ -49,6 +49,8 @@ def cleanup_owned(self): value = json.loads(manifest.read_bytes()) schemas.add('projection_' + value['projection_id'] if value['version'] == 2 else 'catalog_' + value['dataset_id']) + from project_working_store import registered_schemas + schemas.update(registered_schemas(control)) with psycopg.connect(connection_info(), autocommit=True) as connection: for schema in schemas: connection.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(schema))) From a9972b0eb05bc53886f2be0b6e7953a130a9d5fd Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 2 Nov 2026 20:54:26 -0700 Subject: [PATCH 13/13] test(projects): save before reading the project folder Edits stay in the working copy until Save, so the owner-MCP, runtime and native browser proofs save before snapshotting or cloning the folder. The owner-MCP Save As check now compares against the saved source. Co-Authored-By: Claude Opus 5.5 --- tests/browser_project_files_m3.cjs | 2 ++ tests/browser_project_files_native.cjs | 2 ++ tests/test_project_files_owner_mcp.py | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/browser_project_files_m3.cjs b/tests/browser_project_files_m3.cjs index 3f12ed69..5fc1a735 100644 --- a/tests/browser_project_files_m3.cjs +++ b/tests/browser_project_files_m3.cjs @@ -103,6 +103,8 @@ for name in ('torch','numpy'): await activate('#project-runtime-status-form button');await settled();const historical=JSON.parse(await evaluate('document.getElementById("project-runtime-status").textContent'));assert.equal(historical.state,'captured');assert.deepEqual(historical.result,captured); assert.ok(!fs.existsSync(sentinel));assert.ok(!fs.existsSync(audit),'Application never attempted subprocess/exec/install'); check('Fresh explicit confirmed capture copies offline bytes and retains intent/result; no child process or package import executes'); + // Edits reach the project folder on Save; Close keeps them only in the working copy. + await activate('#project-files-save');await settled(); await activate('#project-files-close');await until(async()=>(await state()).state==='no_project_open','close captured project');await stop(server); const source=path.join(external,'alpha'),coldRoot=path.join(temporary,'cold-external');fs.mkdirSync(coldRoot);const clone=path.join(coldRoot,'clone'),canonical=snapshot(source); const git=(...args)=>execFileSync('git',args,{env:gitEnv,stdio:['ignore','pipe','pipe']});git('init','-q','--template=',source);git('-C',source,'add','.');git('-C',source,'-c','user.name=Disposable M3 QA','-c','user.email=qa@example.invalid','-c','commit.gpgsign=false','-c','core.hooksPath=/dev/null','commit','-qm','Inert canonical runtime facts');git('-c','core.hooksPath=/dev/null','clone','-q','--no-hardlinks','--template=',source,clone);if(fs.existsSync(path.join(source,'artifacts')))fs.cpSync(path.join(source,'artifacts'),path.join(clone,'artifacts'),{recursive:true});assert.deepEqual(snapshot(clone),canonical); diff --git a/tests/browser_project_files_native.cjs b/tests/browser_project_files_native.cjs index a4297d5b..b9a978af 100644 --- a/tests/browser_project_files_native.cjs +++ b/tests/browser_project_files_native.cjs @@ -85,6 +85,8 @@ function check(value){checks.push(value);console.log('PASS '+value);} facts=await fixture.command('facts');assert.equal(facts.go_events.length,1);assert.equal(facts.pending,false);assert.equal(facts.available,true);assert.equal(facts.workspace_runs,0); await activate('#project-files-discard');await activate('#project-files-refresh');await settled();sourceFacts=await fixture.command('facts'); check('Named retry refuses absent current Run; real deferrable SQL fault after canonical GO intent sends zero GO, then keyboard health phase selection and explicit reconciliation restore availability without execution'); + // Edits reach the project folder on Save; Close keeps them only in the working copy. + await activate('#project-files-save');await settled(); const source=path.join(sourceHost.ready.external,'alpha'),canonical=snapshot(source),saved=(await state()).project; await activate('#project-files-close');await until(async()=>(await state()).state==='no_project_open','close before cold clone');await sourceHost.stop(); const coldHostRoot=path.join(temporary,'cold-host'),external=path.join(coldHostRoot,'external');fs.mkdirSync(external,{recursive:true});const clone=path.join(external,'clone'); diff --git a/tests/test_project_files_owner_mcp.py b/tests/test_project_files_owner_mcp.py index 1e3d74ca..95094583 100644 --- a/tests/test_project_files_owner_mcp.py +++ b/tests/test_project_files_owner_mcp.py @@ -178,9 +178,10 @@ async def exercise(): imported=await self.tool(session,'import_text',f.envelope(opened,asset=dict( text=http_fixture.TEXT,name='Actual owner MCP authored observation',groups=['owner-mcp'],parents=[],rights='Owned synthetic fixture'))) state=await self.tool(session,'status',{});source_id=state['project']['project_id'] - source_before=http_fixture.canonical_bytes(f.external/'owner-source') saved=await self.tool(session,'save',f.envelope(state)) self.assertEqual(saved['project'],state['project']) + # Edits reach the folder on Save; Save As must leave the saved source exact. + source_before=http_fixture.canonical_bytes(f.external/'owner-source') fork=await self.tool(session,'save_as',f.envelope(saved,root='fixture',directory='owner-fork',name='Independent MCP Ω')) current=fork['state'];self.assertNotEqual(current['project']['project_id'],source_id) self.assertEqual(http_fixture.canonical_bytes(f.external/'owner-source'),source_before)