From 84960fb702e4c0f9eae1fc2fdce5e0791fec358a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 12:53:20 -0700 Subject: [PATCH 01/14] fix: preserve artifact policies before combining project roots --- project_composition.py | 17 ++++++++++++++++- tests/test_project_composition.py | 13 +++++++++++++ tests/test_project_composition_publication.py | 17 +++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/project_composition.py b/project_composition.py index bcb8a795..53bc04f3 100644 --- a/project_composition.py +++ b/project_composition.py @@ -268,8 +268,23 @@ def allocate(family,identity,revision): records=[indexed[key] for key in sorted(indexed)] validate_source_memberships(records,snapshots,destination_project_id) archived={(r['family'],r['id']) for r in merged['archived']+fork['archived']} + # Every input has its own valid current fork. The combined records receive + # their one new current overlay only in the publication owner. Ask each + # source owner for its preservation policy before combining those roots; + # independent editable forks introduce no new payload identities. + plans={} + inventories=[fork['artifacts']] + inventories.extend(artifact_plan(source['records']) for source in snapshots[1:]) + for inventory in inventories: + for edge in inventory: + sha=edge['declaration']['sha256'];previous=plans.get(sha) + if previous is not None and previous['declaration']!=edge['declaration']: + _fail('Composition source artifact declarations conflict.') + if previous is None or edge['preservation']=='unique': + plans[sha]=copy.deepcopy(edge) return dict(records=records,mapping=copy.deepcopy(fork['mapping']), - archived=[dict(family=f,id=i) for f,i in sorted(archived)],artifacts=artifact_plan(records)) + archived=[dict(family=f,id=i) for f,i in sorted(archived)], + artifacts=[plans[sha] for sha in sorted(plans)]) def publish_composition(primary_store, primary_artifacts, retained_sources, trusted_root_fd, directory, diff --git a/tests/test_project_composition.py b/tests/test_project_composition.py index cf42c0d3..c20a90df 100644 --- a/tests/test_project_composition.py +++ b/tests/test_project_composition.py @@ -99,6 +99,19 @@ def test_other_family_conflict_and_repeated_source_project_are_rejected(self): self.assertEqual(repeated.exception.code,'conflict') self.assertIn('source project identity repeats',str(repeated.exception)) + def test_source_unique_policy_overrides_another_sources_runtime_dependency(self): + from project_composition import build_candidate + from test_project_fork_composition import runtime_records + import uuid + sha=hashlib.sha256(b'Z').hexdigest() + runtime,_,_=runtime_records('9'*32,shared_sha=sha,shared_bytes=1) + declaration=next(r for r in runtime if r['family']=='project_artifact' and r['id']==sha) + candidate=build_candidate([snapshot('1',runtime),snapshot('2',[copy.deepcopy(declaration)])], + '3'*32,created_at='2026-10-24T12:00:00+00:00',id_factory=lambda *_:uuid.uuid4().hex, + directory='combined',name='Conservative source custody',request_key='shared-policy') + policies={edge['declaration']['sha256']:edge['preservation'] for edge in candidate['artifacts']} + self.assertEqual(policies[sha],'unique') + def test_source_snapshots_can_be_loaded_lazily(self): records=approved_history() def sources(): diff --git a/tests/test_project_composition_publication.py b/tests/test_project_composition_publication.py index 67764875..0bb6d881 100644 --- a/tests/test_project_composition_publication.py +++ b/tests/test_project_composition_publication.py @@ -73,6 +73,23 @@ def test_complete_publication_preserves_both_sources_and_independent_payloads(se self.assertEqual(target.stat().st_nlink,1) self.assertEqual([s.snapshot() for s in self.stores],before) + def test_already_forked_sources_keep_their_independent_histories(self): + for number in range(2): + result=publish_fork(self.stores[number],self.artifacts[number],self.fd,'forked-source-'+str(number), + root_path=self.root,name='Already forked source',expected_checkpoint=self.snapshots[number]['checkpoint'], + request_key='prepare-fork-'+str(number),quiescent=True, + build_candidate=lambda snap,dest:fork_candidate(snap,dest,created_at=NOW,id_factory=lambda *_:uuid.uuid4().hex), + validate_candidate=lambda records,identity:validate(records,identity,None)) + store=ProjectStore.open(self.root/('forked-source-'+str(number))) + artifacts=ProjectArtifacts(store);self.addCleanup(store.close);self.addCleanup(artifacts.close) + self.stores[number]=store;self.artifacts[number]=artifacts;self.snapshots[number]=result['snapshot'] + source_overlays=[next(r for r in s['records'] if r['family']=='project_fork') for s in self.snapshots] + result=self.publish();records=result['snapshot']['records'];identity=result['snapshot']['project_id'] + self.assertTrue(all(old in records for old in source_overlays)) + self.assertTrue(all(dict(family='project_fork',id=old['id']) in result['fork']['archived'] for old in source_overlays)) + self.assertEqual(decode_fork(records,project_id=identity)['destination_project_id'],identity) + self.assertIn(identity,decode_records(records,project_id=identity)) + def test_missing_unique_secondary_bytes_leave_only_retained_staging(self): (self.stores[1].path/'artifacts/sha256'/self.unique[1]).unlink() self.assert_code('unavailable',self.publish) From 41974aea385b1e496542b1278eb0af8edce8769e Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 15:06:34 -0700 Subject: [PATCH 02/14] feat: recover committed project publication without payload copies --- project_files.py | 43 ++++---- project_fork.py | 92 ++++++++++++++++ project_projection.py | 107 ++++++++++--------- tests/test_project_publication_recovery.py | 116 +++++++++++++++++++++ 4 files changed, 289 insertions(+), 69 deletions(-) create mode 100644 tests/test_project_publication_recovery.py diff --git a/project_files.py b/project_files.py index 08829a6a..c312aa34 100644 --- a/project_files.py +++ b/project_files.py @@ -231,6 +231,30 @@ def _read(root,parts): os.close(parent) +def inspect_captured(records, project_id, root): + """Observe captured working files using only a locked project capability.""" + owned=[r for r in decode_records(records) if r['document']['project_id']==project_id] + items=[] + for envelope in sorted(owned,key=lambda r:r['document']['path']): + doc=envelope['document'];value=doc['current'] + observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], + expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) + try: + parts=_path(doc['kind'],doc['path']) + if value.get('deleted') is True and _missing(root,parts): + observation['state']='deleted' + else: + if value.get('deleted') is True:observation['state']='changed' + _,observed,size=_read(root,parts) + observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') + except _WorkingError as error: + if value.get('deleted') is True: + observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) + else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) + items.append(observation) + return items + + class ProjectFiles: def __init__(self,session): self.session,self.access,self.db,self.store=session,session.access,session.db,session.store @@ -238,25 +262,8 @@ def __init__(self,session): def inspect(self): with self.access.lock,self.session.lock,self.db.lock: self.access.require('inspect','files.inspect');snapshot=self.session.check() - owned=[r for r in decode_records(snapshot['records']) if r['document']['project_id']==snapshot['project_id']];items=[] with self.store.root_directory() as root: - for envelope in sorted(owned,key=lambda r:r['document']['path']): - doc=envelope['document'];value=doc['current'] - observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], - expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) - try: - parts=_path(doc['kind'],doc['path']) - if value.get('deleted') is True and _missing(root,parts): - observation['state']='deleted' - else: - if value.get('deleted') is True:observation['state']='changed' - _,observed,size=_read(root,parts) - observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') - except _WorkingError as error: - if value.get('deleted') is True: - observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) - else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) - items.append(observation) + items=inspect_captured(snapshot['records'],snapshot['project_id'],root) self.session.check() return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) diff --git a/project_fork.py b/project_fork.py index 7aec1a44..bd928ad9 100644 --- a/project_fork.py +++ b/project_fork.py @@ -279,6 +279,98 @@ def publish_fork(source_store, source_artifacts, trusted_root_fd, relative, *, r validate_candidate=lambda records,identity,_artifacts:validate_candidate(records,identity)) +def recover_committed_publication(trusted_root_fd, staging, relative, *, root_path, + expected_project_id, expected_checkpoint, + expected_request, quiescent): + """Publish one explicitly selected committed stage without rebuilding it. + + This operator recovery accepts a frozen historical snapshot, not a new read + of its original sources. The caller owns current filesystem authority and + cessation of the former process. The store's exclusive root lock, complete + domain validation, exact original request/commit receipt, payload custody, + and clean captured files are required before a no-overwrite rename. No SQL + projection, source permission, source runtime or worker is restored here. + After an uncertain rename, repeat this exact recovery to inspect the final + destination; never repeat composition or create another stage. + """ + if quiescent is not True: + _fail('Recovery requires proven cessation of the former publication owner.', 'busy', 409) + _uuid(expected_project_id);_sha(expected_checkpoint) + stage_parts=_relative_parts(staging);parts=_relative_parts(relative) + if (stage_parts[:-1]!=parts[:-1] + or not re.fullmatch(r'\.tuldok-fork-[a-f0-9]{32}',stage_parts[-1]) + or stage_parts==parts): + _fail('Recovery requires an explicit fork stage beside its original destination.') + if type(expected_request) is not dict or expected_request.get('directory')!=relative: + _fail('Recovery requires the exact original publication request.') + key=expected_request.get('request_key') + if type(key) is not str or not REQUEST_KEY.fullmatch(key): + _fail('Recovery requires the original publication request key.') + digest=_digest(expected_request) + published=False + try: + with anchored_directory(trusted_root_fd,'/'.join(parts[:-1]),root_path=root_path) as parent: + def exists(name): + try:os.stat(name,dir_fd=parent,follow_symlinks=False);return True + except FileNotFoundError:return False + stage_exists,final_exists=exists(stage_parts[-1]),exists(parts[-1]) + if stage_exists and final_exists: + _fail('Both stage and destination exist; neither was replaced.', 'conflict', 409) + if not stage_exists and not final_exists: + _fail('Neither the selected stage nor its destination is available.', 'unavailable', 409) + selected=relative if final_exists else staging + with ProjectStore.open_at(trusted_root_fd,selected,root_path=root_path) as store: + snapshot=store.snapshot() + if (snapshot['project_id']!=expected_project_id + or snapshot['checkpoint']!=expected_checkpoint): + _fail('Recovery project identity or checkpoint differs.', 'stale', 409) + overlay=decode_fork(snapshot['records'],project_id=expected_project_id) + if overlay is None:_fail('Recovery requires a committed independent publication.') + from project_composition import decode_records, composition_input + composition=decode_records(snapshot['records'],project_id=expected_project_id).get(expected_project_id) + observed=(composition_input(composition['sources'],composition['directory'],composition['name'],composition['request_key']) + if composition is not None else _input(overlay['source'],overlay['directory'],overlay['name'],overlay['request_key'])) + if observed!=expected_request or store.reconcile(key,digest) is None: + _fail('Committed stage does not match the original request and receipt.', 'conflict', 409) + from project_projection import validate_project_records + from project_files import inspect_captured + declarations=_declarations(snapshot['records']) + with ProjectArtifacts(store) as artifacts: + validate_project_records(snapshot['records'],project_id=expected_project_id,artifacts=artifacts) + checks=artifacts.status_many([declarations[identity] for identity in overlay['artifacts']['copied']]) + if any(item['state']!='verified' for item in checks): + _fail('Committed publication payloads are missing or corrupt.', 'unavailable', 409) + with store.root_directory() as project_root: + if any(item['state'] not in ('unchanged','deleted') + for item in inspect_captured(snapshot['records'],expected_project_id,project_root)): + _fail('Committed stage has changed or missing working files.', 'working_files_changed', 409) + store.save(expected_checkpoint) + if not final_exists: + # Recheck the selected directory entry while the store root + # remains locked; an external rename must not substitute it. + with store.root_directory() as project_root: + entry=os.stat(stage_parts[-1],dir_fd=parent,follow_symlinks=False) + actual=os.fstat(project_root) + if (entry.st_dev,entry.st_ino)!=(actual.st_dev,actual.st_ino): + _fail('Selected stage directory changed.', 'stale', 409) + _rename_noreplace(parent,stage_parts[-1],parts[-1]);published=True + os.fsync(parent) + return dict(project_id=expected_project_id,checkpoint=expected_checkpoint, + directory=relative,request_key=key,reconciled=final_exists, + payloads_copied=0,selected=False) + except BaseException as error: + # Anchored directory owners also translate IO failures. Preserve the + # uncertain post-rename outcome through either exception representation. + if published: + raise ProjectForkError('Committed publication was renamed but acknowledgement is uncertain; reconcile this exact recovery.', + 'unknown',503) from error + if isinstance(error,OSError): + code='conflict' if error.errno==errno.EEXIST else 'unavailable' + raise ProjectForkError('Committed publication recovery is unavailable; no stage was rebuilt.', + code,503 if code!='conflict' else 409) from error + raise + + def _publish_project(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, name, expected_checkpoint, request_key, quiescent, build_candidate, validate_candidate, retained_sources=()): diff --git a/project_projection.py b/project_projection.py index fca264be..1181c457 100644 --- a/project_projection.py +++ b/project_projection.py @@ -58,6 +58,59 @@ def validate_artifact_resolution(operation, result, records): return True +def validate_project_records(records, *, project_id, artifacts): + """Validate one inert candidate through every owning domain, without SQL reconstruction.""" + claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( + MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) + identities = set() + declarations = {} + for item in records: + identity = (item['family'], item['id']) + if item['family'] not in claimed or identity in identities: + raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) + identities.add(identity) + if item['family'] == 'project_artifact': + declaration = validate_declaration(item['document']) + if declaration['sha256'] != item['id']: + raise WorkbenchError('Artifact inventory identity differs.') + declarations[item['id']] = declaration + decode_data_records(records) + from project_fork import decode_fork + decode_fork(records, project_id=project_id) + from project_composition import decode_records as decode_compositions + decode_compositions(records, project_id=project_id) + from project_transfer import decode_records as decode_extraction_records + decode_extraction_records(records) + operation_records(records, project_id=project_id) + decode_file_records(records) + decode_native_records(records,artifacts=artifacts) + materials = MaterialSnapshot(records) + for blob in materials.artifact_blobs(): + if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): + raise WorkbenchError('Material artifact declaration is missing or inconsistent.') + # Resolve every cross-owner reference against THIS candidate, rather + # than a resolver that still sees the preceding committed checkpoint. + decode_producer_records(records, artifacts=artifacts) + decode_history_records(records, materials=materials, artifacts=artifacts) + decode_measurement_records(records, artifacts=artifacts) + for receipt in operation_documents(records): + operation, result = receipt['operation'], receipt['result'] + if (validate_data_result(operation, result, records) + or validate_material_result(operation, result, records) + or validate_history_result(operation, result, records) + or validate_source_result(operation, result, records) + or validate_measurement_result(operation, result, records) + or validate_execution_result(operation, result, records) + or validate_producer_result(operation, receipt['target'], result, records) + or validate_file_result(operation, result, records) + or validate_study_execution_result(operation, result, records) + or validate_runtime_result(operation, result, records) + or validate_preparation_result(operation, result, records) + or validate_artifact_resolution(operation, result, records)): + continue + raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + + class ProjectProjection: def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None): self.store, self.db, self.access = store, db, access @@ -127,57 +180,9 @@ def execute(self, *args, **kwargs): return self.db.execute(*args, **kwargs) def validate_records(self, records, *, project_id=None, artifacts=None): - artifacts = self.artifacts if artifacts is None else artifacts - project_id = self.project_id if project_id is None else project_id - claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( - MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) - identities = set() - declarations = {} - for item in records: - identity = (item['family'], item['id']) - if item['family'] not in claimed or identity in identities: - raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) - identities.add(identity) - if item['family'] == 'project_artifact': - declaration = validate_declaration(item['document']) - if declaration['sha256'] != item['id']: - raise WorkbenchError('Artifact inventory identity differs.') - declarations[item['id']] = declaration - decode_data_records(records) - from project_fork import decode_fork - decode_fork(records, project_id=project_id) - from project_composition import decode_records as decode_compositions - decode_compositions(records, project_id=project_id) - from project_transfer import decode_records as decode_extraction_records - decode_extraction_records(records) - operation_records(records, project_id=project_id) - decode_file_records(records) - decode_native_records(records,artifacts=artifacts) - materials = MaterialSnapshot(records) - for blob in materials.artifact_blobs(): - if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): - raise WorkbenchError('Material artifact declaration is missing or inconsistent.') - # Resolve every cross-owner reference against THIS candidate, rather - # than a resolver that still sees the preceding committed checkpoint. - decode_producer_records(records, artifacts=artifacts) - decode_history_records(records, materials=materials, artifacts=artifacts) - decode_measurement_records(records, artifacts=artifacts) - for receipt in operation_documents(records): - operation, result = receipt['operation'], receipt['result'] - if (validate_data_result(operation, result, records) - or validate_material_result(operation, result, records) - or validate_history_result(operation, result, records) - or validate_source_result(operation, result, records) - or validate_measurement_result(operation, result, records) - or validate_execution_result(operation, result, records) - or validate_producer_result(operation, receipt['target'], result, records) - or validate_file_result(operation, result, records) - or validate_study_execution_result(operation, result, records) - or validate_runtime_result(operation, result, records) - or validate_preparation_result(operation, result, records) - or validate_artifact_resolution(operation, result, records)): - continue - raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + return validate_project_records(records, + project_id=self.project_id if project_id is None else project_id, + artifacts=self.artifacts if artifacts is None else artifacts) def resolve_artifact(self, body, expected_checkpoint, key): """Explicitly supply exact bytes from a configured current local root.""" diff --git a/tests/test_project_publication_recovery.py b/tests/test_project_publication_recovery.py new file mode 100644 index 00000000..36067a6b --- /dev/null +++ b/tests/test_project_publication_recovery.py @@ -0,0 +1,116 @@ +"""Real committed-stage recovery preserves bytes and never reconstructs sources.""" +import errno +import os +import shutil +import unittest +from unittest.mock import patch + +from project_composition import composition_input +from project_fork import recover_committed_publication +from project_store import ProjectStore, ProjectStoreError +from project_files import file_id, materialize_captured +from test_project_composition_publication import CompositionPublicationTests + + +class PublicationRecoveryTests(unittest.TestCase): + setUp=CompositionPublicationTests.setUp + publish=CompositionPublicationTests.publish + assert_code=CompositionPublicationTests.assert_code + + def stage(self): + with patch('project_fork._rename_noreplace',side_effect=OSError(errno.EIO,'interrupted before rename')): + self.assert_code('unavailable',self.publish) + self.staging=next(self.root.glob('.tuldok-fork-*')) + with ProjectStore.open(self.staging) as store: + self.committed=store.snapshot() + self.request=composition_input(self.snapshots,'combined','Combined','compose-exact') + + def recover(self,**changes): + args=dict(root_path=self.root,expected_project_id=self.committed['project_id'], + expected_checkpoint=self.committed['checkpoint'],expected_request=self.request,quiescent=True) + args.update(changes) + return recover_committed_publication(self.fd,self.staging.name,'combined',**args) + + def test_committed_stage_recovers_without_sources_payload_copies_or_new_identity(self): + self.stage() + before={p.name:(p.read_bytes(),p.stat().st_ino) for p in (self.staging/'artifacts/sha256').iterdir()} + for store,artifacts in zip(self.stores,self.artifacts): + artifacts.close();store.close();shutil.rmtree(store.path) + with patch('project_artifacts.ProjectArtifacts.copy_many_to',side_effect=AssertionError('No recopy')), \ + patch('project_composition.build_candidate',side_effect=AssertionError('No rebuild')): + result=self.recover() + self.assertFalse(self.staging.exists()) + self.assertEqual(result['payloads_copied'],0) + self.assertFalse(result['selected']) + with ProjectStore.open(self.root/'combined') as store: + self.assertEqual(store.snapshot(),self.committed) + after={p.name:(p.read_bytes(),p.stat().st_ino) for p in (self.root/'combined/artifacts/sha256').iterdir()} + self.assertEqual(before,after) + self.assertTrue(self.recover()['reconciled']) + + def test_wrong_request_checkpoint_or_identity_never_publishes(self): + self.stage() + self.assert_code('stale',lambda:self.recover(expected_checkpoint='0'*64)) + self.assert_code('stale',lambda:self.recover(expected_project_id='0'*32)) + self.assert_code('conflict',lambda:self.recover(expected_request={**self.request,'name':'Changed'})) + self.assert_code('busy',lambda:self.recover(quiescent=False)) + self.assertTrue(self.staging.exists());self.assertFalse((self.root/'combined').exists()) + + def test_missing_copied_payload_preserves_stage_and_refuses_publication(self): + self.stage();next((self.staging/'artifacts/sha256').iterdir()).unlink() + with self.assertRaises(ProjectStoreError):self.recover() + self.assertTrue(self.staging.exists());self.assertFalse((self.root/'combined').exists()) + + def test_destination_appearance_never_overwrites_either_tree(self): + self.stage();(self.root/'combined').mkdir();(self.root/'combined/unique').write_bytes(b'keep') + self.assert_code('conflict',self.recover) + self.assertTrue(self.staging.exists());self.assertEqual((self.root/'combined/unique').read_bytes(),b'keep') + + def test_lost_rename_ack_reconciles_exact_published_checkpoint(self): + self.stage();fsync=os.fsync + def fail_after_publish(fd): + if (self.root/'combined').exists():raise OSError(errno.EIO,'lost publication acknowledgement') + fsync(fd) + with patch('project_fork.os.fsync',side_effect=fail_after_publish): + self.assert_code('unknown',self.recover) + self.assertFalse(self.staging.exists());self.assertTrue(self.recover()['reconciled']) + + def test_live_writer_and_non_stage_path_are_refused(self): + self.stage() + with ProjectStore.open(self.staging): + self.assert_code('busy',self.recover) + self.assert_code('invalid',lambda:recover_committed_publication(self.fd,'source0','combined', + root_path=self.root,expected_project_id=self.committed['project_id'],expected_checkpoint=self.committed['checkpoint'], + expected_request=self.request,quiescent=True)) + + def test_initial_empty_stage_cannot_be_mistaken_for_committed_publication(self): + self.stage();empty_name='.tuldok-fork-'+'9'*32 + with ProjectStore.create(self.root/empty_name,'Uncommitted','initial','a'*64) as store: + initial=store.snapshot() + self.assert_code('invalid',lambda:recover_committed_publication(self.fd,empty_name,'combined', + root_path=self.root,expected_project_id=initial['project_id'],expected_checkpoint=initial['checkpoint'], + expected_request=self.request,quiescent=True)) + self.assertTrue((self.root/empty_name).exists());self.assertFalse((self.root/'combined').exists()) + + def test_missing_captured_working_file_refuses_without_changing_committed_history(self): + # Seed a real current note before composing; its cloned editable record + # and materialized bytes then pass through the normal publication path. + from datetime import datetime, timezone + import hashlib + store=self.stores[0];snap=store.snapshot();raw=b'original note\n' + version=dict(revision=1,text=raw.decode(),sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw), + actor_id='1'*32,adopted_at=datetime.now(timezone.utc).isoformat()) + doc=dict(format='project_file_v1',project_id=snap['project_id'],kind='note',path='notes/test.md', + current=version,history=[version],origin=None) + record=dict(family='project_file',id=file_id(snap['project_id'],'note','notes/test.md'),document=doc) + self.snapshots[0]=store.commit([*snap['records'],record],snap['checkpoint'],'note','c'*64) + with store.root_directory() as root: + materialize_captured(self.snapshots[0]['records'],snap['project_id'],root) + self.stage();(self.staging/'notes/test.md').unlink() + self.assert_code('working_files_changed',self.recover) + self.assertFalse((self.root/'combined').exists()) + with ProjectStore.open(self.staging) as retained: + self.assertEqual(retained.snapshot(),self.committed) + + +if __name__=='__main__':unittest.main() From 65f6caf022c5b450cd787c3de17f24c7b860844c Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 16:31:42 -0700 Subject: [PATCH 03/14] feat(measurements): admit larger complete comparisons through API v2 Separate combined comparison capacity from the per-packet budget so full-state populations remain complete. Preserve existing public contract hashes and reject oversized views under the selected legacy version. --- agent_api.py | 13 +++++++++++++ project_workspace.py | 4 ++++ retained_measurements.py | 5 +++-- tests/test_measurement_comparison.py | 11 ++++++++++- tests/test_project_measurement_comparison_http.py | 15 +++++++++++++++ tests/test_project_measurements_clients.py | 10 ++++++++++ 6 files changed, 55 insertions(+), 3 deletions(-) diff --git a/agent_api.py b/agent_api.py index 67b99453..ed25af09 100644 --- a/agent_api.py +++ b/agent_api.py @@ -1471,6 +1471,13 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non _LOOKUP = {key: operation for operation in _OPERATIONS for key in (operation['operation_id'], operation['mcp_name'])} # Explicit locally implemented versions. No remotely supplied code or routes. _CONTRACTS = {op['operation_id']: {1: copy.deepcopy(op)} for op in _OPERATIONS} +# Keep the installed v1 bounds/hash exact. Version 2 admits complete larger +# comparisons; each individual packet and every response page retain their caps. +_comparison_capacity=copy.deepcopy(_CONTRACTS['project_files.measurement_compare'][1]) +_comparison_value=_comparison_capacity['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties'] +_comparison_value['observation_count']=integer(1,250000) +_comparison_value['query']['oneOf'][0]['properties']['total']=integer(0,250000) +_CONTRACTS['project_files.measurement_compare'][2]=_comparison_capacity # v1 remains the closed Run-only current custody view. v2 additionally exposes # bounded original local native phases; imported history never populates it. _native_health=copy.deepcopy(_CONTRACTS['project_files.execution_health'][1]) @@ -1974,6 +1981,12 @@ def operation(name, version=None): return copy.deepcopy(versions[max(versions) if version is None else version]) +def measurement_comparison_row_limit(version=None): + """Return the selected immutable public contract's comparison budget.""" + selected=operation('project_files.measurement_compare',version) + return selected['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties']['observation_count']['maximum'] + + def _native_validation_result_ok(value, revision): """Check the finite public truth fields without interpreting native runtime data. diff --git a/project_workspace.py b/project_workspace.py index efe93f9a..072d6c24 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -509,6 +509,10 @@ def dispatch(self, name, arguments, *, selected_version=None): from project_dispatch import dispatch try: value = dispatch(self._projection, name, arguments) + if (name=='project_files.measurement_compare' + and value['value']['observation_count']>agent_api.measurement_comparison_row_limit(selected_version)): + raise WorkbenchError('Comparison exceeds the selected API version observation budget. Select comparison contract version 2 for larger complete populations.', + 'too_large',413) if name == 'project_files.execution_health' and selected_version == 1: value = {key: item for key, item in value.items() if key != 'native_phases'} except BaseException as error: diff --git a/retained_measurements.py b/retained_measurements.py index 796aaa19..4044b017 100644 --- a/retained_measurements.py +++ b/retained_measurements.py @@ -26,6 +26,7 @@ MAX_COMPARISON_SOURCES = 16 MAX_COMPARISON_SUBJECTS = 256 MAX_COMPARISON_BYTES = 128 * 1024 * 1024 +MAX_COMPARISON_ROWS = 250000 STATES = ('measured', 'failed', 'unsupported', 'unavailable') ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') SHA = re.compile(r'^[a-f0-9]{64}$') @@ -411,8 +412,8 @@ def __init__(self, tables): fail('Comparison sources overlap a subject identity.', 'conflict', 409) self._subjects[subject['id']] = dict(copy.deepcopy(subject), source=copy.deepcopy(locator)) observations += metadata['observation_count'] - if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_ROWS: - fail('Comparison exceeds 256 subjects or 100,000 observations.', 'too_large', 413) + if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_COMPARISON_ROWS: + fail('Comparison exceeds 256 subjects or 250,000 observations.', 'too_large', 413) self._tables.append((copy.deepcopy(locator), table)) self._header.update(subjects=list(self._subjects.values()), observation_count=observations) diff --git a/tests/test_measurement_comparison.py b/tests/test_measurement_comparison.py index b289d751..16418875 100644 --- a/tests/test_measurement_comparison.py +++ b/tests/test_measurement_comparison.py @@ -91,7 +91,7 @@ def test_overlapping_subjects_and_repeated_sources_are_explicit_errors(self): self.assertEqual(error.exception.code, 'conflict') def test_bounds_and_invalid_selections_never_return_partial_results(self): - for constant, maximum in [('MAX_COMPARISON_BYTES', 1), ('MAX_COMPARISON_SUBJECTS', 1), ('MAX_ROWS', 7)]: + for constant, maximum in [('MAX_COMPARISON_BYTES', 1), ('MAX_COMPARISON_SUBJECTS', 1), ('MAX_COMPARISON_ROWS', 7)]: with patch.object(owner, constant, maximum), self.assertRaises(WorkbenchError): MeasurementComparison(self.inputs) with self.assertRaises(WorkbenchError): MeasurementComparison(self.inputs * 9) @@ -101,6 +101,15 @@ def test_bounds_and_invalid_selections_never_return_partial_results(self): self.comparison.read('error', **arguments) with self.assertRaises(WorkbenchError): self.comparison.read(filters={}) + def test_packet_and_combined_observation_budgets_are_separate(self): + # Both real source tables have four observations. A packet limit of + # four still permits their complete eight-observation comparison. + with patch.object(owner,'MAX_ROWS',4),patch.object(owner,'MAX_COMPARISON_ROWS',8,create=True): + self.assertEqual(MeasurementComparison(self.inputs).read()['observation_count'],8) + with self.assertRaises(WorkbenchError):MeasurementTable(encode(fixture())) + with patch.object(owner,'MAX_COMPARISON_ROWS',7,create=True),self.assertRaises(WorkbenchError): + MeasurementComparison(self.inputs) + def test_returned_metadata_and_query_values_have_no_mutable_owner_alias(self): expected = self.comparison.read('error') result = self.comparison.read('error') diff --git a/tests/test_project_measurement_comparison_http.py b/tests/test_project_measurement_comparison_http.py index 09063754..df5f9667 100644 --- a/tests/test_project_measurement_comparison_http.py +++ b/tests/test_project_measurement_comparison_http.py @@ -3,6 +3,10 @@ import json import os import unittest +from unittest.mock import patch + +import agent_api +from workbench import WorkbenchError import test_project_files_http as m1 import test_project_files_m2_http as m2 @@ -56,3 +60,14 @@ def test_two_original_sources_compare_and_inspect_through_real_HTTP(self): missing=self.call('measurement_compare',sources=sources,metric_id='latency')['value']['query'] self.assertTrue(all(s['maximum'] is None for s in missing['summary'])) self.assertEqual(self.workspace._projection.session.snapshot,before) + # Exercise version-specific admission at the real workspace/SQL owner + # with small declared fixture budgets; no partial comparison escapes. + arguments=self.arguments('measurement_compare',sources=sources) + with self.owner.bound(self.workspace),patch.object(agent_api,'measurement_comparison_row_limit', + side_effect=lambda version:7 if version==1 else 8): + with self.assertRaises(WorkbenchError) as error: + self.workspace.dispatch('project_files.measurement_compare',arguments,selected_version=1) + self.assertEqual(error.exception.code,'too_large') + value=self.workspace.dispatch('project_files.measurement_compare',arguments,selected_version=2) + self.assertEqual(value['value']['value']['observation_count'],8) + self.assertEqual(self.workspace._projection.session.snapshot,before) diff --git a/tests/test_project_measurements_clients.py b/tests/test_project_measurements_clients.py index e05b6b28..079f6ab2 100644 --- a/tests/test_project_measurements_clients.py +++ b/tests/test_project_measurements_clients.py @@ -24,6 +24,16 @@ def sdk(self,value): return ProjectFilesClient(client({'status':200,'result':dict(binding=BINDING,project_id=PROJECT, checkpoint=CHECKPOINT,value=value)})) + def test_comparison_capacity_is_versioned_without_changing_packet_headers(self): + from retained_measurements import MAX_COMPARISON_ROWS,MAX_ROWS + self.assertEqual(api.measurement_comparison_row_limit(1),100000) + self.assertEqual(api.measurement_comparison_row_limit(2),250000) + self.assertEqual(api.measurement_comparison_row_limit(2),MAX_COMPARISON_ROWS) + self.assertEqual(MAX_ROWS,100000) + self.assertEqual(api._MEASUREMENT_HEADER['properties']['observation_count']['maximum'],100000) + self.assertNotEqual(api.contract('project_files.measurement_compare',1)['sha256'], + api.contract('project_files.measurement_compare',2)['sha256']) + def test_group_comparison_header_and_query_correlate_every_original_source(self): locator=dict(origin_catalog_id=PROJECT,packet_sha256=self.table.sha256) comparison=MeasurementComparison([(locator,self.table)]) From dd1feca9b524fd94e291daee6d6c95a518365390 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 16:45:51 -0700 Subject: [PATCH 04/14] fix(projects): support clients for large retained projects Allow explicit bounded owner waits and regenerate installed browser contracts from the native registry. Complete composition consumer coverage so SDK and browser clients can use the current retained-project capabilities. --- agent_api.py | 8 ++++- docs/contracts/project-files.md | 10 ++++++ scripts/generate_project_files_contracts.py | 35 +++++++++++++++++++++ static/project-files.js | 10 ++++++ tests/test_agent_api.py | 8 +++++ tests/test_project_files_clients.py | 19 +++++++++++ tests/test_project_files_m4_clients.py | 8 +++-- tuldok_client.py | 5 +++ 8 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 scripts/generate_project_files_contracts.py diff --git a/agent_api.py b/agent_api.py index ed25af09..90a534c9 100644 --- a/agent_api.py +++ b/agent_api.py @@ -2395,6 +2395,8 @@ def _private_file(path, maximum): class AgentClient: """No catalog access, automatic retries, redirects, proxies or owner identity.""" + MAX_TIMEOUT_SECONDS = 300 + def __init__(self, endpoint, token, expected_catalog_id, timeout=120): try: parsed = urlsplit(endpoint) @@ -2404,7 +2406,7 @@ def __init__(self, endpoint, token, expected_catalog_id, timeout=120): raise ValueError() if not re.fullmatch(r'[A-Za-z0-9_-]{20,4096}', token) or not re.fullmatch('[a-f0-9]{32}', expected_catalog_id): raise ValueError() - if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= 300: + if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= self.MAX_TIMEOUT_SECONDS: raise ValueError() except (ValueError, TypeError): raise AgentAPIError('Invalid fixed loopback client configuration.', 'configuration') from None @@ -3029,6 +3031,10 @@ def call(self, name, arguments, *, observer=None, observation=None): class ProjectFilesOwnerClient(AgentClient): """Explicit owner file-project access; no unrelated owner operations.""" + # Large retained projects may outlast the generic transport's five minutes. + # Only an explicit owner configuration opts into a longer absolute wait. + MAX_TIMEOUT_SECONDS = 7200 + OPERATIONS = frozenset('project_files.' + action for action in ('status', 'new', 'open', 'save', 'close', 'query', 'get', 'import_text', 'annotate')) | PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4 diff --git a/docs/contracts/project-files.md b/docs/contracts/project-files.md index e04aa7ac..387c9bc6 100644 --- a/docs/contracts/project-files.md +++ b/docs/contracts/project-files.md @@ -105,6 +105,16 @@ annotations; human review still requires the owner. Every project action is boun to an opaque workspace generation and its expected canonical checkpoint. Use `ProjectFilesOwnerClient` only for the narrow owner project-file operations. +Its explicit `timeout` configuration may be up to 7,200 seconds for large retained +projects; the default remains 120 seconds. The absolute request watchdog still +applies and a transport timeout never authorizes an automatic mutation retry. +Generic clients retain their 300-second maximum. This wait budget is not an +execution deadline or a cancellation request. + +Browser contract tables derive from the local `agent_api` registry. Regenerate +them with `python scripts/generate_project_files_contracts.py`; add `--check` +to verify that the installed v1 pins, all supported version pins and fixed +routes match that registry. Remote schemas remain opaque discovery data. `ProjectFilesClient` wraps a negotiated client. CLI domain `file-*` commands take `--arguments` JSON files; lifecycle commands use explicit flags and select `--owner-project-files` with a diff --git a/scripts/generate_project_files_contracts.py b/scripts/generate_project_files_contracts.py new file mode 100644 index 00000000..ac373e53 --- /dev/null +++ b/scripts/generate_project_files_contracts.py @@ -0,0 +1,35 @@ +"""Derive installed browser pins/routes from the local canonical API registry.""" +import argparse +import json +from pathlib import Path +import re +import sys + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT)) +import agent_api + + +def render(original): + names=sorted(name for name in agent_api._CONTRACTS if name.startswith('project_files.')) + tables={ + 'CONTRACTS':{name:agent_api.contract(name,1)['sha256'] for name in names}, + 'VERSIONS':{name:{str(v):agent_api.contract(name,v)['sha256'] for v in sorted(agent_api._CONTRACTS[name])} + for name in names if len(agent_api._CONTRACTS[name])>1}, + 'ROUTES':{name:{'method':agent_api.operation(name)['method'],'path':agent_api.operation(name)['path'], + 'mutation':agent_api.operation(name)['side_effects']} + for name in sorted(agent_api.PROJECT_FILES_M2|agent_api.PROJECT_FILES_M3|agent_api.PROJECT_FILES_M4)}} + for label,value in tables.items(): + pattern=r'(/\* PROJECT_FILES_'+label+r'_START \*/\s*)(.*?)(\s*/\* PROJECT_FILES_'+label+r'_END \*/)' + generated=json.dumps(value,sort_keys=True,indent=2) + original,count=re.subn(pattern,lambda m:m[1]+generated+m[3],original,flags=re.S) + if count!=1:raise ValueError('Exactly one installed '+label+' table required') + return original + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--check',action='store_true') + args=parser.parse_args();path=ROOT/'static/project-files.js';original=path.read_text();expected=render(original) + if args.check: + if original!=expected:raise SystemExit('Installed project-file browser contracts are stale') + elif original!=expected:path.write_text(expected) diff --git a/static/project-files.js b/static/project-files.js index 65cfa499..8e57112d 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -14,6 +14,7 @@ "project_files.candidate": "c519ff04bc06d5e538d36b7f278862d8f282d0e7b97dd7f262444c384a50f854", "project_files.candidate_select": "f043be6923760a208ca50d02a1ff4b60eadf09d48a034acdb18749806a537080", "project_files.close": "95d0099ec2a378a1bbb65676799dd3baae2f36b838301f7aa165b116fba5edf6", + "project_files.compose": "b1dcbbd4ad7ebe17b17e11238f99a93408da24d0d411527e187900cc0f089105", "project_files.data_annotate": "289270f5cb946765c5ced22fbbd18765bb55743145014c81cf54d7a6ba3025ea", "project_files.data_availability": "92332c04973a4fa45d47e384717152ebc4a9ca08e7eb487669688ccd2ed42cbb", "project_files.data_get": "0d19505995cbd8c12e3938cd89cf05955a2a63488a220711d29dd24b27c0c8c8", @@ -167,6 +168,10 @@ "project_files.get": { "1": "6b0f4e16a00aa119b86cbb5493be0bac945ff08a5e63773a8f2c2f9296658d5b", "2": "fdc0858fc03a0866bb743b04251ea5f9a3c1e6be1b211366db7e0652166736ea" + }, + "project_files.measurement_compare": { + "1": "abbb5e406a3c95155e90f08e8a071f330425190b25275f0f6512de08b8afb929", + "2": "d2de9763a9109d0e4599f50e038a3fda0e6bf7c9c60684434825d0108019bc6f" } } /* PROJECT_FILES_VERSIONS_END */; @@ -199,6 +204,11 @@ "mutation": true, "path": "/api/workspace/project-files/studies/{study_id}/candidates" }, + "project_files.compose": { + "method": "POST", + "mutation": true, + "path": "/api/workspace/project-files/compose" + }, "project_files.data_annotate": { "method": "POST", "mutation": true, diff --git a/tests/test_agent_api.py b/tests/test_agent_api.py index 2c9c1f70..004121b4 100644 --- a/tests/test_agent_api.py +++ b/tests/test_agent_api.py @@ -284,6 +284,14 @@ def test_absolute_header_deadline_includes_trickle(self): with self.assertRaises(api.AgentAPIError): client.call('data.query',{}) self.assertLess(time.monotonic()-start,1) + def test_owner_transport_keeps_the_selected_absolute_wait_and_never_retries(self): + with server('trickle') as (endpoint,calls): + client=api.ProjectFilesOwnerClient(endpoint,TOKEN,CATALOG,timeout=.15) + start=time.monotonic() + with self.assertRaises(api.AgentAPIError):client._request('GET','/api/workbench/records') + self.assertLess(time.monotonic()-start,1) + self.assertEqual(len(calls),1) + class OutputTests(unittest.TestCase): @unittest.skipUnless(HAS_JSONSCHEMA, 'Optional agent jsonschema dependency is not installed.') diff --git a/tests/test_project_files_clients.py b/tests/test_project_files_clients.py index e692dc45..2233b259 100644 --- a/tests/test_project_files_clients.py +++ b/tests/test_project_files_clients.py @@ -272,6 +272,9 @@ def test_browser_contract_pins_are_current_registry_output(self): self.assertEqual(pins, {name: api.contract(name)['sha256'] for name in api._CONTRACTS if name.startswith('project_files.')}) for name, digest in pins.items(): self.assertEqual(api.selected_header(name), name + '/1/' + digest) + versions=json.loads(re.search(r'/\* PROJECT_FILES_VERSIONS_START \*/\s*(.*?)\s*/\* PROJECT_FILES_VERSIONS_END \*/',source,re.S)[1]) + self.assertEqual(versions,{name:{str(v):api.contract(name,v)['sha256'] for v in sorted(values)} + for name,values in api._CONTRACTS.items() if name.startswith('project_files.') and len(values)>1}) def test_reads_pin_binding_checkpoint_and_path_record_without_body(self): arguments = {'record_id': RECORD, 'binding': BINDING, 'checkpoint': CHECKPOINT} @@ -352,6 +355,22 @@ def test_closed_record_output_rejects_unknown_and_inconsistent_history(self): class Consumers(unittest.TestCase): + def test_explicit_owner_wait_budget_preserves_generic_bounds_and_private_configuration(self): + endpoint,token='http://127.0.0.1:12345','fixture-agent-token-for-client-tests' + self.assertEqual(api.ProjectFilesOwnerClient(endpoint,token,CATALOG).timeout,120) + self.assertEqual(api.ProjectFilesOwnerClient(endpoint,token,CATALOG,timeout=7200).timeout,7200) + for client_type,invalid in ((api.AgentClient,301),(api.ProjectFilesOwnerClient,7201), + (api.ProjectFilesOwnerClient,float('inf')),(api.ProjectFilesOwnerClient,True)): + with self.subTest(client=client_type.__name__,timeout=invalid),self.assertRaises(api.AgentAPIError): + client_type(endpoint,token,CATALOG,timeout=invalid) + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);credential=root/'token';configuration=root/'config.json' + credential.write_text(token);credential.chmod(0o600) + configuration.write_text(json.dumps(dict(endpoint=endpoint,catalog_id=CATALOG, + token_file=str(credential),timeout=7200)));configuration.chmod(0o600) + self.assertEqual(api.ProjectFilesOwnerClient.from_config(configuration).timeout,7200) + with self.assertRaises(api.AgentAPIError):api.AgentClient.from_config(configuration) + def test_owner_file_client_keeps_unrelated_operations_outside_authority(self): raw = client({'status': 200, 'result': state()}, owner=True) self.assertEqual(ProjectFilesClient(raw).save(**bound()), state()) diff --git a/tests/test_project_files_m4_clients.py b/tests/test_project_files_m4_clients.py index 5136e6af..d24d42a2 100644 --- a/tests/test_project_files_m4_clients.py +++ b/tests/test_project_files_m4_clients.py @@ -84,10 +84,12 @@ def test_capability_profiles_fit_unchanged_budget_and_negotiate_only_intersectio with self.assertRaises(api.AgentAPIError): api.capabilities(CATALOG, profile='arbitrary') self.assertEqual(api.REGISTRY_DIGEST, api._LEGACY_DIGEST) - def test_eleven_operations_have_fixed_consumers_and_closed_inputs(self): - self.assertEqual(len(api.PROJECT_FILES_M4), 11) + def test_complete_m4_operations_have_fixed_consumers_and_closed_inputs(self): + self.assertEqual(api.PROJECT_FILES_M4,{'project_files.'+name for name in ( + 'compose','save_as','files_inspect','file_get','file_history','file_adopt','files_adopt', + 'file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')}) self.assertEqual(project_dispatch.SUPPORTED_OPERATIONS & api.PROJECT_FILES_M4, - api.PROJECT_FILES_M4 - {'project_files.save_as'}) + api.PROJECT_FILES_M4 - {'project_files.save_as','project_files.compose'}) client = ProjectFilesClient(Mock()) parser = project_cli.parser() for operation in api.PROJECT_FILES_M4: diff --git a/tuldok_client.py b/tuldok_client.py index be0daf7f..fdaef8d8 100644 --- a/tuldok_client.py +++ b/tuldok_client.py @@ -706,6 +706,11 @@ def save_as(self, *, root, directory, name, binding, expected_checkpoint, reques return self._call('save_as',dict(root=root,directory=directory,name=name,binding=binding, expected_checkpoint=expected_checkpoint,request_key=request_key)) + def compose(self, *, root, directory, name, sources, binding, expected_checkpoint, request_key): + """Compose explicit retained checkpoints through the negotiated owner.""" + return self._call('compose',dict(root=root,directory=directory,name=name,sources=sources, + binding=binding,expected_checkpoint=expected_checkpoint,request_key=request_key)) + def files_inspect(self, **arguments): return self._call('files_inspect',arguments) From 1f1e3bc5cf6210ed9658256b869389d60d8d3898 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 18:07:00 -0700 Subject: [PATCH 05/14] perf(projects): reuse decoding after verifying current canonical bytes Keep exact manifest-bound decoding inside one Store while freshly checking every canonical record on each read. Borrow rooted handles without cloning unused public snapshots; preserve independent public values and before/after closure checks. --- project_store.py | 47 ++++++++++++++++++++++++--------- tests/test_project_artifacts.py | 6 ++--- tests/test_project_store.py | 31 ++++++++++++++++++++++ 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/project_store.py b/project_store.py index bc4b4071..93ba5ad3 100644 --- a/project_store.py +++ b/project_store.py @@ -468,6 +468,7 @@ def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path): self._origin_path = Path(origin_path) self._meta_fd = None self._manifest = None + self._read_cache = None self._mutex = threading.RLock() self._unknown = False try: @@ -629,12 +630,12 @@ def root_directory(self, *, verify_after=False): if type(verify_after) is not bool: _fail('Root read verification must be an explicit boolean.') with self._mutex: - before = self.snapshot() + before = self._current()[0]['checkpoint'] descriptor = os.dup(self._root_fd) try: yield descriptor self._binding() - if verify_after and self.snapshot()['checkpoint'] != before['checkpoint']: + if verify_after and self._current()[0]['checkpoint'] != before: _fail('Project checkpoint changed during the bounded root read.', 'stale', 409) finally: os.close(descriptor) @@ -642,6 +643,17 @@ def root_directory(self, *, verify_after=False): def _load(self): self._binding() raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) + cached = self._read_cache + if cached is not None and raw == cached[0]: + manifest, loaded = cached[1:] + # Reuse decoding only after checking every actual canonical file. + # Stat identity is not content identity; corruption still fails. + for entry in manifest['inventory']: + self._verified_record_bytes(entry) + if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: + _fail('Project checkpoint changed while reading.', 'stale', 409) + self._binding() + return manifest, loaded manifest = _decode(raw) if type(manifest) is not dict: _fail('Project checkpoint must be an object.') @@ -683,9 +695,7 @@ def _load(self): total += entry['bytes'] if total > MAX_TOTAL_BYTES: _fail('Project records exceed their total byte limit.') - record_raw = self._record_file(family, record_id, digest) - if len(record_raw) != entry['bytes'] or _sha(record_raw) != digest: - _fail('Canonical record bytes changed or are corrupt.', 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + record_raw = self._verified_record_bytes(entry) document = _decode(record_raw) if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') @@ -709,8 +719,16 @@ def _load(self): if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: _fail('Project checkpoint changed while reading.', 'stale', 409) self._binding() + self._read_cache = (raw, manifest, loaded) return manifest, loaded + def _verified_record_bytes(self, entry): + raw = self._record_file(entry['family'], entry['id'], entry['sha256']) + if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: + _fail('Canonical record bytes changed or are corrupt.', + 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + return raw + def _record_file(self, family, record_id, digest, raw=None, sync=False): fds = [] try: @@ -777,15 +795,19 @@ def _snapshot(self, manifest, records): return dict(project_id=manifest['project_id'], name=manifest['name'], checkpoint=manifest['checkpoint'], records=copy.deepcopy(records)) + def _current(self): + try: + manifest, records = self._load() + if manifest != self._manifest: + _fail('Project checkpoint changed; close and reopen.', 'stale', 409) + return manifest, records + except OSError as exc: + _io_error(exc, 'read project snapshot') + def snapshot(self): with self._mutex: - try: - manifest, records = self._load() - if manifest != self._manifest: - _fail('Project checkpoint changed; close and reopen.', 'stale', 409) - return self._snapshot(manifest, records) - except OSError as exc: - _io_error(exc, 'read project snapshot') + manifest, records = self._current() + return self._snapshot(manifest, records) def reconcile(self, request_key, request_sha256): """Inspect retry identity before a domain mutation; never repeat effects.""" @@ -884,6 +906,7 @@ def recover_checkpoint(self): def close(self): with self._mutex: + self._read_cache = None if self._meta_fd is not None: os.close(self._meta_fd) self._meta_fd = None diff --git a/tests/test_project_artifacts.py b/tests/test_project_artifacts.py index 2a68497d..ad06e375 100644 --- a/tests/test_project_artifacts.py +++ b/tests/test_project_artifacts.py @@ -74,7 +74,7 @@ def mutate_source_after_snapshot(fd, value, **options): original.write_bytes(b'changed after verified capture') return publish(fd, value, **options) with patch.object(destination, 'publish_fd', side_effect=mutate_source_after_snapshot): - with patch.object(self.store, 'snapshot', wraps=self.store.snapshot) as checks: + with patch.object(self.store, '_current', wraps=self.store._current) as checks: self.artifacts.copy_many_to(destination, values) self.assertEqual(checks.call_count, 2) for raw, value in zip(raws, values): @@ -118,7 +118,7 @@ def test_status_group_retains_exact_hash_observations_with_two_closure_checks(se self.payload(values[2]).unlink() self.payload(values[2]).write_bytes(b'changed') expected = [self.artifacts.status(value) for value in values] - with patch.object(self.store, 'snapshot', wraps=self.store.snapshot) as checks: + with patch.object(self.store, '_current', wraps=self.store._current) as checks: actual = self.artifacts.status_many(values) self.assertEqual(actual, expected) self.assertEqual([item['state'] for item in actual], ['verified', 'missing', 'corrupt']) @@ -161,7 +161,7 @@ def test_presence_batch_matches_individual_observations_with_two_closure_checks( self.artifacts.publish_bytes(b'present',values[0]) self.commit(values) expected=[self.artifacts.inspect(value) for value in values] - with patch.object(self.store,'snapshot',wraps=self.store.snapshot) as checks: + with patch.object(self.store,'_current',wraps=self.store._current) as checks: actual=self.artifacts.inspect_many(values) self.assertEqual(actual,expected) self.assertEqual(checks.call_count,2) diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 9f321617..9dcf11af 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -97,6 +97,37 @@ def test_appending_one_record_does_not_rewrite_saved_immutable_revisions(self): 'Appending one record must not write disposable copies of every saved immutable revision') self.assertEqual(self.reopen(), committed) + def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_identical_records(self): + rows=[record('a'),record('b','Exact second note\n')] + self.commit(rows,key='repeat-read-fixture') + expected=self.store.snapshot() + canonical_documents={canonical(row['document']) for row in rows} + with patch.object(store_module,'_decode',wraps=store_module._decode) as decode, \ + patch.object(self.store,'_record_file',wraps=self.store._record_file) as read, \ + patch.object(self.store,'_snapshot',wraps=self.store._snapshot) as snapshot: + self.assertEqual(self.store.snapshot(),expected) + with self.store.root_directory(verify_after=True) as descriptor: + self.assertTrue(os.fstat(descriptor).st_ino) + self.assertEqual(self.store.snapshot(),expected) + self.assertEqual(read.call_count,8,'All two records must still be read in each of four checks') + self.assertEqual(snapshot.call_count,2,'Root borrowing must not clone an unused public snapshot') + decoded=[call for call in decode.call_args_list if call.args[0] in canonical_documents] + self.assertEqual(len(decoded),0,'Byte-identical canonical records need no repeat decoding within one Store') + + def test_verified_decode_cache_still_rejects_same_length_corruption_with_restored_mtime(self): + self.commit([record('note','Alpha\n')],key='cache-corruption-fixture') + self.store.snapshot() + entry=self.manifest()['inventory'][0] + path=self.root/'.tuldok'/entry['path'];info=path.stat();raw=path.read_bytes() + changed=raw.replace(b'Alpha',b'Omega') + self.assertEqual(len(changed),len(raw));self.assertNotEqual(changed,raw) + path.chmod(0o600);path.write_bytes(changed) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + self.assert_code('stale',self.store.snapshot) + with self.assertRaises(ProjectStoreError) as caught: + with self.store.root_directory(verify_after=True):pass + self.assertEqual(caught.exception.code,'stale') + def test_reusing_an_unreferenced_revision_still_checks_bytes_and_syncs_it(self): first, second = record('first'), record('second') self.commit([first], key='retain-first') From bf2746e0e067d5125731ed6486d5dd68380a6a82 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 18:26:26 -0700 Subject: [PATCH 06/14] perf(projects): select active receipt records before copying Keep complete fork and historical receipt validation, while copying only operation records for the current request-key lookup. Preserve the default independently owned active-record view for other callers. --- project_fork.py | 10 ++++++++-- project_session.py | 3 ++- tests/test_project_fork.py | 23 +++++++++++++++++++++++ tests/test_project_session.py | 19 +++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/project_fork.py b/project_fork.py index bd928ad9..f35b9812 100644 --- a/project_fork.py +++ b/project_fork.py @@ -174,10 +174,16 @@ def decode_fork(records, *, project_id=None): return copy.deepcopy(overlays[current]) -def active_records(records, *, project_id=None): +def active_records(records, *, project_id=None, families=None): + """Own selected active copies while validating the complete fork history.""" + if families is not None: + if type(families) is not frozenset: + _fail('Active record families must be an explicit frozenset.') + for family in families:_identifier(family,'Active record family') overlay = decode_fork(records, project_id=project_id) archived = set() if overlay is None else {(item['family'], item['id']) for item in overlay['archived']} - return [copy.deepcopy(item) for item in records if (item['family'], item['id']) not in archived] + return [copy.deepcopy(item) for item in records + if (families is None or item['family'] in families) and (item['family'], item['id']) not in archived] def _inside(candidate_fd, ancestor_fd): diff --git a/project_session.py b/project_session.py index 46c106cd..de2293d7 100644 --- a/project_session.py +++ b/project_session.py @@ -65,7 +65,8 @@ def operation_records(records, *, project_id=None): if project_id is None and overlay is not None: project_id = overlay['destination_project_id'] result = {} - for value in operation_documents(active_records(records, project_id=project_id)): + for value in operation_documents(active_records(records, project_id=project_id, + families=frozenset({OPERATION_FAMILY}))): if overlay is not None and value['format'] == 'project_operation_v1': raise ProjectSessionError('A fork must retain legacy operation receipts as archived historical facts.') if project_id is not None and value.get('project_id', project_id) != project_id: diff --git a/tests/test_project_fork.py b/tests/test_project_fork.py index 6241c890..1b4029b8 100644 --- a/tests/test_project_fork.py +++ b/tests/test_project_fork.py @@ -371,5 +371,28 @@ def test_current_overlay_filters_receipts_and_original_editables_without_rewriti self.assert_code('invalid', decode_fork, bad) +class ActiveRecordSelectionTests(unittest.TestCase): + def test_family_selection_keeps_independent_values_and_default_membership(self): + records=[dict(family=family,id='a'*32,document=dict(format='fixture_v1',values=[1])) + for family in ('project_operation','workbench_text')] + self.assertEqual(active_records(records),records) + selected=active_records(records,families=frozenset({'project_operation'})) + self.assertEqual(selected,[records[0]]) + selected[0]['document']['values'].append(2) + self.assertEqual(records[0]['document']['values'],[1]) + self.assertEqual(active_records(records,families=frozenset()),[]) + + def test_family_selection_still_validates_excluded_fork_structure(self): + record=dict(family='workbench_text',id='a'*32,document={'format':'workbench_text_v1'}) + with self.assertRaisesRegex(ProjectForkError,'Duplicate'): + active_records([record,copy.deepcopy(record)],families=frozenset({'project_operation'})) + + def test_family_selection_rejects_implicit_or_invalid_selectors(self): + for invalid in ('project_operation',[],frozenset({1}),frozenset({'bad/family'})): + with self.subTest(selector=invalid),self.assertRaises(ProjectStoreError) as caught: + active_records([],families=invalid) + self.assertEqual(caught.exception.code,'invalid') + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_project_session.py b/tests/test_project_session.py index f9356c44..309f2bc4 100644 --- a/tests/test_project_session.py +++ b/tests/test_project_session.py @@ -14,6 +14,7 @@ from unittest.mock import patch import uuid +import project_fork as fork_module import psycopg from psycopg import sql @@ -27,6 +28,24 @@ class OperationCodecTests(unittest.TestCase): + def test_operation_lookup_does_not_copy_unrelated_active_envelopes(self): + key='selected-receipt' + document=dict(format='project_operation_v1',operation='fixture.capture',target=None, + request_key=key,input_sha256='a'*64,result={}) + receipt=dict(family='project_operation',id=hashlib.sha256(key.encode()).hexdigest(),document=document) + unrelated=dict(family='project_artifact',id='b'*64, + document=dict(format='project_artifact_v1',sha256='b'*64,bytes=4)) + copied=[];deepcopy=copy.deepcopy + def observe(value): + if value is unrelated:copied.append((value['family'],value['id'])) + return deepcopy(value) + with patch.object(fork_module,'copy',SimpleNamespace(deepcopy=observe)): + actual=operation_records([unrelated,receipt]) + self.assertEqual(actual,{key:document}) + self.assertEqual(copied,[],'Receipt selection must not clone unrelated active records') + actual[key]['result']['changed']=True + self.assertEqual(document['result'],{}) + def test_namespaced_receipts_preserve_legacy_identity_and_independent_keys(self): key = 'independent-edit' document = dict(format='project_operation_v1', operation='fixture.capture', target=None, From 0adcc5bea0bcc2cbc5bcf41a0627677582a52335 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 22:06:07 -0700 Subject: [PATCH 07/14] perf(projects): avoid full snapshots for scalar metadata Read project identity and checkpoint through verified Store summaries and bound Session summaries. Preserve full canonical freshness checks and independent snapshots for record consumers. Update coordinated fixtures, including interpreter alias drift on hosts using the system Python. --- project_custody.py | 4 +-- project_data.py | 2 +- project_dispatch.py | 2 +- project_execution.py | 12 ++++---- project_files.py | 10 +++---- project_history.py | 12 ++++---- project_material.py | 6 ++-- project_measurements.py | 2 +- project_native_execution.py | 4 +-- project_native_preparation.py | 12 ++++---- project_producer_execution.py | 2 +- project_producers.py | 2 +- project_projection.py | 2 +- project_runtime.py | 14 +++++----- project_session.py | 8 +++++- project_sources.py | 2 +- project_store.py | 6 ++++ project_study_execution.py | 26 +++++++++--------- project_text.py | 2 +- project_transfer.py | 4 +-- project_workspace.py | 32 +++++++++++----------- tests/test_project_dispatch.py | 5 ++-- tests/test_project_files_fork_contracts.py | 3 +- tests/test_project_material.py | 3 +- tests/test_project_material_fork.py | 9 ++++-- tests/test_project_native_process.py | 8 ++++-- tests/test_project_session.py | 17 ++++++++++++ tests/test_project_store.py | 15 ++++++++++ tests/test_project_workspace.py | 11 ++++++++ 29 files changed, 151 insertions(+), 86 deletions(-) diff --git a/project_custody.py b/project_custody.py index 2e0a97aa..1a188f1d 100644 --- a/project_custody.py +++ b/project_custody.py @@ -197,7 +197,7 @@ def _directory(self, create=False): finally:os.close(parent) def _binding(self, store, projection_id): - snapshot=store.snapshot() + snapshot=store.summary() with store.root_directory() as descriptor: info=os.fstat(descriptor) return dict(host_catalog_id=self.host_catalog_id,project_id=snapshot['project_id'], @@ -247,7 +247,7 @@ def __init__(self, owner, store, binding): persisted=value is not None if value is None: value=dict(format='project_local_custody_v1',**binding, - project_name=store.snapshot()['name'],selected_checkpoint=store.snapshot()['checkpoint'], + project_name=store.summary()['name'],selected_checkpoint=store.summary()['checkpoint'], attempted=None,runs=[],producers=[],runtime_effects=[],package_effects=[]) if any(value[key]!=item for key,item in binding.items()):_fail('Original physical custody binding differs.') self.value=validate(value) diff --git a/project_data.py b/project_data.py index 4a48dd38..6c99e8b2 100644 --- a/project_data.py +++ b/project_data.py @@ -1001,7 +1001,7 @@ def _publish(self, raw, kind, role, media_type): declaration = dict(format='project_artifact_v1', sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) self.artifacts.publish_bytes(raw, declaration) self._pending_artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=self.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=self.session.summary['project_id'], sha256=declaration['sha256']), media_type=media_type, kind=kind, role=role, preservation='unique', required_for=['inspect', 'release']) def _require_record(self, record_id, *, extended=False): diff --git a/project_dispatch.py b/project_dispatch.py index 6c9420a2..bd7929d0 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -149,7 +149,7 @@ def _data(projection, action, a): def _definitions(projection, family, a): docs = projection.materials._documents() values = [doc['current'] for (kind, _), doc in docs.items() - if kind == family and doc['origin_catalog_id'] == projection.session.snapshot['project_id']] + if kind == family and doc['origin_catalog_id'] == projection.session.summary['project_id']] return _page(sorted(values, key=lambda item: item['id']), a) diff --git a/project_execution.py b/project_execution.py index ca5c36d0..fece88a1 100644 --- a/project_execution.py +++ b/project_execution.py @@ -296,7 +296,7 @@ def _restore_file_writer(self, kind, request_key, input_sha256, content_sha256, authority.require(scope,action,owner_only=True) if self.db.in_transaction: raise RuntimeError('File reconciliation requires no outer SQL transaction.') - if self.session.snapshot['checkpoint']!=expected_checkpoint: + if self.session.summary['checkpoint']!=expected_checkpoint: raise WorkbenchError('Expected current file checkpoint changed.','stale_project',409) if self.custody is None: raise WorkbenchError('No original physical file custody exists.','unavailable',409) @@ -464,7 +464,7 @@ def _authorize_observation(self): # The domain owner has already checked submit/cancel authority; # recheck live host credentials without inventing a worker owner. self.runtime_access.require('inspect', 'runs.observe', identity) - if proof['admission']['catalog_id'] != self.session.snapshot['project_id']: + if proof['admission']['catalog_id'] != self.session.summary['project_id']: raise AccessError('Local custody belongs to another project.') def _capture_run(self, identity): @@ -626,7 +626,7 @@ def invoke(): context.consume_admission() owner=self._local_owner();public=owner.submit(dict(body,request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.submit',body['graph_id'],effective,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.submit',body['graph_id'],effective,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_retry(self,run_id,expected_checkpoint,key): @@ -646,7 +646,7 @@ def invoke(): context.consume_admission() owner=self.runs;public=owner.retry(run_id,dict(request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.retry',run_id,body,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.retry',run_id,body,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_reconcile(self,phase_id,expected_checkpoint): @@ -657,7 +657,7 @@ def submit_external(self, body, expected_checkpoint, key): with self.access.lock, self.session.lock, self.db.lock: self.session.check() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: # A retained request never recreates a local Run/admission or wakes @@ -680,7 +680,7 @@ def approve_external(self, body, expected_checkpoint, key): with self.access.lock, self.session.lock, self.db.lock: self.session.check() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: value = prior['result'] diff --git a/project_files.py b/project_files.py index c312aa34..c6e9f6b6 100644 --- a/project_files.py +++ b/project_files.py @@ -300,7 +300,7 @@ def materialize_captured(records,project_id,root_fd): def _adoption_intent(self,body): closed(body,{'kind','path','expected_revision','expected_sha256'},'working-file adoption') - parts=_path(body['kind'],body['path']);project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) + parts=_path(body['kind'],body['path']);project_id=self.session.summary['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 0<=body['expected_revision']<=2**53-1:fail('Invalid expected adoption revision.') return dict(body),parts,record_id @@ -316,12 +316,12 @@ def _capture_adoption(self,intent,prior,root): value=dict(revision=revision+1,sha256=observed,bytes=size,text=text,adopted_at=timestamp(),actor_id=actor.actor_id) history=copy.deepcopy(prior['document']['history']) if prior else [];history.append(value) envelope=dict(family=FAMILY,id=record_id,document=dict(format=prior['document']['format'] if prior else FORMAT, - project_id=self.session.snapshot['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), + project_id=self.session.summary['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), history=history,origin=copy.deepcopy(prior['document']['origin']) if prior else None)) return _result(envelope,True),envelope,size def adopt(self,body,expected_checkpoint,key): - intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.snapshot['project_id'] + intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.summary['project_id'] pending=[] with self.access.lock: self.access.require('edit','files.adopt',record_id) @@ -350,7 +350,7 @@ def adopt_many(self,body,expected_checkpoint,key): intents=[self._adoption_intent(item) for item in body['items']] ids=[intent[2] for intent in intents] if len(set(ids))!=len(ids):fail('Batch adoption repeats a working-file path.') - body=dict(items=[intent[0] for intent in intents]);project_id=self.session.snapshot['project_id'] + body=dict(items=[intent[0] for intent in intents]);project_id=self.session.summary['project_id'] target=digest('\0'.join(ids).encode());pending=[] with self.access.lock: for identity in ids:self.access.require('edit','files.adopt',identity) @@ -377,7 +377,7 @@ def adopt_deletion(self,body,expected_checkpoint,key): closed(body,{'path','expected_revision','expected_sha256'},'note deletion adoption') parts=_path('note',body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 1<=body['expected_revision']<=2**53-1:fail('Invalid expected note revision.') - project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] + project_id=self.session.summary['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] with self.access.lock: self.access.require('edit','files.adopt_deletion',record_id) def invoke(): diff --git a/project_history.py b/project_history.py index 892ae3c2..31c5f645 100644 --- a/project_history.py +++ b/project_history.py @@ -760,7 +760,7 @@ def _authorize(self,action,target=None): self.access.require('inspect',action,target) def _origin(self,origin): - return self.session.snapshot['project_id'] if origin is None else origin + return self.session.summary['project_id'] if origin is None else origin def _record(self,collection,identity,origin=None): origin=self._origin(origin);identifier(origin,'Origin catalog');identifier(identity) @@ -1163,7 +1163,7 @@ def create_study(self,body,expected_checkpoint,request_key): closed(body,('name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']);policy=validate_retained_policy(body['policy']) def build(facts): - origin=self.session.snapshot['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) + origin=self.session.summary['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) prior=next((key for key,value in facts.policies.items() if key[0]==origin and key[3]==sha),None) policy_id=prior[1] if prior else uuid.uuid4().hex ref={'kind':'selection_policy','catalog_id':origin,'id':policy_id,'revision':1,'sha256':sha} @@ -1180,7 +1180,7 @@ def save_study(self,study_id,body,expected_checkpoint,request_key): closed(body,('revision','name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['policy'],'selection_policy',origin) policy=facts.policies.get((origin,ref['id'],ref['revision'],ref['sha256'])) @@ -1201,7 +1201,7 @@ def prepare_trial_from_material(self,study_id,body,expected_checkpoint,request_k if request_key not in operation_records(self.session.snapshot['records']): self.materials.recipe(body['material'],verify=True) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['material'],'recipe_material',origin);self.materials.recipe(ref) @@ -1224,7 +1224,7 @@ def select_candidate(self,study_id,body,expected_checkpoint,request_key): identifier(body['trial_id']);identifier(body['run_id']) # Expensive semantic proof precedes the write transaction. All exact # associations and selected policy are checked again under publication. - origin=self.session.snapshot['project_id'] + origin=self.session.summary['project_id'] self.access.require('edit','studies.select_candidate',study_id) evidence=(self.evidence(body['run_id'],{},origin_catalog_id=origin) if request_key not in operation_records(self.session.snapshot['records']) else None) @@ -1254,7 +1254,7 @@ def build(facts): def record_lineage(self,body,expected_checkpoint,request_key): closed(body,('trial','intent'));body=checked_json(body,INTENT_LIMIT+4096) def build(facts): - origin=self.session.snapshot['project_id'];trial=facts.trial_ref(body['trial']) + origin=self.session.summary['project_id'];trial=facts.trial_ref(body['trial']) if body['trial']['catalog_id']!=origin:_invalid('Inherited Trials remain immutable historical evidence.','conflict') doc=copy.deepcopy(facts.trials[(origin,trial['id'])]) if doc['lineage'] is not None:_invalid('Trial lineage is already recorded and immutable.','conflict') diff --git a/project_material.py b/project_material.py index 194e91a5..7cba65fe 100644 --- a/project_material.py +++ b/project_material.py @@ -381,7 +381,7 @@ def intents(self): from project_material_fork import is_intent documents = self._documents() active = {(r['family'], r['id']) for r in active_records( - self.session.snapshot['records'], project_id=self.session.snapshot['project_id'])} + self.session.snapshot['records'], project_id=self.session.summary['project_id'])} return [dict(family=family, id=locator, current=copy.deepcopy(doc['current']), fork_origin=copy.deepcopy(doc['fork_origin'])) for (family, locator), doc in sorted(documents.items()) @@ -392,7 +392,7 @@ def intent_capabilities(self): intents = self.intents() unresolved = {(r['family'],r['id']) for r in intents} from project_fork import active_records - active = active_records(self.session.snapshot['records'], project_id=self.session.snapshot['project_id']) + active = active_records(self.session.snapshot['records'], project_id=self.session.summary['project_id']) return dict(requires_reselection=[dict(family=r['family'],id=r['id'],fields=r['current']['required_reselection']) for r in intents], editable=[dict(family=r['family'],id=r['id']) for r in active if r['family'] in EDITABLE and (r['family'],r['id']) not in unresolved]) @@ -461,7 +461,7 @@ def snapshot_ref(self, project_id, manifest_sha256, *, catalog_id=None, verify=F return copy.deepcopy(doc['value']) def capture_definition(self, family, value, before_records, *, action, actor_id=None, audit_facts=None): - catalog_id = self.session.snapshot['project_id'] + catalog_id = self.session.summary['project_id'] value = _definition(family, value, catalog_id) key = catalog_id + '.' + value['id'] prior = decode_records(before_records).get((family, key)) diff --git a/project_measurements.py b/project_measurements.py index 2800b50b..2168d1b7 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -127,7 +127,7 @@ def import_packet(self, raw, source, expected_checkpoint, key): self.session.check() table = MeasurementTable(raw) blob = dict(sha256=table.sha256, bytes=len(raw)) - origin = self.session.snapshot['project_id'] + origin = self.session.summary['project_id'] request = dict(blob=blob, source=copy.deepcopy(source)) def invoke(): diff --git a/project_native_execution.py b/project_native_execution.py index 832a990f..9ac6be4a 100644 --- a/project_native_execution.py +++ b/project_native_execution.py @@ -473,7 +473,7 @@ def reconcile(self,phase_id,expected_checkpoint): execution=self.execution;journal=execution.custody if journal is None:_fail('There is no original local native custody.','unavailable') with self.db.lock: - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') local=journal.refresh() selected=next((row for row in local.get('native_phases',[]) if row['phase_id']==phase_id),None) if selected is None:_fail('A retained phase alone cannot select current native custody.','unavailable') @@ -501,7 +501,7 @@ def reconcile(self,phase_id,expected_checkpoint): if self._active:_fail('Current native owner has not finished its bounded observation.','busy') with execution.access.lock,self.session.lock,self.db.lock: self.access.require('cancel','native_execution.reconcile',phase_id,owner_only=True) - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') if journal.refresh()!=local:_fail('Original native custody changed while draining.','stale_project') binding=journal.binding if (binding['project_id']!=self.db.dataset_id or binding['projection_id']!=self.db.projection_id diff --git a/project_native_preparation.py b/project_native_preparation.py index 85a0b1be..7c504be0 100644 --- a/project_native_preparation.py +++ b/project_native_preparation.py @@ -262,7 +262,7 @@ def probe(self,owner,recipe): if self._probe is None: if self.adapter.db.in_transaction:_fail('Native probe cannot run inside a metadata transaction.','unsupported') self.authorize_probe(recipe) - result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.snapshot['checkpoint'], + result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.summary['checkpoint'], request_key=self.key+'.probe',preparation=self) self._probe=copy.deepcopy(result['runtime']);self.phase=copy.deepcopy(result['phase']) return copy.deepcopy(self._probe) @@ -334,7 +334,7 @@ def _scope(self,owner,graph,model,composition,purpose,expected_checkpoint,key): if _CURRENT.get() is not None:_fail('A native preparation is already active.','busy') self.access.require('edit','native_preparation.'+purpose,graph['id'],owner_only=True) self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') context=NativePreparationContext(self,owner,graph,model,composition,purpose,_key(purpose,key)) result=dict(format='project_native_preparation_intent_v1',origin_catalog_id=self.session.project_id, actor_id=context.actor.actor_id,purpose=purpose,graph=dict(id=graph['id'],revision=graph['revision'], @@ -362,13 +362,13 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): import release_validation if adapter_for_graph(graph) is release_validation: self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') value=owner.validate_graph(graph) return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') with self._scope(owner,graph,None,None,'validation',expected_checkpoint,key) as context: value=owner.validate_graph(graph);context._check(graph['recipe']) - return self.session.mutate('native_preparation.validate',graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate('native_preparation.validate',graph_id,body,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') @contextmanager @@ -376,7 +376,7 @@ def current_graph(self,owner,graph_id,revision,expected_checkpoint,key): if self.db.in_transaction:_fail('Current native validation must precede admission.','unsupported') self.access.require('edit','native_preparation.admission',graph_id,owner_only=True) self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') graph=owner.get(graph_id) if graph['revision']!=revision:_fail('Graph changed before native admission.','stale') import release_validation @@ -413,7 +413,7 @@ def capture(value,before): record=self.materials.capture_recipe_material(value,owner.material_path) return dict(records=self.materials._coalesce([record,*self.materials.declarations(record), *self.materials.capture_recipe_dependencies(record,before)]),result=value) - return self.session.mutate(operation,graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate(operation,graph_id,body,self.session.summary['checkpoint'],key, lambda:owner._retain_material(prepared),capture) finally: if prepared['staging'].exists():shutil.rmtree(prepared['staging']) diff --git a/project_producer_execution.py b/project_producer_execution.py index b4456aba..f4ae5915 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -264,7 +264,7 @@ def _settle_drained(self): # Every effectful request is preceded by canonical uncertainty; # a terminal job with no such request is known not admitted. outcome='not_admitted' - observation=(outcome,True,self.session.snapshot['checkpoint']) + observation=(outcome,True,self.session.summary['checkpoint']) if self._journal_observations.get(identity_key)!=observation: self.execution.update_producer_outcome(family,producer_id,outcome,True) self._journal_observations[identity_key]=observation diff --git a/project_producers.py b/project_producers.py index 79b97f20..10eb94be 100644 --- a/project_producers.py +++ b/project_producers.py @@ -915,7 +915,7 @@ def _publish(owner, raw, role, media='application/json'): fail('Conflicting exact producer payload bytes.') owner._pending_payloads[declaration['sha256']] = raw owner._artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=owner.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=owner.session.summary['project_id'], sha256=declaration['sha256']), media_type=media, kind='producer_evidence', role=role, preservation='unique', required_for=['verify_producer']) diff --git a/project_projection.py b/project_projection.py index 1181c457..04aaebba 100644 --- a/project_projection.py +++ b/project_projection.py @@ -116,7 +116,7 @@ def __init__(self, store, db, access, roots, *, custody=None, restore=True, runt self.store, self.db, self.access = store, db, access self.custody = custody self.pending_original = [] - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.artifacts = ProjectArtifacts(store) self.sources = None self._disclosures = set() diff --git a/project_runtime.py b/project_runtime.py index a94cc6e6..21fbd63f 100644 --- a/project_runtime.py +++ b/project_runtime.py @@ -185,7 +185,7 @@ def recover_capture(self, config_request, expected_inventory_sha256, expected_ch body_sha = _intent_input(body) self._authorize('runtime_snapshots.create') self.execution.prepare_runtime_reconciliation(key, body_sha, expected_inventory_sha256, expected_checkpoint) - return self.capture(config_request, expected_inventory_sha256, self.session.snapshot['checkpoint'], key, + return self.capture(config_request, expected_inventory_sha256, self.session.summary['checkpoint'], key, reserve_bytes=reserve_bytes) def abandon(self, capture_key, reason, expected_checkpoint, key): @@ -212,7 +212,7 @@ def forbidden(): _fail('No original local runtime copy exists to abandon.', 'una _fail('Runtime intent differs from its exact original local writer.', 'conflict', 409) if capture_key in receipts: _fail('Capture completed; reconcile its acknowledgement with the exact capture request.', 'conflict', 409) - observed = self.session.snapshot['checkpoint'] + observed = self.session.summary['checkpoint'] def invoke(): current = self._authorize('runtime.capture_abandon') if current.actor_id != actor.actor_id: _fail('Runtime abandonment caller changed.', 'conflict', 409) @@ -273,7 +273,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), invalid, lambda *args: invalid(), scope='admin') else: - if expected_checkpoint != self.session.snapshot['checkpoint']: + if expected_checkpoint != self.session.summary['checkpoint']: _fail('Expected project checkpoint changed.', 'stale_project', 409) config = self.resolve_configuration(copy.deepcopy(config_request)) observed = self.owner.inspect(config) @@ -281,7 +281,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 _fail('Runtime inventory changed after inspection.', 'stale', 409) intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key), + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key), lambda result,before:dict(records=[],result=result), scope='admin') guard.publication_started() if os.path.lexists(self.owner.path / expected_inventory_sha256): @@ -294,7 +294,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 from pipeline import digest from pathlib import Path raw = read_regular(Path(descriptor['root'])/'manifest.json', MAX_MANIFEST) - runtime_ref = dict(kind='runtime',catalog_id=self.session.snapshot['project_id'], + runtime_ref = dict(kind='runtime',catalog_id=self.session.summary['project_id'], id=digest(descriptor),sha256=digest(descriptor)) value = dict(ref=runtime_ref,descriptor=descriptor,environment=None,manifest=decode(raw)) observed = dict(inventory_sha256=descriptor['manifest_sha256'],file_count=descriptor['file_count'],total_bytes=descriptor['total_bytes']) @@ -302,8 +302,8 @@ def capture(result, before): record = self.materials.capture_runtime(value, raw) return dict(records=[*self.materials.declarations(record),record],result=result) result = self.session.mutate('runtime.capture', expected_inventory_sha256, body, - self.session.snapshot['checkpoint'], key, - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') + self.session.summary['checkpoint'], key, + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') guard.confirmed() return result except BaseException as error: diff --git a/project_session.py b/project_session.py index de2293d7..308164c5 100644 --- a/project_session.py +++ b/project_session.py @@ -86,7 +86,7 @@ def __init__(self, store, db, access, artifacts, *, validate_records): self._validate = validate_records self.available = False self._snapshot = None - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.extra_capture = None self.publication_intent = None self.runtime_access = None @@ -177,6 +177,12 @@ def reconcile_selection(self, snapshot): self._recovery_records = None self._recovery_request = None + @property + def summary(self): + """Scalar metadata for the same bound checkpoint as snapshot.""" + with self.lock: + return {key: self._snapshot[key] for key in ('project_id', 'name', 'checkpoint')} + @property def snapshot(self): with self.lock: diff --git a/project_sources.py b/project_sources.py index 08939eba..25007ea1 100644 --- a/project_sources.py +++ b/project_sources.py @@ -119,7 +119,7 @@ def register(self,body,expected_checkpoint,key): actor=self.access.require('edit','source.register',owner_only=True) self.session.check() saved,raw=self._read_registration(body) - origin=self.session.snapshot['project_id'];invoked=False + origin=self.session.summary['project_id'];invoked=False def invoke(): nonlocal invoked invoked=True;self._bind(saved,raw) diff --git a/project_store.py b/project_store.py index 93ba5ad3..cb1c2bf9 100644 --- a/project_store.py +++ b/project_store.py @@ -804,6 +804,12 @@ def _current(self): except OSError as exc: _io_error(exc, 'read project snapshot') + def summary(self): + """Return scalar metadata after the same complete freshness check.""" + with self._mutex: + manifest, _records = self._current() + return {key: manifest[key] for key in ('project_id', 'name', 'checkpoint')} + def snapshot(self): with self._mutex: manifest, records = self._current() diff --git a/project_study_execution.py b/project_study_execution.py index 561c4779..fa6d6ce9 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -267,7 +267,7 @@ def __init__(self,adapter,phase): def __getattr__(self,name):return getattr(self.source,name) def open(self,body): return self.adapter.projection.source_adapter.register(body, - self.adapter.session.snapshot['checkpoint'],self.phase+'-source') + self.adapter.session.summary['checkpoint'],self.phase+'-source') class _PreparationModels: @@ -278,11 +278,11 @@ def _read_blob(self,ref,item,maximum=40*1024**2):return self.adapter.materials.b def create(self,body): adapter=self.adapter return adapter.materials.mutate_definition('model_definition','models.create',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) + adapter.session.summary['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) def publish(self,identity,body): adapter=self.adapter return adapter.materials.publish_model(adapter.projection.models,identity,body, - adapter.session.snapshot['checkpoint'],self.phase+'-publish') + adapter.session.summary['checkpoint'],self.phase+'-publish') class _PreparationPipelines: @@ -295,7 +295,7 @@ def recipe_material(self,ref,*,verify=False):return self.adapter.materials.recip def create_external(self,body): adapter=self.adapter return adapter.materials.mutate_definition('pipeline_graph','graphs.create_external',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) + adapter.session.summary['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) def prepare_external_material(self,*args,**kwargs):return self.owner.prepare_external_material(*args,**kwargs) def _retain_material(self,prepared): adapter=self.adapter;value=self.owner._retain_material(prepared) @@ -310,7 +310,7 @@ def __init__(self,projection,execution): self.projection,self.execution=projection,execution self.session,self.db,self.history=projection.session,projection.db,projection.history self.materials=projection.materials;self.access=execution.runtime_access - self.origin=self.session.snapshot['project_id'] + self.origin=self.session.summary['project_id'] self._dirty=set();self._revisions={};self._epochs={};self._pending_epochs={} self._package_records=[] self.context=copy.copy(projection.context) @@ -573,7 +573,7 @@ def approve(self,study_id,body,expected_checkpoint,key): validate_operation_result(prior['operation'],prior['result'],self.session.snapshot['records']) self._current_grant(prior['result'],current.actor_id) else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self._materialize_trial(study_id,body['trial_id'],body['expected_material_sha256']) def invoke(): grant=self.studies.approve(study_id,body) @@ -597,7 +597,7 @@ def admit(self,study_id,body,expected_checkpoint,key): 'SELECT run_id FROM model_attempt_reservations WHERE run_id=%s',(result.get('run_id'),)).fetchone() is None): _invalid('A historical admission receipt cannot create a current local attempt.','unavailable') else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self.require_current_epoch(self.studies._study(study_id)) self.reject_historical_request(study_id,authority.actor_id,key,None) trial=self._materialize_trial(study_id,body['trial_id']) @@ -626,7 +626,7 @@ def invoke(): # Qualification may launch the explicit probe. It finishes # before Session opens the finite admission transaction. with preparation.admission(pipelines,ref,expected_checkpoint,key): - return self._mutate('study_execution.admit',study_id,body,self.session.snapshot['checkpoint'],key,invoke,'inspect') + return self._mutate('study_execution.admit',study_id,body,self.session.summary['checkpoint'],key,invoke,'inspect') return self._mutate('study_execution.admit',study_id,body,expected_checkpoint,key,invoke,'inspect') def stop(self,study_id,body,expected_checkpoint,key): @@ -645,7 +645,7 @@ def invoke(): # cancellation phase. Receipt replay does not repeat effects. for run_id in invoked: cancel_key='study-stop-'+hashlib.sha256((key+':'+run_id).encode()).hexdigest() - try:self.execution.cancel_run(run_id,self.session.snapshot['checkpoint'],cancel_key) + try:self.execution.cancel_run(run_id,self.session.summary['checkpoint'],cancel_key) except BaseException as error: error.study_stop_receipt=copy.deepcopy(result) raise @@ -683,7 +683,7 @@ def reconcile_preparation(self,study_id,body,expected_checkpoint): if complete.get('native_receipt'): effective=_preparation_body(intent,body['preparation_key']) effective.pop('request_key');effective['actor_id']=intent['actor_id'] - result=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],body['preparation_key'], + result=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],body['preparation_key'], lambda:copy.deepcopy(complete['result']),'cancel') else:result=copy.deepcopy(complete['result']) guard.confirmed() @@ -703,7 +703,7 @@ def abandon_preparation(self,study_id,body,expected_checkpoint,key): operation,current_intent,complete=_preparation_reference(self.session.snapshot['records'],study_id,body['preparation_key']) if current_intent!=intent:_invalid('Preparation intent changed during guarded reconciliation.','conflict') if complete is not None:_invalid('Preparation is complete; reconcile its acknowledgement without abandoning science.','conflict') - observed_checkpoint=self.session.snapshot['checkpoint'] + observed_checkpoint=self.session.summary['checkpoint'] def invoke(): actor=self.access.require('cancel','studies.abandon_preparation',study_id,owner_only=True) if actor.actor_id!=current.actor_id:_invalid('Abandonment caller changed.','conflict') @@ -741,7 +741,7 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): # acknowledges science and cannot repeat filesystem effects. return self._mutate(operation,study_id,effective,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Completed preparation replay invoked effects.')),'edit') - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') receipts=operation_records(self.session.snapshot['records']) if any(_intent_key(other,key) in receipts for other in PREPARATIONS if other!=operation): _invalid('Preparation key already identifies another preparation operation.','conflict') @@ -778,7 +778,7 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): publication=SimpleNamespace(runtime_snapshot=snapshot,before_write=guard.publication_started, read_recipe_blob=self.materials.blob) result=_prepare_operation(studies,study_id,dict(body,request_key=key),kind,publication=publication) - value=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],key, + value=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(result),'edit') guard.confirmed() return value diff --git a/project_text.py b/project_text.py index d9502cb6..fc778528 100644 --- a/project_text.py +++ b/project_text.py @@ -252,7 +252,7 @@ def restore(self, snapshot=None): encode(row['annotation']) if row['annotation'] is not None else None, row['review'], row['revision'], row['created_at'], row['updated_at'])) for state in doc['history']: self._workbench._history(state) - if self.store.snapshot()['checkpoint'] != current['checkpoint']: + if self.store.summary()['checkpoint'] != current['checkpoint']: raise ProjectTextError('Project changed during reconstruction.', 'stale_project', 409) self._snapshot, self.available = current, True return copy.deepcopy(current) diff --git a/project_transfer.py b/project_transfer.py index 10a6b431..91848559 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -720,7 +720,7 @@ def replay(opened): try: store=ProjectStore.create_at(destination_parent_fd,stage_name,name,'extraction-stage.'+digest(request_key.encode())[:32],request_digest,root_path=root_path) with store.root_directory() as owned_fd:stage_identity=_fd_identity(owned_fd) - destination=store.snapshot()['project_id'];created_at=timestamp() + destination=store.summary()['project_id'];created_at=timestamp() from project_data_fork import fork_records as fork_data from project_material_fork import fork_records as fork_material original=copy.deepcopy(records);mappings=[] @@ -755,7 +755,7 @@ def replay(opened): with source.artifacts.open_verified(value,maximum_bytes=value['bytes']) as fd:artifacts.publish_fd(fd,value,maximum_bytes=value['bytes']) with store.root_directory() as fd:materialize_captured(records,destination,fd) _validate_candidate(records,destination,artifacts) - committed=store.commit(records,store.snapshot()['checkpoint'],request_key,request_digest) + committed=store.commit(records,store.summary()['checkpoint'],request_key,request_digest) verification=_cold_verify(store,source) store.save(committed['checkpoint']);_assert_unchanged(source) _rename_noreplace(destination_parent_fd,stage_name,directory);published=True;os.fsync(destination_parent_fd) diff --git a/project_workspace.py b/project_workspace.py index 072d6c24..b006dbe2 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -122,7 +122,7 @@ def _state(self): project = None if self._recovery is None and self._store is not None: try: - snapshot = self._store.snapshot() + snapshot = self._store.summary() except ProjectStoreError as error: if error.code not in ('unknown', 'outcome_unknown', 'stale', 'stale_project', 'invalid', 'unsupported', 'unavailable'): @@ -184,7 +184,7 @@ def _bound(self, arguments, *, may_be_empty=False, check_checkpoint=True, allow_ if arguments.get('expected_checkpoint') is not None: raise WorkbenchError('The empty workspace has no checkpoint.', 'stale_project', 409) return - snapshot = self._store.snapshot() + snapshot = self._store.summary() expected = arguments.get('expected_checkpoint', arguments.get('checkpoint')) if check_checkpoint and expected != snapshot['checkpoint']: raise WorkbenchError('The project checkpoint changed. Refresh before acting.', 'stale_project', 409) @@ -206,7 +206,7 @@ def _root(self, alias, relative): def _projection_for(self, store): identity = uuid.uuid4().hex path = self.path / '.project-projections' / identity - snapshot = store.snapshot() + snapshot = store.summary() projection = Metadata(path, dsn=self._dsn, dataset_id=snapshot['project_id'], projection_id=identity) try: owner = ProjectProjection(store, projection, self.access, @@ -227,7 +227,7 @@ def _retained_projection_for(self, store, projection_id): if type(projection_id) is not str or not re.fullmatch('[a-f0-9]{32}', projection_id): raise WorkbenchError('Invalid original local projection identity.', 'unavailable', 409) db = Metadata(self.path / '.project-projections' / projection_id, dsn=self._dsn, - create=False, dataset_id=store.snapshot()['project_id'], projection_id=projection_id) + create=False, dataset_id=store.summary()['project_id'], projection_id=projection_id) try: return ProjectProjection(store, db, self.access, {alias: value[0] for alias, value in self._roots.items()}, @@ -243,7 +243,7 @@ def _detach(self): self._projection.execution.assert_quiescent() if self._store is not None: # Prove pending project publications and their full closure first. - self._store.save(self._store.snapshot()['checkpoint']) + self._store.save(self._store.summary()['checkpoint']) if self._projection is not None: self._projection.close() if self._store is not None: @@ -272,14 +272,14 @@ def _recover_current(self, arguments): store = projection = None try: store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) - snapshot = store.snapshot() + snapshot = store.summary() if snapshot['project_id'] != self._last_project['project_id']: raise WorkbenchError('Recovery destination has a different project identity.', 'stale_project', 409) # A reread reconciles identity; only a successful durability barrier # resolves a prior checkpoint/directory fsync failure. store.save(snapshot['checkpoint']) projection, text = self._projection_for(store) - snapshot = store.snapshot() + snapshot = store.summary() if self._projection is not None: self._projection.close() self._store, self._projection, self._text = store, projection, text @@ -332,7 +332,7 @@ def _select(self, action, arguments): if store.reconcile(arguments['request_key'], digest) is None: raise error reconciled = True - snapshot = store.snapshot() + snapshot = store.summary() if reconciled: store.save(snapshot['checkpoint']) created = snapshot @@ -342,7 +342,7 @@ def _select(self, action, arguments): self._detach() self._store, self._projection, self._text = store, projection, text self._selection = selection - snapshot = store.snapshot() + snapshot = store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} self._recovery = None self._binding = uuid.uuid4().hex @@ -366,7 +366,7 @@ def _select(self, action, arguments): raise def _record_result(self, record, *, before_revision=None, history_limit=1): - snapshot = self._store.snapshot() + snapshot = self._store.summary() history = [item for item in self._text.history(record['id']) if item['revision'] <= record['revision'] and (before_revision is None or item['revision'] < before_revision)] @@ -402,14 +402,14 @@ def _publish_independent(self, arguments, *, composed=False): if composed: from project_composition import publish_composition from project_artifacts import ProjectArtifacts - others=[];identities={self._store.snapshot()['project_id']} + others=[];identities={self._store.summary()['project_id']} for selected in arguments['sources']: if selected['project_id'] in identities: raise WorkbenchError('Composition source identity repeats.','conflict',409) identities.add(selected['project_id']) directory,descriptor=self._root(selected['root'],selected['directory']) store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory)) - actual=store.snapshot() + actual=store.summary() if actual['project_id']!=selected['project_id'] or actual['checkpoint']!=selected['checkpoint']: raise WorkbenchError('Selected composition source checkpoint differs.','stale_project',409) artifacts=retained.enter_context(ProjectArtifacts(store)) @@ -519,7 +519,7 @@ def dispatch(self, name, arguments, *, selected_version=None): if getattr(error, 'code', None) in ('unknown', 'outcome_unknown', 'projection_unavailable'): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} result = dict(binding=self._binding, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint'], value=value) @@ -629,14 +629,14 @@ def _dispatch_draining(self, name, arguments): raise WorkbenchError('The retained physical recovery owner changed.', 'local_custody_unavailable', 409) recovering = True if incoming is not projection and name == 'project_files.execution_reconcile_current': - value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.snapshot['checkpoint']) + value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.summary['checkpoint']) else: internal = arguments if recovery_operation and recovering and name != 'project_files.execution_reconcile_current': # Public CAS above uses the last acknowledged workspace # view. Only this qualified local owner sees its exact # selected/attempted checkpoint for finite recovery. - internal = dict(arguments, expected_checkpoint=incoming.session.snapshot['checkpoint']) + internal = dict(arguments, expected_checkpoint=incoming.session.summary['checkpoint']) if name == 'project_files.runtime_capture' and recovering: options = {'reserve_bytes': internal['reserve_bytes']} if 'reserve_bytes' in internal else {} value = incoming.runtime.recover_capture({'source': internal['source']}, @@ -660,7 +660,7 @@ def _dispatch_draining(self, name, arguments): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise with self.access.lock, projection.session.lock, projection.db.lock: - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} if name in _RECOVERY_OPERATIONS: self._recovery = None diff --git a/tests/test_project_dispatch.py b/tests/test_project_dispatch.py index f55de4cb..8aab246d 100644 --- a/tests/test_project_dispatch.py +++ b/tests/test_project_dispatch.py @@ -30,7 +30,8 @@ def require(self, *args, **kwargs): def projection(): access = Authority() - session = SimpleNamespace(access=access, check=Mock(), snapshot={'project_id': PROJECT, 'records': []}) + session = SimpleNamespace(access=access, check=Mock(), snapshot={'project_id': PROJECT, 'records': []}, + summary={'project_id': PROJECT, 'name': 'Dispatch fixture', 'checkpoint': CHECKPOINT}) return SimpleNamespace(session=session, data=Mock(), materials=Mock(), history=Mock(), context=SimpleNamespace(models=Mock(), pipelines=Mock()), artifacts=Mock(), producers=Mock(), source_adapter=Mock(), runtime_manifest=Mock(), disclose=Mock(), resolve_artifact=Mock(), execution=SimpleNamespace(study_execution=Mock(), producer_execution=Mock(), health=Mock(), preflight_external=Mock(), approve_external=Mock(), submit_external=Mock(), cancel_run=Mock(), reconcile_current=Mock())) @@ -38,7 +39,7 @@ def projection(): class Dispatch(unittest.TestCase): def test_supported_operations_match_the_finite_public_contract(self): - self.assertEqual(SUPPORTED_OPERATIONS, agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | (agent_api.PROJECT_FILES_M4 - {'project_files.save_as'})) + self.assertEqual(SUPPORTED_OPERATIONS, agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | (agent_api.PROJECT_FILES_M4 - {'project_files.save_as', 'project_files.compose'})) with self.assertRaises(WorkbenchError) as failure: dispatch(object(), 'project_files.runs_execute', {}) self.assertEqual(failure.exception.code, 'unsupported') diff --git a/tests/test_project_files_fork_contracts.py b/tests/test_project_files_fork_contracts.py index 309f8ed2..b731de2c 100644 --- a/tests/test_project_files_fork_contracts.py +++ b/tests/test_project_files_fork_contracts.py @@ -35,8 +35,9 @@ def test_old_pins_stay_exact_and_only_v2_accepts_closed_fork_origins(self): health = 'project_files.execution_health' self.assertEqual(api.contract(health,1)['sha256'], 'c3b1d0017f5a49cf69c3a4c5c33e1d3db9846404d12627563d97ef40de7404a5') + comparison = 'project_files.measurement_compare' self.assertEqual(versions,{name:{str(v):api.contract(name,v)['sha256'] for v in (1,2)} - for name in (*expected, health)}) + for name in (*expected, health, comparison)}) def test_sdk_selects_v2_and_falls_back_to_exact_v1_without_falsifying_provenance(self): value=fork_result();raw=client({'status':200,'result':value});sdk=ProjectFilesClient(raw) diff --git a/tests/test_project_material.py b/tests/test_project_material.py index 8fe2d11a..5e412305 100644 --- a/tests/test_project_material.py +++ b/tests/test_project_material.py @@ -55,7 +55,8 @@ def setUp(self): self.addCleanup(lambda:self.store.close()) self.artifacts=ProjectArtifacts(self.store);self.addCleanup(lambda:self.artifacts.close()) self.catalog=self.store.snapshot()['project_id'] - self.session=SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,check=lambda:None) + self.session=SimpleNamespace(snapshot=self.store.snapshot(),summary=self.store.summary(), + artifacts=self.artifacts,check=lambda:None) self.material=ProjectMaterial(self.session) def capture(self): diff --git a/tests/test_project_material_fork.py b/tests/test_project_material_fork.py index 89681263..d28881ac 100644 --- a/tests/test_project_material_fork.py +++ b/tests/test_project_material_fork.py @@ -52,7 +52,8 @@ def overlay(records, made, source, checkpoint, destination): class MaterialForkCodecs(unittest.TestCase): def setUp(self): self.source=uuid.uuid4().hex;self.checkpoint='a'*64 - self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]),check=lambda:None) + self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]), + summary=dict(project_id=self.source,checkpoint=self.checkpoint),check=lambda:None) self.material=ProjectMaterial(self.session) def capture(self,family,current): @@ -108,7 +109,8 @@ def test_bound_drafts_preserve_full_configuration_as_visible_nonexecutable_inten ('pipeline_graph',graph(external=True)),('recipe_definition',recipe(self.source))] captured=[self.capture(f,v) for f,v in drafts] made,combined,destination=self.fork() - session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined),check=lambda:None) + session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined), + summary=dict(project_id=destination),check=lambda:None) reader=ProjectMaterial(session) intents=reader.intents() self.assertEqual(len(intents),4) @@ -167,7 +169,8 @@ def test_captured_source_selector_stays_in_inert_intent_and_cannot_publish(self) doc=made['records'][0]['document'] self.assertEqual(doc['current']['required_reselection'],['source_selector']) self.assertEqual(doc['current']['definition']['source_selector'],value['source_selector']) - reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined),check=lambda:None)) + reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined), + summary=dict(project_id=destination),check=lambda:None)) with self.assertRaises(WorkbenchError) as caught: reader.publish_model(SimpleNamespace(publish=lambda *args: self.fail('Intent reached owner publication')), doc['current']['id'],{'revision':1},'a'*64,'forbidden-intent') diff --git a/tests/test_project_native_process.py b/tests/test_project_native_process.py index 287de66a..5df959c4 100644 --- a/tests/test_project_native_process.py +++ b/tests/test_project_native_process.py @@ -5,6 +5,7 @@ import os from pathlib import Path import select +import shutil import signal import sys import tempfile @@ -311,8 +312,11 @@ def test_current_interpreter_alias_drift_before_launch_cannot_select_another_int stage=self.root/'venv-alias-drift' prepared=native.prepare(stage,phase='probe',attempt_id=uuid.uuid4().hex,request_sha256='d'*64, runtime=runtime,worker='text_training.py',sources={'text_training.py':WORKER},wall_seconds=8) - Path(executable).unlink();Path(executable).symlink_to(Path('/usr/bin/python3').resolve()) - with self.assertRaises(custody.ProcessError) as error:prepared.launch(stage/'custody') + replacement=self.root/'replacement-python' + shutil.copy2(runtime['executable_source'],replacement) + self.assertNotEqual(replacement.resolve(),Path(executable).resolve()) + Path(executable).unlink();Path(executable).symlink_to(replacement) + with self.assertRaises(custody.ProcessError) as error:self.launch(prepared) self.assertEqual(error.exception.code,'stale');self.assertFalse((stage/'custody').exists()) def test_spawn_callback_is_not_reached_for_current_config_alias_or_capture_refusal(self): diff --git a/tests/test_project_session.py b/tests/test_project_session.py index 309f2bc4..75ff7795 100644 --- a/tests/test_project_session.py +++ b/tests/test_project_session.py @@ -174,6 +174,23 @@ def test_complete_merge_and_original_result_replay_before_stale_cas(self): self.assertIn(('project_artifact', other['sha256']), ids) self.assertEqual(self.artifacts.read_blob({key: declaration[key] for key in ('sha256', 'bytes')}), 'Irreplaceable observation 山'.encode()) + def test_scalar_summary_is_independent_and_tracks_the_bound_publication(self): + before = self.session.summary + arguments, _declaration, _calls = self.mutation() + self.call(arguments) + snapshot = self.session.snapshot + expected = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} + self.assertNotEqual(before['checkpoint'], expected['checkpoint']) + original_copy = copy.deepcopy + def observe(value): + self.assertIsNot(value, self.session._snapshot, 'Summary must not clone the record graph') + return original_copy(value) + with patch('project_session.copy', SimpleNamespace(deepcopy=observe)): + first = self.session.summary + first['name'] = 'Caller mutation' + self.assertEqual(self.session.summary, expected) + self.assertEqual(self.session.snapshot['name'], expected['name']) + def test_key_conflict_stale_and_current_auth_refuse_before_invoke(self): first, _declaration, _calls = self.mutation() self.call(first) diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 9dcf11af..f3e7f0e2 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -97,6 +97,18 @@ def test_appending_one_record_does_not_rewrite_saved_immutable_revisions(self): 'Appending one record must not write disposable copies of every saved immutable revision') self.assertEqual(self.reopen(), committed) + def test_summary_reads_every_current_record_without_cloning_the_record_graph(self): + committed = self.commit([record('a'), record('b')], key='summary-fixture') + expected = {key: committed[key] for key in ('project_id', 'name', 'checkpoint')} + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused record graph copy')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: + first = self.store.summary() + first['name'] = 'Caller mutation' + second = self.store.summary() + self.assertEqual(second, expected) + self.assertEqual(reads.call_count, 4, 'Each summary must still read/hash both actual records') + self.assertEqual(self.store.snapshot(), committed) + def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_identical_records(self): rows=[record('a'),record('b','Exact second note\n')] self.commit(rows,key='repeat-read-fixture') @@ -124,6 +136,7 @@ def test_verified_decode_cache_still_rejects_same_length_corruption_with_restore path.chmod(0o600);path.write_bytes(changed) os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) self.assert_code('stale',self.store.snapshot) + self.assert_code('stale',self.store.summary) with self.assertRaises(ProjectStoreError) as caught: with self.store.root_directory(verify_after=True):pass self.assertEqual(caught.exception.code,'stale') @@ -250,12 +263,14 @@ def test_replaced_physical_root_or_metadata_directory_is_rejected(self): self.root.rename(displaced) self.root.mkdir() self.assert_code('stale', self.store.snapshot) + self.assert_code('stale', self.store.summary) self.root.rmdir() displaced.rename(self.root) meta = self.root / '.tuldok' meta.rename(self.root / '.old-tuldok') meta.mkdir() self.assert_code('stale', self.store.snapshot) + self.assert_code('stale', self.store.summary) def test_changed_checkpoint_is_rejected_before_mutation(self): initial = self.store.snapshot() diff --git a/tests/test_project_workspace.py b/tests/test_project_workspace.py index 0c2dbaee..a78982f4 100644 --- a/tests/test_project_workspace.py +++ b/tests/test_project_workspace.py @@ -72,6 +72,17 @@ def create(self, directory='alpha'): arguments = self.mutation(root='fixture', directory=directory, name='Workspace authored project') return self.call('new', arguments), arguments + def test_status_returns_current_metadata_without_copying_the_record_graph(self): + self.create() + self.import_record('A retained record unrelated to the project summary') + expected = self.state() + with patch.object(self.workspace._store, '_snapshot', + wraps=self.workspace._store._snapshot) as snapshots: + observed = self.state() + self.assertEqual(observed, expected) + self.assertEqual(snapshots.call_count, 0, + 'Status must not copy every retained record to return three metadata fields') + def test_control_project_ancestry_is_rejected_before_destination_or_projection_creation(self): with ProjectStore.create(self.external/'science','Existing portable project','existing-project','b'*64):pass self.start(self.external/'science/control') From 9f08a248378f6ad1d65061976c6015bde16b7e56 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 00:15:42 -0700 Subject: [PATCH 08/14] perf(projects): verify freshness without discarded record copies --- project_data.py | 8 +-- project_execution.py | 22 +++---- project_extraction.py | 2 +- project_files.py | 2 +- project_history.py | 2 +- project_material.py | 4 +- project_measurements.py | 14 ++-- project_native_execution.py | 8 +-- project_native_preparation.py | 12 ++-- project_producer_execution.py | 4 +- project_producers.py | 4 +- project_projection.py | 6 +- project_runtime.py | 6 +- project_session.py | 17 ++++- project_sources.py | 12 ++-- project_study_execution.py | 10 +-- project_transfer_controller.py | 2 +- tests/test_project_extraction.py | 6 +- tests/test_project_files.py | 10 +++ tests/test_project_fork.py | 4 +- tests/test_project_fork_composition.py | 2 +- tests/test_project_history.py | 4 +- tests/test_project_material.py | 14 ++-- tests/test_project_material_fork.py | 6 +- tests/test_project_native_extraction.py | 4 +- tests/test_project_session_freshness.py | 88 +++++++++++++++++++++++++ tests/test_project_transfer.py | 2 +- 27 files changed, 192 insertions(+), 83 deletions(-) create mode 100644 tests/test_project_session_freshness.py diff --git a/project_data.py b/project_data.py index 6c99e8b2..ca70bab0 100644 --- a/project_data.py +++ b/project_data.py @@ -1028,7 +1028,7 @@ def _mutate(self, operation, target, body, expected_checkpoint, key, invoke): fail('Data mutation body must be an object.') self._permission(operation, target, body) with self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() if operation in ('data.annotate','data.correct_rights_note','data.save_image_source','data.delete_image_source'): self._require_record(target,extended=operation!='data.annotate') legacy = self._legacy_replay(operation, target, body, key) @@ -1158,10 +1158,10 @@ def freeze_release(self, body, expected_checkpoint, key): def _read(self, action, invoke, target=None): with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() self._sync_sources() result = invoke() - self.session.check() + self.session.check_current() return result def query(self, options=None): @@ -1399,7 +1399,7 @@ def restore(self, snapshot=None): self.db.execute('INSERT INTO saved_searches VALUES (%s,%s,%s,%s,%s,%s)', (record_id,r['name'],r['revision'],encode_retained(r['criteria']),r['created_at'],r['updated_at'])) elif family == 'dataset_exposure': self.db.execute('INSERT INTO workspace_exposures VALUES (%s,%s,%s,%s,%s,%s,%s)', tuple(doc[k] for k in ('id','record_id','source_hash','reason','run_id','release_id','created_at'))) - self.session.check() + self.session.check_current() self._materialize_images_and_releases(owned) return snapshot diff --git a/project_execution.py b/project_execution.py index fece88a1..0e4941c6 100644 --- a/project_execution.py +++ b/project_execution.py @@ -52,7 +52,7 @@ def publication_started(self): def confirmed(self): if not self.started:raise RuntimeError('File publication must have a durable start before confirmation.') - execution=self.execution;execution.session.check() + execution=self.execution;execution.session.check_current() if execution.db.in_transaction: raise RuntimeError('File publication confirms only after its canonical and outer SQL acknowledgement.') if execution.custody is not None: @@ -164,7 +164,7 @@ def register_custody(self, identity, handle): def before_launch(self, identity): with self.db.lock: - self.session.check();run=self.runs._load(identity);self._proof(run) + self.session.check_current();run=self.runs._load(identity);self._proof(run) self.runtime_access.require_job('runs',identity,run['admission_id'],run['request_sha256']) if self.custody is not None: with self._journal_guard():self.custody.launch_intent(identity) @@ -176,7 +176,7 @@ def _publication_intent(self, before, records, key, sha): def _local_owner(self, *, recover=True): """Construct execution only after the caller checked exact authority.""" - self.session.check() + self.session.check_current() if self._runs is None: from runs import Runs from pipeline import Pipelines @@ -398,7 +398,7 @@ def _producer_authority(self, family, identity, admission_id, request_digest): def register_producer(self, family, identity, admission_id, request_digest): with self.db.lock: - self.session.check();self._producer_authority(family,identity,admission_id,request_digest) + self.session.check_current();self._producer_authority(family,identity,admission_id,request_digest) if self.custody is not None: with self._journal_guard(): if not self.custody.has_pending:self.custody.acknowledge(self.session.snapshot) @@ -406,7 +406,7 @@ def register_producer(self, family, identity, admission_id, request_digest): def producer_effect_intent(self, family, identity): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -416,7 +416,7 @@ def producer_effect_intent(self, family, identity): def update_producer_outcome(self, family, identity, provider_outcome, worker_drained): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -591,7 +591,7 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt def preflight_external(self, graph_id, revision): self.runtime_access.require('inspect', 'pipeline.preflight_external', graph_id) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() return self.projection.pipelines.preflight_external(graph_id, revision) def native_preflight(self,graph_id,revision,expected_checkpoint,key): @@ -615,7 +615,7 @@ def native_submit(self,body,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.submit',body['graph_id'],owner_only=True) effective=dict(body,actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.submit',body['graph_id'],effective,expected_checkpoint,key) if replay is not None:return replay with self.projection.native_preparation.current_graph(self.projection.pipelines,body['graph_id'], @@ -633,7 +633,7 @@ def native_retry(self,run_id,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.retry',run_id,owner_only=True) body=dict(actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.retry',run_id,body,expected_checkpoint,key) if replay is not None:return replay if self._runs is None:raise WorkbenchError('Retry requires the actual current local Run owner.','unavailable',409) @@ -655,7 +655,7 @@ def native_reconcile(self,phase_id,expected_checkpoint): def submit_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require_external_submission() with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) @@ -678,7 +678,7 @@ def invoke(): def approve_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require('approve-execution', 'runs.approve_external', owner_only=True) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) diff --git a/project_extraction.py b/project_extraction.py index 82db29c7..461b0d0f 100644 --- a/project_extraction.py +++ b/project_extraction.py @@ -69,7 +69,7 @@ def __init__(self, source): self.payload_gaps = [] self.deleted_intents = set() self.session = SimpleNamespace(snapshot=source.snapshot, db=self.db, - artifacts=source.artifacts, check=lambda: None) + artifacts=source.artifacts, check_current=lambda: None) # Workbench and each metadata-view constructor only assign references; # none runs SQL, creates directories or constructs Dataset/Jobs/Runs. # Never call get/history: those public owners may enroll images. diff --git a/project_files.py b/project_files.py index c6e9f6b6..e2a5585b 100644 --- a/project_files.py +++ b/project_files.py @@ -264,7 +264,7 @@ def inspect(self): self.access.require('inspect','files.inspect');snapshot=self.session.check() with self.store.root_directory() as root: items=inspect_captured(snapshot['records'],snapshot['project_id'],root) - self.session.check() + self.session.check_current() return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) def read(self,record_id,revision=None): diff --git a/project_history.py b/project_history.py index 31c5f645..8733ca4a 100644 --- a/project_history.py +++ b/project_history.py @@ -752,7 +752,7 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): self.materials=materials;self.authorize_disclosure=authorize_disclosure def _facts(self): - self.session.check() + self.session.check_current() records=self.session.snapshot['records'] return decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) diff --git a/project_material.py b/project_material.py index 7cba65fe..fbfa0189 100644 --- a/project_material.py +++ b/project_material.py @@ -359,7 +359,7 @@ def __init__(self, session): self.owner_access = _MaterialAccess(session) if hasattr(session,'access') else None def _documents(self): - self.session.check() + self.session.check_current() return decode_records(self.session.snapshot['records']) def read_view(self, records): @@ -369,7 +369,7 @@ def read_view(self, records): live publication, verification or execution capability; later reads must select their own checkpoint and construct a new view. """ - self.session.check() + self.session.check_current() return MaterialSnapshot(records) @_boundary diff --git a/project_measurements.py b/project_measurements.py index 2168d1b7..400db842 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -124,7 +124,7 @@ def import_packet(self, raw, source, expected_checkpoint, key): """Called only after SourceAdapter's explicit current rooted capture.""" with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'measurements.import') - self.session.check() + self.session.check_current() table = MeasurementTable(raw) blob = dict(sha256=table.sha256, bytes=len(raw)) origin = self.session.summary['project_id'] @@ -152,11 +152,11 @@ def sources(self, *, offset=0, limit=40): raise WorkbenchError('Invalid measurement source page.') with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', 'measurements.sources') - self.session.check() + self.session.check_current() docs = sorted(self._documents().values(), key=lambda d: (d['origin_catalog_id'], d['id'])) items = [dict(doc, availability=self.artifacts.inspect(declaration(doc['blob']))) for doc in docs[offset:offset+limit]] - self.session.check() + self.session.check_current() self.access.require('inspect', 'measurements.sources') return self._bounded(dict(items=items, total=len(docs), offset=offset, limit=limit)) @@ -172,7 +172,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): target = origin_catalog_id + '.' + packet_sha256 with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() doc = self._documents().get(target) if doc is None: raise WorkbenchError('Measurement source is unavailable.', 'not_found', 404) @@ -181,7 +181,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): if table.metadata()['title'] != doc['title']: raise WorkbenchError('Measurement title differs from the exact retained packet.') value = self._bounded(dict(source=doc, value=select(table))) - self.session.check() + self.session.check_current() self.access.require('inspect', action, target) return value @@ -215,7 +215,7 @@ def compare(self, sources, metric_id=None, **selection): with self.access.lock, self.session.lock, self.db.lock: for target in targets: self.access.require('inspect', 'measurements.compare', target) - self.session.check() + self.session.check_current() documents = self._documents() if any(target not in documents for target in targets): raise WorkbenchError('A comparison source is unavailable.', 'not_found', 404) @@ -231,7 +231,7 @@ def compare(self, sources, metric_id=None, **selection): tables.append((locator, table)) value = MeasurementComparison(tables).read(metric_id, **selection) result = self._bounded(dict(sources=chosen, value=value)) - self.session.check() + self.session.check_current() for target in targets: self.access.require('inspect', 'measurements.compare', target) return result diff --git a/project_native_execution.py b/project_native_execution.py index 9ac6be4a..5895e4d7 100644 --- a/project_native_execution.py +++ b/project_native_execution.py @@ -452,7 +452,7 @@ def identity(info): def _publish(self, document, state, *, helper=None, receipt_sha256=None): with self.db.lock,self.db.transaction(): - self.session.check();actor=self._authorize(document['phase_id']) + self.session.check_current();actor=self._authorize(document['phase_id']) document['events'].append(dict(state=state,at=timestamp(),actor_id=actor.actor_id, helper=copy.deepcopy(helper),receipt_sha256=receipt_sha256)) self._dirty[document['phase_id']]=copy.deepcopy(document) @@ -788,7 +788,7 @@ def _probe_result(recipe, inputs, evidence): def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, run_id=None, job=None, phase_id=None): if self.db.in_transaction:_fail('Native phase cannot launch inside a metadata transaction.','busy') with self.db.lock: - self.session.check() + self.session.check_current() if run_id is None and self._foreground_phase is not None: _fail('Another explicit native preparation phase is active.','busy') if not self.db.claim_runtime('project_native_phases'):_fail('Another current native phase owner is active.','busy') @@ -818,7 +818,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru self._publish(document,'prepared') if self.execution.custody is None:_fail('Native launch requires its original physical custody journal.','unavailable') with self.db.lock,self.execution._journal_guard(): - self._dispatch_authorized(phase_id);self.session.check() + self._dispatch_authorized(phase_id);self.session.check_current() if not self.execution.custody.has_pending:self.execution.custody.acknowledge(self.session.snapshot) self.execution.custody.native_prepared(document) self.execution.custody.native_launch_intent(phase_id) @@ -847,7 +847,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru # Recheck actual authority after both durable ACKs. A # saved GO intent is never permission to retry this send. with self.db.lock: - self.session.check();self._dispatch_authorized(phase_id) + self.session.check_current();self._dispatch_authorized(phase_id) if not self.execution.custody.has_pending:_fail('Native local custody disappeared.','unknown') prepared.check_current_runtime() handle.go(handle.ready) diff --git a/project_native_preparation.py b/project_native_preparation.py index 7c504be0..24113f63 100644 --- a/project_native_preparation.py +++ b/project_native_preparation.py @@ -188,7 +188,7 @@ def validate_operation_result(operation,result,records=None): _fail('Native intent source or frozen input differs from its graph.','invalid') from types import SimpleNamespace from project_material import ProjectMaterial - retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check=lambda:None)) + retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None)) if result['model'] is not None:retained.model(result['model']) if result['composition'] is not None: composition=retained.recipe(result['composition'])['material'] @@ -230,7 +230,7 @@ def __init__(self,adapter,owner,graph,model,composition,purpose,key): def _check(self,recipe): if not self._active or _CURRENT.get() is not self or recipe!=self.graph['recipe']: _fail('An explicit current native preparation is required.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.probe_native',recipe,owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Native preparation caller changed.','conflict') if (os.environ.get('TULDOK_TRAINING_PYTHON') or sys.executable)!=self.configured_executable: @@ -297,7 +297,7 @@ def __init__(self,adapter,owner,graph): def consume_admission(self): if not self._active or self._consumed:_fail('Current integrity qualification has already been consumed.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.snapshot',self.graph['id'],owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Current integrity caller changed.','conflict') if self.owner.get(self.graph['id'])!=self.graph or self.owner.definition_identity(self.recipe)!=self.definition: @@ -333,7 +333,7 @@ def _scope(self,owner,graph,model,composition,purpose,expected_checkpoint,key): if self.db.in_transaction:_fail('Native preparation must precede the metadata transaction.','unsupported') if _CURRENT.get() is not None:_fail('A native preparation is already active.','busy') self.access.require('edit','native_preparation.'+purpose,graph['id'],owner_only=True) - self.session.check() + self.session.check_current() if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') context=NativePreparationContext(self,owner,graph,model,composition,purpose,_key(purpose,key)) result=dict(format='project_native_preparation_intent_v1',origin_catalog_id=self.session.project_id, @@ -361,7 +361,7 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): if graph['revision']!=revision:_fail('Graph changed before native validation.','stale') import release_validation if adapter_for_graph(graph) is release_validation: - self.session.check() + self.session.check_current() if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') value=owner.validate_graph(graph) return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, @@ -375,7 +375,7 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): def current_graph(self,owner,graph_id,revision,expected_checkpoint,key): if self.db.in_transaction:_fail('Current native validation must precede admission.','unsupported') self.access.require('edit','native_preparation.admission',graph_id,owner_only=True) - self.session.check() + self.session.check_current() if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') graph=owner.get(graph_id) if graph['revision']!=revision:_fail('Graph changed before native admission.','stale') diff --git a/project_producer_execution.py b/project_producer_execution.py index f4ae5915..e3729233 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -73,7 +73,7 @@ def _family(self, family): return family def _owner(self, family): - self._family(family);self.session.check() + self._family(family);self.session.check_current() if self._fenced: raise WorkbenchError('Current producer custody is uncertain. Reopen for inert inspection.', 'outcome_unknown', 409) self._refuse_unknown_provider() @@ -429,7 +429,7 @@ def status(self, *, offset=0, limit=40): raise WorkbenchError('Invalid current producer health page.') with self.session.access.lock,self.session.lock,self.db.lock: self.session.access.require('inspect','producers.health') - self.session.check() + self.session.check_current() self._settle_drained() items=[] for family,owner in self._owners.items(): diff --git a/project_producers.py b/project_producers.py index 10eb94be..8e052b34 100644 --- a/project_producers.py +++ b/project_producers.py @@ -592,9 +592,9 @@ def __init__(self, session): def _read(self, action, invoke): with self.session.access.lock, self.session.lock, self.session.db.lock: self.session.access.require('inspect', action) - self.session.check() + self.session.check_current() result = invoke(self.session.snapshot['records']) - self.session.check() + self.session.check_current() return result def list(self, family, *, origin_catalog_id=None, offset=0, limit=40): diff --git a/project_projection.py b/project_projection.py index 04aaebba..1bf45bd2 100644 --- a/project_projection.py +++ b/project_projection.py @@ -190,7 +190,7 @@ def resolve_artifact(self, body, expected_checkpoint, key): raise WorkbenchError('Choose one declared artifact and a current rooted file.') with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'artifacts.resolve', body['sha256'], owner_only=True) - self.session.check() + self.session.check_current() declaration = next((item['document'] for item in self.session.snapshot['records'] if item['family'] == 'project_artifact' and item['id'] == body['sha256']), None) if declaration is None: @@ -246,7 +246,7 @@ def runtime_configuration(self, request): def runtime_sources(self): with self.access.lock, self.session.lock, self.db.lock: self.access.require('admin', 'runtime.sources', owner_only=True) - self.session.check() + self.session.check_current() return dict(sources=sorted(self._runtime_sources)) def authorize_disclosure(self, run, population): @@ -317,7 +317,7 @@ def _tag(self, value): return dict(value, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint']) def records(self): - self.projection.session.check() + self.projection.session.check_current() from project_fork import active_records snapshot = self.projection.session.snapshot records = active_records(snapshot['records'], project_id=snapshot['project_id']) diff --git a/project_runtime.py b/project_runtime.py index 21fbd63f..d63a9eb5 100644 --- a/project_runtime.py +++ b/project_runtime.py @@ -156,7 +156,7 @@ def _authorize(self, action): def inspect(self, config_request): """Read an explicitly selected current source; never provision or execute.""" with self.access.lock, self.session.lock, self.db.lock: - self._authorize('runtime_snapshots.inspect'); self.session.check() + self._authorize('runtime_snapshots.inspect'); self.session.check_current() _body(config_request, '0'*64, 0) with self.execution.runtime_inspection(): return self.owner.inspect(self.resolve_configuration(copy.deepcopy(config_request))) @@ -165,7 +165,7 @@ def status(self, key): """Read historical outcome only; captured does not claim local readiness.""" _key(key) with self.access.lock, self.session.lock, self.db.lock: - self.access.require('inspect', 'runtime.capture_status'); self.session.check() + self.access.require('inspect', 'runtime.capture_status'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) intent = receipts.get(_intent_key(key)); complete = receipts.get(key) if intent is not None and intent['operation'] != 'runtime.capture_intent': intent = None @@ -248,7 +248,7 @@ def capture(self, config_request, expected_inventory_sha256, expected_checkpoint body_sha = _intent_input(body) intent = None with self.access.lock, self.session.lock, self.db.lock: - actor = self._authorize('runtime_snapshots.create'); self.session.check() + actor = self._authorize('runtime_snapshots.create'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) if any(r['operation']=='runtime.abandon' and r['result']['capture_key']==key for r in receipts.values()): _fail('This capture was abandoned; use a fresh key for a new measurement.', 'conflict', 409) diff --git a/project_session.py b/project_session.py index 308164c5..5fa571d1 100644 --- a/project_session.py +++ b/project_session.py @@ -206,15 +206,26 @@ def adopt(self, snapshot): self._snapshot = copy.deepcopy(snapshot) self.available = True - def check(self): - """Check freshness while a caller holds host/session/projection guards.""" + def _checked(self, read): if not self.available: raise ProjectSessionError('Reopen this project to reconstruct its projection.', 'projection_unavailable', 503) - current = self.store.snapshot() + current = read() if current['checkpoint'] != self._snapshot['checkpoint']: raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) return current + def check_current(self): + """Verify actual canonical bytes without returning an unused record graph. + + The caller holds the same host/session/projection guards as check(). + Store.summary reads and hashes all current records before returning. + """ + self._checked(self.store.summary) + + def check(self): + """Return independent current records under caller-owned projection guards.""" + return self._checked(self.store.snapshot) + def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capture, *, scope='edit'): """Invoke existing policy, capture fixed-owner upserts, publish, commit SQL. diff --git a/project_sources.py b/project_sources.py index 25007ea1..95e7c5f2 100644 --- a/project_sources.py +++ b/project_sources.py @@ -117,7 +117,7 @@ def register(self,body,expected_checkpoint,key): self._mutation_identity(expected_checkpoint,key) with self.access.lock,self.session.lock,self.db.lock: actor=self.access.require('edit','source.register',owner_only=True) - self.session.check() + self.session.check_current() saved,raw=self._read_registration(body) origin=self.session.summary['project_id'];invoked=False def invoke(): @@ -153,7 +153,7 @@ def references(self,snapshotref,offset=0,limit=40): raise WorkbenchError('Invalid bounded source reference page.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('inspect','source.references',snapshot['project_id']) - self.session.check() + self.session.check_current() historical=self.materials.snapshot_ref(snapshot['project_id'],snapshot['manifest_sha256'], catalog_id=snapshot['origin_catalog_id']) try:current=self.sources._saved(snapshot['project_id'],extensions=True) @@ -171,7 +171,7 @@ def references(self,snapshotref,offset=0,limit=40): selected=current['manifest']['data'][offset:offset+limit];budget=[INSPECTION_LIMIT] items=[self.sources._inspect(current['root'],copy.deepcopy(ref),budget,DATA_LIMIT) for ref in selected] self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) - self.session.check();self.access.require('inspect','source.references',snapshot['project_id']) + self.session.check_current();self.access.require('inspect','source.references',snapshot['project_id']) return dict(items=items,total=len(current['manifest']['data']),offset=offset,limit=limit) def import_scientific(self,body,expected_checkpoint,key): @@ -183,7 +183,7 @@ def import_scientific(self,body,expected_checkpoint,key): if type(body['document']) is not dict:raise WorkbenchError('Scientific import metadata must be an object.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_scientific',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],DATA_LIMIT) # ProjectData owns the one canonical mutation and native parser; # current rooted reads above never become an imported capability. @@ -204,7 +204,7 @@ def _capture_reference(self,snapshot,reference_id,expected_sha256,maximum_bytes) expected_sha256,maximum_bytes) if source['manifest_sha256']!=snapshot['manifest_sha256']: raise WorkbenchError('Current source snapshot changed during capture.','stale',409) - self.session.check() + self.session.check_current() return raw,source def import_measurements(self,body,expected_checkpoint,key): @@ -216,7 +216,7 @@ def import_measurements(self,body,expected_checkpoint,key): from retained_measurements import MAX_BYTES with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_measurements',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],MAX_BYTES) source=dict(source,origin_catalog_id=snapshot['origin_catalog_id']) self.access.require('edit','source.import_measurements',snapshot['project_id']) diff --git a/project_study_execution.py b/project_study_execution.py index fa6d6ce9..2133bbce 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -142,7 +142,7 @@ def _validate_preparation(operation,result,records): trial=result['trial'];origin=trial['material']['catalog_id'] candidate=facts.candidates[(origin,result['candidate_id'])]['candidate'] if candidate['study_id']!=trial['study_id']:_invalid('Preparation candidate belongs to another Study.') - materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check=lambda:None)) + materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check_current=lambda:None)) composition=materials.recipe(trial['material'])['material'] if (composition['model']!=result['model'] or composition['recipe_id']!=result['graph']['id'] or composition['recipe_revision']!=result['graph']['revision']): @@ -202,7 +202,7 @@ def validate_operation_result(operation,result,records=None): if saved is None or any(saved[key]!=reservation[key] for key in RESERVATION_FIELDS if key not in ('state','settled')): _invalid('Admission receipt has no exact canonical reservation.') trial=facts.trials[(origin,result['trial_id'])]['trial'];run=facts.runs[(origin,result['run_id'])]['run'] - view=SimpleNamespace(snapshot={'records':records},check=lambda:None) + view=SimpleNamespace(snapshot={'records':records},check_current=lambda:None) validate_retained_reservation(reservation,trial=trial, composition=ProjectMaterial(view).recipe(trial['material']),run=run) elif operation=='study_execution.stop': @@ -329,7 +329,7 @@ def _canonical(self,records=None): def restore_authoring(self,records=None): """Reconstruct intent only, under the current project guard.""" with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check();facts=self._canonical(records) + self.session.check_current();facts=self._canonical(records) with self.db.transaction(): for (origin,identity),doc in sorted(facts.studies.items()): if origin!=self.origin:continue @@ -589,7 +589,7 @@ def admit(self,study_id,body,expected_checkpoint,key): authority=self.access.require_external_submission() with self.session.access.lock,self.session.lock,self.db.lock: builtin=None - self.session.check();self.refresh_authoring() + self.session.check_current();self.refresh_authoring() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: result=prior['result'] @@ -734,7 +734,7 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): input_sha=digest(dict(study_id=study_id,body=effective)) intent_key=_intent_key(operation,key);phase=intent_key.removesuffix('-intent') with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check() + self.session.check_current() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: # Current authority is checked above. Historical completion diff --git a/project_transfer_controller.py b/project_transfer_controller.py index 7362e0ea..3ece224e 100644 --- a/project_transfer_controller.py +++ b/project_transfer_controller.py @@ -252,7 +252,7 @@ def extract(self, binding, report_sha256, destination_root, directory, name, req fail('Recover the selected workspace before reconciling extraction.', 'projection_unavailable', 503) guards.enter_context(projection.session.lock) guards.enter_context(projection.db.lock) - projection.session.check() + projection.session.check_current() opened = self.workspace._store receipt = self._transfer.extract(self._selection, report_sha256, destination_fd, directory, name, request_key, opened_store=opened) except BaseException as error: diff --git a/tests/test_project_extraction.py b/tests/test_project_extraction.py index 4ef0ff92..afa628a6 100644 --- a/tests/test_project_extraction.py +++ b/tests/test_project_extraction.py @@ -498,7 +498,7 @@ def test_two_study_lineage_reservation_candidate_and_run_science_are_captured(se before = self.files();source = self.view() captured = self.capture(source, {'studies': [child['id']]}) self.assertEqual(captured['blockers'], [], captured['blockers']) - view = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check=lambda: None) + view = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check_current=lambda: None) from project_material import ProjectMaterial facts = decode_history(captured['records'], materials=ProjectMaterial(view)) self.assertEqual(set(facts.studies), {(self.db.dataset_id, parent['id']), (self.db.dataset_id, child['id'])}) @@ -639,7 +639,7 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure captured = self.capture(source, {'runs': [consumer['id']]}) self.assertEqual(captured['blockers'], [], captured['blockers']) facts = decode_records(captured['records'], materials=ProjectMaterial(SimpleNamespace( - snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check=lambda: None))) + snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check_current=lambda: None))) self.assertEqual(set(facts.runs), {(self.db.dataset_id, run['id']) for run in (producer, middle, consumer)}) for downstream, upstream in ((consumer, middle), (middle, producer)): self.assertIn(dict(source=dict(family='run_history', id=self.db.dataset_id + '.' + downstream['id']), @@ -650,7 +650,7 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure # A fresh inert owner reader reconstructs the exact upstream custody # receipt from retained facts/private captured bytes, never source paths. session = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=copy.deepcopy(captured['records'])), - access=self.access, artifacts=source.artifacts, check=lambda: None) + access=self.access, artifacts=source.artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda run, population: self.access.require('expose-holdout', 'runs.expose_test', run['id'])) with patch.object(source, 'read_catalog', side_effect=AssertionError('No original catalog reread')), \ diff --git a/tests/test_project_files.py b/tests/test_project_files.py index 538501d0..49af9af2 100644 --- a/tests/test_project_files.py +++ b/tests/test_project_files.py @@ -44,6 +44,16 @@ def intent(self,path,text,revision=0,kind='note'): raw=text.encode('utf-8');target=self.root/path;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(raw) return dict(kind=kind,path=path,expected_revision=revision,expected_sha256=hashlib.sha256(raw).hexdigest()) + def test_inspection_copies_only_the_records_it_actually_consumes(self): + self.adopt('notes/inspection.md', 'Exact retained scientific note', kind='note') + expected = self.files.inspect() + with patch.object(self.session.store, '_snapshot', + wraps=self.session.store._snapshot) as snapshots: + observed = self.files.inspect() + self.assertEqual(observed, expected) + self.assertEqual(snapshots.call_count, 1, + 'Inspection needs one record snapshot; final freshness must not clone another') + def test_batch_captures_exact_files_in_one_canonical_mutation(self): items=[self.intent('notes/a.md','Exact Ω\r\n'),self.intent('notes/empty.md','')] before=self.session.snapshot['checkpoint'] diff --git a/tests/test_project_fork.py b/tests/test_project_fork.py index 1b4029b8..0683eef9 100644 --- a/tests/test_project_fork.py +++ b/tests/test_project_fork.py @@ -53,7 +53,7 @@ def setUp(self): capture.mkdir(parents=True) for name, raw in payloads.items(): (capture / name).write_bytes(raw) - session = SimpleNamespace(snapshot=self.source.snapshot(), artifacts=self.artifacts, check=lambda: None) + session = SimpleNamespace(snapshot=self.source.snapshot(), artifacts=self.artifacts, check_current=lambda: None) self.material = ProjectMaterial(session) self.record = self.material.capture_model_material(self.material_value, local) records = [*self.material.declarations(self.record), self.record] @@ -90,7 +90,7 @@ def test_complete_fork_retains_exact_original_material_hashes_and_copies_without self.assertEqual(overlay['source'], dict(project_id=self.origin, checkpoint=self.original['checkpoint'])) self.assertEqual(next(item for item in snapshot['records'] if item['family'] == 'model_material'), self.record) with ProjectStore.open(self.root / 'fork') as destination, ProjectArtifacts(destination) as artifacts: - reader = ProjectMaterial(SimpleNamespace(snapshot=destination.snapshot(), artifacts=artifacts, check=lambda: None)) + reader = ProjectMaterial(SimpleNamespace(snapshot=destination.snapshot(), artifacts=artifacts, check_current=lambda: None)) self.assertEqual(reader.model(self.material_value['ref'], verify=True), self.material_value) source_ref = self.material_value['material']['source_closure'][0]['blob'] self.assertEqual(reader.blob(self.material_value['ref'], source_ref), SOURCE) diff --git a/tests/test_project_fork_composition.py b/tests/test_project_fork_composition.py index b5fcf831..28e48361 100644 --- a/tests/test_project_fork_composition.py +++ b/tests/test_project_fork_composition.py @@ -232,7 +232,7 @@ def setUp(self): def validate(self,records): decode_data_records(records);material_records(records);file_records(records);producer_records(records) - history_records(records,materials=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check=lambda:None))) + history_records(records,materials=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None))) for receipt in operation_documents(records): operation,result=receipt['operation'],receipt['result'] if not (data_result(operation,result,records) or file_result(operation,result,records) diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 700504cd..89e4ecf2 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -137,7 +137,7 @@ def read_view(self, selected): def recipe(self, *args, **kwargs): raise AssertionError('History reread the live material catalog') session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), - artifacts=None,access=SimpleNamespace(),check=lambda:None) + artifacts=None,access=SimpleNamespace(),check_current=lambda:None) facts=ProjectHistory(session,materials=Provider())._facts() self.assertEqual(len(observed),1) self.assertIs(observed[0],records) @@ -653,7 +653,7 @@ def test_cold_actual_composed_proof_keeps_origin_without_live_authority(self): root=self.path/'portable-history';store=ProjectStore.create(root,'Actual cold history','new','f'*64) self.addCleanup(store.close) artifacts=ProjectArtifacts(store);self.addCleanup(artifacts.close) - view=SimpleNamespace(snapshot={'project_id':store.snapshot()['project_id'],'records':[]},artifacts=artifacts,check=lambda:None) + view=SimpleNamespace(snapshot={'project_id':store.snapshot()['project_id'],'records':[]},artifacts=artifacts,check_current=lambda:None) material=ProjectMaterial(view) model_record=material.capture_model_material(model,self.models.path) recipe_record=material.capture_recipe_material(composition,self.dataset.pipelines.material_path) diff --git a/tests/test_project_material.py b/tests/test_project_material.py index 5e412305..9d69056c 100644 --- a/tests/test_project_material.py +++ b/tests/test_project_material.py @@ -56,7 +56,7 @@ def setUp(self): self.artifacts=ProjectArtifacts(self.store);self.addCleanup(lambda:self.artifacts.close()) self.catalog=self.store.snapshot()['project_id'] self.session=SimpleNamespace(snapshot=self.store.snapshot(),summary=self.store.summary(), - artifacts=self.artifacts,check=lambda:None) + artifacts=self.artifacts,check_current=lambda:None) self.material=ProjectMaterial(self.session) def capture(self): @@ -88,7 +88,7 @@ def test_cold_relocation_uses_canonical_artifacts_after_local_capture_deleted(se self.artifacts.close();self.store.close();shutil.rmtree(root) moved=self.root/'moved';(self.root/'project').rename(moved) with ProjectStore.open(moved) as store,ProjectArtifacts(store) as artifacts: - material=ProjectMaterial(SimpleNamespace(snapshot=store.snapshot(),artifacts=artifacts,check=lambda:None)) + material=ProjectMaterial(SimpleNamespace(snapshot=store.snapshot(),artifacts=artifacts,check_current=lambda:None)) self.assertEqual(store.snapshot(),snapshot) self.assertEqual(material.model(value['ref'],verify=True),value) source=value['material']['source_closure'][0]['blob'] @@ -145,7 +145,7 @@ def test_fixed_material_snapshot_checks_complete_reference_identity(self): with self.assertRaises(WorkbenchError):view.runtime(dict(ref,**change)) def test_invalid_live_material_reference_precedes_snapshot_access(self): - with patch.object(self.session,'check',side_effect=AssertionError('Invalid reference accessed snapshot')): + with patch.object(self.session,'check_current',side_effect=AssertionError('Invalid reference accessed snapshot')): with self.assertRaises(WorkbenchError):self.material.runtime({'catalog_id':'invalid'}) def test_material_read_view_uses_supplied_records_without_resnapshotting(self): @@ -159,7 +159,7 @@ def test_material_read_view_uses_supplied_records_without_resnapshotting(self): self.assertEqual(self.session.snapshot['records'],[]) def test_material_read_view_rechecks_session_availability(self): - with patch.object(self.session,'check',side_effect=WorkbenchError('Stale checkpoint.','stale',409)): + with patch.object(self.session,'check_current',side_effect=WorkbenchError('Stale checkpoint.','stale',409)): with self.assertRaisesRegex(WorkbenchError,'Stale checkpoint'): self.material.read_view([]) @@ -361,7 +361,7 @@ def test_retained_one_fit_readers_match_directory_without_source_paths_or_infere patch('subprocess.Popen',side_effect=AssertionError('No process')): expected=evidence.verify(root,run,metrics) self.assertEqual(evidence.verify_blobs(run,metrics,read_blob=lambda name,maximum:payloads[name]),expected) - material=ProjectMaterial(SimpleNamespace(check=lambda:None)) + material=ProjectMaterial(SimpleNamespace(check_current=lambda:None)) result=material.evidence(run,read_blob=lambda name,maximum:payloads[name]) self.assertEqual(result['state'],'verified') checkpoint=result['checkpoint_index']['entries'][0]['checkpoint']['sha256'] @@ -450,7 +450,7 @@ def test_real_deferred_sql_failure_cannot_delete_canonical_source_material(self) self.assertEqual(error.exception.code,'projection_unavailable') snapshot=self.store.snapshot();retained=next(item for item in snapshot['records'] if item['family']=='model_material')['document']['value'] self.assertFalse((self.models.path/retained['ref']['sha256']).exists(),'Legacy rollback removes only disposable projection cache') - cold=ProjectMaterial(SimpleNamespace(snapshot=snapshot,artifacts=self.artifacts,check=lambda:None)) + cold=ProjectMaterial(SimpleNamespace(snapshot=snapshot,artifacts=self.artifacts,check_current=lambda:None)) self.assertEqual(cold.model(retained['ref'],verify=True),retained) self.assertEqual(cold.blob(retained['ref'],retained['material']['source_closure'][0]['blob']),SOURCE) @@ -461,7 +461,7 @@ def test_restore_is_inert_and_explicit_preparation_rebuilds_only_projection_cach draft=self.create();value=self.material.publish_model(self.models,draft['id'],{'revision':1},self.session.snapshot['checkpoint'],'cold-model') cold_db=Metadata(self.path/'cold-projection',dataset_id=self.db.dataset_id,projection_id=uuid.uuid4().hex) self.resources.append(cold_db);self.schemas.append(cold_db.schema) - cold=ProjectMaterial(SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,db=cold_db,check=lambda:None)) + cold=ProjectMaterial(SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,db=cold_db,check_current=lambda:None)) with patch('model_definitions.ModelDefinitions',side_effect=AssertionError('No mutable owner on Open')), \ patch('pipeline.Pipelines',side_effect=AssertionError('No mutable owner on Open')), \ patch('subprocess.Popen',side_effect=AssertionError('No process')): diff --git a/tests/test_project_material_fork.py b/tests/test_project_material_fork.py index d28881ac..fb27a597 100644 --- a/tests/test_project_material_fork.py +++ b/tests/test_project_material_fork.py @@ -53,7 +53,7 @@ class MaterialForkCodecs(unittest.TestCase): def setUp(self): self.source=uuid.uuid4().hex;self.checkpoint='a'*64 self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]), - summary=dict(project_id=self.source,checkpoint=self.checkpoint),check=lambda:None) + summary=dict(project_id=self.source,checkpoint=self.checkpoint),check_current=lambda:None) self.material=ProjectMaterial(self.session) def capture(self,family,current): @@ -110,7 +110,7 @@ def test_bound_drafts_preserve_full_configuration_as_visible_nonexecutable_inten captured=[self.capture(f,v) for f,v in drafts] made,combined,destination=self.fork() session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined), - summary=dict(project_id=destination),check=lambda:None) + summary=dict(project_id=destination),check_current=lambda:None) reader=ProjectMaterial(session) intents=reader.intents() self.assertEqual(len(intents),4) @@ -170,7 +170,7 @@ def test_captured_source_selector_stays_in_inert_intent_and_cannot_publish(self) self.assertEqual(doc['current']['required_reselection'],['source_selector']) self.assertEqual(doc['current']['definition']['source_selector'],value['source_selector']) reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined), - summary=dict(project_id=destination),check=lambda:None)) + summary=dict(project_id=destination),check_current=lambda:None)) with self.assertRaises(WorkbenchError) as caught: reader.publish_model(SimpleNamespace(publish=lambda *args: self.fail('Intent reached owner publication')), doc['current']['id'],{'revision':1},'a'*64,'forbidden-intent') diff --git a/tests/test_project_native_extraction.py b/tests/test_project_native_extraction.py index c2c3372b..fe1d34b3 100644 --- a/tests/test_project_native_extraction.py +++ b/tests/test_project_native_extraction.py @@ -86,7 +86,7 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact captured = self.capture_without_host(run, source) self.assertEqual(captured['blockers'], [], captured['blockers']) materials = ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, - records=captured['records']), check=lambda: None)) + records=captured['records']), check_current=lambda: None)) facts = decode_records(captured['records'], materials=materials) saved = facts.runs[(self.db.dataset_id, run['id'])] self.assertEqual(saved['run']['runtime'], run['runtime']) @@ -115,7 +115,7 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact for name in originals: (stage / name).unlink() cold = ProjectStore.open(destination); self.addCleanup(cold.close) cold_artifacts = ProjectArtifacts(cold); self.addCleanup(cold_artifacts.close) - session = SimpleNamespace(snapshot=cold.snapshot(), access=self.access, artifacts=cold_artifacts, check=lambda: None) + session = SimpleNamespace(snapshot=cold.snapshot(), access=self.access, artifacts=cold_artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda *args: None) for name, raw in originals.items(): self.assertEqual(history.artifact(run['id'], 'source/' + name, origin_catalog_id=self.db.dataset_id), raw) diff --git a/tests/test_project_session_freshness.py b/tests/test_project_session_freshness.py new file mode 100644 index 00000000..16996739 --- /dev/null +++ b/tests/test_project_session_freshness.py @@ -0,0 +1,88 @@ +"""Freshness enforcement over real canonical files, without a SQL projection. + +Only the session's read guard is exercised here. Publication and projection +binding still require the separate real-PostgreSQL session/workspace tests. +""" +import copy +import hashlib +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from project_session import ProjectSession, ProjectSessionError +from project_store import ProjectStore, ProjectStoreError + + +class ProjectSessionFreshnessTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-freshness-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / 'project' + self.store = ProjectStore.create(self.root, 'Freshness fixture', 'new', 'a' * 64) + self.addCleanup(self.store.close) + self.records = [dict(family='workbench_text', id=name, + document=dict(format='workbench_text_v1', text='Alpha ' + name)) for name in ('one', 'two')] + snapshot = self.store.commit(self.records, self.store.summary()['checkpoint'], 'seed', 'b' * 64) + # Isolate the existing read guard; no fake SQL/projection acceptance. + self.session = object.__new__(ProjectSession) + self.session.store = self.store + self.session._snapshot = copy.deepcopy(snapshot) + self.session.available = True + + def test_freshness_reads_all_records_without_copying_an_unused_graph(self): + before = self.store.snapshot() + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused full snapshot')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: + self.assertIsNone(self.session.check_current()) + self.assertIsNone(self.session.check_current()) + self.assertEqual(reads.call_count, 4) + self.assertEqual(self.store.snapshot(), before) + + def test_changed_canonical_checkpoint_refuses_both_read_contracts(self): + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Independently published content' + self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c' * 64) + for check in (self.session.check_current, self.session.check): + with self.assertRaises(ProjectSessionError) as caught: + check() + self.assertEqual(caught.exception.code, 'stale_project') + + def test_unavailable_projection_refuses_before_reading_the_store(self): + self.session.available = False + with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')), \ + patch.object(self.store, 'snapshot', side_effect=AssertionError('Unavailable snapshot read')): + for check in (self.session.check_current, self.session.check): + with self.assertRaises(ProjectSessionError) as caught: + check() + self.assertEqual(caught.exception.code, 'projection_unavailable') + + def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self): + self.session.check_current() + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + path = self.root / '.tuldok' / manifest['inventory'][0]['path'] + info, original = path.stat(), path.read_bytes() + changed = original.replace(b'Alpha', b'Omega') + self.assertEqual(len(original), len(changed)) + self.assertNotEqual(hashlib.sha256(original).digest(), hashlib.sha256(changed).digest()) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + for check in (self.session.check_current, self.session.check): + with self.assertRaises(ProjectStoreError) as caught: + check() + self.assertEqual(caught.exception.code, 'stale') + + def test_record_consumers_keep_independent_complete_snapshots(self): + current = self.session.check() + current['records'][0]['document']['text'] = 'Caller mutation' + current['name'] = 'Caller renamed copy' + self.assertEqual(self.session.check(), self.store.snapshot()) + self.assertEqual(self.session._snapshot, self.store.snapshot()) + self.assertNotEqual(current, self.session.check()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_transfer.py b/tests/test_project_transfer.py index efb639b4..5b798960 100644 --- a/tests/test_project_transfer.py +++ b/tests/test_project_transfer.py @@ -433,7 +433,7 @@ def capture(_,roots): from project_material import ProjectMaterial with ProjectStore.open(self.root/'science') as store,ProjectArtifacts(store) as artifacts: snapshot=store.snapshot();records=snapshot['records'] - view=SimpleNamespace(snapshot=snapshot,artifacts=artifacts,check=lambda:None) + view=SimpleNamespace(snapshot=snapshot,artifacts=artifacts,check_current=lambda:None) facts=history(records,materials=ProjectMaterial(view),artifacts=artifacts) self.assertEqual(len(facts.studies),2);self.assertEqual(len(facts.trials),2) self.assertEqual(len(facts.candidates),1);self.assertEqual(len(facts.reservations),1);self.assertEqual(len(facts.runs),1) From 6713ace099d380ab5c8dd6e4c768e460d5b12b8a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 01:05:24 -0700 Subject: [PATCH 09/14] perf(projects): avoid full history copies for working file reads --- project_dispatch.py | 18 +++++---- project_files.py | 19 +++++---- project_session.py | 16 ++++++++ tests/test_project_dispatch.py | 2 +- tests/test_project_files.py | 6 +-- tests/test_project_files_m4_clients.py | 14 ++++++- tests/test_project_files_m4_http.py | 19 +++++++++ tests/test_project_session_freshness.py | 54 +++++++++++++++++++++++-- 8 files changed, 125 insertions(+), 23 deletions(-) diff --git a/project_dispatch.py b/project_dispatch.py index bd7929d0..de519135 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -370,7 +370,16 @@ def _m4(projection,action,a): if action=='file_adopt_deletion': return projection.files.adopt_deletion(_body(a),a['expected_checkpoint'],a['request_key']) with session.access.lock,session.lock,session.db.lock: - session.access.require('inspect','project_files.'+action);snapshot=session.check() + session.access.require('inspect','project_files.'+action) + if action=='fork_info': + snapshot=session.check() + from project_fork import decode_fork + overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) + return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], + mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, + copied_count=len(overlay['artifacts']['copied']) if overlay else 0, + missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) + session.check_current() if action=='files_inspect':return _page(projection.files.inspect()['items'],a) if action=='file_get':return projection.files.read(a['file_id'],a.get('revision')) if action=='file_history': @@ -390,13 +399,6 @@ def _m4(projection,action,a): items=[dict(item,requires_reselection=[]) for item in value['editable']] items.extend(dict(family=item['family'],id=item['id'],requires_reselection=item['fields']) for item in value['requires_reselection']) return _page(sorted(items,key=lambda item:(item['family'],item['id'])),a) - if action=='fork_info': - from project_fork import decode_fork - overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) - return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], - mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, - copied_count=len(overlay['artifacts']['copied']) if overlay else 0, - missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) raise AssertionError('Closed M4 dispatch is incomplete.') diff --git a/project_files.py b/project_files.py index e2a5585b..12de25fb 100644 --- a/project_files.py +++ b/project_files.py @@ -233,7 +233,12 @@ def _read(root,parts): def inspect_captured(records, project_id, root): """Observe captured working files using only a locked project capability.""" - owned=[r for r in decode_records(records) if r['document']['project_id']==project_id] + return _inspect_validated(decode_records(records), project_id, root) + + +def _inspect_validated(records, project_id, root): + """Consume full-decoder output or fresh values from a validated Session.""" + owned=[r for r in records if r['document']['project_id']==project_id] items=[] for envelope in sorted(owned,key=lambda r:r['document']['path']): doc=envelope['document'];value=doc['current'] @@ -261,16 +266,16 @@ def __init__(self,session): def inspect(self): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.inspect');snapshot=self.session.check() + self.access.require('inspect','files.inspect');snapshot=self.session.check_records(FAMILY) with self.store.root_directory() as root: - items=inspect_captured(snapshot['records'],snapshot['project_id'],root) + items=_inspect_validated(snapshot['records'],snapshot['project_id'],root) self.session.check_current() return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) def read(self,record_id,revision=None): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.read',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.read',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) values=envelope['document']['history'] if revision is None:value=values[-1] @@ -280,8 +285,8 @@ def read(self,record_id,revision=None): def history(self,record_id): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.history',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.history',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) return copy.deepcopy(envelope['document']['history']) diff --git a/project_session.py b/project_session.py index 5fa571d1..c388eaf7 100644 --- a/project_session.py +++ b/project_session.py @@ -226,6 +226,22 @@ def check(self): """Return independent current records under caller-owned projection guards.""" return self._checked(self.store.snapshot) + def check_records(self, family): + """Verify every canonical record, then copy one bound record family. + + The complete snapshot, including cross-family closure, was validated + and isolated by adopt/publication. Returned copies retain that validity + inside the trusted owner boundary; they are not new imported raw input. + Exact current bytes and checkpoint must still match before using it. + Callers hold the same ordered guards as check(); no cached alias escapes. + """ + if type(family) is not str: + raise TypeError('Record-family selection requires a string.') + current = self._checked(self.store.summary) + current['records'] = copy.deepcopy([ + item for item in self._snapshot['records'] if item['family'] == family]) + return current + def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capture, *, scope='edit'): """Invoke existing policy, capture fixed-owner upserts, publish, commit SQL. diff --git a/tests/test_project_dispatch.py b/tests/test_project_dispatch.py index 8aab246d..1aee28e5 100644 --- a/tests/test_project_dispatch.py +++ b/tests/test_project_dispatch.py @@ -30,7 +30,7 @@ def require(self, *args, **kwargs): def projection(): access = Authority() - session = SimpleNamespace(access=access, check=Mock(), snapshot={'project_id': PROJECT, 'records': []}, + session = SimpleNamespace(access=access, check=Mock(), check_current=Mock(), snapshot={'project_id': PROJECT, 'records': []}, summary={'project_id': PROJECT, 'name': 'Dispatch fixture', 'checkpoint': CHECKPOINT}) return SimpleNamespace(session=session, data=Mock(), materials=Mock(), history=Mock(), context=SimpleNamespace(models=Mock(), pipelines=Mock()), artifacts=Mock(), diff --git a/tests/test_project_files.py b/tests/test_project_files.py index 49af9af2..f512d54e 100644 --- a/tests/test_project_files.py +++ b/tests/test_project_files.py @@ -44,15 +44,15 @@ def intent(self,path,text,revision=0,kind='note'): raw=text.encode('utf-8');target=self.root/path;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(raw) return dict(kind=kind,path=path,expected_revision=revision,expected_sha256=hashlib.sha256(raw).hexdigest()) - def test_inspection_copies_only_the_records_it_actually_consumes(self): + def test_inspection_never_copies_the_complete_record_graph(self): self.adopt('notes/inspection.md', 'Exact retained scientific note', kind='note') expected = self.files.inspect() with patch.object(self.session.store, '_snapshot', wraps=self.session.store._snapshot) as snapshots: observed = self.files.inspect() self.assertEqual(observed, expected) - self.assertEqual(snapshots.call_count, 1, - 'Inspection needs one record snapshot; final freshness must not clone another') + self.assertEqual(snapshots.call_count, 0, + 'File inspection needs only independent file records and complete freshness') def test_batch_captures_exact_files_in_one_canonical_mutation(self): items=[self.intent('notes/a.md','Exact Ω\r\n'),self.intent('notes/empty.md','')] diff --git a/tests/test_project_files_m4_clients.py b/tests/test_project_files_m4_clients.py index d24d42a2..90af3962 100644 --- a/tests/test_project_files_m4_clients.py +++ b/tests/test_project_files_m4_clients.py @@ -231,13 +231,25 @@ def test_dispatch_projects_metadata_pages_without_captured_text_or_definition_bu import threading access=SimpleNamespace(lock=threading.RLock(),require=Mock()) session=SimpleNamespace(access=access,lock=threading.RLock(),db=SimpleNamespace(lock=threading.RLock()), - check=Mock(return_value={'project_id':PROJECT,'records':[]})) + check=Mock(side_effect=AssertionError('Unused full dispatcher snapshot')),check_current=Mock()) files=SimpleNamespace(history=Mock(return_value=[version('large text '+str(i),i+1) for i in range(42)])) projection=SimpleNamespace(session=session,files=files) page=project_dispatch.dispatch(projection,'project_files.file_history',dict(READ,file_id=identity(),offset=40,limit=2)) self.assertEqual((page['total'],[item['revision'] for item in page['items']]),(42,[41,42])) self.assertTrue(all('text' not in item for item in page['items'])) + session.check_current.assert_called_once_with() api.validate_output('project_files.file_history',dict(status=200,result=bound(page))) + def test_dispatch_fork_info_retains_the_complete_snapshot_contract(self): + import threading + access=SimpleNamespace(lock=threading.RLock(),require=Mock()) + session=SimpleNamespace(access=access,lock=threading.RLock(),db=SimpleNamespace(lock=threading.RLock()), + check=Mock(return_value={'project_id':PROJECT,'records':[]}), + check_current=Mock(side_effect=AssertionError('Fork must read its records'))) + value=project_dispatch.dispatch(SimpleNamespace(session=session),'project_files.fork_info',dict(READ,offset=0,limit=40)) + session.check.assert_called_once_with() + self.assertEqual(value,dict(source=None,project_id=PROJECT, + mapping=dict(items=[],offset=0,limit=40,total=0),archived_count=0,copied_count=0,missing_regenerable_count=0)) + if __name__=='__main__':unittest.main() diff --git a/tests/test_project_files_m4_http.py b/tests/test_project_files_m4_http.py index 7280a2b4..8d6b0a6f 100644 --- a/tests/test_project_files_m4_http.py +++ b/tests/test_project_files_m4_http.py @@ -10,6 +10,7 @@ import project_workspace from project_store import ProjectStoreError +from workbench import WorkbenchError import test_project_files_http as m1 import test_project_files_m2_http as m2 @@ -46,6 +47,12 @@ def test_batch_adoption_uses_real_http_and_survives_independent_fork(self): items.append(dict(kind='note',path='notes/'+name,expected_revision=0,expected_sha256=hashlib.sha256(raw).hexdigest())) result=self.call('files_adopt',items=items) self.assertEqual(result['changed_count'],2);self.assertEqual(len(self.call('files_inspect')['items']),2) + with patch.object(self.workspace._store,'_snapshot',side_effect=AssertionError('Unrelated full graph copy')): + self.assertEqual(len(self.call('files_inspect')['items']),2) + selected=self.call('file_get',file_id=result['items'][0]['id']) + self.assertEqual(selected['version']['text'].encode(),b'Exact first note\r\n') + history=self.call('file_history',file_id=result['items'][0]['id']) + self.assertEqual([v['revision'] for v in history['items']],[1]) source=copy.deepcopy(self.workspace._projection.session.snapshot) self.fork() current=self.workspace._projection.session.snapshot;indexed={(r['family'],r['id']):r for r in current['records']} @@ -70,6 +77,18 @@ def test_nonempty_independent_fork_exact_science_new_editable_ids_and_same_reque current=self.workspace._projection.session.snapshot indexed={(r['family'],r['id']):r for r in current['records']} for r in original['records']:self.assertEqual(indexed[(r['family'],r['id'])],r) + forged=copy.deepcopy(current) + inherited=next(r['document'] for r in forged['records'] if r['family']=='project_file' + and r['document']['project_id']==current['project_id']) + forged_version=inherited['history'][0];forged_text='Forged inherited script\n' + forged_version.update(text=forged_text,bytes=len(forged_text.encode()),sha256=hashlib.sha256(forged_text.encode()).hexdigest()) + inherited['current']=copy.deepcopy(inherited['history'][-1]) + from project_files import decode_records + with self.assertRaisesRegex(WorkbenchError,'Working-file fork requires its exact archived history'): + decode_records(forged['records']) + with self.assertRaisesRegex(WorkbenchError,'Working-file fork requires its exact archived history'): + self.workspace._projection.session.adopt(forged) + self.assertEqual(self.workspace._projection.session.snapshot,current) active=self.call('data_query')['items'] self.assertEqual(len(active),1);self.assertNotEqual(active[0]['id'],row['id']) self.assertEqual(self.call('data_get',record_id=active[0]['id'])['text'],row['text']) diff --git a/tests/test_project_session_freshness.py b/tests/test_project_session_freshness.py index 16996739..8ab5634e 100644 --- a/tests/test_project_session_freshness.py +++ b/tests/test_project_session_freshness.py @@ -9,6 +9,7 @@ import os from pathlib import Path import tempfile +from types import SimpleNamespace import unittest from unittest.mock import patch @@ -45,7 +46,8 @@ def test_changed_canonical_checkpoint_refuses_both_read_contracts(self): changed = copy.deepcopy(self.records) changed[0]['document']['text'] = 'Independently published content' self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c' * 64) - for check in (self.session.check_current, self.session.check): + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): with self.assertRaises(ProjectSessionError) as caught: check() self.assertEqual(caught.exception.code, 'stale_project') @@ -54,7 +56,8 @@ def test_unavailable_projection_refuses_before_reading_the_store(self): self.session.available = False with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')), \ patch.object(self.store, 'snapshot', side_effect=AssertionError('Unavailable snapshot read')): - for check in (self.session.check_current, self.session.check): + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): with self.assertRaises(ProjectSessionError) as caught: check() self.assertEqual(caught.exception.code, 'projection_unavailable') @@ -70,7 +73,8 @@ def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self) path.chmod(0o600) path.write_bytes(changed) os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) - for check in (self.session.check_current, self.session.check): + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): with self.assertRaises(ProjectStoreError) as caught: check() self.assertEqual(caught.exception.code, 'stale') @@ -83,6 +87,50 @@ def test_record_consumers_keep_independent_complete_snapshots(self): self.assertEqual(self.session._snapshot, self.store.snapshot()) self.assertNotEqual(current, self.session.check()) + def add_unrelated_record(self): + unrelated = dict(family='project_artifact', id='d' * 64, + document=dict(format='project_artifact_v1', sha256='d' * 64, bytes=4)) + self.session._snapshot = self.store.commit([*self.records, unrelated], + self.store.summary()['checkpoint'], 'unrelated', 'e' * 64) + + def test_selected_records_are_exact_independent_and_do_not_copy_other_families(self): + self.add_unrelated_record() + expected = {key: value for key, value in self.store.snapshot().items() if key != 'records'} + expected['records'] = self.records + copied = [] + deepcopy = copy.deepcopy + def observe(value): + self.assertEqual([row['family'] for row in value], ['workbench_text', 'workbench_text']) + copied.append(value) + return deepcopy(value) + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unselected graph copy')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ + patch('project_session.copy', SimpleNamespace(deepcopy=observe)): + selected = self.session.check_records('workbench_text') + self.assertEqual(selected, expected) + self.assertEqual(reads.call_count, 3, 'Unselected actual records must still be read and hashed') + self.assertEqual(len(copied), 1) + selected['records'][0]['document']['text'] = 'Caller mutation' + self.assertEqual(self.session.check_records('workbench_text'), expected) + self.assertEqual(self.session.check_records('project_file')['records'], []) + + def test_selected_read_rejects_corruption_in_an_unselected_family(self): + self.add_unrelated_record() + self.session.check_records('workbench_text') + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + entry = next(row for row in manifest['inventory'] if row['family'] == 'project_artifact') + path = self.root / '.tuldok' / entry['path'] + info, original = path.stat(), path.read_bytes() + changed = original.replace(b'"bytes": 4', b'"bytes": 5') + self.assertNotEqual(changed, original) + self.assertEqual(len(changed), len(original)) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + with self.assertRaises(ProjectStoreError) as caught: + self.session.check_records('workbench_text') + self.assertEqual(caught.exception.code, 'stale') + if __name__ == '__main__': unittest.main() From 9ff98990d524fe885064120c3b77fe198c09a8ac Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 05:07:31 -0700 Subject: [PATCH 10/14] perf(history): reuse decoded facts at a verified checkpoint --- project_history.py | 14 ++++- tests/test_project_extraction.py | 2 + tests/test_project_history.py | 68 +++++++++++++++++++++++++ tests/test_project_native_extraction.py | 5 +- 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/project_history.py b/project_history.py index 8733ca4a..b69eccce 100644 --- a/project_history.py +++ b/project_history.py @@ -750,11 +750,23 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): from project_material import ProjectMaterial materials=ProjectMaterial(session) self.materials=materials;self.authorize_disclosure=authorize_disclosure + self._decoded_checkpoint=None + self._decoded_facts=None def _facts(self): self.session.check_current() + summary=self.session.summary + checkpoint=(summary['project_id'],summary['checkpoint']) + if self._decoded_checkpoint==checkpoint: + return self._decoded_facts records=self.session.snapshot['records'] - return decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) + facts=decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) + # Current canonical bytes are guarded on every call. Only this owner's + # private decoded metadata is retained; returned values and permissions + # remain the responsibility of each read below. + self._decoded_facts=facts + self._decoded_checkpoint=checkpoint + return facts def _authorize(self,action,target=None): self.access.require('inspect',action,target) diff --git a/tests/test_project_extraction.py b/tests/test_project_extraction.py index afa628a6..34e43176 100644 --- a/tests/test_project_extraction.py +++ b/tests/test_project_extraction.py @@ -20,6 +20,7 @@ from access import Access from metadata import Metadata from model_definitions import ModelDefinitions +from pipeline import digest from project_data import decode_data_records from project_extraction import _Capture, capture_legacy from project_material import decode_records as decode_material @@ -650,6 +651,7 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure # A fresh inert owner reader reconstructs the exact upstream custody # receipt from retained facts/private captured bytes, never source paths. session = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=copy.deepcopy(captured['records'])), + summary=dict(project_id=self.db.dataset_id, checkpoint=digest(captured['records'])), access=self.access, artifacts=source.artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda run, population: self.access.require('expose-holdout', 'runs.expose_test', run['id'])) diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 89e4ecf2..c42dd135 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -85,6 +85,47 @@ def legacy_run(origin=ORIGIN): class HistoryCodecTests(unittest.TestCase): + def test_progression_reuses_one_decoded_checkpoint_and_keeps_returned_values_independent(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + checks=[] + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER,checkpoint='c'*64), + summary=dict(project_id=OTHER,checkpoint='c'*64),artifacts=None, + access=SimpleNamespace(require=lambda *args,**kwargs:None), + check_current=lambda:checks.append('actual current-state guard')) + history=ProjectHistory(session,materials=Materials()) + with patch('project_history.decode_records',wraps=decode_records) as decoder: + first=history.progression(ORIGIN,offset=0,limit=20) + first['nodes'][0]['study']['name']='Caller mutation' + first['nodes'][0]['trial']['replicate']=99 + second=history.progression(ORIGIN,offset=0,limit=20) + self.assertEqual(decoder.call_count,1) + self.assertEqual(checks,['actual current-state guard']*2) + self.assertEqual(second['nodes'][0]['study']['name'],'Exact intent λ') + self.assertEqual(second['nodes'][0]['trial']['replicate'],0) + + def test_history_cache_refreshes_on_checkpoint_change_and_does_not_cache_decode_failure(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), + summary=dict(project_id=OTHER,checkpoint='c'*64),artifacts=None, + access=SimpleNamespace(require=lambda *args,**kwargs:None),check_current=lambda:None) + history=ProjectHistory(session,materials=Materials()) + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'Exact intent λ') + updated=copy.deepcopy(records) + for value in (updated[0]['document']['study'],updated[0]['document']['revisions'][0]): + value['name']='New checkpoint name' + session.snapshot=dict(records=updated,project_id=OTHER) + session.summary=dict(project_id=OTHER,checkpoint='d'*64) + with patch('project_history.decode_records',side_effect=WorkbenchError('Decode unavailable','unavailable',503)): + with self.assertRaisesRegex(WorkbenchError,'Decode unavailable') as failed: + history.progression(ORIGIN) + self.assertEqual(failed.exception.code,'unavailable') + with patch('project_history.decode_records',wraps=decode_records) as decoder: + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'New checkpoint name') + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'New checkpoint name') + self.assertEqual(decoder.call_count,1) + def test_fork_preserves_archived_disclosure_observation_without_current_permission(self): from project_fork import _digest,_input from project_session import operation_records @@ -137,6 +178,7 @@ def read_view(self, selected): def recipe(self, *args, **kwargs): raise AssertionError('History reread the live material catalog') session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), + summary=dict(project_id=OTHER,checkpoint='c'*64), artifacts=None,access=SimpleNamespace(),check_current=lambda:None) facts=ProjectHistory(session,materials=Provider())._facts() self.assertEqual(len(observed),1) @@ -312,6 +354,32 @@ def start(self): self.host,self.owner,self.session=host,owner,session self.history=ProjectHistory(session,materials=Materials()) + def test_cached_history_still_reads_actual_bytes_and_rejects_restored_mtime_corruption(self): + from project_store import ProjectStoreError + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + before=self.store.snapshot() + snapshot=self.store.commit(records,before['checkpoint'],'cache-corruption-fixture',digest(records)) + self.session.adopt(snapshot) + with self.owner.bound(self.host): + expected=self.history.progression(ORIGIN) + header=json.loads((self.store.path/'.tuldok/project.json').read_bytes()) + entry=next(item for item in header['inventory'] if item['family']=='study_history') + path=self.store.path/'.tuldok'/entry['path'] + original=path.read_bytes();info=path.stat() + corrupt=original.replace(b'Exact intent',b'Wrong intent',1) + self.assertNotEqual(corrupt,original);self.assertEqual(len(corrupt),len(original)) + try: + path.write_bytes(corrupt) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + with self.assertRaisesRegex(ProjectStoreError,'Canonical record bytes changed or are corrupt') as failed: + self.history.progression(ORIGIN) + self.assertEqual((failed.exception.code,failed.exception.status),('stale',409)) + finally: + path.write_bytes(original) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + self.assertEqual(self.history.progression(ORIGIN),expected) + def test_progression_uses_original_namespaces_exact_edges_and_bounded_pages(self): parent='a'*32;child='b'*32 ref={'catalog_id':ORIGIN,'trial_id':parent,'material_sha256':MATERIAL} diff --git a/tests/test_project_native_extraction.py b/tests/test_project_native_extraction.py index fe1d34b3..291e2247 100644 --- a/tests/test_project_native_extraction.py +++ b/tests/test_project_native_extraction.py @@ -115,7 +115,10 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact for name in originals: (stage / name).unlink() cold = ProjectStore.open(destination); self.addCleanup(cold.close) cold_artifacts = ProjectArtifacts(cold); self.addCleanup(cold_artifacts.close) - session = SimpleNamespace(snapshot=cold.snapshot(), access=self.access, artifacts=cold_artifacts, check_current=lambda: None) + snapshot = cold.snapshot() + session = SimpleNamespace(snapshot=snapshot, + summary={key:snapshot[key] for key in ('project_id','checkpoint')}, + access=self.access, artifacts=cold_artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda *args: None) for name, raw in originals.items(): self.assertEqual(history.artifact(run['id'], 'source/' + name, origin_catalog_id=self.db.dataset_id), raw) From 0490cb14928644d9c6758b32cd2948a9c5f6542f Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 08:17:58 -0700 Subject: [PATCH 11/14] perf(project): reduce validated snapshot and text-list copies --- project_projection.py | 7 +++-- project_session.py | 5 +-- project_store.py | 13 +++++++- tests/test_project_projection.py | 21 ++++++++++++- tests/test_project_store.py | 21 +++++++++++++ tests/test_project_text_tag.py | 52 ++++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 tests/test_project_text_tag.py diff --git a/project_projection.py b/project_projection.py index 1bf45bd2..87d8d133 100644 --- a/project_projection.py +++ b/project_projection.py @@ -313,14 +313,15 @@ def available(self): return self.projection.session.available def _tag(self, value): - snapshot = self.projection.session.snapshot - return dict(value, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint']) + summary = self.projection.session.summary + return dict(value, project_id=summary['project_id'], checkpoint=summary['checkpoint']) def records(self): self.projection.session.check_current() from project_fork import active_records snapshot = self.projection.session.snapshot - records = active_records(snapshot['records'], project_id=snapshot['project_id']) + records = active_records(snapshot['records'], project_id=snapshot['project_id'], + families=frozenset(('workbench_record', 'workbench_text'))) return self._tag(dict(items=[copy.deepcopy(item['document']['current']) for item in records if item['family'] in ('workbench_record', 'workbench_text') and item['document']['current']['kind'] == 'text'])) diff --git a/project_session.py b/project_session.py index c388eaf7..aa471c1e 100644 --- a/project_session.py +++ b/project_session.py @@ -9,6 +9,7 @@ import threading import agent_api +from project_store import copy_validated_json from workbench import WorkbenchError @@ -186,7 +187,7 @@ def summary(self): @property def snapshot(self): with self.lock: - return copy.deepcopy(self._snapshot) + return copy_validated_json(self._snapshot) def _validated(self, records): operation_records(records, project_id=self.project_id) @@ -203,7 +204,7 @@ def adopt(self, snapshot): current = self.store.snapshot() if current != snapshot: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._snapshot = copy.deepcopy(snapshot) + self._snapshot = copy_validated_json(snapshot) self.available = True def _checked(self, read): diff --git a/project_store.py b/project_store.py index cb1c2bf9..d079ca3f 100644 --- a/project_store.py +++ b/project_store.py @@ -124,6 +124,17 @@ def _encode(value): allow_nan=False) + '\n').encode('utf-8') +def copy_validated_json(value): + """Own another tree of already validated JSON without Python graph copying. + + Canonical Store/Session values contain only JSON scalars, lists and string + keyed dictionaries. This is a copy operation inside that proof boundary, + never an input decoder or a substitute for freshness/codec validation. + """ + return json.loads(json.dumps(value, ensure_ascii=True, allow_nan=False, + separators=(',', ':'))) + + def _manifest_raw(manifest): raw = _encode(manifest) if len(raw) > MAX_MANIFEST_BYTES: @@ -793,7 +804,7 @@ def _publish(self, manifest, creating=False): def _snapshot(self, manifest, records): return dict(project_id=manifest['project_id'], name=manifest['name'], - checkpoint=manifest['checkpoint'], records=copy.deepcopy(records)) + checkpoint=manifest['checkpoint'], records=copy_validated_json(records)) def _current(self): try: diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index d08aaadc..3f1e3e98 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -6,7 +6,7 @@ from pathlib import Path import tempfile import unittest -from unittest.mock import patch +from unittest.mock import PropertyMock, patch import uuid import psycopg @@ -120,6 +120,25 @@ def test_unknown_receipt_is_rejected_before_invoke_and_sql_or_checkpoint_changes self.assertEqual(p.store.snapshot(), before) self.assertEqual(p.db.execute('SELECT COUNT(*) FROM workbench_records').fetchone()[0], 0) + def test_text_tag_uses_real_session_summary_and_record_list_remains_owned(self): + text = self.call('import_text', self.mutation(asset=dict(text='Owned note 山', groups=['owned'])))['record'] + p = self.host._projection + with self.owner.bound(self.host): + expected = dict(project_id=p.session.summary['project_id'], + checkpoint=p.session.summary['checkpoint']) + with patch.object(type(p.session), 'snapshot', new_callable=PropertyMock, + side_effect=AssertionError('Scalar tag copied a full Session')): + self.assertEqual(p.text._tag(dict(value='Exact scalar tag')), + dict(expected, value='Exact scalar tag')) + exact_current = [r['document']['current'] for r in p.session.snapshot['records'] + if r['family'] in ('workbench_text', 'workbench_record') + and r['document']['current']['kind'] == 'text'] + result = p.text.records() + self.assertEqual(result, dict(expected, items=exact_current)) + self.assertEqual([r['id'] for r in result['items']], [text['id']]) + result['items'][0]['text'] = 'Caller mutation' + self.assertEqual(p.text.get(text['id'])['text'], 'Owned note 山') + def test_explicit_rooted_artifact_supply_is_durable_and_never_fetches_or_executes(self): p = self.host._projection raw = b'Unique measured bytes; never regenerate these.\n' diff --git a/tests/test_project_store.py b/tests/test_project_store.py index f3e7f0e2..676ab3bc 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -3,10 +3,12 @@ import errno import hashlib import json +import math import os from pathlib import Path import shutil import subprocess +import struct import sys import tempfile import unittest @@ -71,6 +73,25 @@ def write_manifest(self, value): value['checkpoint'] = sha(canonical(value)) (self.root / '.tuldok/project.json').write_bytes(canonical(value)) + def test_snapshot_preserves_scalar_types_float_bits_and_owned_nested_values(self): + floats = [-0.0, 0.0, 5e-324, -5e-324, 1.7976931348623157e308, + math.nextafter(1.0, 0.0), math.nextafter(1.0, math.inf)] + value = record() + value['document']['scalars'] = [None, False, True, -2**63, 2**63-1, + 'Unicode 漢字 \U0001f30a\x00', *floats] + expected = self.commit([value]) + first, second = self.store.snapshot(), self.store.snapshot() + self.assertEqual(first, expected) + self.assertEqual(second, expected) + actual = first['records'][0]['document']['scalars'] + self.assertEqual([type(v) for v in actual], + [type(v) for v in value['document']['scalars']]) + self.assertEqual([struct.pack('>d', v) for v in actual[6:]], + [struct.pack('>d', v) for v in floats]) + first['records'][0]['document']['history'][0]['reviewed'] = True + first['records'][0]['document']['scalars'].clear() + self.assertEqual(self.store.snapshot(), second) + def test_create_commit_save_cold_reopen_and_independent_snapshots(self): initial = self.store.snapshot() self.assertEqual(initial['records'], []) diff --git a/tests/test_project_text_tag.py b/tests/test_project_text_tag.py new file mode 100644 index 00000000..6a944aa6 --- /dev/null +++ b/tests/test_project_text_tag.py @@ -0,0 +1,52 @@ +"""The scalar text response tag must not copy unrelated project metadata.""" +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import project_fork + +from project_projection import _TextView + + +class TextTagTests(unittest.TestCase): + def test_text_records_does_not_clone_unrelated_active_documents(self): + unrelated = dict(family='project_artifact', id='c' * 64, + document=dict(format='project_artifact_v1', sha256='c' * 64, bytes=4)) + note = dict(family='workbench_text', id='d' * 32, + document=dict(current=dict(kind='text', text='Unique note 山'))) + summary = dict(project_id='a' * 32, checkpoint='b' * 64) + session = SimpleNamespace(summary=summary, + snapshot=dict(summary, records=[unrelated, note]), + check_current=lambda: None) + copied = [] + original = project_fork.copy.deepcopy + + def observe(value): + if value is unrelated: + copied.append(value['id']) + return original(value) + + with patch.object(project_fork, 'copy', SimpleNamespace(deepcopy=observe)): + result = _TextView(SimpleNamespace(session=session)).records() + self.assertEqual(result, dict(summary, items=[dict(kind='text', text='Unique note 山')])) + self.assertEqual(copied, [], 'Text list cloned an unrelated active document') + result['items'][0]['text'] = 'Caller mutation' + self.assertEqual(note['document']['current']['text'], 'Unique note 山') + + def test_text_tag_reads_scalar_summary_without_a_record_snapshot(self): + class Session: + summary = dict(project_id='a' * 32, name='Owned project', checkpoint='b' * 64) + + @property + def snapshot(self): + raise AssertionError('Text response tag copied the full record graph') + + session = Session() + view = _TextView(SimpleNamespace(session=session)) + original = dict(items=[dict(text='Authored note 漢字')]) + result = view._tag(original) + self.assertEqual(result, dict(original, project_id='a' * 32, checkpoint='b' * 64)) + self.assertNotIn('name', result) + self.assertEqual(original, dict(items=[dict(text='Authored note 漢字')])) + result['project_id'] = 'caller edit' + self.assertEqual(session.summary['project_id'], 'a' * 32) From d3b8f92ed99ad005e2bb09f6e963c3aebc24d9d0 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 09:52:54 -0700 Subject: [PATCH 12/14] perf(project): validate retained receipts from one owned candidate view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decode each receipt domain once per candidate validation pass while retaining exact revision, source and packet-custody checks. Keep standalone validation fresh and share only a single call’s decoded inventory across batch items. --- project_files.py | 31 +++- project_measurements.py | 26 ++- project_projection.py | 14 +- project_sources.py | 21 ++- tests/test_project_receipt_validation.py | 204 +++++++++++++++++++++++ 5 files changed, 280 insertions(+), 16 deletions(-) create mode 100644 tests/test_project_receipt_validation.py diff --git a/project_files.py b/project_files.py index 12de25fb..ad0f6b3f 100644 --- a/project_files.py +++ b/project_files.py @@ -5,6 +5,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import errno import hashlib import os @@ -128,14 +129,14 @@ def _result(envelope,changed): return result -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): if operation=='files.adopt_many': closed(result,{'items','changed_count'},'batch adoption result') if type(result['items']) is not list or not 1<=len(result['items'])<=MAX_ADOPT_ITEMS: fail('Invalid batch adoption result size.') seen=set();projects=set() for item in result['items']: - validate_operation_result('files.adopt',item,records) + _validate_result('files.adopt',item,lookup) if item['id'] in seen:fail('Batch adoption result repeats a file.') seen.add(item['id']);projects.add(item['project_id']) if (len(projects)!=1 or type(result['changed_count']) is not int @@ -153,14 +154,34 @@ def validate_operation_result(operation,result,records=None): else:sha(result['sha256']) _date(result['adopted_at']) if type(result['revision']) is not int or not 1<=result['revision']<=2**53-1 or type(result['bytes']) is not int or not 0<=result['bytes']<=MAX_FILE_BYTES:fail('Invalid adoption result bounds.') - if records is not None: - owned=decode_records(records) - envelope=next((r for r in owned if r['id']==result['id']),None) + if lookup is not None: + envelope=lookup(result['id']) if envelope is None or not any((value.get('deleted') is True)==deleted and all(value[key]==result[key] for key in ('revision','sha256','bytes','adopted_at')) for value in envelope['document']['history']): fail('Adoption result is not an exact retained working-file revision.') return True +def receipt_validator(records): + """Own one decoded candidate's file histories for its receipt checks. + + The returned validator has no live Session or publication authority. It + owns copied records, so caller mutation cannot change this fixed view. + A different candidate requires a new validator. + """ + owned = {record['id']: record for record in decode_records(records)} + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + # Keep standalone shape/error ordering, and share only this call's lookup + # across batch items. There is no retained cache between invocations. + @cache + def owned(): + return {record['id']: record for record in decode_records(records)} + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class _WorkingError(Exception): def __init__(self,state):self.state=state diff --git a/project_measurements.py b/project_measurements.py index 400db842..d1fed0de 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -6,6 +6,7 @@ """ import copy from datetime import datetime +from functools import cache, partial from project_artifacts import validate_declaration from project_material import decode_records as material_records @@ -101,17 +102,36 @@ def decode_records(records, *, artifacts=None): return result -def validate_operation_result(operation, result, records=None): +def _validate_result(operation, result, lookup=None): if operation != 'measurements.import': return False _document(result) - if records is not None: + if lookup is not None: identity = result['origin_catalog_id'] + '.' + result['id'] - if decode_records(records).get(identity) != result: + if lookup(identity) != result: raise WorkbenchError('Measurement receipt has no exact retained source.') return True +def receipt_validator(records, *, artifacts=None): + """Own one candidate's validated source bindings and packet custody. + + Decode and copy every retained measurement once, including current available + packet bytes when supplied. The returned receipt checker is fixed to those + facts; it carries no live source or publication capability. + """ + owned = decode_records(records, artifacts=artifacts) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return decode_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class ProjectMeasurements: def __init__(self, session): self.session, self.artifacts = session, session.artifacts diff --git a/project_projection.py b/project_projection.py index 87d8d133..25b6bf28 100644 --- a/project_projection.py +++ b/project_projection.py @@ -25,14 +25,17 @@ from project_producer_execution import ProjectProducerExecution, validate_operation_result as validate_producer_result from project_files import ProjectFiles, decode_records as decode_file_records from project_files import validate_operation_result as validate_file_result +from project_files import receipt_validator as file_receipt_validator from project_study_execution import ProjectStudyExecution, validate_operation_result as validate_study_execution_result from project_runtime import ProjectRuntime, validate_operation_result as validate_runtime_result from project_native_execution import FAMILY as NATIVE_FAMILY, decode_records as decode_native_records from project_native_preparation import ProjectNativePreparation, validate_operation_result as validate_preparation_result from project_sources import SourceAdapter, validate_operation_result as validate_source_result +from project_sources import receipt_validator as source_receipt_validator from project_measurements import ProjectMeasurements, FAMILY as MEASUREMENT_FAMILY from project_measurements import decode_records as decode_measurement_records from project_measurements import validate_operation_result as validate_measurement_result +from project_measurements import receipt_validator as measurement_receipt_validator from workbench import WorkbenchError from workbench import timestamp @@ -82,7 +85,7 @@ def validate_project_records(records, *, project_id, artifacts): from project_transfer import decode_records as decode_extraction_records decode_extraction_records(records) operation_records(records, project_id=project_id) - decode_file_records(records) + validate_file_receipt = file_receipt_validator(records) decode_native_records(records,artifacts=artifacts) materials = MaterialSnapshot(records) for blob in materials.artifact_blobs(): @@ -92,17 +95,18 @@ def validate_project_records(records, *, project_id, artifacts): # than a resolver that still sees the preceding committed checkpoint. decode_producer_records(records, artifacts=artifacts) decode_history_records(records, materials=materials, artifacts=artifacts) - decode_measurement_records(records, artifacts=artifacts) + validate_measurement_receipt = measurement_receipt_validator(records, artifacts=artifacts) + validate_source_receipt = source_receipt_validator(records) for receipt in operation_documents(records): operation, result = receipt['operation'], receipt['result'] if (validate_data_result(operation, result, records) or validate_material_result(operation, result, records) or validate_history_result(operation, result, records) - or validate_source_result(operation, result, records) - or validate_measurement_result(operation, result, records) + or validate_source_receipt(operation, result) + or validate_measurement_receipt(operation, result) or validate_execution_result(operation, result, records) or validate_producer_result(operation, receipt['target'], result, records) - or validate_file_result(operation, result, records) + or validate_file_receipt(operation, result) or validate_study_execution_result(operation, result, records) or validate_runtime_result(operation, result, records) or validate_preparation_result(operation, result, records) diff --git a/project_sources.py b/project_sources.py index 95e7c5f2..a17b02a5 100644 --- a/project_sources.py +++ b/project_sources.py @@ -6,6 +6,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import hashlib import os @@ -36,7 +37,7 @@ def _summary(saved, origin): schema_version=saved['manifest']['schema_version']) -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): """Validate the fixed source receipt against its exact retained snapshot.""" if operation!='source.register':return False try: @@ -50,8 +51,8 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Source registration requires an attributed timestamp.') if datetime.fromisoformat(result['registered_at']).utcoffset() is None: raise WorkbenchError('Source registration timestamp requires an offset.') - if records is not None: - document=material_records(records).get(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) + if lookup is not None: + document=lookup(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) expected={key:result[key] for key in ('origin_catalog_id','project_id','manifest_sha256','name','schema_version')} if document is None or _summary(document['value'],document['origin_catalog_id'])!=expected: raise WorkbenchError('Source receipt differs from its exact retained snapshot.') @@ -61,6 +62,20 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Malformed retained source operation result.') from None +def receipt_validator(records): + """Own one candidate's retained material facts for source receipt checks.""" + owned = material_records(records) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return material_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class SourceAdapter: def __init__(self,projection): self.projection=projection diff --git a/tests/test_project_receipt_validation.py b/tests/test_project_receipt_validation.py new file mode 100644 index 00000000..bf69926f --- /dev/null +++ b/tests/test_project_receipt_validation.py @@ -0,0 +1,204 @@ +"""One mixed candidate: retain every receipt check without repeated decoding.""" +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import project_files as files +import project_material as material +import project_measurements as measurements +import project_projection as projection +import project_sources as sources +from project_artifacts import ProjectArtifacts, ProjectArtifactError +from project_store import ProjectStore +from test_retained_measurements import fixture +from workbench import WorkbenchError + +PROJECT = 'a' * 32 +ACTOR = 'c' * 32 +INSTANT = '2026-10-25T00:00:00+00:00' + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def operation(name, key, result): + return dict(family='project_operation', id=PROJECT + '.' + sha(key.encode()), document=dict( + format='project_operation_v2', project_id=PROJECT, operation=name, + target=None, request_key=key, input_sha256='d' * 64, result=copy.deepcopy(result))) + + +def candidate(): + packets = [] + for index in range(2): + value = fixture() + value['title'] = f'Packet {index}' + raw = json.dumps(value).encode() + packets.append(dict(raw=raw, sha256=sha(raw), path=f'packet-{index}.json', + reference_id=f'packet-{index}', title=value['title'])) + manifest = dict(schema_version=1, id='b' * 32, name='Receipt source', + data=[dict(id=p['reference_id'], label=p['title'], path=p['path'], sha256=p['sha256']) + for p in packets], recipes=[], evidence=[], requirements=[]) + raw = json.dumps(manifest).encode() + digest = sha(raw) + snapshot = dict(id=manifest['id'], name=manifest['name'], root='original', + root_identity=[1, 2], manifest_path='source.json', manifest_sha256=digest, manifest=manifest) + retained = dict(family='project_snapshot', id=PROJECT + '.' + digest, document=dict( + format=material.FAMILIES['project_snapshot'], origin_catalog_id=PROJECT, value=snapshot, + artifacts=[dict(name='source.json', blob=dict(sha256=digest, bytes=len(raw)), + role='project_manifest', preservation='unique', required_for=['snapshot_verification'])], audit=[])) + records = [retained] + for hashed, size in [(digest, len(raw)), *[(p['sha256'], len(p['raw'])) for p in packets]]: + records.append(dict(family='project_artifact', id=hashed, + document=dict(format='project_artifact_v1', sha256=hashed, bytes=size))) + source_result = dict(origin_catalog_id=PROJECT, project_id=manifest['id'], manifest_sha256=digest, + name=manifest['name'], schema_version=1, actor_id=ACTOR, registered_at=INSTANT) + for index in range(2): + records.append(operation('source.register', f'source-{index}', source_result)) + for index, packet in enumerate(packets): + doc = dict(format=measurements.FORMAT, origin_catalog_id=PROJECT, id=packet['sha256'], + title=packet['title'], blob=dict(sha256=packet['sha256'], bytes=len(packet['raw'])), + source=dict(origin_catalog_id=PROJECT, project_id=manifest['id'], manifest_sha256=digest, + reference_id=packet['reference_id'], path=packet['path'], measured_sha256=packet['sha256']), + actor_id=ACTOR, imported_at=INSTANT) + records.append(dict(family=measurements.FAMILY, id=PROJECT + '.' + packet['sha256'], document=doc)) + records.append(operation('measurements.import', f'packet-{index}', doc)) + for batch in range(2): + items = [] + for index in range(2): + path = f'notes/note-{batch}-{index}.md' + raw = f'Owned note {batch}-{index} 山'.encode() + version = dict(revision=1, sha256=sha(raw), bytes=len(raw), text=raw.decode(), + adopted_at=INSTANT, actor_id=ACTOR) + identity = files.file_id(PROJECT, 'note', path) + doc = dict(format=files.FORMAT, project_id=PROJECT, kind='note', path=path, + current=copy.deepcopy(version), history=[version], origin=None) + records.append(dict(family=files.FAMILY, id=identity, document=doc)) + items.append(dict(id=identity, project_id=PROJECT, kind='note', path=path, changed=True, + **{key: version[key] for key in ('revision', 'sha256', 'bytes', 'adopted_at')})) + records.append(operation('files.adopt_many', f'notes-{batch}', dict(items=items, changed_count=2))) + return records + + +class ReceiptValidationTests(unittest.TestCase): + def test_whole_candidate_decodes_each_receipt_owner_once(self): + records = candidate() + before = copy.deepcopy(records) + with patch.object(files, 'decode_records', wraps=files.decode_records) as file_decode, \ + patch.object(projection, 'decode_file_records', file_decode), \ + patch.object(measurements, 'decode_records', wraps=measurements.decode_records) as measurement_decode, \ + patch.object(projection, 'decode_measurement_records', measurement_decode), \ + patch.object(material, 'decode_records', wraps=material.decode_records) as material_decode, \ + patch.object(measurements, 'material_records', material_decode), \ + patch.object(sources, 'material_records', material_decode): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + self.assertEqual(records, before) + for name, actual, expected in [('files', file_decode.call_count, 1), + ('measurements', measurement_decode.call_count, 1), + ('material-owning readers', material_decode.call_count, 3)]: + with self.subTest(owner=name): + self.assertEqual(actual, expected, 'Receipt validation redecoded the same candidate') + + def test_standalone_file_batch_decodes_inventory_once(self): + records = candidate() + result = next(r['document']['result'] for r in records + if r['family'] == 'project_operation' + and r['document']['operation'] == 'files.adopt_many') + with patch.object(files, 'decode_records', wraps=files.decode_records) as decode: + self.assertTrue(files.validate_operation_result('files.adopt_many', result, records)) + self.assertEqual(decode.call_count, 1, 'Each batch item decoded the full file inventory') + + def test_forged_late_file_or_measurement_receipt_is_rejected(self): + for owner in ('files.adopt_many', 'measurements.import', 'source.register'): + records = candidate() + result = [r['document']['result'] for r in records if r['family'] == 'project_operation' + and r['document']['operation'] == owner][-1] + if owner == 'files.adopt_many': + result['items'][-1]['sha256'] = 'e' * 64 + else: + result['name' if owner == 'source.register' else 'title'] = 'Forged retained value' + with self.subTest(owner=owner), self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_unknown_operation_still_has_no_accepting_owner(self): + records = candidate() + records.append(operation('unknown.capture', 'unknown', {})) + with self.assertRaises(WorkbenchError) as error: + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + self.assertEqual(error.exception.code, 'unsupported') + + def test_owned_validators_and_standalone_calls_do_not_share_candidates(self): + for name, module in [('files.adopt_many', files), ('measurements.import', measurements), + ('source.register', sources)]: + records = candidate() + original_result = copy.deepcopy(next(r['document']['result'] for r in records + if r['family'] == 'project_operation' and r['document']['operation'] == name)) + checker = module.receipt_validator(records) + self.assertTrue(checker(name, original_result)) + self.assertTrue(module.validate_operation_result(name, original_result, records)) + if module is files: + doc = next(r['document'] for r in records if r['family'] == files.FAMILY) + value = doc['history'][0] + value.update(text='Edited candidate', sha256=sha(b'Edited candidate'), bytes=16) + doc['current'] = copy.deepcopy(value) + elif module is measurements: + next(r['document'] for r in records if r['family'] == measurements.FAMILY)['title'] = 'Edited candidate' + else: + value = next(r['document']['value'] for r in records if r['family'] == 'project_snapshot') + value['name'] = value['manifest']['name'] = 'Edited candidate' + with self.subTest(owner=name): + self.assertTrue(checker(name, original_result), 'Owned candidate changed with caller mutation') + with self.assertRaises(WorkbenchError): + module.receipt_validator(records)(name, original_result) + with self.assertRaises(WorkbenchError): + module.validate_operation_result(name, original_result, records) + + def test_standalone_unrecognized_and_malformed_results_do_not_decode(self): + for name, module in [('files.adopt_many', files), ('measurements.import', measurements), + ('source.register', sources)]: + records = candidate() + attribute = 'material_records' if module is sources else 'decode_records' + with self.subTest(owner=name), patch.object(module, attribute, + side_effect=AssertionError('Result shape must precede candidate decoding')) as decode: + self.assertFalse(module.validate_operation_result('unknown.capture', {}, records)) + with self.assertRaises(WorkbenchError): + module.validate_operation_result(name, {}, records) + decode.assert_not_called() + + def test_unused_malformed_file_and_missing_packet_reference_still_reject(self): + records = candidate() + next(r['document']['history'][0] for r in records if r['family'] == files.FAMILY)['text'] = 'Unhashed' + records = [r for r in records if r['family'] != 'project_operation' + or r['document']['operation'] != 'files.adopt_many'] + with self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + records = candidate() + next(r['document'] for r in records if r['family'] == measurements.FAMILY)['source']['reference_id'] = 'absent' + with self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_each_new_measurement_validator_checks_actual_packet_bytes(self): + records = candidate() + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / 'project' + with ProjectStore.create(root, 'Packet custody fixture', 'create', 'a' * 64) as store: + with ProjectArtifacts(store) as artifacts: + docs = [r['document'] for r in records if r['family'] == measurements.FAMILY] + for index, doc in enumerate(docs): + packet = fixture() + packet['title'] = f'Packet {index}' + raw = json.dumps(packet).encode() + self.assertEqual(sha(raw), doc['id']) + artifacts.publish_bytes(raw, measurements.declaration(doc['blob'])) + checker = measurements.receipt_validator(records, artifacts=artifacts) + self.assertTrue(checker('measurements.import', docs[0])) + payload = root / 'artifacts/sha256' / docs[0]['id'] + raw = payload.read_bytes() + payload.chmod(0o600) + payload.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + with self.assertRaises((WorkbenchError, ProjectArtifactError)): + measurements.receipt_validator(records, artifacts=artifacts) From f8ba4cedd22580f273f8c4fcbc06bca53797a7a7 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 12:21:56 -0700 Subject: [PATCH 13/14] feat(project): browse authored model portfolios from retained notes --- app.py | 1 + docs/contracts/project-model-portfolio.md | 87 ++++++++++ static/project-files.css | 10 ++ static/project-files.html | 2 + static/project-files.js | 13 ++ static/project-model-portfolio.js | 188 +++++++++++++++++++++ tests/browser_model_portfolio.cjs | 129 ++++++++++++++ tests/model_portfolio_fixture.cjs | 16 ++ tests/test_project_model_portfolio.cjs | 47 ++++++ tests/test_project_model_portfolio_http.py | 84 +++++++++ tests/test_workspace_project_files.cjs | 1 + tests/test_workspace_project_files_m4.cjs | 1 + 12 files changed, 579 insertions(+) create mode 100644 docs/contracts/project-model-portfolio.md create mode 100644 static/project-model-portfolio.js create mode 100644 tests/browser_model_portfolio.cjs create mode 100644 tests/model_portfolio_fixture.cjs create mode 100644 tests/test_project_model_portfolio.cjs create mode 100644 tests/test_project_model_portfolio_http.py diff --git a/app.py b/app.py index 87c6c9a7..f818cc0b 100644 --- a/app.py +++ b/app.py @@ -919,6 +919,7 @@ def _get(self): '/workspaces.js': ('workspaces.js', 'text/javascript'), '/experiment-graph.js': ('experiment-graph.js', 'text/javascript'), '/project-progression.js': ('project-progression.js', 'text/javascript'), + '/project-model-portfolio.js': ('project-model-portfolio.js', 'text/javascript'), '/project-measurements.js': ('project-measurements.js', 'text/javascript'), '/project-measurements.css': ('project-measurements.css', 'text/css'), '/experiment-lineage.js': ('experiment-lineage.js', 'text/javascript'), diff --git a/docs/contracts/project-model-portfolio.md b/docs/contracts/project-model-portfolio.md new file mode 100644 index 00000000..d7a0a991 --- /dev/null +++ b/docs/contracts/project-model-portfolio.md @@ -0,0 +1,87 @@ +# Authored model organization + +The project workspace can display an adopted `.portfolio.json` note as an +authored guide to model fits/components. This is separate from **Experiment +progression**, which retains genuine Study/Trial ancestry. Group edges mean +authored grouping; they do not establish derivation, admission or scientific +acceptance. Relationship notes have explicit comparison/parameter meanings and +source excerpts. Missing historical Trial ancestry stays unknown. + +## Retained document + +`project_model_portfolio_v1` is a UTF-8 JSON note of at most1MiB with exactly: + +- `format`: the fixed string above; `project_id`: the owning32hex project ID; +- `title` (1–200characters), `description` (1–8,000characters), and + `authorship`: `retrospective`; +- `groups`: at most64records with exact fields `id`, `label`, `parent`, + `description`. IDs are unique, parent is null or another group, and grouping + is acyclic; +- `models`:1–1,000records with exact fields `id`, `label`, `group_id`, `aliases`, + `review`, `card`, `summary`. Each unique ID names an authored model entry, + not necessarily a complete generator, Trial or unique parameter state; +- `relationships`: at most2,000records with exact fields `from`, `to`, `kind`, + `explanation`, `evidence_model`, `excerpt`. All model references exist, endpoints + differ, and each ordered endpoint/kind tuple is unique. + +Group/model IDs contain1–160ASCII letters/digits or `._:/-`, beginning with a +letter/digit. Group labels are at most200characters, model labels/aliases500. +Each model has at most128unique aliases. A card reference has exact fields +`file_id` (64hex), `path` (a relative `notes/` path up to512characters without +empty/dot/dot-dot components), positive integer `revision`, and64hex `sha256`. +These are exact captured note references, never source-reading capabilities. + +Review is `source_reviewed` or `review_pending`. Reviewed summaries have exactly +six nonempty strings, each up to16,384characters: `method`, `purpose`, +`experiment`, `outcome`, `lesson`, `reproduction`. Pending summaries are null. +This status is an author's source-review statement, not numerical qualification. +Relationship kind is `comparison` or `parameter_derivation`; the latter points +from derived model to parameter source and must be acyclic. Explanation is at +most8,000characters and exact excerpt at most16,384. Comparisons are authored +context, not permission to combine incompatible metrics/populations. + +## Ownership and reads + +The ordinary project-file owner captures/revises the note, authenticates reads, +verifies captured UTF-8 bytes/hashes and retains previous revisions. There is no +new canonical family, SQL table, execution grant, recipe or mutation interface. +Existing `files_inspect` and `file_get` contracts are unchanged. Save/load keeps +the original guide and referenced note history. A fork whose new project/note +identities differ requires an explicitly re-authored guide; the reader refuses +foreign identities instead of retargeting them silently. + +`ProjectModelPortfolioDocument` owns the presentation-document decoder and +cross-reference/literal checks. `ProjectModelPortfolio` owns selection, +presentation and its pending reads. Its injected `read` must be the host's +already negotiated, current-authority, byte-verified project-file reader. +The host owns transport/hash validation and context identity. The controller +checks the exact model card ID/path/revision/SHA/project, and that all six +summary strings and that card's relationship excerpts occur literally in its +captured text before displaying a selected summary. Text is inert; no markup, +scripts, parameter loading, scientific scoring or source execution occurs. + +Only explicit actions read notes/cards. Discovery visits40adopted files per +page; no hidden scan/retry/polling occurs. The guide holds at most1,000models, +shows20per graph window with their ancestor groups, and reports exact total, +reviewed and filtered counts. Group/name/alias/review filters, paging and +textual relationship controls select the same entries as graph buttons. +Paragraphs wrap; exact source text remains in a disclosure. + +The host's binding/project/checkpoint/credential/generation context owns every +read. A context change, permission loss or teardown aborts pending work, clears +old content and restores focus to the host status when needed. Late replies +cannot publish into another context. Errors show their declared diagnostic and +clear old detail instead of displaying a previous model as the new selection. +Unsupported format or foreign-project binding is `unsupported`; malformed +document/reference/summary is `invalid`; transport/availability failures retain +their owning diagnostic. No success or empty-data fallback substitutes for a +failed read. + +## Verification scope + +Pure codec tests cover contract decisions using literal fixtures. Controlled +read and real-browser tests cover selection, unavailable/obsolete responses, +repeated invocation, focus, keyboard activation and wrapping on desktop/mobile. +Those fixtures do not establish real project capture or save/load. Native +acceptance additionally requires an actual adopted guide, exact original cards, +current authority, real project reads and retained checkpoint/custody evidence. diff --git a/static/project-files.css b/static/project-files.css index 515d9ca5..c18e2e30 100644 --- a/static/project-files.css +++ b/static/project-files.css @@ -10,3 +10,13 @@ #project-runtime-files,#project-runtime-summary,#project-runtime-pending,#project-runtime-layout,#project-runtime-status{overflow-wrap:anywhere;min-width:0}#project-runtime-layout,#project-runtime-status{white-space:pre-wrap;max-height:32rem;overflow:auto} #project-candidate-preparation-result,#project-candidate-preparation-status{overflow-wrap:anywhere;min-width:0}#project-candidate-preparation-result{white-space:pre-wrap;max-height:32rem;overflow:auto} + +#project-portfolio-source,#project-portfolio-detail,#project-portfolio-count{overflow-wrap:anywhere;min-width:0} +#project-portfolio-detail p{white-space:pre-wrap;max-width:85ch} +#project-portfolio-detail pre{white-space:pre-wrap;overflow-wrap:anywhere;max-height:32rem;overflow:auto} +#project-portfolio-detail details{margin:1rem 0} +#project-portfolio-detail h4{margin-bottom:.25rem} +#project-portfolio-detail h3{scroll-margin-top:1rem} +#project-portfolio-filter-form{display:grid;grid-template-columns:1fr 2fr 1fr;gap:.7rem;align-items:end} +#project-portfolio-filter-form button{justify-self:start} +@media(max-width:760px){#project-portfolio-filter-form{grid-template-columns:1fr}} diff --git a/static/project-files.html b/static/project-files.html index de9c6f1a..52f1ccca 100644 --- a/static/project-files.html +++ b/static/project-files.html @@ -14,6 +14,7 @@ + @@ -50,6 +51,7 @@

Project file

No unsubmitted edits.

+

Prepare a Candidate package or verifier

Choose a current Study, Candidate and captured CPU runtime. Preparation saves an inert Trial and exact sources; approval and Run admission remain separate actions.

diff --git a/static/project-files.js b/static/project-files.js index 8e57112d..3146719e 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -902,6 +902,17 @@ onContextFocusLost:()=>$('project-files-status').focus() }); window.addEventListener('pagehide',()=>projectProgression.clear()); + const projectPortfolio=window.ProjectModelPortfolio({ + root:$('project-model-portfolio'), + async read(action,args,signal){ + if(!m2Allowed(action))throw Error('Model organization inspection is unavailable for the selected project.'); + const value=await m2Request(action,args,epoch,{signal}); + if(value===null)throw new DOMException('Project selection changed.','AbortError'); + return value; + }, + onContextFocusLost:()=>$('project-files-status').focus() + }); + window.addEventListener('pagehide',()=>projectPortfolio.clear()); const projectMeasurements=window.ProjectMeasurements({ root:$('project-measurements'), async read(action,args,signal){ @@ -1167,6 +1178,8 @@ } const m2Allowed=action=>!busy&&!blocked&&state?.state==='open'&&selectedContracts.has('project_files.'+action); function m2Controls(){ + projectPortfolio.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), + project_id:state?.project?.project_id,enabled:m2Allowed('files_inspect')&&m2Allowed('file_get')}); projectProgression.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), project_id:state?.project?.project_id,enabled:m2Allowed('progression')&&m2Allowed('progression_origins')}); projectMeasurements.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), diff --git a/static/project-model-portfolio.js b/static/project-model-portfolio.js new file mode 100644 index 00000000..4f1b7fa3 --- /dev/null +++ b/static/project-model-portfolio.js @@ -0,0 +1,188 @@ +'use strict'; +// Authored organization is a retained note, not Trial lineage or qualification. +// This module owns its document contract, finite view and cancellable reads. +(() => { + const fields=['method','purpose','experiment','outcome','lesson','reproduction']; + const exact=(value,keys)=>value&&typeof value==='object'&&!Array.isArray(value)&& + Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); + const text=(value,limit)=>typeof value==='string'&&value.length>0&&value.length<=limit; + const identity=value=>text(value,160)&&/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value); + const sha=value=>typeof value==='string'&&/^[a-f0-9]{64}$/.test(value); + const natural=value=>Number.isSafeInteger(value)&&value>=0; + function fail(message,code='invalid'){throw Object.assign(Error(message),{code});} + function noteRef(value){ + if(!exact(value,['file_id','path','revision','sha256'])||!sha(value.file_id)||!sha(value.sha256)|| + !natural(value.revision)||value.revision<1||!text(value.path,512)||!value.path.startsWith('notes/')|| + value.path.split('/').some(part=>!part||part==='.'||part==='..'))fail('Invalid captured model-note reference.'); + } + function decode(raw){ + if(typeof raw!=='string'||new TextEncoder().encode(raw).length>1048576)fail('Organization note exceeds its retained text limit.'); + let value;try{value=JSON.parse(raw);}catch{fail('Organization note is not valid JSON.');} + if(value?.format!=='project_model_portfolio_v1')fail('Unsupported model-organization document format.','unsupported'); + if(!exact(value,['format','project_id','title','description','authorship','groups','models','relationships'])|| + !/^[a-f0-9]{32}$/.test(value.project_id)||!text(value.title,200)||!text(value.description,8000)|| + value.authorship!=='retrospective'||!Array.isArray(value.groups)||value.groups.length>64|| + !Array.isArray(value.models)||!value.models.length||value.models.length>1000|| + !Array.isArray(value.relationships)||value.relationships.length>2000)fail('Invalid model-organization document.'); + const groups=new Map(),models=new Map(); + for(const group of value.groups){ + if(!exact(group,['id','label','parent','description'])||!identity(group.id)||groups.has(group.id)|| + !text(group.label,200)||!text(group.description,8000)||group.parent!==null&&!identity(group.parent))fail('Invalid or duplicate organization group.'); + groups.set(group.id,group); + } + for(const group of groups.values()){ + const visited=new Set([group.id]);let parent=group.parent; + while(parent!==null){if(!groups.has(parent))fail('Organization group parent is unavailable.'); + if(visited.has(parent))fail('Organization grouping contains a cycle.');visited.add(parent);parent=groups.get(parent).parent;} + } + for(const model of value.models){ + if(!exact(model,['id','label','group_id','aliases','review','card','summary'])||!identity(model.id)||models.has(model.id)|| + !text(model.label,500)||!groups.has(model.group_id)||!Array.isArray(model.aliases)||model.aliases.length>128|| + new Set(model.aliases).size!==model.aliases.length||model.aliases.some(alias=>!text(alias,500))|| + !['source_reviewed','review_pending'].includes(model.review))fail('Invalid or duplicate organized model.'); + noteRef(model.card); + if(model.review==='source_reviewed'){ + if(!exact(model.summary,fields)||fields.some(field=>!text(model.summary[field],16384)))fail('Reviewed model requires all six authored summary fields.'); + }else if(model.summary!==null)fail('Pending review cannot expose a reviewed summary.'); + models.set(model.id,model); + } + const pairs=new Set(); + for(const relation of value.relationships){ + if(!exact(relation,['from','to','kind','explanation','evidence_model','excerpt'])|| + !models.has(relation.from)||!models.has(relation.to)||relation.from===relation.to|| + !['comparison','parameter_derivation'].includes(relation.kind)||!text(relation.explanation,8000)|| + !models.has(relation.evidence_model)||!text(relation.excerpt,16384))fail('Invalid authored model relationship.'); + const key=JSON.stringify([relation.from,relation.to,relation.kind]);if(pairs.has(key))fail('Duplicate authored model relationship.');pairs.add(key); + } + const parents=new Map(),complete=new Set(),visiting=new Set(); + for(const relation of value.relationships)if(relation.kind==='parameter_derivation'){ + if(!parents.has(relation.from))parents.set(relation.from,[]);parents.get(relation.from).push(relation.to); + } + function visit(id){if(complete.has(id))return;if(visiting.has(id))fail('Authored parameter derivation contains a cycle.'); + visiting.add(id);for(const parent of parents.get(id)||[])visit(parent);visiting.delete(id);complete.add(id);} + for(const id of parents.keys())visit(id); + return value; + } + function checkCard(document,model,result){ + const ref=model.card,version=result?.version; + if(result?.project_id!==document.project_id||result.id!==ref.file_id||result.path!==ref.path||result.kind!=='note'|| + version?.deleted===true||version?.revision!==ref.revision||version.sha256!==ref.sha256||typeof version.text!=='string') + fail('Model card differs from its exact retained organization reference.'); + if(model.summary&&fields.some(field=>!version.text.includes(model.summary[field])))fail('Authored summary differs from the captured model card.'); + for(const relation of document.relationships)if(relation.evidence_model===model.id&&!version.text.includes(relation.excerpt)) + fail('Authored relationship excerpt differs from its captured model card.'); + return version.text; + } + window.ProjectModelPortfolioDocument=Object.freeze({decode,checkCard}); + const node=(tag,value)=>{const element=document.createElement(tag);if(value!==undefined)element.textContent=value;return element;}; + window.ProjectModelPortfolio=function({root,read,onContextFocusLost=()=>{}}){ + root.innerHTML='

Authored model organization

Browse a retained guide to models and their methods. Grouping and retrospective comparison notes are separate from recorded Trial ancestry and scientific qualification.

'; + const $=name=>root.querySelector('#project-portfolio-'+name); + let context=null,generation=0,pending=null,busy=false,notes=new Map(),noteOffset=0,noteTotal=0; + let guide=null,groups=new Map(),models=new Map(),applied={group:'',search:'',review:''},offset=0,selected=''; + const pageSize=20; + function controls(){ + const enabled=!!context?.enabled; + for(const name of ['notes','notes-more','note','load','group','search','review','prev','next'])$(name).disabled=!enabled||busy; + $('note').disabled ||= !notes.size;$('load').disabled ||= !notes.has($('note').value); + $('notes-more').hidden=noteOffset>=noteTotal;$('notes-more').disabled ||= noteOffset>=noteTotal; + $('prev').disabled ||= !guide||offset===0;$('next').disabled ||= !guide||offset+pageSize>=filtered().length; + for(const button of root.querySelectorAll('#project-portfolio-filter-form button,[data-lineage-node],#project-portfolio-detail button'))button.disabled=!enabled||busy; + root.setAttribute('aria-busy',String(busy)); + } + function clearGuide(){guide=null;groups.clear();models.clear();selected='';offset=0;applied={group:'',search:'',review:''};$('content').hidden=true;$('detail').replaceChildren();$('graph').replaceChildren();$('search').value='';$('review').value='';$('group').replaceChildren();} + function reset(){++generation;pending?.abort();pending=null;busy=false;notes.clear();noteOffset=noteTotal=0;$('note').replaceChildren();$('note-count').textContent='';$('status').textContent='';clearGuide();controls();} + async function perform(work){ + if(!context?.enabled||busy)return; + const ticket=++generation,key=context.key;pending=new AbortController();const signal=pending.signal;busy=true;controls(); + const active=()=>ticket===generation&&context?.enabled&&context.key===key&&!signal.aborted; + try{await work(active,signal);}catch(error){if(active()){$('detail').replaceChildren();$('status').textContent=(error.code?error.code+': ':'')+error.message;$('status').focus();}} + finally{if(active()){busy=false;pending=null;controls();}} + } + async function findNotes(more=false){ + return perform(async(active,signal)=>{ + const requested=more?noteOffset:0,value=await read('files_inspect',{offset:requested,limit:40},signal);if(!active())return; + if(value?.offset!==requested||value.limit!==40||!natural(value.total)||!Array.isArray(value.items)|| + value.items.length!==Math.min(40,Math.max(0,value.total-requested)))fail('Organization note page differs from its continuation.'); + if(!more){clearGuide();notes.clear();$('note').replaceChildren();} + for(const item of value.items)if(item.kind==='note'&&item.path?.endsWith('.portfolio.json')){ + if(!sha(item.id)||!natural(item.revision)||item.revision<1||!sha(item.expected_sha256)||notes.has(item.id))fail('Invalid organization note selection.'); + notes.set(item.id,structuredClone(item));const option=node('option',item.path+' · revision '+item.revision);option.value=item.id;$('note').append(option); + } + noteOffset=requested+value.items.length;noteTotal=value.total; + $('note-count').textContent=`${notes.size} organization notes found; inspected ${noteOffset} of ${noteTotal} adopted files.`; + $('status').textContent=notes.size?'Choose a retained organization note.':'No organization note on these pages. Adopt a .portfolio.json note or inspect the next page.'; + }); + } + function inGroup(model,wanted){let id=model.group_id;while(id!==null){if(id===wanted)return true;id=groups.get(id).parent;}return false;} + function filtered(){if(!guide)return [];return guide.models.filter(model=>(!applied.group||inGroup(model,applied.group))&& + (!applied.review||model.review===applied.review)&&(!applied.search||[model.label,...model.aliases].some(value=>value.toLocaleLowerCase().includes(applied.search))));} + function render(){ + if(!guide)return; + const focused=$('graph').contains(document.activeElement)?document.activeElement.id:null; + const matching=filtered(),shown=matching.slice(offset,offset+pageSize),included=new Set(); + for(const model of shown){let id=model.group_id;while(id!==null){included.add(id);id=groups.get(id).parent;}} + const graphNodes=[...included].map(id=>{const group=groups.get(id);return {id:'group/'+id,title:group.label,recordedAt:'',parent:group.parent===null?null:'group/'+group.parent,baseline:null,lines:['Authored method group']};}); + graphNodes.push(...shown.map(model=>({id:'model/'+model.id,title:model.label,recordedAt:'',parent:'group/'+model.group_id,baseline:null, + lines:[model.aliases.slice(0,3).join(' · ')||model.id,model.review==='source_reviewed'?'Source reviewed · authored summary':'Source review pending']}))); + $('graph').innerHTML=window.WorkspaceExperimentGraph.render(graphNodes,{selected:selected?'model/'+selected:'',zoom:1}).html; + for(const button of $('graph').querySelectorAll('[data-lineage-node]')){ + const value=button.dataset.lineageNode; + button.onclick=()=>{if(value.startsWith('model/'))void selectModel(value.slice(6));else{$('group').value=value.slice(6);applyFilters();}}; + } + $('count').textContent=`${guide.models.length} models in this guide; ${guide.models.filter(model=>model.review==='source_reviewed').length} source reviewed. ${matching.length} match the applied filters. Showing ${shown.length?offset+1:0}–${offset+shown.length}.`; + controls(); + if(focused){const replacement=$('graph').querySelector('#'+CSS.escape(focused)); + if(replacement)replacement.focus({preventScroll:true});else{$('count').tabIndex=-1;$('count').focus();}} + } + function applyFilters(){if(!guide||busy)return;applied={group:$('group').value,search:$('search').value.trim().toLocaleLowerCase(),review:$('review').value}; + if(applied.group&&!groups.has(applied.group))return;offset=0;selected='';$('detail').replaceChildren();render();$('status').textContent='Applied model filters.';} + async function loadGuide(){ + const choice=notes.get($('note').value);if(!choice)return; + return perform(async(active,signal)=>{ + clearGuide();const value=await read('file_get',{file_id:choice.id,revision:choice.revision},signal);if(!active())return; + if(value?.id!==choice.id||value.project_id!==context.project_id||value.kind!=='note'||value.path!==choice.path|| + value.version?.revision!==choice.revision||value.version.sha256!==choice.expected_sha256||value.version.deleted===true)fail('Organization note differs from the selected captured revision.'); + const parsed=decode(value.version.text); + if(parsed.project_id!==context.project_id)fail('This guide belongs to another project; re-author its captured note bindings.','unsupported'); + guide=parsed;groups=new Map(guide.groups.map(group=>[group.id,group]));models=new Map(guide.models.map(model=>[model.id,model])); + $('title').textContent=guide.title;$('description').textContent=guide.description; + $('source').textContent=`Retrospective authored guide · ${choice.path} · revision ${choice.revision} · SHA256 ${choice.expected_sha256}`; + const all=node('option','All groups');all.value='';$('group').append(all); + for(const group of guide.groups){const option=node('option',group.label);option.value=group.id;$('group').append(option);} + $('content').hidden=false;render();$('status').textContent='Loaded retained authored model organization.'; + }); + } + async function selectModel(id){ + const model=models.get(id);if(!model||busy)return; + return perform(async(active,signal)=>{ + if(!filtered().some(item=>item.id===id)){ + applied={group:'',search:'',review:''};$('group').value='';$('search').value='';$('review').value=''; + } + offset=Math.floor(filtered().findIndex(item=>item.id===id)/pageSize)*pageSize; + selected=id;$('detail').replaceChildren();render(); + const value=await read('file_get',{file_id:model.card.file_id,revision:model.card.revision},signal);if(!active())return; + const captured=checkCard(guide,model,value),heading=node('h3',model.label);heading.id='project-portfolio-model-heading';heading.tabIndex=-1;$('detail').append(heading); + $('detail').append(node('p',(model.aliases.join(' · ')||model.id)+' · '+(model.review==='source_reviewed'?'Source reviewed':'Source review pending'))); + if(model.summary){const labels={method:'Method',purpose:'Purpose',experiment:'Training and experiment',outcome:'Observed outcome',lesson:'Lesson',reproduction:'Reproduction'}; + for(const field of fields){$('detail').append(node('h4',labels[field]),node('p',model.summary[field]));}} + else $('detail').append(node('p','A source-reviewed interpretation has not been recorded for this fit. Its original captured card remains available below.')); + const relations=guide.relationships.filter(relation=>relation.from===id||relation.to===id); + if(relations.length){$('detail').append(node('h4','Authored model relationships')); + for(const relation of relations){const box=node('div'),other=models.get(relation.from===id?relation.to:relation.from),button=node('button', + (relation.kind==='comparison'?'Comparison: ':relation.from===id?'Parameters from: ':'Derived model: ')+other.label);button.type='button';button.onclick=()=>void selectModel(other.id); + box.append(button,node('p',relation.explanation));const evidence=node('details');evidence.append(node('summary','Exact retained relationship excerpt'),node('p',relation.excerpt)); + const source=node('button','Read evidence card: '+models.get(relation.evidence_model).label);source.type='button';source.onclick=()=>void selectModel(relation.evidence_model);evidence.append(source);box.append(evidence);$('detail').append(box);}} + const raw=node('details'),summary=node('summary','Exact captured card and original receipts');raw.append(summary,node('p',`${model.card.path} · revision ${model.card.revision} · SHA256 ${model.card.sha256}`),node('pre',captured));$('detail').append(raw); + $('status').textContent='Exact retained model summary and captured evidence.';heading.focus(); + }); + } + $('notes').onclick=()=>void findNotes();$('notes-more').onclick=()=>void findNotes(true); + $('note').onchange=()=>{clearGuide();controls();}; + $('source-form').onsubmit=event=>{event.preventDefault();void loadGuide();}; + $('filter-form').onsubmit=event=>{event.preventDefault();applyFilters();}; + for(const [name,direction] of [['prev',-1],['next',1]])$(name).onclick=()=>{if(busy||!guide)return;offset=Math.max(0,offset+direction*pageSize);selected='';$('detail').replaceChildren();render();}; + controls(); + return {setContext(value){if(value.key!==context?.key||context?.enabled&&!value.enabled){const owned=root.contains(document.activeElement);context=value;reset();if(owned)onContextFocusLost();}else context=value;root.hidden=!value.enabled;controls();},clear(){context=null;reset();root.hidden=true;}}; + }; +})(); diff --git a/tests/browser_model_portfolio.cjs b/tests/browser_model_portfolio.cjs new file mode 100644 index 00000000..851e650f --- /dev/null +++ b/tests/browser_model_portfolio.cjs @@ -0,0 +1,129 @@ +'use strict'; +// Real Chromium interaction/layout and controlled read-lifecycle evidence. +// This isolated static fixture does not claim native project persistence. +const fs=require('node:fs'),path=require('node:path'),http=require('node:http'),crypto=require('node:crypto'); +const {spawn}=require('node:child_process'); +const {waitForDebugger}=require('./browser_startup.cjs'),{cdpSession}=require('./browser_cdp_session.cjs'); +const output=path.resolve(process.argv[2]||''),engine=process.env.TULDOK_BROWSER_EXECUTABLE; +if(!process.argv[2]||!engine||fs.existsSync(output))throw Error('Explicit fresh output directory and browser executable required'); +fs.mkdirSync(output,{recursive:true,mode:0o700}); +const fixture=structuredClone(require('./model_portfolio_fixture.cjs')); +const hash=value=>crypto.createHash('sha256').update(value).digest('hex'); +const first=fixture.document.models[0]; +const oldMethod=first.summary.method;first.summary.method+=' '+('A physical pressure field with explicit case and work scope. '.repeat(60)); +first.label='First model '; +fixture.cards[0].version.text=fixture.cards[0].version.text.replace(oldMethod,first.summary.method); +fixture.cards[0].version.sha256=first.card.sha256=hash(fixture.cards[0].version.text); +fixture.cards[0].version.bytes=Buffer.byteLength(fixture.cards[0].version.text); +for(let index=0;index<20;index++){ + const model=structuredClone(fixture.document.models[1]);model.id='extra-'+index;model.label='Extra model '+index;model.aliases=[]; + model.card.path='notes/extra-'+index+'.md';model.card.file_id=hash(fixture.project+'\0note\0'+model.card.path); + fixture.document.models.push(model);fixture.cards.push({...structuredClone(fixture.cards[1]),id:model.card.file_id,path:model.card.path}); +} +fixture.guide.version.text=JSON.stringify(fixture.document); +fixture.guide.version.sha256=fixture.inspection.expected_sha256=fixture.inspection.observed_sha256=hash(fixture.guide.version.text); +fixture.guide.version.bytes=fixture.inspection.expected_bytes=fixture.inspection.observed_bytes=Buffer.byteLength(fixture.guide.version.text); +const inline=JSON.stringify(fixture).replaceAll('<','\\u003c'); +const html=`Authored organization fixture
`; +const staticRoot=path.join(__dirname,'../static'),allowed=new Map([ + ['/project-files.css',['project-files.css','text/css']],['/experiment-lineage.css',['experiment-lineage.css','text/css']], + ['/experiment-graph.js',['experiment-graph.js','text/javascript']],['/project-model-portfolio.js',['project-model-portfolio.js','text/javascript']]]); +const server=http.createServer((request,response)=>{ + if(request.url==='/'){response.writeHead(200,{'Content-Type':'text/html; charset=utf-8'});response.end(html);return;} + const asset=allowed.get(request.url);if(!asset){response.writeHead(404);response.end();return;} + response.writeHead(200,{'Content-Type':asset[1]});response.end(fs.readFileSync(path.join(staticRoot,asset[0]))); +}); +let child,session; +const report={status:'starting',engine,verification_only:true,native_persistence_claim:false,graphics:'Headless Chromium with GPU disabled; no model or rendering speed claim.',checks:[],errors:[],browser_stopped:false}; +const sourceFiles=['app.py','static/project-files.css','static/project-files.html','static/project-files.js','static/project-model-portfolio.js', + 'docs/contracts/project-model-portfolio.md','tests/model_portfolio_fixture.cjs','tests/test_project_model_portfolio.cjs', + 'tests/browser_model_portfolio.cjs','tests/test_project_model_portfolio_http.py','tests/test_workspace_project_files.cjs','tests/test_workspace_project_files_m4.cjs']; +report.source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); +const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); +async function main(){ + await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); + const origin='http://127.0.0.1:'+server.address().port; + try{ + const flags=['--headless','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,1000','--user-data-dir='+path.join(output,'profile'),'about:blank']; + report.flags=flags;report.origin=origin; + const log=fs.openSync(path.join(output,'browser.log'),'w');child=spawn(engine,flags,{stdio:['ignore',log,log]});fs.closeSync(log); + const port=await waitForDebugger(child,path.join(output,'profile/DevToolsActivePort')); + const targets=await(await fetch('http://127.0.0.1:'+port+'/json/list')).json(),target=targets.find(target=>target.type==='page'); + const socket=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{socket.addEventListener('open',resolve,{once:true});socket.addEventListener('error',reject,{once:true});}); + session=cdpSession(socket,{onEvent:event=>{if(event.method==='Runtime.exceptionThrown')report.errors.push(event.params.exceptionDetails.text);}}); + const send=session.send; + await send('Page.enable');await send('Runtime.enable'); + async function evaluate(expression){const result=await send('Runtime.evaluate',{expression,returnByValue:true,awaitPromise:true});if(result.exceptionDetails)throw Error(result.exceptionDetails.exception?.description||result.exceptionDetails.text);return result.result.value;} + async function until(expression){const end=Date.now()+15000;while(Date.now(){const element=document.querySelector(${JSON.stringify(selector)});if(!element||element.disabled)throw Error('Unavailable fixture control');element.scrollIntoView({block:'center'});element.focus();})()`); + await send('Input.dispatchKeyEvent',{type:'keyDown',key:'Enter',code:'Enter',windowsVirtualKeyCode:13,text:'\r'});await send('Input.dispatchKeyEvent',{type:'keyUp',key:'Enter',code:'Enter',windowsVirtualKeyCode:13});} + async function fill(selector,value){await evaluate(`(()=>{const element=document.querySelector(${JSON.stringify(selector)});element.value=${JSON.stringify(value)};element.dispatchEvent(new Event('input',{bubbles:true}));element.dispatchEvent(new Event('change',{bubbles:true}));})()`);} + async function screenshot(name){const shot=await send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,name),Buffer.from(shot.data,'base64'));} + async function load(){await keyClick('#project-portfolio-notes');await until("document.querySelector('#project-portfolio-note').options.length===1");await keyClick('#project-portfolio-load');await until("document.querySelector('#project-portfolio-status').textContent==='Loaded retained authored model organization.'");} + async function selectFirst(){await keyClick('[data-lineage-node="model/fit-first"]');await until("document.querySelector('#project-portfolio-status').textContent==='Exact retained model summary and captured evidence.'");} + await send('Page.navigate',{url:origin+'/'});await until("!!window.fixtureUI"); + await load();await check('readCalls.length===2','Guide loading reads no model card or payload'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('22 models in this guide; 1 source reviewed')",'Exact complete and reviewed guide counts'); + const point=await evaluate("(()=>{const element=document.querySelector('[data-lineage-node=\"group/pressure\"]');element.scrollIntoView({block:'center'});const box=element.getBoundingClientRect();return {x:box.x+box.width/2,y:box.y+box.height/2};})()"); + await send('Input.dispatchMouseEvent',{type:'mousePressed',button:'left',clickCount:1,...point});await send('Input.dispatchMouseEvent',{type:'mouseReleased',button:'left',clickCount:1,...point}); + await check("document.querySelector('#project-portfolio-group').value==='pressure'&&document.activeElement.dataset.lineageNode==='group/pressure'",'Pointer group activation applies the group and retains visible keyboard focus'); + await check("!document.querySelector('#root img')&&!window.injected",'Authored graph labels remain inert text'); + await keyClick('#project-portfolio-next');await check("document.querySelector('#project-portfolio-count').textContent.includes('Showing 21–22.')",'Bounded model paging uses the full population'); + await keyClick('#project-portfolio-prev');await selectFirst(); + await check("document.activeElement.id==='project-portfolio-model-heading'",'Keyboard selection moves focus to selected summary'); + await check("document.querySelector('#project-portfolio-detail').innerText.includes(fixtureData.document.models[0].summary.method)",'Complete long method paragraph is readable retained text'); + await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Desktop detail wraps without horizontal overflow'); + await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('desktop-summary.png'); + await send('Emulation.setDeviceMetricsOverride',{width:390,height:844,deviceScaleFactor:1,mobile:true}); + await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Mobile detail wraps without horizontal overflow'); + await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('mobile-summary.png'); + await send('Emulation.clearDeviceMetricsOverride'); + await fill('#project-portfolio-search','C107');await keyClick('#project-portfolio-filter-form button'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('1 match the applied filters')&&document.querySelector('#project-portfolio-next').disabled",'Alias filtering has exact counts and paging'); + await selectFirst();await keyClick('#project-portfolio-detail button');await until("document.querySelector('#project-portfolio-model-heading')?.textContent==='Second model'"); + await check("document.querySelector('#project-portfolio-detail').innerText.includes('A source-reviewed interpretation has not been recorded')",'Pending review does not invent a method summary'); + await check("document.querySelector('[data-lineage-node=\"model/fit-second\"]').getAttribute('aria-pressed')==='true'",'Off-filter relationship navigation brings selected model into the graph window'); + await selectFirst();await evaluate('window.failNext=true');await keyClick('#project-portfolio-detail button'); + await until("document.querySelector('#project-portfolio-status').textContent==='unavailable: Retained card unavailable.'"); + await check("document.querySelector('#project-portfolio-detail').children.length===0",'Unavailable card clears the previous model detail'); + await evaluate('window.corruptNext=true');await keyClick('[data-lineage-node="model/fit-first"]'); + await until("document.querySelector('#project-portfolio-status').textContent==='invalid: Model card differs from its exact retained organization reference.'"); + await check("document.querySelector('#project-portfolio-detail').children.length===0",'Foreign card revision is refused without stale detail'); + await evaluate('window.holdNext=true');await keyClick('[data-lineage-node="model/fit-first"]');await until('!!window.releaseHeld'); + await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:true});window.releaseHeld()"); + await check("window.heldSignal.aborted&&document.querySelector('#project-portfolio-content').hidden",'Context replacement aborts and clears pending view'); + await load();await check("document.querySelector('#project-portfolio-detail').children.length===0",'Reload after supersession starts without old detail'); + await selectFirst();await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:false})"); + await check("document.querySelector('#root').hidden&&document.querySelector('#project-portfolio-detail').children.length===0&&document.activeElement.id==='outside'",'Permission loss clears private view and restores host focus'); + await evaluate("fixtureUI.setContext({key:'three',project_id:fixtureData.project,enabled:true})");await load(); + await fill('#project-portfolio-search','No match');await keyClick('#project-portfolio-filter-form button'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('0 match the applied filters. Showing 0–0.')",'Empty filter result retains exact zero denominator'); + await check('readCalls.every(call=>[\'files_inspect\',\'file_get\'].includes(call.action))','All interactions use read-only project-file actions'); + if(report.errors.length)throw Error('Browser runtime exception');report.status='passed_authored_portfolio_real_browser_component'; + }catch(error){report.status='failed_authored_portfolio_real_browser_component';report.error=String(error); + if(session){try{const state=await session.send('Runtime.evaluate',{expression:'document.body.innerText',returnByValue:true});report.failure_page=state.result?.value; + const shot=await session.send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,'failed-page.png'),Buffer.from(shot.data,'base64'));}catch(captureError){report.failure_capture_error=String(captureError);}} + throw error;} + finally{session?.close();if(child&&child.exitCode===null&&child.signalCode===null){child.kill('SIGTERM');for(let i=0;i<100&&child.exitCode===null&&child.signalCode===null;i++)await pause(50);if(child.exitCode===null&&child.signalCode===null)child.kill('SIGKILL');} + report.browser_stopped=!child||child.exitCode!==null||child.signalCode!==null;await new Promise(resolve=>server.close(resolve)); + report.end_source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); + report.source_stable_through_tests=JSON.stringify(report.source_sha256)===JSON.stringify(report.end_source_sha256); + if(!report.source_stable_through_tests){report.status='failed_authored_portfolio_real_browser_component';report.errors.push('Source changed during browser qualification');process.exitCode=1;} + report.producer_sha256=hash(fs.readFileSync(__filename));fs.writeFileSync(path.join(output,'report.json'),JSON.stringify(report,null,2)+'\n');} + console.log(report.status,report.checks.length,'checks'); +} +main().catch(error=>{console.error(String(error));process.exitCode=1;}); diff --git a/tests/model_portfolio_fixture.cjs b/tests/model_portfolio_fixture.cjs new file mode 100644 index 00000000..cb63135a --- /dev/null +++ b/tests/model_portfolio_fixture.cjs @@ -0,0 +1,16 @@ +'use strict'; +// Literal presentation-document fixtures, independent of the decoder. +const crypto=require('node:crypto'); +const hash=value=>crypto.createHash('sha256').update(value).digest('hex'); +const project='a'.repeat(32),path='notes/00-methods.portfolio.json'; +const summary={method:'A small pressure correction model.',purpose:'Measure one pressure correction.',experiment:'Two retained training scenes.',outcome:'Retained diagnostic result; not a rollout qualification.',lesson:'Compare work and accuracy on the same cases.',reproduction:'Use the retained source recipe; clean retraining remains pending.'}; +const cardText=Object.values(summary).join('\n\n')+'\nOriginal receipt bytes remain here.\n'; +const ref=(path,text)=>({file_id:hash(project+'\0note\0'+path),path,revision:2,sha256:hash(text)}); +const first={id:'fit-first',label:'First model',group_id:'pressure',aliases:['C107'],review:'source_reviewed',card:ref('notes/first.md',cardText),summary}; +const second={id:'fit-second',label:'Second model',group_id:'pressure',aliases:['C106'],review:'review_pending',card:ref('notes/second.md','Original pending card.\n'),summary:null}; +const document={format:'project_model_portfolio_v1',project_id:project,title:'Retained fit guide',description:'Authored retrospective organization; original Trial history is unchanged.',authorship:'retrospective',groups:[{id:'pressure',label:'Pressure methods',parent:null,description:'Pressure requests, not complete simulation steps.'}],models:[first,second],relationships:[{from:first.id,to:second.id,kind:'comparison',explanation:'Authored comparison with the earlier control.',evidence_model:first.id,excerpt:summary.lesson}]}; +const result=(model,text)=>({id:model.card.file_id,project_id:project,kind:'note',path:model.card.path,version:{revision:2,sha256:hash(text),bytes:Buffer.byteLength(text),text,actor_id:'b'.repeat(32),adopted_at:'2026-10-25T00:00:00+00:00'}}); +const cards=[result(first,cardText),result(second,'Original pending card.\n')]; +const raw=JSON.stringify(document),guideRef=ref(path,raw),guide={...result({card:guideRef},raw),version:{...result({card:guideRef},raw).version,revision:1}}; +const inspection={id:guideRef.file_id,kind:'note',path,revision:1,expected_sha256:guideRef.sha256,expected_bytes:Buffer.byteLength(raw),observed_sha256:guideRef.sha256,observed_bytes:Buffer.byteLength(raw),state:'unchanged',reason:null}; +module.exports={project,document,summary,cards,guide,inspection}; diff --git a/tests/test_project_model_portfolio.cjs b/tests/test_project_model_portfolio.cjs new file mode 100644 index 00000000..704e4035 --- /dev/null +++ b/tests/test_project_model_portfolio.cjs @@ -0,0 +1,47 @@ +'use strict'; +const assert=require('node:assert/strict'),fs=require('node:fs'),vm=require('node:vm'),path=require('node:path'); +const fixture=require('./model_portfolio_fixture.cjs'),window={}; +vm.runInNewContext(fs.readFileSync(path.join(__dirname,'../static/project-model-portfolio.js'),'utf8'),{window,TextEncoder}); +const {decode,checkCard}=window.ProjectModelPortfolioDocument; +const valid=()=>structuredClone(fixture.document),parse=value=>decode(JSON.stringify(value)); +assert.deepEqual(JSON.parse(JSON.stringify(parse(valid()))),fixture.document); +let checks=1; +function bad(change,message,code='invalid'){ + const value=valid();change(value); + assert.throws(()=>parse(value),error=>error.code===code&&error.message===message);checks++; +} +bad(d=>{d.format='other';},'Unsupported model-organization document format.','unsupported'); +bad(d=>{d.extra=true;},'Invalid model-organization document.'); +bad(d=>{d.models=[];},'Invalid model-organization document.'); +bad(d=>{d.groups.push(structuredClone(d.groups[0]));},'Invalid or duplicate organization group.'); +bad(d=>{d.groups[0].parent='missing';},'Organization group parent is unavailable.'); +bad(d=>{d.groups[0].parent='pressure';},'Organization grouping contains a cycle.'); +bad(d=>{d.models.push(structuredClone(d.models[0]));},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].group_id='missing';},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].aliases.push('C107');},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].review='qualified';},'Invalid or duplicate organized model.'); +bad(d=>{delete d.models[0].summary.method;},'Reviewed model requires all six authored summary fields.'); +bad(d=>{d.models[1].summary={};},'Pending review cannot expose a reviewed summary.'); +bad(d=>{d.models[0].card.path='notes/../private';},'Invalid captured model-note reference.'); +bad(d=>{d.models[0].card.revision=0;},'Invalid captured model-note reference.'); +bad(d=>{d.relationships[0].to='unknown';},'Invalid authored model relationship.'); +bad(d=>{d.relationships[0].to=d.relationships[0].from;},'Invalid authored model relationship.'); +bad(d=>{d.relationships[0].kind='trial_parent';},'Invalid authored model relationship.'); +bad(d=>{d.relationships.push(structuredClone(d.relationships[0]));},'Duplicate authored model relationship.'); +bad(d=>{const a=d.relationships[0];a.kind='parameter_derivation';d.relationships.push({...a,from:a.to,to:a.from});},'Authored parameter derivation contains a cycle.'); +assert.throws(()=>decode('not JSON'),error=>error.code==='invalid'&&error.message==='Organization note is not valid JSON.');checks++; +assert.throws(()=>decode(' '.repeat(1048577)),error=>error.code==='invalid'&&error.message==='Organization note exceeds its retained text limit.');checks++; +assert.equal(checkCard(valid(),fixture.document.models[0],fixture.cards[0]),fixture.cards[0].version.text);checks++; +for(const change of [r=>{r.project_id='c'.repeat(32);},r=>{r.id='d'.repeat(64);},r=>{r.path='notes/other.md';},r=>{r.version.revision=3;},r=>{r.version.sha256='e'.repeat(64);},r=>{r.version.deleted=true;}]){ + const result=structuredClone(fixture.cards[0]);change(result); + assert.throws(()=>checkCard(valid(),fixture.document.models[0],result),error=>error.code==='invalid'&&error.message==='Model card differs from its exact retained organization reference.');checks++; +} +{ + const value=valid();value.models[0].summary.method='Not present in the captured card.'; + assert.throws(()=>checkCard(value,value.models[0],fixture.cards[0]),error=>error.message==='Authored summary differs from the captured model card.');checks++; +} +{ + const value=valid();value.relationships[0].excerpt='Not present in the captured card.'; + assert.throws(()=>checkCard(value,value.models[0],fixture.cards[0]),error=>error.message==='Authored relationship excerpt differs from its captured model card.');checks++; +} +console.log('PASS authored portfolio document:',checks,'literal contract/reference decisions; no native capture or browser claim.'); diff --git a/tests/test_project_model_portfolio_http.py b/tests/test_project_model_portfolio_http.py new file mode 100644 index 00000000..8fbda68f --- /dev/null +++ b/tests/test_project_model_portfolio_http.py @@ -0,0 +1,84 @@ +"""Real authenticated note transport/cold reload; no model or recipe execution.""" +import hashlib +import json +import os +from pathlib import Path +import unittest +from unittest.mock import patch + +import project_workspace +import test_project_files_http as m1 +import test_project_files_m2_http as m2 + +ROOT = Path(__file__).resolve().parents[1] + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class ModelPortfolioHTTP(unittest.TestCase): + cleanup_owned = m1.ProjectFilesHTTP.cleanup_owned + select_host = m1.ProjectFilesHTTP.select_host + request = m1.ProjectFilesHTTP.request + state = m1.ProjectFilesHTTP.state + envelope = m1.ProjectFilesHTTP.envelope + action = m1.ProjectFilesHTTP.action + create = m1.ProjectFilesHTTP.create + arguments = m2.ProjectFilesM2HTTP.arguments + transport = m2.ProjectFilesM2HTTP.transport + call = m2.ProjectFilesM2HTTP.call + + def setUp(self): + self.operations = [] + m1.ProjectFilesHTTP.setUp(self) + gate = patch.object(project_workspace, 'M4_ENABLED', True) + gate.start() + self.addCleanup(gate.stop) + + def test_native_static_module_exact_note_revision_and_cold_reload(self): + status, html = self.request('/project-files') + self.assertEqual(status, 200) + self.assertIn(b'/project-model-portfolio.js', html) + self.assertIn(b'id="project-model-portfolio"', html) + status, script = self.request('/project-model-portfolio.js') + self.assertEqual(status, 200) + self.assertEqual(script, (ROOT/'static/project-model-portfolio.js').read_bytes()) + opened = self.create() + root = self.external/'alpha' + (root/'notes').mkdir() + paragraphs = {key: 'Literal retained '+key+' paragraph.' for key in + ('method','purpose','experiment','outcome','lesson','reproduction')} + card_text = '\n\n'.join(paragraphs.values())+'\nOriginal evidence remains.\n' + (root/'notes/model.md').write_text(card_text) + card = self.call('file_adopt',kind='note',path='notes/model.md',expected_revision=0, + expected_sha256=hashlib.sha256(card_text.encode()).hexdigest()) + guide = dict(format='project_model_portfolio_v1',project_id=opened['project']['project_id'], + title='Authored retained fixture',description='Synthetic presentation fixture; no scientific quality claim.', + authorship='retrospective',groups=[dict(id='pressure',label='Pressure methods',parent=None,description='An authored group.')], + models=[dict(id='fixture-fit',label='One retained fit',group_id='pressure',aliases=['Fixture'],review='source_reviewed', + card=dict(file_id=card['id'],path='notes/model.md',revision=1,sha256=card['sha256']),summary=paragraphs)],relationships=[]) + raw=(json.dumps(guide,ensure_ascii=False)+'\n').encode() + (root/'notes/00-fixture.portfolio.json').write_bytes(raw) + saved=self.call('file_adopt',kind='note',path='notes/00-fixture.portfolio.json',expected_revision=0, + expected_sha256=hashlib.sha256(raw).hexdigest()) + observed=self.call('file_get',file_id=saved['id'],revision=1) + self.assertEqual(observed['version']['text'].encode(),raw) + self.assertEqual(json.loads(observed['version']['text']),guide) + changed='A later authored card revision.\n' + (root/'notes/model.md').write_text(changed) + self.call('file_adopt',kind='note',path='notes/model.md',expected_revision=1, + expected_sha256=hashlib.sha256(changed.encode()).hexdigest()) + self.assertEqual(self.call('file_get',file_id=card['id'],revision=1)['version']['text'],card_text) + self.assertEqual(self.call('file_get',file_id=card['id'])['version']['text'],changed) + self.action('save') + expected=(root/'.tuldok/project.json').read_bytes() + self.action('close') + cold=os.environ.get('TULDOK_PROJECT_FILES_COLD_DATABASE_URL') + self.assertTrue(cold,'Independent cold role required for retained-note acceptance') + self.select_host(self.path/'cold-control',{'fixture':self.external},cold) + self.action('open',root='fixture',directory='alpha') + self.assertEqual((root/'.tuldok/project.json').read_bytes(),expected) + self.assertEqual(self.call('file_get',file_id=saved['id'],revision=1)['version']['text'].encode(),raw) + retained=self.call('file_get',file_id=card['id'],revision=1) + self.assertEqual(retained['version']['text'],card_text) + self.assertEqual(retained['project_id'],guide['project_id']) + self.assertEqual(self.call('file_get',file_id=card['id'])['version']['text'],changed) + self.action('close') diff --git a/tests/test_workspace_project_files.cjs b/tests/test_workspace_project_files.cjs index 31339b01..86582638 100644 --- a/tests/test_workspace_project_files.cjs +++ b/tests/test_workspace_project_files.cjs @@ -38,6 +38,7 @@ function mount(initial=[reply(state()),reply(records())],capabilityValue=capabil context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); context.ProjectMeasurements=()=>({setContext(){},clear(){}}); + context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,events,discoveries,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, diff --git a/tests/test_workspace_project_files_m4.cjs b/tests/test_workspace_project_files_m4.cjs index dbdb85b7..80e343ca 100644 --- a/tests/test_workspace_project_files_m4.cjs +++ b/tests/test_workspace_project_files_m4.cjs @@ -43,6 +43,7 @@ function mount(capabilityValue=capabilities){ context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); context.ProjectMeasurements=()=>({setContext(){},clear(){}}); + context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, From 00a02ee25cb30d254c059a9b372036f9f6f36696 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 26 Oct 2026 08:36:28 -0700 Subject: [PATCH 14/14] perf(project): read retained measurement sources by validated family Chart-source lookup copied the entire retained graph and decoded material again. Read the already validated measurement family through the existing canonical freshness guard; keep candidate and packet validation at their original owners. 122 distinct filesystem, native SQL, permission, import, receipt and client checks pass across two retained fixtures. Fixed-checkpoint lookup removes 19-20 CPU seconds of copying/decoding; native endpoint and whole-Open latency remain separate pending qualifications. No global cache or schema change. --- project_measurements.py | 5 +- tests/test_project_measurement_reads.py | 106 ++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 tests/test_project_measurement_reads.py diff --git a/project_measurements.py b/project_measurements.py index d1fed0de..e64a307e 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -138,7 +138,10 @@ def __init__(self, session): self.access, self.db = session.access, session.db def _documents(self): - return decode_records(self.session.snapshot['records']) + # Session adoption/publication already validates the complete candidate. + # Verify current canonical bytes, then own only the measurement family. + records = self.session.check_records(FAMILY)['records'] + return {item['id']: item['document'] for item in records} def import_packet(self, raw, source, expected_checkpoint, key): """Called only after SourceAdapter's explicit current rooted capture.""" diff --git a/tests/test_project_measurement_reads.py b/tests/test_project_measurement_reads.py new file mode 100644 index 00000000..1365d63a --- /dev/null +++ b/tests/test_project_measurement_reads.py @@ -0,0 +1,106 @@ +"""Read the adopted measurement family without cloning unrelated retained facts. + +Real canonical files and real session freshness guards; no SQL reconstruction +or packet import is claimed by this focused fixture. +""" +import copy +import json +import os +from pathlib import Path +import tempfile +import threading +import unittest +from unittest.mock import patch + +import project_measurements +import project_session +from project_projection import validate_project_records +from project_store import ProjectStore, ProjectStoreError +from test_project_receipt_validation import candidate, PROJECT + + +class MeasurementReadTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-measurement-read-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name)/'project' + self.store = ProjectStore.create(self.root, 'Measurement read fixture', 'new', 'a'*64) + self.addCleanup(self.store.close) + self.records = candidate() + self.records.append(dict(family='project_artifact', id='f'*64, + document=dict(format='project_artifact_v1', sha256='f'*64, bytes=7))) + validate_project_records(self.records, project_id=PROJECT, artifacts=None) + snapshot = self.store.commit(self.records, self.store.summary()['checkpoint'], 'seed', 'b'*64) + self.session = object.__new__(project_session.ProjectSession) + self.session.store, self.session.available = self.store, True + self.session._snapshot, self.session.lock = copy.deepcopy(snapshot), threading.RLock() + self.owner = object.__new__(project_measurements.ProjectMeasurements) + self.owner.session = self.session + self.expected = {r['id']: copy.deepcopy(r['document']) for r in self.records + if r['family'] == project_measurements.FAMILY} + + def test_documents_are_literal_independent_and_do_not_copy_unrelated_graph(self): + original = project_session.copy_validated_json + unrelated = [] + def observe(value): + if isinstance(value, dict) and 'records' in value: + unrelated.extend(r['id'] for r in value['records'] if r['id'] == 'f'*64) + return original(value) + with patch.object(project_session, 'copy_validated_json', observe): + result = self.owner._documents() + self.assertEqual(result, self.expected) + self.assertEqual(unrelated, [], 'Measurement read copied an unrelated retained envelope') + next(iter(result.values()))['source']['path'] = 'caller-edited.json' + self.assertEqual(self.owner._documents(), self.expected) + self.assertEqual(self.session._snapshot, self.store.snapshot()) + + def test_lookup_reads_every_canonical_file_without_redecoding_adopted_material(self): + with patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ + patch.object(project_measurements, 'material_records', + side_effect=AssertionError('Read-only lookup redecoded adopted material')): + self.assertEqual(self.owner._documents(), self.expected) + self.assertEqual(reads.call_count, len(self.records), 'All current files must still be read and hashed') + + def test_unavailable_projection_fails_before_any_store_read(self): + self.session.available = False + with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')): + with self.assertRaises(project_session.ProjectSessionError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'projection_unavailable') + + def test_other_family_publication_requires_reopen(self): + changed = copy.deepcopy(self.records) + changed[-1]['document']['bytes'] = 8 + self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c'*64) + with self.assertRaises(project_session.ProjectSessionError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'stale_project') + + def test_same_length_unselected_corruption_with_restored_mtime_is_rejected(self): + inventory = json.loads((self.root/'.tuldok/project.json').read_bytes())['inventory'] + entry = next(r for r in inventory if r['id'] == 'f'*64) + path = self.root/'.tuldok'/entry['path'] + before, info = path.read_bytes(), path.stat() + changed = before.replace(b'"bytes": 7', b'"bytes": 8') + self.assertNotEqual(changed, before) + self.assertEqual(len(changed), len(before)) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + with self.assertRaises(ProjectStoreError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'stale') + + def test_family_can_be_empty_and_returned_snapshot_contract_stays_complete(self): + records = [r for r in self.records if r['family'] not in (project_measurements.FAMILY,'project_operation')] + validate_project_records(records, project_id=PROJECT, artifacts=None) + self.session._snapshot = self.store.commit(records, self.store.summary()['checkpoint'], 'empty', 'd'*64) + self.assertEqual(self.owner._documents(), {}) + snapshot = self.session.snapshot + self.assertEqual(snapshot, self.store.snapshot()) + snapshot['records'].clear() + self.assertEqual(self.session.snapshot, self.store.snapshot()) + + +if __name__ == '__main__': + unittest.main()