From e9c107b4a6f8acc3ec6e8d416cfb0b589b8fea5f Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 21:58:32 -0700 Subject: [PATCH 01/33] Bound source closure scans during independent project forks --- project_artifacts.py | 120 ++++++++++++++++++++++++-------- project_fork.py | 11 ++- tests/test_project_artifacts.py | 99 ++++++++++++++++++++++++++ tests/test_project_fork.py | 53 ++++++++++++++ 4 files changed, 247 insertions(+), 36 deletions(-) diff --git a/project_artifacts.py b/project_artifacts.py index 743ba5ce..fe2a0ee0 100644 --- a/project_artifacts.py +++ b/project_artifacts.py @@ -247,12 +247,34 @@ def status(self, declaration): self._ensure_open() try: with self.store.root_directory() as root: - try: - with self._source(root, declaration) as source: - return _observe(source, declaration) - except _Missing: - return dict(state='missing', expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], - observed_sha256=None, observed_bytes=None) + return self._status_at(root, declaration) + except OSError as error: + _io(error) + + def _status_at(self, root, declaration): + try: + with self._source(root, declaration) as source: + return _observe(source, declaration) + except _Missing: + return dict(state='missing', expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], + observed_sha256=None, observed_bytes=None) + + def status_many(self, declarations): + """Hash each explicit artifact under one before/after-verified root. + + Unlike presence inspection, these observations verify present payload + hashes. An unchanged canonical closure is proved for the entire group; + every individual file retains its anchored source-location checks. + """ + if type(declarations) is not list: + _fail('Artifact status groups must be explicit finite lists.') + declarations = [validate_declaration(value) for value in declarations] + with self._mutex: + self._ensure_open() + try: + with self.store.root_directory(verify_after=True) as root: + result = [self._status_at(root, value) for value in declarations] + return result except OSError as error: _io(error) @@ -327,36 +349,74 @@ def _snapshot(self, declaration): snapshot = None try: with self.store.root_directory() as root: - local = staging = None - try: - with self._source(root, declaration) as source: - local = _directory(root, '.tuldok-local', create=True) - staging = _directory(local, 'artifact-snapshots', create=True) - temporary = '.snapshot-' + uuid.uuid4().hex + '.tmp' - output = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, - 0o600, dir_fd=staging) - try: - _verified(_observe(source, declaration, output=output)) - os.fchmod(output, 0o400) - os.fsync(output) - finally: - os.close(output) - snapshot = os.open(temporary, FILE_FLAGS, dir_fd=staging) - _info(snapshot, rooted=True) - os.unlink(temporary, dir_fd=staging) - except _Missing as error: - raise ProjectArtifactError('Required artifact bytes are missing.', 'unavailable', 404) from error - finally: - if staging is not None: - os.close(staging) - if local is not None: - os.close(local) + snapshot = self._snapshot_at(root, declaration) return snapshot except BaseException: if snapshot is not None: os.close(snapshot) raise + def _snapshot_at(self, root, declaration): + snapshot = None + try: + local = staging = None + try: + with self._source(root, declaration) as source: + local = _directory(root, '.tuldok-local', create=True) + staging = _directory(local, 'artifact-snapshots', create=True) + temporary = '.snapshot-' + uuid.uuid4().hex + '.tmp' + output = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, dir_fd=staging) + try: + _verified(_observe(source, declaration, output=output)) + os.fchmod(output, 0o400) + os.fsync(output) + finally: + os.close(output) + snapshot = os.open(temporary, FILE_FLAGS, dir_fd=staging) + _info(snapshot, rooted=True) + os.unlink(temporary, dir_fd=staging) + except _Missing as error: + raise ProjectArtifactError('Required artifact bytes are missing.', 'unavailable', 404) from error + finally: + if staging is not None: + os.close(staging) + if local is not None: + os.close(local) + return snapshot + except BaseException: + if snapshot is not None: + os.close(snapshot) + raise + + def copy_many_to(self, destination, declarations): + """Copy a finite inventory with one before/after-verified source root. + + Each payload uses its own verified unlinked snapshot and ordinary atomic + destination publication. Only one temporary payload is retained at once. + A later failure can leave earlier verified destination blobs inert; the + caller owns publication of the complete destination project. + """ + if not isinstance(destination, ProjectArtifacts) or destination is self: + _fail('Artifact copying requires a distinct destination owner.') + if type(declarations) is not list: + _fail('Artifact copy groups must be explicit finite lists.') + declarations = [validate_declaration(value) for value in declarations] + first, second = sorted((self, destination), key=id) + with first._mutex, second._mutex: + self._ensure_open() + destination._ensure_open() + try: + with self.store.root_directory(verify_after=True) as root: + for declaration in declarations: + snapshot = self._snapshot_at(root, declaration) + try: + destination.publish_fd(snapshot, declaration, maximum_bytes=declaration['bytes']) + finally: + os.close(snapshot) + except OSError as error: + _io(error) + @contextmanager def open_verified(self, declaration, *, maximum_bytes=None): """Borrow a verified unlinked read-only snapshot, not a mutable source fd.""" diff --git a/project_fork.py b/project_fork.py index ff75fec4..959dde8a 100644 --- a/project_fork.py +++ b/project_fork.py @@ -353,8 +353,10 @@ def publish_fork(source_store, source_artifacts, trusted_root_fd, relative, *, r _fail('Fork must retain the source payload inventory without regenerating bytes.') plan = _artifact_plan(candidate['artifacts'], declarations) copied, missing = [], [] - for identity in sorted(plan): - status = source_artifacts.status(declarations[identity])['state'] + identities = sorted(plan) + observations = source_artifacts.status_many([declarations[identity] for identity in identities]) + for identity, observation in zip(identities, observations): + status = observation['state'] if status == 'corrupt': _fail('Source fork payload is corrupt.', 'invalid', 400) if status == 'missing': @@ -374,10 +376,7 @@ def publish_fork(source_store, source_artifacts, trusted_root_fd, relative, *, r validate_candidate(copy.deepcopy(records), destination_id) _source_current(source_store, source) artifacts = ProjectArtifacts(staged) - for identity in copied: - value = declarations[identity] - with source_artifacts.open_verified(value, maximum_bytes=value['bytes']) as source_fd: - artifacts.publish_fd(source_fd, value, maximum_bytes=value['bytes']) + source_artifacts.copy_many_to(artifacts, [declarations[identity] for identity in copied]) # Ordinary authored files are explicitly adopted text. Only a new # task-owned fork stage materializes them; Open never calls this. from project_files import materialize_captured diff --git a/tests/test_project_artifacts.py b/tests/test_project_artifacts.py index ecd81586..2a68497d 100644 --- a/tests/test_project_artifacts.py +++ b/tests/test_project_artifacts.py @@ -57,6 +57,105 @@ def commit(self, values, key='payloads'): return self.store.commit(records, self.store.snapshot()['checkpoint'], key, request(records)) class ProjectArtifactTests(_ArtifactFixture, unittest.TestCase): + def test_copy_group_keeps_exact_independent_snapshots_with_two_source_closure_checks(self): + raws = [b'first exact output', b'second exact output'] + values = [declaration(raw) for raw in raws] + for raw, value in zip(raws, values): + self.artifacts.publish_bytes(raw, value) + self.commit(values) + target = self.path / 'copy' + with ProjectStore.create(target, 'Copy', 'copy', 'b' * 64) as store, ProjectArtifacts(store) as destination: + publish = destination.publish_fd + def mutate_source_after_snapshot(fd, value, **options): + # A mutable original source must not be the fd consumed by the + # destination. Its already-verified snapshot stays independent. + original = self.payload(value) + original.unlink() + original.write_bytes(b'changed after verified capture') + return publish(fd, value, **options) + with patch.object(destination, 'publish_fd', side_effect=mutate_source_after_snapshot): + with patch.object(self.store, 'snapshot', wraps=self.store.snapshot) as checks: + self.artifacts.copy_many_to(destination, values) + self.assertEqual(checks.call_count, 2) + for raw, value in zip(raws, values): + copied = target / 'artifacts/sha256' / value['sha256'] + self.assertEqual(copied.read_bytes(), raw) + self.assertEqual(copied.stat().st_nlink, 1) + self.assertNotEqual(copied.stat().st_ino, self.payload(value).stat().st_ino) + + def test_copy_group_rejects_source_canonical_corruption_before_success(self): + value = declaration(b'present') + self.artifacts.publish_bytes(b'present', value) + self.commit([value]) + manifest = json.loads((self.project / '.tuldok/project.json').read_bytes()) + record = self.project / '.tuldok' / manifest['inventory'][0]['path'] + with ProjectStore.create(self.path / 'copy', 'Copy', 'copy', 'b' * 64) as store, ProjectArtifacts(store) as destination: + publish = destination.publish_fd + def corrupt_after_copy(*args, **options): + result = publish(*args, **options) + record.write_bytes(b'{"corrupt":true}\n') + return result + with patch.object(destination, 'publish_fd', side_effect=corrupt_after_copy): + self.assert_code('stale', self.artifacts.copy_many_to, destination, [value]) + + def test_copy_group_prevalidates_inventory_and_preserves_missing_payload_failure(self): + value, missing = declaration(b'present'), declaration(b'missing') + self.artifacts.publish_bytes(b'present', value) + self.commit([value, missing]) + with ProjectStore.create(self.path / 'copy', 'Copy', 'copy', 'b' * 64) as store, ProjectArtifacts(store) as destination: + with patch.object(destination, 'publish_fd', wraps=destination.publish_fd) as publish: + self.assert_code('invalid', self.artifacts.copy_many_to, destination, [value, dict(missing, bytes=True)]) + publish.assert_not_called() + self.assert_code('unavailable', self.artifacts.copy_many_to, destination, [value, missing]) + self.assertEqual(destination.status(value)['state'], 'verified') + self.assertEqual(destination.status(missing)['state'], 'missing') + + def test_status_group_retains_exact_hash_observations_with_two_closure_checks(self): + values = [declaration(b'present'), declaration(b'missing'), declaration(b'correct')] + self.artifacts.publish_bytes(b'present', values[0]) + self.artifacts.publish_bytes(b'correct', values[2]) + self.commit(values) + self.payload(values[2]).unlink() + self.payload(values[2]).write_bytes(b'changed') + expected = [self.artifacts.status(value) for value in values] + with patch.object(self.store, 'snapshot', wraps=self.store.snapshot) as checks: + actual = self.artifacts.status_many(values) + self.assertEqual(actual, expected) + self.assertEqual([item['state'] for item in actual], ['verified', 'missing', 'corrupt']) + self.assertEqual(checks.call_count, 2) + self.assertEqual(actual[0]['observed_sha256'], values[0]['sha256']) + self.assertEqual(actual[2]['observed_sha256'], hashlib.sha256(b'changed').hexdigest()) + + def test_status_group_rejects_canonical_corruption_during_observation(self): + value = declaration(b'present') + self.artifacts.publish_bytes(b'present', value) + self.commit([value]) + manifest = json.loads((self.project / '.tuldok/project.json').read_bytes()) + record = self.project / '.tuldok' / manifest['inventory'][0]['path'] + original = self.artifacts._source + @contextmanager + def corrupt_after_observation(*args): + with original(*args) as source: + yield source + record.write_bytes(b'{"corrupt":true}\n') + with patch.object(self.artifacts, '_source', side_effect=corrupt_after_observation): + self.assert_code('stale', self.artifacts.status_many, [value]) + + def test_status_group_rejects_checkpoint_change_and_unsafe_aliases(self): + value = declaration(b'present') + self.artifacts.publish_bytes(b'present', value) + self.commit([value]) + original = self.artifacts._source + @contextmanager + def change_checkpoint(*args): + with original(*args) as source: + yield source + self.commit([], key='new-checkpoint') + with patch.object(self.artifacts, '_source', side_effect=change_checkpoint): + self.assert_code('stale', self.artifacts.status_many, [value]) + os.link(self.payload(value), self.path / 'outside-alias') + self.assert_code('invalid', self.artifacts.status_many, [value]) + def test_presence_batch_matches_individual_observations_with_two_closure_checks(self): values=[declaration(b'present'),declaration(b'missing'),declaration(b'other')] self.artifacts.publish_bytes(b'present',values[0]) diff --git a/tests/test_project_fork.py b/tests/test_project_fork.py index 946f8b75..6241c890 100644 --- a/tests/test_project_fork.py +++ b/tests/test_project_fork.py @@ -120,6 +120,59 @@ def test_exact_retry_reconciles_and_changed_body_or_unrelated_destination_never_ self.assert_code('conflict', self.fork, 'unrelated') self.assertEqual((other / '.tuldok/project.json').read_bytes(), unrelated) + def test_missing_runtime_inventory_does_not_repeat_full_source_closure_per_entry(self): + # Each snapshot validates the complete canonical file closure. Missing + # runtime declarations must not multiply that work by inventory length. + snapshot = self.source.snapshot + with patch.object(self.source, 'snapshot', wraps=snapshot) as scans: + self.fork('small-inventory') + baseline = scans.call_count + records = copy.deepcopy(self.original['records']) + runtime = set() + for index in range(32): + raw = ('unmaterialized runtime dependency %d' % index).encode() + sha = hashlib.sha256(raw).hexdigest() + runtime.add(sha) + records.append(dict(family='project_artifact', id=sha, + document=dict(format='project_artifact_v1', sha256=sha, bytes=len(raw)))) + self.original = self.source.commit(records, self.original['checkpoint'], + 'larger-runtime-inventory', body(records)) + def candidate(source, destination): + value = self.candidate(source, destination) + for edge in value['artifacts']: + if edge['declaration']['sha256'] in runtime: + edge['preservation'] = 'regenerable' + return value + with patch.object(self.source, 'snapshot', wraps=snapshot) as scans: + result = self.fork('large-inventory', build_candidate=candidate) + self.assertEqual(set(result['fork']['artifacts']['missing_regenerable']), runtime) + self.assertLessEqual(scans.call_count, baseline + 4, + 'Full source closure scans must stay bounded when missing runtime inventory grows') + + def test_present_payload_copy_does_not_repeat_full_source_closure_per_file(self): + snapshot = self.source.snapshot + with patch.object(self.source, 'snapshot', wraps=snapshot) as scans: + self.fork('small-payload-set') + baseline = scans.call_count + records = copy.deepcopy(self.original['records']) + payloads = {} + for index in range(8): + raw = ('retained unique output %d' % index).encode() + sha = hashlib.sha256(raw).hexdigest() + value = dict(format='project_artifact_v1', sha256=sha, bytes=len(raw)) + self.artifacts.publish_bytes(raw, value) + records.append(dict(family='project_artifact', id=sha, document=value)) + payloads[sha] = raw + self.original = self.source.commit(records, self.original['checkpoint'], + 'larger-payload-set', body(records)) + with patch.object(self.source, 'snapshot', wraps=snapshot) as scans: + result = self.fork('large-payload-set') + for sha, raw in payloads.items(): + self.assertIn(sha, result['fork']['artifacts']['copied']) + self.assertEqual((self.root / 'large-payload-set/artifacts/sha256' / sha).read_bytes(), raw) + self.assertLessEqual(scans.call_count, baseline + 4, + 'Full source closure scans must stay bounded when present payload count grows') + def test_quiescence_checkpoint_overlap_and_symlink_guards_precede_publication(self): before = self.canonical_bytes() self.assert_code('busy', self.fork, quiescent=False) From 25f997af0b07d6396ccf7d66e3910bd1dd43a624 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 22:52:26 -0700 Subject: [PATCH 02/33] Reuse verified immutable revisions without disposable rewrites --- project_store.py | 21 +++++++++++++------ tests/test_project_store.py | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/project_store.py b/project_store.py index 0531b8ea..60a12b81 100644 --- a/project_store.py +++ b/project_store.py @@ -717,14 +717,23 @@ def _record_file(self, family, record_id, digest, raw=None, sync=False): fds.append(parent) name = digest + '.json' if raw is not None: - temporary = '.record-' + uuid.uuid4().hex + '.tmp' - _write_file(parent, temporary, raw) try: - _rename_noreplace(parent, temporary, name) - except FileExistsError: - if _read(parent, name, MAX_RECORD_BYTES) != raw: + existing = _read(parent, name, MAX_RECORD_BYTES) + except FileNotFoundError: + temporary = '.record-' + uuid.uuid4().hex + '.tmp' + _write_file(parent, temporary, raw) + try: + _rename_noreplace(parent, temporary, name) + except FileExistsError: + if _read(parent, name, MAX_RECORD_BYTES) != raw: + _fail('Immutable project revision already has different bytes.') + os.unlink(temporary, dir_fd=parent) + else: + if existing != raw: _fail('Immutable project revision already has different bytes.') - os.unlink(temporary, dir_fd=parent) + # Existing revisions can come from a copied project or an + # interrupted earlier commit. Retain all ordinary sync checks; + # only the disposable write/rename/unlink cycle is avoided. sync = True if sync: _sync_file(parent, name) diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 0f65339d..d3d93570 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -86,6 +86,48 @@ def test_create_commit_save_cold_reopen_and_independent_snapshots(self): committed['records'][0]['document']['text'] = 'Unchecked returned mutation' self.assertEqual(self.store.snapshot()['records'][0]['document']['text'], 'Notes preserve \r\n authored values') + def test_appending_one_record_does_not_rewrite_saved_immutable_revisions(self): + values = [record(str(index), 'Retained authored note %d' % index) for index in range(8)] + self.commit(values, key='initial-inventory') + writer = store_module._write_file + with patch.object(store_module, '_write_file', wraps=writer) as writes: + committed = self.commit([*values, record('new')], key='append-one') + record_writes = [call for call in writes.call_args_list if call.args[1].startswith('.record-')] + self.assertEqual(len(record_writes), 1, + 'Appending one record must not write disposable copies of every saved immutable revision') + self.assertEqual(self.reopen(), committed) + + def test_reusing_an_unreferenced_revision_still_checks_bytes_and_syncs_it(self): + first, second = record('first'), record('second') + self.commit([first], key='retain-first') + self.commit([second], key='retain-second') + writer, sync = store_module._write_file, store_module._sync_file + with patch.object(store_module, '_write_file', wraps=writer) as writes: + with patch.object(store_module, '_sync_file', wraps=sync) as syncs: + committed = self.commit([first, second], key='reuse-first') + self.assertFalse(any(call.args[1].startswith('.record-') for call in writes.call_args_list)) + self.assertEqual(syncs.call_count, 2) + self.assertEqual(self.reopen(), committed) + + def test_reusing_a_corrupt_unreferenced_revision_never_publishes_it(self): + first = record('first') + self.commit([first], key='retain-first') + entry = self.manifest()['inventory'][0] + self.commit([], key='exclude-first') + before = self.store.snapshot() + path = self.root / '.tuldok' / entry['path'] + path.unlink() + path.write_bytes(b'{"corrupt":"unreferenced revision"}\n') + self.assert_code('invalid', self.commit, [first], key='reuse-corrupt') + self.assertEqual(self.reopen(), before) + + def test_existing_revision_sync_failure_keeps_the_previous_checkpoint(self): + self.commit([record()], key='initial') + before = self.store.snapshot() + with patch.object(store_module, '_sync_file', side_effect=OSError(errno.EIO, 'existing revision sync failed')): + self.assert_code('unavailable', self.commit, [record()], key='reuse-with-sync-error') + self.assertEqual(self.reopen(), before) + def test_fresh_copy_and_moved_folder_keep_identity_and_values(self): committed = self.commit() self.store.close() From fecd3fca2f383ddd11d88440ab259af3a33bceba Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 00:30:35 -0700 Subject: [PATCH 03/33] Adopt bounded groups of reviewed project files atomically --- agent_api.py | 15 +++++ project_dispatch.py | 4 +- project_files.py | 90 ++++++++++++++++++++++---- static/project-files.js | 15 +++-- tests/test_project_files.py | 83 ++++++++++++++++++++++++ tests/test_project_files_m4_clients.py | 18 +++++- tests/test_project_files_m4_http.py | 13 ++++ tuldok_client.py | 4 ++ 8 files changed, 219 insertions(+), 23 deletions(-) diff --git a/agent_api.py b/agent_api.py index 4bcd9ce6..d07cdfff 100644 --- a/agent_api.py +++ b/agent_api.py @@ -1327,6 +1327,9 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non output=obj(dict(_PROJECT_FILES_PAGE['properties'],id=SHA,items=array({'oneOf':[obj(_PROJECT_FILE_VERSION_META),obj(_PROJECT_FILE_DELETED_META)]},40))),milestone=4) _project_file_operation('file_adopt','POST','/files/adopt',{'kind':_PROJECT_FILE_KIND,'path':text(512), 'expected_revision':SAFE_NAT,'expected_sha256':SHA},output=_PROJECT_FILE_RESULT,mutation=True,milestone=4) +_project_file_operation('files_adopt','POST','/files/adopt-many',{'items':array(obj({'kind':_PROJECT_FILE_KIND, + 'path':text(512),'expected_revision':SAFE_NAT,'expected_sha256':SHA}),40,1)}, + output=obj({'items':array(_PROJECT_FILE_RESULT,40,1),'changed_count':integer(0,40)}),mutation=True,milestone=4) _project_file_operation('file_adopt_deletion','POST','/files/adopt-deletion',{'path':text(512), 'expected_revision':SAFE_POS,'expected_sha256':SHA},output=obj(dict(_PROJECT_FILE_RESULT['properties'], kind=const('note'),revision=integer(2,9007199254740991),deleted=const(True),sha256={'type':'null'},bytes=const(0),changed=const(True))), @@ -2671,6 +2674,18 @@ def call(self, name, arguments, *, observer=None, observation=None): raise AgentAPIError('Project read response belongs to a different checkpoint.', 'transport', 502) page = value['value'] action=op['operation_id'].removeprefix('project_files.') + if action=='files_adopt': + if (len(page['items'])!=len(arguments['items']) + or page['changed_count']!=sum(item['changed'] for item in page['items']) + or sum(item['bytes'] for item in page['items'])>8*1024*1024): + raise AgentAPIError('Batch adoption count or capture bound differs.','transport',502) + for item,reviewed in zip(page['items'],arguments['items']): + identity=hashlib.sha256((value['project_id']+'\0'+reviewed['kind']+'\0'+reviewed['path']).encode()).hexdigest() + if (item['id']!=identity or item['project_id']!=value['project_id'] + or item['kind']!=reviewed['kind'] or item['path']!=reviewed['path'] + or item['sha256']!=reviewed['expected_sha256'] + or item['revision']!=reviewed['expected_revision']+int(item['changed'])): + raise AgentAPIError('Batch adoption differs from its reviewed files.','transport',502) if action in ('file_get','file_adopt','file_adopt_deletion'): kind,path=page['kind'],page['path'] expected_id=hashlib.sha256((value['project_id']+'\0'+kind+'\0'+path).encode()).hexdigest() diff --git a/project_dispatch.py b/project_dispatch.py index bdbdc250..50bedbc4 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -38,7 +38,7 @@ _RUNTIME_ACTIONS=frozenset(('runtime_sources','runtime_inspect','runtime_capture','runtime_status','runtime_abandon')) _MEASUREMENT_ACTIONS=frozenset(('measurement_import','measurement_sources','measurement_metadata', 'measurement_subjects','measurement_query','measurement_inspect')) -_M4_ACTIONS=frozenset(('files_inspect','file_get','file_history','file_adopt','file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')) +_M4_ACTIONS=frozenset(('files_inspect','file_get','file_history','file_adopt','files_adopt','file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')) SUPPORTED_OPERATIONS = frozenset('project_files.' + action for action in _DATA_ACTIONS | _MATERIAL_ACTIONS | _HISTORY_ACTIONS | _OTHER_ACTIONS | _EXECUTION_ACTIONS | _M4_ACTIONS | _RUNTIME_ACTIONS | _MEASUREMENT_ACTIONS) @@ -365,6 +365,8 @@ def _m4(projection,action,a): session=projection.session if action=='file_adopt': return projection.files.adopt(_body(a),a['expected_checkpoint'],a['request_key']) + if action=='files_adopt': + return projection.files.adopt_many(_body(a),a['expected_checkpoint'],a['request_key']) if action=='file_adopt_deletion': return projection.files.adopt_deletion(_body(a),a['expected_checkpoint'],a['request_key']) with session.access.lock,session.lock,session.db.lock: diff --git a/project_files.py b/project_files.py index a1de61fc..08829a6a 100644 --- a/project_files.py +++ b/project_files.py @@ -18,6 +18,8 @@ DELETION_FORMAT='project_file_v2' MAX_FILE_BYTES=1024*1024 MAX_FILES=1000 +MAX_ADOPT_ITEMS=40 +MAX_ADOPT_BYTES=8*1024*1024 KINDS={'script':'scripts','note':'notes'} VERSION_FIELDS={'revision','sha256','bytes','text','adopted_at','actor_id'} RESULT_FIELDS={'id','project_id','kind','path','revision','sha256','bytes','adopted_at','changed'} @@ -127,6 +129,20 @@ def _result(envelope,changed): def validate_operation_result(operation,result,records=None): + if operation=='files.adopt_many': + closed(result,{'items','changed_count'},'batch adoption result') + if type(result['items']) is not list or not 1<=len(result['items'])<=MAX_ADOPT_ITEMS: + fail('Invalid batch adoption result size.') + seen=set();projects=set() + for item in result['items']: + validate_operation_result('files.adopt',item,records) + if item['id'] in seen:fail('Batch adoption result repeats a file.') + seen.add(item['id']);projects.add(item['project_id']) + if (len(projects)!=1 or type(result['changed_count']) is not int + or result['changed_count']!=sum(item['changed'] for item in result['items']) + or sum(item['bytes'] for item in result['items'])>MAX_ADOPT_BYTES): + fail('Batch adoption result count or project differs.') + return True if operation not in ('files.adopt','files.adopt_deletion'):return False deleted=operation=='files.adopt_deletion' closed(result,RESULT_FIELDS|{'deleted'} if deleted else RESULT_FIELDS,'adoption result') @@ -275,33 +291,81 @@ def require_clean(self): def materialize_captured(records,project_id,root_fd): return materialize_captured(records,project_id,root_fd) - def adopt(self,body,expected_checkpoint,key): + def _adoption_intent(self,body): closed(body,{'kind','path','expected_revision','expected_sha256'},'working-file adoption') parts=_path(body['kind'],body['path']);project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 0<=body['expected_revision']<=2**53-1:fail('Invalid expected adoption revision.') + return dict(body),parts,record_id + + def _capture_adoption(self,intent,prior,root): + body,parts,record_id=intent + revision=prior['document']['current']['revision'] if prior else 0 + if revision!=body['expected_revision']:fail('Working-file adoption revision changed.','conflict',409) + try:text,observed,size=_read(root,parts) + except _WorkingError as error:fail('Working file is '+error.state+'.','unavailable' if error.state=='missing' else 'invalid',404 if error.state=='missing' else 400) + if observed!=body['expected_sha256']:fail('Working file changed after review.','conflict',409) + if prior is not None and observed==prior['document']['current']['sha256']:return _result(prior,False),None,size + actor=self.access.require('edit','files.adopt',record_id) + value=dict(revision=revision+1,sha256=observed,bytes=size,text=text,adopted_at=timestamp(),actor_id=actor.actor_id) + history=copy.deepcopy(prior['document']['history']) if prior else [];history.append(value) + envelope=dict(family=FAMILY,id=record_id,document=dict(format=prior['document']['format'] if prior else FORMAT, + project_id=self.session.snapshot['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), + history=history,origin=copy.deepcopy(prior['document']['origin']) if prior else None)) + return _result(envelope,True),envelope,size + + def adopt(self,body,expected_checkpoint,key): + intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.snapshot['project_id'] pending=[] with self.access.lock: self.access.require('edit','files.adopt',record_id) def invoke(): snapshot=self.session.snapshot;owned=[r for r in decode_records(snapshot['records']) if r['document']['project_id']==project_id] prior=next((r for r in owned if r['id']==record_id),None) - revision=prior['document']['current']['revision'] if prior else 0 - if revision!=body['expected_revision']:fail('Working-file adoption revision changed.','conflict',409) if prior is None and len(owned)>=MAX_FILES:fail('Project already tracks 1,000 working files.','too_large',413) with self.store.root_directory() as root: - try:text,observed,size=_read(root,parts) - except _WorkingError as error:fail('Working file is '+error.state+'.','unavailable' if error.state=='missing' else 'invalid',404 if error.state=='missing' else 400) - if observed!=body['expected_sha256']:fail('Working file changed after review.','conflict',409) - if prior is not None and observed==prior['document']['current']['sha256']:return _result(prior,False) - actor=self.access.require('edit','files.adopt',record_id) - value=dict(revision=revision+1,sha256=observed,bytes=size,text=text,adopted_at=timestamp(),actor_id=actor.actor_id) - history=copy.deepcopy(prior['document']['history']) if prior else [];history.append(value) - envelope=dict(family=FAMILY,id=record_id,document=dict(format=prior['document']['format'] if prior else FORMAT,project_id=project_id,kind=body['kind'],path=body['path'],current=copy.deepcopy(value),history=history,origin=copy.deepcopy(prior['document']['origin']) if prior else None)) - decode_records([r for r in snapshot['records'] if not (r['family']==FAMILY and r['id']==record_id)]+[envelope]);pending.append(envelope) - return _result(envelope,True) + result,envelope,_=self._capture_adoption(intent,prior,root) + if envelope is not None: + decode_records([r for r in snapshot['records'] if not (r['family']==FAMILY and r['id']==record_id)]+[envelope]);pending.append(envelope) + return result return self.session.mutate('files.adopt',record_id,body,expected_checkpoint,key,invoke, lambda result,before:dict(records=copy.deepcopy(pending),result=copy.deepcopy(result))) + def adopt_many(self,body,expected_checkpoint,key): + """Capture one bounded, all-or-nothing group through ordinary adoption. + + Every file retains its permission/revision/hash and anchored read checks. + One verified root scope and one canonical publication cover the group. + No working file is rewritten, even when another input rejects the group. + """ + closed(body,{'items'},'batch working-file adoption') + if type(body['items']) is not list or not 1<=len(body['items'])<=MAX_ADOPT_ITEMS: + fail('Choose 1 through 40 working files for adoption.','too_large',413) + intents=[self._adoption_intent(item) for item in body['items']] + ids=[intent[2] for intent in intents] + if len(set(ids))!=len(ids):fail('Batch adoption repeats a working-file path.') + body=dict(items=[intent[0] for intent in intents]);project_id=self.session.snapshot['project_id'] + target=digest('\0'.join(ids).encode());pending=[] + with self.access.lock: + for identity in ids:self.access.require('edit','files.adopt',identity) + def invoke(): + snapshot=self.session.snapshot + owned={r['id']:r for r in decode_records(snapshot['records']) if r['document']['project_id']==project_id} + if len(owned)+sum(identity not in owned for identity in ids)>MAX_FILES: + fail('Project already tracks 1,000 working files.','too_large',413) + results=[];total=0 + with self.store.root_directory(verify_after=True) as root: + for intent in intents: + result,envelope,size=self._capture_adoption(intent,owned.get(intent[2]),root) + total+=size + if total>MAX_ADOPT_BYTES:fail('Batch working-file capture exceeds 8 MiB.','too_large',413) + if envelope is not None:pending.append(envelope) + results.append(result) + changed={r['id'] for r in pending} + decode_records([r for r in snapshot['records'] if not (r['family']==FAMILY and r['id'] in changed)]+pending) + return dict(items=results,changed_count=sum(item['changed'] for item in results)) + return self.session.mutate('files.adopt_many',target,body,expected_checkpoint,key,invoke, + lambda result,before:dict(records=copy.deepcopy(pending),result=copy.deepcopy(result))) + def adopt_deletion(self,body,expected_checkpoint,key): closed(body,{'path','expected_revision','expected_sha256'},'note deletion adoption') parts=_path('note',body['path']);sha(body['expected_sha256']) diff --git a/static/project-files.js b/static/project-files.js index 708658ed..ee86c1cd 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -66,6 +66,7 @@ "project_files.file_adopt_deletion": "7b6d661856a614d1b83d8011d1ffc7f9e2619853495faf9250888739f880685c", "project_files.file_get": "a22c27e2fd890ffcb479e0aa8ea40a0cb32d06d1e5750b970b40f27915fe4d24", "project_files.file_history": "c83477f7b0f8616c96ad79cebaf0eb3887fe289567f5d5ceb1d1502bebed7423", + "project_files.files_adopt": "62215260bf40bfa5db2521f908413bb6dfe47656fb8ea30b38039d637e5daf51", "project_files.files_inspect": "0c2ff3ed93109ea19b5c92ffa24ac8f67e4bcfa3fe386921e984b5dfa3bfe7eb", "project_files.fork_info": "958bc1aa269b5459c7e2d9c6bcb6725c06f7055502d73c49a95330cc4065aea4", "project_files.get": "6b0f4e16a00aa119b86cbb5493be0bac945ff08a5e63773a8f2c2f9296658d5b", @@ -82,6 +83,12 @@ "project_files.material_intent_capabilities": "86e90e2d0f094beb1c277a680da9a35e28d78c827390ac1c4e51fc3003c1ce32", "project_files.material_intents": "6431fd2780d706c77d2c4dd82f15794017368fd3bade28ac62bbdff5c8b0c547", "project_files.materials": "92a6a6a1750fed89aef9574ab6e82be2c6c6351bfa89fbfcb1f3866d4ea3cb13", + "project_files.measurement_import": "c8e500396138fc60a31fc916c441bb95454f410231a6508609a7ebba5cd31f18", + "project_files.measurement_inspect": "fcbe9846ca49c3da4713b54a3cb4a8237189d8eea5799f275d33992ef268914f", + "project_files.measurement_metadata": "5a92ed5259f8876a3c5b8fd1886a9e0f44f0aea8e53d7f63902ac765ddac0de7", + "project_files.measurement_query": "65b1456a6870dd6d4b7499669777a2c66157142ccc27cac1864a995664220e1c", + "project_files.measurement_sources": "7805e9c5a6b2e2ab566ca134c18697116c89e08a1991b5192b0b5c152a34041d", + "project_files.measurement_subjects": "37bd3f717c7158e34b9c221bfa1dbce79490b9d55248803d13dab2ea17b35ea5", "project_files.model_create": "34cf790fb2a3355ccab7c3a6c7174ced8dae861812d9479582fd8d5a9212745c", "project_files.model_get": "95cc198ab6f084b596324f7251803ad4670a9f8e786e1a9639f1b9ed041798c3", "project_files.model_material": "46afde2b06e0e5b08565f12ec86d2d7ec43ef6b63d97c86c264fc9c2f451f20b", @@ -142,13 +149,7 @@ "project_files.study_create": "8d2aac7c073a6a7b3ebb41227b5b466cf03606625404f93a59b325038a4ad7a1", "project_files.study_save": "5bf3fb814588fd7d9c35836e9d964a64481b3cc4169caf76cad6ce776ff0fe15", "project_files.trial": "79afa4bed6d0907f3d4116cc04cb784e94e5a24ab89725ec24f5d7bce70e869f", - "project_files.trial_prepare": "e1f6973536fd4c356c4a92fc674d6766f7df0d8a113b8ee71bb05d0adc40b04f", - "project_files.measurement_import": "c8e500396138fc60a31fc916c441bb95454f410231a6508609a7ebba5cd31f18", - "project_files.measurement_sources": "7805e9c5a6b2e2ab566ca134c18697116c89e08a1991b5192b0b5c152a34041d", - "project_files.measurement_metadata": "5a92ed5259f8876a3c5b8fd1886a9e0f44f0aea8e53d7f63902ac765ddac0de7", - "project_files.measurement_subjects": "37bd3f717c7158e34b9c221bfa1dbce79490b9d55248803d13dab2ea17b35ea5", - "project_files.measurement_query": "65b1456a6870dd6d4b7499669777a2c66157142ccc27cac1864a995664220e1c", - "project_files.measurement_inspect": "fcbe9846ca49c3da4713b54a3cb4a8237189d8eea5799f275d33992ef268914f" + "project_files.trial_prepare": "e1f6973536fd4c356c4a92fc674d6766f7df0d8a113b8ee71bb05d0adc40b04f" } /* PROJECT_FILES_CONTRACTS_END */; // Generated fixed operation routes from agent_api; remote schemas are never executed. diff --git a/tests/test_project_files.py b/tests/test_project_files.py index 3fbdcfab..538501d0 100644 --- a/tests/test_project_files.py +++ b/tests/test_project_files.py @@ -7,6 +7,7 @@ from unittest.mock import patch import test_project_data as data_fixture +import project_files as files_module from project_files import (ProjectFiles, decode_records, validate_operation_result, file_id, fork_records, materialize_captured, MAX_FILE_BYTES, MAX_FILES) from project_session import ProjectSession, operation_records @@ -39,6 +40,88 @@ def adopt(self,path,text,revision=0,kind='script',key=None): return self.files.adopt(dict(kind=kind,path=path,expected_revision=revision, expected_sha256=hashlib.sha256(raw).hexdigest()),self.session.snapshot['checkpoint'],key or 'files-'+str(self.ordinal)) + def intent(self,path,text,revision=0,kind='note'): + raw=text.encode('utf-8');target=self.root/path;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(raw) + return dict(kind=kind,path=path,expected_revision=revision,expected_sha256=hashlib.sha256(raw).hexdigest()) + + def test_batch_captures_exact_files_in_one_canonical_mutation(self): + items=[self.intent('notes/a.md','Exact Ω\r\n'),self.intent('notes/empty.md','')] + before=self.session.snapshot['checkpoint'] + with patch.object(self.session,'mutate',wraps=self.session.mutate) as mutations: + result=self.files.adopt_many(dict(items=items),before,'batch') + self.assertEqual(mutations.call_count,1);self.assertEqual(result['changed_count'],2) + self.assertEqual([self.files.read(r['id'])['version']['text'] for r in result['items']],['Exact Ω\r\n','']) + operations=list(operation_records(self.session.snapshot['records']).values()) + self.assertEqual(sum(o['operation']=='files.adopt_many' for o in operations),1) + self.assertTrue(validate_operation_result('files.adopt_many',result,self.session.snapshot['records'])) + + def test_batch_changed_missing_or_repeated_file_never_partially_publishes(self): + first=self.intent('notes/a.md','First');second=self.intent('notes/b.md','Second') + before=copy.deepcopy(self.session.snapshot) + for label,items in [('hash',[first,dict(second,expected_sha256='0'*64)]), + ('duplicate',[first,first]),('revision',[first,dict(second,expected_revision=1)])]: + with self.subTest(label=label),self.assertRaises(WorkbenchError): + self.files.adopt_many(dict(items=items),before['checkpoint'],'reject-'+label) + self.assertEqual(self.session.snapshot,before) + (self.root/'notes/b.md').unlink() + with self.assertRaises(WorkbenchError):self.files.adopt_many(dict(items=[first,second]),before['checkpoint'],'missing') + self.assertEqual(self.session.snapshot,before);self.assertEqual(decode_records(self.session.snapshot['records']),[]) + + def test_batch_permission_precedes_file_reads_and_replay_preserves_exact_history(self): + items=[self.intent('notes/a.md','Original'),self.intent('notes/b.md','Other')] + token=self.fixture.access.issue(dict(label='Batch inspect only',scopes=['inspect']))['token'] + with self.fixture.access.bind(token),patch.object(files_module,'_read',side_effect=AssertionError('Unauthorized read')): + with self.assertRaises(AccessError):self.files.adopt_many(dict(items=items),self.session.snapshot['checkpoint'],'deny') + before=self.session.snapshot['checkpoint'];result=self.files.adopt_many(dict(items=items),before,'original-batch') + saved=copy.deepcopy(self.session.snapshot);(self.root/'notes/a.md').write_text('Unadopted edit') + with patch.object(files_module,'_read',side_effect=AssertionError('Replay must not recapture')): + self.assertEqual(self.files.adopt_many(dict(items=items),'0'*64,'original-batch'),result) + self.assertEqual(self.session.snapshot,saved) + self.intent('notes/a.md','Original');noop=[dict(item,expected_revision=1) for item in items] + value=self.files.adopt_many(dict(items=noop),self.session.snapshot['checkpoint'],'noop-batch') + self.assertEqual(value['changed_count'],0);self.assertTrue(all(len(self.files.history(r['id']))==1 for r in value['items'])) + + def test_batch_item_and_total_byte_bounds_keep_checkpoint(self): + before=copy.deepcopy(self.session.snapshot) + with self.assertRaises(WorkbenchError):self.files.adopt_many(dict(items=[]),before['checkpoint'],'empty') + items=[dict(kind='note',path=f'notes/{i}.md',expected_revision=0,expected_sha256='0'*64) for i in range(41)] + with self.assertRaises(WorkbenchError):self.files.adopt_many(dict(items=items),before['checkpoint'],'too-many') + large=[self.intent(f'notes/large-{i}.md','x'*MAX_FILE_BYTES) for i in range(9)] + with self.assertRaises(WorkbenchError):self.files.adopt_many(dict(items=large),before['checkpoint'],'aggregate-too-large') + self.assertEqual(self.session.snapshot,before) + + def test_batch_unsafe_last_file_and_project_capacity_never_publish(self): + first=self.intent('notes/a.md','First');last=self.intent('notes/b.md','Second') + before=copy.deepcopy(self.session.snapshot);path=self.root/'notes/b.md' + for label in ('symlink','utf8'): + path.unlink() + if label=='symlink':path.symlink_to(self.root/'notes/a.md') + else:path.write_bytes(b'\xff') + with self.subTest(label=label),self.assertRaises(WorkbenchError): + self.files.adopt_many(dict(items=[first,last]),before['checkpoint'],'unsafe-'+label) + self.assertEqual(self.session.snapshot,before) + self.intent('notes/b.md','Second') + with patch.object(files_module,'MAX_FILES',1),self.assertRaises(WorkbenchError): + self.files.adopt_many(dict(items=[first,last]),before['checkpoint'],'capacity') + self.assertEqual(self.session.snapshot,before) + + def test_batch_revision_history_and_forged_receipt_are_bound_to_retained_bytes(self): + item=self.intent('notes/a.md','First');body=dict(items=[item]) + first=self.files.adopt_many(body,self.session.snapshot['checkpoint'],'first-batch') + second=self.intent('notes/a.md','Second',1) + result=self.files.adopt_many(dict(items=[second]),self.session.snapshot['checkpoint'],'second-batch') + self.assertEqual(self.files.read(result['items'][0]['id'])['version']['revision'],2) + records=self.session.snapshot['records'] + self.assertTrue(validate_operation_result('files.adopt_many',first,records)) + for label in ('hash','count','duplicate','project'): + bad=copy.deepcopy(result) + if label=='hash':bad['items'][0]['sha256']='0'*64 + elif label=='count':bad['changed_count']=0 + elif label=='duplicate':bad['items'].append(copy.deepcopy(bad['items'][0]));bad['changed_count']=2 + else:bad['items'][0]['project_id']='f'*32 + with self.subTest(label=label),self.assertRaises(WorkbenchError): + validate_operation_result('files.adopt_many',bad,records) + def test_adoption_exact_utf8_history_changed_missing_and_cold_no_restore(self): source='#!/usr/bin/env python3\r\n# authored cafe\u0301 😀\r\n' first=self.adopt('scripts/authored.py',source,key='first') diff --git a/tests/test_project_files_m4_clients.py b/tests/test_project_files_m4_clients.py index adf2cc56..5136e6af 100644 --- a/tests/test_project_files_m4_clients.py +++ b/tests/test_project_files_m4_clients.py @@ -58,6 +58,20 @@ def ready(reply, *, owner=False): class M4Clients(unittest.TestCase): + def test_batch_reply_must_match_every_reviewed_file_and_changed_count(self): + text='Exact note\n';raw=text.encode();path='notes/first.md' + item=dict(id=identity('note',path),project_id=PROJECT,kind='note',path=path, + revision=1,sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw),adopted_at=STAMP,changed=True) + arguments=dict(WRITE,items=[dict(kind='note',path=path,expected_revision=0,expected_sha256=item['sha256'])]) + reply=dict(status=200,result=bound(dict(items=[item],changed_count=1),NEXT)) + self.assertEqual(ProjectFilesClient(ready(reply)).files_adopt(**arguments),reply['result']) + for field,value in [('project_id','f'*32),('sha256','a'*64),('path','notes/other.md'),('revision',2)]: + bad=copy.deepcopy(reply);bad['result']['value']['items'][0][field]=value + with self.subTest(field=field),self.assertRaises(api.AgentAPIError): + ProjectFilesClient(ready(bad)).files_adopt(**arguments) + bad=copy.deepcopy(reply);bad['result']['value']['changed_count']=0 + with self.assertRaises(api.AgentAPIError):ProjectFilesClient(ready(bad)).files_adopt(**arguments) + def test_capability_profiles_fit_unchanged_budget_and_negotiate_only_intersections(self): native = api.capabilities(CATALOG) project = api.capabilities(CATALOG, profile='project') @@ -70,8 +84,8 @@ def test_capability_profiles_fit_unchanged_budget_and_negotiate_only_intersectio with self.assertRaises(api.AgentAPIError): api.capabilities(CATALOG, profile='arbitrary') self.assertEqual(api.REGISTRY_DIGEST, api._LEGACY_DIGEST) - def test_ten_operations_have_fixed_consumers_and_closed_inputs(self): - self.assertEqual(len(api.PROJECT_FILES_M4), 10) + def test_eleven_operations_have_fixed_consumers_and_closed_inputs(self): + self.assertEqual(len(api.PROJECT_FILES_M4), 11) self.assertEqual(project_dispatch.SUPPORTED_OPERATIONS & api.PROJECT_FILES_M4, api.PROJECT_FILES_M4 - {'project_files.save_as'}) client = ProjectFilesClient(Mock()) diff --git a/tests/test_project_files_m4_http.py b/tests/test_project_files_m4_http.py index a520c416..7280a2b4 100644 --- a/tests/test_project_files_m4_http.py +++ b/tests/test_project_files_m4_http.py @@ -39,6 +39,19 @@ def fork(self,**fields): self.assertEqual(status,200,value) return arguments,value + def test_batch_adoption_uses_real_http_and_survives_independent_fork(self): + self.create();root=self.external/'alpha';(root/'notes').mkdir();items=[] + for name,raw in [('first.md',b'Exact first note\r\n'),('second.md','Exact Ω note\n'.encode())]: + (root/'notes'/name).write_bytes(raw) + items.append(dict(kind='note',path='notes/'+name,expected_revision=0,expected_sha256=hashlib.sha256(raw).hexdigest())) + result=self.call('files_adopt',items=items) + self.assertEqual(result['changed_count'],2);self.assertEqual(len(self.call('files_inspect')['items']),2) + source=copy.deepcopy(self.workspace._projection.session.snapshot) + self.fork() + current=self.workspace._projection.session.snapshot;indexed={(r['family'],r['id']):r for r in current['records']} + for record in source['records']:self.assertEqual(indexed[(record['family'],record['id'])],record) + for item in items:self.assertEqual((self.external/'fork'/item['path']).read_bytes(),(root/item['path']).read_bytes()) + def test_nonempty_independent_fork_exact_science_new_editable_ids_and_same_request_key(self): self.create() row=self.call('data_import_text',text='Unique observation café 山',name='Original', diff --git a/tuldok_client.py b/tuldok_client.py index 76d09cc5..ce838bcb 100644 --- a/tuldok_client.py +++ b/tuldok_client.py @@ -714,6 +714,10 @@ def file_history(self, **arguments): def file_adopt(self, **arguments): return self._call('file_adopt',arguments) + def files_adopt(self, **arguments): + """Atomically adopt a bounded group of reviewed working files.""" + return self._call('files_adopt',arguments) + def file_adopt_deletion(self, **arguments): """Retain an explicitly reviewed missing note as a deletion revision.""" return self._call('file_adopt_deletion',arguments) From b21904f2d5fccb0f151a61223618e77f2f2587e7 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 00:51:49 -0700 Subject: [PATCH 04/33] Return structured busy errors from authentication boundaries --- app.py | 10 ++++++++++ tests/test_access_http.py | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/app.py b/app.py index b7a59bb2..87c6c9a7 100644 --- a/app.py +++ b/app.py @@ -486,9 +486,15 @@ def _access_body(self): except (ValueError, workbench.WorkbenchError): raise access.AccessError('Access requests require a bounded JSON object.', 'invalid', 400) from None + def _catalog_busy(self): + return self.reply({'error': 'Tuldok is busy completing another operation. Try again after it finishes.', + 'code': 'storage'}, 503) + def _access_error(self, error): try: dataset.access.denial(error, 'http.request', urlsplit(self.path).path[:512], anonymous=True) + except CatalogBusy: + return self._catalog_busy() except CommitOutcomeUnknown: return self.reply({'error': 'Commit outcome is unknown; inspect state before retrying.', 'code': 'commit_unknown'}, 503) except psycopg.Error: @@ -583,6 +589,8 @@ def _handle_GET(self): return self.reply({'error': str(error), 'code': error.code}, error.status) except access.AccessError as error: return self._access_error(error) + except CatalogBusy: + return self._catalog_busy() except CommitOutcomeUnknown: return self.reply({'error': 'Commit outcome is unknown; inspect state before retrying.', 'code': 'commit_unknown'}, 503) except psycopg.Error: @@ -633,6 +641,8 @@ def _handle_POST(self): return self.reply({'error': str(error), 'code': error.code}, error.status) except access.AccessError as error: return self._access_error(error) + except CatalogBusy: + return self._catalog_busy() except CommitOutcomeUnknown: return self.reply({'error': 'Commit outcome is unknown; inspect state before retrying.', 'code': 'commit_unknown'}, 503) except psycopg.Error: diff --git a/tests/test_access_http.py b/tests/test_access_http.py index cb9e5366..9f73fe3a 100644 --- a/tests/test_access_http.py +++ b/tests/test_access_http.py @@ -4,6 +4,7 @@ import threading import unittest import uuid +from unittest.mock import patch from http.server import ThreadingHTTPServer from urllib.error import HTTPError from urllib.request import Request, urlopen @@ -67,6 +68,31 @@ def test_denial_audit_storage_outage_still_returns_structured_error(self): status, body, _ = self.request('/api/access/session', headers={'Authorization': 'Bearer'}) self.assertEqual((status, body['code']), (503, 'storage')) + def test_busy_catalog_authentication_returns_json_and_keeps_login_code_usable(self): + code = self.dataset.access._local_owner_token() + held, release = threading.Event(), threading.Event() + def hold_catalog(): + with self.dataset.access.lock: + held.set() + release.wait(5) + holder = threading.Thread(target=hold_catalog) + holder.start() + try: + self.assertTrue(held.wait(2)) + with patch('metadata.CATALOG_LOCAL_TIMEOUT', .05): + for body, headers in (({'token':code}, {'Origin':self.root}), (None, self.owner.headers), + (None, {'Authorization':'Bearer'}), + ({'token':code}, {'Origin':'http://elsewhere.invalid'})): + status, value, _ = self.request('/api/access/session', body, headers) + self.assertEqual((status, value['code']), (503, 'storage')) + self.assertIn('busy completing another operation', value['error']) + finally: + release.set() + holder.join(5) + self.assertFalse(holder.is_alive()) + status, identity, _ = self.request('/api/access/session', {'token':code}, {'Origin':self.root}) + self.assertEqual((status, identity['kind']), (200, 'owner')) + def test_agent_review_and_execution_denied_before_mutation(self): with self.owner.bound(self.dataset): agent = self.dataset.access.issue({'label': 'Editor', 'scopes': ['inspect', 'edit', 'execute']}) From c5992d7be3dd7186c9f6d525367f8881cc16671a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 02:16:49 -0700 Subject: [PATCH 05/33] Compare compatible native measurement packets with complete subject populations --- agent_api.py | 63 ++ project_dispatch.py | 3 +- project_measurements.py | 39 +- retained_measurements.py | 113 ++++ static/project-files.js | 581 +++++++++--------- static/project-measurements.js | 159 +++-- tests/test_measurement_comparison.py | 110 ++++ ...est_project_measurement_comparison_http.py | 58 ++ tests/test_project_measurements_clients.py | 19 +- tuldok_client.py | 4 + 10 files changed, 810 insertions(+), 339 deletions(-) create mode 100644 tests/test_measurement_comparison.py create mode 100644 tests/test_project_measurement_comparison_http.py diff --git a/agent_api.py b/agent_api.py index d07cdfff..64e6e0bf 100644 --- a/agent_api.py +++ b/agent_api.py @@ -952,6 +952,23 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non required=[*list(_MEASUREMENT_SELECTION),'metric_id'],output=_MEASUREMENT_BOUND(_MEASUREMENT_QUERY)) _project_file_operation('measurement_inspect','GET','/measurements/{packet_sha256}/observations/{observation_id}', dict(_MEASUREMENT_SELECTION,observation_id=_MEASUREMENT_ID),output=_MEASUREMENT_BOUND(_MEASUREMENT_INSPECTION)) + +_COMPARISON_QUERY = obj(dict( + {key:_MEASUREMENT_QUERY['properties'][key] for key in ('metric','definition','filters','subject_ids','total','offset','limit','sampling')}, + subject_ids=array(_MEASUREMENT_ID,256,1), + summary=array(obj(dict(_MEASUREMENT_SUMMARY['properties'],source=obj(_MEASUREMENT_SELECTION))),256), + rows=array(obj(dict(_MEASUREMENT_OBSERVATION['properties'],source=obj(_MEASUREMENT_SELECTION))),100))) +_COMPARISON_VALUE = obj({'format':const('retained_measurement_comparison_v1'), + 'scope':_MEASUREMENT_SCOPE,'scope_definition':{'type':'object'}, + 'dimensions':_MEASUREMENT_HEADER['properties']['dimensions'],'metrics':array(_MEASUREMENT_METRIC,32,1), + 'definition_source':obj(_MEASUREMENT_SELECTION),'case_population_sha256':SHA, + 'case_count':integer(1,4096),'observation_count':integer(1,100000), + 'subjects':array(obj(dict(_MEASUREMENT_SUBJECT['properties'],source=obj(_MEASUREMENT_SELECTION))),256,1), + 'query':{'oneOf':[_COMPARISON_QUERY,_NULL]}}) +_project_file_operation('measurement_compare','POST','/measurements/compare', + dict(sources=array(obj(_MEASUREMENT_SELECTION),16,1),metric_id=_MEASUREMENT_ID, + subject_ids=array(_MEASUREMENT_ID,256,1),filters=_MEASUREMENT_DIMENSIONS,offset=SAFE_NAT,limit=integer(1,100)), + required=['sources'],output=obj({'sources':array(_MEASUREMENT_SOURCE,16,1),'value':_COMPARISON_VALUE})) _project_file_operation('data_annotate', 'POST', '/data/{record_id}', { 'record_id': ID, 'revision': SAFE_POS, 'source_revision': SAFE_POS, 'task': {'enum': list(ANNOTATIONS)}, 'annotation': JSON_OBJECT, @@ -3040,6 +3057,8 @@ def _validate_measurement_selection(action, page, arguments, project_id): """Correlate fixed scalar replies; scientific validity remains upstream.""" def refuse(): raise AgentAPIError('Retained measurements differ from the selected source or continuation.', 'transport', 502) + if action=='measurement_compare': + return _validate_comparison_selection(page, arguments) if action=='measurement_sources': offset,limit=arguments.get('offset',0),arguments.get('limit',40) keys=[(d['origin_catalog_id'],d['id']) for d in page['items']] @@ -3072,6 +3091,50 @@ def refuse(): or any(r['metric_id']!=arguments['metric_id'] or r['subject_id'] not in subjects for r in result['rows'])):refuse() +def _validate_comparison_selection(page, arguments): + """Bind aggregate replies to reviewed packets, subjects and exact page intent.""" + def refuse(): + raise AgentAPIError('Measurement comparison differs from its selected packets or continuation.', 'transport', 502) + def key(locator): + return locator['origin_catalog_id'], locator['packet_sha256'] + selected = arguments['sources'] + keys = [key(item) for item in selected] + documents, value = page['sources'], page['value'] + if (len(keys)!=len(set(keys)) or len(documents)!=len(keys) + or [(doc['origin_catalog_id'],doc['id']) for doc in documents]!=keys + or any(doc['blob']['sha256']!=doc['id'] or doc['source']['measured_sha256']!=doc['id'] for doc in documents) + or sum(doc['blob']['bytes'] for doc in documents)>128*1024*1024 + or value['definition_source']!=selected[0]):refuse() + subjects = value['subjects'] + identities = [subject['id'] for subject in subjects] + if (len(identities)!=len(set(identities)) or any(key(subject['source']) not in keys for subject in subjects) + or value['observation_count']!=len(subjects)*value['case_count']*len(value['metrics'])):refuse() + owners = {subject['id']:subject['source'] for subject in subjects} + query = value['query'] + if 'metric_id' not in arguments: + if query is not None:refuse() + return + if query is None:refuse() + wanted = arguments.get('subject_ids', identities) + offset,limit = arguments.get('offset',0),arguments.get('limit',50) + metrics = [metric for metric in value['metrics'] if metric['id']==arguments['metric_id']] + if (len(metrics)!=1 or query['metric']!=metrics[0] or query['subject_ids']!=wanted + or len(wanted)!=len(set(wanted)) or any(identity not in owners for identity in wanted) + or query['filters']!=arguments.get('filters',{}) + or query['offset']!=offset or query['limit']!=limit + or len(query['rows'])!=min(limit,max(0,query['total']-offset)) + or [summary['subject_id'] for summary in query['summary']]!=wanted + or sum(summary['count'] for summary in query['summary'])!=query['total']):refuse() + for summary in query['summary']: + if (summary['source']!=owners[summary['subject_id']] + or sum(summary['states'].values())!=summary['count'] + or (summary['maximum'] is None)!=(summary['states']['measured']==0)):refuse() + for row in query['rows']: + if (row['subject_id'] not in wanted or row['metric_id']!=arguments['metric_id'] + or row['source']!=owners[row['subject_id']] + or (row['value'] is None)!=(row['state']!='measured')):refuse() + + class PublicationOwnerClient(AgentClient): """Explicit owner publication actions only; ordinary AgentClient stays agent-only.""" OPERATIONS = frozenset(('runs.publication_status', 'runs.recover_publication', 'runs.cancel_publication', diff --git a/project_dispatch.py b/project_dispatch.py index 50bedbc4..6c9420a2 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -37,7 +37,7 @@ 'execution_study_prepare_package','execution_study_prepare_verifier','execution_study_abandon_preparation','execution_study_reconcile_preparation')) _RUNTIME_ACTIONS=frozenset(('runtime_sources','runtime_inspect','runtime_capture','runtime_status','runtime_abandon')) _MEASUREMENT_ACTIONS=frozenset(('measurement_import','measurement_sources','measurement_metadata', - 'measurement_subjects','measurement_query','measurement_inspect')) + 'measurement_subjects','measurement_query','measurement_inspect','measurement_compare')) _M4_ACTIONS=frozenset(('files_inspect','file_get','file_history','file_adopt','files_adopt','file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')) SUPPORTED_OPERATIONS = frozenset('project_files.' + action for action in _DATA_ACTIONS | _MATERIAL_ACTIONS | _HISTORY_ACTIONS | _OTHER_ACTIONS | _EXECUTION_ACTIONS | _M4_ACTIONS | _RUNTIME_ACTIONS | _MEASUREMENT_ACTIONS) @@ -440,6 +440,7 @@ def _measurements(projection,action,a): if action=='measurement_import': return projection.source_adapter.import_measurements(_body(a),a['expected_checkpoint'],a['request_key']) if action=='measurement_sources':return owner.sources(offset=a.get('offset',0),limit=a.get('limit',40)) + if action=='measurement_compare':return owner.compare(a['sources'],**_body(a,'sources')) origin,packet=a['origin_catalog_id'],a['packet_sha256'] if action=='measurement_metadata':return owner.metadata(origin,packet) if action=='measurement_subjects':return owner.subjects(origin,packet,offset=a.get('offset',0),limit=a.get('limit',40)) diff --git a/project_measurements.py b/project_measurements.py index b97bca56..2800b50b 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -10,7 +10,8 @@ from project_artifacts import validate_declaration from project_material import decode_records as material_records from projects import ID, NAME, SHA, relative_path -from retained_measurements import MeasurementTable, MAX_BYTES, MAX_RESPONSE_BYTES, encode, text +from retained_measurements import (MeasurementTable, MeasurementComparison, MAX_BYTES, MAX_RESPONSE_BYTES, + MAX_COMPARISON_SOURCES, MAX_COMPARISON_BYTES, encode, text) from workbench import WorkbenchError, timestamp @@ -198,3 +199,39 @@ def query(self, origin_catalog_id, packet_sha256, metric_id, **selection): def inspect(self, origin_catalog_id, packet_sha256, observation_id): return self._read(origin_catalog_id, packet_sha256, 'measurements.inspect', lambda table: table.inspect(observation_id)) + + def compare(self, sources, metric_id=None, **selection): + """Read one checkpoint's bounded group with every original source intact.""" + if type(sources) is not list or not 1 <= len(sources) <= MAX_COMPARISON_SOURCES: + raise WorkbenchError('Choose 1 through 16 native measurement sources.', 'too_large', 413) + targets = [] + for source in sources: + _closed(source, ('origin_catalog_id','packet_sha256')) + _identity(source['origin_catalog_id'], ID) + _identity(source['packet_sha256'], SHA) + targets.append(source['origin_catalog_id'] + '.' + source['packet_sha256']) + if len(set(targets)) != len(targets): + raise WorkbenchError('Native comparison repeats a measurement source.') + with self.access.lock, self.session.lock, self.db.lock: + for target in targets: + self.access.require('inspect', 'measurements.compare', target) + self.session.check() + documents = self._documents() + if any(target not in documents for target in targets): + raise WorkbenchError('A comparison source is unavailable.', 'not_found', 404) + chosen = [documents[target] for target in targets] + if sum(doc['blob']['bytes'] for doc in chosen) > MAX_COMPARISON_BYTES: + raise WorkbenchError('Native comparison exceeds 128 MiB of original packets.', 'too_large', 413) + tables = [] + for locator, doc in zip(sources, chosen): + raw = self.artifacts.read_blob(doc['blob'], maximum_bytes=MAX_BYTES) + table = MeasurementTable(raw) + if table.metadata()['title'] != doc['title']: + raise WorkbenchError('Comparison title differs from its original packet.') + tables.append((locator, table)) + value = MeasurementComparison(tables).read(metric_id, **selection) + result = self._bounded(dict(sources=chosen, value=value)) + self.session.check() + for target in targets: + self.access.require('inspect', 'measurements.compare', target) + return result diff --git a/retained_measurements.py b/retained_measurements.py index 11188802..796aaa19 100644 --- a/retained_measurements.py +++ b/retained_measurements.py @@ -23,6 +23,9 @@ MAX_SOURCE_TOTAL = 16 * 1024 * 1024 MAX_RESPONSE_BYTES = 256 * 1024 MAX_ROWS = 100000 +MAX_COMPARISON_SOURCES = 16 +MAX_COMPARISON_SUBJECTS = 256 +MAX_COMPARISON_BYTES = 128 * 1024 * 1024 STATES = ('measured', 'failed', 'unsupported', 'unavailable') ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') SHA = re.compile(r'^[a-f0-9]{64}$') @@ -137,6 +140,7 @@ def __init__(self, raw): if type(raw) is not bytes or not 0 < len(raw) <= MAX_BYTES: fail('A bounded complete byte snapshot is required.', 'too_large', 413) self.sha256 = hashlib.sha256(raw).hexdigest() + self.bytes = len(raw) doc = decode(raw) exact(doc, ('format', 'schema_version', 'title', 'scope', 'dimensions', 'subjects', 'cases', 'metrics', 'observations', 'sources')) @@ -348,3 +352,112 @@ def _inspection_reference(self,reference): state='exceeds_inspection_bound';value=None return {'reference':copy.deepcopy(reference),'source_sha256':self._sources[reference['source_id']]['sha256'], 'state':state,'value':value} + + +class MeasurementComparison: + """One bounded scientific comparison over immutable, source-bound tables. + + The native custody owner supplies validated locator/table pairs. Compatibility + depends on original definitions and actual case evidence, not display names. + Returned rows keep their packet locator; no observations or counts are pooled + across subjects. Pagination affects rows only, never the maximum denominator. + """ + def __init__(self, tables): + if type(tables) is not list or not 1 <= len(tables) <= MAX_COMPARISON_SOURCES: + fail('Choose 1 through 16 comparison sources.', 'too_large', 413) + self._tables = [] + self._subjects = {} + seen = set() + basis = None + observations = raw_bytes = 0 + for locator, table in tables: + exact(locator, ('origin_catalog_id', 'packet_sha256')) + if (type(locator['origin_catalog_id']) is not str + or not re.fullmatch('[a-f0-9]{32}', locator['origin_catalog_id']) + or not isinstance(table, MeasurementTable) or locator['packet_sha256'] != table.sha256): + fail('Comparison source differs from its validated packet.') + key = (locator['origin_catalog_id'], locator['packet_sha256']) + if key in seen: + fail('Comparison repeats a source.') + seen.add(key) + raw_bytes += table.bytes + if raw_bytes > MAX_COMPARISON_BYTES: + fail('Comparison exceeds 128 MiB of original packets.', 'too_large', 413) + metadata = table.metadata() + cases = [] + for case in table._cases.values(): + reference = case['reference'] + evidence = table._reference(reference) + cases.append(dict(id=case['id'], dimensions=case['dimensions'], evidence=evidence, + opaque_source_sha256=table._sources[reference['source_id']]['sha256'] + if reference['pointer'] is None else None)) + current = dict(scope=metadata['scope_definition'], dimensions=metadata['dimensions'], + metrics=[table._definition(metric, ('id','label','units')) for metric in table._metrics.values()], + cases=cases) + # Canonical source bytes distinguish Boolean/numeric values too; + # ordinary Python equality is not the scientific identity relation. + identity = digest(current) + if basis is None: + basis = identity + self._header = dict(format='retained_measurement_comparison_v1', + scope=metadata['scope'], scope_definition=metadata['scope_definition'], + dimensions=metadata['dimensions'], metrics=metadata['metrics'], + definition_source=copy.deepcopy(locator), case_population_sha256=digest(cases), + case_count=metadata['case_count']) + elif identity != basis: + fail('Comparison sources have different scope, metric, dimension or original case definitions.', 'incompatible', 409) + for subject in table._subjects.values(): + if subject['id'] in self._subjects: + fail('Comparison sources overlap a subject identity.', 'conflict', 409) + self._subjects[subject['id']] = dict(copy.deepcopy(subject), source=copy.deepcopy(locator)) + observations += metadata['observation_count'] + if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_ROWS: + fail('Comparison exceeds 256 subjects or 100,000 observations.', 'too_large', 413) + self._tables.append((copy.deepcopy(locator), table)) + self._header.update(subjects=list(self._subjects.values()), observation_count=observations) + + def read(self, metric_id=None, *, subject_ids=None, filters=None, offset=0, limit=50): + result = copy.deepcopy(self._header) + result['query'] = None + if metric_id is None: + if subject_ids is not None or filters is not None or offset != 0 or limit != 50: + fail('A comparison selection requires an explicit metric.') + return self._bounded(result) + selected = list(self._subjects) if subject_ids is None else subject_ids + if (type(selected) is not list or not selected + or any(type(value) is not str or value not in self._subjects for value in selected) + or len(set(selected)) != len(selected)): + fail('Choose existing unique comparison subjects.') + if type(offset) is not int or not 0 <= offset <= 2**53-1 or type(limit) is not int or not 1 <= limit <= 100: + fail('Comparison page bounds must be finite integers.') + summaries = {} + rows = [] + total = 0 + template = None + for locator, table in self._tables: + own = [identity for identity in selected if identity in table._subjects] + if not own: + continue + page = table.query(metric_id, subject_ids=own, filters=filters, offset=0, limit=1) + if template is None: + template = page + for summary in page['summary']: + summaries[summary['subject_id']] = dict(summary, source=copy.deepcopy(locator)) + start = max(0, offset-total) + if total+page['total'] > offset and len(rows) < limit: + current = table.query(metric_id, subject_ids=own, filters=filters, + offset=start, limit=limit-len(rows)) + rows.extend(dict(row, source=copy.deepcopy(locator)) for row in current['rows']) + total += page['total'] + metric = next(item for item in self._header['metrics'] if item['id'] == metric_id) + result['query'] = dict(metric=copy.deepcopy(metric), definition=template['definition'], + filters=template['filters'], subject_ids=list(selected), + summary=[summaries[identity] for identity in selected], rows=rows, + total=total, offset=offset, limit=limit, sampling='none') + return self._bounded(result) + + @staticmethod + def _bounded(value): + if len(encode(value)) > MAX_RESPONSE_BYTES: + fail('Comparison response exceeds its bound; choose fewer sources or a smaller page.', 'too_large', 413) + return value diff --git a/static/project-files.js b/static/project-files.js index ee86c1cd..65cfa499 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -83,6 +83,7 @@ "project_files.material_intent_capabilities": "86e90e2d0f094beb1c277a680da9a35e28d78c827390ac1c4e51fc3003c1ce32", "project_files.material_intents": "6431fd2780d706c77d2c4dd82f15794017368fd3bade28ac62bbdff5c8b0c547", "project_files.materials": "92a6a6a1750fed89aef9574ab6e82be2c6c6351bfa89fbfcb1f3866d4ea3cb13", + "project_files.measurement_compare": "abbb5e406a3c95155e90f08e8a071f330425190b25275f0f6512de08b8afb929", "project_files.measurement_import": "c8e500396138fc60a31fc916c441bb95454f410231a6508609a7ebba5cd31f18", "project_files.measurement_inspect": "fcbe9846ca49c3da4713b54a3cb4a8237189d8eea5799f275d33992ef268914f", "project_files.measurement_metadata": "5a92ed5259f8876a3c5b8fd1886a9e0f44f0aea8e53d7f63902ac765ddac0de7", @@ -175,668 +176,678 @@ { "project_files.artifact_resolve": { "method": "POST", - "path": "/api/workspace/project-files/artifacts/resolve", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/artifacts/resolve" }, "project_files.artifact_status": { "method": "POST", - "path": "/api/workspace/project-files/artifacts/status", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/artifacts/status" }, "project_files.artifacts": { "method": "GET", - "path": "/api/workspace/project-files/artifacts", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/artifacts" }, "project_files.candidate": { "method": "GET", - "path": "/api/workspace/project-files/candidates/{candidate_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/candidates/{candidate_id}" }, "project_files.candidate_select": { "method": "POST", - "path": "/api/workspace/project-files/studies/{study_id}/candidates", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies/{study_id}/candidates" }, "project_files.data_annotate": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}" }, "project_files.data_availability": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/availability", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/availability" }, "project_files.data_get": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}" }, "project_files.data_history": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/history" }, "project_files.data_import_audio": { "method": "POST", - "path": "/api/workspace/project-files/data/import-audio", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-audio" }, "project_files.data_import_image": { "method": "POST", - "path": "/api/workspace/project-files/data/import-image", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-image" }, "project_files.data_import_mesh": { "method": "POST", - "path": "/api/workspace/project-files/data/import-mesh", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-mesh" }, "project_files.data_import_pointcloud": { "method": "POST", - "path": "/api/workspace/project-files/data/import-pointcloud", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-pointcloud" }, "project_files.data_import_scientific": { "method": "POST", - "path": "/api/workspace/project-files/data/import-scientific", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-scientific" }, "project_files.data_import_sequence": { "method": "POST", - "path": "/api/workspace/project-files/data/import-sequence", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-sequence" }, "project_files.data_import_text": { "method": "POST", - "path": "/api/workspace/project-files/data/import-text", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-text" }, "project_files.data_import_volume": { "method": "POST", - "path": "/api/workspace/project-files/data/import-volume", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-volume" }, "project_files.data_preference_delete": { "method": "POST", - "path": "/api/workspace/project-files/preferences/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/preferences/delete" }, "project_files.data_preference_history": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/preference-history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/preference-history" }, "project_files.data_preference_save": { "method": "POST", - "path": "/api/workspace/project-files/preferences", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/preferences" }, "project_files.data_preferences": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/preferences", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/preferences" }, "project_files.data_query": { "method": "GET", - "path": "/api/workspace/project-files/data", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data" }, "project_files.data_response_history": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/response-history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/response-history" }, "project_files.data_response_save": { "method": "POST", - "path": "/api/workspace/project-files/responses", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/responses" }, "project_files.data_responses": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/responses", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/responses" }, "project_files.data_rights": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}/rights", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}/rights" }, "project_files.data_scientific": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/scientific", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/scientific" }, "project_files.data_source_delete": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}/source/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}/source/delete" }, "project_files.data_source_save": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}/source", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}/source" }, "project_files.definitions": { "method": "GET", - "path": "/api/workspace/project-files/definitions", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/definitions" }, "project_files.dependencies": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}/dependencies", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/dependencies" }, "project_files.evidence": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/evidence", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/evidence" }, "project_files.execution_approve_external": { "method": "POST", - "path": "/api/workspace/project-files/execution/approvals", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/approvals" }, "project_files.execution_cancel": { "method": "POST", - "path": "/api/workspace/project-files/execution/runs/{run_id}/cancel", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/runs/{run_id}/cancel" }, "project_files.execution_generation_resume": { "method": "POST", - "path": "/api/workspace/project-files/execution/generation/{producer_id}/resume", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/generation/{producer_id}/resume" }, "project_files.execution_grounded_review": { "method": "POST", - "path": "/api/workspace/project-files/execution/grounded/{producer_id}/review", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/grounded/{producer_id}/review" }, "project_files.execution_health": { "method": "GET", - "path": "/api/workspace/project-files/execution", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/execution" }, "project_files.execution_preflight_external": { "method": "POST", - "path": "/api/workspace/project-files/execution/preflight", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/execution/preflight" }, "project_files.execution_producer_cancel": { "method": "POST", - "path": "/api/workspace/project-files/execution/producers/{producer_id}/cancel", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/producers/{producer_id}/cancel" }, "project_files.execution_producer_decide": { "method": "POST", - "path": "/api/workspace/project-files/execution/producers/{producer_id}/decide", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/producers/{producer_id}/decide" }, "project_files.execution_producer_health": { "method": "GET", - "path": "/api/workspace/project-files/execution/producers/health", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/execution/producers/health" }, "project_files.execution_producer_start": { "method": "POST", - "path": "/api/workspace/project-files/execution/producers", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/producers" }, "project_files.execution_reconcile_current": { "method": "POST", - "path": "/api/workspace/project-files/execution/reconcile-current", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/reconcile-current" }, "project_files.execution_study_abandon_preparation": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/abandon", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/abandon" }, "project_files.execution_study_admit": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/runs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/runs" }, "project_files.execution_study_approve": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/approvals", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/approvals" }, "project_files.execution_study_continue": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/continue", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/continue" }, "project_files.execution_study_prepare_package": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/packages", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/packages" }, "project_files.execution_study_prepare_verifier": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/verifiers", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/verifiers" }, "project_files.execution_study_reconcile_preparation": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/reconcile", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/reconcile" }, "project_files.execution_study_stop": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/stop", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/stop" }, "project_files.execution_submit_external": { "method": "POST", - "path": "/api/workspace/project-files/execution/runs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/runs" }, "project_files.expose_holdout": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/disclose", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/runs/{run_id}/disclose" }, "project_files.file_adopt": { "method": "POST", - "path": "/api/workspace/project-files/files/adopt", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/files/adopt" }, "project_files.file_adopt_deletion": { "method": "POST", - "path": "/api/workspace/project-files/files/adopt-deletion", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/files/adopt-deletion" }, "project_files.file_get": { "method": "GET", - "path": "/api/workspace/project-files/files/{file_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/files/{file_id}" }, "project_files.file_history": { "method": "GET", - "path": "/api/workspace/project-files/files/{file_id}/history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/files/{file_id}/history" + }, + "project_files.files_adopt": { + "method": "POST", + "mutation": true, + "path": "/api/workspace/project-files/files/adopt-many" }, "project_files.files_inspect": { "method": "GET", - "path": "/api/workspace/project-files/files", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/files" }, "project_files.fork_info": { "method": "GET", - "path": "/api/workspace/project-files/fork", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/fork" }, "project_files.graph_clone": { "method": "POST", - "path": "/api/workspace/project-files/graphs/{graph_id}/clone", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs/{graph_id}/clone" }, "project_files.graph_create": { "method": "POST", - "path": "/api/workspace/project-files/graphs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs" }, "project_files.graph_create_external": { "method": "POST", - "path": "/api/workspace/project-files/graphs/external", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs/external" }, "project_files.graph_get": { "method": "GET", - "path": "/api/workspace/project-files/graphs/{graph_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/graphs/{graph_id}" }, "project_files.graph_save": { "method": "POST", - "path": "/api/workspace/project-files/graphs/{graph_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs/{graph_id}" }, "project_files.graphs": { "method": "GET", - "path": "/api/workspace/project-files/graphs", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/graphs" }, "project_files.lineage": { "method": "GET", - "path": "/api/workspace/project-files/lineage", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/lineage" }, "project_files.lineage_record": { "method": "POST", - "path": "/api/workspace/project-files/lineage", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/lineage" }, "project_files.material_intent": { "method": "GET", - "path": "/api/workspace/project-files/material-intents/{family}/{definition_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/material-intents/{family}/{definition_id}" }, "project_files.material_intent_capabilities": { "method": "GET", - "path": "/api/workspace/project-files/material-intents/capabilities", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/material-intents/capabilities" }, "project_files.material_intents": { "method": "GET", - "path": "/api/workspace/project-files/material-intents", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/material-intents" }, "project_files.materials": { "method": "GET", - "path": "/api/workspace/project-files/materials", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials" + }, + "project_files.measurement_compare": { + "method": "POST", + "mutation": false, + "path": "/api/workspace/project-files/measurements/compare" + }, + "project_files.measurement_import": { + "method": "POST", + "mutation": true, + "path": "/api/workspace/project-files/measurements/import" + }, + "project_files.measurement_inspect": { + "method": "GET", + "mutation": false, + "path": "/api/workspace/project-files/measurements/{packet_sha256}/observations/{observation_id}" + }, + "project_files.measurement_metadata": { + "method": "GET", + "mutation": false, + "path": "/api/workspace/project-files/measurements/{packet_sha256}" + }, + "project_files.measurement_query": { + "method": "POST", + "mutation": false, + "path": "/api/workspace/project-files/measurements/{packet_sha256}/query" + }, + "project_files.measurement_sources": { + "method": "GET", + "mutation": false, + "path": "/api/workspace/project-files/measurements" + }, + "project_files.measurement_subjects": { + "method": "GET", + "mutation": false, + "path": "/api/workspace/project-files/measurements/{packet_sha256}/subjects" }, "project_files.model_create": { "method": "POST", - "path": "/api/workspace/project-files/models", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/models" }, "project_files.model_get": { "method": "GET", - "path": "/api/workspace/project-files/models/{model_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/models/{model_id}" }, "project_files.model_material": { "method": "POST", - "path": "/api/workspace/project-files/materials/model", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/model" }, "project_files.model_publish": { "method": "POST", - "path": "/api/workspace/project-files/models/{model_id}/publish", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/models/{model_id}/publish" }, "project_files.model_save": { "method": "POST", - "path": "/api/workspace/project-files/models/{model_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/models/{model_id}" }, "project_files.models": { "method": "GET", - "path": "/api/workspace/project-files/models", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/models" }, "project_files.native_preflight": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/preflight", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/preflight" }, "project_files.native_reconcile": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/phases/{phase_id}/reconcile", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/phases/{phase_id}/reconcile" }, "project_files.native_retry": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/runs/{run_id}/retry", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/runs/{run_id}/retry" }, "project_files.native_submit": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/runs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/runs" }, "project_files.package_state": { "method": "GET", - "path": "/api/workspace/project-files/candidates/{candidate_id}/package-state", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/candidates/{candidate_id}/package-state" }, "project_files.policies": { "method": "GET", - "path": "/api/workspace/project-files/policies", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/policies" }, "project_files.policy": { "method": "POST", - "path": "/api/workspace/project-files/policies/get", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/policies/get" }, "project_files.producer": { "method": "GET", - "path": "/api/workspace/project-files/producers/{origin_catalog_id}/{producer_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/producers/{origin_catalog_id}/{producer_id}" }, "project_files.producers": { "method": "GET", - "path": "/api/workspace/project-files/producers", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/producers" }, "project_files.progression": { "method": "GET", - "path": "/api/workspace/project-files/progression", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/progression" }, "project_files.progression_origins": { "method": "GET", - "path": "/api/workspace/project-files/progression/origins", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/progression/origins" }, "project_files.publication_result": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}/publication-result", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/publication-result" }, "project_files.recipe_create": { "method": "POST", - "path": "/api/workspace/project-files/recipes", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/recipes" }, "project_files.recipe_get": { "method": "GET", - "path": "/api/workspace/project-files/recipes/{recipe_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/recipes/{recipe_id}" }, "project_files.recipe_material": { "method": "POST", - "path": "/api/workspace/project-files/materials/recipe", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/recipe" }, "project_files.recipe_prepare": { "method": "POST", - "path": "/api/workspace/project-files/recipes/{graph_id}/prepare", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/recipes/{graph_id}/prepare" }, "project_files.recipe_save": { "method": "POST", - "path": "/api/workspace/project-files/recipes/{recipe_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/recipes/{recipe_id}" }, "project_files.recipes": { "method": "GET", - "path": "/api/workspace/project-files/recipes", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/recipes" }, "project_files.release_freeze": { "method": "POST", - "path": "/api/workspace/project-files/releases/freeze", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/releases/freeze" }, "project_files.release_preview": { "method": "POST", - "path": "/api/workspace/project-files/releases/preview", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/releases/preview" }, "project_files.retained_analysis": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/analysis", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/analysis" }, "project_files.retained_analysis_export": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/analysis/export", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/analysis/export" }, "project_files.run": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}" }, "project_files.run_document_segment": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}/document-segment", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/document-segment" }, "project_files.runs": { "method": "GET", - "path": "/api/workspace/project-files/runs", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs" }, "project_files.runtime_abandon": { "method": "POST", - "path": "/api/workspace/project-files/runtime/abandon", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/runtime/abandon" }, "project_files.runtime_capture": { "method": "POST", - "path": "/api/workspace/project-files/runtime/capture", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/runtime/capture" }, "project_files.runtime_inspect": { "method": "GET", - "path": "/api/workspace/project-files/runtime/inspect", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runtime/inspect" }, "project_files.runtime_material": { "method": "POST", - "path": "/api/workspace/project-files/materials/runtime", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/runtime" }, "project_files.runtime_sources": { "method": "GET", - "path": "/api/workspace/project-files/runtime/sources", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runtime/sources" }, "project_files.runtime_status": { "method": "GET", - "path": "/api/workspace/project-files/runtime/status", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runtime/status" }, "project_files.save_as": { "method": "POST", - "path": "/api/workspace/project-files/save-as", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/save-as" }, "project_files.search_create": { "method": "POST", - "path": "/api/workspace/project-files/searches", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/searches" }, "project_files.search_delete": { "method": "POST", - "path": "/api/workspace/project-files/searches/{search_id}/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/searches/{search_id}/delete" }, "project_files.search_rename": { "method": "POST", - "path": "/api/workspace/project-files/searches/{search_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/searches/{search_id}" }, "project_files.searches": { "method": "GET", - "path": "/api/workspace/project-files/searches", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/searches" }, "project_files.selection_create": { "method": "POST", - "path": "/api/workspace/project-files/selections", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/selections" }, "project_files.selection_delete": { "method": "POST", - "path": "/api/workspace/project-files/selections/{selection_id}/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/selections/{selection_id}/delete" }, "project_files.selection_get": { "method": "GET", - "path": "/api/workspace/project-files/selections/{selection_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/selections/{selection_id}" }, "project_files.selection_rename": { "method": "POST", - "path": "/api/workspace/project-files/selections/{selection_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/selections/{selection_id}" }, "project_files.selections": { "method": "GET", - "path": "/api/workspace/project-files/selections", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/selections" }, "project_files.snapshot": { "method": "POST", - "path": "/api/workspace/project-files/materials/snapshot", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/snapshot" }, "project_files.source_references": { "method": "GET", - "path": "/api/workspace/project-files/sources/{project_id}/references", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/sources/{project_id}/references" }, "project_files.source_register": { "method": "POST", - "path": "/api/workspace/project-files/sources/register", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/sources/register" }, "project_files.studies": { "method": "GET", - "path": "/api/workspace/project-files/studies", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/studies" }, "project_files.study": { "method": "GET", - "path": "/api/workspace/project-files/studies/{study_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/studies/{study_id}" }, "project_files.study_create": { "method": "POST", - "path": "/api/workspace/project-files/studies", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies" }, "project_files.study_save": { "method": "POST", - "path": "/api/workspace/project-files/studies/{study_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies/{study_id}" }, "project_files.trial": { "method": "GET", - "path": "/api/workspace/project-files/trials/{trial_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/trials/{trial_id}" }, "project_files.trial_prepare": { "method": "POST", - "path": "/api/workspace/project-files/studies/{study_id}/trials", - "mutation": true - }, - "project_files.measurement_import": { - "method": "POST", - "path": "/api/workspace/project-files/measurements/import", - "mutation": true - }, - "project_files.measurement_inspect": { - "method": "GET", - "path": "/api/workspace/project-files/measurements/{packet_sha256}/observations/{observation_id}", - "mutation": false - }, - "project_files.measurement_metadata": { - "method": "GET", - "path": "/api/workspace/project-files/measurements/{packet_sha256}", - "mutation": false - }, - "project_files.measurement_query": { - "method": "POST", - "path": "/api/workspace/project-files/measurements/{packet_sha256}/query", - "mutation": false - }, - "project_files.measurement_sources": { - "method": "GET", - "path": "/api/workspace/project-files/measurements", - "mutation": false - }, - "project_files.measurement_subjects": { - "method": "GET", - "path": "/api/workspace/project-files/measurements/{packet_sha256}/subjects", - "mutation": false + "mutation": true, + "path": "/api/workspace/project-files/studies/{study_id}/trials" } } /* PROJECT_FILES_ROUTES_END */; @@ -1149,7 +1160,7 @@ projectProgression.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), project_id:state?.project?.project_id,enabled:m2Allowed('progression')&&m2Allowed('progression_origins')}); projectMeasurements.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), - enabled:['measurement_sources','measurement_metadata','measurement_subjects','measurement_query','measurement_inspect'].every(m2Allowed)}); + enabled:['measurement_sources','measurement_compare','measurement_inspect'].every(m2Allowed)}); if(blocked||state?.state!=='open'||scientificEpoch!==null&&(scientificEpoch!==epoch||explored?.checkpoint!==checkpoint()))clearScientific(); const action=$('project-explore-kind').value; $('project-explore-load').disabled=!m2Allowed(action); diff --git a/static/project-measurements.js b/static/project-measurements.js index 7f99a043..314d3f33 100644 --- a/static/project-measurements.js +++ b/static/project-measurements.js @@ -20,11 +20,11 @@ details.append(node('summary', title), node('pre', JSON.stringify(value, null, 2))); return details; } window.ProjectMeasurements = function({root, read, onContextFocusLost = () => {}}) { - root.innerHTML = '

Retained measurements

Compare exact saved scalars and inspect their original cases. Maxima use every matching case; failed and unavailable observations remain counted.

'; + root.innerHTML = '

Retained measurements

Compare exact saved scalars and inspect their original cases. Maxima use every matching case; failed and unavailable observations remain counted.

'; const $ = name => root.querySelector('#project-measurements-' + name); let context = null, generation = 0, plotGeneration = 0, pending, busy = false, pendingFocus = false; - let sources = [], sourceOffset = 0, sourceTotal = 0, selected = null, descriptor = null, metadata = null; - let subjects = [], subjectOffset = 0, subjectTotal = 0, filters = new Map(), applied = null, query = null; + let sources = [], sourceOffset = 0, sourceTotal = 0, selected = null, descriptors = null, metadata = null; + let subjects = [], subjectOffset = 0, subjectTotal = 0, filters = new Map(), applied = null, query = null, selectedSubjects = new Set(); const renderer = window.RetainedChart.mount($('chart'), spec, {onError: error => { const ownsFocus = root.contains(document.activeElement); clearResults(); $('status').textContent = error.message; controls(); if (ownsFocus) $('compare').focus(); @@ -37,15 +37,17 @@ function controls() { for (const control of root.querySelectorAll('input,select,button')) control.disabled = !context?.enabled || busy; $('sources-prev').hidden = sourceOffset === 0; $('sources-next').hidden = sourceOffset + sources.length >= sourceTotal; - $('source').disabled ||= !sources.length; $('load').disabled ||= !$('source').value; - $('subjects-prev').hidden = subjectOffset === 0; $('subjects-next').hidden = subjectOffset + subjects.length >= subjectTotal; - $('compare').disabled ||= !metadata || !$('subjects').selectedOptions.length; + $('source').disabled ||= !sources.length; $('source-all').disabled ||= !sources.length || sources.length > 16; + $('load').disabled ||= !$('source').selectedOptions.length || $('source').selectedOptions.length > 16; + $('subjects-prev').hidden = subjectOffset === 0; $('subjects-next').hidden = subjectOffset + 40 >= subjectTotal; + $('compare').disabled ||= !metadata || !selectedSubjects.size; + $('subjects-all').disabled ||= !metadata; $('subjects-clear').disabled ||= !metadata; $('prev').disabled ||= !query || query.offset === 0; $('next').disabled ||= !query || query.offset + query.rows.length >= query.total; root.setAttribute('aria-busy', String(busy)); } function clearSource() { - selected = descriptor = metadata = null; subjects = []; subjectOffset = subjectTotal = 0; filters.clear(); + selected = descriptors = metadata = null; subjects = []; selectedSubjects.clear(); subjectOffset = subjectTotal = 0; filters.clear(); $('form').hidden = true; $('context').replaceChildren(); $('filters').replaceChildren(); $('metric').replaceChildren(); $('subjects').replaceChildren(); clearResults(); } @@ -84,34 +86,68 @@ seen.add(key); } sources = value.items; sourceOffset = offset; sourceTotal = value.total; $('source').replaceChildren(); - for (const item of sources) { + for (const [index, item] of sources.entries()) { const option = node('option', item.title + ' · ' + item.id.slice(0, 12) + (item.availability?.state === 'missing' ? ' · missing packet' : '')); - option.value = sourceKey(item); $('source').append(option); + option.value = sourceKey(item); option.selected = index === 0; $('source').append(option); } $('source-page').textContent = `${sources.length ? offset + 1 : 0}–${offset + sources.length} of ${sourceTotal} sources`; - $('status').textContent = sources.length ? 'Choose a source and load its exact retained metadata.' : 'No retained measurement sources in this project.'; + $('status').textContent = sources.length ? 'Choose compatible sources and load their exact retained metadata.' : 'No retained measurement sources in this project.'; return $('source'); }); } - function checkRead(value, source = selected) { - if (!value || value.source?.origin_catalog_id !== source?.origin_catalog_id || value.source?.id !== source?.id - || value.source.blob?.sha256 !== source.id || value.source.source?.measured_sha256 !== source.id - || descriptor && (value.source.blob.bytes !== descriptor.blob.bytes || value.source.title !== descriptor.title) - || new TextEncoder().encode(JSON.stringify(value)).length > 256 * 1024) throw Error('Measurement reply differs from the selected packet.'); - return value.value; + const same = (left, right) => { + if (Object.is(left, right)) return true; + if (!left || !right || typeof left !== 'object' || typeof right !== 'object' || Array.isArray(left) !== Array.isArray(right)) return false; + const keys = Object.keys(left); + return keys.length === Object.keys(right).length && keys.every(key => Object.hasOwn(right, key) && same(left[key], right[key])); + }; + const locatorKey = value => value.origin_catalog_id + '.' + value.packet_sha256; + const selection = () => ({sources:selected.map(item => ({origin_catalog_id:item.origin_catalog_id, packet_sha256:item.id}))}); + function checkComparison(response) { + if (!response || !Array.isArray(response.sources) || response.sources.length !== selected.length + || new TextEncoder().encode(JSON.stringify(response)).length > 256 * 1024) throw Error('Comparison differs from selected sources.'); + for (const [index, source] of response.sources.entries()) { + const expected = selected[index]; + if (source.origin_catalog_id !== expected.origin_catalog_id || source.id !== expected.id + || source.title !== expected.title || source.blob?.sha256 !== source.id || source.blob.bytes !== expected.blob.bytes + || source.source?.measured_sha256 !== source.id) throw Error('Comparison source identity changed.'); + } + if (descriptors && !same(descriptors, response.sources)) throw Error('Original comparison capture facts changed.'); + const value = response.value, keys = selected.map(sourceKey); + if (value?.format !== 'retained_measurement_comparison_v1' || !hash(value.case_population_sha256) + || !Array.isArray(value.subjects) || !value.subjects.length || value.subjects.length > 256 + || !natural(value.case_count) || !value.case_count || !natural(value.observation_count) + || !Array.isArray(value.metrics) || !value.metrics.length || value.metrics.length > 32 + || !Array.isArray(value.dimensions) || value.dimensions.length > 32 + || value.observation_count !== value.subjects.length * value.case_count * value.metrics.length + || locatorKey(value.definition_source) !== keys[0] + || value.subjects.some(item => !identity(item.id) || typeof item.label !== 'string' || !keys.includes(locatorKey(item.source))) + || new Set(value.subjects.map(item => item.id)).size !== value.subjects.length) throw Error('Invalid comparison metadata.'); + if (metadata && ['scope','scope_definition','dimensions','metrics','definition_source','case_population_sha256','case_count','observation_count','subjects'].some(key => !same(value[key], metadata[key]))) throw Error('Comparison population changed.'); + return value; + } + function checkRead(response, source) { + const original = descriptors.find(item => sourceKey(item) === locatorKey(source)); + if (!original || response?.source?.origin_catalog_id !== source.origin_catalog_id || response.source.id !== source.packet_sha256 + || response.source.blob?.sha256 !== source.packet_sha256 || !same(response.source, original) + || new TextEncoder().encode(JSON.stringify(response)).length > 256 * 1024) throw Error('Observation differs from its original packet.'); + return response.value; } - const selection = () => ({origin_catalog_id: selected.origin_catalog_id, packet_sha256: selected.id}); - async function subjectPage(offset, active, signal) { - const value = checkRead(await read('measurement_subjects', {...selection(), offset, limit: 40}, signal)); if (!active()) return; - page(value, offset, 40, value.items); - if (value.total !== metadata.subject_count) throw Error('Subject page differs from the retained packet population.'); - if (value.items.some(item => !identity(item.id) || typeof item.label !== 'string') - || new Set(value.items.map(item => item.id)).size !== value.items.length) throw Error('Invalid retained subjects.'); - subjects = value.items; subjectOffset = offset; subjectTotal = value.total; $('subjects').replaceChildren(); - for (const item of subjects) { const option = node('option', item.label + ' · ' + item.id); option.value = item.id; option.selected = true; $('subjects').append(option); } - $('subject-page').textContent = `Subjects ${subjects.length ? offset + 1 : 0}–${offset + subjects.length} of ${subjectTotal}. Only explicitly selected subjects on this page will be compared.`; + function subjectPage(offset) { + if (!metadata || !natural(offset) || offset > subjectTotal) throw Error('Invalid comparison subject page.'); + subjectOffset = offset; $('subjects').replaceChildren(); + const items = subjects.slice(offset, offset + 40); + for (const item of items) { + const option = node('option', item.label + ' · ' + item.id); option.value = item.id; + option.selected = selectedSubjects.has(item.id); $('subjects').append(option); + } + subjectSelectionStatus(); return $('subjects'); } + function subjectSelectionStatus() { + const count = Math.min(40, subjectTotal - subjectOffset); + $('subject-page').textContent = `Subjects ${count ? subjectOffset + 1 : 0}–${subjectOffset + count} of ${subjectTotal}. ${selectedSubjects.size} selected across all pages; selection is retained when paging.`; + } function dimensions() { filters.clear(); $('filters').replaceChildren(); for (const [index, dimension] of metadata.dimensions.entries()) { @@ -158,12 +194,13 @@ for (const text of headings) { const cell = node('th', text); cell.scope = 'col'; row.append(cell); } head.append(row); result.append(node('caption', caption), head, body); return {result, body}; } - async function inspect(observation_id) { + async function inspect(observation_id, source) { clearInspection(); return perform(async(active, signal) => { - const value = checkRead(await read('measurement_inspect', {...selection(), observation_id}, signal)); if (!active()) return; - if (value.packet_sha256 !== selected.id || value.observation?.id !== observation_id - || value.observation.metric_id !== applied?.metric_id || !applied.subject_ids.includes(value.observation.subject_id)) throw Error('Observation differs from the selected comparison.'); + const value = checkRead(await read('measurement_inspect', {...source, observation_id}, signal), source); if (!active()) return; + if (value.packet_sha256 !== source.packet_sha256 || value.observation?.id !== observation_id + || value.observation.metric_id !== applied?.metric_id || !applied.subject_ids.includes(value.observation.subject_id) + || !same(subjects.find(item => item.id === value.observation.subject_id)?.source, source)) throw Error('Observation differs from the selected comparison.'); const heading = node('h3', value.subject.label + ' · ' + value.case.id); heading.tabIndex = -1; const observation = value.observation; $('inspection').append(heading, node('p', `${value.metric.label} · ${value.metric.units} · ${observation.state} · ${observation.value === null ? observation.reason : display(observation.value)}`), @@ -175,27 +212,31 @@ }); } async function showQuery(value, request, active, signal, authority) { - if (value.packet_sha256 !== selected.id || value.metric?.id !== request.metric_id + if (!value || value.metric?.id !== request.metric_id + || !same(value.metric, metadata.metrics.find(metric => metric.id === request.metric_id)) || JSON.stringify(value.subject_ids) !== JSON.stringify(request.subject_ids) || !value.filters || Object.keys(value.filters).length !== Object.keys(request.filters).length || Object.entries(request.filters).some(([key, wanted]) => !Object.hasOwn(value.filters, key) || value.filters[key] !== wanted) || !Array.isArray(value.summary) || value.summary.length !== request.subject_ids.length || value.sampling !== 'none') throw Error('Measurement comparison differs from its requested selection.'); page(value, request.offset, request.limit, value.rows); + if (value.summary.reduce((count, summary) => count + summary.count, 0) !== value.total) throw Error('Comparison denominator differs from its complete summaries.'); const maxima = table('Maximum over all matching cases, including every observation status', ['Subject / execution attributes','Maximum','Measured','Failed','Unsupported','Unavailable']); - const points = []; + const points = [], inspections = new Map(); for (const [index, summary] of value.summary.entries()) { if (summary.subject_id !== request.subject_ids[index] || !natural(summary.count) || STATES.some(state => !natural(summary.states?.[state])) || STATES.reduce((sum, state) => sum + summary.states[state], 0) !== summary.count || (summary.maximum === null) !== (summary.states.measured === 0)) throw Error('Invalid complete measurement summary.'); - const subject = subjects.find(item => item.id === summary.subject_id); if (!subject) throw Error('Subject is outside the selected page.'); + const subject = subjects.find(item => item.id === summary.subject_id); if (!subject || !same(subject.source, summary.source)) throw Error('Subject differs from its original comparison packet.'); const row = node('tr'), label = node('td'); label.append(node('strong', subject.label), attributes('Execution attributes', subject.dimensions)); const maximum = node('td'); if (summary.maximum === null) maximum.textContent = 'No measured value'; else { const point = summary.maximum; if (!finite(point.value) || !identity(point.id) || !identity(point.case_id)) throw Error('Invalid retained maximum.'); - maximum.append(button(display(point.value) + ' · inspect maximum for ' + subject.label, () => void inspect(point.id))); - points.push({subject_id:subject.id, value:point.value, observation_id:point.id}); + maximum.append(button(display(point.value) + ' · inspect maximum for ' + subject.label, () => void inspect(point.id, summary.source))); + const token = JSON.stringify([summary.source.origin_catalog_id, summary.source.packet_sha256, point.id]); + inspections.set(token, {id:point.id, source:summary.source}); + points.push({subject_id:subject.id, value:point.value, observation_id:token}); } row.append(label, maximum, ...STATES.map(state => node('td', String(summary.states[state])))); maxima.body.append(row); } @@ -204,7 +245,8 @@ if (!request.subject_ids.includes(observation.subject_id) || observation.metric_id !== request.metric_id || !identity(observation.id) || !STATES.includes(observation.state) || (observation.state === 'measured' ? !finite(observation.value) : observation.value !== null)) throw Error('Invalid retained observation row.'); const subject = subjects.find(item => item.id === observation.subject_id), row = node('tr'), detail = node('td'); - detail.append(button('Inspect ' + observation.id, () => void inspect(observation.id)), attributes('Case and execution attributes', observation.dimensions)); + if (!subject || !same(subject.source, observation.source)) throw Error('Observation belongs to another packet.'); + detail.append(button('Inspect ' + observation.id, () => void inspect(observation.id, observation.source)), attributes('Case and execution attributes', observation.dimensions)); row.append(node('td', subject.label), node('td', observation.case_id), node('td', observation.value === null ? observation.state + ': ' + observation.reason : display(observation.value)), detail); observations.body.append(row); } query = value; applied = structuredClone(request); $('results').hidden = false; @@ -214,44 +256,59 @@ const plotTicket = ++plotGeneration; if (points.length) await renderer.render({maxima:points}, {signal, isCurrent: () => context?.enabled && context.key === authority && plotTicket === plotGeneration, - onSignal:{measurement_select: selection => { const id = selection?.observation_id?.[0]; if (points.some(point => point.observation_id === id)) void inspect(id); }}}); + onSignal:{measurement_select: selection => { const id = selection?.observation_id?.[0]; const original = inspections.get(id); if (original) void inspect(original.id, original.source); }}}); else { renderer.clear(); $('chart').append(node('p', 'No measured values for this selection. Missing and failed observations remain counted below.')); } if (active()) { $('heading').focus(); $('status').textContent = 'Exact retained comparison. Metric and scope definitions are available above.'; } } function compare(offset = 0, saved = null) { return perform(async(active, signal, authority) => { const request = saved ? {...saved, offset} : {metric_id:$('metric').value, - subject_ids:[...$('subjects').selectedOptions].map(option => option.value), filters:filterValues(), offset, limit:50}; + subject_ids:subjects.filter(item => selectedSubjects.has(item.id)).map(item => item.id), filters:filterValues(), offset, limit:50}; if (!request.subject_ids.length) throw Error('Choose at least one subject.'); clearResults(); - const value = checkRead(await read('measurement_query', {...selection(), ...request}, signal)); if (!active()) return; + const value = checkComparison(await read('measurement_compare', {...selection(), ...request}, signal)).query; if (!active()) return; await showQuery(value, request, active, signal, authority); }); } $('sources').onclick = () => void loadSources(); $('sources-prev').onclick = () => void loadSources(Math.max(0, sourceOffset - 40)); $('sources-next').onclick = () => void loadSources(sourceOffset + 40); - $('source').onchange = () => { invalidate(); clearSource(); controls(); }; + $('source').onchange = () => { invalidate(); clearSource(); $('status').textContent = 'Load the selected compatible sources.'; controls(); }; $('source-form').onsubmit = event => { event.preventDefault(); void perform(async(active, signal) => { - const chosen = sources.find(source => sourceKey(source) === $('source').value); clearSource(); selected = chosen; - if (!selected) throw Error('Choose a retained measurement source.'); - const response = await read('measurement_metadata', selection(), signal); if (!active()) return; - metadata = checkRead(response); descriptor = response.source; - if (metadata.packet_sha256 !== selected.id || metadata.title !== selected.title || !Array.isArray(metadata.metrics) || metadata.metrics.length > 32 - || !Array.isArray(metadata.dimensions) || metadata.dimensions.length > 32) throw Error('Invalid exact measurement metadata.'); - $('context').append(node('h3', metadata.title), node('p', metadata.scope.label + ' · population: ' + metadata.scope.population), + const chosen = [...$('source').selectedOptions].map(option => sources.find(item => sourceKey(item) === option.value)); + clearSource(); selected = chosen; + if (!selected.length || selected.length > 16 || selected.some(item => !item)) throw Error('Choose 1 through 16 retained sources.'); + const response = await read('measurement_compare', selection(), signal); if (!active()) return; + metadata = checkComparison(response); descriptors = response.sources; + if (metadata.query !== null) throw Error('Metadata reply unexpectedly contains a query.'); + subjects = metadata.subjects; subjectTotal = subjects.length; selectedSubjects = new Set(subjects.map(item => item.id)); + $('context').append(node('h3', `Comparison of ${selected.length} retained sources`), node('p', metadata.scope.label + ' · population: ' + metadata.scope.population), node('p', typeof metadata.scope_definition.task === 'string' ? metadata.scope_definition.task : 'Interpret results using the retained scope definition.'), - jsonDetails('Exact scope, metric definitions and packet source', {metadata, source:descriptor})); + jsonDetails('Exact compatible scope, metric definitions and original packet sources', {metadata, sources:descriptors})); for (const metric of metadata.metrics) { const option = node('option', metric.label + ' · ' + metric.units); option.value = metric.id; $('metric').append(option); } - dimensions(); await subjectPage(0, active, signal); if (!active()) return; + dimensions(); subjectPage(0); if (!active()) return; $('form').hidden = false; $('status').textContent = 'Choose a metric, subjects and optional exact attribute filters.'; return $('metric'); }); }; - $('metric').onchange = invalidate; $('subjects').onchange = invalidate; + $('source-all').onclick = () => { + for (const option of $('source').options) option.selected = true; + $('source').onchange(); + }; + $('metric').onchange = invalidate; + $('subjects').onchange = () => { + for (const option of $('subjects').options) { + if (option.selected) selectedSubjects.add(option.value); else selectedSubjects.delete(option.value); + } + invalidate(); subjectSelectionStatus(); controls(); + }; + for (const [name, all] of [['subjects-all',true],['subjects-clear',false]]) $(name).onclick = () => { + selectedSubjects = new Set(all ? subjects.map(item => item.id) : []); + invalidate(); subjectPage(subjectOffset); controls(); + }; for (const [name, direction] of [['subjects-prev',-1],['subjects-next',1]]) $(name).onclick = () => { - clearResults(); void perform((active, signal) => subjectPage(subjectOffset + direction * 40, active, signal)); + clearResults(); void perform(() => subjectPage(subjectOffset + direction * 40)); }; $('form').onsubmit = event => { event.preventDefault(); void compare(); }; $('prev').onclick = () => void compare(Math.max(0, query.offset - query.limit), applied); diff --git a/tests/test_measurement_comparison.py b/tests/test_measurement_comparison.py new file mode 100644 index 00000000..b289d751 --- /dev/null +++ b/tests/test_measurement_comparison.py @@ -0,0 +1,110 @@ +"""Scientific composition of real retained scalar tables, no native transport.""" +import copy +import struct +import unittest +from unittest.mock import patch + +import retained_measurements as owner +from retained_measurements import MeasurementComparison, MeasurementTable, encode +from test_retained_measurements import fixture, source +from workbench import WorkbenchError + + +def partition(subject): + document = fixture() + document['title'] = 'Original partition ' + subject + document['subjects'] = [item for item in document['subjects'] if item['id'] == subject] + document['observations'] = [row for row in document['observations'] if row['subject_id'] == subject] + return document + + +def selected(document, origin='a'*32): + table = MeasurementTable(encode(document)) + return dict(origin_catalog_id=origin, packet_sha256=table.sha256), table + + +class ComparisonTests(unittest.TestCase): + def setUp(self): + self.inputs = [selected(partition('a')), selected(partition('b'))] + self.comparison = MeasurementComparison(self.inputs) + + def test_complete_maxima_and_missing_denominators_keep_original_packet(self): + result = self.comparison.read('error') + self.assertEqual(len(result['subjects']), 2) + self.assertEqual(result['case_count'], 2) + self.assertEqual(result['observation_count'], 8) + self.assertEqual([s['maximum']['value'] for s in result['query']['summary']], [2e-15, .02]) + for row in result['query']['rows']: + locator, original = next(item for item in self.inputs if item[0] == row['source']) + evidence = original.inspect(row['id']) + self.assertEqual(struct.pack('!d', row['value']), struct.pack('!d', evidence['scalar_evidence']['value'])) + missing = self.comparison.read('latency')['query'] + self.assertEqual(missing['total'], 4) + self.assertEqual([s['states'] for s in missing['summary']], [ + dict(measured=0, failed=0, unsupported=0, unavailable=2), + dict(measured=0, failed=1, unsupported=0, unavailable=1)]) + self.assertTrue(all(s['maximum'] is None for s in missing['summary'])) + + def test_global_pages_span_packets_without_changing_maxima(self): + full = self.comparison.read('error')['query'] + rows = [] + for offset in range(5): + page = self.comparison.read('error', offset=offset, limit=1)['query'] + self.assertEqual(page['summary'], full['summary']) + self.assertEqual(page['total'], 4) + rows.extend(page['rows']) + self.assertEqual(rows, full['rows']) + self.assertEqual([r['subject_id'] for r in rows], ['a','a','b','b']) + ordered = self.comparison.read('error', subject_ids=['b','a'])['query'] + self.assertEqual([s['subject_id'] for s in ordered['summary']], ['b','a']) + + def test_density_domain_device_and_unknown_filters_are_separate(self): + density = self.comparison.read('error', filters={'cpm':16})['query'] + domain = self.comparison.read('error', filters={'grid_x':16})['query'] + self.assertEqual([s['maximum']['value'] for s in density['summary']], [1e-17,.01]) + self.assertEqual([s['maximum']['value'] for s in domain['summary']], [2e-15,.02]) + cpu = self.comparison.read('error', filters={'device':'CPU'})['query'] + self.assertEqual([s['count'] for s in cpu['summary']], [0,2]) + unknown = self.comparison.read('error', filters={'device':None})['query'] + self.assertEqual(unknown['total'], 0) + self.assertTrue(all(s['maximum'] is None for s in unknown['summary'])) + + def test_changed_original_case_evidence_or_dimension_meaning_refuses_comparison(self): + changed = partition('b') + context = fixture()['sources'][0] + import base64, json + original = json.loads(base64.b64decode(context['data_base64'])) + original['different_original_case'] = True + changed['sources'][0] = source('context', original) + for document in (changed, partition('b')): + if document is not changed: + document['dimensions'][0]['units'] = 'different units' + with self.assertRaises(WorkbenchError) as error: + MeasurementComparison([self.inputs[0], selected(document)]) + self.assertEqual(error.exception.code, 'incompatible') + + def test_overlapping_subjects_and_repeated_sources_are_explicit_errors(self): + with self.assertRaises(WorkbenchError): MeasurementComparison([self.inputs[0], self.inputs[0]]) + other = partition('a'); other['title'] = 'Different packet, same subject' + with self.assertRaises(WorkbenchError) as error: + MeasurementComparison([self.inputs[0], selected(other)]) + self.assertEqual(error.exception.code, 'conflict') + + def test_bounds_and_invalid_selections_never_return_partial_results(self): + for constant, maximum in [('MAX_COMPARISON_BYTES', 1), ('MAX_COMPARISON_SUBJECTS', 1), ('MAX_ROWS', 7)]: + with patch.object(owner, constant, maximum), self.assertRaises(WorkbenchError): + MeasurementComparison(self.inputs) + with self.assertRaises(WorkbenchError): MeasurementComparison(self.inputs * 9) + for arguments in ({'subject_ids':[['bad']]}, {'subject_ids':[]}, {'subject_ids':['a','a']}, + {'subject_ids':['missing']}, {'offset':True}, {'limit':101}): + with self.subTest(arguments=arguments), self.assertRaises(WorkbenchError): + self.comparison.read('error', **arguments) + with self.assertRaises(WorkbenchError): self.comparison.read(filters={}) + + def test_returned_metadata_and_query_values_have_no_mutable_owner_alias(self): + expected = self.comparison.read('error') + result = self.comparison.read('error') + result['query']['metric']['label'] = 'Changed' + result['query']['summary'][0]['states']['measured'] = 99 + result['subjects'][0]['source']['origin_catalog_id'] = 'b'*32 + self.assertEqual(self.comparison.read('error'), expected) diff --git a/tests/test_project_measurement_comparison_http.py b/tests/test_project_measurement_comparison_http.py new file mode 100644 index 00000000..09063754 --- /dev/null +++ b/tests/test_project_measurement_comparison_http.py @@ -0,0 +1,58 @@ +"""Real native HTTP source import, atomic comparison and scalar inspection.""" +import hashlib +import json +import os +import unittest + +import test_project_files_http as m1 +import test_project_files_m2_http as m2 +from retained_measurements import encode +from test_measurement_comparison import partition + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class ComparisonHTTP(unittest.TestCase): + cleanup_owned=m1.ProjectFilesHTTP.cleanup_owned + select_host=m1.ProjectFilesHTTP.select_host + request=m1.ProjectFilesHTTP.request + state=m1.ProjectFilesHTTP.state + envelope=m1.ProjectFilesHTTP.envelope + action=m1.ProjectFilesHTTP.action + create=m1.ProjectFilesHTTP.create + arguments=m2.ProjectFilesM2HTTP.arguments + transport=m2.ProjectFilesM2HTTP.transport + call=m2.ProjectFilesM2HTTP.call + def setUp(self): + self.operations=[] + m1.ProjectFilesHTTP.setUp(self) + + def test_two_original_sources_compare_and_inspect_through_real_HTTP(self): + self.create() + sources=[] + for subject, identity in [('a','b'*32),('b','c'*32)]: + raw=encode(partition(subject));digest=hashlib.sha256(raw).hexdigest() + filename=subject+'-measurements.json';manifest_name=subject+'-source.json' + (self.external/filename).write_bytes(raw) + manifest=dict(schema_version=1,id=identity,name='Original '+subject, + data=[dict(id='scalars',label='Retained scalars',path=filename,sha256=digest)], + recipes=[],evidence=[],requirements=[]) + (self.external/manifest_name).write_text(json.dumps(manifest)) + registered=self.call('source_register',root='fixture',manifest_path=manifest_name) + snapshot={key:registered[key] for key in ('origin_catalog_id','project_id','manifest_sha256')} + admitted=self.call('measurement_import',snapshot=snapshot,reference_id='scalars',expected_sha256=digest) + sources.append(dict(origin_catalog_id=admitted['origin_catalog_id'],packet_sha256=digest)) + before=self.workspace._projection.session.snapshot + metadata=self.call('measurement_compare',sources=sources)['value'] + self.assertEqual([s['id'] for s in metadata['subjects']],['a','b']) + self.assertIsNone(metadata['query']) + query=self.call('measurement_compare',sources=sources,metric_id='error',offset=1,limit=2)['value']['query'] + self.assertEqual(query['total'],4) + self.assertEqual([s['maximum']['value'] for s in query['summary']],[2e-15,.02]) + self.assertEqual([r['subject_id'] for r in query['rows']],['a','b']) + maximum=query['summary'][1] + inspected=self.call('measurement_inspect',**maximum['source'],observation_id=maximum['maximum']['id'])['value'] + self.assertEqual(inspected['scalar_evidence']['value'],.02) + self.assertEqual(inspected['case']['dimensions']['cpm'],8) + missing=self.call('measurement_compare',sources=sources,metric_id='latency')['value']['query'] + self.assertTrue(all(s['maximum'] is None for s in missing['summary'])) + self.assertEqual(self.workspace._projection.session.snapshot,before) diff --git a/tests/test_project_measurements_clients.py b/tests/test_project_measurements_clients.py index bc415313..e05b6b28 100644 --- a/tests/test_project_measurements_clients.py +++ b/tests/test_project_measurements_clients.py @@ -4,7 +4,7 @@ from unittest.mock import patch import agent_api as api -from retained_measurements import MeasurementTable, encode +from retained_measurements import MeasurementTable, MeasurementComparison, encode from test_project_files_clients import BINDING, PROJECT, CHECKPOINT, OTHER, bound, client from test_retained_measurements import fixture from tuldok_client import ProjectFilesClient @@ -24,6 +24,23 @@ def sdk(self,value): return ProjectFilesClient(client({'status':200,'result':dict(binding=BINDING,project_id=PROJECT, checkpoint=CHECKPOINT,value=value)})) + def test_group_comparison_header_and_query_correlate_every_original_source(self): + locator=dict(origin_catalog_id=PROJECT,packet_sha256=self.table.sha256) + comparison=MeasurementComparison([(locator,self.table)]) + args=dict(binding=BINDING,checkpoint=CHECKPOINT,sources=[locator]) + page=dict(sources=[self.source],value=comparison.read()) + self.assertEqual(self.sdk(page).measurement_compare(**args)['value'],page) + query=dict(sources=[self.source],value=comparison.read('error',offset=1,limit=1)) + request=dict(args,metric_id='error',offset=1,limit=1) + self.assertEqual(self.sdk(query).measurement_compare(**request)['value'],query) + for mutate in (lambda p:p['sources'][0].update(id=OTHER), + lambda p:p['value']['query'].update(offset=0), + lambda p:p['value']['query']['summary'][0]['states'].update(measured=99), + lambda p:p['value']['query']['rows'][0]['source'].update(packet_sha256=OTHER), + lambda p:p['value']['subjects'][0]['source'].update(origin_catalog_id='f'*32)): + bad=copy.deepcopy(query);mutate(bad) + with self.assertRaises(api.AgentAPIError):self.sdk(bad).measurement_compare(**request) + def test_capability_bound_applies_to_the_actual_advertised_profile(self): sizes={profile:len(api.capabilities(PROJECT,profile=profile)['operations']) for profile in ('catalog','project')} maximum=max(sizes.values()) diff --git a/tuldok_client.py b/tuldok_client.py index ce838bcb..be0daf7f 100644 --- a/tuldok_client.py +++ b/tuldok_client.py @@ -417,6 +417,10 @@ def measurement_subjects(self, **arguments): def measurement_query(self, **arguments): return self._call('measurement_query', arguments) + def measurement_compare(self, **arguments): + """Compare compatible retained sources at one selected checkpoint.""" + return self._call('measurement_compare', arguments) + def measurement_inspect(self, **arguments): return self._call('measurement_inspect', arguments) From 7d73fe7a99e883bbcabf2d06a7c418f98bad11ba Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 02:43:24 -0700 Subject: [PATCH 06/33] Keep applied comparison visible while paging its subject selector --- static/project-measurements.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/project-measurements.js b/static/project-measurements.js index 314d3f33..64e3f9c0 100644 --- a/static/project-measurements.js +++ b/static/project-measurements.js @@ -308,7 +308,7 @@ invalidate(); subjectPage(subjectOffset); controls(); }; for (const [name, direction] of [['subjects-prev',-1],['subjects-next',1]]) $(name).onclick = () => { - clearResults(); void perform(() => subjectPage(subjectOffset + direction * 40)); + void perform(() => subjectPage(subjectOffset + direction * 40)); }; $('form').onsubmit = event => { event.preventDefault(); void compare(); }; $('prev').onclick = () => void compare(Math.max(0, query.offset - query.limit), applied); From e0b8948a9a92828644788dac3dfb8f934fa2a30d Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 04:04:15 -0700 Subject: [PATCH 07/33] Index extraction diagnostics while retaining their original order and meaning --- project_extraction.py | 13 ++++++--- tests/test_project_extraction.py | 46 ++++++++++++++++++++++++++++++-- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/project_extraction.py b/project_extraction.py index 79872c6d..a4b53760 100644 --- a/project_extraction.py +++ b/project_extraction.py @@ -60,6 +60,8 @@ def __init__(self, source): self.origin = data_codec.identity(self.db.dataset_id) self.records, self.inventory, self.dependencies = {}, {}, {} self.blockers, self.warnings, self.working_files = [], [], [] + # Preserve emitted order without scanning prior runtime diagnostics. + self._issue_keys = {False: set(), True: set()} self.anchor = None self.queue, self.visited, self.queued = deque(), set(), set() self.snapshot_by_id = {} @@ -121,10 +123,13 @@ def need(self, family, identity, parent=None, reason='required scientific depend return target def issue(self, key, code, detail, *, warning=False): - value = dict(family=key[0], id=key[1], code=code, detail=str(detail)[:1000]) - destination = self.warnings if warning else self.blockers - if value not in destination: - destination.append(value) + detail = str(detail)[:1000] + identity = key[0], key[1], code, detail + seen = self._issue_keys[bool(warning)] + if identity not in seen: + seen.add(identity) + destination = self.warnings if warning else self.blockers + destination.append(dict(family=key[0], id=key[1], code=code, detail=detail)) def add(self, record): key = record['family'], record['id'] diff --git a/tests/test_project_extraction.py b/tests/test_project_extraction.py index 59d4ab79..e4ce169b 100644 --- a/tests/test_project_extraction.py +++ b/tests/test_project_extraction.py @@ -9,6 +9,7 @@ import os from pathlib import Path import tempfile +import threading from types import SimpleNamespace import unittest from unittest.mock import patch @@ -20,14 +21,55 @@ from metadata import Metadata from model_definitions import ModelDefinitions from project_data import decode_data_records -from project_extraction import capture_legacy +from project_extraction import _Capture, capture_legacy from project_material import decode_records as decode_material -from project_transfer import ReadonlyCatalog, SourceView +from project_transfer import CapturedArtifacts, ReadonlyCatalog, SourceView from projects import Projects from saved_searches import SavedSearches from workbench import Workbench, WorkbenchError +class CaptureDiagnostics(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-capture-diagnostics-') + self.addCleanup(temporary.cleanup) + # These constructors assign owner references only. This fixture never + # calls a data reader, SQL, a model runtime or the source capability. + self.source = SimpleNamespace( + db=SimpleNamespace(dataset_id='a' * 32, path=Path(temporary.name), lock=threading.RLock()), + snapshot=dict(project_id='a' * 32, records=[]), artifacts=CapturedArtifacts()) + self.capture = _Capture(self.source) + + def test_severity_parent_family_and_code_keep_distinct_diagnostics(self): + capture = self.capture + parent = 'run_history', 'b' * 32 + capture.issue(parent, 'missing_source', 'Original source unavailable', warning=True) + capture.issue(parent, 'missing_source', 'Original source unavailable', warning=True) + capture.issue(parent, 'missing_source', 'Original source unavailable') + capture.issue(('study_history', parent[1]), 'missing_source', 'Original source unavailable', warning=True) + capture.issue(('run_history', 'c' * 32), 'missing_source', 'Original source unavailable', warning=True) + capture.issue(parent, 'unsupported', 'Original source unavailable', warning=True) + self.assertEqual(capture.warnings, [ + dict(family='run_history', id=parent[1], code='missing_source', detail='Original source unavailable'), + dict(family='study_history', id=parent[1], code='missing_source', detail='Original source unavailable'), + dict(family='run_history', id='c' * 32, code='missing_source', detail='Original source unavailable'), + dict(family='run_history', id=parent[1], code='unsupported', detail='Original source unavailable')]) + self.assertEqual(capture.blockers, [capture.warnings[0]]) + + def test_retained_detail_normalization_and_first_observation_order(self): + capture = self.capture + parent = 'run_history', 'b' * 32 + prefix = 'λ' * 1000 + capture.issue(parent, 'missing_source', prefix + 'first', warning=True) + capture.issue(parent, 'missing_source', prefix + 'second', warning=True) + capture.issue(parent, 'missing_source', 42, warning=True) + capture.issue(parent, 'missing_source', '42', warning=True) + self.assertEqual(capture.warnings, [ + dict(family='run_history', id=parent[1], code='missing_source', detail=prefix), + dict(family='run_history', id=parent[1], code='missing_source', detail='42')]) + self.assertEqual(capture.blockers, []) + + @unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') class LegacyCapture(unittest.TestCase): def setUp(self): From 0afbb1171f3893167fe5a0df608fa718915904e1 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 04:24:44 -0700 Subject: [PATCH 08/33] Read explicitly bound original manifests without reviving historical root authority --- project_extraction.py | 12 ++++++--- project_transfer.py | 12 +++++++++ tests/test_project_extraction.py | 46 ++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/project_extraction.py b/project_extraction.py index a4b53760..bc27b2ab 100644 --- a/project_extraction.py +++ b/project_extraction.py @@ -379,10 +379,14 @@ def capture_snapshot(self, sha, key): if len(matches) != 1: _missing('An exact selected project snapshot is not available in this pinned source.') saved = matches[0] - root = self.source.roots.get(saved['root']) - if root is None or list(root[1]) != saved['root_identity']: - _missing('The retained registration has no matching current explicit source-root capability.') - raw = self.source.read_root(saved['root'], saved['manifest_path'], 65536) + # Explicitly recovered bytes retain the original registration as inert + # history. Its former directory/inode is not current read authority. + raw = self.source.read_bound_payload(sha, maximum_bytes=65536) + if raw is None: + root = self.source.roots.get(saved['root']) + if root is None or list(root[1]) != saved['root_identity']: + _missing('The retained registration has no matching current explicit source-root capability.') + raw = self.source.read_root(saved['root'], saved['manifest_path'], 65536) if hashlib.sha256(raw).hexdigest() != sha or manifest(decode_manifest(raw)) != saved['manifest']: raise WorkbenchError('Original registered manifest bytes differ.', 'corrupt_payload', 409) blob = self.raw(key, raw, 'project_manifest') diff --git a/project_transfer.py b/project_transfer.py index 932d0963..10a6b431 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -226,6 +226,18 @@ def resolve_payload(self,sha256,size): raw=self.read_root(location['root'],location['relative'],MAX_SOURCE_FILE) if len(raw)!=size or digest(raw)!=sha256:fail('External source payload differs from its declaration.') return raw + def read_bound_payload(self,sha256,*,maximum_bytes=MAX_SOURCE_FILE): + """Read exact operator-bound bytes; None means no explicit binding. + + Historical paths or root identities never create this capability. A + changed, oversized or unavailable bound file remains an error, even if + another working file exists at the original historical location. + """ + self.require();sha(sha256) + location=self._payload_locations.get(sha256) + if location is None:return None + if location['bytes']>maximum_bytes:fail('Bound original source exceeds its owner bound.','too_large',413) + return self.resolve_payload(sha256,location['bytes']) @contextmanager def payload_fd(self,sha256,size): sha(sha256) diff --git a/tests/test_project_extraction.py b/tests/test_project_extraction.py index e4ce169b..368fbb88 100644 --- a/tests/test_project_extraction.py +++ b/tests/test_project_extraction.py @@ -127,6 +127,52 @@ def capture(self, source, roots): patch('runs.Runs.__init__', side_effect=AssertionError('Extraction constructed Runs')): return capture_legacy(source, selection) + def bind_manifest_copy(self, source): + recovered = self.root / 'recovered-original' + recovered.mkdir() + path = recovered / 'manifest.json' + path.write_bytes(self.manifest_raw) + descriptor = os.open(recovered, os.O_RDONLY | os.O_DIRECTORY) + info = os.fstat(descriptor) + source.roots['recovered'] = descriptor, [info.st_dev, info.st_ino] + source.bind_payload(hashlib.sha256(self.manifest_raw).hexdigest(), len(self.manifest_raw), + 'recovered', 'manifest.json') + return path + + def test_explicit_original_manifest_binding_keeps_history_without_old_root_authority(self): + source = self.view() + self.bind_manifest_copy(source) + descriptor, _ = source.roots.pop('fixture') + os.close(descriptor) + captured = self.capture(source, {}) + self.assertEqual(captured['blockers'], []) + snapshot = next(row['document'] for row in captured['records'] if row['family'] == 'project_snapshot') + self.assertEqual(snapshot['value']['root'], 'fixture') + self.assertEqual(snapshot['value']['manifest'], self.manifest) + declaration = next(row['document'] for row in captured['records'] if row['family'] == 'project_artifact') + self.assertEqual(source.artifacts.read_blob(declaration), self.manifest_raw) + self.assertNotIn('fixture', source.roots) + + def test_explicit_original_manifest_bytes_do_not_read_changed_working_manifest(self): + source = self.view() + self.bind_manifest_copy(source) + changed = b'{"unrelated_current_working_file":true}\n' + (self.project_root / 'source.json').write_bytes(changed) + captured = self.capture(source, {}) + self.assertEqual(captured['blockers'], []) + declaration = next(row['document'] for row in captured['records'] if row['family'] == 'project_artifact') + self.assertEqual(source.artifacts.read_blob(declaration), self.manifest_raw) + self.assertEqual((self.project_root / 'source.json').read_bytes(), changed) + + def test_changed_explicit_manifest_binding_blocks_instead_of_using_working_file(self): + source = self.view() + recovered = self.bind_manifest_copy(source) + recovered.write_bytes(b'changed after explicit binding') + captured = self.capture(source, {}) + self.assertTrue(any(row['code'] == 'stale_source' for row in captured['blockers'])) + self.assertFalse(any(row['family'] == 'project_snapshot' for row in captured['records'])) + self.assertEqual((self.project_root / 'source.json').read_bytes(), self.manifest_raw) + def test_text_responses_history_and_unrelated_inventory_are_read_only(self): selected = self.text();unrelated = self.text('Unrelated original data') response_id = uuid.uuid4().hex From 1d76b47797bb681dc0ed9411299096af498995b8 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 05:09:35 -0700 Subject: [PATCH 09/33] Reuse owned material validation within each fixed candidate snapshot --- project_extraction.py | 3 +-- project_material.py | 46 ++++++++++++++++++++++++++++------ project_projection.py | 17 +++++-------- tests/test_project_material.py | 35 +++++++++++++++++++++++++- 4 files changed, 80 insertions(+), 21 deletions(-) diff --git a/project_extraction.py b/project_extraction.py index bc27b2ab..b03ec2d9 100644 --- a/project_extraction.py +++ b/project_extraction.py @@ -760,8 +760,7 @@ def validate(self): if self.blockers:return try: data_codec.decode_data_records(records) - material_codec.decode_records(records) - materials = material_codec.ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=self.origin, records=records), check=lambda: None)) + materials = material_codec.MaterialSnapshot(records) history_codec.decode_records(records, materials=materials) producer_codec.decode_records(records, artifacts=self.source.artifacts) except WorkbenchError as error: diff --git a/project_material.py b/project_material.py index 530ce34a..c9411142 100644 --- a/project_material.py +++ b/project_material.py @@ -315,6 +315,44 @@ def validate_operation_result(operation, result, records=None): return True +def _retained_value(get_documents, family, ref): + catalog_id = _catalog(ref['catalog_id']) + reference(ref, {'model_material':'model', 'recipe_material':'recipe_material', + 'runtime_material':'runtime'}[family], catalog_id) + key = catalog_id + '.' + ref['sha256'] + document = get_documents().get((family, key)) + if document is None: + raise WorkbenchError('Retained material is unavailable.', 'unavailable', 409) + if document['value']['ref'] != ref: + raise WorkbenchError('Retained material reference differs.', 'invalid', 400) + return copy.deepcopy(document['value']) + + +class MaterialSnapshot: + """Owned, validated material facts for one fixed candidate validation. + + The decoder copies the selected documents. Neither input mutation nor a + returned value can alter this view. It has no live Session, artifact reader, + publication operation or current execution authority. + """ + @_boundary + def __init__(self, records): + self.__documents = decode_records(records) + + @_boundary + def recipe(self, ref): + return _retained_value(lambda: self.__documents, 'recipe_material', ref) + + @_boundary + def runtime(self, ref): + return _retained_value(lambda: self.__documents, 'runtime_material', ref) + + def artifact_blobs(self): + for document in self.__documents.values(): + for edge in document.get('artifacts', []): + yield copy.deepcopy(edge['blob']) + + class ProjectMaterial: def __init__(self, session): self.session = session @@ -356,13 +394,7 @@ def _require_resolved(self, family, identity): @_boundary def _value(self, family, ref): - catalog_id = _catalog(ref['catalog_id']) - reference(ref,{'model_material':'model','recipe_material':'recipe_material','runtime_material':'runtime'}[family],catalog_id) - key = catalog_id + '.' + ref['sha256'] - document = self._documents().get((family, key)) - if document is None: raise WorkbenchError('Retained material is unavailable.', 'unavailable', 409) - if document['value']['ref'] != ref: raise WorkbenchError('Retained material reference differs.', 'invalid', 400) - return copy.deepcopy(document['value']) + return _retained_value(self._documents, family, ref) @_boundary def model(self, ref, *, verify=False): diff --git a/project_projection.py b/project_projection.py index e8a81a40..a9ca2f94 100644 --- a/project_projection.py +++ b/project_projection.py @@ -15,8 +15,7 @@ from project_history import ProjectHistory, FAMILY_FORMATS as HISTORY_FAMILIES from project_history import decode_records as decode_history_records from project_history import validate_operation_result as validate_history_result -from project_material import ProjectMaterial, FAMILIES as MATERIAL_FAMILIES -from project_material import decode_records as decode_material_records +from project_material import ProjectMaterial, MaterialSnapshot, FAMILIES as MATERIAL_FAMILIES from project_material import validate_operation_result as validate_material_result from project_producers import ProjectProducers, FAMILIES as PRODUCER_FAMILIES from project_producers import decode_records as decode_producer_records @@ -151,18 +150,14 @@ def validate_records(self, records, *, project_id=None): operation_records(records, project_id=project_id) decode_file_records(records) decode_native_records(records,artifacts=self.artifacts) - materials = decode_material_records(records) - for document in materials.values(): - for edge in document.get('artifacts', []): - blob = edge['blob'] - if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): - raise WorkbenchError('Material artifact declaration is missing or inconsistent.') + materials = MaterialSnapshot(records) + for blob in materials.artifact_blobs(): + if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): + raise WorkbenchError('Material artifact declaration is missing or inconsistent.') # Resolve every cross-owner reference against THIS candidate, rather # than a resolver that still sees the preceding committed checkpoint. - candidate = SimpleNamespace(snapshot=dict(project_id=project_id, records=records), - artifacts=self.artifacts, check=lambda: None) decode_producer_records(records, artifacts=self.artifacts) - decode_history_records(records, materials=ProjectMaterial(candidate), artifacts=self.artifacts) + decode_history_records(records, materials=materials, artifacts=self.artifacts) decode_measurement_records(records, artifacts=self.artifacts) for receipt in operation_documents(records): operation, result = receipt['operation'], receipt['result'] diff --git a/tests/test_project_material.py b/tests/test_project_material.py index 8893d6ae..4fc298d3 100644 --- a/tests/test_project_material.py +++ b/tests/test_project_material.py @@ -20,7 +20,7 @@ from model_definitions import ModelDefinitions, blob, decode_model_draft, decode_model_material, verify_model_closure from pipeline import digest, decode_recipe_material, execution_digest, verify_recipe_closure from project_artifacts import ProjectArtifacts, ProjectArtifactError, validate_declaration -from project_material import ProjectMaterial, decode_records, validate_operation_result +from project_material import ProjectMaterial, MaterialSnapshot, decode_records, validate_operation_result from project_store import ProjectStore from project_session import ProjectSession, operation_records from workbench import WorkbenchError @@ -123,6 +123,30 @@ def test_runtime_environment_is_historical_and_does_not_probe_origin_paths(self) with patch('subprocess.Popen',side_effect=AssertionError('no process')),patch('os.stat',side_effect=AssertionError('no host paths')): self.assertEqual(self.material.runtime(ref,verify=True)['environment'],environment) + def test_fixed_material_snapshot_owns_input_and_returns_independent_values(self): + environment={'python':'/unavailable/original/python','packages':{'neutral':'1.0'},'definition_files':{'adapter.py':'a'*64}} + sha=digest(environment);ref=dict(kind='runtime',catalog_id=self.catalog,id=sha,sha256=sha) + original=dict(ref=ref,descriptor=None,environment=environment,manifest=None) + record=self.material.capture_runtime(original) + records=[record];view=MaterialSnapshot(records) + records[0]['document']['value']['environment']['packages']['neutral']='changed input' + value=view.runtime(ref);self.assertEqual(value,original) + value['environment']['packages']['neutral']='changed return' + self.assertEqual(view.runtime(ref),original) + with self.assertRaises(WorkbenchError):MaterialSnapshot(records) + + def test_fixed_material_snapshot_checks_complete_reference_identity(self): + environment={'python':'/unavailable/original/python','packages':{},'definition_files':{}} + sha=digest(environment);ref=dict(kind='runtime',catalog_id=self.catalog,id=sha,sha256=sha) + record=self.material.capture_runtime(dict(ref=ref,descriptor=None,environment=environment,manifest=None)) + view=MaterialSnapshot([record]) + for change in ({'catalog_id':'f'*32},{'sha256':'f'*64},{'id':'f'*64},{'kind':'recipe_material'}): + with self.assertRaises(WorkbenchError):view.runtime(dict(ref,**change)) + + def test_invalid_live_material_reference_precedes_snapshot_access(self): + with patch.object(self.session,'check',side_effect=AssertionError('Invalid reference accessed snapshot')): + with self.assertRaises(WorkbenchError):self.material.runtime({'catalog_id':'invalid'}) + def test_operation_result_requires_exact_retained_revision_or_material(self): first=definition(self.catalog);record=self.material.capture_definition('model_definition',first,[],action='models.create') changed=copy.deepcopy(first);changed.update(revision=2,name='Changed') @@ -238,6 +262,15 @@ def read(ref,item,maximum):return (payloads if ref['kind']=='model' else recipe_ dependencies=self.material.capture_recipe_dependencies(dict(document={'value':retained}),[]) self.assertEqual(dependencies[-1]['document']['value']['environment'],runtime) self.assertEqual(self.material.capture_recipe_dependencies(dict(document={'value':retained}),dependencies),[]) + recipe_root=self.root/'retained-recipe';target=recipe_root/retained['ref']['sha256'];target.mkdir(parents=True) + for name,raw in recipe_payloads.items():(target/name).write_bytes(raw) + record=self.material.capture_recipe_material(retained,recipe_root) + view=MaterialSnapshot([record,*dependencies]) + self.assertEqual(view.recipe(retained['ref']),retained) + expected=[edge['blob'] for item in [record,*dependencies] for edge in item['document']['artifacts']] + self.assertEqual(list(view.artifact_blobs()),expected) + changed=list(view.artifact_blobs());changed[0]['bytes']+=1 + self.assertEqual(list(view.artifact_blobs()),expected) for field,replacement in (('data',{}),('executable_sha256','wrong'),('captures',[dict(item,media_type='text/plain')])): bad=copy.deepcopy(retained);bad['material'][field]=replacement changed=digest(bad['material']);bad['ref'].update(id=changed,sha256=changed) From a3c94728891e85416735c671a4635e95e06b524a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 06:49:00 -0700 Subject: [PATCH 10/33] Support scientific portfolios within a 2 GiB canonical record budget --- docs/contracts/project-files.md | 7 +++++-- project_store.py | 4 +++- tests/test_project_store.py | 27 +++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/contracts/project-files.md b/docs/contracts/project-files.md index 7959d37d..e04aa7ac 100644 --- a/docs/contracts/project-files.md +++ b/docs/contracts/project-files.md @@ -32,9 +32,12 @@ request receipts and a SHA256 checkpoint. Its digest covers canonical bytes of the document with `checkpoint` omitted. The store supports only its declared format and family; malformed and future versions fail before projection writes. The canonical checkpoint index is limited to 64 MiB, separately from the -100,000-record, 40 MiB-per-record and 256 MiB-total-record limits. Commit checks +100,000-record, 40 MiB-per-record and 2 GiB-total-record limits. The aggregate +budget covers canonical record bytes, excluding artifact payloads; scientific +catalogs retain original runtime/file observations across many Runs. Commit +and cold Open enforce the same aggregate budget. Commit checks the index size before staging immutable records. This format is unchanged; -older installations with the former 16 MiB index limit cannot open larger +older installations with the former 16 MiB index or 256 MiB aggregate limit cannot open larger checkpoints and must be upgraded before consuming them. Verified stopped-source backups and ordinary catalog backups share identical diff --git a/project_store.py b/project_store.py index 60a12b81..31a184bf 100644 --- a/project_store.py +++ b/project_store.py @@ -62,7 +62,9 @@ 'text_proposal_history': 'project_text_proposal_history_v1', } MAX_RECORDS = 100_000 -MAX_TOTAL_BYTES = 256 * 1024 * 1024 +# Scientific history can retain original runtime/file observations in hundreds +# of Runs. Bound the whole local checkpoint separately from each document. +MAX_TOTAL_BYTES = 2 * 1024 * 1024 * 1024 IDENTIFIER = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$') REQUEST_KEY = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') DOMAIN_FORMAT = re.compile(r'^[a-z][a-z0-9_]{0,95}_v([1-9][0-9]*)$') diff --git a/tests/test_project_store.py b/tests/test_project_store.py index d3d93570..9f321617 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -308,6 +308,33 @@ def test_closed_document_envelopes_and_supported_family_format(self): for value in invalid: with self.subTest(value=repr(value)): self.assert_code('invalid', self.store.commit, [value], checkpoint, 'bad', body('bad')) + + def test_portfolio_above_256_MiB_commits_and_cold_reopens(self): + # Each document fits the 40 MiB owner bound. Aggregate project metadata + # must also support scientific catalogs larger than the old 256 MiB. + text = 'p' * (30 * 1024 * 1024) + values = [record(str(index), text) for index in range(9)] + committed = self.store.commit(values, self.store.snapshot()['checkpoint'], + 'large-portfolio', body('large-portfolio')) + self.assertGreater(sum(row['bytes'] for row in self.manifest()['inventory']), + 256 * 1024 * 1024) + self.assertEqual(self.reopen(), committed) + self.assertEqual(self.store.snapshot()['records'], values) + + def test_aggregate_byte_bound_rejects_commit_and_cold_open(self): + values = [record('a'), record('b')] + total = sum(len(canonical(value['document'])) for value in values) + before = self.bytes_tree() + checkpoint = self.store.snapshot()['checkpoint'] + with patch.object(store_module, 'MAX_TOTAL_BYTES', total - 1): + self.assert_code('invalid', self.store.commit, values, checkpoint, + 'over-budget', body('over-budget')) + self.assertEqual(self.bytes_tree(), before) + committed = self.commit(values) + self.store.close() + with patch.object(store_module, 'MAX_TOTAL_BYTES', total - 1): + self.assert_code('invalid', ProjectStore.open, self.root) + self.assertEqual(self.reopen(), committed) for value in [dict(record(), family='access_credentials'), dict(record(), document={'format': 'workbench_text_v2'}), dict(record(), document={'format': 'unknown_v1'})]: From 195ba9b262f179e12542c0946aa525141a0e1e90 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 07:12:35 -0700 Subject: [PATCH 11/33] Read historical material closure once per selected checkpoint --- project_history.py | 3 ++- project_material.py | 12 +++++++++++- tests/test_project_history.py | 20 ++++++++++++++++++++ tests/test_project_material.py | 15 +++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/project_history.py b/project_history.py index 2fec6803..eb14389a 100644 --- a/project_history.py +++ b/project_history.py @@ -738,7 +738,8 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): def _facts(self): self.session.check() - return decode_records(self.session.snapshot['records'],materials=self.materials,artifacts=self.artifacts) + records=self.session.snapshot['records'] + return decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) def _authorize(self,action,target=None): self.access.require('inspect',action,target) diff --git a/project_material.py b/project_material.py index c9411142..9d26fea6 100644 --- a/project_material.py +++ b/project_material.py @@ -329,7 +329,7 @@ def _retained_value(get_documents, family, ref): class MaterialSnapshot: - """Owned, validated material facts for one fixed candidate validation. + """Owned, validated material facts for one fixed checkpoint read. The decoder copies the selected documents. Neither input mutation nor a returned value can alter this view. It has no live Session, artifact reader, @@ -362,6 +362,16 @@ def _documents(self): self.session.check() return decode_records(self.session.snapshot['records']) + def read_view(self, records): + """Own validated material facts from one guarded checkpoint read. + + The history reader supplies its selected records. This view carries no + live publication, verification or execution capability; later reads + must select their own checkpoint and construct a new view. + """ + self.session.check() + return MaterialSnapshot(records) + @_boundary def intents(self): """Inspect unresolved active definitions; they confer no execution binding.""" diff --git a/tests/test_project_history.py b/tests/test_project_history.py index c505f663..700504cd 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -60,6 +60,8 @@ def trial(identity,study_id,origin=ORIGIN,parent=None,baseline=None): class Materials: """Only material lookup is substituted; history/lineage validators are real.""" + def read_view(self, records): + return self def recipe(self,ref,*,verify=False): if ref!={'kind':'recipe_material','catalog_id':ORIGIN,'id':MATERIAL,'sha256':MATERIAL}: raise WorkbenchError('Unknown exact fixture material.','unavailable',409) @@ -124,6 +126,23 @@ def test_cross_study_parent_baseline_and_origin_namespace_remain_exact(self): changed=copy.deepcopy(records);changed[-1]['document']['lineage']['intent']['baseline']['catalog_id']=OTHER with self.assertRaises(WorkbenchError):decode_records(changed,materials=Materials()) + def test_history_read_uses_one_material_view_of_the_selected_records(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + observed=[] + class Provider: + def read_view(self, selected): + observed.append(selected) + return Materials() + def recipe(self, *args, **kwargs): + raise AssertionError('History reread the live material catalog') + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), + artifacts=None,access=SimpleNamespace(),check=lambda:None) + facts=ProjectHistory(session,materials=Provider())._facts() + self.assertEqual(len(observed),1) + self.assertIs(observed[0],records) + self.assertEqual(facts.trials[(ORIGIN,identity)]['trial'],records[1]['document']['trial']) + def test_closure_omissions_cycles_and_history_completeness_are_rejected(self): parent_id='9'*32;child_id='a'*32 ref=lambda identity:{'catalog_id':ORIGIN,'trial_id':identity,'material_sha256':MATERIAL} @@ -370,6 +389,7 @@ def test_metadata_edits_cold_reopen_keep_history_without_active_authority(self): def test_trial_and_cross_study_lineage_checkpoint_preserve_previous_revision(self): class OwnedMaterial: def __init__(self,origin):self.origin=origin + def read_view(self,records):return self def recipe(self,ref,*,verify=False): self.ref=ref if ref!={'kind':'recipe_material','catalog_id':self.origin,'id':MATERIAL,'sha256':MATERIAL}: diff --git a/tests/test_project_material.py b/tests/test_project_material.py index 4fc298d3..cba2f75b 100644 --- a/tests/test_project_material.py +++ b/tests/test_project_material.py @@ -147,6 +147,21 @@ def test_invalid_live_material_reference_precedes_snapshot_access(self): with patch.object(self.session,'check',side_effect=AssertionError('Invalid reference accessed snapshot')): with self.assertRaises(WorkbenchError):self.material.runtime({'catalog_id':'invalid'}) + def test_material_read_view_uses_supplied_records_without_resnapshotting(self): + environment={'python':'/unavailable/original/python','packages':{},'definition_files':{}} + sha=digest(environment);ref=dict(kind='runtime',catalog_id=self.catalog,id=sha,sha256=sha) + original=dict(ref=ref,descriptor=None,environment=environment,manifest=None) + records=[self.material.capture_runtime(original)] + self.assertEqual(self.session.snapshot['records'],[]) + view=self.material.read_view(records) + self.assertEqual(view.runtime(ref),original) + self.assertEqual(self.session.snapshot['records'],[]) + + def test_material_read_view_rechecks_session_availability(self): + with patch.object(self.session,'check',side_effect=WorkbenchError('Stale checkpoint.','stale',409)): + with self.assertRaisesRegex(WorkbenchError,'Stale checkpoint'): + self.material.read_view([]) + def test_operation_result_requires_exact_retained_revision_or_material(self): first=definition(self.catalog);record=self.material.capture_definition('model_definition',first,[],action='models.create') changed=copy.deepcopy(first);changed.update(revision=2,name='Changed') From d582987a9d03d1b2bef98be9659ea85eaaa805f2 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 07:23:27 -0700 Subject: [PATCH 12/33] Create every captured runtime directory with private permissions --- runtime_snapshots.py | 9 ++++++++- tests/test_runtime_snapshots.py | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/runtime_snapshots.py b/runtime_snapshots.py index 09ce69ca..e1795760 100644 --- a/runtime_snapshots.py +++ b/runtime_snapshots.py @@ -316,7 +316,14 @@ def create(self,config,expected_inventory_sha256,*,quiesced=False,reserve_bytes= try: for index,row in enumerate(manifest['files']): if index%128==0 and shutil.disk_usage(self.path).free Date: Sat, 24 Oct 2026 08:42:58 -0700 Subject: [PATCH 13/33] Retain legacy activity manifest fields without enabling current activity access --- project_extraction.py | 4 ++-- project_material.py | 8 ++++---- projects.py | 18 +++++++++++++++++- tests/test_project_extraction.py | 18 ++++++++++++++++++ tests/test_project_material.py | 21 +++++++++++++++++++++ 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/project_extraction.py b/project_extraction.py index b03ec2d9..82db29c7 100644 --- a/project_extraction.py +++ b/project_extraction.py @@ -18,7 +18,7 @@ import project_material as material_codec import project_history as history_codec import project_producers as producer_codec -from projects import decode_retained_snapshot, manifest, decode as decode_manifest, relative_path +from projects import decode_retained_snapshot, decode_retained_manifest, decode as decode_manifest, relative_path from project_files import file_id, MAX_FILE_BYTES MAX_PAYLOAD = 40 * 1024 * 1024 @@ -387,7 +387,7 @@ def capture_snapshot(self, sha, key): if root is None or list(root[1]) != saved['root_identity']: _missing('The retained registration has no matching current explicit source-root capability.') raw = self.source.read_root(saved['root'], saved['manifest_path'], 65536) - if hashlib.sha256(raw).hexdigest() != sha or manifest(decode_manifest(raw)) != saved['manifest']: + if hashlib.sha256(raw).hexdigest() != sha or decode_retained_manifest(decode_manifest(raw)) != saved['manifest']: raise WorkbenchError('Original registered manifest bytes differ.', 'corrupt_payload', 409) blob = self.raw(key, raw, 'project_manifest') record = dict(family='project_snapshot', id=key[1], document=dict(format=material_codec.FAMILIES['project_snapshot'], diff --git a/project_material.py b/project_material.py index 9d26fea6..4ddd29a2 100644 --- a/project_material.py +++ b/project_material.py @@ -453,10 +453,10 @@ def snapshot_ref(self, project_id, manifest_sha256, *, catalog_id=None, verify=F if len(candidates) != 1: raise WorkbenchError('Retained project snapshot is unavailable or ambiguous.', 'unavailable', 409) doc = candidates[0] if verify: - from projects import decode, manifest + from projects import decode, decode_retained_manifest edge = doc['artifacts'][0]['blob'] raw = self.session.artifacts.read_blob(edge, maximum_bytes=64*1024) - if manifest(decode(raw)) != doc['value']['manifest']: + if decode_retained_manifest(decode(raw)) != doc['value']['manifest']: raise WorkbenchError('Retained project manifest differs.', 'invalid', 400) return copy.deepcopy(doc['value']) @@ -589,9 +589,9 @@ def capture_recipe_dependencies(self,record,before_records,*,runtime_manifest=No @_boundary def capture_snapshot(self, value, raw, *, catalog_id): - from projects import decode_retained_snapshot,decode,manifest + from projects import decode_retained_snapshot,decode,decode_retained_manifest value=decode_retained_snapshot(value);_catalog(catalog_id) - if type(raw) is not bytes or len(raw)>64*1024 or hashlib.sha256(raw).hexdigest()!=value['manifest_sha256'] or manifest(decode(raw))!=value['manifest']: + if type(raw) is not bytes or len(raw)>64*1024 or hashlib.sha256(raw).hexdigest()!=value['manifest_sha256'] or decode_retained_manifest(decode(raw))!=value['manifest']: raise WorkbenchError('Exact original project manifest bytes differ.') key=catalog_id+'.'+value['manifest_sha256'] prior=decode_records(self.session.snapshot['records']).get(('project_snapshot',key)) diff --git a/projects.py b/projects.py index 4a55f852..5860d846 100644 --- a/projects.py +++ b/projects.py @@ -105,6 +105,22 @@ def manifest(value): return value +def decode_retained_manifest(value): + """Keep the closed schema1 activity field as inert historical content. + + Current registration still uses manifest(), which does not accept this + retired field. Reading a retained manifest grants no activity-file access. + """ + import copy + if type(value) is dict and value.get('schema_version') == 1 and 'activity_path' in value: + relative_path(value['activity_path']) + current = {key: item for key, item in value.items() if key != 'activity_path'} + manifest(current) + else: + manifest(value) + return copy.deepcopy(value) + + def decode_retained_snapshot(value): """Validate immutable registration facts without binding any current root.""" import copy @@ -115,7 +131,7 @@ def decode_retained_snapshot(value): if type(value['manifest_sha256']) is not str or not SHA.fullmatch(value['manifest_sha256']):raise WorkbenchError('Invalid retained manifest hash.') root=value['root_identity'] if type(root) is not list or len(root)!=2 or any(type(number) is not int or number<0 for number in root):raise WorkbenchError('Invalid historical root identity.') - document=manifest(value['manifest']) + document=decode_retained_manifest(value['manifest']) if document['id']!=value['id'] or document['name']!=value['name']:raise WorkbenchError('Retained snapshot manifest identity differs.') return copy.deepcopy(value) diff --git a/tests/test_project_extraction.py b/tests/test_project_extraction.py index 368fbb88..4ef0ff92 100644 --- a/tests/test_project_extraction.py +++ b/tests/test_project_extraction.py @@ -139,6 +139,24 @@ def bind_manifest_copy(self, source): 'recovered', 'manifest.json') return path + def test_exact_legacy_activity_manifest_is_captured_without_reading_activity(self): + self.manifest['activity_path']='activity-must-remain-inert.json' + self.manifest_raw=(json.dumps(self.manifest,ensure_ascii=False,indent=3)+'\n\n').encode() + saved=json.loads(self.dataset.db.execute('SELECT document FROM workspace_projects WHERE id=%s', + (self.project_id,)).fetchone()[0]) + saved['manifest']=copy.deepcopy(self.manifest) + saved['manifest_sha256']=hashlib.sha256(self.manifest_raw).hexdigest() + self.dataset.db.execute('UPDATE workspace_projects SET document=%s WHERE id=%s', + (json.dumps(saved),self.project_id)) + self.selection['project_snapshot']['manifest_sha256']=saved['manifest_sha256'] + source=self.view();self.bind_manifest_copy(source) + fd,_=source.roots.pop('fixture');os.close(fd) + captured=self.capture(source,{}) + self.assertEqual(captured['blockers'],[]) + snapshot=next(row['document'] for row in captured['records'] if row['family']=='project_snapshot') + self.assertEqual(snapshot['value'],saved) + self.assertEqual(len(snapshot['artifacts']),1) + def test_explicit_original_manifest_binding_keeps_history_without_old_root_authority(self): source = self.view() self.bind_manifest_copy(source) diff --git a/tests/test_project_material.py b/tests/test_project_material.py index cba2f75b..8fe2d11a 100644 --- a/tests/test_project_material.py +++ b/tests/test_project_material.py @@ -224,6 +224,27 @@ def supplied(descriptor):seen.append(descriptor);return raw self.assertEqual(restored[-1],record) self.assertEqual(self.material.runtime(ref,verify=True),value) + def test_legacy_activity_path_is_retained_without_becoming_a_current_manifest(self): + from projects import manifest as current_manifest, decode_retained_snapshot + manifest = dict(schema_version=1, id='c'*32, name='Archived activity', + data=[], recipes=[], evidence=[], requirements=[], activity_path='old/activity.json') + raw = (json.dumps(manifest, indent=3)+'\n\n').encode() + saved = dict(id=manifest['id'], name=manifest['name'], root='gone', root_identity=[1,2], + manifest_path='original.json', manifest_sha256=hashlib.sha256(raw).hexdigest(), manifest=manifest) + with self.assertRaises(WorkbenchError): current_manifest(manifest) + with patch('subprocess.Popen', side_effect=AssertionError('Retained activity is inert')): + self.assertEqual(decode_retained_snapshot(saved),saved) + record=self.material.capture_snapshot(saved,raw,catalog_id=self.catalog) + self.session.snapshot=dict(self.session.snapshot,records=[*self.material.declarations(record),record]) + self.assertEqual(self.material.snapshot_ref(manifest['id'],saved['manifest_sha256'],verify=True),saved) + documents=decode_records(self.session.snapshot['records']) + self.assertEqual(documents[('project_snapshot',self.catalog+'.'+saved['manifest_sha256'])]['value'],saved) + for bad_path in ('../escape.json','/absolute.json','a\\b.json',False): + bad=copy.deepcopy(saved);bad['manifest']['activity_path']=bad_path + with self.assertRaises(WorkbenchError): decode_retained_snapshot(bad) + bad=copy.deepcopy(saved);bad['manifest']['unknown_activity_field']='anything' + with self.assertRaises(WorkbenchError): decode_retained_snapshot(bad) + def test_original_snapshot_manifest_whitespace_is_not_reserialized_for_identity(self): manifest=dict(schema_version=1,id='c'*32,name='Snapshot',data=[],recipes=[],evidence=[],requirements=[]) import json From 1bbb31197772d45bcb0b273ad9013da0d1e554e9 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 08:48:17 -0700 Subject: [PATCH 14/33] Decode original pre-link Torch manifests as inert retained evidence --- project_material.py | 12 ++++++------ tests/test_torch_runtime.py | 21 +++++++++++++++++++++ torch_runtime.py | 27 ++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/project_material.py b/project_material.py index 4ddd29a2..194e91a5 100644 --- a/project_material.py +++ b/project_material.py @@ -277,8 +277,8 @@ def decode_runtime_material(value, *, catalog_id): from recipe_process import validate_cpu_snapshot_manifest,encoded manifest=validate_cpu_snapshot_manifest(descriptor,value['manifest']) elif descriptor.get('profile') == 'managed_torch_python_v1': - from torch_runtime import verify_descriptor,encoded - manifest=verify_descriptor(descriptor,value['manifest']) + from torch_runtime import verify_retained_descriptor,encoded + manifest=verify_retained_descriptor(descriptor,value['manifest']) else: raise WorkbenchError('Unsupported retained runtime profile.', 'unsupported', 409) raw=encoded(manifest) if len(raw)!=descriptor['manifest_bytes'] or hashlib.sha256(raw).hexdigest()!=descriptor['manifest_sha256']: @@ -440,8 +440,8 @@ def runtime(self, ref, *, verify=False): from recipe_process import decode, encoded, validate_cpu_manifest manifest = validate_cpu_manifest(decode(raw)) else: - from torch_runtime import decode, encoded, validate_manifest - manifest = validate_manifest(decode(raw)) + from torch_runtime import decode, encoded, validate_retained_manifest + manifest = validate_retained_manifest(decode(raw)) if encoded(manifest) != raw: raise WorkbenchError('Retained runtime manifest encoding differs.', 'invalid', 400) return value @@ -581,8 +581,8 @@ def capture_recipe_dependencies(self,record,before_records,*,runtime_manifest=No from recipe_process import decode,validate_cpu_manifest manifest=validate_cpu_manifest(decode(manifest_raw)) else: - from torch_runtime import decode,validate_manifest - manifest=validate_manifest(decode(manifest_raw)) + from torch_runtime import decode,validate_retained_manifest + manifest=validate_retained_manifest(decode(manifest_raw)) runtime=self.capture_runtime(dict(ref=ref,descriptor=descriptor, environment=None if descriptor is not None else environment,manifest=manifest),manifest_raw) return [*self.declarations(runtime),runtime] diff --git a/tests/test_torch_runtime.py b/tests/test_torch_runtime.py index c93d202b..03fe4089 100644 --- a/tests/test_torch_runtime.py +++ b/tests/test_torch_runtime.py @@ -89,6 +89,27 @@ def checkpoint(): with self.assertRaises(Cancelled):runtime.file_record(Path(self.config['python']),checkpoint=checkpoint) self.assertEqual(len(calls),2) + def test_retained_prelink_manifest_keeps_exact_hash_without_current_launch_authority(self): + from pipeline import digest + from project_material import decode_runtime_material + manifest=runtime.inventory(self.config);manifest.pop('file_links') + raw=runtime.encoded(manifest) + descriptor=dict(schema_version=1,profile=runtime.PROFILE,catalog_id='a'*32, + manifest_sha256=runtime.sha(raw),manifest_bytes=len(raw), + file_count=len(manifest['files']),total_bytes=manifest['total_bytes']) + with self.assertRaisesRegex(ValueError,'closed manifest'): + runtime.verify_descriptor(descriptor,manifest) + path=Path(self.temp.name)/'old-manifest.json';path.write_bytes(raw) + with self.assertRaisesRegex(ValueError,'closed manifest'):runtime.load(path) + value=dict(ref=dict(kind='runtime',catalog_id='a'*32,id=digest(descriptor),sha256=digest(descriptor)), + descriptor=descriptor,environment=None,manifest=manifest) + self.assertEqual(decode_runtime_material(value,catalog_id='a'*32),value) + self.assertEqual(runtime.verify_retained_descriptor(descriptor,manifest),manifest) + self.assertNotIn('file_links',manifest) + changed=dict(manifest,unknown_field=[]) + with self.assertRaises(ValueError):runtime.verify_retained_descriptor(descriptor,changed) + with self.assertRaises(ValueError):runtime.verify_retained_descriptor(dict(descriptor,manifest_sha256='f'*64),manifest) + def test_manifest_link_noncanonical_encoding_and_descriptor_mismatch_are_rejected(self): manifest=runtime.inventory(self.config);raw=runtime.encoded(manifest) root=Path(self.temp.name);path=root/'manifest.json';path.write_bytes(raw) diff --git a/torch_runtime.py b/torch_runtime.py index f664b9fe..e20cb384 100644 --- a/torch_runtime.py +++ b/torch_runtime.py @@ -199,9 +199,20 @@ def validate_descriptor(value): return value -def verify_descriptor(value,manifest): - """Bind a captured manifest to the catalog descriptor before installed asset reads.""" - validate_descriptor(value);validate_manifest(manifest) +def validate_retained_manifest(value): + """Validate original pre-link inventories solely as historical evidence. + + The measured bytes keep their original shape/hash. Current load and launch + still require validate_manifest and its explicit link inventory. + """ + if type(value) is dict and 'file_links' not in value: + validate_manifest(dict(value,file_links=[])) + return value + return validate_manifest(value) + + +def _verify_descriptor(value,manifest,validate): + validate_descriptor(value);validate(manifest) raw=encoded(manifest) require(value['manifest_sha256']==sha(raw) and value['manifest_bytes']==len(raw) and value['file_count']==len(manifest['files']) and value['total_bytes']==manifest['total_bytes'], @@ -209,6 +220,16 @@ def verify_descriptor(value,manifest): return manifest +def verify_descriptor(value,manifest): + """Bind a captured manifest to the catalog descriptor before installed asset reads.""" + return _verify_descriptor(value,manifest,validate_manifest) + + +def verify_retained_descriptor(value,manifest): + """Bind an original current or pre-link manifest without granting execution.""" + return _verify_descriptor(value,manifest,validate_retained_manifest) + + def launch(path,device,entry,arguments): require(device in ('cpu','cuda'),'explicit CPU or CUDA device') value=load(path) From 7f841a6e63eb16cca4d4b4f7aef209ab4ddd2adc Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 09:01:34 -0700 Subject: [PATCH 15/33] Validate original Torch material policy only through retained Run specifications --- recipe_contract.py | 42 ++++++++++++++++++++++++++++++++++++- runs.py | 2 +- tests/test_torch_runtime.py | 33 +++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/recipe_contract.py b/recipe_contract.py index 2c1f85fc..2c3de494 100644 --- a/recipe_contract.py +++ b/recipe_contract.py @@ -830,8 +830,23 @@ def prepare(projects, graph, execution_sha256, profile=None, *, releases=None, r 'graph_revision':graph['revision'], 'execution_sha256':execution_sha256}, captured +def _legacy_torch_policy(material): + return (material['schema_version']==1 and material['recipe']['schema_version']==1 + and material['recipe']['runtime']==TORCH_PROFILE + and material['environment']=={'LANG':'C.UTF-8'}) + + def validate_material(material): """Cold structural custody checks; no filesystem access or current-code comparison.""" + return _validate_material(material, retained=False) + + +def validate_retained_material(material): + """Validate current and original schema1 execution facts without admission.""" + return _validate_material(material, retained=True) + + +def _validate_material(material, *, retained): closed(material, ('schema_version', 'catalog_id', 'project', 'graph_execution_sha256', 'recipe', 'parameters', 'inputs', 'captures', 'runtime', 'host_files', 'permissions', 'environment', 'resources'), ('native_policies','cpu_snapshot','torch_runtime','extension_binding')) if type(material['schema_version']) is not int or material['schema_version'] not in (1,2,3) or not isinstance(material['catalog_id'], str) or not material['catalog_id']: @@ -856,7 +871,11 @@ def receipt(value): raise WorkbenchError('Material leaves no generated context capacity.') if encode(validate_parameters(definition, material['parameters'])) != encode(material['parameters']): raise WorkbenchError('Retained parameters must include every typed value.') - if material['resources'] != definition['resources'] or material['environment'] != ({} if definition['runtime']==TORCH_PROFILE else {'LANG':'C.UTF-8'}) or material['permissions'] != {'filesystem':'trusted-local', 'network':'not_isolated'}: + expected_environment = {} if definition['runtime']==TORCH_PROFILE else {'LANG':'C.UTF-8'} + legacy_environment = retained and _legacy_torch_policy(material) + if (material['resources'] != definition['resources'] + or material['environment'] != expected_environment and not legacy_environment + or material['permissions'] != {'filesystem':'trusted-local', 'network':'not_isolated'}): raise WorkbenchError('Material execution policy is inconsistent.') if not isinstance(material['inputs'], dict) or set(material['inputs']) != set(definition['inputs']): raise WorkbenchError('Material input inventory differs.') expected = {'source/'+item['path']:item['sha256'] for item in definition['sources']} @@ -991,6 +1010,10 @@ def receipt(value): def spec_for_material(material, staging, *, context=None): """Pure reconstruction: retained approvals never depend on current source/runtime.""" validate_material(material) + return _spec_for_material(material, staging, context=context) + + +def _spec_for_material(material, staging, *, context=None): staging = Path(staging).absolute() output = staging/'output' definition = material['recipe']; runtime = material['runtime'] @@ -1041,6 +1064,23 @@ def validate_spec(material, spec, staging, *, context=None): if is_evidence(material['recipe']) and context is None: raise WorkbenchError('Final model-evidence specification requires authoritative attempt context.') expected = spec_for_material(material, staging, context=context) + return _validate_spec(expected, spec) + + +def validate_retained_spec(material, spec, staging, *, context=None): + """Prove an exact historical spec; never select this path for new work.""" + validate_retained_material(material) + if is_evidence(material['recipe']) and context is None: + raise WorkbenchError('Final model-evidence specification requires authoritative attempt context.') + expected = _spec_for_material(material, staging, context=context) + if _legacy_torch_policy(material): + # This original policy predates descriptor duplication in the spec. + # Its descriptor remains mandatory in the exact retained material. + expected.pop('torch_runtime') + return _validate_spec(expected, spec) + + +def _validate_spec(expected, spec): actual = {key:value for key,value in spec.items() if key not in ('attempt_id', 'request_sha256')} if encode(actual) != encode(expected): raise WorkbenchError('Execution specification differs from the approved material.', 'conflict', 409) diff --git a/runs.py b/runs.py index 8d8a9398..c4cfcfc7 100644 --- a/runs.py +++ b/runs.py @@ -193,7 +193,7 @@ def validate_retained_run(run, *, catalog_id, run_id, request_key, request_sha25 or Path(declared).parent.name != 'workspace-runs'): raise ValueError('Invalid retained external staging location.') admitted_context=extension_context - recipe_contract.validate_spec(run['material'], run['spec'], declared,context=admitted_context) + recipe_contract.validate_retained_spec(run['material'], run['spec'], declared,context=admitted_context) if (run['graph'].get('recipe') != 'external_recipe' or run['material']['catalog_id'] != catalog_id or digest(run['material']) != run['material_sha256'] diff --git a/tests/test_torch_runtime.py b/tests/test_torch_runtime.py index 03fe4089..73beb56c 100644 --- a/tests/test_torch_runtime.py +++ b/tests/test_torch_runtime.py @@ -130,6 +130,39 @@ def test_manifest_link_noncanonical_encoding_and_descriptor_mismatch_are_rejecte class TorchRecipe(unittest.TestCase): setUp=fixtures.RecipePipelineTests.setUp + def test_retained_language_policy_keeps_exact_spec_and_cannot_enter_current_admission(self): + InstalledAssets.setUp(self) + manifest=runtime.inventory(self.config);raw=runtime.encoded(manifest) + identity=runtime.sha(raw) + descriptor=dict(schema_version=1,profile=runtime.PROFILE,catalog_id=self.dataset.db.dataset_id, + manifest_sha256=identity,manifest_bytes=len(raw),file_count=len(manifest['files']), + total_bytes=manifest['total_bytes']) + self.value.update(runtime=runtime.PROFILE,runtime_device='cuda',runtime_snapshot_sha256=identity) + self.recipe_raw=json.dumps(self.value).encode();(self.project/'recipe.json').write_bytes(self.recipe_raw) + self.body['expected_sha256']=runtime.sha(self.recipe_raw) + graph=self.dataset.pipelines.create_external(self.body) + with patch('torch_runtimes.TorchRuntimes.get',return_value=(descriptor,raw)), \ + patch('subprocess.Popen',side_effect=AssertionError('Retained validation executes no source')): + preflight=self.dataset.pipelines.preflight_external(graph['id'],1) + captured=self.dataset.pipelines.capture_external(graph['id'],1,preflight['material_sha256'],self.root/'capture') + material=copy.deepcopy(captured['material']);spec=copy.deepcopy(captured['spec']) + material['environment']={'LANG':'C.UTF-8'};spec['env']={'LANG':'C.UTF-8'} + descriptor_in_current_spec=spec.pop('torch_runtime') + original_material=copy.deepcopy(material);original_spec=copy.deepcopy(spec) + with self.assertRaises(WorkbenchError):contract.validate_material(material) + with self.assertRaises(WorkbenchError):contract.spec_for_material(material,self.root/'capture') + with patch('subprocess.Popen',side_effect=AssertionError('Retained spec is inert')): + self.assertEqual(contract.validate_retained_spec(material,spec,self.root/'capture'),spec) + self.assertEqual(material,original_material);self.assertEqual(spec,original_spec) + with self.assertRaises(WorkbenchError):contract.validate_spec(material,spec,self.root/'capture') + for changed_env in ({'LANG':'elsewhere'},{'LANG':'C.UTF-8','PYTHONPATH':'/unsafe'}): + bad=copy.deepcopy(material);bad['environment']=changed_env + with self.assertRaises(WorkbenchError):contract.validate_retained_spec(bad,spec,self.root/'capture') + bad=copy.deepcopy(spec);bad['env']={} + with self.assertRaises(WorkbenchError):contract.validate_retained_spec(material,bad,self.root/'capture') + bad=copy.deepcopy(spec);bad['torch_runtime']=descriptor_in_current_spec + with self.assertRaises(WorkbenchError):contract.validate_retained_spec(material,bad,self.root/'capture') + def test_capture_binds_manifest_bootstrap_and_explicit_device(self): InstalledAssets.setUp(self) manifest=runtime.inventory(self.config) From e74760c036f5327b8c4f168f46cf97b44e3f10b6 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 09:06:53 -0700 Subject: [PATCH 16/33] Skip surrogate scans for proven ASCII JSON strings --- agent_api.py | 2 +- tests/test_agent_api.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/agent_api.py b/agent_api.py index 64e6e0bf..ee23b2b5 100644 --- a/agent_api.py +++ b/agent_api.py @@ -1994,7 +1994,7 @@ def _finite(value, depth=0, *, nul_path=None, _path=()): if depth > DEPTH_LIMIT: raise AgentAPIError('JSON nesting exceeds the transport bound.') if isinstance(value, str): - if any(0xD800 <= ord(ch) <= 0xDFFF for ch in value) or ('\x00' in value and _path != nul_path): + if (not str.isascii(value) and any(0xD800 <= ord(ch) <= 0xDFFF for ch in value)) or ('\x00' in value and _path != nul_path): raise AgentAPIError('Text contains an invalid code point.') elif isinstance(value, float) and not math.isfinite(value): raise AgentAPIError('JSON numbers must be finite.') diff --git a/tests/test_agent_api.py b/tests/test_agent_api.py index 7ccee78f..2c9c1f70 100644 --- a/tests/test_agent_api.py +++ b/tests/test_agent_api.py @@ -50,6 +50,26 @@ def sample(schema): class RegistryTests(unittest.TestCase): + def test_ascii_shortcut_preserves_unicode_and_scoped_nul_rejection(self): + for codepoint in range(128): + value=chr(codepoint) + if codepoint==0: + with self.assertRaises(api.AgentAPIError):api.decode(json.dumps(value)) + else: + self.assertEqual(api.decode(json.dumps(value)),value) + for codepoint in range(0xd800,0xe000): + with self.assertRaises(api.AgentAPIError):api.decode(json.dumps(chr(codepoint))) + for codepoint in (0x80,0x7ff,0x800,0xd7ff,0xe000,0xffff,0x10000,0x10ffff): + value=chr(codepoint) + self.assertEqual(api.decode(json.dumps(value,ensure_ascii=False)),value) + allowed={'entry':'left\x00right'} + self.assertEqual(api.decode(json.dumps(allowed),nul_path=('entry',)),allowed) + for denied in ({'entry':'ok','other':'\x00'},{'\x00':'ok'}): + with self.assertRaises(api.AgentAPIError):api.decode(json.dumps(denied),nul_path=('entry',)) + class FalseAscii(str): + def isascii(self):return True + with self.assertRaises(api.AgentAPIError):api._finite(FalseAscii('\ud800')) + def test_discovery_is_stdlib_only_in_cold_process(self): code = 'import sys;sys.path.insert(0,sys.argv[1]);import agent_api;agent_api.discovery();assert not any(x in sys.modules for x in ("jsonschema","app","workbench","mcp","psycopg"))' subprocess.run([sys.executable, '-I', '-S', '-c', code, str(Path(api.__file__).parent)], check=True, capture_output=True, timeout=10) From bf6b39b2b5a21b613f64922758f109462603c5f4 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 09:36:30 -0700 Subject: [PATCH 17/33] Use fixed material views for Study history reconstruction --- project_study_execution.py | 7 +++---- tests/test_project_study_execution.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/project_study_execution.py b/project_study_execution.py index 9e19b3c2..561c4779 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -16,7 +16,7 @@ from pipeline import closed, identifier, digest from project_history import (FAMILY_FORMATS, _audit, _sha, envelope, decode_records as history_records) -from project_material import ProjectMaterial +from project_material import ProjectMaterial, MaterialSnapshot from project_session import operation_records from workbench import WorkbenchError, encode, timestamp @@ -36,8 +36,7 @@ def _invalid(message,code='invalid'): def _facts(records): - view=SimpleNamespace(snapshot={'records':records},check=lambda:None) - return history_records(records,materials=ProjectMaterial(view)) + return history_records(records,materials=MaterialSnapshot(records)) def _intent_key(operation,key): @@ -325,7 +324,7 @@ def __init__(self,projection,execution): def _canonical(self,records=None): records=self.session.snapshot['records'] if records is None else records - return history_records(records,materials=self.materials) + return history_records(records,materials=self.materials.read_view(records)) def restore_authoring(self,records=None): """Reconstruct intent only, under the current project guard.""" diff --git a/tests/test_project_study_execution.py b/tests/test_project_study_execution.py index 321cae09..7227e00d 100644 --- a/tests/test_project_study_execution.py +++ b/tests/test_project_study_execution.py @@ -95,6 +95,31 @@ def before_write(): class StudyExecutionCodecTests(unittest.TestCase): + def test_canonical_reconstruction_uses_one_selected_material_view(self): + records=[];view=object();calls=[] + class Materials: + def read_view(self,selected): + calls.append(selected);return view + adapter=ProjectStudyExecution.__new__(ProjectStudyExecution) + adapter.session=SimpleNamespace(snapshot={'records':records});adapter.materials=Materials() + def decoded(selected,*,materials): + self.assertIs(selected,records);self.assertIs(materials,view);return 'selected facts' + with patch('project_study_execution.history_records',side_effect=decoded): + self.assertEqual(adapter._canonical(),'selected facts') + self.assertEqual(adapter._canonical(records),'selected facts') + self.assertEqual(len(calls),2) + self.assertTrue(all(selected is records for selected in calls)) + + def test_fixed_record_validation_has_no_live_material_session(self): + import project_study_execution as owner + from project_material import MaterialSnapshot + records=[] + def decoded(selected,*,materials): + self.assertIs(selected,records);self.assertIsInstance(materials,MaterialSnapshot);return 'fixed facts' + with patch('project_study_execution.ProjectMaterial',side_effect=AssertionError('No live material lookups')), \ + patch('project_study_execution.history_records',side_effect=decoded): + self.assertEqual(owner._facts(records),'fixed facts') + def test_admission_has_only_exact_reservation_and_references(self): value=trial('9'*32,'3'*32)['document']['trial'];composition=Materials().recipe(value['material']) charge=composition['material']['resource_envelope'] From afa7ef49c40212823fa62698614fd9a490d8895d Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 11:19:34 -0700 Subject: [PATCH 18/33] feat: preserve exact retained sources for project composition --- project_composition.py | 109 +++++++++++++++++++++++++++ project_history.py | 23 ++++-- tests/test_project_composition.py | 121 ++++++++++++++++++++++++++++++ 3 files changed, 248 insertions(+), 5 deletions(-) create mode 100644 project_composition.py create mode 100644 tests/test_project_composition.py diff --git a/project_composition.py b/project_composition.py new file mode 100644 index 00000000..a5bcf86d --- /dev/null +++ b/project_composition.py @@ -0,0 +1,109 @@ +"""Pure retained-source assembly for native independent project composition. + +Sources are ordered: the primary supplies the base envelope for an overlapping +identity. Exact duplicates share custody. Only admission-proof grant/consumption +variants of otherwise byte-identical Runs can coexist as native Run observations. +No source root, catalog permission, runtime or editable overlay is activated here. +The lifecycle owner must validate each source and the complete final candidate, +clone primary edits and publish all required bytes through the native owners. +""" +import hashlib + +from project_fork import ARCHIVE_FAMILIES, _sha, _uuid, decode_fork +from project_history import capture_run_observation +from project_store import (MAX_RECORDS, MAX_TOTAL_BYTES, ProjectStoreError, + _encode, _records) + + +class ProjectCompositionError(ProjectStoreError): + pass + + +def _fail(message, code='invalid', status=400): + raise ProjectCompositionError(message, code, status) + + +def _record_sha(record): + return hashlib.sha256(_encode(record)).hexdigest() + + +def _approval_variant(original, alternate): + if original['family'] != 'run_history': + return False + left, right = original['document'], alternate['document'] + if _encode({k:v for k,v in left.items() if k != 'admission_proof'}) != _encode( + {k:v for k,v in right.items() if k != 'admission_proof'}): + return False + left, right = left.get('admission_proof'), right.get('admission_proof') + fields = {'grant', 'consumption', 'admission'} + return (type(left) is dict and type(right) is dict and set(left) == fields and set(right) == fields + and _encode(left['admission']) == _encode(right['admission'])) + + +def merge_retained_sources(snapshots): + """Return owned originals, exact source memberships and inactive dispositions. + + This internal construction step is not a publication or a fresh domain + admission boundary. Each source must already have passed its domain owners. + Store shape/size checks are retained here. Sources may be supplied lazily + to keep one temporary source snapshot in memory at a time. Same-ID conflicts outside the one + explicit Run-proof case are rejected; nothing is renamed or overwritten. + Membership digests cover complete canonical envelopes, not Python equality. + """ + try: + snapshots = iter(snapshots) + except TypeError: + _fail('Composition requires explicit complete source snapshots.') + projects, indexed, hashes, sizes = set(), {}, {}, {} + memberships, archived = [], set() + for snapshot in snapshots: + if type(snapshot) is not dict or set(snapshot) != {'project_id','name','checkpoint','records'}: + _fail('Composition requires complete canonical source snapshots.') + _uuid(snapshot['project_id']); _sha(snapshot['checkpoint']) + if snapshot['project_id'] in projects: + _fail('Composition source project identity repeats.', 'conflict', 409) + projects.add(snapshot['project_id']) + if type(snapshot['name']) is not str or not snapshot['name'].strip() or len(snapshot['name']) > 200: + _fail('Composition source name differs from the project contract.') + previous = decode_fork(snapshot['records'], project_id=snapshot['project_id']) + if previous is not None: + archived.update((item['family'], item['id']) for item in previous['archived']) + membership = dict(project_id=snapshot['project_id'], name=snapshot['name'], + checkpoint=snapshot['checkpoint'], records={}, run_observations={}) + if len(snapshot['records']) > MAX_RECORDS: + _fail('Composition source exceeds the project record budget.', 'too_large', 413) + source_bytes = 0 + for original in snapshot['records']: + # Validate/copy one envelope at a time. Holding another complete + # source-sized serialized/deep-copy list is unnecessary for custody. + family, identity, document, raw = _records([original])[0] + source_bytes += len(raw) + if source_bytes > MAX_TOTAL_BYTES: + _fail('Composition source exceeds the project byte budget.', 'too_large', 413) + record = dict(family=family, id=identity, document=document) + key = family, identity + sha = _record_sha(record) + membership['records'].setdefault(family, {})[identity] = sha + if family in ARCHIVE_FAMILIES: + archived.add(key) + if key not in indexed: + indexed[key], hashes[key], sizes[key] = record, sha, len(raw) + elif hashes[key] != sha: + if not _approval_variant(indexed[key], record): + _fail('Composition has conflicting retained envelopes: '+family+'/'+identity+'.', 'conflict', 409) + observation = capture_run_observation(document) + observation_key = observation['family'], observation['id'] + observation_sha = _record_sha(observation) + if observation_key in indexed and hashes[observation_key] != observation_sha: + _fail('Composition Run observation identity conflicts.', 'conflict', 409) + indexed[observation_key], hashes[observation_key] = observation, observation_sha + sizes[observation_key] = len(_records([observation])[0][3]) + membership['run_observations'][identity] = observation['id'] + memberships.append(membership) + del snapshot + if len(projects) < 2: + _fail('Composition requires at least two explicit complete source snapshots.') + if len(indexed) > MAX_RECORDS or sum(sizes.values()) > MAX_TOTAL_BYTES: + _fail('Composed retained sources exceed the project record budget.', 'too_large', 413) + return dict(records=[indexed[key] for key in sorted(indexed)], sources=memberships, + archived=[dict(family=family,id=identity) for family,identity in sorted(archived)]) diff --git a/project_history.py b/project_history.py index eb14389a..892ae3c2 100644 --- a/project_history.py +++ b/project_history.py @@ -273,6 +273,21 @@ def _observation_document(record): return doc +def capture_run_observation(document): + """Retain one already validated Run-history document as inert exact evidence. + + This creates no disclosure fact or execution permission. The complete + history decoder still owns Run, admission and payload closure validation. + """ + from project_store import _encode as canonical + sha=hashlib.sha256(canonical(document)).hexdigest() + record={'family':'run_observation','id':sha,'document':{ + 'format':FAMILY_FORMATS['run_observation'],'origin_catalog_id':document['origin_catalog_id'], + 'run_id':document['run']['id'],'sha256':sha,'observation':copy.deepcopy(document)}} + _observation_document(record) + return record + + def declarations(record): if record['family']=='run_observation': return declarations({'family':'run_history','document':record['document']['observation']}) @@ -925,11 +940,9 @@ def invoke(): current=self.access.require('expose-holdout','history.disclose',run_id) if current.actor_id!=actor_id:_invalid('Disclosure caller changed.','conflict') doc=self._record(self._facts().runs,run_id,body['origin_catalog_id']) - from project_store import _encode as canonical - sha=hashlib.sha256(canonical(doc)).hexdigest() - captured.append({'family':'run_observation','id':sha,'document':{ - 'format':FAMILY_FORMATS['run_observation'],'origin_catalog_id':body['origin_catalog_id'], - 'run_id':run_id,'sha256':sha,'observation':copy.deepcopy(doc)}}) + observation=capture_run_observation(doc) + sha=observation['id'] + captured.append(observation) return validate_disclosure_fact({'format':'project_disclosure_v2','origin_catalog_id':body['origin_catalog_id'], 'run_id':run_id,'payload_set_sha256':payload_set_sha256(doc['origin_catalog_id'],run_id,doc['payloads']), 'population':'all','reason':body['reason'],'actor_id':actor_id,'occurred_at':timestamp(), diff --git a/tests/test_project_composition.py b/tests/test_project_composition.py new file mode 100644 index 00000000..7ad20a9a --- /dev/null +++ b/tests/test_project_composition.py @@ -0,0 +1,121 @@ +"""Exact retained-source assembly; native history remains the proof oracle.""" +import copy +import hashlib +import unittest +from unittest.mock import patch + +from project_composition import ProjectCompositionError, merge_retained_sources +from project_history import capture_run, declarations, decode_records +from project_store import _encode +from test_project_history import ORIGIN, legacy_run + + +def snapshot(identity, records): + return dict(project_id=identity*32, name='Source '+identity, checkpoint=identity*64, records=records) + + +def approved_history(): + run=legacy_run();run['approval_id']='9'*32 + grant=dict(id=run['approval_id'],catalog_id=ORIGIN,approver_actor_id='a'*32,approver_credential_id='b'*32, + consumer_actor_id='5'*32,material_sha256='f'*64,created_at=1,expires_at=100,revoked_at=None) + consumption=dict(grant_id=grant['id'],run_id=run['id'],admission_id=run['admission_id'], + consumer_credential_id='6'*32,request_sha256=run['request_sha256'],consumed_at=2) + effects={'pipeline.inspect_release':[run['release_id']],'dataset_releases.locate':[run['release_id']], + 'pipeline.probe':['image_classification'],'pipeline.probe_native':['image_classification'], + 'pipeline.definition_identity':['image_classification'],'pipeline.exposures':[None],'runs.update':[run['id']]} + admission=dict(id=run['admission_id'],catalog_id=ORIGIN,actor_id='5'*32,credential_id='6'*32, + kind='runs',job_id=run['id'],request_digest=run['request_sha256'],effects=effects,created_at=1) + record=capture_run(run,admission_proof=dict(grant=grant,consumption=consumption,admission=admission),payloads=[]) + records=[record,*declarations(record)] + decode_records(records) + return records + + +class RetainedCompositionTests(unittest.TestCase): + def test_exact_duplicate_is_shared_and_each_source_membership_is_retained(self): + records=approved_history();left=snapshot('1',records);right=snapshot('2',copy.deepcopy(records)) + result=merge_retained_sources([left,right]) + self.assertEqual(result['records'],sorted(records,key=lambda item:(item['family'],item['id']))) + self.assertEqual(len(result['sources']),2) + for source in result['sources']: + for record in records: + expected=hashlib.sha256(_encode(record)).hexdigest() + self.assertEqual(source['records'][record['family']][record['id']],expected) + self.assertEqual(source['run_observations'],{}) + + def test_alternate_grant_is_exact_native_observation_without_duplicate_run(self): + original=approved_history();alternate=copy.deepcopy(original) + alternate[0]['document']['admission_proof']['grant']['revoked_at']=3 + alternate[0]['document']['admission_proof']['consumption']['consumed_at']=3 + decode_records(alternate) + with patch('os.open',side_effect=AssertionError('Pure construction has no filesystem access')): + result=merge_retained_sources([snapshot('1',original),snapshot('2',alternate),snapshot('3',copy.deepcopy(alternate))]) + facts=decode_records(result['records']) + self.assertEqual(len(facts.runs),1) + self.assertEqual(len(facts.observations),1) + base=next(r for r in result['records'] if r['family']=='run_history') + observation=next(r for r in result['records'] if r['family']=='run_observation') + self.assertEqual(_encode(base),_encode(original[0])) + self.assertEqual(_encode(observation['document']['observation']),_encode(alternate[0]['document'])) + expected=hashlib.sha256(_encode(alternate[0]['document'])).hexdigest() + self.assertEqual(observation['id'],expected) + for source in result['sources'][1:]: + self.assertEqual(source['run_observations'],{original[0]['id']:expected}) + self.assertFalse(result['sources'][0]['run_observations']) + + def test_broader_run_conflicts_are_rejected_at_the_named_conflict_boundary(self): + original=approved_history() + for field in ('metrics','population','admission'): + alternate=copy.deepcopy(original) + if field=='metrics':alternate[0]['document']['run']['metrics']={'accuracy':0.5} + elif field=='population':alternate[0]['document']['payloads'][0]['population']='test' + else:alternate[0]['document']['admission_proof']['admission']['created_at']=2 + decode_records(alternate) + with self.subTest(field=field),self.assertRaises(ProjectCompositionError) as caught: + merge_retained_sources([snapshot('1',original),snapshot('2',alternate)]) + self.assertEqual(caught.exception.code,'conflict') + self.assertIn('run_history/'+original[0]['id'],str(caught.exception)) + + def test_equal_python_numbers_do_not_hide_different_canonical_bytes(self): + original=approved_history();original[0]['document']['run']['parameters']={'scale':1} + alternate=copy.deepcopy(original);alternate[0]['document']['run']['parameters']['scale']=1.0 + self.assertEqual(original,alternate) + decode_records(original);decode_records(alternate) + with self.assertRaises(ProjectCompositionError) as caught: + merge_retained_sources([snapshot('1',original),snapshot('2',alternate)]) + self.assertEqual(caught.exception.code,'conflict') + + def test_other_family_conflict_and_repeated_source_project_are_rejected(self): + original=approved_history();alternate=copy.deepcopy(original) + alternate[1]['document']['bytes']+=1 + with self.assertRaises(ProjectCompositionError) as caught: + merge_retained_sources([snapshot('1',original),snapshot('2',alternate)]) + self.assertEqual(caught.exception.code,'conflict') + self.assertIn('project_artifact/',str(caught.exception)) + with self.assertRaises(ProjectCompositionError) as repeated: + merge_retained_sources([snapshot('1',original),snapshot('1',original)]) + self.assertEqual(repeated.exception.code,'conflict') + self.assertIn('source project identity repeats',str(repeated.exception)) + + def test_source_snapshots_can_be_loaded_lazily(self): + records=approved_history() + def sources(): + yield snapshot('1',records) + yield snapshot('2',copy.deepcopy(records)) + self.assertEqual(len(merge_retained_sources(sources())['sources']),2) + with self.assertRaises(ProjectCompositionError) as caught: + merge_retained_sources(iter([snapshot('1',records)])) + self.assertEqual(caught.exception.code,'invalid') + self.assertIn('at least two',str(caught.exception)) + + def test_output_owns_its_documents_and_does_not_mutate_sources(self): + original=approved_history();before=copy.deepcopy(original) + result=merge_retained_sources([snapshot('1',original),snapshot('2',copy.deepcopy(original))]) + result['records'][0]['document']['bytes']=123 + self.assertEqual(original,before) + original[0]['document']['run']['metrics']={'accuracy':0.25} + retained=next(r for r in result['records'] if r['family']=='run_history') + self.assertIsNone(retained['document']['run']['metrics']) + + +if __name__=='__main__':unittest.main() From f1cdb23cd01ecfeb2afc20785b7e3f595f7c6aed Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 11:53:26 -0700 Subject: [PATCH 19/33] feat: publish native multi-source project compositions --- agent_api.py | 18 +- docs/runbooks/project-composition.md | 54 +++ project_composition.py | 179 ++++++++++ project_fork.py | 310 +++++++++--------- project_projection.py | 15 +- project_store.py | 1 + project_workspace.py | 62 +++- tests/test_project_composition.py | 2 + tests/test_project_composition_publication.py | 155 +++++++++ tests/test_project_composition_workspace.py | 87 +++++ 10 files changed, 712 insertions(+), 171 deletions(-) create mode 100644 docs/runbooks/project-composition.md create mode 100644 tests/test_project_composition_publication.py create mode 100644 tests/test_project_composition_workspace.py diff --git a/agent_api.py b/agent_api.py index ee23b2b5..67b99453 100644 --- a/agent_api.py +++ b/agent_api.py @@ -1329,6 +1329,17 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non scopes=('inspect','edit'),mutation=True,description='Create an independent idle project fork at an explicit configured local destination; retain inert origins and exact copy completeness.') _OPERATIONS[-1]['outputSchema']['oneOf'][1]['properties']['result']['properties']['receipt']=PROJECT_FILES_RECEIPT PROJECT_FILES_M4.add('project_files.save_as');_OWNER_OPERATION_IDS.add('project_files.save_as');_MODEL_RECEIPT_IDS.add('project_files.save_as') +# Composition is a new owner-only operation; existing Save As pins/state stay frozen. +_PROJECT_COMPOSITION_SOURCE=obj({'root':text(160),'directory':text(512),'project_id':ID,'checkpoint':SHA}) +model_operation('project_files.compose','POST','/api/workspace/project-files/compose', + dict(_PROJECT_FILES_MUTATION,root=text(160),directory=text(512),name=text(200), + sources=array(_PROJECT_COMPOSITION_SOURCE,15,1)), + output=obj({'state':PROJECT_FILES_STATE,'fork':_PROJECT_FORK_RESULT, + 'sources':array(obj({'project_id':ID,'checkpoint':SHA}),16,2)}), + scopes=('inspect','edit'),mutation=True, + description='Compose explicit retained project checkpoints with the current editable primary into one independent inert project.') +_OPERATIONS[-1]['outputSchema']['oneOf'][1]['properties']['result']['properties']['receipt']=PROJECT_FILES_RECEIPT +PROJECT_FILES_M4.add('project_files.compose');_OWNER_OPERATION_IDS.add('project_files.compose');_MODEL_RECEIPT_IDS.add('project_files.compose') _PROJECT_FILE_INSPECTION=obj({'id':SHA,'kind':_PROJECT_FILE_KIND,'path':text(512),'revision':SAFE_POS, 'expected_sha256':{'oneOf':[SHA,{'type':'null'}]},'expected_bytes':integer(0,1048576), 'observed_sha256':{'oneOf':[SHA,{'type':'null'}]},'observed_bytes':{'oneOf':[integer(0,1048576),{'type':'null'}]}, @@ -2670,7 +2681,7 @@ def call(self, name, arguments, *, observer=None, observation=None): elif (value['binding']!=arguments['binding'] or value['receipt']['request_key']!=arguments['request_key'] or value['receipt']['report_sha256']!=arguments['report_sha256']): raise AgentAPIError('Extraction receipt belongs to another controller report or request.','transport',502) - if op['operation_id']=='project_files.save_as' and result['status']<400: + if op['operation_id'] in ('project_files.save_as','project_files.compose') and result['status']<400: value=result['result'];fork=value['fork'];state=value['state'] if (state['state']!='open' or state['project']['project_id']!=fork['project_id'] or state['project']['checkpoint']!=fork['checkpoint'] or state['project']['name']!=arguments['name'] or state['binding']==arguments['binding'] @@ -2678,6 +2689,11 @@ def call(self, name, arguments, *, observer=None, observation=None): or fork['source_checkpoint']!=arguments['expected_checkpoint'] or any(fork[field]!=arguments[field] for field in ('root','directory','request_key'))): raise AgentAPIError('Save As result differs from its source, destination or selected state.','transport',502) + if op['operation_id']=='project_files.compose': + expected=[dict(project_id=fork['source_project_id'],checkpoint=arguments['expected_checkpoint'])] + expected.extend({key:item[key] for key in ('project_id','checkpoint')} for item in arguments['sources']) + if value['sources']!=expected: + raise AgentAPIError('Composition result differs from its exact ordered sources.','transport',502) elif op['operation_id'] in PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4: value = result['result'] if result['status'] >= 400: diff --git a/docs/runbooks/project-composition.md b/docs/runbooks/project-composition.md new file mode 100644 index 00000000..76757fd6 --- /dev/null +++ b/docs/runbooks/project-composition.md @@ -0,0 +1,54 @@ +# Compose retained projects + +The owner operation `project_files.compose` creates one independent project from +an open editable primary and one or more explicit retained project checkpoints. +It preserves the original scientific evidence and forks the primary editable +notes, data and definitions through the same owners as Save As. Secondary drafts +and operation receipts remain inert historical sources. No source project is +modified and no imported program, runtime or credential becomes active. + +Negotiate the operation's version-1 contract through the native project owner +client. The route is `POST /api/workspace/project-files/compose`. In addition to +the current `binding`, `expected_checkpoint` and one fresh `request_key`, supply: + +- `root`, `directory` and `name` for a new destination under a configured root; +- `sources`, an ordered list of additional selections, each containing `root`, + `directory`, `project_id` and `checkpoint`. + +The current project is implicitly the first source. At most 15 additional +sources are accepted by the HTTP contract. All source projects must be idle and +available for native exclusive store ownership. The primary's captured working +files must be clean. The destination cannot contain, lie inside or alias any +source, and existing unrelated destinations are never overwritten. + +Equal canonical envelopes share one retained representation. Same-ID conflicts +are rejected except for Run documents that differ only in their historical +approval grant/consumption while retaining the same admission and every other +field. The complete alternate document is retained as a native Run observation; +it does not become a second Run or create a disclosure. Both documents remain +subject to the history owner's proof and payload validation. + +The persisted `project_composition_v1` record binds the ordered source identities +and exact canonical envelope hashes to the destination's ordinary fork overlay. +A source membership explicitly identifies any alternate Run observation needed +to recover that source's document. Its decoder rejects missing, altered or +contradictory memberships. Subsequent ordinary forks preserve and archive this +provenance. Older software that does not support this family must reject the +project explicitly; records must not be dropped for compatibility. + +Payloads are verified and copied as independent ordinary files. A missing copy +can be supplied by another source with the same verified declaration; a corrupt +selected source is rejected. Unique bytes cannot be omitted. Only dependencies +classified as regenerable by their domain owners can remain explicitly missing. + +On success, `state` describes the newly selected project, `fork` reports the +primary origin and copy completeness, and `sources` lists every selected source +identity/checkpoint in order. The local owner installation retains its own +credentials. Existing Save As and lifecycle-state contracts are unchanged. + +Retain a request before dispatch. An interrupted publication can leave an inert +stage or a complete unselected destination. Inspect the original request and +owner state before reconciling with the same key and arguments. Native receipts +bind reconciliation to the complete ordered source set. A timeout does not +justify another request key or deleting a retained stage. Selection failure +returns a durable unselected receipt while preserving the current primary. diff --git a/project_composition.py b/project_composition.py index a5bcf86d..bcb8a795 100644 --- a/project_composition.py +++ b/project_composition.py @@ -107,3 +107,182 @@ def merge_retained_sources(snapshots): _fail('Composed retained sources exceed the project record budget.', 'too_large', 413) return dict(records=[indexed[key] for key in sorted(indexed)], sources=memberships, archived=[dict(family=family,id=identity) for family,identity in sorted(archived)]) + + +FAMILY = 'project_composition' +FORMAT = 'project_composition_v1' + + +def composition_input(sources, directory, name, request_key): + return dict(format='project_composition_request_v1', + sources=[dict(project_id=s['project_id'],checkpoint=s['checkpoint']) for s in sources], + directory=directory,name=name,request_key=request_key) + + +def _request_digest(sources, directory, name, request_key): + return hashlib.sha256(_encode(composition_input(sources,directory,name,request_key))).hexdigest() + + +def _shape(value, fields, label): + if type(value) is not dict or set(value) != set(fields): + _fail('Invalid composition '+label+' fields.') + + +def _decode_memberships(document, indexed): + from project_store import REQUEST_KEY, _identifier, _relative_parts + _shape(document,('format','destination_project_id','directory','name','request_key','request_sha256','sources'),'provenance') + if document['format'] != FORMAT: + _fail('Unsupported composition provenance format.','unsupported',422) + _uuid(document['destination_project_id']);_relative_parts(document['directory']) + if type(document['name']) is not str or not document['name'].strip() or len(document['name'])>200: + _fail('Invalid composition name.') + if type(document['request_key']) is not str or not REQUEST_KEY.fullmatch(document['request_key']): + _fail('Invalid composition request key.') + _sha(document['request_sha256']) + sources=document['sources'] + if type(sources) is not list or len(sources)<2: + _fail('Composition provenance needs at least two sources.') + seen=set() + for source in sources: + _shape(source,('project_id','name','checkpoint','records','run_observations'),'source membership') + _uuid(source['project_id']);_sha(source['checkpoint']) + if source['project_id'] in seen or source['project_id']==document['destination_project_id']: + _fail('Composition source identities repeat or alias the destination.') + seen.add(source['project_id']) + if type(source['name']) is not str or not source['name'].strip() or len(source['name'])>200: + _fail('Invalid composition source name.') + if type(source['records']) is not dict or type(source['run_observations']) is not dict: + _fail('Composition membership must use explicit inventories.') + runs=source['records'].get('run_history',{}) + if type(runs) is not dict:_fail('Composition Run membership must be an object.') + for identity,sha in source['run_observations'].items(): + _identifier(identity,'Composition original Run ID');_sha(sha) + if not set(source['run_observations'])<=set(runs): + _fail('Composition has an observation mapping without an original Run.') + for family,members in source['records'].items(): + _identifier(family,'Composition member family') + if type(members) is not dict or not members: + _fail('Composition family membership must be a nonempty object.') + for identity,sha in members.items(): + _identifier(identity,'Composition member ID');_sha(sha) + alternate=source['run_observations'].get(identity) if family=='run_history' else None + if alternate is None: + retained=indexed.get((family,identity)) + if retained is None:_fail('Composition source member is missing.','unavailable',409) + else: + from project_history import _observation_document + _sha(alternate) + wrapped=indexed.get(('run_observation',alternate)) + if wrapped is None:_fail('Composition source observation is missing.','unavailable',409) + observed=_observation_document(wrapped) + retained=dict(family='run_history',id=identity,document=observed['observation']) + base=indexed.get(('run_history',identity)) + if (base is None or not _approval_variant(base,retained) + or _record_sha(base)==_record_sha(retained)): + _fail('Composition alternate observation is not an admission-proof-only variant.') + if _record_sha(retained)!=sha: + _fail('Composition source membership differs from its exact retained envelope.') + if document['request_sha256']!=_request_digest(sources,document['directory'],document['name'],document['request_key']): + _fail('Composition request receipt differs from its exact sources.') + + +def decode_records(records, *, project_id=None): + """Validate inert source provenance and its independently owned fork binding.""" + from project_fork import _decode_document + indexed={(r['family'],r['id']):r for r in records} + if len(indexed)!=len(records):_fail('Composition record identity repeats.') + result={} + for record in records: + if record['family']!=FAMILY:continue + document=record['document'];_decode_memberships(document,indexed) + if record['id']!=document['destination_project_id']: + _fail('Composition envelope belongs to a different destination.') + fork=indexed.get(('project_fork',record['id'])) + if fork is None:_fail('Composition has no native fork overlay.','unavailable',409) + _decode_document(fork['document'],record['id'],indexed) + overlay=fork['document'];primary=document['sources'][0] + if (overlay['source']!={key:primary[key] for key in ('project_id','checkpoint')} + or any(overlay[key]!=document[key] for key in ('directory','name','request_key'))): + _fail('Composition provenance differs from its primary fork binding.') + archived={(r['family'],r['id']) for r in overlay['archived']} + required={(family,identity) for source in document['sources'] for family,members in source['records'].items() + for identity in members if family in ARCHIVE_FAMILIES} + if not required<=archived: + _fail('Composition must archive all source editable values and operation receipts.') + result[record['id']]=document + if project_id is not None: + current=decode_fork(records,project_id=project_id) + archived=set() if current is None else {(r['family'],r['id']) for r in current['archived']} + if any(identity!=project_id and (FAMILY,identity) not in archived for identity in result): + _fail('Prior composition provenance must remain explicitly archived.') + return result + + +def validate_source_memberships(records, snapshots, destination_project_id): + """Bind a constructed candidate to the actual owner-selected complete inputs.""" + document=next((r['document'] for r in records if r['family']==FAMILY and r['id']==destination_project_id),None) + if document is None:_fail('Composition candidate has no destination provenance.') + if len(document['sources'])!=len(snapshots):_fail('Composition candidate source count differs.') + for membership,snapshot in zip(document['sources'],snapshots): + if any(membership[key]!=snapshot[key] for key in ('project_id','name','checkpoint')): + _fail('Composition candidate belongs to different source identities.') + expected={} + for record in snapshot['records']: + expected.setdefault(record['family'],{})[record['id']]=_record_sha(record) + if membership['records']!=expected: + _fail('Composition candidate does not preserve the complete source membership.') + _decode_memberships(document,{(r['family'],r['id']):r for r in records}) + + +def build_candidate(snapshots, destination_project_id, *, created_at, id_factory, + directory, name, request_key): + """Preserve all sources and independently fork only the primary editable state.""" + import copy + from project_fork_composition import build_candidate as fork_candidate, artifact_plan + from project_fork import ID + merged=merge_retained_sources(snapshots) + _uuid(destination_project_id) + used={s['project_id'] for s in snapshots} + if destination_project_id in used:_fail('Composition destination must have a fresh identity.') + used.update(r['id'] for r in merged['records'] if ID.fullmatch(r['id'])) + used.update(r['document']['current']['id'] for r in merged['records'] + if r['family'] in ('model_definition','pipeline_graph','recipe_definition')) + used.add(destination_project_id) + def allocate(family,identity,revision): + value=id_factory(family,identity,revision);_uuid(value) + if value in used:_fail('Composition allocator reused a source or allocated identity.') + used.add(value);return value + fork=fork_candidate(snapshots[0],destination_project_id,created_at=created_at,id_factory=allocate) + indexed={(r['family'],r['id']):r for r in merged['records']} + for record in fork['records']: + key=record['family'],record['id'] + if key in indexed: + if _record_sha(indexed[key])!=_record_sha(record):_fail('Primary fork changed an original envelope.') + else:indexed[key]=record + document=dict(format=FORMAT,destination_project_id=destination_project_id,directory=directory,name=name, + request_key=request_key,request_sha256=_request_digest(merged['sources'],directory,name,request_key), + sources=merged['sources']) + record=dict(family=FAMILY,id=destination_project_id,document=document) + if (FAMILY,destination_project_id) in indexed:_fail('Composition provenance identity collides.') + indexed[(FAMILY,destination_project_id)]=record + records=[indexed[key] for key in sorted(indexed)] + validate_source_memberships(records,snapshots,destination_project_id) + archived={(r['family'],r['id']) for r in merged['archived']+fork['archived']} + return dict(records=records,mapping=copy.deepcopy(fork['mapping']), + archived=[dict(family=f,id=i) for f,i in sorted(archived)],artifacts=artifact_plan(records)) + + +def publish_composition(primary_store, primary_artifacts, retained_sources, trusted_root_fd, directory, + *, root_path, name, expected_checkpoint, request_key, quiescent, + created_at, id_factory, validate_candidate): + """Publish through the native fork lifecycle with explicit additional owners.""" + from functools import partial + from project_fork import _publish_project + if type(retained_sources) not in (list,tuple) or not retained_sources: + _fail('Composition requires at least one explicit secondary source owner.') + return _publish_project(primary_store,primary_artifacts,trusted_root_fd,directory, + root_path=root_path,name=name,expected_checkpoint=expected_checkpoint,request_key=request_key, + quiescent=quiescent,retained_sources=retained_sources, + build_candidate=partial(build_candidate,created_at=created_at,id_factory=id_factory, + directory=directory,name=name,request_key=request_key), + validate_candidate=validate_candidate) diff --git a/project_fork.py b/project_fork.py index 959dde8a..7aec1a44 100644 --- a/project_fork.py +++ b/project_fork.py @@ -24,7 +24,7 @@ # formerly active values/receipts become inactive through the fork overlay. ARCHIVE_FAMILIES = frozenset(('workbench_text', 'workbench_record', 'workbench_response', 'workbench_preference', 'saved_selection', 'saved_search', 'model_definition', - 'pipeline_graph', 'recipe_definition', 'project_file', 'project_operation', FAMILY)) + 'pipeline_graph', 'recipe_definition', 'project_file', 'project_operation', 'project_composition', FAMILY)) ID = re.compile(r'^[a-f0-9]{32}$') SHA = re.compile(r'^[a-f0-9]{64}$') @@ -236,188 +236,196 @@ def _result(snapshot, overlay, reconciled): return dict(snapshot=copy.deepcopy(snapshot), fork=copy.deepcopy(overlay), reconciled=reconciled) -def _reconcile_existing(source_store, source, trusted_root_fd, relative, root_path, - expected_input, request_key, digest, validate_candidate): +def _reconcile_existing(sources, trusted_root_fd, relative, root_path, + expected_input, request_key, digest, validate_candidate, *, composed): with ProjectStore.open_at(trusted_root_fd, relative, root_path=root_path) as destination: snapshot = destination.snapshot() overlay = decode_fork(snapshot['records'], project_id=snapshot['project_id']) - if (overlay is None or _input(overlay['source'], overlay['directory'], overlay['name'], overlay['request_key']) != expected_input - or overlay['request_sha256'] != digest): + if composed: + from project_composition import decode_records, composition_input + document = decode_records(snapshot['records'],project_id=snapshot['project_id']).get(snapshot['project_id']) + observed_input = None if document is None else composition_input(document['sources'],document['directory'],document['name'],document['request_key']) + observed_digest = None if document is None else document['request_sha256'] + else: + observed_input = None if overlay is None else _input(overlay['source'],overlay['directory'],overlay['name'],overlay['request_key']) + observed_digest = None if overlay is None else overlay['request_sha256'] + if observed_input != expected_input or observed_digest != digest: _fail('Existing destination does not belong to this exact fork request.', 'conflict', 409) if destination.reconcile(request_key, digest) is None: _fail('Existing destination lacks the canonical fork publication receipt.', 'conflict', 409) - validate_candidate(copy.deepcopy(snapshot['records']), snapshot['project_id']) declarations = _declarations(snapshot['records']) with ProjectArtifacts(destination) as artifacts: + validate_candidate(copy.deepcopy(snapshot['records']),snapshot['project_id'],artifacts) for identity in overlay['artifacts']['copied']: if artifacts.status(declarations[identity])['state'] != 'verified': _fail('Previously copied fork payloads are missing or corrupt.', 'unavailable', 409) - _source_current(source_store, source) - # Rereading reconciles identity; this barrier resolves a lost parent - # fsync acknowledgement without guessing durability from mere presence. + for item in sources: + _source_current(item['store'],item['identity']) destination.save(snapshot['checkpoint']) - return _result(destination.snapshot(), overlay, True) + result = _result(destination.snapshot(),overlay,True) + if composed:result['composition']=copy.deepcopy(document) + return result def publish_fork(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, name, expected_checkpoint, request_key, quiescent, build_candidate, validate_candidate): - """Publish a complete independent directory while preserving the source. - - Caller must hold its lifecycle/authorization guard and prove quiescence. - build_candidate(source_snapshot, destination_project_id) is one pure owned - composition callback returning exactly records/mapping/archived/artifacts. - It preserves every original envelope and appends fresh domain-constructed - values. validate_candidate(records, project_id) owns semantic closure. - No temporary or final destination is deleted after failed/unknown outcomes. + """Publish an independent single-source fork through the shared lifecycle.""" + if not callable(validate_candidate): + _fail('Fork requires complete validation.','unavailable',503) + return _publish_project(source_store,source_artifacts,trusted_root_fd,relative, + root_path=root_path,name=name,expected_checkpoint=expected_checkpoint,request_key=request_key, + quiescent=quiescent,build_candidate=build_candidate, + validate_candidate=lambda records,identity,_artifacts:validate_candidate(records,identity)) + + +def _publish_project(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, + name, expected_checkpoint, request_key, quiescent, build_candidate, + validate_candidate, retained_sources=()): + """One native stage/commit/no-clobber lifecycle for forks and composition. + + The workspace owns authorization, quiescence and all borrowed source owners. + Every source is pinned and rechecked. Composed candidates additionally prove + complete source provenance. Validators receive the actual source/destination + artifact owner; no imported runtime, permission or source path is activated. + No failed or unknown stage is deleted or selected by this owner. """ if quiescent is not True: _fail('Fork requires proven quiescence under the workspace lifecycle guard.', 'busy', 409) if not callable(build_candidate) or not callable(validate_candidate): _fail('Fork requires pure domain construction and complete validation.', 'unavailable', 503) - if type(request_key) is not str or not REQUEST_KEY.fullmatch(request_key): - _fail('Invalid fork request key.') + if type(retained_sources) not in (tuple,list):_fail('Explicit retained source owners required.') + if type(request_key) is not str or not REQUEST_KEY.fullmatch(request_key):_fail('Invalid fork request key.') _sha(expected_checkpoint) - if type(name) is not str or not name.strip() or len(name) > 200: - _fail('Fork name must contain 1–200 characters.') - _encode(name) - parts = _relative_parts(relative) - snapshot = source_store.snapshot() - if snapshot['checkpoint'] != expected_checkpoint: - _fail('Expected source checkpoint changed.', 'stale', 409) - validate_candidate(copy.deepcopy(snapshot['records']), snapshot['project_id']) - source = dict(project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint']) - expected_input = _input(source, relative, name, request_key) - digest = _digest(expected_input) - source_root = staged = artifacts = None - staging_relative = None - overlay = committed = None - published = False + if type(name) is not str or not name.strip() or len(name)>200:_fail('Fork name must contain 1–200 characters.') + _encode(name);parts=_relative_parts(relative) + owners=[dict(store=source_store,artifacts=source_artifacts,checkpoint=expected_checkpoint)] + for item in retained_sources: + _shape(item,('store','artifacts','checkpoint'),'retained source owner') + _sha(item['checkpoint']);owners.append(dict(item)) + sources=[];snapshots=[];seen=set() + for item in owners: + snapshot=item['store'].snapshot() + if snapshot['checkpoint']!=item['checkpoint']:_fail('Expected source checkpoint changed.','stale',409) + if snapshot['project_id'] in seen:_fail('Retained source project identity repeats.','conflict',409) + seen.add(snapshot['project_id']) + validate_candidate(copy.deepcopy(snapshot['records']),snapshot['project_id'],item['artifacts']) + identity={key:snapshot[key] for key in ('project_id','checkpoint')} + sources.append(dict(store=item['store'],artifacts=item['artifacts'],identity=identity)) + snapshots.append(snapshot) + composed=bool(retained_sources);source=sources[0]['identity'] + if composed: + from project_composition import composition_input + expected_input=composition_input(snapshots,relative,name,request_key) + else:expected_input=_input(source,relative,name,request_key) + digest=_digest(expected_input) + source_roots=[];staged=artifacts=None;staging_relative=None;committed=None;published=False + def sources_current(): + for item in sources:_source_current(item['store'],item['identity']) try: - with source_store.root_directory() as borrowed: - source_root = os.dup(borrowed) - with anchored_directory(trusted_root_fd, '/'.join(parts[:-1]), root_path=root_path) as parent: - if _inside(parent, source_root): - _fail('Fork destination cannot overlap the source project.') - try: - existing = os.stat(parts[-1], dir_fd=parent, follow_symlinks=False) - except FileNotFoundError: - existing = None + for item in sources: + with item['store'].root_directory() as borrowed:source_roots.append(os.dup(borrowed)) + with anchored_directory(trusted_root_fd,'/'.join(parts[:-1]),root_path=root_path) as parent: + if any(_inside(parent,root) for root in source_roots):_fail('Fork destination cannot overlap any source project.') + try:existing=os.stat(parts[-1],dir_fd=parent,follow_symlinks=False) + except FileNotFoundError:existing=None if existing is not None: - if not stat.S_ISDIR(existing.st_mode): - _fail('Fork destination is an existing or unsafe filesystem object.', 'conflict', 409) - selected = os.open(parts[-1], DIR_FLAGS, dir_fd=parent) + if not stat.S_ISDIR(existing.st_mode):_fail('Fork destination is an existing or unsafe filesystem object.','conflict',409) + selected=os.open(parts[-1],DIR_FLAGS,dir_fd=parent) try: - if _inside(source_root, selected) or _inside(selected, source_root): - _fail('Fork destination aliases or overlaps the source project.') - finally: - os.close(selected) - return _reconcile_existing(source_store, source, trusted_root_fd, relative, root_path, - expected_input, request_key, digest, validate_candidate) - staging_name = '.tuldok-fork-' + uuid.uuid4().hex - staging_relative = '/'.join([*parts[:-1], staging_name]) - staged = ProjectStore.create_at(trusted_root_fd, staging_relative, name, - 'fork-stage.' + hashlib.sha256(request_key.encode()).hexdigest(), digest, root_path=root_path) - destination_id = staged.snapshot()['project_id'] - if destination_id == source['project_id']: - _fail('Fork destination identity must be new.') - candidate = build_candidate(copy.deepcopy(snapshot), destination_id) - _shape(candidate, ('records', 'mapping', 'archived', 'artifacts'), 'owned candidate') - if type(candidate['records']) is not list: - _fail('Fork candidate records must be a list.') - records = copy.deepcopy(candidate['records']) - indexed = {} + if any(_inside(root,selected) or _inside(selected,root) for root in source_roots): + _fail('Fork destination aliases or overlaps a source project.') + finally:os.close(selected) + return _reconcile_existing(sources,trusted_root_fd,relative,root_path, + expected_input,request_key,digest,validate_candidate,composed=composed) + staging_name='.tuldok-fork-'+uuid.uuid4().hex + staging_relative='/'.join([*parts[:-1],staging_name]) + staged=ProjectStore.create_at(trusted_root_fd,staging_relative,name, + 'fork-stage.'+hashlib.sha256(request_key.encode()).hexdigest(),digest,root_path=root_path) + destination_id=staged.snapshot()['project_id'] + if destination_id in seen:_fail('Fork destination identity must be new.') + candidate=build_candidate(copy.deepcopy(snapshots if composed else snapshots[0]),destination_id) + _shape(candidate,('records','mapping','archived','artifacts'),'owned candidate') + if type(candidate['records']) is not list:_fail('Fork candidate records must be a list.') + records=copy.deepcopy(candidate['records']);indexed={} for item in records: - _shape(item, ('family', 'id', 'document'), 'candidate envelope') - _identifier(item['family'], 'Fork candidate family') - _identifier(item['id'], 'Fork candidate record ID') - identity = item['family'], item['id'] - if identity in indexed: - _fail('Fork candidate repeats a record identity.') - indexed[identity] = item - source_index = {(item['family'], item['id']): item for item in snapshot['records']} - for identity, original in source_index.items(): - if identity not in indexed or _encode(indexed[identity]) != _encode(original): - _fail('Fork must preserve every original canonical envelope exactly.') - archived = [_locator(item) for item in candidate['archived']] - if not set(archived) <= set(source_index): - _fail('Fork may archive only retained source records.') - required = {identity for identity in source_index if identity[0] in ARCHIVE_FAMILIES} - previous = decode_fork(snapshot['records'], project_id=source['project_id']) - if previous is not None: - required.update(_locator(item) for item in previous['archived']) - if not required <= set(archived): - _fail('Fork must archive every source editable value and operation receipt.') - declarations = _declarations(records) - if declarations != _declarations(snapshot['records']): - _fail('Fork must retain the source payload inventory without regenerating bytes.') - plan = _artifact_plan(candidate['artifacts'], declarations) - copied, missing = [], [] - identities = sorted(plan) - observations = source_artifacts.status_many([declarations[identity] for identity in identities]) - for identity, observation in zip(identities, observations): - status = observation['state'] - if status == 'corrupt': - _fail('Source fork payload is corrupt.', 'invalid', 400) - if status == 'missing': - if plan[identity] == 'unique': - _fail('Irreplaceable source bytes are missing; complete fork is unavailable.', 'unavailable', 409) - missing.append(identity) - elif status == 'verified': - copied.append(identity) - else: - _fail('Fork requires verified payload custody.', 'unavailable', 409) - overlay = dict(format=FORMAT, source=source, destination_project_id=destination_id, - request_key=request_key, request_sha256=digest, directory=relative, name=name, - mapping=copy.deepcopy(candidate['mapping']), archived=copy.deepcopy(candidate['archived']), - artifacts=dict(copied=copied, missing_regenerable=missing)) - records.append(dict(family=FAMILY, id=destination_id, document=overlay)) - decode_fork(records, project_id=destination_id) - validate_candidate(copy.deepcopy(records), destination_id) - _source_current(source_store, source) - artifacts = ProjectArtifacts(staged) - source_artifacts.copy_many_to(artifacts, [declarations[identity] for identity in copied]) - # Ordinary authored files are explicitly adopted text. Only a new - # task-owned fork stage materializes them; Open never calls this. + _shape(item,('family','id','document'),'candidate envelope') + _identifier(item['family'],'Fork candidate family');_identifier(item['id'],'Fork candidate record ID') + key=item['family'],item['id'] + if key in indexed:_fail('Fork candidate repeats a record identity.') + indexed[key]=item + source_index={(r['family'],r['id']):r for snapshot in snapshots for r in snapshot['records']} + if composed: + from project_composition import validate_source_memberships + validate_source_memberships(records,snapshots,destination_id) + else: + for identity,original in source_index.items(): + if identity not in indexed or _encode(indexed[identity])!=_encode(original): + _fail('Fork must preserve every original canonical envelope exactly.') + archived=[_locator(item) for item in candidate['archived']] + if not set(archived)<=set(source_index):_fail('Fork may archive only retained source records.') + required={key for key in source_index if key[0] in ARCHIVE_FAMILIES} + for snapshot in snapshots: + previous=decode_fork(snapshot['records'],project_id=snapshot['project_id']) + if previous is not None:required.update(_locator(item) for item in previous['archived']) + if not required<=set(archived):_fail('Fork must archive every source editable value and operation receipt.') + declarations=_declarations(records);expected_declarations={} + for snapshot in snapshots: + for identity,value in _declarations(snapshot['records']).items(): + if identity in expected_declarations and expected_declarations[identity]!=value: + _fail('Source artifact declarations conflict.') + expected_declarations[identity]=value + if declarations!=expected_declarations:_fail('Fork must retain the source payload inventory without regenerating bytes.') + plan=_artifact_plan(candidate['artifacts'],declarations);selected={} + for number,item in enumerate(sources): + own=_declarations(snapshots[number]['records']);identities=sorted(own) + observations=item['artifacts'].status_many([own[key] for key in identities]) + for identity,observation in zip(identities,observations): + status=observation['state'] + if status=='corrupt':_fail('Source fork payload is corrupt.','invalid',400) + if status=='verified':selected.setdefault(identity,number) + elif status!='missing':_fail('Fork requires verified payload custody.','unavailable',409) + copied=sorted(selected);missing=sorted(set(plan)-set(selected)) + if any(plan[key]=='unique' for key in missing): + _fail('Irreplaceable source bytes are missing; complete fork is unavailable.','unavailable',409) + overlay=dict(format=FORMAT,source=source,destination_project_id=destination_id, + request_key=request_key,request_sha256=_digest(_input(source,relative,name,request_key)),directory=relative,name=name, + mapping=copy.deepcopy(candidate['mapping']),archived=copy.deepcopy(candidate['archived']), + artifacts=dict(copied=copied,missing_regenerable=missing)) + records.append(dict(family=FAMILY,id=destination_id,document=overlay)) + decode_fork(records,project_id=destination_id) + validate_candidate(copy.deepcopy(records),destination_id,source_artifacts) + sources_current();artifacts=ProjectArtifacts(staged) + for number,item in enumerate(sources): + group=[declarations[key] for key in copied if selected[key]==number] + if group:item['artifacts'].copy_many_to(artifacts,group) from project_files import materialize_captured - with staged.root_directory() as destination_root: - materialize_captured(records, destination_id, destination_root) - _source_current(source_store, source) - validate_candidate(copy.deepcopy(records), destination_id) - committed = staged.commit(records, staged.snapshot()['checkpoint'], request_key, digest) - staged.save(committed['checkpoint']) - _source_current(source_store, source) - # All named content and its one canonical pointer are durable. The - # destination appears in one atomic no-overwrite directory rename. - try: - _rename_noreplace(parent, staging_name, parts[-1]) + with staged.root_directory() as destination_root:materialize_captured(records,destination_id,destination_root) + sources_current();validate_candidate(copy.deepcopy(records),destination_id,artifacts) + committed=staged.commit(records,staged.snapshot()['checkpoint'],request_key,digest) + staged.save(committed['checkpoint']);sources_current() + try:_rename_noreplace(parent,staging_name,parts[-1]) except FileExistsError as error: - raise ProjectForkError('Fork destination appeared during publication; it was not overwritten.', 'conflict', 409) from error - published = True - os.fsync(parent) - _source_current(source_store, source) - return _result(committed, overlay, False) + raise ProjectForkError('Fork destination appeared during publication; it was not overwritten.','conflict',409) from error + published=True;os.fsync(parent);sources_current() + result=_result(committed,overlay,False) + if composed: + from project_composition import decode_records + result['composition']=copy.deepcopy(decode_records(records,project_id=destination_id)[destination_id]) + return result except BaseException as error: if published: - failure = ProjectForkError('Fork destination was published but acknowledgement is uncertain; retry this exact request.', - 'unknown', 503) + failure=ProjectForkError('Fork destination was published but acknowledgement is uncertain; retry this exact request.','unknown',503) if committed is not None: - failure.receipt = dict(committed=True, selected=False, project_id=committed['project_id'], - checkpoint=committed['checkpoint'], request_key=request_key, directory=relative) - raise failure from error - if isinstance(error, OSError): - code = 'conflict' if error.errno == errno.EEXIST else 'unavailable' - failure = ProjectForkError('Fork publication is unavailable; retained staging has not been selected.', code, - 409 if code == 'conflict' else 503) - failure.staging_directory = staging_relative + failure.receipt=dict(committed=True,selected=False,project_id=committed['project_id'],checkpoint=committed['checkpoint'],request_key=request_key,directory=relative) raise failure from error - if staging_relative is not None: - error.staging_directory = staging_relative + if isinstance(error,OSError): + code='conflict' if error.errno==errno.EEXIST else 'unavailable' + failure=ProjectForkError('Fork publication is unavailable; retained staging has not been selected.',code,409 if code=='conflict' else 503) + failure.staging_directory=staging_relative;raise failure from error + if staging_relative is not None:error.staging_directory=staging_relative raise finally: - if artifacts is not None: - artifacts.close() - if staged is not None: - staged.close() - if source_root is not None: - os.close(source_root) + if artifacts is not None:artifacts.close() + if staged is not None:staged.close() + for descriptor in source_roots:os.close(descriptor) diff --git a/project_projection.py b/project_projection.py index a9ca2f94..fca264be 100644 --- a/project_projection.py +++ b/project_projection.py @@ -126,9 +126,10 @@ def schema(self): def execute(self, *args, **kwargs): return self.db.execute(*args, **kwargs) - def validate_records(self, records, *, project_id=None): + def validate_records(self, records, *, project_id=None, artifacts=None): + artifacts = self.artifacts if artifacts is None else artifacts project_id = self.project_id if project_id is None else project_id - claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( + claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) identities = set() declarations = {} @@ -145,20 +146,22 @@ def validate_records(self, records, *, project_id=None): decode_data_records(records) from project_fork import decode_fork decode_fork(records, project_id=project_id) + from project_composition import decode_records as decode_compositions + decode_compositions(records, project_id=project_id) from project_transfer import decode_records as decode_extraction_records decode_extraction_records(records) operation_records(records, project_id=project_id) decode_file_records(records) - decode_native_records(records,artifacts=self.artifacts) + decode_native_records(records,artifacts=artifacts) materials = MaterialSnapshot(records) for blob in materials.artifact_blobs(): if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): raise WorkbenchError('Material artifact declaration is missing or inconsistent.') # Resolve every cross-owner reference against THIS candidate, rather # than a resolver that still sees the preceding committed checkpoint. - decode_producer_records(records, artifacts=self.artifacts) - decode_history_records(records, materials=materials, artifacts=self.artifacts) - decode_measurement_records(records, artifacts=self.artifacts) + decode_producer_records(records, artifacts=artifacts) + decode_history_records(records, materials=materials, artifacts=artifacts) + decode_measurement_records(records, artifacts=artifacts) for receipt in operation_documents(records): operation, result = receipt['operation'], receipt['result'] if (validate_data_result(operation, result, records) diff --git a/project_store.py b/project_store.py index 31a184bf..bc4b4071 100644 --- a/project_store.py +++ b/project_store.py @@ -54,6 +54,7 @@ 'project_artifact': 'project_artifact_v1', 'project_operation': 'project_operation_v1', 'project_fork': 'project_fork_v1', + 'project_composition': 'project_composition_v1', 'project_file': 'project_file_v1', 'project_extraction': 'project_extraction_v1', 'generation_history': 'project_generation_history_v1', diff --git a/project_workspace.py b/project_workspace.py index 83f9597b..efe93f9a 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -5,6 +5,7 @@ The installation catalog holds local credentials; it is never portable content. """ import copy +from contextlib import ExitStack import hashlib import json import os @@ -377,6 +378,12 @@ def _record_result(self, record, *, before_revision=None, history_limit=1): next_history_before_revision=page[-1]['revision'] if len(history) > len(page) else None) def _save_as(self, arguments): + return self._publish_independent(arguments) + + def _compose(self, arguments): + return self._publish_independent(arguments, composed=True) + + def _publish_independent(self, arguments, *, composed=False): """Publish a new independent project, then select a fresh inert cache.""" from functools import partial from project_fork import publish_fork @@ -390,13 +397,35 @@ def _save_as(self, arguments): self._store.save(arguments['expected_checkpoint']) published = None incoming = projection = None + retained = ExitStack() try: - published = publish_fork(self._store, source.artifacts, root_fd, arguments['directory'], - root_path=root_path, name=arguments['name'], - expected_checkpoint=arguments['expected_checkpoint'], request_key=arguments['request_key'], - quiescent=True, build_candidate=partial(build_candidate, created_at=timestamp(), - id_factory=lambda _family, _source_id, _revision: uuid.uuid4().hex), - validate_candidate=lambda records, identity: source.validate_records(records, project_id=identity)) + if composed: + from project_composition import publish_composition + from project_artifacts import ProjectArtifacts + others=[];identities={self._store.snapshot()['project_id']} + for selected in arguments['sources']: + if selected['project_id'] in identities: + raise WorkbenchError('Composition source identity repeats.','conflict',409) + identities.add(selected['project_id']) + directory,descriptor=self._root(selected['root'],selected['directory']) + store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory)) + actual=store.snapshot() + if actual['project_id']!=selected['project_id'] or actual['checkpoint']!=selected['checkpoint']: + raise WorkbenchError('Selected composition source checkpoint differs.','stale_project',409) + artifacts=retained.enter_context(ProjectArtifacts(store)) + others.append(dict(store=store,artifacts=artifacts,checkpoint=selected['checkpoint'])) + published=publish_composition(self._store,source.artifacts,others,root_fd,arguments['directory'], + root_path=root_path,name=arguments['name'],expected_checkpoint=arguments['expected_checkpoint'], + request_key=arguments['request_key'],quiescent=True,created_at=timestamp(), + id_factory=lambda *_:uuid.uuid4().hex, + validate_candidate=lambda records,identity,artifacts:source.validate_records(records,project_id=identity,artifacts=artifacts)) + else: + published = publish_fork(self._store, source.artifacts, root_fd, arguments['directory'], + root_path=root_path, name=arguments['name'], + expected_checkpoint=arguments['expected_checkpoint'], request_key=arguments['request_key'], + quiescent=True, build_candidate=partial(build_candidate, created_at=timestamp(), + id_factory=lambda _family, _source_id, _revision: uuid.uuid4().hex), + validate_candidate=lambda records, identity: source.validate_records(records, project_id=identity)) # A filesystem editor can act outside the workspace lock. Leave the # complete fork unselected if its source working files changed. source.files.require_clean() @@ -415,7 +444,10 @@ def _save_as(self, arguments): request_key=arguments['request_key'], reconciled=published['reconciled'], mapping_count=len(overlay['mapping']), copied_count=len(overlay['artifacts']['copied']), missing_regenerable_count=len(overlay['artifacts']['missing_regenerable'])) - return dict(state=self._state(), fork=summary) + result=dict(state=self._state(), fork=summary) + if composed: + result['sources']=[{key:item[key] for key in ('project_id','checkpoint')} for item in published['composition']['sources']] + return result except BaseException as error: if projection is not None and projection is not self._projection: projection.close() @@ -430,11 +462,13 @@ def _save_as(self, arguments): elif receipt is not None: receipt = dict(receipt, root=arguments['root']) if receipt is not None: - failure = WorkbenchError('Independent project was published but could not be selected; retry this exact Save As.', + failure = WorkbenchError('Independent project was published but could not be selected; reconcile this exact request.', 'project_selection_unavailable', 503) failure.receipt = receipt raise failure from error raise + finally: + retained.close() def dispatch(self, name, arguments, *, selected_version=None): """Pin workspace -> current host authority -> projection through return. @@ -457,7 +491,7 @@ def dispatch(self, name, arguments, *, selected_version=None): if scope == 'execute' and action in ('execution_submit_external', 'execution_study_admit'): self.access.require_external_submission() else: - self.access.require(scope, name, owner_only=action in ('new', 'open', 'save', 'save_as', 'close')) + self.access.require(scope, name, owner_only=action in ('new', 'open', 'save', 'save_as', 'compose', 'close')) if selected_version == 1 and action in ('get', 'annotate'): self._bound(arguments, check_checkpoint=action == 'get') record = self._projection.data._require_record(arguments['record_id']) @@ -466,7 +500,7 @@ def dispatch(self, name, arguments, *, selected_version=None): 'unsupported', 409) if name in agent_api.PROJECT_FILES_M4 and not M4_ENABLED: raise WorkbenchError('This project capability is not yet available.', 'unavailable', 409) - if name in (agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | agent_api.PROJECT_FILES_M4) - {'project_files.save_as'}: + if name in (agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | agent_api.PROJECT_FILES_M4) - {'project_files.save_as','project_files.compose'}: if ((name in agent_api.PROJECT_FILES_M2 and not M2_ENABLED) or (name in agent_api.PROJECT_FILES_M3 and not M3_ENABLED) or (name in agent_api.PROJECT_FILES_M4 and not M4_ENABLED)): @@ -496,7 +530,7 @@ def dispatch(self, name, arguments, *, selected_version=None): if prior is not None: if prior[0] != signature: raise WorkbenchError('Request key already has different input.', 'conflict', 409) - prior_state = prior[1]['state'] if action == 'save_as' else prior[1] + prior_state = prior[1]['state'] if action in ('save_as','compose') else prior[1] if prior_state != self._state(): raise WorkbenchError('This lifecycle receipt belongs to an obsolete workspace state.', 'stale_project', 409) result = copy.deepcopy(prior[1]) @@ -510,6 +544,8 @@ def dispatch(self, name, arguments, *, selected_version=None): result = self._state() elif action == 'save_as': result = self._save_as(arguments) + elif action == 'compose': + result = self._compose(arguments) elif action == 'close': self._bound(arguments) self._detach() @@ -542,11 +578,11 @@ def dispatch(self, name, arguments, *, selected_version=None): raise WorkbenchError('Unsupported project-file operation.', 'unsupported', 400) except BaseException as error: code = getattr(error, 'code', None) - if action != 'save_as' and code in ('unknown', 'outcome_unknown', 'projection_unavailable'): + if action not in ('save_as','compose') and code in ('unknown', 'outcome_unknown', 'projection_unavailable'): reason = 'unknown_publication' if code in ('unknown', 'outcome_unknown') else 'projection_unavailable' self._recovery_required(reason, arguments.get('request_key')) raise - if arguments.get('request_key') and action in ('new', 'open', 'save', 'save_as', 'close'): + if arguments.get('request_key') and action in ('new', 'open', 'save', 'save_as', 'compose', 'close'): self._receipts[key] = (signature, copy.deepcopy(result)) if len(agent_api.encoded(result)) > agent_api.RESPONSE_LIMIT: raise WorkbenchError('This project result exceeds the synchronous response bound; canonical work is retained.', diff --git a/tests/test_project_composition.py b/tests/test_project_composition.py index 7ad20a9a..cf42c0d3 100644 --- a/tests/test_project_composition.py +++ b/tests/test_project_composition.py @@ -16,6 +16,8 @@ def snapshot(identity, records): def approved_history(): run=legacy_run();run['approval_id']='9'*32 + run['release_id']=run['release_sha256']=hashlib.sha256(b'reference input').hexdigest() + run['release_bytes']=len(b'reference input') grant=dict(id=run['approval_id'],catalog_id=ORIGIN,approver_actor_id='a'*32,approver_credential_id='b'*32, consumer_actor_id='5'*32,material_sha256='f'*64,created_at=1,expires_at=100,revoked_at=None) consumption=dict(grant_id=grant['id'],run_id=run['id'],admission_id=run['admission_id'], diff --git a/tests/test_project_composition_publication.py b/tests/test_project_composition_publication.py new file mode 100644 index 00000000..67764875 --- /dev/null +++ b/tests/test_project_composition_publication.py @@ -0,0 +1,155 @@ +"""Real native stage/custody/reconciliation across multiple independent sources.""" +import copy +import errno +import hashlib +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import uuid + +from project_artifacts import ProjectArtifacts +from project_composition import decode_records, publish_composition +from project_fork import decode_fork, publish_fork +from project_fork_composition import build_candidate as fork_candidate +from project_history import decode_records as history_records +from project_store import DIR_FLAGS, ProjectStore, ProjectStoreError +from test_project_composition import approved_history + +NOW='2026-10-24T12:00:00+00:00' + + +def validate(records,identity,artifacts): + decode_fork(records,project_id=identity) + decode_records(records,project_id=identity) + history_records(records,artifacts=artifacts) + + +class CompositionPublicationTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory(prefix='native-composition-');self.addCleanup(self.tmp.cleanup) + self.root=Path(self.tmp.name);self.fd=os.open(self.root,DIR_FLAGS);self.addCleanup(os.close,self.fd) + self.stores=[];self.artifacts=[];self.snapshots=[];self.unique=[] + for number in range(2): + store=ProjectStore.create(self.root/('source'+str(number)),'Source '+str(number),'create','a'*64) + self.addCleanup(store.close);artifacts=ProjectArtifacts(store);self.addCleanup(artifacts.close) + records=approved_history() + if number:records[0]['document']['admission_proof']['grant']['revoked_at']=3 + unique=('unique source '+str(number)).encode();sha=hashlib.sha256(unique).hexdigest() + declaration=dict(format='project_artifact_v1',sha256=sha,bytes=len(unique)) + records.append(dict(family='project_artifact',id=sha,document=declaration)) + for raw in (b'reference input',unique): + artifacts.publish_bytes(raw,dict(format='project_artifact_v1',sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw))) + snap=store.commit(records,store.snapshot()['checkpoint'],'seed','b'*64) + self.stores.append(store);self.artifacts.append(artifacts);self.snapshots.append(snap);self.unique.append(sha) + + def publish(self,directory='combined',**options): + arguments=dict(root_path=self.root,name='Combined',expected_checkpoint=self.snapshots[0]['checkpoint'], + request_key='compose-exact',quiescent=True,created_at=NOW,id_factory=lambda *_:uuid.uuid4().hex,validate_candidate=validate) + arguments.update(options) + return publish_composition(self.stores[0],self.artifacts[0], + [dict(store=self.stores[1],artifacts=self.artifacts[1],checkpoint=self.snapshots[1]['checkpoint'])], + self.fd,directory,**arguments) + + def assert_code(self,code,call): + with self.assertRaises(ProjectStoreError) as caught:call() + self.assertEqual(caught.exception.code,code) + return caught.exception + + def test_complete_publication_preserves_both_sources_and_independent_payloads(self): + before=copy.deepcopy(self.snapshots) + result=self.publish();snapshot=result['snapshot'];identity=snapshot['project_id'] + self.assertFalse(result['reconciled']) + self.assertEqual(len(history_records(snapshot['records']).runs),1) + self.assertEqual(len(history_records(snapshot['records']).observations),1) + self.assertEqual(len(decode_records(snapshot['records'],project_id=identity)[identity]['sources']),2) + with ProjectStore.open(self.root/'combined') as destination,ProjectArtifacts(destination) as owner: + for sha in result['fork']['artifacts']['copied']: + source=next(s.path/'artifacts/sha256'/sha for s in self.stores if (s.path/'artifacts/sha256'/sha).exists()) + target=destination.path/'artifacts/sha256'/sha + self.assertEqual(target.read_bytes(),source.read_bytes()) + self.assertNotEqual((source.stat().st_dev,source.stat().st_ino),(target.stat().st_dev,target.stat().st_ino)) + self.assertEqual(target.stat().st_nlink,1) + self.assertEqual([s.snapshot() for s in self.stores],before) + + def test_missing_unique_secondary_bytes_leave_only_retained_staging(self): + (self.stores[1].path/'artifacts/sha256'/self.unique[1]).unlink() + self.assert_code('unavailable',self.publish) + self.assertFalse((self.root/'combined').exists()) + self.assertTrue(list(self.root.glob('.tuldok-fork-*'))) + + def test_missing_duplicate_is_satisfied_by_verified_primary_bytes(self): + sha=hashlib.sha256(b'reference input').hexdigest() + (self.stores[1].path/'artifacts/sha256'/sha).unlink() + result=self.publish() + self.assertIn(sha,result['fork']['artifacts']['copied']) + self.assertEqual((self.root/'combined/artifacts/sha256'/sha).read_bytes(),b'reference input') + + def test_corrupt_shared_secondary_copy_does_not_fall_back_to_good_primary(self): + sha=hashlib.sha256(b'reference input').hexdigest() + target=self.stores[1].path/'artifacts/sha256'/sha + target.chmod(0o600);target.write_bytes(b'wrong') + self.assert_code('invalid',self.publish) + self.assertFalse((self.root/'combined').exists()) + + def test_secondary_checkpoint_change_during_copy_prevents_publication(self): + copy_many=self.artifacts[0].copy_many_to + def changed(destination,declarations): + copy_many(destination,declarations) + source=self.stores[1];snapshot=source.snapshot() + source.commit(snapshot['records'],snapshot['checkpoint'],'new-source-receipt','c'*64) + with patch.object(self.artifacts[0],'copy_many_to',side_effect=changed): + self.assert_code('stale',self.publish) + self.assertFalse((self.root/'combined').exists()) + + def test_lost_publication_ack_reconciles_exact_sources_without_reconstruction(self): + fsync=os.fsync;injected=False + def fail_after_publish(fd): + nonlocal injected + if (self.root/'combined').exists() and not injected: + injected=True;raise OSError(errno.EIO,'bounded lost-ack fixture') + fsync(fd) + with patch('project_fork.os.fsync',side_effect=fail_after_publish): + failure=self.assert_code('unknown',self.publish) + self.assertTrue(failure.receipt['committed']) + with patch('project_composition.build_candidate',side_effect=AssertionError('No replayed construction')): + replay=self.publish() + self.assertTrue(replay['reconciled']) + self.assertEqual(replay['snapshot']['checkpoint'],failure.receipt['checkpoint']) + self.assert_code('conflict',lambda:self.publish(name='Changed input')) + + def test_destination_cannot_overlap_secondary_source(self): + self.assert_code('invalid',lambda:self.publish('source1/nested')) + self.assertFalse((self.stores[1].path/'nested').exists()) + + def test_persisted_membership_tampering_is_rejected(self): + result=self.publish();records=copy.deepcopy(result['snapshot']['records']) + doc=next(r['document'] for r in records if r['family']=='project_composition') + source=doc['sources'][1];family=next(iter(source['records']));identity=next(iter(source['records'][family])) + source['records'][family][identity]='0'*64 + error=self.assert_code('invalid',lambda:decode_records(records,project_id=result['snapshot']['project_id'])) + self.assertIn('exact retained envelope',str(error)) + + def test_malformed_persisted_run_memberships_have_typed_invalid_outcomes(self): + result=self.publish() + for kind in ('non-object','null-observation'): + records=copy.deepcopy(result['snapshot']['records']) + doc=next(r['document'] for r in records if r['family']=='project_composition') + source=doc['sources'][1] + if kind=='non-object':source['records']['run_history']=1 + else:source['run_observations'][next(iter(source['records']['run_history']))]=None + with self.subTest(kind=kind):self.assert_code('invalid',lambda:decode_records(records)) + + def test_future_ordinary_fork_preserves_and_archives_composition_provenance(self): + first=self.publish() + with ProjectStore.open(self.root/'combined') as source,ProjectArtifacts(source) as artifacts: + result=publish_fork(source,artifacts,self.fd,'descendant',root_path=self.root,name='Descendant', + expected_checkpoint=first['snapshot']['checkpoint'],request_key='fork-composition',quiescent=True, + build_candidate=lambda snap,dest:fork_candidate(snap,dest,created_at=NOW,id_factory=lambda *_:uuid.uuid4().hex), + validate_candidate=lambda records,identity:validate(records,identity,artifacts)) + self.assertIn(dict(family='project_composition',id=first['snapshot']['project_id']),result['fork']['archived']) + self.assertIn(first['snapshot']['project_id'],decode_records(result['snapshot']['records'],project_id=result['snapshot']['project_id'])) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_project_composition_workspace.py b/tests/test_project_composition_workspace.py new file mode 100644 index 00000000..af5134e3 --- /dev/null +++ b/tests/test_project_composition_workspace.py @@ -0,0 +1,87 @@ +"""Owner-authorized native composition, editable notes and selection recovery.""" +import copy +import hashlib +import os +import unittest +from unittest.mock import patch + +import agent_api +from access import AccessError +from project_artifacts import ProjectArtifacts +from project_composition import decode_records +from project_files import file_id +from project_store import ProjectStore +from workbench import WorkbenchError +from test_project_composition import approved_history +from test_project_workspace import ProjectWorkspaceTests + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'),'Requires disposable PostgreSQL') +class CompositionWorkspaceTests(unittest.TestCase): + def setUp(self): + self.fixture=ProjectWorkspaceTests('test_lifecycle_replay_is_idempotent_only_while_resulting_state_is_current') + self.fixture.setUp();self.addCleanup(self.fixture.doCleanups) + f=self.fixture;created,_=f.create();self.primary_id=created['project']['project_id'] + for revision,text in enumerate(('Original decision Ω\r\n','Updated decision Ω\r\n')): + raw=text.encode();target=f.external/'alpha/notes/decision.md';target.parent.mkdir(exist_ok=True) + target.write_bytes(raw) + f.call('file_adopt',f.mutation(kind='note',path='notes/decision.md',expected_revision=revision, + expected_sha256=hashlib.sha256(raw).hexdigest())) + self.primary=f.workspace._store.snapshot() + with ProjectStore.create(f.external/'retained','Retained source','create','a'*64) as store,ProjectArtifacts(store) as artifacts: + records=approved_history();raw=b'reference input' + artifacts.publish_bytes(raw,dict(format='project_artifact_v1',sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw))) + self.secondary=store.commit(records,store.snapshot()['checkpoint'],'seed','b'*64) + self.arguments=f.mutation(root='fixture',directory='combined',name='Combined scientific history',sources=[dict( + root='fixture',directory='retained',project_id=self.secondary['project_id'],checkpoint=self.secondary['checkpoint'])]) + + def compose(self):return self.fixture.call('compose',self.arguments) + + def test_native_owner_composition_preserves_note_history_and_selects_editable_primary_fork(self): + f=self.fixture + with patch('subprocess.Popen',side_effect=AssertionError('Imported projects never execute')): + result=self.compose() + state=result['state'];self.assertEqual(state['state'],'open') + self.assertNotEqual(state['project']['project_id'],self.primary_id) + self.assertEqual(result['sources'],[{key:s[key] for key in ('project_id','checkpoint')} for s in (self.primary,self.secondary)]) + snapshot=f.workspace._store.snapshot();identity=snapshot['project_id'] + decode_records(snapshot['records'],project_id=identity) + old_note=next(r for r in self.primary['records'] if r['family']=='project_file') + self.assertIn(old_note,snapshot['records']);self.assertEqual(len(old_note['document']['history']),2) + adopted=file_id(identity,'note','notes/decision.md') + value=f.call('file_get',dict(binding=state['binding'],checkpoint=state['project']['checkpoint'],file_id=adopted))['value'] + self.assertEqual(value['version']['text'],'Updated decision Ω\r\n') + raw=b'Next independent decision\n';target=f.external/'combined/notes/decision.md';target.write_bytes(raw) + updated=f.call('file_adopt',f.mutation(kind='note',path='notes/decision.md',expected_revision=value['version']['revision'], + expected_sha256=hashlib.sha256(raw).hexdigest())) + self.assertEqual(updated['value']['revision'],value['version']['revision']+1) + agent_api.validate_output('project_files.compose',dict(status=200,result=result),version=1) + + def test_failed_selection_preserves_primary_and_exact_retry_reconciles_destination(self): + f=self.fixture;project=f.workspace._projection_for + def fail_new(store): + if store.path.name=='combined':raise WorkbenchError('Bounded selection fixture failure.','unavailable',503) + return project(store) + with patch.object(f.workspace,'_projection_for',side_effect=fail_new),self.assertRaises(WorkbenchError) as caught:self.compose() + self.assertEqual(caught.exception.code,'project_selection_unavailable') + self.assertTrue(caught.exception.receipt['committed']);self.assertFalse(caught.exception.receipt['selected']) + self.assertEqual(f.state()['project']['project_id'],self.primary_id) + replay=self.compose();self.assertTrue(replay['fork']['reconciled']) + self.assertEqual(replay['state']['project']['project_id'],caught.exception.receipt['project_id']) + + def test_wrong_secondary_checkpoint_is_rejected_before_destination_publication(self): + self.arguments['sources'][0]['checkpoint']='0'*64 + with self.assertRaises(WorkbenchError) as caught:self.compose() + self.assertEqual(caught.exception.code,'stale_project') + self.assertFalse((self.fixture.external/'combined').exists()) + self.assertEqual(self.fixture.state()['project']['project_id'],self.primary_id) + + def test_anonymous_caller_is_denied_before_opening_retained_sources(self): + with patch.object(ProjectStore,'open_at',side_effect=AssertionError('No unauthorized filesystem selection')): + with self.assertRaises(AccessError) as caught: + self.fixture.workspace.dispatch('project_files.compose',self.arguments) + self.assertEqual(caught.exception.status,401) + self.assertFalse((self.fixture.external/'combined').exists()) + + +if __name__=='__main__':unittest.main() From 84960fb702e4c0f9eae1fc2fdce5e0791fec358a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 12:53:20 -0700 Subject: [PATCH 20/33] fix: preserve artifact policies before combining project roots --- project_composition.py | 17 ++++++++++++++++- tests/test_project_composition.py | 13 +++++++++++++ tests/test_project_composition_publication.py | 17 +++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/project_composition.py b/project_composition.py index bcb8a795..53bc04f3 100644 --- a/project_composition.py +++ b/project_composition.py @@ -268,8 +268,23 @@ def allocate(family,identity,revision): records=[indexed[key] for key in sorted(indexed)] validate_source_memberships(records,snapshots,destination_project_id) archived={(r['family'],r['id']) for r in merged['archived']+fork['archived']} + # Every input has its own valid current fork. The combined records receive + # their one new current overlay only in the publication owner. Ask each + # source owner for its preservation policy before combining those roots; + # independent editable forks introduce no new payload identities. + plans={} + inventories=[fork['artifacts']] + inventories.extend(artifact_plan(source['records']) for source in snapshots[1:]) + for inventory in inventories: + for edge in inventory: + sha=edge['declaration']['sha256'];previous=plans.get(sha) + if previous is not None and previous['declaration']!=edge['declaration']: + _fail('Composition source artifact declarations conflict.') + if previous is None or edge['preservation']=='unique': + plans[sha]=copy.deepcopy(edge) return dict(records=records,mapping=copy.deepcopy(fork['mapping']), - archived=[dict(family=f,id=i) for f,i in sorted(archived)],artifacts=artifact_plan(records)) + archived=[dict(family=f,id=i) for f,i in sorted(archived)], + artifacts=[plans[sha] for sha in sorted(plans)]) def publish_composition(primary_store, primary_artifacts, retained_sources, trusted_root_fd, directory, diff --git a/tests/test_project_composition.py b/tests/test_project_composition.py index cf42c0d3..c20a90df 100644 --- a/tests/test_project_composition.py +++ b/tests/test_project_composition.py @@ -99,6 +99,19 @@ def test_other_family_conflict_and_repeated_source_project_are_rejected(self): self.assertEqual(repeated.exception.code,'conflict') self.assertIn('source project identity repeats',str(repeated.exception)) + def test_source_unique_policy_overrides_another_sources_runtime_dependency(self): + from project_composition import build_candidate + from test_project_fork_composition import runtime_records + import uuid + sha=hashlib.sha256(b'Z').hexdigest() + runtime,_,_=runtime_records('9'*32,shared_sha=sha,shared_bytes=1) + declaration=next(r for r in runtime if r['family']=='project_artifact' and r['id']==sha) + candidate=build_candidate([snapshot('1',runtime),snapshot('2',[copy.deepcopy(declaration)])], + '3'*32,created_at='2026-10-24T12:00:00+00:00',id_factory=lambda *_:uuid.uuid4().hex, + directory='combined',name='Conservative source custody',request_key='shared-policy') + policies={edge['declaration']['sha256']:edge['preservation'] for edge in candidate['artifacts']} + self.assertEqual(policies[sha],'unique') + def test_source_snapshots_can_be_loaded_lazily(self): records=approved_history() def sources(): diff --git a/tests/test_project_composition_publication.py b/tests/test_project_composition_publication.py index 67764875..0bb6d881 100644 --- a/tests/test_project_composition_publication.py +++ b/tests/test_project_composition_publication.py @@ -73,6 +73,23 @@ def test_complete_publication_preserves_both_sources_and_independent_payloads(se self.assertEqual(target.stat().st_nlink,1) self.assertEqual([s.snapshot() for s in self.stores],before) + def test_already_forked_sources_keep_their_independent_histories(self): + for number in range(2): + result=publish_fork(self.stores[number],self.artifacts[number],self.fd,'forked-source-'+str(number), + root_path=self.root,name='Already forked source',expected_checkpoint=self.snapshots[number]['checkpoint'], + request_key='prepare-fork-'+str(number),quiescent=True, + build_candidate=lambda snap,dest:fork_candidate(snap,dest,created_at=NOW,id_factory=lambda *_:uuid.uuid4().hex), + validate_candidate=lambda records,identity:validate(records,identity,None)) + store=ProjectStore.open(self.root/('forked-source-'+str(number))) + artifacts=ProjectArtifacts(store);self.addCleanup(store.close);self.addCleanup(artifacts.close) + self.stores[number]=store;self.artifacts[number]=artifacts;self.snapshots[number]=result['snapshot'] + source_overlays=[next(r for r in s['records'] if r['family']=='project_fork') for s in self.snapshots] + result=self.publish();records=result['snapshot']['records'];identity=result['snapshot']['project_id'] + self.assertTrue(all(old in records for old in source_overlays)) + self.assertTrue(all(dict(family='project_fork',id=old['id']) in result['fork']['archived'] for old in source_overlays)) + self.assertEqual(decode_fork(records,project_id=identity)['destination_project_id'],identity) + self.assertIn(identity,decode_records(records,project_id=identity)) + def test_missing_unique_secondary_bytes_leave_only_retained_staging(self): (self.stores[1].path/'artifacts/sha256'/self.unique[1]).unlink() self.assert_code('unavailable',self.publish) From 41974aea385b1e496542b1278eb0af8edce8769e Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 15:06:34 -0700 Subject: [PATCH 21/33] feat: recover committed project publication without payload copies --- project_files.py | 43 ++++---- project_fork.py | 92 ++++++++++++++++ project_projection.py | 107 ++++++++++--------- tests/test_project_publication_recovery.py | 116 +++++++++++++++++++++ 4 files changed, 289 insertions(+), 69 deletions(-) create mode 100644 tests/test_project_publication_recovery.py diff --git a/project_files.py b/project_files.py index 08829a6a..c312aa34 100644 --- a/project_files.py +++ b/project_files.py @@ -231,6 +231,30 @@ def _read(root,parts): os.close(parent) +def inspect_captured(records, project_id, root): + """Observe captured working files using only a locked project capability.""" + owned=[r for r in decode_records(records) if r['document']['project_id']==project_id] + items=[] + for envelope in sorted(owned,key=lambda r:r['document']['path']): + doc=envelope['document'];value=doc['current'] + observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], + expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) + try: + parts=_path(doc['kind'],doc['path']) + if value.get('deleted') is True and _missing(root,parts): + observation['state']='deleted' + else: + if value.get('deleted') is True:observation['state']='changed' + _,observed,size=_read(root,parts) + observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') + except _WorkingError as error: + if value.get('deleted') is True: + observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) + else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) + items.append(observation) + return items + + class ProjectFiles: def __init__(self,session): self.session,self.access,self.db,self.store=session,session.access,session.db,session.store @@ -238,25 +262,8 @@ def __init__(self,session): def inspect(self): with self.access.lock,self.session.lock,self.db.lock: self.access.require('inspect','files.inspect');snapshot=self.session.check() - owned=[r for r in decode_records(snapshot['records']) if r['document']['project_id']==snapshot['project_id']];items=[] with self.store.root_directory() as root: - for envelope in sorted(owned,key=lambda r:r['document']['path']): - doc=envelope['document'];value=doc['current'] - observation=dict(id=envelope['id'],kind=doc['kind'],path=doc['path'],revision=value['revision'], - expected_sha256=value['sha256'],expected_bytes=value['bytes'],observed_sha256=None,observed_bytes=None,state='missing',reason=None) - try: - parts=_path(doc['kind'],doc['path']) - if value.get('deleted') is True and _missing(root,parts): - observation['state']='deleted' - else: - if value.get('deleted') is True:observation['state']='changed' - _,observed,size=_read(root,parts) - observation.update(observed_sha256=observed,observed_bytes=size,state='unchanged' if observed==value['sha256'] else 'changed') - except _WorkingError as error: - if value.get('deleted') is True: - observation.update(state='changed' if error.state in ('invalid_utf8','too_large') else 'invalid',reason=error.state) - else:observation.update(state='missing' if error.state=='missing' else 'invalid',reason=error.state) - items.append(observation) + items=inspect_captured(snapshot['records'],snapshot['project_id'],root) self.session.check() return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) diff --git a/project_fork.py b/project_fork.py index 7aec1a44..bd928ad9 100644 --- a/project_fork.py +++ b/project_fork.py @@ -279,6 +279,98 @@ def publish_fork(source_store, source_artifacts, trusted_root_fd, relative, *, r validate_candidate=lambda records,identity,_artifacts:validate_candidate(records,identity)) +def recover_committed_publication(trusted_root_fd, staging, relative, *, root_path, + expected_project_id, expected_checkpoint, + expected_request, quiescent): + """Publish one explicitly selected committed stage without rebuilding it. + + This operator recovery accepts a frozen historical snapshot, not a new read + of its original sources. The caller owns current filesystem authority and + cessation of the former process. The store's exclusive root lock, complete + domain validation, exact original request/commit receipt, payload custody, + and clean captured files are required before a no-overwrite rename. No SQL + projection, source permission, source runtime or worker is restored here. + After an uncertain rename, repeat this exact recovery to inspect the final + destination; never repeat composition or create another stage. + """ + if quiescent is not True: + _fail('Recovery requires proven cessation of the former publication owner.', 'busy', 409) + _uuid(expected_project_id);_sha(expected_checkpoint) + stage_parts=_relative_parts(staging);parts=_relative_parts(relative) + if (stage_parts[:-1]!=parts[:-1] + or not re.fullmatch(r'\.tuldok-fork-[a-f0-9]{32}',stage_parts[-1]) + or stage_parts==parts): + _fail('Recovery requires an explicit fork stage beside its original destination.') + if type(expected_request) is not dict or expected_request.get('directory')!=relative: + _fail('Recovery requires the exact original publication request.') + key=expected_request.get('request_key') + if type(key) is not str or not REQUEST_KEY.fullmatch(key): + _fail('Recovery requires the original publication request key.') + digest=_digest(expected_request) + published=False + try: + with anchored_directory(trusted_root_fd,'/'.join(parts[:-1]),root_path=root_path) as parent: + def exists(name): + try:os.stat(name,dir_fd=parent,follow_symlinks=False);return True + except FileNotFoundError:return False + stage_exists,final_exists=exists(stage_parts[-1]),exists(parts[-1]) + if stage_exists and final_exists: + _fail('Both stage and destination exist; neither was replaced.', 'conflict', 409) + if not stage_exists and not final_exists: + _fail('Neither the selected stage nor its destination is available.', 'unavailable', 409) + selected=relative if final_exists else staging + with ProjectStore.open_at(trusted_root_fd,selected,root_path=root_path) as store: + snapshot=store.snapshot() + if (snapshot['project_id']!=expected_project_id + or snapshot['checkpoint']!=expected_checkpoint): + _fail('Recovery project identity or checkpoint differs.', 'stale', 409) + overlay=decode_fork(snapshot['records'],project_id=expected_project_id) + if overlay is None:_fail('Recovery requires a committed independent publication.') + from project_composition import decode_records, composition_input + composition=decode_records(snapshot['records'],project_id=expected_project_id).get(expected_project_id) + observed=(composition_input(composition['sources'],composition['directory'],composition['name'],composition['request_key']) + if composition is not None else _input(overlay['source'],overlay['directory'],overlay['name'],overlay['request_key'])) + if observed!=expected_request or store.reconcile(key,digest) is None: + _fail('Committed stage does not match the original request and receipt.', 'conflict', 409) + from project_projection import validate_project_records + from project_files import inspect_captured + declarations=_declarations(snapshot['records']) + with ProjectArtifacts(store) as artifacts: + validate_project_records(snapshot['records'],project_id=expected_project_id,artifacts=artifacts) + checks=artifacts.status_many([declarations[identity] for identity in overlay['artifacts']['copied']]) + if any(item['state']!='verified' for item in checks): + _fail('Committed publication payloads are missing or corrupt.', 'unavailable', 409) + with store.root_directory() as project_root: + if any(item['state'] not in ('unchanged','deleted') + for item in inspect_captured(snapshot['records'],expected_project_id,project_root)): + _fail('Committed stage has changed or missing working files.', 'working_files_changed', 409) + store.save(expected_checkpoint) + if not final_exists: + # Recheck the selected directory entry while the store root + # remains locked; an external rename must not substitute it. + with store.root_directory() as project_root: + entry=os.stat(stage_parts[-1],dir_fd=parent,follow_symlinks=False) + actual=os.fstat(project_root) + if (entry.st_dev,entry.st_ino)!=(actual.st_dev,actual.st_ino): + _fail('Selected stage directory changed.', 'stale', 409) + _rename_noreplace(parent,stage_parts[-1],parts[-1]);published=True + os.fsync(parent) + return dict(project_id=expected_project_id,checkpoint=expected_checkpoint, + directory=relative,request_key=key,reconciled=final_exists, + payloads_copied=0,selected=False) + except BaseException as error: + # Anchored directory owners also translate IO failures. Preserve the + # uncertain post-rename outcome through either exception representation. + if published: + raise ProjectForkError('Committed publication was renamed but acknowledgement is uncertain; reconcile this exact recovery.', + 'unknown',503) from error + if isinstance(error,OSError): + code='conflict' if error.errno==errno.EEXIST else 'unavailable' + raise ProjectForkError('Committed publication recovery is unavailable; no stage was rebuilt.', + code,503 if code!='conflict' else 409) from error + raise + + def _publish_project(source_store, source_artifacts, trusted_root_fd, relative, *, root_path, name, expected_checkpoint, request_key, quiescent, build_candidate, validate_candidate, retained_sources=()): diff --git a/project_projection.py b/project_projection.py index fca264be..1181c457 100644 --- a/project_projection.py +++ b/project_projection.py @@ -58,6 +58,59 @@ def validate_artifact_resolution(operation, result, records): return True +def validate_project_records(records, *, project_id, artifacts): + """Validate one inert candidate through every owning domain, without SQL reconstruction.""" + claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( + MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) + identities = set() + declarations = {} + for item in records: + identity = (item['family'], item['id']) + if item['family'] not in claimed or identity in identities: + raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) + identities.add(identity) + if item['family'] == 'project_artifact': + declaration = validate_declaration(item['document']) + if declaration['sha256'] != item['id']: + raise WorkbenchError('Artifact inventory identity differs.') + declarations[item['id']] = declaration + decode_data_records(records) + from project_fork import decode_fork + decode_fork(records, project_id=project_id) + from project_composition import decode_records as decode_compositions + decode_compositions(records, project_id=project_id) + from project_transfer import decode_records as decode_extraction_records + decode_extraction_records(records) + operation_records(records, project_id=project_id) + decode_file_records(records) + decode_native_records(records,artifacts=artifacts) + materials = MaterialSnapshot(records) + for blob in materials.artifact_blobs(): + if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): + raise WorkbenchError('Material artifact declaration is missing or inconsistent.') + # Resolve every cross-owner reference against THIS candidate, rather + # than a resolver that still sees the preceding committed checkpoint. + decode_producer_records(records, artifacts=artifacts) + decode_history_records(records, materials=materials, artifacts=artifacts) + decode_measurement_records(records, artifacts=artifacts) + for receipt in operation_documents(records): + operation, result = receipt['operation'], receipt['result'] + if (validate_data_result(operation, result, records) + or validate_material_result(operation, result, records) + or validate_history_result(operation, result, records) + or validate_source_result(operation, result, records) + or validate_measurement_result(operation, result, records) + or validate_execution_result(operation, result, records) + or validate_producer_result(operation, receipt['target'], result, records) + or validate_file_result(operation, result, records) + or validate_study_execution_result(operation, result, records) + or validate_runtime_result(operation, result, records) + or validate_preparation_result(operation, result, records) + or validate_artifact_resolution(operation, result, records)): + continue + raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + + class ProjectProjection: def __init__(self, store, db, access, roots, *, custody=None, restore=True, runtime_sources=None): self.store, self.db, self.access = store, db, access @@ -127,57 +180,9 @@ def execute(self, *args, **kwargs): return self.db.execute(*args, **kwargs) def validate_records(self, records, *, project_id=None, artifacts=None): - artifacts = self.artifacts if artifacts is None else artifacts - project_id = self.project_id if project_id is None else project_id - claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_composition', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( - MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) - identities = set() - declarations = {} - for item in records: - identity = (item['family'], item['id']) - if item['family'] not in claimed or identity in identities: - raise WorkbenchError('Unknown or duplicate owned project family.', 'unsupported', 409) - identities.add(identity) - if item['family'] == 'project_artifact': - declaration = validate_declaration(item['document']) - if declaration['sha256'] != item['id']: - raise WorkbenchError('Artifact inventory identity differs.') - declarations[item['id']] = declaration - decode_data_records(records) - from project_fork import decode_fork - decode_fork(records, project_id=project_id) - from project_composition import decode_records as decode_compositions - decode_compositions(records, project_id=project_id) - from project_transfer import decode_records as decode_extraction_records - decode_extraction_records(records) - operation_records(records, project_id=project_id) - decode_file_records(records) - decode_native_records(records,artifacts=artifacts) - materials = MaterialSnapshot(records) - for blob in materials.artifact_blobs(): - if declarations.get(blob['sha256']) != dict(format='project_artifact_v1', **blob): - raise WorkbenchError('Material artifact declaration is missing or inconsistent.') - # Resolve every cross-owner reference against THIS candidate, rather - # than a resolver that still sees the preceding committed checkpoint. - decode_producer_records(records, artifacts=artifacts) - decode_history_records(records, materials=materials, artifacts=artifacts) - decode_measurement_records(records, artifacts=artifacts) - for receipt in operation_documents(records): - operation, result = receipt['operation'], receipt['result'] - if (validate_data_result(operation, result, records) - or validate_material_result(operation, result, records) - or validate_history_result(operation, result, records) - or validate_source_result(operation, result, records) - or validate_measurement_result(operation, result, records) - or validate_execution_result(operation, result, records) - or validate_producer_result(operation, receipt['target'], result, records) - or validate_file_result(operation, result, records) - or validate_study_execution_result(operation, result, records) - or validate_runtime_result(operation, result, records) - or validate_preparation_result(operation, result, records) - or validate_artifact_resolution(operation, result, records)): - continue - raise WorkbenchError('No project owner accepts this operation receipt.', 'unsupported', 409) + return validate_project_records(records, + project_id=self.project_id if project_id is None else project_id, + artifacts=self.artifacts if artifacts is None else artifacts) def resolve_artifact(self, body, expected_checkpoint, key): """Explicitly supply exact bytes from a configured current local root.""" diff --git a/tests/test_project_publication_recovery.py b/tests/test_project_publication_recovery.py new file mode 100644 index 00000000..36067a6b --- /dev/null +++ b/tests/test_project_publication_recovery.py @@ -0,0 +1,116 @@ +"""Real committed-stage recovery preserves bytes and never reconstructs sources.""" +import errno +import os +import shutil +import unittest +from unittest.mock import patch + +from project_composition import composition_input +from project_fork import recover_committed_publication +from project_store import ProjectStore, ProjectStoreError +from project_files import file_id, materialize_captured +from test_project_composition_publication import CompositionPublicationTests + + +class PublicationRecoveryTests(unittest.TestCase): + setUp=CompositionPublicationTests.setUp + publish=CompositionPublicationTests.publish + assert_code=CompositionPublicationTests.assert_code + + def stage(self): + with patch('project_fork._rename_noreplace',side_effect=OSError(errno.EIO,'interrupted before rename')): + self.assert_code('unavailable',self.publish) + self.staging=next(self.root.glob('.tuldok-fork-*')) + with ProjectStore.open(self.staging) as store: + self.committed=store.snapshot() + self.request=composition_input(self.snapshots,'combined','Combined','compose-exact') + + def recover(self,**changes): + args=dict(root_path=self.root,expected_project_id=self.committed['project_id'], + expected_checkpoint=self.committed['checkpoint'],expected_request=self.request,quiescent=True) + args.update(changes) + return recover_committed_publication(self.fd,self.staging.name,'combined',**args) + + def test_committed_stage_recovers_without_sources_payload_copies_or_new_identity(self): + self.stage() + before={p.name:(p.read_bytes(),p.stat().st_ino) for p in (self.staging/'artifacts/sha256').iterdir()} + for store,artifacts in zip(self.stores,self.artifacts): + artifacts.close();store.close();shutil.rmtree(store.path) + with patch('project_artifacts.ProjectArtifacts.copy_many_to',side_effect=AssertionError('No recopy')), \ + patch('project_composition.build_candidate',side_effect=AssertionError('No rebuild')): + result=self.recover() + self.assertFalse(self.staging.exists()) + self.assertEqual(result['payloads_copied'],0) + self.assertFalse(result['selected']) + with ProjectStore.open(self.root/'combined') as store: + self.assertEqual(store.snapshot(),self.committed) + after={p.name:(p.read_bytes(),p.stat().st_ino) for p in (self.root/'combined/artifacts/sha256').iterdir()} + self.assertEqual(before,after) + self.assertTrue(self.recover()['reconciled']) + + def test_wrong_request_checkpoint_or_identity_never_publishes(self): + self.stage() + self.assert_code('stale',lambda:self.recover(expected_checkpoint='0'*64)) + self.assert_code('stale',lambda:self.recover(expected_project_id='0'*32)) + self.assert_code('conflict',lambda:self.recover(expected_request={**self.request,'name':'Changed'})) + self.assert_code('busy',lambda:self.recover(quiescent=False)) + self.assertTrue(self.staging.exists());self.assertFalse((self.root/'combined').exists()) + + def test_missing_copied_payload_preserves_stage_and_refuses_publication(self): + self.stage();next((self.staging/'artifacts/sha256').iterdir()).unlink() + with self.assertRaises(ProjectStoreError):self.recover() + self.assertTrue(self.staging.exists());self.assertFalse((self.root/'combined').exists()) + + def test_destination_appearance_never_overwrites_either_tree(self): + self.stage();(self.root/'combined').mkdir();(self.root/'combined/unique').write_bytes(b'keep') + self.assert_code('conflict',self.recover) + self.assertTrue(self.staging.exists());self.assertEqual((self.root/'combined/unique').read_bytes(),b'keep') + + def test_lost_rename_ack_reconciles_exact_published_checkpoint(self): + self.stage();fsync=os.fsync + def fail_after_publish(fd): + if (self.root/'combined').exists():raise OSError(errno.EIO,'lost publication acknowledgement') + fsync(fd) + with patch('project_fork.os.fsync',side_effect=fail_after_publish): + self.assert_code('unknown',self.recover) + self.assertFalse(self.staging.exists());self.assertTrue(self.recover()['reconciled']) + + def test_live_writer_and_non_stage_path_are_refused(self): + self.stage() + with ProjectStore.open(self.staging): + self.assert_code('busy',self.recover) + self.assert_code('invalid',lambda:recover_committed_publication(self.fd,'source0','combined', + root_path=self.root,expected_project_id=self.committed['project_id'],expected_checkpoint=self.committed['checkpoint'], + expected_request=self.request,quiescent=True)) + + def test_initial_empty_stage_cannot_be_mistaken_for_committed_publication(self): + self.stage();empty_name='.tuldok-fork-'+'9'*32 + with ProjectStore.create(self.root/empty_name,'Uncommitted','initial','a'*64) as store: + initial=store.snapshot() + self.assert_code('invalid',lambda:recover_committed_publication(self.fd,empty_name,'combined', + root_path=self.root,expected_project_id=initial['project_id'],expected_checkpoint=initial['checkpoint'], + expected_request=self.request,quiescent=True)) + self.assertTrue((self.root/empty_name).exists());self.assertFalse((self.root/'combined').exists()) + + def test_missing_captured_working_file_refuses_without_changing_committed_history(self): + # Seed a real current note before composing; its cloned editable record + # and materialized bytes then pass through the normal publication path. + from datetime import datetime, timezone + import hashlib + store=self.stores[0];snap=store.snapshot();raw=b'original note\n' + version=dict(revision=1,text=raw.decode(),sha256=hashlib.sha256(raw).hexdigest(),bytes=len(raw), + actor_id='1'*32,adopted_at=datetime.now(timezone.utc).isoformat()) + doc=dict(format='project_file_v1',project_id=snap['project_id'],kind='note',path='notes/test.md', + current=version,history=[version],origin=None) + record=dict(family='project_file',id=file_id(snap['project_id'],'note','notes/test.md'),document=doc) + self.snapshots[0]=store.commit([*snap['records'],record],snap['checkpoint'],'note','c'*64) + with store.root_directory() as root: + materialize_captured(self.snapshots[0]['records'],snap['project_id'],root) + self.stage();(self.staging/'notes/test.md').unlink() + self.assert_code('working_files_changed',self.recover) + self.assertFalse((self.root/'combined').exists()) + with ProjectStore.open(self.staging) as retained: + self.assertEqual(retained.snapshot(),self.committed) + + +if __name__=='__main__':unittest.main() From 65f6caf022c5b450cd787c3de17f24c7b860844c Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 16:31:42 -0700 Subject: [PATCH 22/33] feat(measurements): admit larger complete comparisons through API v2 Separate combined comparison capacity from the per-packet budget so full-state populations remain complete. Preserve existing public contract hashes and reject oversized views under the selected legacy version. --- agent_api.py | 13 +++++++++++++ project_workspace.py | 4 ++++ retained_measurements.py | 5 +++-- tests/test_measurement_comparison.py | 11 ++++++++++- tests/test_project_measurement_comparison_http.py | 15 +++++++++++++++ tests/test_project_measurements_clients.py | 10 ++++++++++ 6 files changed, 55 insertions(+), 3 deletions(-) diff --git a/agent_api.py b/agent_api.py index 67b99453..ed25af09 100644 --- a/agent_api.py +++ b/agent_api.py @@ -1471,6 +1471,13 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non _LOOKUP = {key: operation for operation in _OPERATIONS for key in (operation['operation_id'], operation['mcp_name'])} # Explicit locally implemented versions. No remotely supplied code or routes. _CONTRACTS = {op['operation_id']: {1: copy.deepcopy(op)} for op in _OPERATIONS} +# Keep the installed v1 bounds/hash exact. Version 2 admits complete larger +# comparisons; each individual packet and every response page retain their caps. +_comparison_capacity=copy.deepcopy(_CONTRACTS['project_files.measurement_compare'][1]) +_comparison_value=_comparison_capacity['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties'] +_comparison_value['observation_count']=integer(1,250000) +_comparison_value['query']['oneOf'][0]['properties']['total']=integer(0,250000) +_CONTRACTS['project_files.measurement_compare'][2]=_comparison_capacity # v1 remains the closed Run-only current custody view. v2 additionally exposes # bounded original local native phases; imported history never populates it. _native_health=copy.deepcopy(_CONTRACTS['project_files.execution_health'][1]) @@ -1974,6 +1981,12 @@ def operation(name, version=None): return copy.deepcopy(versions[max(versions) if version is None else version]) +def measurement_comparison_row_limit(version=None): + """Return the selected immutable public contract's comparison budget.""" + selected=operation('project_files.measurement_compare',version) + return selected['outputSchema']['oneOf'][0]['properties']['result']['properties']['value']['properties']['value']['properties']['observation_count']['maximum'] + + def _native_validation_result_ok(value, revision): """Check the finite public truth fields without interpreting native runtime data. diff --git a/project_workspace.py b/project_workspace.py index efe93f9a..072d6c24 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -509,6 +509,10 @@ def dispatch(self, name, arguments, *, selected_version=None): from project_dispatch import dispatch try: value = dispatch(self._projection, name, arguments) + if (name=='project_files.measurement_compare' + and value['value']['observation_count']>agent_api.measurement_comparison_row_limit(selected_version)): + raise WorkbenchError('Comparison exceeds the selected API version observation budget. Select comparison contract version 2 for larger complete populations.', + 'too_large',413) if name == 'project_files.execution_health' and selected_version == 1: value = {key: item for key, item in value.items() if key != 'native_phases'} except BaseException as error: diff --git a/retained_measurements.py b/retained_measurements.py index 796aaa19..4044b017 100644 --- a/retained_measurements.py +++ b/retained_measurements.py @@ -26,6 +26,7 @@ MAX_COMPARISON_SOURCES = 16 MAX_COMPARISON_SUBJECTS = 256 MAX_COMPARISON_BYTES = 128 * 1024 * 1024 +MAX_COMPARISON_ROWS = 250000 STATES = ('measured', 'failed', 'unsupported', 'unavailable') ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') SHA = re.compile(r'^[a-f0-9]{64}$') @@ -411,8 +412,8 @@ def __init__(self, tables): fail('Comparison sources overlap a subject identity.', 'conflict', 409) self._subjects[subject['id']] = dict(copy.deepcopy(subject), source=copy.deepcopy(locator)) observations += metadata['observation_count'] - if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_ROWS: - fail('Comparison exceeds 256 subjects or 100,000 observations.', 'too_large', 413) + if len(self._subjects) > MAX_COMPARISON_SUBJECTS or observations > MAX_COMPARISON_ROWS: + fail('Comparison exceeds 256 subjects or 250,000 observations.', 'too_large', 413) self._tables.append((copy.deepcopy(locator), table)) self._header.update(subjects=list(self._subjects.values()), observation_count=observations) diff --git a/tests/test_measurement_comparison.py b/tests/test_measurement_comparison.py index b289d751..16418875 100644 --- a/tests/test_measurement_comparison.py +++ b/tests/test_measurement_comparison.py @@ -91,7 +91,7 @@ def test_overlapping_subjects_and_repeated_sources_are_explicit_errors(self): self.assertEqual(error.exception.code, 'conflict') def test_bounds_and_invalid_selections_never_return_partial_results(self): - for constant, maximum in [('MAX_COMPARISON_BYTES', 1), ('MAX_COMPARISON_SUBJECTS', 1), ('MAX_ROWS', 7)]: + for constant, maximum in [('MAX_COMPARISON_BYTES', 1), ('MAX_COMPARISON_SUBJECTS', 1), ('MAX_COMPARISON_ROWS', 7)]: with patch.object(owner, constant, maximum), self.assertRaises(WorkbenchError): MeasurementComparison(self.inputs) with self.assertRaises(WorkbenchError): MeasurementComparison(self.inputs * 9) @@ -101,6 +101,15 @@ def test_bounds_and_invalid_selections_never_return_partial_results(self): self.comparison.read('error', **arguments) with self.assertRaises(WorkbenchError): self.comparison.read(filters={}) + def test_packet_and_combined_observation_budgets_are_separate(self): + # Both real source tables have four observations. A packet limit of + # four still permits their complete eight-observation comparison. + with patch.object(owner,'MAX_ROWS',4),patch.object(owner,'MAX_COMPARISON_ROWS',8,create=True): + self.assertEqual(MeasurementComparison(self.inputs).read()['observation_count'],8) + with self.assertRaises(WorkbenchError):MeasurementTable(encode(fixture())) + with patch.object(owner,'MAX_COMPARISON_ROWS',7,create=True),self.assertRaises(WorkbenchError): + MeasurementComparison(self.inputs) + def test_returned_metadata_and_query_values_have_no_mutable_owner_alias(self): expected = self.comparison.read('error') result = self.comparison.read('error') diff --git a/tests/test_project_measurement_comparison_http.py b/tests/test_project_measurement_comparison_http.py index 09063754..df5f9667 100644 --- a/tests/test_project_measurement_comparison_http.py +++ b/tests/test_project_measurement_comparison_http.py @@ -3,6 +3,10 @@ import json import os import unittest +from unittest.mock import patch + +import agent_api +from workbench import WorkbenchError import test_project_files_http as m1 import test_project_files_m2_http as m2 @@ -56,3 +60,14 @@ def test_two_original_sources_compare_and_inspect_through_real_HTTP(self): missing=self.call('measurement_compare',sources=sources,metric_id='latency')['value']['query'] self.assertTrue(all(s['maximum'] is None for s in missing['summary'])) self.assertEqual(self.workspace._projection.session.snapshot,before) + # Exercise version-specific admission at the real workspace/SQL owner + # with small declared fixture budgets; no partial comparison escapes. + arguments=self.arguments('measurement_compare',sources=sources) + with self.owner.bound(self.workspace),patch.object(agent_api,'measurement_comparison_row_limit', + side_effect=lambda version:7 if version==1 else 8): + with self.assertRaises(WorkbenchError) as error: + self.workspace.dispatch('project_files.measurement_compare',arguments,selected_version=1) + self.assertEqual(error.exception.code,'too_large') + value=self.workspace.dispatch('project_files.measurement_compare',arguments,selected_version=2) + self.assertEqual(value['value']['value']['observation_count'],8) + self.assertEqual(self.workspace._projection.session.snapshot,before) diff --git a/tests/test_project_measurements_clients.py b/tests/test_project_measurements_clients.py index e05b6b28..079f6ab2 100644 --- a/tests/test_project_measurements_clients.py +++ b/tests/test_project_measurements_clients.py @@ -24,6 +24,16 @@ def sdk(self,value): return ProjectFilesClient(client({'status':200,'result':dict(binding=BINDING,project_id=PROJECT, checkpoint=CHECKPOINT,value=value)})) + def test_comparison_capacity_is_versioned_without_changing_packet_headers(self): + from retained_measurements import MAX_COMPARISON_ROWS,MAX_ROWS + self.assertEqual(api.measurement_comparison_row_limit(1),100000) + self.assertEqual(api.measurement_comparison_row_limit(2),250000) + self.assertEqual(api.measurement_comparison_row_limit(2),MAX_COMPARISON_ROWS) + self.assertEqual(MAX_ROWS,100000) + self.assertEqual(api._MEASUREMENT_HEADER['properties']['observation_count']['maximum'],100000) + self.assertNotEqual(api.contract('project_files.measurement_compare',1)['sha256'], + api.contract('project_files.measurement_compare',2)['sha256']) + def test_group_comparison_header_and_query_correlate_every_original_source(self): locator=dict(origin_catalog_id=PROJECT,packet_sha256=self.table.sha256) comparison=MeasurementComparison([(locator,self.table)]) From dd1feca9b524fd94e291daee6d6c95a518365390 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 16:45:51 -0700 Subject: [PATCH 23/33] fix(projects): support clients for large retained projects Allow explicit bounded owner waits and regenerate installed browser contracts from the native registry. Complete composition consumer coverage so SDK and browser clients can use the current retained-project capabilities. --- agent_api.py | 8 ++++- docs/contracts/project-files.md | 10 ++++++ scripts/generate_project_files_contracts.py | 35 +++++++++++++++++++++ static/project-files.js | 10 ++++++ tests/test_agent_api.py | 8 +++++ tests/test_project_files_clients.py | 19 +++++++++++ tests/test_project_files_m4_clients.py | 8 +++-- tuldok_client.py | 5 +++ 8 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 scripts/generate_project_files_contracts.py diff --git a/agent_api.py b/agent_api.py index ed25af09..90a534c9 100644 --- a/agent_api.py +++ b/agent_api.py @@ -2395,6 +2395,8 @@ def _private_file(path, maximum): class AgentClient: """No catalog access, automatic retries, redirects, proxies or owner identity.""" + MAX_TIMEOUT_SECONDS = 300 + def __init__(self, endpoint, token, expected_catalog_id, timeout=120): try: parsed = urlsplit(endpoint) @@ -2404,7 +2406,7 @@ def __init__(self, endpoint, token, expected_catalog_id, timeout=120): raise ValueError() if not re.fullmatch(r'[A-Za-z0-9_-]{20,4096}', token) or not re.fullmatch('[a-f0-9]{32}', expected_catalog_id): raise ValueError() - if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= 300: + if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= self.MAX_TIMEOUT_SECONDS: raise ValueError() except (ValueError, TypeError): raise AgentAPIError('Invalid fixed loopback client configuration.', 'configuration') from None @@ -3029,6 +3031,10 @@ def call(self, name, arguments, *, observer=None, observation=None): class ProjectFilesOwnerClient(AgentClient): """Explicit owner file-project access; no unrelated owner operations.""" + # Large retained projects may outlast the generic transport's five minutes. + # Only an explicit owner configuration opts into a longer absolute wait. + MAX_TIMEOUT_SECONDS = 7200 + OPERATIONS = frozenset('project_files.' + action for action in ('status', 'new', 'open', 'save', 'close', 'query', 'get', 'import_text', 'annotate')) | PROJECT_FILES_M2 | PROJECT_FILES_M3 | PROJECT_FILES_M4 diff --git a/docs/contracts/project-files.md b/docs/contracts/project-files.md index e04aa7ac..387c9bc6 100644 --- a/docs/contracts/project-files.md +++ b/docs/contracts/project-files.md @@ -105,6 +105,16 @@ annotations; human review still requires the owner. Every project action is boun to an opaque workspace generation and its expected canonical checkpoint. Use `ProjectFilesOwnerClient` only for the narrow owner project-file operations. +Its explicit `timeout` configuration may be up to 7,200 seconds for large retained +projects; the default remains 120 seconds. The absolute request watchdog still +applies and a transport timeout never authorizes an automatic mutation retry. +Generic clients retain their 300-second maximum. This wait budget is not an +execution deadline or a cancellation request. + +Browser contract tables derive from the local `agent_api` registry. Regenerate +them with `python scripts/generate_project_files_contracts.py`; add `--check` +to verify that the installed v1 pins, all supported version pins and fixed +routes match that registry. Remote schemas remain opaque discovery data. `ProjectFilesClient` wraps a negotiated client. CLI domain `file-*` commands take `--arguments` JSON files; lifecycle commands use explicit flags and select `--owner-project-files` with a diff --git a/scripts/generate_project_files_contracts.py b/scripts/generate_project_files_contracts.py new file mode 100644 index 00000000..ac373e53 --- /dev/null +++ b/scripts/generate_project_files_contracts.py @@ -0,0 +1,35 @@ +"""Derive installed browser pins/routes from the local canonical API registry.""" +import argparse +import json +from pathlib import Path +import re +import sys + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT)) +import agent_api + + +def render(original): + names=sorted(name for name in agent_api._CONTRACTS if name.startswith('project_files.')) + tables={ + 'CONTRACTS':{name:agent_api.contract(name,1)['sha256'] for name in names}, + 'VERSIONS':{name:{str(v):agent_api.contract(name,v)['sha256'] for v in sorted(agent_api._CONTRACTS[name])} + for name in names if len(agent_api._CONTRACTS[name])>1}, + 'ROUTES':{name:{'method':agent_api.operation(name)['method'],'path':agent_api.operation(name)['path'], + 'mutation':agent_api.operation(name)['side_effects']} + for name in sorted(agent_api.PROJECT_FILES_M2|agent_api.PROJECT_FILES_M3|agent_api.PROJECT_FILES_M4)}} + for label,value in tables.items(): + pattern=r'(/\* PROJECT_FILES_'+label+r'_START \*/\s*)(.*?)(\s*/\* PROJECT_FILES_'+label+r'_END \*/)' + generated=json.dumps(value,sort_keys=True,indent=2) + original,count=re.subn(pattern,lambda m:m[1]+generated+m[3],original,flags=re.S) + if count!=1:raise ValueError('Exactly one installed '+label+' table required') + return original + + +if __name__=='__main__': + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--check',action='store_true') + args=parser.parse_args();path=ROOT/'static/project-files.js';original=path.read_text();expected=render(original) + if args.check: + if original!=expected:raise SystemExit('Installed project-file browser contracts are stale') + elif original!=expected:path.write_text(expected) diff --git a/static/project-files.js b/static/project-files.js index 65cfa499..8e57112d 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -14,6 +14,7 @@ "project_files.candidate": "c519ff04bc06d5e538d36b7f278862d8f282d0e7b97dd7f262444c384a50f854", "project_files.candidate_select": "f043be6923760a208ca50d02a1ff4b60eadf09d48a034acdb18749806a537080", "project_files.close": "95d0099ec2a378a1bbb65676799dd3baae2f36b838301f7aa165b116fba5edf6", + "project_files.compose": "b1dcbbd4ad7ebe17b17e11238f99a93408da24d0d411527e187900cc0f089105", "project_files.data_annotate": "289270f5cb946765c5ced22fbbd18765bb55743145014c81cf54d7a6ba3025ea", "project_files.data_availability": "92332c04973a4fa45d47e384717152ebc4a9ca08e7eb487669688ccd2ed42cbb", "project_files.data_get": "0d19505995cbd8c12e3938cd89cf05955a2a63488a220711d29dd24b27c0c8c8", @@ -167,6 +168,10 @@ "project_files.get": { "1": "6b0f4e16a00aa119b86cbb5493be0bac945ff08a5e63773a8f2c2f9296658d5b", "2": "fdc0858fc03a0866bb743b04251ea5f9a3c1e6be1b211366db7e0652166736ea" + }, + "project_files.measurement_compare": { + "1": "abbb5e406a3c95155e90f08e8a071f330425190b25275f0f6512de08b8afb929", + "2": "d2de9763a9109d0e4599f50e038a3fda0e6bf7c9c60684434825d0108019bc6f" } } /* PROJECT_FILES_VERSIONS_END */; @@ -199,6 +204,11 @@ "mutation": true, "path": "/api/workspace/project-files/studies/{study_id}/candidates" }, + "project_files.compose": { + "method": "POST", + "mutation": true, + "path": "/api/workspace/project-files/compose" + }, "project_files.data_annotate": { "method": "POST", "mutation": true, diff --git a/tests/test_agent_api.py b/tests/test_agent_api.py index 2c9c1f70..004121b4 100644 --- a/tests/test_agent_api.py +++ b/tests/test_agent_api.py @@ -284,6 +284,14 @@ def test_absolute_header_deadline_includes_trickle(self): with self.assertRaises(api.AgentAPIError): client.call('data.query',{}) self.assertLess(time.monotonic()-start,1) + def test_owner_transport_keeps_the_selected_absolute_wait_and_never_retries(self): + with server('trickle') as (endpoint,calls): + client=api.ProjectFilesOwnerClient(endpoint,TOKEN,CATALOG,timeout=.15) + start=time.monotonic() + with self.assertRaises(api.AgentAPIError):client._request('GET','/api/workbench/records') + self.assertLess(time.monotonic()-start,1) + self.assertEqual(len(calls),1) + class OutputTests(unittest.TestCase): @unittest.skipUnless(HAS_JSONSCHEMA, 'Optional agent jsonschema dependency is not installed.') diff --git a/tests/test_project_files_clients.py b/tests/test_project_files_clients.py index e692dc45..2233b259 100644 --- a/tests/test_project_files_clients.py +++ b/tests/test_project_files_clients.py @@ -272,6 +272,9 @@ def test_browser_contract_pins_are_current_registry_output(self): self.assertEqual(pins, {name: api.contract(name)['sha256'] for name in api._CONTRACTS if name.startswith('project_files.')}) for name, digest in pins.items(): self.assertEqual(api.selected_header(name), name + '/1/' + digest) + versions=json.loads(re.search(r'/\* PROJECT_FILES_VERSIONS_START \*/\s*(.*?)\s*/\* PROJECT_FILES_VERSIONS_END \*/',source,re.S)[1]) + self.assertEqual(versions,{name:{str(v):api.contract(name,v)['sha256'] for v in sorted(values)} + for name,values in api._CONTRACTS.items() if name.startswith('project_files.') and len(values)>1}) def test_reads_pin_binding_checkpoint_and_path_record_without_body(self): arguments = {'record_id': RECORD, 'binding': BINDING, 'checkpoint': CHECKPOINT} @@ -352,6 +355,22 @@ def test_closed_record_output_rejects_unknown_and_inconsistent_history(self): class Consumers(unittest.TestCase): + def test_explicit_owner_wait_budget_preserves_generic_bounds_and_private_configuration(self): + endpoint,token='http://127.0.0.1:12345','fixture-agent-token-for-client-tests' + self.assertEqual(api.ProjectFilesOwnerClient(endpoint,token,CATALOG).timeout,120) + self.assertEqual(api.ProjectFilesOwnerClient(endpoint,token,CATALOG,timeout=7200).timeout,7200) + for client_type,invalid in ((api.AgentClient,301),(api.ProjectFilesOwnerClient,7201), + (api.ProjectFilesOwnerClient,float('inf')),(api.ProjectFilesOwnerClient,True)): + with self.subTest(client=client_type.__name__,timeout=invalid),self.assertRaises(api.AgentAPIError): + client_type(endpoint,token,CATALOG,timeout=invalid) + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);credential=root/'token';configuration=root/'config.json' + credential.write_text(token);credential.chmod(0o600) + configuration.write_text(json.dumps(dict(endpoint=endpoint,catalog_id=CATALOG, + token_file=str(credential),timeout=7200)));configuration.chmod(0o600) + self.assertEqual(api.ProjectFilesOwnerClient.from_config(configuration).timeout,7200) + with self.assertRaises(api.AgentAPIError):api.AgentClient.from_config(configuration) + def test_owner_file_client_keeps_unrelated_operations_outside_authority(self): raw = client({'status': 200, 'result': state()}, owner=True) self.assertEqual(ProjectFilesClient(raw).save(**bound()), state()) diff --git a/tests/test_project_files_m4_clients.py b/tests/test_project_files_m4_clients.py index 5136e6af..d24d42a2 100644 --- a/tests/test_project_files_m4_clients.py +++ b/tests/test_project_files_m4_clients.py @@ -84,10 +84,12 @@ def test_capability_profiles_fit_unchanged_budget_and_negotiate_only_intersectio with self.assertRaises(api.AgentAPIError): api.capabilities(CATALOG, profile='arbitrary') self.assertEqual(api.REGISTRY_DIGEST, api._LEGACY_DIGEST) - def test_eleven_operations_have_fixed_consumers_and_closed_inputs(self): - self.assertEqual(len(api.PROJECT_FILES_M4), 11) + def test_complete_m4_operations_have_fixed_consumers_and_closed_inputs(self): + self.assertEqual(api.PROJECT_FILES_M4,{'project_files.'+name for name in ( + 'compose','save_as','files_inspect','file_get','file_history','file_adopt','files_adopt', + 'file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')}) self.assertEqual(project_dispatch.SUPPORTED_OPERATIONS & api.PROJECT_FILES_M4, - api.PROJECT_FILES_M4 - {'project_files.save_as'}) + api.PROJECT_FILES_M4 - {'project_files.save_as','project_files.compose'}) client = ProjectFilesClient(Mock()) parser = project_cli.parser() for operation in api.PROJECT_FILES_M4: diff --git a/tuldok_client.py b/tuldok_client.py index be0daf7f..fdaef8d8 100644 --- a/tuldok_client.py +++ b/tuldok_client.py @@ -706,6 +706,11 @@ def save_as(self, *, root, directory, name, binding, expected_checkpoint, reques return self._call('save_as',dict(root=root,directory=directory,name=name,binding=binding, expected_checkpoint=expected_checkpoint,request_key=request_key)) + def compose(self, *, root, directory, name, sources, binding, expected_checkpoint, request_key): + """Compose explicit retained checkpoints through the negotiated owner.""" + return self._call('compose',dict(root=root,directory=directory,name=name,sources=sources, + binding=binding,expected_checkpoint=expected_checkpoint,request_key=request_key)) + def files_inspect(self, **arguments): return self._call('files_inspect',arguments) From 1f1e3bc5cf6210ed9658256b869389d60d8d3898 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 18:07:00 -0700 Subject: [PATCH 24/33] perf(projects): reuse decoding after verifying current canonical bytes Keep exact manifest-bound decoding inside one Store while freshly checking every canonical record on each read. Borrow rooted handles without cloning unused public snapshots; preserve independent public values and before/after closure checks. --- project_store.py | 47 ++++++++++++++++++++++++--------- tests/test_project_artifacts.py | 6 ++--- tests/test_project_store.py | 31 ++++++++++++++++++++++ 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/project_store.py b/project_store.py index bc4b4071..93ba5ad3 100644 --- a/project_store.py +++ b/project_store.py @@ -468,6 +468,7 @@ def __init__(self, path, root_fd, origin_fd, relative_parts, origin_path): self._origin_path = Path(origin_path) self._meta_fd = None self._manifest = None + self._read_cache = None self._mutex = threading.RLock() self._unknown = False try: @@ -629,12 +630,12 @@ def root_directory(self, *, verify_after=False): if type(verify_after) is not bool: _fail('Root read verification must be an explicit boolean.') with self._mutex: - before = self.snapshot() + before = self._current()[0]['checkpoint'] descriptor = os.dup(self._root_fd) try: yield descriptor self._binding() - if verify_after and self.snapshot()['checkpoint'] != before['checkpoint']: + if verify_after and self._current()[0]['checkpoint'] != before: _fail('Project checkpoint changed during the bounded root read.', 'stale', 409) finally: os.close(descriptor) @@ -642,6 +643,17 @@ def root_directory(self, *, verify_after=False): def _load(self): self._binding() raw = _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) + cached = self._read_cache + if cached is not None and raw == cached[0]: + manifest, loaded = cached[1:] + # Reuse decoding only after checking every actual canonical file. + # Stat identity is not content identity; corruption still fails. + for entry in manifest['inventory']: + self._verified_record_bytes(entry) + if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: + _fail('Project checkpoint changed while reading.', 'stale', 409) + self._binding() + return manifest, loaded manifest = _decode(raw) if type(manifest) is not dict: _fail('Project checkpoint must be an object.') @@ -683,9 +695,7 @@ def _load(self): total += entry['bytes'] if total > MAX_TOTAL_BYTES: _fail('Project records exceed their total byte limit.') - record_raw = self._record_file(family, record_id, digest) - if len(record_raw) != entry['bytes'] or _sha(record_raw) != digest: - _fail('Canonical record bytes changed or are corrupt.', 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + record_raw = self._verified_record_bytes(entry) document = _decode(record_raw) if type(document) is not dict or document.get('format') != entry['format']: _fail('Domain document format disagrees with its inventory.') @@ -709,8 +719,16 @@ def _load(self): if _read(self._meta_fd, 'project.json', MAX_MANIFEST_BYTES) != raw: _fail('Project checkpoint changed while reading.', 'stale', 409) self._binding() + self._read_cache = (raw, manifest, loaded) return manifest, loaded + def _verified_record_bytes(self, entry): + raw = self._record_file(entry['family'], entry['id'], entry['sha256']) + if len(raw) != entry['bytes'] or _sha(raw) != entry['sha256']: + _fail('Canonical record bytes changed or are corrupt.', + 'stale' if self._manifest else 'invalid', 409 if self._manifest else 400) + return raw + def _record_file(self, family, record_id, digest, raw=None, sync=False): fds = [] try: @@ -777,15 +795,19 @@ def _snapshot(self, manifest, records): return dict(project_id=manifest['project_id'], name=manifest['name'], checkpoint=manifest['checkpoint'], records=copy.deepcopy(records)) + def _current(self): + try: + manifest, records = self._load() + if manifest != self._manifest: + _fail('Project checkpoint changed; close and reopen.', 'stale', 409) + return manifest, records + except OSError as exc: + _io_error(exc, 'read project snapshot') + def snapshot(self): with self._mutex: - try: - manifest, records = self._load() - if manifest != self._manifest: - _fail('Project checkpoint changed; close and reopen.', 'stale', 409) - return self._snapshot(manifest, records) - except OSError as exc: - _io_error(exc, 'read project snapshot') + manifest, records = self._current() + return self._snapshot(manifest, records) def reconcile(self, request_key, request_sha256): """Inspect retry identity before a domain mutation; never repeat effects.""" @@ -884,6 +906,7 @@ def recover_checkpoint(self): def close(self): with self._mutex: + self._read_cache = None if self._meta_fd is not None: os.close(self._meta_fd) self._meta_fd = None diff --git a/tests/test_project_artifacts.py b/tests/test_project_artifacts.py index 2a68497d..ad06e375 100644 --- a/tests/test_project_artifacts.py +++ b/tests/test_project_artifacts.py @@ -74,7 +74,7 @@ def mutate_source_after_snapshot(fd, value, **options): original.write_bytes(b'changed after verified capture') return publish(fd, value, **options) with patch.object(destination, 'publish_fd', side_effect=mutate_source_after_snapshot): - with patch.object(self.store, 'snapshot', wraps=self.store.snapshot) as checks: + with patch.object(self.store, '_current', wraps=self.store._current) as checks: self.artifacts.copy_many_to(destination, values) self.assertEqual(checks.call_count, 2) for raw, value in zip(raws, values): @@ -118,7 +118,7 @@ def test_status_group_retains_exact_hash_observations_with_two_closure_checks(se self.payload(values[2]).unlink() self.payload(values[2]).write_bytes(b'changed') expected = [self.artifacts.status(value) for value in values] - with patch.object(self.store, 'snapshot', wraps=self.store.snapshot) as checks: + with patch.object(self.store, '_current', wraps=self.store._current) as checks: actual = self.artifacts.status_many(values) self.assertEqual(actual, expected) self.assertEqual([item['state'] for item in actual], ['verified', 'missing', 'corrupt']) @@ -161,7 +161,7 @@ def test_presence_batch_matches_individual_observations_with_two_closure_checks( self.artifacts.publish_bytes(b'present',values[0]) self.commit(values) expected=[self.artifacts.inspect(value) for value in values] - with patch.object(self.store,'snapshot',wraps=self.store.snapshot) as checks: + with patch.object(self.store,'_current',wraps=self.store._current) as checks: actual=self.artifacts.inspect_many(values) self.assertEqual(actual,expected) self.assertEqual(checks.call_count,2) diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 9f321617..9dcf11af 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -97,6 +97,37 @@ def test_appending_one_record_does_not_rewrite_saved_immutable_revisions(self): 'Appending one record must not write disposable copies of every saved immutable revision') self.assertEqual(self.reopen(), committed) + def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_identical_records(self): + rows=[record('a'),record('b','Exact second note\n')] + self.commit(rows,key='repeat-read-fixture') + expected=self.store.snapshot() + canonical_documents={canonical(row['document']) for row in rows} + with patch.object(store_module,'_decode',wraps=store_module._decode) as decode, \ + patch.object(self.store,'_record_file',wraps=self.store._record_file) as read, \ + patch.object(self.store,'_snapshot',wraps=self.store._snapshot) as snapshot: + self.assertEqual(self.store.snapshot(),expected) + with self.store.root_directory(verify_after=True) as descriptor: + self.assertTrue(os.fstat(descriptor).st_ino) + self.assertEqual(self.store.snapshot(),expected) + self.assertEqual(read.call_count,8,'All two records must still be read in each of four checks') + self.assertEqual(snapshot.call_count,2,'Root borrowing must not clone an unused public snapshot') + decoded=[call for call in decode.call_args_list if call.args[0] in canonical_documents] + self.assertEqual(len(decoded),0,'Byte-identical canonical records need no repeat decoding within one Store') + + def test_verified_decode_cache_still_rejects_same_length_corruption_with_restored_mtime(self): + self.commit([record('note','Alpha\n')],key='cache-corruption-fixture') + self.store.snapshot() + entry=self.manifest()['inventory'][0] + path=self.root/'.tuldok'/entry['path'];info=path.stat();raw=path.read_bytes() + changed=raw.replace(b'Alpha',b'Omega') + self.assertEqual(len(changed),len(raw));self.assertNotEqual(changed,raw) + path.chmod(0o600);path.write_bytes(changed) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + self.assert_code('stale',self.store.snapshot) + with self.assertRaises(ProjectStoreError) as caught: + with self.store.root_directory(verify_after=True):pass + self.assertEqual(caught.exception.code,'stale') + def test_reusing_an_unreferenced_revision_still_checks_bytes_and_syncs_it(self): first, second = record('first'), record('second') self.commit([first], key='retain-first') From bf2746e0e067d5125731ed6486d5dd68380a6a82 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 18:26:26 -0700 Subject: [PATCH 25/33] perf(projects): select active receipt records before copying Keep complete fork and historical receipt validation, while copying only operation records for the current request-key lookup. Preserve the default independently owned active-record view for other callers. --- project_fork.py | 10 ++++++++-- project_session.py | 3 ++- tests/test_project_fork.py | 23 +++++++++++++++++++++++ tests/test_project_session.py | 19 +++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/project_fork.py b/project_fork.py index bd928ad9..f35b9812 100644 --- a/project_fork.py +++ b/project_fork.py @@ -174,10 +174,16 @@ def decode_fork(records, *, project_id=None): return copy.deepcopy(overlays[current]) -def active_records(records, *, project_id=None): +def active_records(records, *, project_id=None, families=None): + """Own selected active copies while validating the complete fork history.""" + if families is not None: + if type(families) is not frozenset: + _fail('Active record families must be an explicit frozenset.') + for family in families:_identifier(family,'Active record family') overlay = decode_fork(records, project_id=project_id) archived = set() if overlay is None else {(item['family'], item['id']) for item in overlay['archived']} - return [copy.deepcopy(item) for item in records if (item['family'], item['id']) not in archived] + return [copy.deepcopy(item) for item in records + if (families is None or item['family'] in families) and (item['family'], item['id']) not in archived] def _inside(candidate_fd, ancestor_fd): diff --git a/project_session.py b/project_session.py index 46c106cd..de2293d7 100644 --- a/project_session.py +++ b/project_session.py @@ -65,7 +65,8 @@ def operation_records(records, *, project_id=None): if project_id is None and overlay is not None: project_id = overlay['destination_project_id'] result = {} - for value in operation_documents(active_records(records, project_id=project_id)): + for value in operation_documents(active_records(records, project_id=project_id, + families=frozenset({OPERATION_FAMILY}))): if overlay is not None and value['format'] == 'project_operation_v1': raise ProjectSessionError('A fork must retain legacy operation receipts as archived historical facts.') if project_id is not None and value.get('project_id', project_id) != project_id: diff --git a/tests/test_project_fork.py b/tests/test_project_fork.py index 6241c890..1b4029b8 100644 --- a/tests/test_project_fork.py +++ b/tests/test_project_fork.py @@ -371,5 +371,28 @@ def test_current_overlay_filters_receipts_and_original_editables_without_rewriti self.assert_code('invalid', decode_fork, bad) +class ActiveRecordSelectionTests(unittest.TestCase): + def test_family_selection_keeps_independent_values_and_default_membership(self): + records=[dict(family=family,id='a'*32,document=dict(format='fixture_v1',values=[1])) + for family in ('project_operation','workbench_text')] + self.assertEqual(active_records(records),records) + selected=active_records(records,families=frozenset({'project_operation'})) + self.assertEqual(selected,[records[0]]) + selected[0]['document']['values'].append(2) + self.assertEqual(records[0]['document']['values'],[1]) + self.assertEqual(active_records(records,families=frozenset()),[]) + + def test_family_selection_still_validates_excluded_fork_structure(self): + record=dict(family='workbench_text',id='a'*32,document={'format':'workbench_text_v1'}) + with self.assertRaisesRegex(ProjectForkError,'Duplicate'): + active_records([record,copy.deepcopy(record)],families=frozenset({'project_operation'})) + + def test_family_selection_rejects_implicit_or_invalid_selectors(self): + for invalid in ('project_operation',[],frozenset({1}),frozenset({'bad/family'})): + with self.subTest(selector=invalid),self.assertRaises(ProjectStoreError) as caught: + active_records([],families=invalid) + self.assertEqual(caught.exception.code,'invalid') + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_project_session.py b/tests/test_project_session.py index f9356c44..309f2bc4 100644 --- a/tests/test_project_session.py +++ b/tests/test_project_session.py @@ -14,6 +14,7 @@ from unittest.mock import patch import uuid +import project_fork as fork_module import psycopg from psycopg import sql @@ -27,6 +28,24 @@ class OperationCodecTests(unittest.TestCase): + def test_operation_lookup_does_not_copy_unrelated_active_envelopes(self): + key='selected-receipt' + document=dict(format='project_operation_v1',operation='fixture.capture',target=None, + request_key=key,input_sha256='a'*64,result={}) + receipt=dict(family='project_operation',id=hashlib.sha256(key.encode()).hexdigest(),document=document) + unrelated=dict(family='project_artifact',id='b'*64, + document=dict(format='project_artifact_v1',sha256='b'*64,bytes=4)) + copied=[];deepcopy=copy.deepcopy + def observe(value): + if value is unrelated:copied.append((value['family'],value['id'])) + return deepcopy(value) + with patch.object(fork_module,'copy',SimpleNamespace(deepcopy=observe)): + actual=operation_records([unrelated,receipt]) + self.assertEqual(actual,{key:document}) + self.assertEqual(copied,[],'Receipt selection must not clone unrelated active records') + actual[key]['result']['changed']=True + self.assertEqual(document['result'],{}) + def test_namespaced_receipts_preserve_legacy_identity_and_independent_keys(self): key = 'independent-edit' document = dict(format='project_operation_v1', operation='fixture.capture', target=None, From 0adcc5bea0bcc2cbc5bcf41a0627677582a52335 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sat, 24 Oct 2026 22:06:07 -0700 Subject: [PATCH 26/33] perf(projects): avoid full snapshots for scalar metadata Read project identity and checkpoint through verified Store summaries and bound Session summaries. Preserve full canonical freshness checks and independent snapshots for record consumers. Update coordinated fixtures, including interpreter alias drift on hosts using the system Python. --- project_custody.py | 4 +-- project_data.py | 2 +- project_dispatch.py | 2 +- project_execution.py | 12 ++++---- project_files.py | 10 +++---- project_history.py | 12 ++++---- project_material.py | 6 ++-- project_measurements.py | 2 +- project_native_execution.py | 4 +-- project_native_preparation.py | 12 ++++---- project_producer_execution.py | 2 +- project_producers.py | 2 +- project_projection.py | 2 +- project_runtime.py | 14 +++++----- project_session.py | 8 +++++- project_sources.py | 2 +- project_store.py | 6 ++++ project_study_execution.py | 26 +++++++++--------- project_text.py | 2 +- project_transfer.py | 4 +-- project_workspace.py | 32 +++++++++++----------- tests/test_project_dispatch.py | 5 ++-- tests/test_project_files_fork_contracts.py | 3 +- tests/test_project_material.py | 3 +- tests/test_project_material_fork.py | 9 ++++-- tests/test_project_native_process.py | 8 ++++-- tests/test_project_session.py | 17 ++++++++++++ tests/test_project_store.py | 15 ++++++++++ tests/test_project_workspace.py | 11 ++++++++ 29 files changed, 151 insertions(+), 86 deletions(-) diff --git a/project_custody.py b/project_custody.py index 2e0a97aa..1a188f1d 100644 --- a/project_custody.py +++ b/project_custody.py @@ -197,7 +197,7 @@ def _directory(self, create=False): finally:os.close(parent) def _binding(self, store, projection_id): - snapshot=store.snapshot() + snapshot=store.summary() with store.root_directory() as descriptor: info=os.fstat(descriptor) return dict(host_catalog_id=self.host_catalog_id,project_id=snapshot['project_id'], @@ -247,7 +247,7 @@ def __init__(self, owner, store, binding): persisted=value is not None if value is None: value=dict(format='project_local_custody_v1',**binding, - project_name=store.snapshot()['name'],selected_checkpoint=store.snapshot()['checkpoint'], + project_name=store.summary()['name'],selected_checkpoint=store.summary()['checkpoint'], attempted=None,runs=[],producers=[],runtime_effects=[],package_effects=[]) if any(value[key]!=item for key,item in binding.items()):_fail('Original physical custody binding differs.') self.value=validate(value) diff --git a/project_data.py b/project_data.py index 4a48dd38..6c99e8b2 100644 --- a/project_data.py +++ b/project_data.py @@ -1001,7 +1001,7 @@ def _publish(self, raw, kind, role, media_type): declaration = dict(format='project_artifact_v1', sha256=hashlib.sha256(raw).hexdigest(), bytes=len(raw)) self.artifacts.publish_bytes(raw, declaration) self._pending_artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=self.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=self.session.summary['project_id'], sha256=declaration['sha256']), media_type=media_type, kind=kind, role=role, preservation='unique', required_for=['inspect', 'release']) def _require_record(self, record_id, *, extended=False): diff --git a/project_dispatch.py b/project_dispatch.py index 6c9420a2..bd7929d0 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -149,7 +149,7 @@ def _data(projection, action, a): def _definitions(projection, family, a): docs = projection.materials._documents() values = [doc['current'] for (kind, _), doc in docs.items() - if kind == family and doc['origin_catalog_id'] == projection.session.snapshot['project_id']] + if kind == family and doc['origin_catalog_id'] == projection.session.summary['project_id']] return _page(sorted(values, key=lambda item: item['id']), a) diff --git a/project_execution.py b/project_execution.py index ca5c36d0..fece88a1 100644 --- a/project_execution.py +++ b/project_execution.py @@ -296,7 +296,7 @@ def _restore_file_writer(self, kind, request_key, input_sha256, content_sha256, authority.require(scope,action,owner_only=True) if self.db.in_transaction: raise RuntimeError('File reconciliation requires no outer SQL transaction.') - if self.session.snapshot['checkpoint']!=expected_checkpoint: + if self.session.summary['checkpoint']!=expected_checkpoint: raise WorkbenchError('Expected current file checkpoint changed.','stale_project',409) if self.custody is None: raise WorkbenchError('No original physical file custody exists.','unavailable',409) @@ -464,7 +464,7 @@ def _authorize_observation(self): # The domain owner has already checked submit/cancel authority; # recheck live host credentials without inventing a worker owner. self.runtime_access.require('inspect', 'runs.observe', identity) - if proof['admission']['catalog_id'] != self.session.snapshot['project_id']: + if proof['admission']['catalog_id'] != self.session.summary['project_id']: raise AccessError('Local custody belongs to another project.') def _capture_run(self, identity): @@ -626,7 +626,7 @@ def invoke(): context.consume_admission() owner=self._local_owner();public=owner.submit(dict(body,request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.submit',body['graph_id'],effective,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.submit',body['graph_id'],effective,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_retry(self,run_id,expected_checkpoint,key): @@ -646,7 +646,7 @@ def invoke(): context.consume_admission() owner=self.runs;public=owner.retry(run_id,dict(request_key=key)) return _admission_result(owner._load(public['id'])) - return self.mutate('runs.retry',run_id,body,self.session.snapshot['checkpoint'],key, + return self.mutate('runs.retry',run_id,body,self.session.summary['checkpoint'],key, invoke,lambda value,before:dict(records=[],result=value),scope='execute') def native_reconcile(self,phase_id,expected_checkpoint): @@ -657,7 +657,7 @@ def submit_external(self, body, expected_checkpoint, key): with self.access.lock, self.session.lock, self.db.lock: self.session.check() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: # A retained request never recreates a local Run/admission or wakes @@ -680,7 +680,7 @@ def approve_external(self, body, expected_checkpoint, key): with self.access.lock, self.session.lock, self.db.lock: self.session.check() prior = operation_records(self.session.snapshot['records']).get(key) - if prior is None and expected_checkpoint != self.session.snapshot['checkpoint']: + if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) if prior is not None: value = prior['result'] diff --git a/project_files.py b/project_files.py index c312aa34..c6e9f6b6 100644 --- a/project_files.py +++ b/project_files.py @@ -300,7 +300,7 @@ def materialize_captured(records,project_id,root_fd): def _adoption_intent(self,body): closed(body,{'kind','path','expected_revision','expected_sha256'},'working-file adoption') - parts=_path(body['kind'],body['path']);project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) + parts=_path(body['kind'],body['path']);project_id=self.session.summary['project_id'];record_id=file_id(project_id,body['kind'],body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 0<=body['expected_revision']<=2**53-1:fail('Invalid expected adoption revision.') return dict(body),parts,record_id @@ -316,12 +316,12 @@ def _capture_adoption(self,intent,prior,root): value=dict(revision=revision+1,sha256=observed,bytes=size,text=text,adopted_at=timestamp(),actor_id=actor.actor_id) history=copy.deepcopy(prior['document']['history']) if prior else [];history.append(value) envelope=dict(family=FAMILY,id=record_id,document=dict(format=prior['document']['format'] if prior else FORMAT, - project_id=self.session.snapshot['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), + project_id=self.session.summary['project_id'],kind=body['kind'],path=body['path'],current=copy.deepcopy(value), history=history,origin=copy.deepcopy(prior['document']['origin']) if prior else None)) return _result(envelope,True),envelope,size def adopt(self,body,expected_checkpoint,key): - intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.snapshot['project_id'] + intent=self._adoption_intent(body);body,parts,record_id=intent;project_id=self.session.summary['project_id'] pending=[] with self.access.lock: self.access.require('edit','files.adopt',record_id) @@ -350,7 +350,7 @@ def adopt_many(self,body,expected_checkpoint,key): intents=[self._adoption_intent(item) for item in body['items']] ids=[intent[2] for intent in intents] if len(set(ids))!=len(ids):fail('Batch adoption repeats a working-file path.') - body=dict(items=[intent[0] for intent in intents]);project_id=self.session.snapshot['project_id'] + body=dict(items=[intent[0] for intent in intents]);project_id=self.session.summary['project_id'] target=digest('\0'.join(ids).encode());pending=[] with self.access.lock: for identity in ids:self.access.require('edit','files.adopt',identity) @@ -377,7 +377,7 @@ def adopt_deletion(self,body,expected_checkpoint,key): closed(body,{'path','expected_revision','expected_sha256'},'note deletion adoption') parts=_path('note',body['path']);sha(body['expected_sha256']) if type(body['expected_revision']) is not int or not 1<=body['expected_revision']<=2**53-1:fail('Invalid expected note revision.') - project_id=self.session.snapshot['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] + project_id=self.session.summary['project_id'];record_id=file_id(project_id,'note',body['path']);pending=[] with self.access.lock: self.access.require('edit','files.adopt_deletion',record_id) def invoke(): diff --git a/project_history.py b/project_history.py index 892ae3c2..31c5f645 100644 --- a/project_history.py +++ b/project_history.py @@ -760,7 +760,7 @@ def _authorize(self,action,target=None): self.access.require('inspect',action,target) def _origin(self,origin): - return self.session.snapshot['project_id'] if origin is None else origin + return self.session.summary['project_id'] if origin is None else origin def _record(self,collection,identity,origin=None): origin=self._origin(origin);identifier(origin,'Origin catalog');identifier(identity) @@ -1163,7 +1163,7 @@ def create_study(self,body,expected_checkpoint,request_key): closed(body,('name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']);policy=validate_retained_policy(body['policy']) def build(facts): - origin=self.session.snapshot['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) + origin=self.session.summary['project_id'];identity=uuid.uuid4().hex;sha=digest(policy) prior=next((key for key,value in facts.policies.items() if key[0]==origin and key[3]==sha),None) policy_id=prior[1] if prior else uuid.uuid4().hex ref={'kind':'selection_policy','catalog_id':origin,'id':policy_id,'revision':1,'sha256':sha} @@ -1180,7 +1180,7 @@ def save_study(self,study_id,body,expected_checkpoint,request_key): closed(body,('revision','name','limits','policy'));body=checked_json(body) text(body['name'],'Study name');limits=budget(body['limits']) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin));study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['policy'],'selection_policy',origin) policy=facts.policies.get((origin,ref['id'],ref['revision'],ref['sha256'])) @@ -1201,7 +1201,7 @@ def prepare_trial_from_material(self,study_id,body,expected_checkpoint,request_k if request_key not in operation_records(self.session.snapshot['records']): self.materials.recipe(body['material'],verify=True) def build(facts): - origin=self.session.snapshot['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) + origin=self.session.summary['project_id'];doc=copy.deepcopy(self._record(facts.studies,study_id,origin)) study=doc['study'] if body['revision']!=study['revision']:_invalid('Study changed. Reload.','stale') ref=reference(body['material'],'recipe_material',origin);self.materials.recipe(ref) @@ -1224,7 +1224,7 @@ def select_candidate(self,study_id,body,expected_checkpoint,request_key): identifier(body['trial_id']);identifier(body['run_id']) # Expensive semantic proof precedes the write transaction. All exact # associations and selected policy are checked again under publication. - origin=self.session.snapshot['project_id'] + origin=self.session.summary['project_id'] self.access.require('edit','studies.select_candidate',study_id) evidence=(self.evidence(body['run_id'],{},origin_catalog_id=origin) if request_key not in operation_records(self.session.snapshot['records']) else None) @@ -1254,7 +1254,7 @@ def build(facts): def record_lineage(self,body,expected_checkpoint,request_key): closed(body,('trial','intent'));body=checked_json(body,INTENT_LIMIT+4096) def build(facts): - origin=self.session.snapshot['project_id'];trial=facts.trial_ref(body['trial']) + origin=self.session.summary['project_id'];trial=facts.trial_ref(body['trial']) if body['trial']['catalog_id']!=origin:_invalid('Inherited Trials remain immutable historical evidence.','conflict') doc=copy.deepcopy(facts.trials[(origin,trial['id'])]) if doc['lineage'] is not None:_invalid('Trial lineage is already recorded and immutable.','conflict') diff --git a/project_material.py b/project_material.py index 194e91a5..7cba65fe 100644 --- a/project_material.py +++ b/project_material.py @@ -381,7 +381,7 @@ def intents(self): from project_material_fork import is_intent documents = self._documents() active = {(r['family'], r['id']) for r in active_records( - self.session.snapshot['records'], project_id=self.session.snapshot['project_id'])} + self.session.snapshot['records'], project_id=self.session.summary['project_id'])} return [dict(family=family, id=locator, current=copy.deepcopy(doc['current']), fork_origin=copy.deepcopy(doc['fork_origin'])) for (family, locator), doc in sorted(documents.items()) @@ -392,7 +392,7 @@ def intent_capabilities(self): intents = self.intents() unresolved = {(r['family'],r['id']) for r in intents} from project_fork import active_records - active = active_records(self.session.snapshot['records'], project_id=self.session.snapshot['project_id']) + active = active_records(self.session.snapshot['records'], project_id=self.session.summary['project_id']) return dict(requires_reselection=[dict(family=r['family'],id=r['id'],fields=r['current']['required_reselection']) for r in intents], editable=[dict(family=r['family'],id=r['id']) for r in active if r['family'] in EDITABLE and (r['family'],r['id']) not in unresolved]) @@ -461,7 +461,7 @@ def snapshot_ref(self, project_id, manifest_sha256, *, catalog_id=None, verify=F return copy.deepcopy(doc['value']) def capture_definition(self, family, value, before_records, *, action, actor_id=None, audit_facts=None): - catalog_id = self.session.snapshot['project_id'] + catalog_id = self.session.summary['project_id'] value = _definition(family, value, catalog_id) key = catalog_id + '.' + value['id'] prior = decode_records(before_records).get((family, key)) diff --git a/project_measurements.py b/project_measurements.py index 2800b50b..2168d1b7 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -127,7 +127,7 @@ def import_packet(self, raw, source, expected_checkpoint, key): self.session.check() table = MeasurementTable(raw) blob = dict(sha256=table.sha256, bytes=len(raw)) - origin = self.session.snapshot['project_id'] + origin = self.session.summary['project_id'] request = dict(blob=blob, source=copy.deepcopy(source)) def invoke(): diff --git a/project_native_execution.py b/project_native_execution.py index 832a990f..9ac6be4a 100644 --- a/project_native_execution.py +++ b/project_native_execution.py @@ -473,7 +473,7 @@ def reconcile(self,phase_id,expected_checkpoint): execution=self.execution;journal=execution.custody if journal is None:_fail('There is no original local native custody.','unavailable') with self.db.lock: - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected native recovery checkpoint changed.','stale_project') local=journal.refresh() selected=next((row for row in local.get('native_phases',[]) if row['phase_id']==phase_id),None) if selected is None:_fail('A retained phase alone cannot select current native custody.','unavailable') @@ -501,7 +501,7 @@ def reconcile(self,phase_id,expected_checkpoint): if self._active:_fail('Current native owner has not finished its bounded observation.','busy') with execution.access.lock,self.session.lock,self.db.lock: self.access.require('cancel','native_execution.reconcile',phase_id,owner_only=True) - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Native checkpoint changed while draining.','stale_project') if journal.refresh()!=local:_fail('Original native custody changed while draining.','stale_project') binding=journal.binding if (binding['project_id']!=self.db.dataset_id or binding['projection_id']!=self.db.projection_id diff --git a/project_native_preparation.py b/project_native_preparation.py index 85a0b1be..7c504be0 100644 --- a/project_native_preparation.py +++ b/project_native_preparation.py @@ -262,7 +262,7 @@ def probe(self,owner,recipe): if self._probe is None: if self.adapter.db.in_transaction:_fail('Native probe cannot run inside a metadata transaction.','unsupported') self.authorize_probe(recipe) - result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.snapshot['checkpoint'], + result=self.adapter.execution.native.probe(recipe,expected_checkpoint=self.adapter.session.summary['checkpoint'], request_key=self.key+'.probe',preparation=self) self._probe=copy.deepcopy(result['runtime']);self.phase=copy.deepcopy(result['phase']) return copy.deepcopy(self._probe) @@ -334,7 +334,7 @@ def _scope(self,owner,graph,model,composition,purpose,expected_checkpoint,key): if _CURRENT.get() is not None:_fail('A native preparation is already active.','busy') self.access.require('edit','native_preparation.'+purpose,graph['id'],owner_only=True) self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') context=NativePreparationContext(self,owner,graph,model,composition,purpose,_key(purpose,key)) result=dict(format='project_native_preparation_intent_v1',origin_catalog_id=self.session.project_id, actor_id=context.actor.actor_id,purpose=purpose,graph=dict(id=graph['id'],revision=graph['revision'], @@ -362,13 +362,13 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): import release_validation if adapter_for_graph(graph) is release_validation: self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') value=owner.validate_graph(graph) return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') with self._scope(owner,graph,None,None,'validation',expected_checkpoint,key) as context: value=owner.validate_graph(graph);context._check(graph['recipe']) - return self.session.mutate('native_preparation.validate',graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate('native_preparation.validate',graph_id,body,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(value),lambda value,before:dict(records=[],result=value),scope='execute') @contextmanager @@ -376,7 +376,7 @@ def current_graph(self,owner,graph_id,revision,expected_checkpoint,key): if self.db.in_transaction:_fail('Current native validation must precede admission.','unsupported') self.access.require('edit','native_preparation.admission',graph_id,owner_only=True) self.session.check() - if self.session.snapshot['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') + if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') graph=owner.get(graph_id) if graph['revision']!=revision:_fail('Graph changed before native admission.','stale') import release_validation @@ -413,7 +413,7 @@ def capture(value,before): record=self.materials.capture_recipe_material(value,owner.material_path) return dict(records=self.materials._coalesce([record,*self.materials.declarations(record), *self.materials.capture_recipe_dependencies(record,before)]),result=value) - return self.session.mutate(operation,graph_id,body,self.session.snapshot['checkpoint'],key, + return self.session.mutate(operation,graph_id,body,self.session.summary['checkpoint'],key, lambda:owner._retain_material(prepared),capture) finally: if prepared['staging'].exists():shutil.rmtree(prepared['staging']) diff --git a/project_producer_execution.py b/project_producer_execution.py index b4456aba..f4ae5915 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -264,7 +264,7 @@ def _settle_drained(self): # Every effectful request is preceded by canonical uncertainty; # a terminal job with no such request is known not admitted. outcome='not_admitted' - observation=(outcome,True,self.session.snapshot['checkpoint']) + observation=(outcome,True,self.session.summary['checkpoint']) if self._journal_observations.get(identity_key)!=observation: self.execution.update_producer_outcome(family,producer_id,outcome,True) self._journal_observations[identity_key]=observation diff --git a/project_producers.py b/project_producers.py index 79b97f20..10eb94be 100644 --- a/project_producers.py +++ b/project_producers.py @@ -915,7 +915,7 @@ def _publish(owner, raw, role, media='application/json'): fail('Conflicting exact producer payload bytes.') owner._pending_payloads[declaration['sha256']] = raw owner._artifacts[declaration['sha256']] = declaration - return dict(artifact=dict(namespace=owner.session.snapshot['project_id'], sha256=declaration['sha256']), + return dict(artifact=dict(namespace=owner.session.summary['project_id'], sha256=declaration['sha256']), media_type=media, kind='producer_evidence', role=role, preservation='unique', required_for=['verify_producer']) diff --git a/project_projection.py b/project_projection.py index 1181c457..04aaebba 100644 --- a/project_projection.py +++ b/project_projection.py @@ -116,7 +116,7 @@ def __init__(self, store, db, access, roots, *, custody=None, restore=True, runt self.store, self.db, self.access = store, db, access self.custody = custody self.pending_original = [] - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.artifacts = ProjectArtifacts(store) self.sources = None self._disclosures = set() diff --git a/project_runtime.py b/project_runtime.py index a94cc6e6..21fbd63f 100644 --- a/project_runtime.py +++ b/project_runtime.py @@ -185,7 +185,7 @@ def recover_capture(self, config_request, expected_inventory_sha256, expected_ch body_sha = _intent_input(body) self._authorize('runtime_snapshots.create') self.execution.prepare_runtime_reconciliation(key, body_sha, expected_inventory_sha256, expected_checkpoint) - return self.capture(config_request, expected_inventory_sha256, self.session.snapshot['checkpoint'], key, + return self.capture(config_request, expected_inventory_sha256, self.session.summary['checkpoint'], key, reserve_bytes=reserve_bytes) def abandon(self, capture_key, reason, expected_checkpoint, key): @@ -212,7 +212,7 @@ def forbidden(): _fail('No original local runtime copy exists to abandon.', 'una _fail('Runtime intent differs from its exact original local writer.', 'conflict', 409) if capture_key in receipts: _fail('Capture completed; reconcile its acknowledgement with the exact capture request.', 'conflict', 409) - observed = self.session.snapshot['checkpoint'] + observed = self.session.summary['checkpoint'] def invoke(): current = self._authorize('runtime.capture_abandon') if current.actor_id != actor.actor_id: _fail('Runtime abandonment caller changed.', 'conflict', 409) @@ -273,7 +273,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), invalid, lambda *args: invalid(), scope='admin') else: - if expected_checkpoint != self.session.snapshot['checkpoint']: + if expected_checkpoint != self.session.summary['checkpoint']: _fail('Expected project checkpoint changed.', 'stale_project', 409) config = self.resolve_configuration(copy.deepcopy(config_request)) observed = self.owner.inspect(config) @@ -281,7 +281,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 _fail('Runtime inventory changed after inspection.', 'stale', 409) intent = self.session.mutate('runtime.capture_intent', expected_inventory_sha256, body, expected_checkpoint, _intent_key(key), - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key), + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key), lambda result,before:dict(records=[],result=result), scope='admin') guard.publication_started() if os.path.lexists(self.owner.path / expected_inventory_sha256): @@ -294,7 +294,7 @@ def invalid(): _fail('Runtime intent replay is unavailable.', 'unavailable', 409 from pipeline import digest from pathlib import Path raw = read_regular(Path(descriptor['root'])/'manifest.json', MAX_MANIFEST) - runtime_ref = dict(kind='runtime',catalog_id=self.session.snapshot['project_id'], + runtime_ref = dict(kind='runtime',catalog_id=self.session.summary['project_id'], id=digest(descriptor),sha256=digest(descriptor)) value = dict(ref=runtime_ref,descriptor=descriptor,environment=None,manifest=decode(raw)) observed = dict(inventory_sha256=descriptor['manifest_sha256'],file_count=descriptor['file_count'],total_bytes=descriptor['total_bytes']) @@ -302,8 +302,8 @@ def capture(result, before): record = self.materials.capture_runtime(value, raw) return dict(records=[*self.materials.declarations(record),record],result=result) result = self.session.mutate('runtime.capture', expected_inventory_sha256, body, - self.session.snapshot['checkpoint'], key, - lambda:_summary(self.session.snapshot['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') + self.session.summary['checkpoint'], key, + lambda:_summary(self.session.summary['project_id'], observed, actor.actor_id, key, runtime_ref=runtime_ref), capture, scope='admin') guard.confirmed() return result except BaseException as error: diff --git a/project_session.py b/project_session.py index de2293d7..308164c5 100644 --- a/project_session.py +++ b/project_session.py @@ -86,7 +86,7 @@ def __init__(self, store, db, access, artifacts, *, validate_records): self._validate = validate_records self.available = False self._snapshot = None - self.project_id = store.snapshot()['project_id'] + self.project_id = store.summary()['project_id'] self.extra_capture = None self.publication_intent = None self.runtime_access = None @@ -177,6 +177,12 @@ def reconcile_selection(self, snapshot): self._recovery_records = None self._recovery_request = None + @property + def summary(self): + """Scalar metadata for the same bound checkpoint as snapshot.""" + with self.lock: + return {key: self._snapshot[key] for key in ('project_id', 'name', 'checkpoint')} + @property def snapshot(self): with self.lock: diff --git a/project_sources.py b/project_sources.py index 08939eba..25007ea1 100644 --- a/project_sources.py +++ b/project_sources.py @@ -119,7 +119,7 @@ def register(self,body,expected_checkpoint,key): actor=self.access.require('edit','source.register',owner_only=True) self.session.check() saved,raw=self._read_registration(body) - origin=self.session.snapshot['project_id'];invoked=False + origin=self.session.summary['project_id'];invoked=False def invoke(): nonlocal invoked invoked=True;self._bind(saved,raw) diff --git a/project_store.py b/project_store.py index 93ba5ad3..cb1c2bf9 100644 --- a/project_store.py +++ b/project_store.py @@ -804,6 +804,12 @@ def _current(self): except OSError as exc: _io_error(exc, 'read project snapshot') + def summary(self): + """Return scalar metadata after the same complete freshness check.""" + with self._mutex: + manifest, _records = self._current() + return {key: manifest[key] for key in ('project_id', 'name', 'checkpoint')} + def snapshot(self): with self._mutex: manifest, records = self._current() diff --git a/project_study_execution.py b/project_study_execution.py index 561c4779..fa6d6ce9 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -267,7 +267,7 @@ def __init__(self,adapter,phase): def __getattr__(self,name):return getattr(self.source,name) def open(self,body): return self.adapter.projection.source_adapter.register(body, - self.adapter.session.snapshot['checkpoint'],self.phase+'-source') + self.adapter.session.summary['checkpoint'],self.phase+'-source') class _PreparationModels: @@ -278,11 +278,11 @@ def _read_blob(self,ref,item,maximum=40*1024**2):return self.adapter.materials.b def create(self,body): adapter=self.adapter return adapter.materials.mutate_definition('model_definition','models.create',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) + adapter.session.summary['checkpoint'],self.phase+'-model',lambda:adapter.projection.models.create(body)) def publish(self,identity,body): adapter=self.adapter return adapter.materials.publish_model(adapter.projection.models,identity,body, - adapter.session.snapshot['checkpoint'],self.phase+'-publish') + adapter.session.summary['checkpoint'],self.phase+'-publish') class _PreparationPipelines: @@ -295,7 +295,7 @@ def recipe_material(self,ref,*,verify=False):return self.adapter.materials.recip def create_external(self,body): adapter=self.adapter return adapter.materials.mutate_definition('pipeline_graph','graphs.create_external',None,body, - adapter.session.snapshot['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) + adapter.session.summary['checkpoint'],self.phase+'-graph',lambda:self.owner.create_external(body)) def prepare_external_material(self,*args,**kwargs):return self.owner.prepare_external_material(*args,**kwargs) def _retain_material(self,prepared): adapter=self.adapter;value=self.owner._retain_material(prepared) @@ -310,7 +310,7 @@ def __init__(self,projection,execution): self.projection,self.execution=projection,execution self.session,self.db,self.history=projection.session,projection.db,projection.history self.materials=projection.materials;self.access=execution.runtime_access - self.origin=self.session.snapshot['project_id'] + self.origin=self.session.summary['project_id'] self._dirty=set();self._revisions={};self._epochs={};self._pending_epochs={} self._package_records=[] self.context=copy.copy(projection.context) @@ -573,7 +573,7 @@ def approve(self,study_id,body,expected_checkpoint,key): validate_operation_result(prior['operation'],prior['result'],self.session.snapshot['records']) self._current_grant(prior['result'],current.actor_id) else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self._materialize_trial(study_id,body['trial_id'],body['expected_material_sha256']) def invoke(): grant=self.studies.approve(study_id,body) @@ -597,7 +597,7 @@ def admit(self,study_id,body,expected_checkpoint,key): 'SELECT run_id FROM model_attempt_reservations WHERE run_id=%s',(result.get('run_id'),)).fetchone() is None): _invalid('A historical admission receipt cannot create a current local attempt.','unavailable') else: - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') self.require_current_epoch(self.studies._study(study_id)) self.reject_historical_request(study_id,authority.actor_id,key,None) trial=self._materialize_trial(study_id,body['trial_id']) @@ -626,7 +626,7 @@ def invoke(): # Qualification may launch the explicit probe. It finishes # before Session opens the finite admission transaction. with preparation.admission(pipelines,ref,expected_checkpoint,key): - return self._mutate('study_execution.admit',study_id,body,self.session.snapshot['checkpoint'],key,invoke,'inspect') + return self._mutate('study_execution.admit',study_id,body,self.session.summary['checkpoint'],key,invoke,'inspect') return self._mutate('study_execution.admit',study_id,body,expected_checkpoint,key,invoke,'inspect') def stop(self,study_id,body,expected_checkpoint,key): @@ -645,7 +645,7 @@ def invoke(): # cancellation phase. Receipt replay does not repeat effects. for run_id in invoked: cancel_key='study-stop-'+hashlib.sha256((key+':'+run_id).encode()).hexdigest() - try:self.execution.cancel_run(run_id,self.session.snapshot['checkpoint'],cancel_key) + try:self.execution.cancel_run(run_id,self.session.summary['checkpoint'],cancel_key) except BaseException as error: error.study_stop_receipt=copy.deepcopy(result) raise @@ -683,7 +683,7 @@ def reconcile_preparation(self,study_id,body,expected_checkpoint): if complete.get('native_receipt'): effective=_preparation_body(intent,body['preparation_key']) effective.pop('request_key');effective['actor_id']=intent['actor_id'] - result=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],body['preparation_key'], + result=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],body['preparation_key'], lambda:copy.deepcopy(complete['result']),'cancel') else:result=copy.deepcopy(complete['result']) guard.confirmed() @@ -703,7 +703,7 @@ def abandon_preparation(self,study_id,body,expected_checkpoint,key): operation,current_intent,complete=_preparation_reference(self.session.snapshot['records'],study_id,body['preparation_key']) if current_intent!=intent:_invalid('Preparation intent changed during guarded reconciliation.','conflict') if complete is not None:_invalid('Preparation is complete; reconcile its acknowledgement without abandoning science.','conflict') - observed_checkpoint=self.session.snapshot['checkpoint'] + observed_checkpoint=self.session.summary['checkpoint'] def invoke(): actor=self.access.require('cancel','studies.abandon_preparation',study_id,owner_only=True) if actor.actor_id!=current.actor_id:_invalid('Abandonment caller changed.','conflict') @@ -741,7 +741,7 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): # acknowledges science and cannot repeat filesystem effects. return self._mutate(operation,study_id,effective,expected_checkpoint,key, lambda:(_ for _ in ()).throw(RuntimeError('Completed preparation replay invoked effects.')),'edit') - if expected_checkpoint!=self.session.snapshot['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') + if expected_checkpoint!=self.session.summary['checkpoint']:_invalid('Expected project checkpoint changed.','stale_project') receipts=operation_records(self.session.snapshot['records']) if any(_intent_key(other,key) in receipts for other in PREPARATIONS if other!=operation): _invalid('Preparation key already identifies another preparation operation.','conflict') @@ -778,7 +778,7 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): publication=SimpleNamespace(runtime_snapshot=snapshot,before_write=guard.publication_started, read_recipe_blob=self.materials.blob) result=_prepare_operation(studies,study_id,dict(body,request_key=key),kind,publication=publication) - value=self._mutate(operation,study_id,effective,self.session.snapshot['checkpoint'],key, + value=self._mutate(operation,study_id,effective,self.session.summary['checkpoint'],key, lambda:copy.deepcopy(result),'edit') guard.confirmed() return value diff --git a/project_text.py b/project_text.py index d9502cb6..fc778528 100644 --- a/project_text.py +++ b/project_text.py @@ -252,7 +252,7 @@ def restore(self, snapshot=None): encode(row['annotation']) if row['annotation'] is not None else None, row['review'], row['revision'], row['created_at'], row['updated_at'])) for state in doc['history']: self._workbench._history(state) - if self.store.snapshot()['checkpoint'] != current['checkpoint']: + if self.store.summary()['checkpoint'] != current['checkpoint']: raise ProjectTextError('Project changed during reconstruction.', 'stale_project', 409) self._snapshot, self.available = current, True return copy.deepcopy(current) diff --git a/project_transfer.py b/project_transfer.py index 10a6b431..91848559 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -720,7 +720,7 @@ def replay(opened): try: store=ProjectStore.create_at(destination_parent_fd,stage_name,name,'extraction-stage.'+digest(request_key.encode())[:32],request_digest,root_path=root_path) with store.root_directory() as owned_fd:stage_identity=_fd_identity(owned_fd) - destination=store.snapshot()['project_id'];created_at=timestamp() + destination=store.summary()['project_id'];created_at=timestamp() from project_data_fork import fork_records as fork_data from project_material_fork import fork_records as fork_material original=copy.deepcopy(records);mappings=[] @@ -755,7 +755,7 @@ def replay(opened): with source.artifacts.open_verified(value,maximum_bytes=value['bytes']) as fd:artifacts.publish_fd(fd,value,maximum_bytes=value['bytes']) with store.root_directory() as fd:materialize_captured(records,destination,fd) _validate_candidate(records,destination,artifacts) - committed=store.commit(records,store.snapshot()['checkpoint'],request_key,request_digest) + committed=store.commit(records,store.summary()['checkpoint'],request_key,request_digest) verification=_cold_verify(store,source) store.save(committed['checkpoint']);_assert_unchanged(source) _rename_noreplace(destination_parent_fd,stage_name,directory);published=True;os.fsync(destination_parent_fd) diff --git a/project_workspace.py b/project_workspace.py index 072d6c24..b006dbe2 100644 --- a/project_workspace.py +++ b/project_workspace.py @@ -122,7 +122,7 @@ def _state(self): project = None if self._recovery is None and self._store is not None: try: - snapshot = self._store.snapshot() + snapshot = self._store.summary() except ProjectStoreError as error: if error.code not in ('unknown', 'outcome_unknown', 'stale', 'stale_project', 'invalid', 'unsupported', 'unavailable'): @@ -184,7 +184,7 @@ def _bound(self, arguments, *, may_be_empty=False, check_checkpoint=True, allow_ if arguments.get('expected_checkpoint') is not None: raise WorkbenchError('The empty workspace has no checkpoint.', 'stale_project', 409) return - snapshot = self._store.snapshot() + snapshot = self._store.summary() expected = arguments.get('expected_checkpoint', arguments.get('checkpoint')) if check_checkpoint and expected != snapshot['checkpoint']: raise WorkbenchError('The project checkpoint changed. Refresh before acting.', 'stale_project', 409) @@ -206,7 +206,7 @@ def _root(self, alias, relative): def _projection_for(self, store): identity = uuid.uuid4().hex path = self.path / '.project-projections' / identity - snapshot = store.snapshot() + snapshot = store.summary() projection = Metadata(path, dsn=self._dsn, dataset_id=snapshot['project_id'], projection_id=identity) try: owner = ProjectProjection(store, projection, self.access, @@ -227,7 +227,7 @@ def _retained_projection_for(self, store, projection_id): if type(projection_id) is not str or not re.fullmatch('[a-f0-9]{32}', projection_id): raise WorkbenchError('Invalid original local projection identity.', 'unavailable', 409) db = Metadata(self.path / '.project-projections' / projection_id, dsn=self._dsn, - create=False, dataset_id=store.snapshot()['project_id'], projection_id=projection_id) + create=False, dataset_id=store.summary()['project_id'], projection_id=projection_id) try: return ProjectProjection(store, db, self.access, {alias: value[0] for alias, value in self._roots.items()}, @@ -243,7 +243,7 @@ def _detach(self): self._projection.execution.assert_quiescent() if self._store is not None: # Prove pending project publications and their full closure first. - self._store.save(self._store.snapshot()['checkpoint']) + self._store.save(self._store.summary()['checkpoint']) if self._projection is not None: self._projection.close() if self._store is not None: @@ -272,14 +272,14 @@ def _recover_current(self, arguments): store = projection = None try: store = ProjectStore.open_at(root_fd, arguments['directory'], root_path=root_path) - snapshot = store.snapshot() + snapshot = store.summary() if snapshot['project_id'] != self._last_project['project_id']: raise WorkbenchError('Recovery destination has a different project identity.', 'stale_project', 409) # A reread reconciles identity; only a successful durability barrier # resolves a prior checkpoint/directory fsync failure. store.save(snapshot['checkpoint']) projection, text = self._projection_for(store) - snapshot = store.snapshot() + snapshot = store.summary() if self._projection is not None: self._projection.close() self._store, self._projection, self._text = store, projection, text @@ -332,7 +332,7 @@ def _select(self, action, arguments): if store.reconcile(arguments['request_key'], digest) is None: raise error reconciled = True - snapshot = store.snapshot() + snapshot = store.summary() if reconciled: store.save(snapshot['checkpoint']) created = snapshot @@ -342,7 +342,7 @@ def _select(self, action, arguments): self._detach() self._store, self._projection, self._text = store, projection, text self._selection = selection - snapshot = store.snapshot() + snapshot = store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} self._recovery = None self._binding = uuid.uuid4().hex @@ -366,7 +366,7 @@ def _select(self, action, arguments): raise def _record_result(self, record, *, before_revision=None, history_limit=1): - snapshot = self._store.snapshot() + snapshot = self._store.summary() history = [item for item in self._text.history(record['id']) if item['revision'] <= record['revision'] and (before_revision is None or item['revision'] < before_revision)] @@ -402,14 +402,14 @@ def _publish_independent(self, arguments, *, composed=False): if composed: from project_composition import publish_composition from project_artifacts import ProjectArtifacts - others=[];identities={self._store.snapshot()['project_id']} + others=[];identities={self._store.summary()['project_id']} for selected in arguments['sources']: if selected['project_id'] in identities: raise WorkbenchError('Composition source identity repeats.','conflict',409) identities.add(selected['project_id']) directory,descriptor=self._root(selected['root'],selected['directory']) store=retained.enter_context(ProjectStore.open_at(descriptor,selected['directory'],root_path=directory)) - actual=store.snapshot() + actual=store.summary() if actual['project_id']!=selected['project_id'] or actual['checkpoint']!=selected['checkpoint']: raise WorkbenchError('Selected composition source checkpoint differs.','stale_project',409) artifacts=retained.enter_context(ProjectArtifacts(store)) @@ -519,7 +519,7 @@ def dispatch(self, name, arguments, *, selected_version=None): if getattr(error, 'code', None) in ('unknown', 'outcome_unknown', 'projection_unavailable'): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} result = dict(binding=self._binding, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint'], value=value) @@ -629,14 +629,14 @@ def _dispatch_draining(self, name, arguments): raise WorkbenchError('The retained physical recovery owner changed.', 'local_custody_unavailable', 409) recovering = True if incoming is not projection and name == 'project_files.execution_reconcile_current': - value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.snapshot['checkpoint']) + value = incoming.execution.reconcile_original(expected_checkpoint=incoming.session.summary['checkpoint']) else: internal = arguments if recovery_operation and recovering and name != 'project_files.execution_reconcile_current': # Public CAS above uses the last acknowledged workspace # view. Only this qualified local owner sees its exact # selected/attempted checkpoint for finite recovery. - internal = dict(arguments, expected_checkpoint=incoming.session.snapshot['checkpoint']) + internal = dict(arguments, expected_checkpoint=incoming.session.summary['checkpoint']) if name == 'project_files.runtime_capture' and recovering: options = {'reserve_bytes': internal['reserve_bytes']} if 'reserve_bytes' in internal else {} value = incoming.runtime.recover_capture({'source': internal['source']}, @@ -660,7 +660,7 @@ def _dispatch_draining(self, name, arguments): self._recovery_required('projection_unavailable', arguments.get('request_key')) raise with self.access.lock, projection.session.lock, projection.db.lock: - snapshot = self._store.snapshot() + snapshot = self._store.summary() self._last_project = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} if name in _RECOVERY_OPERATIONS: self._recovery = None diff --git a/tests/test_project_dispatch.py b/tests/test_project_dispatch.py index f55de4cb..8aab246d 100644 --- a/tests/test_project_dispatch.py +++ b/tests/test_project_dispatch.py @@ -30,7 +30,8 @@ def require(self, *args, **kwargs): def projection(): access = Authority() - session = SimpleNamespace(access=access, check=Mock(), snapshot={'project_id': PROJECT, 'records': []}) + session = SimpleNamespace(access=access, check=Mock(), snapshot={'project_id': PROJECT, 'records': []}, + summary={'project_id': PROJECT, 'name': 'Dispatch fixture', 'checkpoint': CHECKPOINT}) return SimpleNamespace(session=session, data=Mock(), materials=Mock(), history=Mock(), context=SimpleNamespace(models=Mock(), pipelines=Mock()), artifacts=Mock(), producers=Mock(), source_adapter=Mock(), runtime_manifest=Mock(), disclose=Mock(), resolve_artifact=Mock(), execution=SimpleNamespace(study_execution=Mock(), producer_execution=Mock(), health=Mock(), preflight_external=Mock(), approve_external=Mock(), submit_external=Mock(), cancel_run=Mock(), reconcile_current=Mock())) @@ -38,7 +39,7 @@ def projection(): class Dispatch(unittest.TestCase): def test_supported_operations_match_the_finite_public_contract(self): - self.assertEqual(SUPPORTED_OPERATIONS, agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | (agent_api.PROJECT_FILES_M4 - {'project_files.save_as'})) + self.assertEqual(SUPPORTED_OPERATIONS, agent_api.PROJECT_FILES_M2 | agent_api.PROJECT_FILES_M3 | (agent_api.PROJECT_FILES_M4 - {'project_files.save_as', 'project_files.compose'})) with self.assertRaises(WorkbenchError) as failure: dispatch(object(), 'project_files.runs_execute', {}) self.assertEqual(failure.exception.code, 'unsupported') diff --git a/tests/test_project_files_fork_contracts.py b/tests/test_project_files_fork_contracts.py index 309f8ed2..b731de2c 100644 --- a/tests/test_project_files_fork_contracts.py +++ b/tests/test_project_files_fork_contracts.py @@ -35,8 +35,9 @@ def test_old_pins_stay_exact_and_only_v2_accepts_closed_fork_origins(self): health = 'project_files.execution_health' self.assertEqual(api.contract(health,1)['sha256'], 'c3b1d0017f5a49cf69c3a4c5c33e1d3db9846404d12627563d97ef40de7404a5') + comparison = 'project_files.measurement_compare' self.assertEqual(versions,{name:{str(v):api.contract(name,v)['sha256'] for v in (1,2)} - for name in (*expected, health)}) + for name in (*expected, health, comparison)}) def test_sdk_selects_v2_and_falls_back_to_exact_v1_without_falsifying_provenance(self): value=fork_result();raw=client({'status':200,'result':value});sdk=ProjectFilesClient(raw) diff --git a/tests/test_project_material.py b/tests/test_project_material.py index 8fe2d11a..5e412305 100644 --- a/tests/test_project_material.py +++ b/tests/test_project_material.py @@ -55,7 +55,8 @@ def setUp(self): self.addCleanup(lambda:self.store.close()) self.artifacts=ProjectArtifacts(self.store);self.addCleanup(lambda:self.artifacts.close()) self.catalog=self.store.snapshot()['project_id'] - self.session=SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,check=lambda:None) + self.session=SimpleNamespace(snapshot=self.store.snapshot(),summary=self.store.summary(), + artifacts=self.artifacts,check=lambda:None) self.material=ProjectMaterial(self.session) def capture(self): diff --git a/tests/test_project_material_fork.py b/tests/test_project_material_fork.py index 89681263..d28881ac 100644 --- a/tests/test_project_material_fork.py +++ b/tests/test_project_material_fork.py @@ -52,7 +52,8 @@ def overlay(records, made, source, checkpoint, destination): class MaterialForkCodecs(unittest.TestCase): def setUp(self): self.source=uuid.uuid4().hex;self.checkpoint='a'*64 - self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]),check=lambda:None) + self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]), + summary=dict(project_id=self.source,checkpoint=self.checkpoint),check=lambda:None) self.material=ProjectMaterial(self.session) def capture(self,family,current): @@ -108,7 +109,8 @@ def test_bound_drafts_preserve_full_configuration_as_visible_nonexecutable_inten ('pipeline_graph',graph(external=True)),('recipe_definition',recipe(self.source))] captured=[self.capture(f,v) for f,v in drafts] made,combined,destination=self.fork() - session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined),check=lambda:None) + session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined), + summary=dict(project_id=destination),check=lambda:None) reader=ProjectMaterial(session) intents=reader.intents() self.assertEqual(len(intents),4) @@ -167,7 +169,8 @@ def test_captured_source_selector_stays_in_inert_intent_and_cannot_publish(self) doc=made['records'][0]['document'] self.assertEqual(doc['current']['required_reselection'],['source_selector']) self.assertEqual(doc['current']['definition']['source_selector'],value['source_selector']) - reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined),check=lambda:None)) + reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined), + summary=dict(project_id=destination),check=lambda:None)) with self.assertRaises(WorkbenchError) as caught: reader.publish_model(SimpleNamespace(publish=lambda *args: self.fail('Intent reached owner publication')), doc['current']['id'],{'revision':1},'a'*64,'forbidden-intent') diff --git a/tests/test_project_native_process.py b/tests/test_project_native_process.py index 287de66a..5df959c4 100644 --- a/tests/test_project_native_process.py +++ b/tests/test_project_native_process.py @@ -5,6 +5,7 @@ import os from pathlib import Path import select +import shutil import signal import sys import tempfile @@ -311,8 +312,11 @@ def test_current_interpreter_alias_drift_before_launch_cannot_select_another_int stage=self.root/'venv-alias-drift' prepared=native.prepare(stage,phase='probe',attempt_id=uuid.uuid4().hex,request_sha256='d'*64, runtime=runtime,worker='text_training.py',sources={'text_training.py':WORKER},wall_seconds=8) - Path(executable).unlink();Path(executable).symlink_to(Path('/usr/bin/python3').resolve()) - with self.assertRaises(custody.ProcessError) as error:prepared.launch(stage/'custody') + replacement=self.root/'replacement-python' + shutil.copy2(runtime['executable_source'],replacement) + self.assertNotEqual(replacement.resolve(),Path(executable).resolve()) + Path(executable).unlink();Path(executable).symlink_to(replacement) + with self.assertRaises(custody.ProcessError) as error:self.launch(prepared) self.assertEqual(error.exception.code,'stale');self.assertFalse((stage/'custody').exists()) def test_spawn_callback_is_not_reached_for_current_config_alias_or_capture_refusal(self): diff --git a/tests/test_project_session.py b/tests/test_project_session.py index 309f2bc4..75ff7795 100644 --- a/tests/test_project_session.py +++ b/tests/test_project_session.py @@ -174,6 +174,23 @@ def test_complete_merge_and_original_result_replay_before_stale_cas(self): self.assertIn(('project_artifact', other['sha256']), ids) self.assertEqual(self.artifacts.read_blob({key: declaration[key] for key in ('sha256', 'bytes')}), 'Irreplaceable observation 山'.encode()) + def test_scalar_summary_is_independent_and_tracks_the_bound_publication(self): + before = self.session.summary + arguments, _declaration, _calls = self.mutation() + self.call(arguments) + snapshot = self.session.snapshot + expected = {key: snapshot[key] for key in ('project_id', 'name', 'checkpoint')} + self.assertNotEqual(before['checkpoint'], expected['checkpoint']) + original_copy = copy.deepcopy + def observe(value): + self.assertIsNot(value, self.session._snapshot, 'Summary must not clone the record graph') + return original_copy(value) + with patch('project_session.copy', SimpleNamespace(deepcopy=observe)): + first = self.session.summary + first['name'] = 'Caller mutation' + self.assertEqual(self.session.summary, expected) + self.assertEqual(self.session.snapshot['name'], expected['name']) + def test_key_conflict_stale_and_current_auth_refuse_before_invoke(self): first, _declaration, _calls = self.mutation() self.call(first) diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 9dcf11af..f3e7f0e2 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -97,6 +97,18 @@ def test_appending_one_record_does_not_rewrite_saved_immutable_revisions(self): 'Appending one record must not write disposable copies of every saved immutable revision') self.assertEqual(self.reopen(), committed) + def test_summary_reads_every_current_record_without_cloning_the_record_graph(self): + committed = self.commit([record('a'), record('b')], key='summary-fixture') + expected = {key: committed[key] for key in ('project_id', 'name', 'checkpoint')} + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused record graph copy')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: + first = self.store.summary() + first['name'] = 'Caller mutation' + second = self.store.summary() + self.assertEqual(second, expected) + self.assertEqual(reads.call_count, 4, 'Each summary must still read/hash both actual records') + self.assertEqual(self.store.snapshot(), committed) + def test_repeated_snapshot_and_root_reads_recheck_bytes_without_redecoding_identical_records(self): rows=[record('a'),record('b','Exact second note\n')] self.commit(rows,key='repeat-read-fixture') @@ -124,6 +136,7 @@ def test_verified_decode_cache_still_rejects_same_length_corruption_with_restore path.chmod(0o600);path.write_bytes(changed) os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) self.assert_code('stale',self.store.snapshot) + self.assert_code('stale',self.store.summary) with self.assertRaises(ProjectStoreError) as caught: with self.store.root_directory(verify_after=True):pass self.assertEqual(caught.exception.code,'stale') @@ -250,12 +263,14 @@ def test_replaced_physical_root_or_metadata_directory_is_rejected(self): self.root.rename(displaced) self.root.mkdir() self.assert_code('stale', self.store.snapshot) + self.assert_code('stale', self.store.summary) self.root.rmdir() displaced.rename(self.root) meta = self.root / '.tuldok' meta.rename(self.root / '.old-tuldok') meta.mkdir() self.assert_code('stale', self.store.snapshot) + self.assert_code('stale', self.store.summary) def test_changed_checkpoint_is_rejected_before_mutation(self): initial = self.store.snapshot() diff --git a/tests/test_project_workspace.py b/tests/test_project_workspace.py index 0c2dbaee..a78982f4 100644 --- a/tests/test_project_workspace.py +++ b/tests/test_project_workspace.py @@ -72,6 +72,17 @@ def create(self, directory='alpha'): arguments = self.mutation(root='fixture', directory=directory, name='Workspace authored project') return self.call('new', arguments), arguments + def test_status_returns_current_metadata_without_copying_the_record_graph(self): + self.create() + self.import_record('A retained record unrelated to the project summary') + expected = self.state() + with patch.object(self.workspace._store, '_snapshot', + wraps=self.workspace._store._snapshot) as snapshots: + observed = self.state() + self.assertEqual(observed, expected) + self.assertEqual(snapshots.call_count, 0, + 'Status must not copy every retained record to return three metadata fields') + def test_control_project_ancestry_is_rejected_before_destination_or_projection_creation(self): with ProjectStore.create(self.external/'science','Existing portable project','existing-project','b'*64):pass self.start(self.external/'science/control') From 9f08a248378f6ad1d65061976c6015bde16b7e56 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 00:15:42 -0700 Subject: [PATCH 27/33] perf(projects): verify freshness without discarded record copies --- project_data.py | 8 +-- project_execution.py | 22 +++---- project_extraction.py | 2 +- project_files.py | 2 +- project_history.py | 2 +- project_material.py | 4 +- project_measurements.py | 14 ++-- project_native_execution.py | 8 +-- project_native_preparation.py | 12 ++-- project_producer_execution.py | 4 +- project_producers.py | 4 +- project_projection.py | 6 +- project_runtime.py | 6 +- project_session.py | 17 ++++- project_sources.py | 12 ++-- project_study_execution.py | 10 +-- project_transfer_controller.py | 2 +- tests/test_project_extraction.py | 6 +- tests/test_project_files.py | 10 +++ tests/test_project_fork.py | 4 +- tests/test_project_fork_composition.py | 2 +- tests/test_project_history.py | 4 +- tests/test_project_material.py | 14 ++-- tests/test_project_material_fork.py | 6 +- tests/test_project_native_extraction.py | 4 +- tests/test_project_session_freshness.py | 88 +++++++++++++++++++++++++ tests/test_project_transfer.py | 2 +- 27 files changed, 192 insertions(+), 83 deletions(-) create mode 100644 tests/test_project_session_freshness.py diff --git a/project_data.py b/project_data.py index 6c99e8b2..ca70bab0 100644 --- a/project_data.py +++ b/project_data.py @@ -1028,7 +1028,7 @@ def _mutate(self, operation, target, body, expected_checkpoint, key, invoke): fail('Data mutation body must be an object.') self._permission(operation, target, body) with self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() if operation in ('data.annotate','data.correct_rights_note','data.save_image_source','data.delete_image_source'): self._require_record(target,extended=operation!='data.annotate') legacy = self._legacy_replay(operation, target, body, key) @@ -1158,10 +1158,10 @@ def freeze_release(self, body, expected_checkpoint, key): def _read(self, action, invoke, target=None): with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() self._sync_sources() result = invoke() - self.session.check() + self.session.check_current() return result def query(self, options=None): @@ -1399,7 +1399,7 @@ def restore(self, snapshot=None): self.db.execute('INSERT INTO saved_searches VALUES (%s,%s,%s,%s,%s,%s)', (record_id,r['name'],r['revision'],encode_retained(r['criteria']),r['created_at'],r['updated_at'])) elif family == 'dataset_exposure': self.db.execute('INSERT INTO workspace_exposures VALUES (%s,%s,%s,%s,%s,%s,%s)', tuple(doc[k] for k in ('id','record_id','source_hash','reason','run_id','release_id','created_at'))) - self.session.check() + self.session.check_current() self._materialize_images_and_releases(owned) return snapshot diff --git a/project_execution.py b/project_execution.py index fece88a1..0e4941c6 100644 --- a/project_execution.py +++ b/project_execution.py @@ -52,7 +52,7 @@ def publication_started(self): def confirmed(self): if not self.started:raise RuntimeError('File publication must have a durable start before confirmation.') - execution=self.execution;execution.session.check() + execution=self.execution;execution.session.check_current() if execution.db.in_transaction: raise RuntimeError('File publication confirms only after its canonical and outer SQL acknowledgement.') if execution.custody is not None: @@ -164,7 +164,7 @@ def register_custody(self, identity, handle): def before_launch(self, identity): with self.db.lock: - self.session.check();run=self.runs._load(identity);self._proof(run) + self.session.check_current();run=self.runs._load(identity);self._proof(run) self.runtime_access.require_job('runs',identity,run['admission_id'],run['request_sha256']) if self.custody is not None: with self._journal_guard():self.custody.launch_intent(identity) @@ -176,7 +176,7 @@ def _publication_intent(self, before, records, key, sha): def _local_owner(self, *, recover=True): """Construct execution only after the caller checked exact authority.""" - self.session.check() + self.session.check_current() if self._runs is None: from runs import Runs from pipeline import Pipelines @@ -398,7 +398,7 @@ def _producer_authority(self, family, identity, admission_id, request_digest): def register_producer(self, family, identity, admission_id, request_digest): with self.db.lock: - self.session.check();self._producer_authority(family,identity,admission_id,request_digest) + self.session.check_current();self._producer_authority(family,identity,admission_id,request_digest) if self.custody is not None: with self._journal_guard(): if not self.custody.has_pending:self.custody.acknowledge(self.session.snapshot) @@ -406,7 +406,7 @@ def register_producer(self, family, identity, admission_id, request_digest): def producer_effect_intent(self, family, identity): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -416,7 +416,7 @@ def producer_effect_intent(self, family, identity): def update_producer_outcome(self, family, identity, provider_outcome, worker_drained): with self.db.lock: - self.session.check() + self.session.check_current() if self.custody is not None: value=next((item for item in self.custody.snapshot()['producers'] if (item['family'],item['producer_id'])==(family,identity)),None) @@ -591,7 +591,7 @@ def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capt def preflight_external(self, graph_id, revision): self.runtime_access.require('inspect', 'pipeline.preflight_external', graph_id) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() return self.projection.pipelines.preflight_external(graph_id, revision) def native_preflight(self,graph_id,revision,expected_checkpoint,key): @@ -615,7 +615,7 @@ def native_submit(self,body,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.submit',body['graph_id'],owner_only=True) effective=dict(body,actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.submit',body['graph_id'],effective,expected_checkpoint,key) if replay is not None:return replay with self.projection.native_preparation.current_graph(self.projection.pipelines,body['graph_id'], @@ -633,7 +633,7 @@ def native_retry(self,run_id,expected_checkpoint,key): actor=self.runtime_access.require('execute','runs.retry',run_id,owner_only=True) body=dict(actor_id=actor.actor_id) with self.db.lock: - self.session.check() + self.session.check_current() replay=self._native_replay('runs.retry',run_id,body,expected_checkpoint,key) if replay is not None:return replay if self._runs is None:raise WorkbenchError('Retry requires the actual current local Run owner.','unavailable',409) @@ -655,7 +655,7 @@ def native_reconcile(self,phase_id,expected_checkpoint): def submit_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require_external_submission() with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) @@ -678,7 +678,7 @@ def invoke(): def approve_external(self, body, expected_checkpoint, key): authority = self.runtime_access.require('approve-execution', 'runs.approve_external', owner_only=True) with self.access.lock, self.session.lock, self.db.lock: - self.session.check() + self.session.check_current() prior = operation_records(self.session.snapshot['records']).get(key) if prior is None and expected_checkpoint != self.session.summary['checkpoint']: raise WorkbenchError('Expected project checkpoint changed.', 'stale_project', 409) diff --git a/project_extraction.py b/project_extraction.py index 82db29c7..461b0d0f 100644 --- a/project_extraction.py +++ b/project_extraction.py @@ -69,7 +69,7 @@ def __init__(self, source): self.payload_gaps = [] self.deleted_intents = set() self.session = SimpleNamespace(snapshot=source.snapshot, db=self.db, - artifacts=source.artifacts, check=lambda: None) + artifacts=source.artifacts, check_current=lambda: None) # Workbench and each metadata-view constructor only assign references; # none runs SQL, creates directories or constructs Dataset/Jobs/Runs. # Never call get/history: those public owners may enroll images. diff --git a/project_files.py b/project_files.py index c6e9f6b6..e2a5585b 100644 --- a/project_files.py +++ b/project_files.py @@ -264,7 +264,7 @@ def inspect(self): self.access.require('inspect','files.inspect');snapshot=self.session.check() with self.store.root_directory() as root: items=inspect_captured(snapshot['records'],snapshot['project_id'],root) - self.session.check() + self.session.check_current() return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) def read(self,record_id,revision=None): diff --git a/project_history.py b/project_history.py index 31c5f645..8733ca4a 100644 --- a/project_history.py +++ b/project_history.py @@ -752,7 +752,7 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): self.materials=materials;self.authorize_disclosure=authorize_disclosure def _facts(self): - self.session.check() + self.session.check_current() records=self.session.snapshot['records'] return decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) diff --git a/project_material.py b/project_material.py index 7cba65fe..fbfa0189 100644 --- a/project_material.py +++ b/project_material.py @@ -359,7 +359,7 @@ def __init__(self, session): self.owner_access = _MaterialAccess(session) if hasattr(session,'access') else None def _documents(self): - self.session.check() + self.session.check_current() return decode_records(self.session.snapshot['records']) def read_view(self, records): @@ -369,7 +369,7 @@ def read_view(self, records): live publication, verification or execution capability; later reads must select their own checkpoint and construct a new view. """ - self.session.check() + self.session.check_current() return MaterialSnapshot(records) @_boundary diff --git a/project_measurements.py b/project_measurements.py index 2168d1b7..400db842 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -124,7 +124,7 @@ def import_packet(self, raw, source, expected_checkpoint, key): """Called only after SourceAdapter's explicit current rooted capture.""" with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'measurements.import') - self.session.check() + self.session.check_current() table = MeasurementTable(raw) blob = dict(sha256=table.sha256, bytes=len(raw)) origin = self.session.summary['project_id'] @@ -152,11 +152,11 @@ def sources(self, *, offset=0, limit=40): raise WorkbenchError('Invalid measurement source page.') with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', 'measurements.sources') - self.session.check() + self.session.check_current() docs = sorted(self._documents().values(), key=lambda d: (d['origin_catalog_id'], d['id'])) items = [dict(doc, availability=self.artifacts.inspect(declaration(doc['blob']))) for doc in docs[offset:offset+limit]] - self.session.check() + self.session.check_current() self.access.require('inspect', 'measurements.sources') return self._bounded(dict(items=items, total=len(docs), offset=offset, limit=limit)) @@ -172,7 +172,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): target = origin_catalog_id + '.' + packet_sha256 with self.access.lock, self.session.lock, self.db.lock: self.access.require('inspect', action, target) - self.session.check() + self.session.check_current() doc = self._documents().get(target) if doc is None: raise WorkbenchError('Measurement source is unavailable.', 'not_found', 404) @@ -181,7 +181,7 @@ def _read(self, origin_catalog_id, packet_sha256, action, select): if table.metadata()['title'] != doc['title']: raise WorkbenchError('Measurement title differs from the exact retained packet.') value = self._bounded(dict(source=doc, value=select(table))) - self.session.check() + self.session.check_current() self.access.require('inspect', action, target) return value @@ -215,7 +215,7 @@ def compare(self, sources, metric_id=None, **selection): with self.access.lock, self.session.lock, self.db.lock: for target in targets: self.access.require('inspect', 'measurements.compare', target) - self.session.check() + self.session.check_current() documents = self._documents() if any(target not in documents for target in targets): raise WorkbenchError('A comparison source is unavailable.', 'not_found', 404) @@ -231,7 +231,7 @@ def compare(self, sources, metric_id=None, **selection): tables.append((locator, table)) value = MeasurementComparison(tables).read(metric_id, **selection) result = self._bounded(dict(sources=chosen, value=value)) - self.session.check() + self.session.check_current() for target in targets: self.access.require('inspect', 'measurements.compare', target) return result diff --git a/project_native_execution.py b/project_native_execution.py index 9ac6be4a..5895e4d7 100644 --- a/project_native_execution.py +++ b/project_native_execution.py @@ -452,7 +452,7 @@ def identity(info): def _publish(self, document, state, *, helper=None, receipt_sha256=None): with self.db.lock,self.db.transaction(): - self.session.check();actor=self._authorize(document['phase_id']) + self.session.check_current();actor=self._authorize(document['phase_id']) document['events'].append(dict(state=state,at=timestamp(),actor_id=actor.actor_id, helper=copy.deepcopy(helper),receipt_sha256=receipt_sha256)) self._dirty[document['phase_id']]=copy.deepcopy(document) @@ -788,7 +788,7 @@ def _probe_result(recipe, inputs, evidence): def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, run_id=None, job=None, phase_id=None): if self.db.in_transaction:_fail('Native phase cannot launch inside a metadata transaction.','busy') with self.db.lock: - self.session.check() + self.session.check_current() if run_id is None and self._foreground_phase is not None: _fail('Another explicit native preparation phase is active.','busy') if not self.db.claim_runtime('project_native_phases'):_fail('Another current native phase owner is active.','busy') @@ -818,7 +818,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru self._publish(document,'prepared') if self.execution.custody is None:_fail('Native launch requires its original physical custody journal.','unavailable') with self.db.lock,self.execution._journal_guard(): - self._dispatch_authorized(phase_id);self.session.check() + self._dispatch_authorized(phase_id);self.session.check_current() if not self.execution.custody.has_pending:self.execution.custody.acknowledge(self.session.snapshot) self.execution.custody.native_prepared(document) self.execution.custody.native_launch_intent(phase_id) @@ -847,7 +847,7 @@ def _run_phase(self, phase, recipe, request_key, actor, inputs, authority, *, ru # Recheck actual authority after both durable ACKs. A # saved GO intent is never permission to retry this send. with self.db.lock: - self.session.check();self._dispatch_authorized(phase_id) + self.session.check_current();self._dispatch_authorized(phase_id) if not self.execution.custody.has_pending:_fail('Native local custody disappeared.','unknown') prepared.check_current_runtime() handle.go(handle.ready) diff --git a/project_native_preparation.py b/project_native_preparation.py index 7c504be0..24113f63 100644 --- a/project_native_preparation.py +++ b/project_native_preparation.py @@ -188,7 +188,7 @@ def validate_operation_result(operation,result,records=None): _fail('Native intent source or frozen input differs from its graph.','invalid') from types import SimpleNamespace from project_material import ProjectMaterial - retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check=lambda:None)) + retained=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None)) if result['model'] is not None:retained.model(result['model']) if result['composition'] is not None: composition=retained.recipe(result['composition'])['material'] @@ -230,7 +230,7 @@ def __init__(self,adapter,owner,graph,model,composition,purpose,key): def _check(self,recipe): if not self._active or _CURRENT.get() is not self or recipe!=self.graph['recipe']: _fail('An explicit current native preparation is required.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.probe_native',recipe,owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Native preparation caller changed.','conflict') if (os.environ.get('TULDOK_TRAINING_PYTHON') or sys.executable)!=self.configured_executable: @@ -297,7 +297,7 @@ def __init__(self,adapter,owner,graph): def consume_admission(self): if not self._active or self._consumed:_fail('Current integrity qualification has already been consumed.') - self.adapter.session.check() + self.adapter.session.check_current() actor=self.adapter.access.require('execute','pipeline.snapshot',self.graph['id'],owner_only=True) if actor.actor_id!=self.actor.actor_id:_fail('Current integrity caller changed.','conflict') if self.owner.get(self.graph['id'])!=self.graph or self.owner.definition_identity(self.recipe)!=self.definition: @@ -333,7 +333,7 @@ def _scope(self,owner,graph,model,composition,purpose,expected_checkpoint,key): if self.db.in_transaction:_fail('Native preparation must precede the metadata transaction.','unsupported') if _CURRENT.get() is not None:_fail('A native preparation is already active.','busy') self.access.require('edit','native_preparation.'+purpose,graph['id'],owner_only=True) - self.session.check() + self.session.check_current() if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') context=NativePreparationContext(self,owner,graph,model,composition,purpose,_key(purpose,key)) result=dict(format='project_native_preparation_intent_v1',origin_catalog_id=self.session.project_id, @@ -361,7 +361,7 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): if graph['revision']!=revision:_fail('Graph changed before native validation.','stale') import release_validation if adapter_for_graph(graph) is release_validation: - self.session.check() + self.session.check_current() if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') value=owner.validate_graph(graph) return self.session.mutate('native_preparation.validate',graph_id,body,expected_checkpoint,key, @@ -375,7 +375,7 @@ def validate_graph(self,owner,graph_id,revision,expected_checkpoint,key): def current_graph(self,owner,graph_id,revision,expected_checkpoint,key): if self.db.in_transaction:_fail('Current native validation must precede admission.','unsupported') self.access.require('edit','native_preparation.admission',graph_id,owner_only=True) - self.session.check() + self.session.check_current() if self.session.summary['checkpoint']!=expected_checkpoint:_fail('Expected project checkpoint changed.','stale_project') graph=owner.get(graph_id) if graph['revision']!=revision:_fail('Graph changed before native admission.','stale') diff --git a/project_producer_execution.py b/project_producer_execution.py index f4ae5915..e3729233 100644 --- a/project_producer_execution.py +++ b/project_producer_execution.py @@ -73,7 +73,7 @@ def _family(self, family): return family def _owner(self, family): - self._family(family);self.session.check() + self._family(family);self.session.check_current() if self._fenced: raise WorkbenchError('Current producer custody is uncertain. Reopen for inert inspection.', 'outcome_unknown', 409) self._refuse_unknown_provider() @@ -429,7 +429,7 @@ def status(self, *, offset=0, limit=40): raise WorkbenchError('Invalid current producer health page.') with self.session.access.lock,self.session.lock,self.db.lock: self.session.access.require('inspect','producers.health') - self.session.check() + self.session.check_current() self._settle_drained() items=[] for family,owner in self._owners.items(): diff --git a/project_producers.py b/project_producers.py index 10eb94be..8e052b34 100644 --- a/project_producers.py +++ b/project_producers.py @@ -592,9 +592,9 @@ def __init__(self, session): def _read(self, action, invoke): with self.session.access.lock, self.session.lock, self.session.db.lock: self.session.access.require('inspect', action) - self.session.check() + self.session.check_current() result = invoke(self.session.snapshot['records']) - self.session.check() + self.session.check_current() return result def list(self, family, *, origin_catalog_id=None, offset=0, limit=40): diff --git a/project_projection.py b/project_projection.py index 04aaebba..1bf45bd2 100644 --- a/project_projection.py +++ b/project_projection.py @@ -190,7 +190,7 @@ def resolve_artifact(self, body, expected_checkpoint, key): raise WorkbenchError('Choose one declared artifact and a current rooted file.') with self.access.lock, self.session.lock, self.db.lock: actor = self.access.require('edit', 'artifacts.resolve', body['sha256'], owner_only=True) - self.session.check() + self.session.check_current() declaration = next((item['document'] for item in self.session.snapshot['records'] if item['family'] == 'project_artifact' and item['id'] == body['sha256']), None) if declaration is None: @@ -246,7 +246,7 @@ def runtime_configuration(self, request): def runtime_sources(self): with self.access.lock, self.session.lock, self.db.lock: self.access.require('admin', 'runtime.sources', owner_only=True) - self.session.check() + self.session.check_current() return dict(sources=sorted(self._runtime_sources)) def authorize_disclosure(self, run, population): @@ -317,7 +317,7 @@ def _tag(self, value): return dict(value, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint']) def records(self): - self.projection.session.check() + self.projection.session.check_current() from project_fork import active_records snapshot = self.projection.session.snapshot records = active_records(snapshot['records'], project_id=snapshot['project_id']) diff --git a/project_runtime.py b/project_runtime.py index 21fbd63f..d63a9eb5 100644 --- a/project_runtime.py +++ b/project_runtime.py @@ -156,7 +156,7 @@ def _authorize(self, action): def inspect(self, config_request): """Read an explicitly selected current source; never provision or execute.""" with self.access.lock, self.session.lock, self.db.lock: - self._authorize('runtime_snapshots.inspect'); self.session.check() + self._authorize('runtime_snapshots.inspect'); self.session.check_current() _body(config_request, '0'*64, 0) with self.execution.runtime_inspection(): return self.owner.inspect(self.resolve_configuration(copy.deepcopy(config_request))) @@ -165,7 +165,7 @@ def status(self, key): """Read historical outcome only; captured does not claim local readiness.""" _key(key) with self.access.lock, self.session.lock, self.db.lock: - self.access.require('inspect', 'runtime.capture_status'); self.session.check() + self.access.require('inspect', 'runtime.capture_status'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) intent = receipts.get(_intent_key(key)); complete = receipts.get(key) if intent is not None and intent['operation'] != 'runtime.capture_intent': intent = None @@ -248,7 +248,7 @@ def capture(self, config_request, expected_inventory_sha256, expected_checkpoint body_sha = _intent_input(body) intent = None with self.access.lock, self.session.lock, self.db.lock: - actor = self._authorize('runtime_snapshots.create'); self.session.check() + actor = self._authorize('runtime_snapshots.create'); self.session.check_current() receipts = operation_records(self.session.snapshot['records']) if any(r['operation']=='runtime.abandon' and r['result']['capture_key']==key for r in receipts.values()): _fail('This capture was abandoned; use a fresh key for a new measurement.', 'conflict', 409) diff --git a/project_session.py b/project_session.py index 308164c5..5fa571d1 100644 --- a/project_session.py +++ b/project_session.py @@ -206,15 +206,26 @@ def adopt(self, snapshot): self._snapshot = copy.deepcopy(snapshot) self.available = True - def check(self): - """Check freshness while a caller holds host/session/projection guards.""" + def _checked(self, read): if not self.available: raise ProjectSessionError('Reopen this project to reconstruct its projection.', 'projection_unavailable', 503) - current = self.store.snapshot() + current = read() if current['checkpoint'] != self._snapshot['checkpoint']: raise ProjectSessionError('Canonical project changed. Close and reopen it.', 'stale_project', 409) return current + def check_current(self): + """Verify actual canonical bytes without returning an unused record graph. + + The caller holds the same host/session/projection guards as check(). + Store.summary reads and hashes all current records before returning. + """ + self._checked(self.store.summary) + + def check(self): + """Return independent current records under caller-owned projection guards.""" + return self._checked(self.store.snapshot) + def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capture, *, scope='edit'): """Invoke existing policy, capture fixed-owner upserts, publish, commit SQL. diff --git a/project_sources.py b/project_sources.py index 25007ea1..95e7c5f2 100644 --- a/project_sources.py +++ b/project_sources.py @@ -117,7 +117,7 @@ def register(self,body,expected_checkpoint,key): self._mutation_identity(expected_checkpoint,key) with self.access.lock,self.session.lock,self.db.lock: actor=self.access.require('edit','source.register',owner_only=True) - self.session.check() + self.session.check_current() saved,raw=self._read_registration(body) origin=self.session.summary['project_id'];invoked=False def invoke(): @@ -153,7 +153,7 @@ def references(self,snapshotref,offset=0,limit=40): raise WorkbenchError('Invalid bounded source reference page.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('inspect','source.references',snapshot['project_id']) - self.session.check() + self.session.check_current() historical=self.materials.snapshot_ref(snapshot['project_id'],snapshot['manifest_sha256'], catalog_id=snapshot['origin_catalog_id']) try:current=self.sources._saved(snapshot['project_id'],extensions=True) @@ -171,7 +171,7 @@ def references(self,snapshotref,offset=0,limit=40): selected=current['manifest']['data'][offset:offset+limit];budget=[INSPECTION_LIMIT] items=[self.sources._inspect(current['root'],copy.deepcopy(ref),budget,DATA_LIMIT) for ref in selected] self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) - self.session.check();self.access.require('inspect','source.references',snapshot['project_id']) + self.session.check_current();self.access.require('inspect','source.references',snapshot['project_id']) return dict(items=items,total=len(current['manifest']['data']),offset=offset,limit=limit) def import_scientific(self,body,expected_checkpoint,key): @@ -183,7 +183,7 @@ def import_scientific(self,body,expected_checkpoint,key): if type(body['document']) is not dict:raise WorkbenchError('Scientific import metadata must be an object.') with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_scientific',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],DATA_LIMIT) # ProjectData owns the one canonical mutation and native parser; # current rooted reads above never become an imported capability. @@ -204,7 +204,7 @@ def _capture_reference(self,snapshot,reference_id,expected_sha256,maximum_bytes) expected_sha256,maximum_bytes) if source['manifest_sha256']!=snapshot['manifest_sha256']: raise WorkbenchError('Current source snapshot changed during capture.','stale',409) - self.session.check() + self.session.check_current() return raw,source def import_measurements(self,body,expected_checkpoint,key): @@ -216,7 +216,7 @@ def import_measurements(self,body,expected_checkpoint,key): from retained_measurements import MAX_BYTES with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_measurements',snapshot['project_id']) - self.session.check() + self.session.check_current() raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],MAX_BYTES) source=dict(source,origin_catalog_id=snapshot['origin_catalog_id']) self.access.require('edit','source.import_measurements',snapshot['project_id']) diff --git a/project_study_execution.py b/project_study_execution.py index fa6d6ce9..2133bbce 100644 --- a/project_study_execution.py +++ b/project_study_execution.py @@ -142,7 +142,7 @@ def _validate_preparation(operation,result,records): trial=result['trial'];origin=trial['material']['catalog_id'] candidate=facts.candidates[(origin,result['candidate_id'])]['candidate'] if candidate['study_id']!=trial['study_id']:_invalid('Preparation candidate belongs to another Study.') - materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check=lambda:None)) + materials=ProjectMaterial(SimpleNamespace(snapshot={'records':records},check_current=lambda:None)) composition=materials.recipe(trial['material'])['material'] if (composition['model']!=result['model'] or composition['recipe_id']!=result['graph']['id'] or composition['recipe_revision']!=result['graph']['revision']): @@ -202,7 +202,7 @@ def validate_operation_result(operation,result,records=None): if saved is None or any(saved[key]!=reservation[key] for key in RESERVATION_FIELDS if key not in ('state','settled')): _invalid('Admission receipt has no exact canonical reservation.') trial=facts.trials[(origin,result['trial_id'])]['trial'];run=facts.runs[(origin,result['run_id'])]['run'] - view=SimpleNamespace(snapshot={'records':records},check=lambda:None) + view=SimpleNamespace(snapshot={'records':records},check_current=lambda:None) validate_retained_reservation(reservation,trial=trial, composition=ProjectMaterial(view).recipe(trial['material']),run=run) elif operation=='study_execution.stop': @@ -329,7 +329,7 @@ def _canonical(self,records=None): def restore_authoring(self,records=None): """Reconstruct intent only, under the current project guard.""" with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check();facts=self._canonical(records) + self.session.check_current();facts=self._canonical(records) with self.db.transaction(): for (origin,identity),doc in sorted(facts.studies.items()): if origin!=self.origin:continue @@ -589,7 +589,7 @@ def admit(self,study_id,body,expected_checkpoint,key): authority=self.access.require_external_submission() with self.session.access.lock,self.session.lock,self.db.lock: builtin=None - self.session.check();self.refresh_authoring() + self.session.check_current();self.refresh_authoring() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: result=prior['result'] @@ -734,7 +734,7 @@ def _prepare_candidate(self,study_id,body,expected_checkpoint,key,kind): input_sha=digest(dict(study_id=study_id,body=effective)) intent_key=_intent_key(operation,key);phase=intent_key.removesuffix('-intent') with self.session.access.lock,self.session.lock,self.db.lock: - self.session.check() + self.session.check_current() prior=operation_records(self.session.snapshot['records']).get(key) if prior is not None: # Current authority is checked above. Historical completion diff --git a/project_transfer_controller.py b/project_transfer_controller.py index 7362e0ea..3ece224e 100644 --- a/project_transfer_controller.py +++ b/project_transfer_controller.py @@ -252,7 +252,7 @@ def extract(self, binding, report_sha256, destination_root, directory, name, req fail('Recover the selected workspace before reconciling extraction.', 'projection_unavailable', 503) guards.enter_context(projection.session.lock) guards.enter_context(projection.db.lock) - projection.session.check() + projection.session.check_current() opened = self.workspace._store receipt = self._transfer.extract(self._selection, report_sha256, destination_fd, directory, name, request_key, opened_store=opened) except BaseException as error: diff --git a/tests/test_project_extraction.py b/tests/test_project_extraction.py index 4ef0ff92..afa628a6 100644 --- a/tests/test_project_extraction.py +++ b/tests/test_project_extraction.py @@ -498,7 +498,7 @@ def test_two_study_lineage_reservation_candidate_and_run_science_are_captured(se before = self.files();source = self.view() captured = self.capture(source, {'studies': [child['id']]}) self.assertEqual(captured['blockers'], [], captured['blockers']) - view = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check=lambda: None) + view = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check_current=lambda: None) from project_material import ProjectMaterial facts = decode_history(captured['records'], materials=ProjectMaterial(view)) self.assertEqual(set(facts.studies), {(self.db.dataset_id, parent['id']), (self.db.dataset_id, child['id'])}) @@ -639,7 +639,7 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure captured = self.capture(source, {'runs': [consumer['id']]}) self.assertEqual(captured['blockers'], [], captured['blockers']) facts = decode_records(captured['records'], materials=ProjectMaterial(SimpleNamespace( - snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check=lambda: None))) + snapshot=dict(project_id=self.db.dataset_id, records=captured['records']), check_current=lambda: None))) self.assertEqual(set(facts.runs), {(self.db.dataset_id, run['id']) for run in (producer, middle, consumer)}) for downstream, upstream in ((consumer, middle), (middle, producer)): self.assertIn(dict(source=dict(family='run_history', id=self.db.dataset_id + '.' + downstream['id']), @@ -650,7 +650,7 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure # A fresh inert owner reader reconstructs the exact upstream custody # receipt from retained facts/private captured bytes, never source paths. session = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=copy.deepcopy(captured['records'])), - access=self.access, artifacts=source.artifacts, check=lambda: None) + access=self.access, artifacts=source.artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda run, population: self.access.require('expose-holdout', 'runs.expose_test', run['id'])) with patch.object(source, 'read_catalog', side_effect=AssertionError('No original catalog reread')), \ diff --git a/tests/test_project_files.py b/tests/test_project_files.py index 538501d0..49af9af2 100644 --- a/tests/test_project_files.py +++ b/tests/test_project_files.py @@ -44,6 +44,16 @@ def intent(self,path,text,revision=0,kind='note'): raw=text.encode('utf-8');target=self.root/path;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(raw) return dict(kind=kind,path=path,expected_revision=revision,expected_sha256=hashlib.sha256(raw).hexdigest()) + def test_inspection_copies_only_the_records_it_actually_consumes(self): + self.adopt('notes/inspection.md', 'Exact retained scientific note', kind='note') + expected = self.files.inspect() + with patch.object(self.session.store, '_snapshot', + wraps=self.session.store._snapshot) as snapshots: + observed = self.files.inspect() + self.assertEqual(observed, expected) + self.assertEqual(snapshots.call_count, 1, + 'Inspection needs one record snapshot; final freshness must not clone another') + def test_batch_captures_exact_files_in_one_canonical_mutation(self): items=[self.intent('notes/a.md','Exact Ω\r\n'),self.intent('notes/empty.md','')] before=self.session.snapshot['checkpoint'] diff --git a/tests/test_project_fork.py b/tests/test_project_fork.py index 1b4029b8..0683eef9 100644 --- a/tests/test_project_fork.py +++ b/tests/test_project_fork.py @@ -53,7 +53,7 @@ def setUp(self): capture.mkdir(parents=True) for name, raw in payloads.items(): (capture / name).write_bytes(raw) - session = SimpleNamespace(snapshot=self.source.snapshot(), artifacts=self.artifacts, check=lambda: None) + session = SimpleNamespace(snapshot=self.source.snapshot(), artifacts=self.artifacts, check_current=lambda: None) self.material = ProjectMaterial(session) self.record = self.material.capture_model_material(self.material_value, local) records = [*self.material.declarations(self.record), self.record] @@ -90,7 +90,7 @@ def test_complete_fork_retains_exact_original_material_hashes_and_copies_without self.assertEqual(overlay['source'], dict(project_id=self.origin, checkpoint=self.original['checkpoint'])) self.assertEqual(next(item for item in snapshot['records'] if item['family'] == 'model_material'), self.record) with ProjectStore.open(self.root / 'fork') as destination, ProjectArtifacts(destination) as artifacts: - reader = ProjectMaterial(SimpleNamespace(snapshot=destination.snapshot(), artifacts=artifacts, check=lambda: None)) + reader = ProjectMaterial(SimpleNamespace(snapshot=destination.snapshot(), artifacts=artifacts, check_current=lambda: None)) self.assertEqual(reader.model(self.material_value['ref'], verify=True), self.material_value) source_ref = self.material_value['material']['source_closure'][0]['blob'] self.assertEqual(reader.blob(self.material_value['ref'], source_ref), SOURCE) diff --git a/tests/test_project_fork_composition.py b/tests/test_project_fork_composition.py index b5fcf831..28e48361 100644 --- a/tests/test_project_fork_composition.py +++ b/tests/test_project_fork_composition.py @@ -232,7 +232,7 @@ def setUp(self): def validate(self,records): decode_data_records(records);material_records(records);file_records(records);producer_records(records) - history_records(records,materials=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check=lambda:None))) + history_records(records,materials=ProjectMaterial(SimpleNamespace(snapshot=dict(records=records),check_current=lambda:None))) for receipt in operation_documents(records): operation,result=receipt['operation'],receipt['result'] if not (data_result(operation,result,records) or file_result(operation,result,records) diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 700504cd..89e4ecf2 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -137,7 +137,7 @@ def read_view(self, selected): def recipe(self, *args, **kwargs): raise AssertionError('History reread the live material catalog') session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), - artifacts=None,access=SimpleNamespace(),check=lambda:None) + artifacts=None,access=SimpleNamespace(),check_current=lambda:None) facts=ProjectHistory(session,materials=Provider())._facts() self.assertEqual(len(observed),1) self.assertIs(observed[0],records) @@ -653,7 +653,7 @@ def test_cold_actual_composed_proof_keeps_origin_without_live_authority(self): root=self.path/'portable-history';store=ProjectStore.create(root,'Actual cold history','new','f'*64) self.addCleanup(store.close) artifacts=ProjectArtifacts(store);self.addCleanup(artifacts.close) - view=SimpleNamespace(snapshot={'project_id':store.snapshot()['project_id'],'records':[]},artifacts=artifacts,check=lambda:None) + view=SimpleNamespace(snapshot={'project_id':store.snapshot()['project_id'],'records':[]},artifacts=artifacts,check_current=lambda:None) material=ProjectMaterial(view) model_record=material.capture_model_material(model,self.models.path) recipe_record=material.capture_recipe_material(composition,self.dataset.pipelines.material_path) diff --git a/tests/test_project_material.py b/tests/test_project_material.py index 5e412305..9d69056c 100644 --- a/tests/test_project_material.py +++ b/tests/test_project_material.py @@ -56,7 +56,7 @@ def setUp(self): self.artifacts=ProjectArtifacts(self.store);self.addCleanup(lambda:self.artifacts.close()) self.catalog=self.store.snapshot()['project_id'] self.session=SimpleNamespace(snapshot=self.store.snapshot(),summary=self.store.summary(), - artifacts=self.artifacts,check=lambda:None) + artifacts=self.artifacts,check_current=lambda:None) self.material=ProjectMaterial(self.session) def capture(self): @@ -88,7 +88,7 @@ def test_cold_relocation_uses_canonical_artifacts_after_local_capture_deleted(se self.artifacts.close();self.store.close();shutil.rmtree(root) moved=self.root/'moved';(self.root/'project').rename(moved) with ProjectStore.open(moved) as store,ProjectArtifacts(store) as artifacts: - material=ProjectMaterial(SimpleNamespace(snapshot=store.snapshot(),artifacts=artifacts,check=lambda:None)) + material=ProjectMaterial(SimpleNamespace(snapshot=store.snapshot(),artifacts=artifacts,check_current=lambda:None)) self.assertEqual(store.snapshot(),snapshot) self.assertEqual(material.model(value['ref'],verify=True),value) source=value['material']['source_closure'][0]['blob'] @@ -145,7 +145,7 @@ def test_fixed_material_snapshot_checks_complete_reference_identity(self): with self.assertRaises(WorkbenchError):view.runtime(dict(ref,**change)) def test_invalid_live_material_reference_precedes_snapshot_access(self): - with patch.object(self.session,'check',side_effect=AssertionError('Invalid reference accessed snapshot')): + with patch.object(self.session,'check_current',side_effect=AssertionError('Invalid reference accessed snapshot')): with self.assertRaises(WorkbenchError):self.material.runtime({'catalog_id':'invalid'}) def test_material_read_view_uses_supplied_records_without_resnapshotting(self): @@ -159,7 +159,7 @@ def test_material_read_view_uses_supplied_records_without_resnapshotting(self): self.assertEqual(self.session.snapshot['records'],[]) def test_material_read_view_rechecks_session_availability(self): - with patch.object(self.session,'check',side_effect=WorkbenchError('Stale checkpoint.','stale',409)): + with patch.object(self.session,'check_current',side_effect=WorkbenchError('Stale checkpoint.','stale',409)): with self.assertRaisesRegex(WorkbenchError,'Stale checkpoint'): self.material.read_view([]) @@ -361,7 +361,7 @@ def test_retained_one_fit_readers_match_directory_without_source_paths_or_infere patch('subprocess.Popen',side_effect=AssertionError('No process')): expected=evidence.verify(root,run,metrics) self.assertEqual(evidence.verify_blobs(run,metrics,read_blob=lambda name,maximum:payloads[name]),expected) - material=ProjectMaterial(SimpleNamespace(check=lambda:None)) + material=ProjectMaterial(SimpleNamespace(check_current=lambda:None)) result=material.evidence(run,read_blob=lambda name,maximum:payloads[name]) self.assertEqual(result['state'],'verified') checkpoint=result['checkpoint_index']['entries'][0]['checkpoint']['sha256'] @@ -450,7 +450,7 @@ def test_real_deferred_sql_failure_cannot_delete_canonical_source_material(self) self.assertEqual(error.exception.code,'projection_unavailable') snapshot=self.store.snapshot();retained=next(item for item in snapshot['records'] if item['family']=='model_material')['document']['value'] self.assertFalse((self.models.path/retained['ref']['sha256']).exists(),'Legacy rollback removes only disposable projection cache') - cold=ProjectMaterial(SimpleNamespace(snapshot=snapshot,artifacts=self.artifacts,check=lambda:None)) + cold=ProjectMaterial(SimpleNamespace(snapshot=snapshot,artifacts=self.artifacts,check_current=lambda:None)) self.assertEqual(cold.model(retained['ref'],verify=True),retained) self.assertEqual(cold.blob(retained['ref'],retained['material']['source_closure'][0]['blob']),SOURCE) @@ -461,7 +461,7 @@ def test_restore_is_inert_and_explicit_preparation_rebuilds_only_projection_cach draft=self.create();value=self.material.publish_model(self.models,draft['id'],{'revision':1},self.session.snapshot['checkpoint'],'cold-model') cold_db=Metadata(self.path/'cold-projection',dataset_id=self.db.dataset_id,projection_id=uuid.uuid4().hex) self.resources.append(cold_db);self.schemas.append(cold_db.schema) - cold=ProjectMaterial(SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,db=cold_db,check=lambda:None)) + cold=ProjectMaterial(SimpleNamespace(snapshot=self.store.snapshot(),artifacts=self.artifacts,db=cold_db,check_current=lambda:None)) with patch('model_definitions.ModelDefinitions',side_effect=AssertionError('No mutable owner on Open')), \ patch('pipeline.Pipelines',side_effect=AssertionError('No mutable owner on Open')), \ patch('subprocess.Popen',side_effect=AssertionError('No process')): diff --git a/tests/test_project_material_fork.py b/tests/test_project_material_fork.py index d28881ac..fb27a597 100644 --- a/tests/test_project_material_fork.py +++ b/tests/test_project_material_fork.py @@ -53,7 +53,7 @@ class MaterialForkCodecs(unittest.TestCase): def setUp(self): self.source=uuid.uuid4().hex;self.checkpoint='a'*64 self.session=SimpleNamespace(snapshot=dict(project_id=self.source,checkpoint=self.checkpoint,records=[]), - summary=dict(project_id=self.source,checkpoint=self.checkpoint),check=lambda:None) + summary=dict(project_id=self.source,checkpoint=self.checkpoint),check_current=lambda:None) self.material=ProjectMaterial(self.session) def capture(self,family,current): @@ -110,7 +110,7 @@ def test_bound_drafts_preserve_full_configuration_as_visible_nonexecutable_inten captured=[self.capture(f,v) for f,v in drafts] made,combined,destination=self.fork() session=SimpleNamespace(snapshot=dict(project_id=destination,records=combined), - summary=dict(project_id=destination),check=lambda:None) + summary=dict(project_id=destination),check_current=lambda:None) reader=ProjectMaterial(session) intents=reader.intents() self.assertEqual(len(intents),4) @@ -170,7 +170,7 @@ def test_captured_source_selector_stays_in_inert_intent_and_cannot_publish(self) self.assertEqual(doc['current']['required_reselection'],['source_selector']) self.assertEqual(doc['current']['definition']['source_selector'],value['source_selector']) reader=ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=destination,records=combined), - summary=dict(project_id=destination),check=lambda:None)) + summary=dict(project_id=destination),check_current=lambda:None)) with self.assertRaises(WorkbenchError) as caught: reader.publish_model(SimpleNamespace(publish=lambda *args: self.fail('Intent reached owner publication')), doc['current']['id'],{'revision':1},'a'*64,'forbidden-intent') diff --git a/tests/test_project_native_extraction.py b/tests/test_project_native_extraction.py index c2c3372b..fe1d34b3 100644 --- a/tests/test_project_native_extraction.py +++ b/tests/test_project_native_extraction.py @@ -86,7 +86,7 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact captured = self.capture_without_host(run, source) self.assertEqual(captured['blockers'], [], captured['blockers']) materials = ProjectMaterial(SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, - records=captured['records']), check=lambda: None)) + records=captured['records']), check_current=lambda: None)) facts = decode_records(captured['records'], materials=materials) saved = facts.runs[(self.db.dataset_id, run['id'])] self.assertEqual(saved['run']['runtime'], run['runtime']) @@ -115,7 +115,7 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact for name in originals: (stage / name).unlink() cold = ProjectStore.open(destination); self.addCleanup(cold.close) cold_artifacts = ProjectArtifacts(cold); self.addCleanup(cold_artifacts.close) - session = SimpleNamespace(snapshot=cold.snapshot(), access=self.access, artifacts=cold_artifacts, check=lambda: None) + session = SimpleNamespace(snapshot=cold.snapshot(), access=self.access, artifacts=cold_artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda *args: None) for name, raw in originals.items(): self.assertEqual(history.artifact(run['id'], 'source/' + name, origin_catalog_id=self.db.dataset_id), raw) diff --git a/tests/test_project_session_freshness.py b/tests/test_project_session_freshness.py new file mode 100644 index 00000000..16996739 --- /dev/null +++ b/tests/test_project_session_freshness.py @@ -0,0 +1,88 @@ +"""Freshness enforcement over real canonical files, without a SQL projection. + +Only the session's read guard is exercised here. Publication and projection +binding still require the separate real-PostgreSQL session/workspace tests. +""" +import copy +import hashlib +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from project_session import ProjectSession, ProjectSessionError +from project_store import ProjectStore, ProjectStoreError + + +class ProjectSessionFreshnessTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-freshness-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) / 'project' + self.store = ProjectStore.create(self.root, 'Freshness fixture', 'new', 'a' * 64) + self.addCleanup(self.store.close) + self.records = [dict(family='workbench_text', id=name, + document=dict(format='workbench_text_v1', text='Alpha ' + name)) for name in ('one', 'two')] + snapshot = self.store.commit(self.records, self.store.summary()['checkpoint'], 'seed', 'b' * 64) + # Isolate the existing read guard; no fake SQL/projection acceptance. + self.session = object.__new__(ProjectSession) + self.session.store = self.store + self.session._snapshot = copy.deepcopy(snapshot) + self.session.available = True + + def test_freshness_reads_all_records_without_copying_an_unused_graph(self): + before = self.store.snapshot() + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unused full snapshot')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads: + self.assertIsNone(self.session.check_current()) + self.assertIsNone(self.session.check_current()) + self.assertEqual(reads.call_count, 4) + self.assertEqual(self.store.snapshot(), before) + + def test_changed_canonical_checkpoint_refuses_both_read_contracts(self): + changed = copy.deepcopy(self.records) + changed[0]['document']['text'] = 'Independently published content' + self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c' * 64) + for check in (self.session.check_current, self.session.check): + with self.assertRaises(ProjectSessionError) as caught: + check() + self.assertEqual(caught.exception.code, 'stale_project') + + def test_unavailable_projection_refuses_before_reading_the_store(self): + self.session.available = False + with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')), \ + patch.object(self.store, 'snapshot', side_effect=AssertionError('Unavailable snapshot read')): + for check in (self.session.check_current, self.session.check): + with self.assertRaises(ProjectSessionError) as caught: + check() + self.assertEqual(caught.exception.code, 'projection_unavailable') + + def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self): + self.session.check_current() + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + path = self.root / '.tuldok' / manifest['inventory'][0]['path'] + info, original = path.stat(), path.read_bytes() + changed = original.replace(b'Alpha', b'Omega') + self.assertEqual(len(original), len(changed)) + self.assertNotEqual(hashlib.sha256(original).digest(), hashlib.sha256(changed).digest()) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + for check in (self.session.check_current, self.session.check): + with self.assertRaises(ProjectStoreError) as caught: + check() + self.assertEqual(caught.exception.code, 'stale') + + def test_record_consumers_keep_independent_complete_snapshots(self): + current = self.session.check() + current['records'][0]['document']['text'] = 'Caller mutation' + current['name'] = 'Caller renamed copy' + self.assertEqual(self.session.check(), self.store.snapshot()) + self.assertEqual(self.session._snapshot, self.store.snapshot()) + self.assertNotEqual(current, self.session.check()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_project_transfer.py b/tests/test_project_transfer.py index efb639b4..5b798960 100644 --- a/tests/test_project_transfer.py +++ b/tests/test_project_transfer.py @@ -433,7 +433,7 @@ def capture(_,roots): from project_material import ProjectMaterial with ProjectStore.open(self.root/'science') as store,ProjectArtifacts(store) as artifacts: snapshot=store.snapshot();records=snapshot['records'] - view=SimpleNamespace(snapshot=snapshot,artifacts=artifacts,check=lambda:None) + view=SimpleNamespace(snapshot=snapshot,artifacts=artifacts,check_current=lambda:None) facts=history(records,materials=ProjectMaterial(view),artifacts=artifacts) self.assertEqual(len(facts.studies),2);self.assertEqual(len(facts.trials),2) self.assertEqual(len(facts.candidates),1);self.assertEqual(len(facts.reservations),1);self.assertEqual(len(facts.runs),1) From 6713ace099d380ab5c8dd6e4c768e460d5b12b8a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 01:05:24 -0700 Subject: [PATCH 28/33] perf(projects): avoid full history copies for working file reads --- project_dispatch.py | 18 +++++---- project_files.py | 19 +++++---- project_session.py | 16 ++++++++ tests/test_project_dispatch.py | 2 +- tests/test_project_files.py | 6 +-- tests/test_project_files_m4_clients.py | 14 ++++++- tests/test_project_files_m4_http.py | 19 +++++++++ tests/test_project_session_freshness.py | 54 +++++++++++++++++++++++-- 8 files changed, 125 insertions(+), 23 deletions(-) diff --git a/project_dispatch.py b/project_dispatch.py index bd7929d0..de519135 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -370,7 +370,16 @@ def _m4(projection,action,a): if action=='file_adopt_deletion': return projection.files.adopt_deletion(_body(a),a['expected_checkpoint'],a['request_key']) with session.access.lock,session.lock,session.db.lock: - session.access.require('inspect','project_files.'+action);snapshot=session.check() + session.access.require('inspect','project_files.'+action) + if action=='fork_info': + snapshot=session.check() + from project_fork import decode_fork + overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) + return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], + mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, + copied_count=len(overlay['artifacts']['copied']) if overlay else 0, + missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) + session.check_current() if action=='files_inspect':return _page(projection.files.inspect()['items'],a) if action=='file_get':return projection.files.read(a['file_id'],a.get('revision')) if action=='file_history': @@ -390,13 +399,6 @@ def _m4(projection,action,a): items=[dict(item,requires_reselection=[]) for item in value['editable']] items.extend(dict(family=item['family'],id=item['id'],requires_reselection=item['fields']) for item in value['requires_reselection']) return _page(sorted(items,key=lambda item:(item['family'],item['id'])),a) - if action=='fork_info': - from project_fork import decode_fork - overlay=decode_fork(snapshot['records'],project_id=snapshot['project_id']) - return dict(source=overlay['source'] if overlay else None,project_id=snapshot['project_id'], - mapping=_page(overlay['mapping'] if overlay else [],a),archived_count=len(overlay['archived']) if overlay else 0, - copied_count=len(overlay['artifacts']['copied']) if overlay else 0, - missing_regenerable_count=len(overlay['artifacts']['missing_regenerable']) if overlay else 0) raise AssertionError('Closed M4 dispatch is incomplete.') diff --git a/project_files.py b/project_files.py index e2a5585b..12de25fb 100644 --- a/project_files.py +++ b/project_files.py @@ -233,7 +233,12 @@ def _read(root,parts): def inspect_captured(records, project_id, root): """Observe captured working files using only a locked project capability.""" - owned=[r for r in decode_records(records) if r['document']['project_id']==project_id] + return _inspect_validated(decode_records(records), project_id, root) + + +def _inspect_validated(records, project_id, root): + """Consume full-decoder output or fresh values from a validated Session.""" + owned=[r for r in records if r['document']['project_id']==project_id] items=[] for envelope in sorted(owned,key=lambda r:r['document']['path']): doc=envelope['document'];value=doc['current'] @@ -261,16 +266,16 @@ def __init__(self,session): def inspect(self): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.inspect');snapshot=self.session.check() + self.access.require('inspect','files.inspect');snapshot=self.session.check_records(FAMILY) with self.store.root_directory() as root: - items=inspect_captured(snapshot['records'],snapshot['project_id'],root) + items=_inspect_validated(snapshot['records'],snapshot['project_id'],root) self.session.check_current() return dict(items=items,project_id=snapshot['project_id'],checkpoint=snapshot['checkpoint']) def read(self,record_id,revision=None): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.read',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.read',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) values=envelope['document']['history'] if revision is None:value=values[-1] @@ -280,8 +285,8 @@ def read(self,record_id,revision=None): def history(self,record_id): with self.access.lock,self.session.lock,self.db.lock: - self.access.require('inspect','files.history',record_id);snapshot=self.session.check() - envelope=next((r for r in decode_records(snapshot['records']) if r['id']==record_id),None) + self.access.require('inspect','files.history',record_id);snapshot=self.session.check_records(FAMILY) + envelope=next((r for r in snapshot['records'] if r['id']==record_id),None) if envelope is None:fail('Adopted working file is unavailable.','unavailable',404) return copy.deepcopy(envelope['document']['history']) diff --git a/project_session.py b/project_session.py index 5fa571d1..c388eaf7 100644 --- a/project_session.py +++ b/project_session.py @@ -226,6 +226,22 @@ def check(self): """Return independent current records under caller-owned projection guards.""" return self._checked(self.store.snapshot) + def check_records(self, family): + """Verify every canonical record, then copy one bound record family. + + The complete snapshot, including cross-family closure, was validated + and isolated by adopt/publication. Returned copies retain that validity + inside the trusted owner boundary; they are not new imported raw input. + Exact current bytes and checkpoint must still match before using it. + Callers hold the same ordered guards as check(); no cached alias escapes. + """ + if type(family) is not str: + raise TypeError('Record-family selection requires a string.') + current = self._checked(self.store.summary) + current['records'] = copy.deepcopy([ + item for item in self._snapshot['records'] if item['family'] == family]) + return current + def mutate(self, operation, target, body, expected_checkpoint, key, invoke, capture, *, scope='edit'): """Invoke existing policy, capture fixed-owner upserts, publish, commit SQL. diff --git a/tests/test_project_dispatch.py b/tests/test_project_dispatch.py index 8aab246d..1aee28e5 100644 --- a/tests/test_project_dispatch.py +++ b/tests/test_project_dispatch.py @@ -30,7 +30,7 @@ def require(self, *args, **kwargs): def projection(): access = Authority() - session = SimpleNamespace(access=access, check=Mock(), snapshot={'project_id': PROJECT, 'records': []}, + session = SimpleNamespace(access=access, check=Mock(), check_current=Mock(), snapshot={'project_id': PROJECT, 'records': []}, summary={'project_id': PROJECT, 'name': 'Dispatch fixture', 'checkpoint': CHECKPOINT}) return SimpleNamespace(session=session, data=Mock(), materials=Mock(), history=Mock(), context=SimpleNamespace(models=Mock(), pipelines=Mock()), artifacts=Mock(), diff --git a/tests/test_project_files.py b/tests/test_project_files.py index 49af9af2..f512d54e 100644 --- a/tests/test_project_files.py +++ b/tests/test_project_files.py @@ -44,15 +44,15 @@ def intent(self,path,text,revision=0,kind='note'): raw=text.encode('utf-8');target=self.root/path;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(raw) return dict(kind=kind,path=path,expected_revision=revision,expected_sha256=hashlib.sha256(raw).hexdigest()) - def test_inspection_copies_only_the_records_it_actually_consumes(self): + def test_inspection_never_copies_the_complete_record_graph(self): self.adopt('notes/inspection.md', 'Exact retained scientific note', kind='note') expected = self.files.inspect() with patch.object(self.session.store, '_snapshot', wraps=self.session.store._snapshot) as snapshots: observed = self.files.inspect() self.assertEqual(observed, expected) - self.assertEqual(snapshots.call_count, 1, - 'Inspection needs one record snapshot; final freshness must not clone another') + self.assertEqual(snapshots.call_count, 0, + 'File inspection needs only independent file records and complete freshness') def test_batch_captures_exact_files_in_one_canonical_mutation(self): items=[self.intent('notes/a.md','Exact Ω\r\n'),self.intent('notes/empty.md','')] diff --git a/tests/test_project_files_m4_clients.py b/tests/test_project_files_m4_clients.py index d24d42a2..90af3962 100644 --- a/tests/test_project_files_m4_clients.py +++ b/tests/test_project_files_m4_clients.py @@ -231,13 +231,25 @@ def test_dispatch_projects_metadata_pages_without_captured_text_or_definition_bu import threading access=SimpleNamespace(lock=threading.RLock(),require=Mock()) session=SimpleNamespace(access=access,lock=threading.RLock(),db=SimpleNamespace(lock=threading.RLock()), - check=Mock(return_value={'project_id':PROJECT,'records':[]})) + check=Mock(side_effect=AssertionError('Unused full dispatcher snapshot')),check_current=Mock()) files=SimpleNamespace(history=Mock(return_value=[version('large text '+str(i),i+1) for i in range(42)])) projection=SimpleNamespace(session=session,files=files) page=project_dispatch.dispatch(projection,'project_files.file_history',dict(READ,file_id=identity(),offset=40,limit=2)) self.assertEqual((page['total'],[item['revision'] for item in page['items']]),(42,[41,42])) self.assertTrue(all('text' not in item for item in page['items'])) + session.check_current.assert_called_once_with() api.validate_output('project_files.file_history',dict(status=200,result=bound(page))) + def test_dispatch_fork_info_retains_the_complete_snapshot_contract(self): + import threading + access=SimpleNamespace(lock=threading.RLock(),require=Mock()) + session=SimpleNamespace(access=access,lock=threading.RLock(),db=SimpleNamespace(lock=threading.RLock()), + check=Mock(return_value={'project_id':PROJECT,'records':[]}), + check_current=Mock(side_effect=AssertionError('Fork must read its records'))) + value=project_dispatch.dispatch(SimpleNamespace(session=session),'project_files.fork_info',dict(READ,offset=0,limit=40)) + session.check.assert_called_once_with() + self.assertEqual(value,dict(source=None,project_id=PROJECT, + mapping=dict(items=[],offset=0,limit=40,total=0),archived_count=0,copied_count=0,missing_regenerable_count=0)) + if __name__=='__main__':unittest.main() diff --git a/tests/test_project_files_m4_http.py b/tests/test_project_files_m4_http.py index 7280a2b4..8d6b0a6f 100644 --- a/tests/test_project_files_m4_http.py +++ b/tests/test_project_files_m4_http.py @@ -10,6 +10,7 @@ import project_workspace from project_store import ProjectStoreError +from workbench import WorkbenchError import test_project_files_http as m1 import test_project_files_m2_http as m2 @@ -46,6 +47,12 @@ def test_batch_adoption_uses_real_http_and_survives_independent_fork(self): items.append(dict(kind='note',path='notes/'+name,expected_revision=0,expected_sha256=hashlib.sha256(raw).hexdigest())) result=self.call('files_adopt',items=items) self.assertEqual(result['changed_count'],2);self.assertEqual(len(self.call('files_inspect')['items']),2) + with patch.object(self.workspace._store,'_snapshot',side_effect=AssertionError('Unrelated full graph copy')): + self.assertEqual(len(self.call('files_inspect')['items']),2) + selected=self.call('file_get',file_id=result['items'][0]['id']) + self.assertEqual(selected['version']['text'].encode(),b'Exact first note\r\n') + history=self.call('file_history',file_id=result['items'][0]['id']) + self.assertEqual([v['revision'] for v in history['items']],[1]) source=copy.deepcopy(self.workspace._projection.session.snapshot) self.fork() current=self.workspace._projection.session.snapshot;indexed={(r['family'],r['id']):r for r in current['records']} @@ -70,6 +77,18 @@ def test_nonempty_independent_fork_exact_science_new_editable_ids_and_same_reque current=self.workspace._projection.session.snapshot indexed={(r['family'],r['id']):r for r in current['records']} for r in original['records']:self.assertEqual(indexed[(r['family'],r['id'])],r) + forged=copy.deepcopy(current) + inherited=next(r['document'] for r in forged['records'] if r['family']=='project_file' + and r['document']['project_id']==current['project_id']) + forged_version=inherited['history'][0];forged_text='Forged inherited script\n' + forged_version.update(text=forged_text,bytes=len(forged_text.encode()),sha256=hashlib.sha256(forged_text.encode()).hexdigest()) + inherited['current']=copy.deepcopy(inherited['history'][-1]) + from project_files import decode_records + with self.assertRaisesRegex(WorkbenchError,'Working-file fork requires its exact archived history'): + decode_records(forged['records']) + with self.assertRaisesRegex(WorkbenchError,'Working-file fork requires its exact archived history'): + self.workspace._projection.session.adopt(forged) + self.assertEqual(self.workspace._projection.session.snapshot,current) active=self.call('data_query')['items'] self.assertEqual(len(active),1);self.assertNotEqual(active[0]['id'],row['id']) self.assertEqual(self.call('data_get',record_id=active[0]['id'])['text'],row['text']) diff --git a/tests/test_project_session_freshness.py b/tests/test_project_session_freshness.py index 16996739..8ab5634e 100644 --- a/tests/test_project_session_freshness.py +++ b/tests/test_project_session_freshness.py @@ -9,6 +9,7 @@ import os from pathlib import Path import tempfile +from types import SimpleNamespace import unittest from unittest.mock import patch @@ -45,7 +46,8 @@ def test_changed_canonical_checkpoint_refuses_both_read_contracts(self): changed = copy.deepcopy(self.records) changed[0]['document']['text'] = 'Independently published content' self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c' * 64) - for check in (self.session.check_current, self.session.check): + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): with self.assertRaises(ProjectSessionError) as caught: check() self.assertEqual(caught.exception.code, 'stale_project') @@ -54,7 +56,8 @@ def test_unavailable_projection_refuses_before_reading_the_store(self): self.session.available = False with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')), \ patch.object(self.store, 'snapshot', side_effect=AssertionError('Unavailable snapshot read')): - for check in (self.session.check_current, self.session.check): + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): with self.assertRaises(ProjectSessionError) as caught: check() self.assertEqual(caught.exception.code, 'projection_unavailable') @@ -70,7 +73,8 @@ def test_same_length_corruption_with_restored_mtime_is_not_cached_as_fresh(self) path.chmod(0o600) path.write_bytes(changed) os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) - for check in (self.session.check_current, self.session.check): + for check in (self.session.check_current, self.session.check, + lambda: self.session.check_records('workbench_text')): with self.assertRaises(ProjectStoreError) as caught: check() self.assertEqual(caught.exception.code, 'stale') @@ -83,6 +87,50 @@ def test_record_consumers_keep_independent_complete_snapshots(self): self.assertEqual(self.session._snapshot, self.store.snapshot()) self.assertNotEqual(current, self.session.check()) + def add_unrelated_record(self): + unrelated = dict(family='project_artifact', id='d' * 64, + document=dict(format='project_artifact_v1', sha256='d' * 64, bytes=4)) + self.session._snapshot = self.store.commit([*self.records, unrelated], + self.store.summary()['checkpoint'], 'unrelated', 'e' * 64) + + def test_selected_records_are_exact_independent_and_do_not_copy_other_families(self): + self.add_unrelated_record() + expected = {key: value for key, value in self.store.snapshot().items() if key != 'records'} + expected['records'] = self.records + copied = [] + deepcopy = copy.deepcopy + def observe(value): + self.assertEqual([row['family'] for row in value], ['workbench_text', 'workbench_text']) + copied.append(value) + return deepcopy(value) + with patch.object(self.store, '_snapshot', side_effect=AssertionError('Unselected graph copy')), \ + patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ + patch('project_session.copy', SimpleNamespace(deepcopy=observe)): + selected = self.session.check_records('workbench_text') + self.assertEqual(selected, expected) + self.assertEqual(reads.call_count, 3, 'Unselected actual records must still be read and hashed') + self.assertEqual(len(copied), 1) + selected['records'][0]['document']['text'] = 'Caller mutation' + self.assertEqual(self.session.check_records('workbench_text'), expected) + self.assertEqual(self.session.check_records('project_file')['records'], []) + + def test_selected_read_rejects_corruption_in_an_unselected_family(self): + self.add_unrelated_record() + self.session.check_records('workbench_text') + manifest = json.loads((self.root / '.tuldok/project.json').read_bytes()) + entry = next(row for row in manifest['inventory'] if row['family'] == 'project_artifact') + path = self.root / '.tuldok' / entry['path'] + info, original = path.stat(), path.read_bytes() + changed = original.replace(b'"bytes": 4', b'"bytes": 5') + self.assertNotEqual(changed, original) + self.assertEqual(len(changed), len(original)) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + with self.assertRaises(ProjectStoreError) as caught: + self.session.check_records('workbench_text') + self.assertEqual(caught.exception.code, 'stale') + if __name__ == '__main__': unittest.main() From 9ff98990d524fe885064120c3b77fe198c09a8ac Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 05:07:31 -0700 Subject: [PATCH 29/33] perf(history): reuse decoded facts at a verified checkpoint --- project_history.py | 14 ++++- tests/test_project_extraction.py | 2 + tests/test_project_history.py | 68 +++++++++++++++++++++++++ tests/test_project_native_extraction.py | 5 +- 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/project_history.py b/project_history.py index 8733ca4a..b69eccce 100644 --- a/project_history.py +++ b/project_history.py @@ -750,11 +750,23 @@ def __init__(self,session,*,materials=None,authorize_disclosure=None): from project_material import ProjectMaterial materials=ProjectMaterial(session) self.materials=materials;self.authorize_disclosure=authorize_disclosure + self._decoded_checkpoint=None + self._decoded_facts=None def _facts(self): self.session.check_current() + summary=self.session.summary + checkpoint=(summary['project_id'],summary['checkpoint']) + if self._decoded_checkpoint==checkpoint: + return self._decoded_facts records=self.session.snapshot['records'] - return decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) + facts=decode_records(records,materials=self.materials.read_view(records),artifacts=self.artifacts) + # Current canonical bytes are guarded on every call. Only this owner's + # private decoded metadata is retained; returned values and permissions + # remain the responsibility of each read below. + self._decoded_facts=facts + self._decoded_checkpoint=checkpoint + return facts def _authorize(self,action,target=None): self.access.require('inspect',action,target) diff --git a/tests/test_project_extraction.py b/tests/test_project_extraction.py index afa628a6..34e43176 100644 --- a/tests/test_project_extraction.py +++ b/tests/test_project_extraction.py @@ -20,6 +20,7 @@ from access import Access from metadata import Metadata from model_definitions import ModelDefinitions +from pipeline import digest from project_data import decode_data_records from project_extraction import _Capture, capture_legacy from project_material import decode_records as decode_material @@ -650,6 +651,7 @@ def test_selected_run_automatically_captures_exact_upstream_run_artifact_closure # A fresh inert owner reader reconstructs the exact upstream custody # receipt from retained facts/private captured bytes, never source paths. session = SimpleNamespace(snapshot=dict(project_id=self.db.dataset_id, records=copy.deepcopy(captured['records'])), + summary=dict(project_id=self.db.dataset_id, checkpoint=digest(captured['records'])), access=self.access, artifacts=source.artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda run, population: self.access.require('expose-holdout', 'runs.expose_test', run['id'])) diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 89e4ecf2..c42dd135 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -85,6 +85,47 @@ def legacy_run(origin=ORIGIN): class HistoryCodecTests(unittest.TestCase): + def test_progression_reuses_one_decoded_checkpoint_and_keeps_returned_values_independent(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + checks=[] + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER,checkpoint='c'*64), + summary=dict(project_id=OTHER,checkpoint='c'*64),artifacts=None, + access=SimpleNamespace(require=lambda *args,**kwargs:None), + check_current=lambda:checks.append('actual current-state guard')) + history=ProjectHistory(session,materials=Materials()) + with patch('project_history.decode_records',wraps=decode_records) as decoder: + first=history.progression(ORIGIN,offset=0,limit=20) + first['nodes'][0]['study']['name']='Caller mutation' + first['nodes'][0]['trial']['replicate']=99 + second=history.progression(ORIGIN,offset=0,limit=20) + self.assertEqual(decoder.call_count,1) + self.assertEqual(checks,['actual current-state guard']*2) + self.assertEqual(second['nodes'][0]['study']['name'],'Exact intent λ') + self.assertEqual(second['nodes'][0]['trial']['replicate'],0) + + def test_history_cache_refreshes_on_checkpoint_change_and_does_not_cache_decode_failure(self): + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), + summary=dict(project_id=OTHER,checkpoint='c'*64),artifacts=None, + access=SimpleNamespace(require=lambda *args,**kwargs:None),check_current=lambda:None) + history=ProjectHistory(session,materials=Materials()) + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'Exact intent λ') + updated=copy.deepcopy(records) + for value in (updated[0]['document']['study'],updated[0]['document']['revisions'][0]): + value['name']='New checkpoint name' + session.snapshot=dict(records=updated,project_id=OTHER) + session.summary=dict(project_id=OTHER,checkpoint='d'*64) + with patch('project_history.decode_records',side_effect=WorkbenchError('Decode unavailable','unavailable',503)): + with self.assertRaisesRegex(WorkbenchError,'Decode unavailable') as failed: + history.progression(ORIGIN) + self.assertEqual(failed.exception.code,'unavailable') + with patch('project_history.decode_records',wraps=decode_records) as decoder: + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'New checkpoint name') + self.assertEqual(history.progression(ORIGIN)['nodes'][0]['study']['name'],'New checkpoint name') + self.assertEqual(decoder.call_count,1) + def test_fork_preserves_archived_disclosure_observation_without_current_permission(self): from project_fork import _digest,_input from project_session import operation_records @@ -137,6 +178,7 @@ def read_view(self, selected): def recipe(self, *args, **kwargs): raise AssertionError('History reread the live material catalog') session=SimpleNamespace(snapshot=dict(records=records,project_id=OTHER), + summary=dict(project_id=OTHER,checkpoint='c'*64), artifacts=None,access=SimpleNamespace(),check_current=lambda:None) facts=ProjectHistory(session,materials=Provider())._facts() self.assertEqual(len(observed),1) @@ -312,6 +354,32 @@ def start(self): self.host,self.owner,self.session=host,owner,session self.history=ProjectHistory(session,materials=Materials()) + def test_cached_history_still_reads_actual_bytes_and_rejects_restored_mtime_corruption(self): + from project_store import ProjectStoreError + identity='9'*32 + records=[study(trials=[identity]),trial(identity,'3'*32)] + before=self.store.snapshot() + snapshot=self.store.commit(records,before['checkpoint'],'cache-corruption-fixture',digest(records)) + self.session.adopt(snapshot) + with self.owner.bound(self.host): + expected=self.history.progression(ORIGIN) + header=json.loads((self.store.path/'.tuldok/project.json').read_bytes()) + entry=next(item for item in header['inventory'] if item['family']=='study_history') + path=self.store.path/'.tuldok'/entry['path'] + original=path.read_bytes();info=path.stat() + corrupt=original.replace(b'Exact intent',b'Wrong intent',1) + self.assertNotEqual(corrupt,original);self.assertEqual(len(corrupt),len(original)) + try: + path.write_bytes(corrupt) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + with self.assertRaisesRegex(ProjectStoreError,'Canonical record bytes changed or are corrupt') as failed: + self.history.progression(ORIGIN) + self.assertEqual((failed.exception.code,failed.exception.status),('stale',409)) + finally: + path.write_bytes(original) + os.utime(path,ns=(info.st_atime_ns,info.st_mtime_ns)) + self.assertEqual(self.history.progression(ORIGIN),expected) + def test_progression_uses_original_namespaces_exact_edges_and_bounded_pages(self): parent='a'*32;child='b'*32 ref={'catalog_id':ORIGIN,'trial_id':parent,'material_sha256':MATERIAL} diff --git a/tests/test_project_native_extraction.py b/tests/test_project_native_extraction.py index fe1d34b3..291e2247 100644 --- a/tests/test_project_native_extraction.py +++ b/tests/test_project_native_extraction.py @@ -115,7 +115,10 @@ def test_original_sources_are_unique_canonical_payloads_and_cold_read_byte_exact for name in originals: (stage / name).unlink() cold = ProjectStore.open(destination); self.addCleanup(cold.close) cold_artifacts = ProjectArtifacts(cold); self.addCleanup(cold_artifacts.close) - session = SimpleNamespace(snapshot=cold.snapshot(), access=self.access, artifacts=cold_artifacts, check_current=lambda: None) + snapshot = cold.snapshot() + session = SimpleNamespace(snapshot=snapshot, + summary={key:snapshot[key] for key in ('project_id','checkpoint')}, + access=self.access, artifacts=cold_artifacts, check_current=lambda: None) history = ProjectHistory(session, authorize_disclosure=lambda *args: None) for name, raw in originals.items(): self.assertEqual(history.artifact(run['id'], 'source/' + name, origin_catalog_id=self.db.dataset_id), raw) From 0490cb14928644d9c6758b32cd2948a9c5f6542f Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 08:17:58 -0700 Subject: [PATCH 30/33] perf(project): reduce validated snapshot and text-list copies --- project_projection.py | 7 +++-- project_session.py | 5 +-- project_store.py | 13 +++++++- tests/test_project_projection.py | 21 ++++++++++++- tests/test_project_store.py | 21 +++++++++++++ tests/test_project_text_tag.py | 52 ++++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 tests/test_project_text_tag.py diff --git a/project_projection.py b/project_projection.py index 1bf45bd2..87d8d133 100644 --- a/project_projection.py +++ b/project_projection.py @@ -313,14 +313,15 @@ def available(self): return self.projection.session.available def _tag(self, value): - snapshot = self.projection.session.snapshot - return dict(value, project_id=snapshot['project_id'], checkpoint=snapshot['checkpoint']) + summary = self.projection.session.summary + return dict(value, project_id=summary['project_id'], checkpoint=summary['checkpoint']) def records(self): self.projection.session.check_current() from project_fork import active_records snapshot = self.projection.session.snapshot - records = active_records(snapshot['records'], project_id=snapshot['project_id']) + records = active_records(snapshot['records'], project_id=snapshot['project_id'], + families=frozenset(('workbench_record', 'workbench_text'))) return self._tag(dict(items=[copy.deepcopy(item['document']['current']) for item in records if item['family'] in ('workbench_record', 'workbench_text') and item['document']['current']['kind'] == 'text'])) diff --git a/project_session.py b/project_session.py index c388eaf7..aa471c1e 100644 --- a/project_session.py +++ b/project_session.py @@ -9,6 +9,7 @@ import threading import agent_api +from project_store import copy_validated_json from workbench import WorkbenchError @@ -186,7 +187,7 @@ def summary(self): @property def snapshot(self): with self.lock: - return copy.deepcopy(self._snapshot) + return copy_validated_json(self._snapshot) def _validated(self, records): operation_records(records, project_id=self.project_id) @@ -203,7 +204,7 @@ def adopt(self, snapshot): current = self.store.snapshot() if current != snapshot: raise ProjectSessionError('Project changed while binding its projection.', 'stale_project', 409) - self._snapshot = copy.deepcopy(snapshot) + self._snapshot = copy_validated_json(snapshot) self.available = True def _checked(self, read): diff --git a/project_store.py b/project_store.py index cb1c2bf9..d079ca3f 100644 --- a/project_store.py +++ b/project_store.py @@ -124,6 +124,17 @@ def _encode(value): allow_nan=False) + '\n').encode('utf-8') +def copy_validated_json(value): + """Own another tree of already validated JSON without Python graph copying. + + Canonical Store/Session values contain only JSON scalars, lists and string + keyed dictionaries. This is a copy operation inside that proof boundary, + never an input decoder or a substitute for freshness/codec validation. + """ + return json.loads(json.dumps(value, ensure_ascii=True, allow_nan=False, + separators=(',', ':'))) + + def _manifest_raw(manifest): raw = _encode(manifest) if len(raw) > MAX_MANIFEST_BYTES: @@ -793,7 +804,7 @@ def _publish(self, manifest, creating=False): def _snapshot(self, manifest, records): return dict(project_id=manifest['project_id'], name=manifest['name'], - checkpoint=manifest['checkpoint'], records=copy.deepcopy(records)) + checkpoint=manifest['checkpoint'], records=copy_validated_json(records)) def _current(self): try: diff --git a/tests/test_project_projection.py b/tests/test_project_projection.py index d08aaadc..3f1e3e98 100644 --- a/tests/test_project_projection.py +++ b/tests/test_project_projection.py @@ -6,7 +6,7 @@ from pathlib import Path import tempfile import unittest -from unittest.mock import patch +from unittest.mock import PropertyMock, patch import uuid import psycopg @@ -120,6 +120,25 @@ def test_unknown_receipt_is_rejected_before_invoke_and_sql_or_checkpoint_changes self.assertEqual(p.store.snapshot(), before) self.assertEqual(p.db.execute('SELECT COUNT(*) FROM workbench_records').fetchone()[0], 0) + def test_text_tag_uses_real_session_summary_and_record_list_remains_owned(self): + text = self.call('import_text', self.mutation(asset=dict(text='Owned note 山', groups=['owned'])))['record'] + p = self.host._projection + with self.owner.bound(self.host): + expected = dict(project_id=p.session.summary['project_id'], + checkpoint=p.session.summary['checkpoint']) + with patch.object(type(p.session), 'snapshot', new_callable=PropertyMock, + side_effect=AssertionError('Scalar tag copied a full Session')): + self.assertEqual(p.text._tag(dict(value='Exact scalar tag')), + dict(expected, value='Exact scalar tag')) + exact_current = [r['document']['current'] for r in p.session.snapshot['records'] + if r['family'] in ('workbench_text', 'workbench_record') + and r['document']['current']['kind'] == 'text'] + result = p.text.records() + self.assertEqual(result, dict(expected, items=exact_current)) + self.assertEqual([r['id'] for r in result['items']], [text['id']]) + result['items'][0]['text'] = 'Caller mutation' + self.assertEqual(p.text.get(text['id'])['text'], 'Owned note 山') + def test_explicit_rooted_artifact_supply_is_durable_and_never_fetches_or_executes(self): p = self.host._projection raw = b'Unique measured bytes; never regenerate these.\n' diff --git a/tests/test_project_store.py b/tests/test_project_store.py index f3e7f0e2..676ab3bc 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -3,10 +3,12 @@ import errno import hashlib import json +import math import os from pathlib import Path import shutil import subprocess +import struct import sys import tempfile import unittest @@ -71,6 +73,25 @@ def write_manifest(self, value): value['checkpoint'] = sha(canonical(value)) (self.root / '.tuldok/project.json').write_bytes(canonical(value)) + def test_snapshot_preserves_scalar_types_float_bits_and_owned_nested_values(self): + floats = [-0.0, 0.0, 5e-324, -5e-324, 1.7976931348623157e308, + math.nextafter(1.0, 0.0), math.nextafter(1.0, math.inf)] + value = record() + value['document']['scalars'] = [None, False, True, -2**63, 2**63-1, + 'Unicode 漢字 \U0001f30a\x00', *floats] + expected = self.commit([value]) + first, second = self.store.snapshot(), self.store.snapshot() + self.assertEqual(first, expected) + self.assertEqual(second, expected) + actual = first['records'][0]['document']['scalars'] + self.assertEqual([type(v) for v in actual], + [type(v) for v in value['document']['scalars']]) + self.assertEqual([struct.pack('>d', v) for v in actual[6:]], + [struct.pack('>d', v) for v in floats]) + first['records'][0]['document']['history'][0]['reviewed'] = True + first['records'][0]['document']['scalars'].clear() + self.assertEqual(self.store.snapshot(), second) + def test_create_commit_save_cold_reopen_and_independent_snapshots(self): initial = self.store.snapshot() self.assertEqual(initial['records'], []) diff --git a/tests/test_project_text_tag.py b/tests/test_project_text_tag.py new file mode 100644 index 00000000..6a944aa6 --- /dev/null +++ b/tests/test_project_text_tag.py @@ -0,0 +1,52 @@ +"""The scalar text response tag must not copy unrelated project metadata.""" +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import project_fork + +from project_projection import _TextView + + +class TextTagTests(unittest.TestCase): + def test_text_records_does_not_clone_unrelated_active_documents(self): + unrelated = dict(family='project_artifact', id='c' * 64, + document=dict(format='project_artifact_v1', sha256='c' * 64, bytes=4)) + note = dict(family='workbench_text', id='d' * 32, + document=dict(current=dict(kind='text', text='Unique note 山'))) + summary = dict(project_id='a' * 32, checkpoint='b' * 64) + session = SimpleNamespace(summary=summary, + snapshot=dict(summary, records=[unrelated, note]), + check_current=lambda: None) + copied = [] + original = project_fork.copy.deepcopy + + def observe(value): + if value is unrelated: + copied.append(value['id']) + return original(value) + + with patch.object(project_fork, 'copy', SimpleNamespace(deepcopy=observe)): + result = _TextView(SimpleNamespace(session=session)).records() + self.assertEqual(result, dict(summary, items=[dict(kind='text', text='Unique note 山')])) + self.assertEqual(copied, [], 'Text list cloned an unrelated active document') + result['items'][0]['text'] = 'Caller mutation' + self.assertEqual(note['document']['current']['text'], 'Unique note 山') + + def test_text_tag_reads_scalar_summary_without_a_record_snapshot(self): + class Session: + summary = dict(project_id='a' * 32, name='Owned project', checkpoint='b' * 64) + + @property + def snapshot(self): + raise AssertionError('Text response tag copied the full record graph') + + session = Session() + view = _TextView(SimpleNamespace(session=session)) + original = dict(items=[dict(text='Authored note 漢字')]) + result = view._tag(original) + self.assertEqual(result, dict(original, project_id='a' * 32, checkpoint='b' * 64)) + self.assertNotIn('name', result) + self.assertEqual(original, dict(items=[dict(text='Authored note 漢字')])) + result['project_id'] = 'caller edit' + self.assertEqual(session.summary['project_id'], 'a' * 32) From d3b8f92ed99ad005e2bb09f6e963c3aebc24d9d0 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 09:52:54 -0700 Subject: [PATCH 31/33] perf(project): validate retained receipts from one owned candidate view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decode each receipt domain once per candidate validation pass while retaining exact revision, source and packet-custody checks. Keep standalone validation fresh and share only a single call’s decoded inventory across batch items. --- project_files.py | 31 +++- project_measurements.py | 26 ++- project_projection.py | 14 +- project_sources.py | 21 ++- tests/test_project_receipt_validation.py | 204 +++++++++++++++++++++++ 5 files changed, 280 insertions(+), 16 deletions(-) create mode 100644 tests/test_project_receipt_validation.py diff --git a/project_files.py b/project_files.py index 12de25fb..ad0f6b3f 100644 --- a/project_files.py +++ b/project_files.py @@ -5,6 +5,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import errno import hashlib import os @@ -128,14 +129,14 @@ def _result(envelope,changed): return result -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): if operation=='files.adopt_many': closed(result,{'items','changed_count'},'batch adoption result') if type(result['items']) is not list or not 1<=len(result['items'])<=MAX_ADOPT_ITEMS: fail('Invalid batch adoption result size.') seen=set();projects=set() for item in result['items']: - validate_operation_result('files.adopt',item,records) + _validate_result('files.adopt',item,lookup) if item['id'] in seen:fail('Batch adoption result repeats a file.') seen.add(item['id']);projects.add(item['project_id']) if (len(projects)!=1 or type(result['changed_count']) is not int @@ -153,14 +154,34 @@ def validate_operation_result(operation,result,records=None): else:sha(result['sha256']) _date(result['adopted_at']) if type(result['revision']) is not int or not 1<=result['revision']<=2**53-1 or type(result['bytes']) is not int or not 0<=result['bytes']<=MAX_FILE_BYTES:fail('Invalid adoption result bounds.') - if records is not None: - owned=decode_records(records) - envelope=next((r for r in owned if r['id']==result['id']),None) + if lookup is not None: + envelope=lookup(result['id']) if envelope is None or not any((value.get('deleted') is True)==deleted and all(value[key]==result[key] for key in ('revision','sha256','bytes','adopted_at')) for value in envelope['document']['history']): fail('Adoption result is not an exact retained working-file revision.') return True +def receipt_validator(records): + """Own one decoded candidate's file histories for its receipt checks. + + The returned validator has no live Session or publication authority. It + owns copied records, so caller mutation cannot change this fixed view. + A different candidate requires a new validator. + """ + owned = {record['id']: record for record in decode_records(records)} + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + # Keep standalone shape/error ordering, and share only this call's lookup + # across batch items. There is no retained cache between invocations. + @cache + def owned(): + return {record['id']: record for record in decode_records(records)} + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class _WorkingError(Exception): def __init__(self,state):self.state=state diff --git a/project_measurements.py b/project_measurements.py index 400db842..d1fed0de 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -6,6 +6,7 @@ """ import copy from datetime import datetime +from functools import cache, partial from project_artifacts import validate_declaration from project_material import decode_records as material_records @@ -101,17 +102,36 @@ def decode_records(records, *, artifacts=None): return result -def validate_operation_result(operation, result, records=None): +def _validate_result(operation, result, lookup=None): if operation != 'measurements.import': return False _document(result) - if records is not None: + if lookup is not None: identity = result['origin_catalog_id'] + '.' + result['id'] - if decode_records(records).get(identity) != result: + if lookup(identity) != result: raise WorkbenchError('Measurement receipt has no exact retained source.') return True +def receipt_validator(records, *, artifacts=None): + """Own one candidate's validated source bindings and packet custody. + + Decode and copy every retained measurement once, including current available + packet bytes when supplied. The returned receipt checker is fixed to those + facts; it carries no live source or publication capability. + """ + owned = decode_records(records, artifacts=artifacts) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return decode_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class ProjectMeasurements: def __init__(self, session): self.session, self.artifacts = session, session.artifacts diff --git a/project_projection.py b/project_projection.py index 87d8d133..25b6bf28 100644 --- a/project_projection.py +++ b/project_projection.py @@ -25,14 +25,17 @@ from project_producer_execution import ProjectProducerExecution, validate_operation_result as validate_producer_result from project_files import ProjectFiles, decode_records as decode_file_records from project_files import validate_operation_result as validate_file_result +from project_files import receipt_validator as file_receipt_validator from project_study_execution import ProjectStudyExecution, validate_operation_result as validate_study_execution_result from project_runtime import ProjectRuntime, validate_operation_result as validate_runtime_result from project_native_execution import FAMILY as NATIVE_FAMILY, decode_records as decode_native_records from project_native_preparation import ProjectNativePreparation, validate_operation_result as validate_preparation_result from project_sources import SourceAdapter, validate_operation_result as validate_source_result +from project_sources import receipt_validator as source_receipt_validator from project_measurements import ProjectMeasurements, FAMILY as MEASUREMENT_FAMILY from project_measurements import decode_records as decode_measurement_records from project_measurements import validate_operation_result as validate_measurement_result +from project_measurements import receipt_validator as measurement_receipt_validator from workbench import WorkbenchError from workbench import timestamp @@ -82,7 +85,7 @@ def validate_project_records(records, *, project_id, artifacts): from project_transfer import decode_records as decode_extraction_records decode_extraction_records(records) operation_records(records, project_id=project_id) - decode_file_records(records) + validate_file_receipt = file_receipt_validator(records) decode_native_records(records,artifacts=artifacts) materials = MaterialSnapshot(records) for blob in materials.artifact_blobs(): @@ -92,17 +95,18 @@ def validate_project_records(records, *, project_id, artifacts): # than a resolver that still sees the preceding committed checkpoint. decode_producer_records(records, artifacts=artifacts) decode_history_records(records, materials=materials, artifacts=artifacts) - decode_measurement_records(records, artifacts=artifacts) + validate_measurement_receipt = measurement_receipt_validator(records, artifacts=artifacts) + validate_source_receipt = source_receipt_validator(records) for receipt in operation_documents(records): operation, result = receipt['operation'], receipt['result'] if (validate_data_result(operation, result, records) or validate_material_result(operation, result, records) or validate_history_result(operation, result, records) - or validate_source_result(operation, result, records) - or validate_measurement_result(operation, result, records) + or validate_source_receipt(operation, result) + or validate_measurement_receipt(operation, result) or validate_execution_result(operation, result, records) or validate_producer_result(operation, receipt['target'], result, records) - or validate_file_result(operation, result, records) + or validate_file_receipt(operation, result) or validate_study_execution_result(operation, result, records) or validate_runtime_result(operation, result, records) or validate_preparation_result(operation, result, records) diff --git a/project_sources.py b/project_sources.py index 95e7c5f2..a17b02a5 100644 --- a/project_sources.py +++ b/project_sources.py @@ -6,6 +6,7 @@ """ import copy from datetime import datetime +from functools import cache, partial import hashlib import os @@ -36,7 +37,7 @@ def _summary(saved, origin): schema_version=saved['manifest']['schema_version']) -def validate_operation_result(operation,result,records=None): +def _validate_result(operation,result,lookup=None): """Validate the fixed source receipt against its exact retained snapshot.""" if operation!='source.register':return False try: @@ -50,8 +51,8 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Source registration requires an attributed timestamp.') if datetime.fromisoformat(result['registered_at']).utcoffset() is None: raise WorkbenchError('Source registration timestamp requires an offset.') - if records is not None: - document=material_records(records).get(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) + if lookup is not None: + document=lookup(('project_snapshot',result['origin_catalog_id']+'.'+result['manifest_sha256'])) expected={key:result[key] for key in ('origin_catalog_id','project_id','manifest_sha256','name','schema_version')} if document is None or _summary(document['value'],document['origin_catalog_id'])!=expected: raise WorkbenchError('Source receipt differs from its exact retained snapshot.') @@ -61,6 +62,20 @@ def validate_operation_result(operation,result,records=None): raise WorkbenchError('Malformed retained source operation result.') from None +def receipt_validator(records): + """Own one candidate's retained material facts for source receipt checks.""" + owned = material_records(records) + return partial(_validate_result, lookup=owned.get) + + +def validate_operation_result(operation, result, records=None): + @cache + def owned(): + return material_records(records) + lookup = None if records is None else lambda identity: owned().get(identity) + return _validate_result(operation, result, lookup) + + class SourceAdapter: def __init__(self,projection): self.projection=projection diff --git a/tests/test_project_receipt_validation.py b/tests/test_project_receipt_validation.py new file mode 100644 index 00000000..bf69926f --- /dev/null +++ b/tests/test_project_receipt_validation.py @@ -0,0 +1,204 @@ +"""One mixed candidate: retain every receipt check without repeated decoding.""" +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import project_files as files +import project_material as material +import project_measurements as measurements +import project_projection as projection +import project_sources as sources +from project_artifacts import ProjectArtifacts, ProjectArtifactError +from project_store import ProjectStore +from test_retained_measurements import fixture +from workbench import WorkbenchError + +PROJECT = 'a' * 32 +ACTOR = 'c' * 32 +INSTANT = '2026-10-25T00:00:00+00:00' + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def operation(name, key, result): + return dict(family='project_operation', id=PROJECT + '.' + sha(key.encode()), document=dict( + format='project_operation_v2', project_id=PROJECT, operation=name, + target=None, request_key=key, input_sha256='d' * 64, result=copy.deepcopy(result))) + + +def candidate(): + packets = [] + for index in range(2): + value = fixture() + value['title'] = f'Packet {index}' + raw = json.dumps(value).encode() + packets.append(dict(raw=raw, sha256=sha(raw), path=f'packet-{index}.json', + reference_id=f'packet-{index}', title=value['title'])) + manifest = dict(schema_version=1, id='b' * 32, name='Receipt source', + data=[dict(id=p['reference_id'], label=p['title'], path=p['path'], sha256=p['sha256']) + for p in packets], recipes=[], evidence=[], requirements=[]) + raw = json.dumps(manifest).encode() + digest = sha(raw) + snapshot = dict(id=manifest['id'], name=manifest['name'], root='original', + root_identity=[1, 2], manifest_path='source.json', manifest_sha256=digest, manifest=manifest) + retained = dict(family='project_snapshot', id=PROJECT + '.' + digest, document=dict( + format=material.FAMILIES['project_snapshot'], origin_catalog_id=PROJECT, value=snapshot, + artifacts=[dict(name='source.json', blob=dict(sha256=digest, bytes=len(raw)), + role='project_manifest', preservation='unique', required_for=['snapshot_verification'])], audit=[])) + records = [retained] + for hashed, size in [(digest, len(raw)), *[(p['sha256'], len(p['raw'])) for p in packets]]: + records.append(dict(family='project_artifact', id=hashed, + document=dict(format='project_artifact_v1', sha256=hashed, bytes=size))) + source_result = dict(origin_catalog_id=PROJECT, project_id=manifest['id'], manifest_sha256=digest, + name=manifest['name'], schema_version=1, actor_id=ACTOR, registered_at=INSTANT) + for index in range(2): + records.append(operation('source.register', f'source-{index}', source_result)) + for index, packet in enumerate(packets): + doc = dict(format=measurements.FORMAT, origin_catalog_id=PROJECT, id=packet['sha256'], + title=packet['title'], blob=dict(sha256=packet['sha256'], bytes=len(packet['raw'])), + source=dict(origin_catalog_id=PROJECT, project_id=manifest['id'], manifest_sha256=digest, + reference_id=packet['reference_id'], path=packet['path'], measured_sha256=packet['sha256']), + actor_id=ACTOR, imported_at=INSTANT) + records.append(dict(family=measurements.FAMILY, id=PROJECT + '.' + packet['sha256'], document=doc)) + records.append(operation('measurements.import', f'packet-{index}', doc)) + for batch in range(2): + items = [] + for index in range(2): + path = f'notes/note-{batch}-{index}.md' + raw = f'Owned note {batch}-{index} 山'.encode() + version = dict(revision=1, sha256=sha(raw), bytes=len(raw), text=raw.decode(), + adopted_at=INSTANT, actor_id=ACTOR) + identity = files.file_id(PROJECT, 'note', path) + doc = dict(format=files.FORMAT, project_id=PROJECT, kind='note', path=path, + current=copy.deepcopy(version), history=[version], origin=None) + records.append(dict(family=files.FAMILY, id=identity, document=doc)) + items.append(dict(id=identity, project_id=PROJECT, kind='note', path=path, changed=True, + **{key: version[key] for key in ('revision', 'sha256', 'bytes', 'adopted_at')})) + records.append(operation('files.adopt_many', f'notes-{batch}', dict(items=items, changed_count=2))) + return records + + +class ReceiptValidationTests(unittest.TestCase): + def test_whole_candidate_decodes_each_receipt_owner_once(self): + records = candidate() + before = copy.deepcopy(records) + with patch.object(files, 'decode_records', wraps=files.decode_records) as file_decode, \ + patch.object(projection, 'decode_file_records', file_decode), \ + patch.object(measurements, 'decode_records', wraps=measurements.decode_records) as measurement_decode, \ + patch.object(projection, 'decode_measurement_records', measurement_decode), \ + patch.object(material, 'decode_records', wraps=material.decode_records) as material_decode, \ + patch.object(measurements, 'material_records', material_decode), \ + patch.object(sources, 'material_records', material_decode): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + self.assertEqual(records, before) + for name, actual, expected in [('files', file_decode.call_count, 1), + ('measurements', measurement_decode.call_count, 1), + ('material-owning readers', material_decode.call_count, 3)]: + with self.subTest(owner=name): + self.assertEqual(actual, expected, 'Receipt validation redecoded the same candidate') + + def test_standalone_file_batch_decodes_inventory_once(self): + records = candidate() + result = next(r['document']['result'] for r in records + if r['family'] == 'project_operation' + and r['document']['operation'] == 'files.adopt_many') + with patch.object(files, 'decode_records', wraps=files.decode_records) as decode: + self.assertTrue(files.validate_operation_result('files.adopt_many', result, records)) + self.assertEqual(decode.call_count, 1, 'Each batch item decoded the full file inventory') + + def test_forged_late_file_or_measurement_receipt_is_rejected(self): + for owner in ('files.adopt_many', 'measurements.import', 'source.register'): + records = candidate() + result = [r['document']['result'] for r in records if r['family'] == 'project_operation' + and r['document']['operation'] == owner][-1] + if owner == 'files.adopt_many': + result['items'][-1]['sha256'] = 'e' * 64 + else: + result['name' if owner == 'source.register' else 'title'] = 'Forged retained value' + with self.subTest(owner=owner), self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_unknown_operation_still_has_no_accepting_owner(self): + records = candidate() + records.append(operation('unknown.capture', 'unknown', {})) + with self.assertRaises(WorkbenchError) as error: + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + self.assertEqual(error.exception.code, 'unsupported') + + def test_owned_validators_and_standalone_calls_do_not_share_candidates(self): + for name, module in [('files.adopt_many', files), ('measurements.import', measurements), + ('source.register', sources)]: + records = candidate() + original_result = copy.deepcopy(next(r['document']['result'] for r in records + if r['family'] == 'project_operation' and r['document']['operation'] == name)) + checker = module.receipt_validator(records) + self.assertTrue(checker(name, original_result)) + self.assertTrue(module.validate_operation_result(name, original_result, records)) + if module is files: + doc = next(r['document'] for r in records if r['family'] == files.FAMILY) + value = doc['history'][0] + value.update(text='Edited candidate', sha256=sha(b'Edited candidate'), bytes=16) + doc['current'] = copy.deepcopy(value) + elif module is measurements: + next(r['document'] for r in records if r['family'] == measurements.FAMILY)['title'] = 'Edited candidate' + else: + value = next(r['document']['value'] for r in records if r['family'] == 'project_snapshot') + value['name'] = value['manifest']['name'] = 'Edited candidate' + with self.subTest(owner=name): + self.assertTrue(checker(name, original_result), 'Owned candidate changed with caller mutation') + with self.assertRaises(WorkbenchError): + module.receipt_validator(records)(name, original_result) + with self.assertRaises(WorkbenchError): + module.validate_operation_result(name, original_result, records) + + def test_standalone_unrecognized_and_malformed_results_do_not_decode(self): + for name, module in [('files.adopt_many', files), ('measurements.import', measurements), + ('source.register', sources)]: + records = candidate() + attribute = 'material_records' if module is sources else 'decode_records' + with self.subTest(owner=name), patch.object(module, attribute, + side_effect=AssertionError('Result shape must precede candidate decoding')) as decode: + self.assertFalse(module.validate_operation_result('unknown.capture', {}, records)) + with self.assertRaises(WorkbenchError): + module.validate_operation_result(name, {}, records) + decode.assert_not_called() + + def test_unused_malformed_file_and_missing_packet_reference_still_reject(self): + records = candidate() + next(r['document']['history'][0] for r in records if r['family'] == files.FAMILY)['text'] = 'Unhashed' + records = [r for r in records if r['family'] != 'project_operation' + or r['document']['operation'] != 'files.adopt_many'] + with self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + records = candidate() + next(r['document'] for r in records if r['family'] == measurements.FAMILY)['source']['reference_id'] = 'absent' + with self.assertRaises(WorkbenchError): + projection.validate_project_records(records, project_id=PROJECT, artifacts=None) + + def test_each_new_measurement_validator_checks_actual_packet_bytes(self): + records = candidate() + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / 'project' + with ProjectStore.create(root, 'Packet custody fixture', 'create', 'a' * 64) as store: + with ProjectArtifacts(store) as artifacts: + docs = [r['document'] for r in records if r['family'] == measurements.FAMILY] + for index, doc in enumerate(docs): + packet = fixture() + packet['title'] = f'Packet {index}' + raw = json.dumps(packet).encode() + self.assertEqual(sha(raw), doc['id']) + artifacts.publish_bytes(raw, measurements.declaration(doc['blob'])) + checker = measurements.receipt_validator(records, artifacts=artifacts) + self.assertTrue(checker('measurements.import', docs[0])) + payload = root / 'artifacts/sha256' / docs[0]['id'] + raw = payload.read_bytes() + payload.chmod(0o600) + payload.write_bytes(bytes([raw[0] ^ 1]) + raw[1:]) + with self.assertRaises((WorkbenchError, ProjectArtifactError)): + measurements.receipt_validator(records, artifacts=artifacts) From f8ba4cedd22580f273f8c4fcbc06bca53797a7a7 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Sun, 25 Oct 2026 12:21:56 -0700 Subject: [PATCH 32/33] feat(project): browse authored model portfolios from retained notes --- app.py | 1 + docs/contracts/project-model-portfolio.md | 87 ++++++++++ static/project-files.css | 10 ++ static/project-files.html | 2 + static/project-files.js | 13 ++ static/project-model-portfolio.js | 188 +++++++++++++++++++++ tests/browser_model_portfolio.cjs | 129 ++++++++++++++ tests/model_portfolio_fixture.cjs | 16 ++ tests/test_project_model_portfolio.cjs | 47 ++++++ tests/test_project_model_portfolio_http.py | 84 +++++++++ tests/test_workspace_project_files.cjs | 1 + tests/test_workspace_project_files_m4.cjs | 1 + 12 files changed, 579 insertions(+) create mode 100644 docs/contracts/project-model-portfolio.md create mode 100644 static/project-model-portfolio.js create mode 100644 tests/browser_model_portfolio.cjs create mode 100644 tests/model_portfolio_fixture.cjs create mode 100644 tests/test_project_model_portfolio.cjs create mode 100644 tests/test_project_model_portfolio_http.py diff --git a/app.py b/app.py index 87c6c9a7..f818cc0b 100644 --- a/app.py +++ b/app.py @@ -919,6 +919,7 @@ def _get(self): '/workspaces.js': ('workspaces.js', 'text/javascript'), '/experiment-graph.js': ('experiment-graph.js', 'text/javascript'), '/project-progression.js': ('project-progression.js', 'text/javascript'), + '/project-model-portfolio.js': ('project-model-portfolio.js', 'text/javascript'), '/project-measurements.js': ('project-measurements.js', 'text/javascript'), '/project-measurements.css': ('project-measurements.css', 'text/css'), '/experiment-lineage.js': ('experiment-lineage.js', 'text/javascript'), diff --git a/docs/contracts/project-model-portfolio.md b/docs/contracts/project-model-portfolio.md new file mode 100644 index 00000000..d7a0a991 --- /dev/null +++ b/docs/contracts/project-model-portfolio.md @@ -0,0 +1,87 @@ +# Authored model organization + +The project workspace can display an adopted `.portfolio.json` note as an +authored guide to model fits/components. This is separate from **Experiment +progression**, which retains genuine Study/Trial ancestry. Group edges mean +authored grouping; they do not establish derivation, admission or scientific +acceptance. Relationship notes have explicit comparison/parameter meanings and +source excerpts. Missing historical Trial ancestry stays unknown. + +## Retained document + +`project_model_portfolio_v1` is a UTF-8 JSON note of at most1MiB with exactly: + +- `format`: the fixed string above; `project_id`: the owning32hex project ID; +- `title` (1–200characters), `description` (1–8,000characters), and + `authorship`: `retrospective`; +- `groups`: at most64records with exact fields `id`, `label`, `parent`, + `description`. IDs are unique, parent is null or another group, and grouping + is acyclic; +- `models`:1–1,000records with exact fields `id`, `label`, `group_id`, `aliases`, + `review`, `card`, `summary`. Each unique ID names an authored model entry, + not necessarily a complete generator, Trial or unique parameter state; +- `relationships`: at most2,000records with exact fields `from`, `to`, `kind`, + `explanation`, `evidence_model`, `excerpt`. All model references exist, endpoints + differ, and each ordered endpoint/kind tuple is unique. + +Group/model IDs contain1–160ASCII letters/digits or `._:/-`, beginning with a +letter/digit. Group labels are at most200characters, model labels/aliases500. +Each model has at most128unique aliases. A card reference has exact fields +`file_id` (64hex), `path` (a relative `notes/` path up to512characters without +empty/dot/dot-dot components), positive integer `revision`, and64hex `sha256`. +These are exact captured note references, never source-reading capabilities. + +Review is `source_reviewed` or `review_pending`. Reviewed summaries have exactly +six nonempty strings, each up to16,384characters: `method`, `purpose`, +`experiment`, `outcome`, `lesson`, `reproduction`. Pending summaries are null. +This status is an author's source-review statement, not numerical qualification. +Relationship kind is `comparison` or `parameter_derivation`; the latter points +from derived model to parameter source and must be acyclic. Explanation is at +most8,000characters and exact excerpt at most16,384. Comparisons are authored +context, not permission to combine incompatible metrics/populations. + +## Ownership and reads + +The ordinary project-file owner captures/revises the note, authenticates reads, +verifies captured UTF-8 bytes/hashes and retains previous revisions. There is no +new canonical family, SQL table, execution grant, recipe or mutation interface. +Existing `files_inspect` and `file_get` contracts are unchanged. Save/load keeps +the original guide and referenced note history. A fork whose new project/note +identities differ requires an explicitly re-authored guide; the reader refuses +foreign identities instead of retargeting them silently. + +`ProjectModelPortfolioDocument` owns the presentation-document decoder and +cross-reference/literal checks. `ProjectModelPortfolio` owns selection, +presentation and its pending reads. Its injected `read` must be the host's +already negotiated, current-authority, byte-verified project-file reader. +The host owns transport/hash validation and context identity. The controller +checks the exact model card ID/path/revision/SHA/project, and that all six +summary strings and that card's relationship excerpts occur literally in its +captured text before displaying a selected summary. Text is inert; no markup, +scripts, parameter loading, scientific scoring or source execution occurs. + +Only explicit actions read notes/cards. Discovery visits40adopted files per +page; no hidden scan/retry/polling occurs. The guide holds at most1,000models, +shows20per graph window with their ancestor groups, and reports exact total, +reviewed and filtered counts. Group/name/alias/review filters, paging and +textual relationship controls select the same entries as graph buttons. +Paragraphs wrap; exact source text remains in a disclosure. + +The host's binding/project/checkpoint/credential/generation context owns every +read. A context change, permission loss or teardown aborts pending work, clears +old content and restores focus to the host status when needed. Late replies +cannot publish into another context. Errors show their declared diagnostic and +clear old detail instead of displaying a previous model as the new selection. +Unsupported format or foreign-project binding is `unsupported`; malformed +document/reference/summary is `invalid`; transport/availability failures retain +their owning diagnostic. No success or empty-data fallback substitutes for a +failed read. + +## Verification scope + +Pure codec tests cover contract decisions using literal fixtures. Controlled +read and real-browser tests cover selection, unavailable/obsolete responses, +repeated invocation, focus, keyboard activation and wrapping on desktop/mobile. +Those fixtures do not establish real project capture or save/load. Native +acceptance additionally requires an actual adopted guide, exact original cards, +current authority, real project reads and retained checkpoint/custody evidence. diff --git a/static/project-files.css b/static/project-files.css index 515d9ca5..c18e2e30 100644 --- a/static/project-files.css +++ b/static/project-files.css @@ -10,3 +10,13 @@ #project-runtime-files,#project-runtime-summary,#project-runtime-pending,#project-runtime-layout,#project-runtime-status{overflow-wrap:anywhere;min-width:0}#project-runtime-layout,#project-runtime-status{white-space:pre-wrap;max-height:32rem;overflow:auto} #project-candidate-preparation-result,#project-candidate-preparation-status{overflow-wrap:anywhere;min-width:0}#project-candidate-preparation-result{white-space:pre-wrap;max-height:32rem;overflow:auto} + +#project-portfolio-source,#project-portfolio-detail,#project-portfolio-count{overflow-wrap:anywhere;min-width:0} +#project-portfolio-detail p{white-space:pre-wrap;max-width:85ch} +#project-portfolio-detail pre{white-space:pre-wrap;overflow-wrap:anywhere;max-height:32rem;overflow:auto} +#project-portfolio-detail details{margin:1rem 0} +#project-portfolio-detail h4{margin-bottom:.25rem} +#project-portfolio-detail h3{scroll-margin-top:1rem} +#project-portfolio-filter-form{display:grid;grid-template-columns:1fr 2fr 1fr;gap:.7rem;align-items:end} +#project-portfolio-filter-form button{justify-self:start} +@media(max-width:760px){#project-portfolio-filter-form{grid-template-columns:1fr}} diff --git a/static/project-files.html b/static/project-files.html index de9c6f1a..52f1ccca 100644 --- a/static/project-files.html +++ b/static/project-files.html @@ -14,6 +14,7 @@ + @@ -50,6 +51,7 @@

Project file

No unsubmitted edits.

+

Prepare a Candidate package or verifier

Choose a current Study, Candidate and captured CPU runtime. Preparation saves an inert Trial and exact sources; approval and Run admission remain separate actions.

diff --git a/static/project-files.js b/static/project-files.js index 8e57112d..3146719e 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -902,6 +902,17 @@ onContextFocusLost:()=>$('project-files-status').focus() }); window.addEventListener('pagehide',()=>projectProgression.clear()); + const projectPortfolio=window.ProjectModelPortfolio({ + root:$('project-model-portfolio'), + async read(action,args,signal){ + if(!m2Allowed(action))throw Error('Model organization inspection is unavailable for the selected project.'); + const value=await m2Request(action,args,epoch,{signal}); + if(value===null)throw new DOMException('Project selection changed.','AbortError'); + return value; + }, + onContextFocusLost:()=>$('project-files-status').focus() + }); + window.addEventListener('pagehide',()=>projectPortfolio.clear()); const projectMeasurements=window.ProjectMeasurements({ root:$('project-measurements'), async read(action,args,signal){ @@ -1167,6 +1178,8 @@ } const m2Allowed=action=>!busy&&!blocked&&state?.state==='open'&&selectedContracts.has('project_files.'+action); function m2Controls(){ + projectPortfolio.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), + project_id:state?.project?.project_id,enabled:m2Allowed('files_inspect')&&m2Allowed('file_get')}); projectProgression.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), project_id:state?.project?.project_id,enabled:m2Allowed('progression')&&m2Allowed('progression_origins')}); projectMeasurements.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), diff --git a/static/project-model-portfolio.js b/static/project-model-portfolio.js new file mode 100644 index 00000000..4f1b7fa3 --- /dev/null +++ b/static/project-model-portfolio.js @@ -0,0 +1,188 @@ +'use strict'; +// Authored organization is a retained note, not Trial lineage or qualification. +// This module owns its document contract, finite view and cancellable reads. +(() => { + const fields=['method','purpose','experiment','outcome','lesson','reproduction']; + const exact=(value,keys)=>value&&typeof value==='object'&&!Array.isArray(value)&& + Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); + const text=(value,limit)=>typeof value==='string'&&value.length>0&&value.length<=limit; + const identity=value=>text(value,160)&&/^[A-Za-z0-9][A-Za-z0-9._:/-]*$/.test(value); + const sha=value=>typeof value==='string'&&/^[a-f0-9]{64}$/.test(value); + const natural=value=>Number.isSafeInteger(value)&&value>=0; + function fail(message,code='invalid'){throw Object.assign(Error(message),{code});} + function noteRef(value){ + if(!exact(value,['file_id','path','revision','sha256'])||!sha(value.file_id)||!sha(value.sha256)|| + !natural(value.revision)||value.revision<1||!text(value.path,512)||!value.path.startsWith('notes/')|| + value.path.split('/').some(part=>!part||part==='.'||part==='..'))fail('Invalid captured model-note reference.'); + } + function decode(raw){ + if(typeof raw!=='string'||new TextEncoder().encode(raw).length>1048576)fail('Organization note exceeds its retained text limit.'); + let value;try{value=JSON.parse(raw);}catch{fail('Organization note is not valid JSON.');} + if(value?.format!=='project_model_portfolio_v1')fail('Unsupported model-organization document format.','unsupported'); + if(!exact(value,['format','project_id','title','description','authorship','groups','models','relationships'])|| + !/^[a-f0-9]{32}$/.test(value.project_id)||!text(value.title,200)||!text(value.description,8000)|| + value.authorship!=='retrospective'||!Array.isArray(value.groups)||value.groups.length>64|| + !Array.isArray(value.models)||!value.models.length||value.models.length>1000|| + !Array.isArray(value.relationships)||value.relationships.length>2000)fail('Invalid model-organization document.'); + const groups=new Map(),models=new Map(); + for(const group of value.groups){ + if(!exact(group,['id','label','parent','description'])||!identity(group.id)||groups.has(group.id)|| + !text(group.label,200)||!text(group.description,8000)||group.parent!==null&&!identity(group.parent))fail('Invalid or duplicate organization group.'); + groups.set(group.id,group); + } + for(const group of groups.values()){ + const visited=new Set([group.id]);let parent=group.parent; + while(parent!==null){if(!groups.has(parent))fail('Organization group parent is unavailable.'); + if(visited.has(parent))fail('Organization grouping contains a cycle.');visited.add(parent);parent=groups.get(parent).parent;} + } + for(const model of value.models){ + if(!exact(model,['id','label','group_id','aliases','review','card','summary'])||!identity(model.id)||models.has(model.id)|| + !text(model.label,500)||!groups.has(model.group_id)||!Array.isArray(model.aliases)||model.aliases.length>128|| + new Set(model.aliases).size!==model.aliases.length||model.aliases.some(alias=>!text(alias,500))|| + !['source_reviewed','review_pending'].includes(model.review))fail('Invalid or duplicate organized model.'); + noteRef(model.card); + if(model.review==='source_reviewed'){ + if(!exact(model.summary,fields)||fields.some(field=>!text(model.summary[field],16384)))fail('Reviewed model requires all six authored summary fields.'); + }else if(model.summary!==null)fail('Pending review cannot expose a reviewed summary.'); + models.set(model.id,model); + } + const pairs=new Set(); + for(const relation of value.relationships){ + if(!exact(relation,['from','to','kind','explanation','evidence_model','excerpt'])|| + !models.has(relation.from)||!models.has(relation.to)||relation.from===relation.to|| + !['comparison','parameter_derivation'].includes(relation.kind)||!text(relation.explanation,8000)|| + !models.has(relation.evidence_model)||!text(relation.excerpt,16384))fail('Invalid authored model relationship.'); + const key=JSON.stringify([relation.from,relation.to,relation.kind]);if(pairs.has(key))fail('Duplicate authored model relationship.');pairs.add(key); + } + const parents=new Map(),complete=new Set(),visiting=new Set(); + for(const relation of value.relationships)if(relation.kind==='parameter_derivation'){ + if(!parents.has(relation.from))parents.set(relation.from,[]);parents.get(relation.from).push(relation.to); + } + function visit(id){if(complete.has(id))return;if(visiting.has(id))fail('Authored parameter derivation contains a cycle.'); + visiting.add(id);for(const parent of parents.get(id)||[])visit(parent);visiting.delete(id);complete.add(id);} + for(const id of parents.keys())visit(id); + return value; + } + function checkCard(document,model,result){ + const ref=model.card,version=result?.version; + if(result?.project_id!==document.project_id||result.id!==ref.file_id||result.path!==ref.path||result.kind!=='note'|| + version?.deleted===true||version?.revision!==ref.revision||version.sha256!==ref.sha256||typeof version.text!=='string') + fail('Model card differs from its exact retained organization reference.'); + if(model.summary&&fields.some(field=>!version.text.includes(model.summary[field])))fail('Authored summary differs from the captured model card.'); + for(const relation of document.relationships)if(relation.evidence_model===model.id&&!version.text.includes(relation.excerpt)) + fail('Authored relationship excerpt differs from its captured model card.'); + return version.text; + } + window.ProjectModelPortfolioDocument=Object.freeze({decode,checkCard}); + const node=(tag,value)=>{const element=document.createElement(tag);if(value!==undefined)element.textContent=value;return element;}; + window.ProjectModelPortfolio=function({root,read,onContextFocusLost=()=>{}}){ + root.innerHTML='

Authored model organization

Browse a retained guide to models and their methods. Grouping and retrospective comparison notes are separate from recorded Trial ancestry and scientific qualification.

'; + const $=name=>root.querySelector('#project-portfolio-'+name); + let context=null,generation=0,pending=null,busy=false,notes=new Map(),noteOffset=0,noteTotal=0; + let guide=null,groups=new Map(),models=new Map(),applied={group:'',search:'',review:''},offset=0,selected=''; + const pageSize=20; + function controls(){ + const enabled=!!context?.enabled; + for(const name of ['notes','notes-more','note','load','group','search','review','prev','next'])$(name).disabled=!enabled||busy; + $('note').disabled ||= !notes.size;$('load').disabled ||= !notes.has($('note').value); + $('notes-more').hidden=noteOffset>=noteTotal;$('notes-more').disabled ||= noteOffset>=noteTotal; + $('prev').disabled ||= !guide||offset===0;$('next').disabled ||= !guide||offset+pageSize>=filtered().length; + for(const button of root.querySelectorAll('#project-portfolio-filter-form button,[data-lineage-node],#project-portfolio-detail button'))button.disabled=!enabled||busy; + root.setAttribute('aria-busy',String(busy)); + } + function clearGuide(){guide=null;groups.clear();models.clear();selected='';offset=0;applied={group:'',search:'',review:''};$('content').hidden=true;$('detail').replaceChildren();$('graph').replaceChildren();$('search').value='';$('review').value='';$('group').replaceChildren();} + function reset(){++generation;pending?.abort();pending=null;busy=false;notes.clear();noteOffset=noteTotal=0;$('note').replaceChildren();$('note-count').textContent='';$('status').textContent='';clearGuide();controls();} + async function perform(work){ + if(!context?.enabled||busy)return; + const ticket=++generation,key=context.key;pending=new AbortController();const signal=pending.signal;busy=true;controls(); + const active=()=>ticket===generation&&context?.enabled&&context.key===key&&!signal.aborted; + try{await work(active,signal);}catch(error){if(active()){$('detail').replaceChildren();$('status').textContent=(error.code?error.code+': ':'')+error.message;$('status').focus();}} + finally{if(active()){busy=false;pending=null;controls();}} + } + async function findNotes(more=false){ + return perform(async(active,signal)=>{ + const requested=more?noteOffset:0,value=await read('files_inspect',{offset:requested,limit:40},signal);if(!active())return; + if(value?.offset!==requested||value.limit!==40||!natural(value.total)||!Array.isArray(value.items)|| + value.items.length!==Math.min(40,Math.max(0,value.total-requested)))fail('Organization note page differs from its continuation.'); + if(!more){clearGuide();notes.clear();$('note').replaceChildren();} + for(const item of value.items)if(item.kind==='note'&&item.path?.endsWith('.portfolio.json')){ + if(!sha(item.id)||!natural(item.revision)||item.revision<1||!sha(item.expected_sha256)||notes.has(item.id))fail('Invalid organization note selection.'); + notes.set(item.id,structuredClone(item));const option=node('option',item.path+' · revision '+item.revision);option.value=item.id;$('note').append(option); + } + noteOffset=requested+value.items.length;noteTotal=value.total; + $('note-count').textContent=`${notes.size} organization notes found; inspected ${noteOffset} of ${noteTotal} adopted files.`; + $('status').textContent=notes.size?'Choose a retained organization note.':'No organization note on these pages. Adopt a .portfolio.json note or inspect the next page.'; + }); + } + function inGroup(model,wanted){let id=model.group_id;while(id!==null){if(id===wanted)return true;id=groups.get(id).parent;}return false;} + function filtered(){if(!guide)return [];return guide.models.filter(model=>(!applied.group||inGroup(model,applied.group))&& + (!applied.review||model.review===applied.review)&&(!applied.search||[model.label,...model.aliases].some(value=>value.toLocaleLowerCase().includes(applied.search))));} + function render(){ + if(!guide)return; + const focused=$('graph').contains(document.activeElement)?document.activeElement.id:null; + const matching=filtered(),shown=matching.slice(offset,offset+pageSize),included=new Set(); + for(const model of shown){let id=model.group_id;while(id!==null){included.add(id);id=groups.get(id).parent;}} + const graphNodes=[...included].map(id=>{const group=groups.get(id);return {id:'group/'+id,title:group.label,recordedAt:'',parent:group.parent===null?null:'group/'+group.parent,baseline:null,lines:['Authored method group']};}); + graphNodes.push(...shown.map(model=>({id:'model/'+model.id,title:model.label,recordedAt:'',parent:'group/'+model.group_id,baseline:null, + lines:[model.aliases.slice(0,3).join(' · ')||model.id,model.review==='source_reviewed'?'Source reviewed · authored summary':'Source review pending']}))); + $('graph').innerHTML=window.WorkspaceExperimentGraph.render(graphNodes,{selected:selected?'model/'+selected:'',zoom:1}).html; + for(const button of $('graph').querySelectorAll('[data-lineage-node]')){ + const value=button.dataset.lineageNode; + button.onclick=()=>{if(value.startsWith('model/'))void selectModel(value.slice(6));else{$('group').value=value.slice(6);applyFilters();}}; + } + $('count').textContent=`${guide.models.length} models in this guide; ${guide.models.filter(model=>model.review==='source_reviewed').length} source reviewed. ${matching.length} match the applied filters. Showing ${shown.length?offset+1:0}–${offset+shown.length}.`; + controls(); + if(focused){const replacement=$('graph').querySelector('#'+CSS.escape(focused)); + if(replacement)replacement.focus({preventScroll:true});else{$('count').tabIndex=-1;$('count').focus();}} + } + function applyFilters(){if(!guide||busy)return;applied={group:$('group').value,search:$('search').value.trim().toLocaleLowerCase(),review:$('review').value}; + if(applied.group&&!groups.has(applied.group))return;offset=0;selected='';$('detail').replaceChildren();render();$('status').textContent='Applied model filters.';} + async function loadGuide(){ + const choice=notes.get($('note').value);if(!choice)return; + return perform(async(active,signal)=>{ + clearGuide();const value=await read('file_get',{file_id:choice.id,revision:choice.revision},signal);if(!active())return; + if(value?.id!==choice.id||value.project_id!==context.project_id||value.kind!=='note'||value.path!==choice.path|| + value.version?.revision!==choice.revision||value.version.sha256!==choice.expected_sha256||value.version.deleted===true)fail('Organization note differs from the selected captured revision.'); + const parsed=decode(value.version.text); + if(parsed.project_id!==context.project_id)fail('This guide belongs to another project; re-author its captured note bindings.','unsupported'); + guide=parsed;groups=new Map(guide.groups.map(group=>[group.id,group]));models=new Map(guide.models.map(model=>[model.id,model])); + $('title').textContent=guide.title;$('description').textContent=guide.description; + $('source').textContent=`Retrospective authored guide · ${choice.path} · revision ${choice.revision} · SHA256 ${choice.expected_sha256}`; + const all=node('option','All groups');all.value='';$('group').append(all); + for(const group of guide.groups){const option=node('option',group.label);option.value=group.id;$('group').append(option);} + $('content').hidden=false;render();$('status').textContent='Loaded retained authored model organization.'; + }); + } + async function selectModel(id){ + const model=models.get(id);if(!model||busy)return; + return perform(async(active,signal)=>{ + if(!filtered().some(item=>item.id===id)){ + applied={group:'',search:'',review:''};$('group').value='';$('search').value='';$('review').value=''; + } + offset=Math.floor(filtered().findIndex(item=>item.id===id)/pageSize)*pageSize; + selected=id;$('detail').replaceChildren();render(); + const value=await read('file_get',{file_id:model.card.file_id,revision:model.card.revision},signal);if(!active())return; + const captured=checkCard(guide,model,value),heading=node('h3',model.label);heading.id='project-portfolio-model-heading';heading.tabIndex=-1;$('detail').append(heading); + $('detail').append(node('p',(model.aliases.join(' · ')||model.id)+' · '+(model.review==='source_reviewed'?'Source reviewed':'Source review pending'))); + if(model.summary){const labels={method:'Method',purpose:'Purpose',experiment:'Training and experiment',outcome:'Observed outcome',lesson:'Lesson',reproduction:'Reproduction'}; + for(const field of fields){$('detail').append(node('h4',labels[field]),node('p',model.summary[field]));}} + else $('detail').append(node('p','A source-reviewed interpretation has not been recorded for this fit. Its original captured card remains available below.')); + const relations=guide.relationships.filter(relation=>relation.from===id||relation.to===id); + if(relations.length){$('detail').append(node('h4','Authored model relationships')); + for(const relation of relations){const box=node('div'),other=models.get(relation.from===id?relation.to:relation.from),button=node('button', + (relation.kind==='comparison'?'Comparison: ':relation.from===id?'Parameters from: ':'Derived model: ')+other.label);button.type='button';button.onclick=()=>void selectModel(other.id); + box.append(button,node('p',relation.explanation));const evidence=node('details');evidence.append(node('summary','Exact retained relationship excerpt'),node('p',relation.excerpt)); + const source=node('button','Read evidence card: '+models.get(relation.evidence_model).label);source.type='button';source.onclick=()=>void selectModel(relation.evidence_model);evidence.append(source);box.append(evidence);$('detail').append(box);}} + const raw=node('details'),summary=node('summary','Exact captured card and original receipts');raw.append(summary,node('p',`${model.card.path} · revision ${model.card.revision} · SHA256 ${model.card.sha256}`),node('pre',captured));$('detail').append(raw); + $('status').textContent='Exact retained model summary and captured evidence.';heading.focus(); + }); + } + $('notes').onclick=()=>void findNotes();$('notes-more').onclick=()=>void findNotes(true); + $('note').onchange=()=>{clearGuide();controls();}; + $('source-form').onsubmit=event=>{event.preventDefault();void loadGuide();}; + $('filter-form').onsubmit=event=>{event.preventDefault();applyFilters();}; + for(const [name,direction] of [['prev',-1],['next',1]])$(name).onclick=()=>{if(busy||!guide)return;offset=Math.max(0,offset+direction*pageSize);selected='';$('detail').replaceChildren();render();}; + controls(); + return {setContext(value){if(value.key!==context?.key||context?.enabled&&!value.enabled){const owned=root.contains(document.activeElement);context=value;reset();if(owned)onContextFocusLost();}else context=value;root.hidden=!value.enabled;controls();},clear(){context=null;reset();root.hidden=true;}}; + }; +})(); diff --git a/tests/browser_model_portfolio.cjs b/tests/browser_model_portfolio.cjs new file mode 100644 index 00000000..851e650f --- /dev/null +++ b/tests/browser_model_portfolio.cjs @@ -0,0 +1,129 @@ +'use strict'; +// Real Chromium interaction/layout and controlled read-lifecycle evidence. +// This isolated static fixture does not claim native project persistence. +const fs=require('node:fs'),path=require('node:path'),http=require('node:http'),crypto=require('node:crypto'); +const {spawn}=require('node:child_process'); +const {waitForDebugger}=require('./browser_startup.cjs'),{cdpSession}=require('./browser_cdp_session.cjs'); +const output=path.resolve(process.argv[2]||''),engine=process.env.TULDOK_BROWSER_EXECUTABLE; +if(!process.argv[2]||!engine||fs.existsSync(output))throw Error('Explicit fresh output directory and browser executable required'); +fs.mkdirSync(output,{recursive:true,mode:0o700}); +const fixture=structuredClone(require('./model_portfolio_fixture.cjs')); +const hash=value=>crypto.createHash('sha256').update(value).digest('hex'); +const first=fixture.document.models[0]; +const oldMethod=first.summary.method;first.summary.method+=' '+('A physical pressure field with explicit case and work scope. '.repeat(60)); +first.label='First model '; +fixture.cards[0].version.text=fixture.cards[0].version.text.replace(oldMethod,first.summary.method); +fixture.cards[0].version.sha256=first.card.sha256=hash(fixture.cards[0].version.text); +fixture.cards[0].version.bytes=Buffer.byteLength(fixture.cards[0].version.text); +for(let index=0;index<20;index++){ + const model=structuredClone(fixture.document.models[1]);model.id='extra-'+index;model.label='Extra model '+index;model.aliases=[]; + model.card.path='notes/extra-'+index+'.md';model.card.file_id=hash(fixture.project+'\0note\0'+model.card.path); + fixture.document.models.push(model);fixture.cards.push({...structuredClone(fixture.cards[1]),id:model.card.file_id,path:model.card.path}); +} +fixture.guide.version.text=JSON.stringify(fixture.document); +fixture.guide.version.sha256=fixture.inspection.expected_sha256=fixture.inspection.observed_sha256=hash(fixture.guide.version.text); +fixture.guide.version.bytes=fixture.inspection.expected_bytes=fixture.inspection.observed_bytes=Buffer.byteLength(fixture.guide.version.text); +const inline=JSON.stringify(fixture).replaceAll('<','\\u003c'); +const html=`Authored organization fixture
`; +const staticRoot=path.join(__dirname,'../static'),allowed=new Map([ + ['/project-files.css',['project-files.css','text/css']],['/experiment-lineage.css',['experiment-lineage.css','text/css']], + ['/experiment-graph.js',['experiment-graph.js','text/javascript']],['/project-model-portfolio.js',['project-model-portfolio.js','text/javascript']]]); +const server=http.createServer((request,response)=>{ + if(request.url==='/'){response.writeHead(200,{'Content-Type':'text/html; charset=utf-8'});response.end(html);return;} + const asset=allowed.get(request.url);if(!asset){response.writeHead(404);response.end();return;} + response.writeHead(200,{'Content-Type':asset[1]});response.end(fs.readFileSync(path.join(staticRoot,asset[0]))); +}); +let child,session; +const report={status:'starting',engine,verification_only:true,native_persistence_claim:false,graphics:'Headless Chromium with GPU disabled; no model or rendering speed claim.',checks:[],errors:[],browser_stopped:false}; +const sourceFiles=['app.py','static/project-files.css','static/project-files.html','static/project-files.js','static/project-model-portfolio.js', + 'docs/contracts/project-model-portfolio.md','tests/model_portfolio_fixture.cjs','tests/test_project_model_portfolio.cjs', + 'tests/browser_model_portfolio.cjs','tests/test_project_model_portfolio_http.py','tests/test_workspace_project_files.cjs','tests/test_workspace_project_files_m4.cjs']; +report.source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); +const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); +async function main(){ + await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve)); + const origin='http://127.0.0.1:'+server.address().port; + try{ + const flags=['--headless','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,1000','--user-data-dir='+path.join(output,'profile'),'about:blank']; + report.flags=flags;report.origin=origin; + const log=fs.openSync(path.join(output,'browser.log'),'w');child=spawn(engine,flags,{stdio:['ignore',log,log]});fs.closeSync(log); + const port=await waitForDebugger(child,path.join(output,'profile/DevToolsActivePort')); + const targets=await(await fetch('http://127.0.0.1:'+port+'/json/list')).json(),target=targets.find(target=>target.type==='page'); + const socket=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{socket.addEventListener('open',resolve,{once:true});socket.addEventListener('error',reject,{once:true});}); + session=cdpSession(socket,{onEvent:event=>{if(event.method==='Runtime.exceptionThrown')report.errors.push(event.params.exceptionDetails.text);}}); + const send=session.send; + await send('Page.enable');await send('Runtime.enable'); + async function evaluate(expression){const result=await send('Runtime.evaluate',{expression,returnByValue:true,awaitPromise:true});if(result.exceptionDetails)throw Error(result.exceptionDetails.exception?.description||result.exceptionDetails.text);return result.result.value;} + async function until(expression){const end=Date.now()+15000;while(Date.now(){const element=document.querySelector(${JSON.stringify(selector)});if(!element||element.disabled)throw Error('Unavailable fixture control');element.scrollIntoView({block:'center'});element.focus();})()`); + await send('Input.dispatchKeyEvent',{type:'keyDown',key:'Enter',code:'Enter',windowsVirtualKeyCode:13,text:'\r'});await send('Input.dispatchKeyEvent',{type:'keyUp',key:'Enter',code:'Enter',windowsVirtualKeyCode:13});} + async function fill(selector,value){await evaluate(`(()=>{const element=document.querySelector(${JSON.stringify(selector)});element.value=${JSON.stringify(value)};element.dispatchEvent(new Event('input',{bubbles:true}));element.dispatchEvent(new Event('change',{bubbles:true}));})()`);} + async function screenshot(name){const shot=await send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,name),Buffer.from(shot.data,'base64'));} + async function load(){await keyClick('#project-portfolio-notes');await until("document.querySelector('#project-portfolio-note').options.length===1");await keyClick('#project-portfolio-load');await until("document.querySelector('#project-portfolio-status').textContent==='Loaded retained authored model organization.'");} + async function selectFirst(){await keyClick('[data-lineage-node="model/fit-first"]');await until("document.querySelector('#project-portfolio-status').textContent==='Exact retained model summary and captured evidence.'");} + await send('Page.navigate',{url:origin+'/'});await until("!!window.fixtureUI"); + await load();await check('readCalls.length===2','Guide loading reads no model card or payload'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('22 models in this guide; 1 source reviewed')",'Exact complete and reviewed guide counts'); + const point=await evaluate("(()=>{const element=document.querySelector('[data-lineage-node=\"group/pressure\"]');element.scrollIntoView({block:'center'});const box=element.getBoundingClientRect();return {x:box.x+box.width/2,y:box.y+box.height/2};})()"); + await send('Input.dispatchMouseEvent',{type:'mousePressed',button:'left',clickCount:1,...point});await send('Input.dispatchMouseEvent',{type:'mouseReleased',button:'left',clickCount:1,...point}); + await check("document.querySelector('#project-portfolio-group').value==='pressure'&&document.activeElement.dataset.lineageNode==='group/pressure'",'Pointer group activation applies the group and retains visible keyboard focus'); + await check("!document.querySelector('#root img')&&!window.injected",'Authored graph labels remain inert text'); + await keyClick('#project-portfolio-next');await check("document.querySelector('#project-portfolio-count').textContent.includes('Showing 21–22.')",'Bounded model paging uses the full population'); + await keyClick('#project-portfolio-prev');await selectFirst(); + await check("document.activeElement.id==='project-portfolio-model-heading'",'Keyboard selection moves focus to selected summary'); + await check("document.querySelector('#project-portfolio-detail').innerText.includes(fixtureData.document.models[0].summary.method)",'Complete long method paragraph is readable retained text'); + await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Desktop detail wraps without horizontal overflow'); + await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('desktop-summary.png'); + await send('Emulation.setDeviceMetricsOverride',{width:390,height:844,deviceScaleFactor:1,mobile:true}); + await check("document.querySelector('#project-portfolio-detail').scrollWidth<=document.querySelector('#project-portfolio-detail').clientWidth",'Mobile detail wraps without horizontal overflow'); + await evaluate("document.querySelector('#project-portfolio-model-heading').scrollIntoView({block:'start'})");await screenshot('mobile-summary.png'); + await send('Emulation.clearDeviceMetricsOverride'); + await fill('#project-portfolio-search','C107');await keyClick('#project-portfolio-filter-form button'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('1 match the applied filters')&&document.querySelector('#project-portfolio-next').disabled",'Alias filtering has exact counts and paging'); + await selectFirst();await keyClick('#project-portfolio-detail button');await until("document.querySelector('#project-portfolio-model-heading')?.textContent==='Second model'"); + await check("document.querySelector('#project-portfolio-detail').innerText.includes('A source-reviewed interpretation has not been recorded')",'Pending review does not invent a method summary'); + await check("document.querySelector('[data-lineage-node=\"model/fit-second\"]').getAttribute('aria-pressed')==='true'",'Off-filter relationship navigation brings selected model into the graph window'); + await selectFirst();await evaluate('window.failNext=true');await keyClick('#project-portfolio-detail button'); + await until("document.querySelector('#project-portfolio-status').textContent==='unavailable: Retained card unavailable.'"); + await check("document.querySelector('#project-portfolio-detail').children.length===0",'Unavailable card clears the previous model detail'); + await evaluate('window.corruptNext=true');await keyClick('[data-lineage-node="model/fit-first"]'); + await until("document.querySelector('#project-portfolio-status').textContent==='invalid: Model card differs from its exact retained organization reference.'"); + await check("document.querySelector('#project-portfolio-detail').children.length===0",'Foreign card revision is refused without stale detail'); + await evaluate('window.holdNext=true');await keyClick('[data-lineage-node="model/fit-first"]');await until('!!window.releaseHeld'); + await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:true});window.releaseHeld()"); + await check("window.heldSignal.aborted&&document.querySelector('#project-portfolio-content').hidden",'Context replacement aborts and clears pending view'); + await load();await check("document.querySelector('#project-portfolio-detail').children.length===0",'Reload after supersession starts without old detail'); + await selectFirst();await evaluate("fixtureUI.setContext({key:'two',project_id:fixtureData.project,enabled:false})"); + await check("document.querySelector('#root').hidden&&document.querySelector('#project-portfolio-detail').children.length===0&&document.activeElement.id==='outside'",'Permission loss clears private view and restores host focus'); + await evaluate("fixtureUI.setContext({key:'three',project_id:fixtureData.project,enabled:true})");await load(); + await fill('#project-portfolio-search','No match');await keyClick('#project-portfolio-filter-form button'); + await check("document.querySelector('#project-portfolio-count').textContent.includes('0 match the applied filters. Showing 0–0.')",'Empty filter result retains exact zero denominator'); + await check('readCalls.every(call=>[\'files_inspect\',\'file_get\'].includes(call.action))','All interactions use read-only project-file actions'); + if(report.errors.length)throw Error('Browser runtime exception');report.status='passed_authored_portfolio_real_browser_component'; + }catch(error){report.status='failed_authored_portfolio_real_browser_component';report.error=String(error); + if(session){try{const state=await session.send('Runtime.evaluate',{expression:'document.body.innerText',returnByValue:true});report.failure_page=state.result?.value; + const shot=await session.send('Page.captureScreenshot',{format:'png',captureBeyondViewport:false});fs.writeFileSync(path.join(output,'failed-page.png'),Buffer.from(shot.data,'base64'));}catch(captureError){report.failure_capture_error=String(captureError);}} + throw error;} + finally{session?.close();if(child&&child.exitCode===null&&child.signalCode===null){child.kill('SIGTERM');for(let i=0;i<100&&child.exitCode===null&&child.signalCode===null;i++)await pause(50);if(child.exitCode===null&&child.signalCode===null)child.kill('SIGKILL');} + report.browser_stopped=!child||child.exitCode!==null||child.signalCode!==null;await new Promise(resolve=>server.close(resolve)); + report.end_source_sha256=Object.fromEntries(sourceFiles.map(name=>[name,hash(fs.readFileSync(path.join(__dirname,'..',name)))])); + report.source_stable_through_tests=JSON.stringify(report.source_sha256)===JSON.stringify(report.end_source_sha256); + if(!report.source_stable_through_tests){report.status='failed_authored_portfolio_real_browser_component';report.errors.push('Source changed during browser qualification');process.exitCode=1;} + report.producer_sha256=hash(fs.readFileSync(__filename));fs.writeFileSync(path.join(output,'report.json'),JSON.stringify(report,null,2)+'\n');} + console.log(report.status,report.checks.length,'checks'); +} +main().catch(error=>{console.error(String(error));process.exitCode=1;}); diff --git a/tests/model_portfolio_fixture.cjs b/tests/model_portfolio_fixture.cjs new file mode 100644 index 00000000..cb63135a --- /dev/null +++ b/tests/model_portfolio_fixture.cjs @@ -0,0 +1,16 @@ +'use strict'; +// Literal presentation-document fixtures, independent of the decoder. +const crypto=require('node:crypto'); +const hash=value=>crypto.createHash('sha256').update(value).digest('hex'); +const project='a'.repeat(32),path='notes/00-methods.portfolio.json'; +const summary={method:'A small pressure correction model.',purpose:'Measure one pressure correction.',experiment:'Two retained training scenes.',outcome:'Retained diagnostic result; not a rollout qualification.',lesson:'Compare work and accuracy on the same cases.',reproduction:'Use the retained source recipe; clean retraining remains pending.'}; +const cardText=Object.values(summary).join('\n\n')+'\nOriginal receipt bytes remain here.\n'; +const ref=(path,text)=>({file_id:hash(project+'\0note\0'+path),path,revision:2,sha256:hash(text)}); +const first={id:'fit-first',label:'First model',group_id:'pressure',aliases:['C107'],review:'source_reviewed',card:ref('notes/first.md',cardText),summary}; +const second={id:'fit-second',label:'Second model',group_id:'pressure',aliases:['C106'],review:'review_pending',card:ref('notes/second.md','Original pending card.\n'),summary:null}; +const document={format:'project_model_portfolio_v1',project_id:project,title:'Retained fit guide',description:'Authored retrospective organization; original Trial history is unchanged.',authorship:'retrospective',groups:[{id:'pressure',label:'Pressure methods',parent:null,description:'Pressure requests, not complete simulation steps.'}],models:[first,second],relationships:[{from:first.id,to:second.id,kind:'comparison',explanation:'Authored comparison with the earlier control.',evidence_model:first.id,excerpt:summary.lesson}]}; +const result=(model,text)=>({id:model.card.file_id,project_id:project,kind:'note',path:model.card.path,version:{revision:2,sha256:hash(text),bytes:Buffer.byteLength(text),text,actor_id:'b'.repeat(32),adopted_at:'2026-10-25T00:00:00+00:00'}}); +const cards=[result(first,cardText),result(second,'Original pending card.\n')]; +const raw=JSON.stringify(document),guideRef=ref(path,raw),guide={...result({card:guideRef},raw),version:{...result({card:guideRef},raw).version,revision:1}}; +const inspection={id:guideRef.file_id,kind:'note',path,revision:1,expected_sha256:guideRef.sha256,expected_bytes:Buffer.byteLength(raw),observed_sha256:guideRef.sha256,observed_bytes:Buffer.byteLength(raw),state:'unchanged',reason:null}; +module.exports={project,document,summary,cards,guide,inspection}; diff --git a/tests/test_project_model_portfolio.cjs b/tests/test_project_model_portfolio.cjs new file mode 100644 index 00000000..704e4035 --- /dev/null +++ b/tests/test_project_model_portfolio.cjs @@ -0,0 +1,47 @@ +'use strict'; +const assert=require('node:assert/strict'),fs=require('node:fs'),vm=require('node:vm'),path=require('node:path'); +const fixture=require('./model_portfolio_fixture.cjs'),window={}; +vm.runInNewContext(fs.readFileSync(path.join(__dirname,'../static/project-model-portfolio.js'),'utf8'),{window,TextEncoder}); +const {decode,checkCard}=window.ProjectModelPortfolioDocument; +const valid=()=>structuredClone(fixture.document),parse=value=>decode(JSON.stringify(value)); +assert.deepEqual(JSON.parse(JSON.stringify(parse(valid()))),fixture.document); +let checks=1; +function bad(change,message,code='invalid'){ + const value=valid();change(value); + assert.throws(()=>parse(value),error=>error.code===code&&error.message===message);checks++; +} +bad(d=>{d.format='other';},'Unsupported model-organization document format.','unsupported'); +bad(d=>{d.extra=true;},'Invalid model-organization document.'); +bad(d=>{d.models=[];},'Invalid model-organization document.'); +bad(d=>{d.groups.push(structuredClone(d.groups[0]));},'Invalid or duplicate organization group.'); +bad(d=>{d.groups[0].parent='missing';},'Organization group parent is unavailable.'); +bad(d=>{d.groups[0].parent='pressure';},'Organization grouping contains a cycle.'); +bad(d=>{d.models.push(structuredClone(d.models[0]));},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].group_id='missing';},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].aliases.push('C107');},'Invalid or duplicate organized model.'); +bad(d=>{d.models[0].review='qualified';},'Invalid or duplicate organized model.'); +bad(d=>{delete d.models[0].summary.method;},'Reviewed model requires all six authored summary fields.'); +bad(d=>{d.models[1].summary={};},'Pending review cannot expose a reviewed summary.'); +bad(d=>{d.models[0].card.path='notes/../private';},'Invalid captured model-note reference.'); +bad(d=>{d.models[0].card.revision=0;},'Invalid captured model-note reference.'); +bad(d=>{d.relationships[0].to='unknown';},'Invalid authored model relationship.'); +bad(d=>{d.relationships[0].to=d.relationships[0].from;},'Invalid authored model relationship.'); +bad(d=>{d.relationships[0].kind='trial_parent';},'Invalid authored model relationship.'); +bad(d=>{d.relationships.push(structuredClone(d.relationships[0]));},'Duplicate authored model relationship.'); +bad(d=>{const a=d.relationships[0];a.kind='parameter_derivation';d.relationships.push({...a,from:a.to,to:a.from});},'Authored parameter derivation contains a cycle.'); +assert.throws(()=>decode('not JSON'),error=>error.code==='invalid'&&error.message==='Organization note is not valid JSON.');checks++; +assert.throws(()=>decode(' '.repeat(1048577)),error=>error.code==='invalid'&&error.message==='Organization note exceeds its retained text limit.');checks++; +assert.equal(checkCard(valid(),fixture.document.models[0],fixture.cards[0]),fixture.cards[0].version.text);checks++; +for(const change of [r=>{r.project_id='c'.repeat(32);},r=>{r.id='d'.repeat(64);},r=>{r.path='notes/other.md';},r=>{r.version.revision=3;},r=>{r.version.sha256='e'.repeat(64);},r=>{r.version.deleted=true;}]){ + const result=structuredClone(fixture.cards[0]);change(result); + assert.throws(()=>checkCard(valid(),fixture.document.models[0],result),error=>error.code==='invalid'&&error.message==='Model card differs from its exact retained organization reference.');checks++; +} +{ + const value=valid();value.models[0].summary.method='Not present in the captured card.'; + assert.throws(()=>checkCard(value,value.models[0],fixture.cards[0]),error=>error.message==='Authored summary differs from the captured model card.');checks++; +} +{ + const value=valid();value.relationships[0].excerpt='Not present in the captured card.'; + assert.throws(()=>checkCard(value,value.models[0],fixture.cards[0]),error=>error.message==='Authored relationship excerpt differs from its captured model card.');checks++; +} +console.log('PASS authored portfolio document:',checks,'literal contract/reference decisions; no native capture or browser claim.'); diff --git a/tests/test_project_model_portfolio_http.py b/tests/test_project_model_portfolio_http.py new file mode 100644 index 00000000..8fbda68f --- /dev/null +++ b/tests/test_project_model_portfolio_http.py @@ -0,0 +1,84 @@ +"""Real authenticated note transport/cold reload; no model or recipe execution.""" +import hashlib +import json +import os +from pathlib import Path +import unittest +from unittest.mock import patch + +import project_workspace +import test_project_files_http as m1 +import test_project_files_m2_http as m2 + +ROOT = Path(__file__).resolve().parents[1] + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class ModelPortfolioHTTP(unittest.TestCase): + cleanup_owned = m1.ProjectFilesHTTP.cleanup_owned + select_host = m1.ProjectFilesHTTP.select_host + request = m1.ProjectFilesHTTP.request + state = m1.ProjectFilesHTTP.state + envelope = m1.ProjectFilesHTTP.envelope + action = m1.ProjectFilesHTTP.action + create = m1.ProjectFilesHTTP.create + arguments = m2.ProjectFilesM2HTTP.arguments + transport = m2.ProjectFilesM2HTTP.transport + call = m2.ProjectFilesM2HTTP.call + + def setUp(self): + self.operations = [] + m1.ProjectFilesHTTP.setUp(self) + gate = patch.object(project_workspace, 'M4_ENABLED', True) + gate.start() + self.addCleanup(gate.stop) + + def test_native_static_module_exact_note_revision_and_cold_reload(self): + status, html = self.request('/project-files') + self.assertEqual(status, 200) + self.assertIn(b'/project-model-portfolio.js', html) + self.assertIn(b'id="project-model-portfolio"', html) + status, script = self.request('/project-model-portfolio.js') + self.assertEqual(status, 200) + self.assertEqual(script, (ROOT/'static/project-model-portfolio.js').read_bytes()) + opened = self.create() + root = self.external/'alpha' + (root/'notes').mkdir() + paragraphs = {key: 'Literal retained '+key+' paragraph.' for key in + ('method','purpose','experiment','outcome','lesson','reproduction')} + card_text = '\n\n'.join(paragraphs.values())+'\nOriginal evidence remains.\n' + (root/'notes/model.md').write_text(card_text) + card = self.call('file_adopt',kind='note',path='notes/model.md',expected_revision=0, + expected_sha256=hashlib.sha256(card_text.encode()).hexdigest()) + guide = dict(format='project_model_portfolio_v1',project_id=opened['project']['project_id'], + title='Authored retained fixture',description='Synthetic presentation fixture; no scientific quality claim.', + authorship='retrospective',groups=[dict(id='pressure',label='Pressure methods',parent=None,description='An authored group.')], + models=[dict(id='fixture-fit',label='One retained fit',group_id='pressure',aliases=['Fixture'],review='source_reviewed', + card=dict(file_id=card['id'],path='notes/model.md',revision=1,sha256=card['sha256']),summary=paragraphs)],relationships=[]) + raw=(json.dumps(guide,ensure_ascii=False)+'\n').encode() + (root/'notes/00-fixture.portfolio.json').write_bytes(raw) + saved=self.call('file_adopt',kind='note',path='notes/00-fixture.portfolio.json',expected_revision=0, + expected_sha256=hashlib.sha256(raw).hexdigest()) + observed=self.call('file_get',file_id=saved['id'],revision=1) + self.assertEqual(observed['version']['text'].encode(),raw) + self.assertEqual(json.loads(observed['version']['text']),guide) + changed='A later authored card revision.\n' + (root/'notes/model.md').write_text(changed) + self.call('file_adopt',kind='note',path='notes/model.md',expected_revision=1, + expected_sha256=hashlib.sha256(changed.encode()).hexdigest()) + self.assertEqual(self.call('file_get',file_id=card['id'],revision=1)['version']['text'],card_text) + self.assertEqual(self.call('file_get',file_id=card['id'])['version']['text'],changed) + self.action('save') + expected=(root/'.tuldok/project.json').read_bytes() + self.action('close') + cold=os.environ.get('TULDOK_PROJECT_FILES_COLD_DATABASE_URL') + self.assertTrue(cold,'Independent cold role required for retained-note acceptance') + self.select_host(self.path/'cold-control',{'fixture':self.external},cold) + self.action('open',root='fixture',directory='alpha') + self.assertEqual((root/'.tuldok/project.json').read_bytes(),expected) + self.assertEqual(self.call('file_get',file_id=saved['id'],revision=1)['version']['text'].encode(),raw) + retained=self.call('file_get',file_id=card['id'],revision=1) + self.assertEqual(retained['version']['text'],card_text) + self.assertEqual(retained['project_id'],guide['project_id']) + self.assertEqual(self.call('file_get',file_id=card['id'])['version']['text'],changed) + self.action('close') diff --git a/tests/test_workspace_project_files.cjs b/tests/test_workspace_project_files.cjs index 31339b01..86582638 100644 --- a/tests/test_workspace_project_files.cjs +++ b/tests/test_workspace_project_files.cjs @@ -38,6 +38,7 @@ function mount(initial=[reply(state()),reply(records())],capabilityValue=capabil context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); context.ProjectMeasurements=()=>({setContext(){},clear(){}}); + context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,events,discoveries,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, diff --git a/tests/test_workspace_project_files_m4.cjs b/tests/test_workspace_project_files_m4.cjs index dbdb85b7..80e343ca 100644 --- a/tests/test_workspace_project_files_m4.cjs +++ b/tests/test_workspace_project_files_m4.cjs @@ -43,6 +43,7 @@ function mount(capabilityValue=capabilities){ context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); context.ProjectMeasurements=()=>({setContext(){},clear(){}}); + context.ProjectModelPortfolio=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, From 00a02ee25cb30d254c059a9b372036f9f6f36696 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Mon, 26 Oct 2026 08:36:28 -0700 Subject: [PATCH 33/33] perf(project): read retained measurement sources by validated family Chart-source lookup copied the entire retained graph and decoded material again. Read the already validated measurement family through the existing canonical freshness guard; keep candidate and packet validation at their original owners. 122 distinct filesystem, native SQL, permission, import, receipt and client checks pass across two retained fixtures. Fixed-checkpoint lookup removes 19-20 CPU seconds of copying/decoding; native endpoint and whole-Open latency remain separate pending qualifications. No global cache or schema change. --- project_measurements.py | 5 +- tests/test_project_measurement_reads.py | 106 ++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 tests/test_project_measurement_reads.py diff --git a/project_measurements.py b/project_measurements.py index d1fed0de..e64a307e 100644 --- a/project_measurements.py +++ b/project_measurements.py @@ -138,7 +138,10 @@ def __init__(self, session): self.access, self.db = session.access, session.db def _documents(self): - return decode_records(self.session.snapshot['records']) + # Session adoption/publication already validates the complete candidate. + # Verify current canonical bytes, then own only the measurement family. + records = self.session.check_records(FAMILY)['records'] + return {item['id']: item['document'] for item in records} def import_packet(self, raw, source, expected_checkpoint, key): """Called only after SourceAdapter's explicit current rooted capture.""" diff --git a/tests/test_project_measurement_reads.py b/tests/test_project_measurement_reads.py new file mode 100644 index 00000000..1365d63a --- /dev/null +++ b/tests/test_project_measurement_reads.py @@ -0,0 +1,106 @@ +"""Read the adopted measurement family without cloning unrelated retained facts. + +Real canonical files and real session freshness guards; no SQL reconstruction +or packet import is claimed by this focused fixture. +""" +import copy +import json +import os +from pathlib import Path +import tempfile +import threading +import unittest +from unittest.mock import patch + +import project_measurements +import project_session +from project_projection import validate_project_records +from project_store import ProjectStore, ProjectStoreError +from test_project_receipt_validation import candidate, PROJECT + + +class MeasurementReadTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory(prefix='tuldok-measurement-read-') + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name)/'project' + self.store = ProjectStore.create(self.root, 'Measurement read fixture', 'new', 'a'*64) + self.addCleanup(self.store.close) + self.records = candidate() + self.records.append(dict(family='project_artifact', id='f'*64, + document=dict(format='project_artifact_v1', sha256='f'*64, bytes=7))) + validate_project_records(self.records, project_id=PROJECT, artifacts=None) + snapshot = self.store.commit(self.records, self.store.summary()['checkpoint'], 'seed', 'b'*64) + self.session = object.__new__(project_session.ProjectSession) + self.session.store, self.session.available = self.store, True + self.session._snapshot, self.session.lock = copy.deepcopy(snapshot), threading.RLock() + self.owner = object.__new__(project_measurements.ProjectMeasurements) + self.owner.session = self.session + self.expected = {r['id']: copy.deepcopy(r['document']) for r in self.records + if r['family'] == project_measurements.FAMILY} + + def test_documents_are_literal_independent_and_do_not_copy_unrelated_graph(self): + original = project_session.copy_validated_json + unrelated = [] + def observe(value): + if isinstance(value, dict) and 'records' in value: + unrelated.extend(r['id'] for r in value['records'] if r['id'] == 'f'*64) + return original(value) + with patch.object(project_session, 'copy_validated_json', observe): + result = self.owner._documents() + self.assertEqual(result, self.expected) + self.assertEqual(unrelated, [], 'Measurement read copied an unrelated retained envelope') + next(iter(result.values()))['source']['path'] = 'caller-edited.json' + self.assertEqual(self.owner._documents(), self.expected) + self.assertEqual(self.session._snapshot, self.store.snapshot()) + + def test_lookup_reads_every_canonical_file_without_redecoding_adopted_material(self): + with patch.object(self.store, '_record_file', wraps=self.store._record_file) as reads, \ + patch.object(project_measurements, 'material_records', + side_effect=AssertionError('Read-only lookup redecoded adopted material')): + self.assertEqual(self.owner._documents(), self.expected) + self.assertEqual(reads.call_count, len(self.records), 'All current files must still be read and hashed') + + def test_unavailable_projection_fails_before_any_store_read(self): + self.session.available = False + with patch.object(self.store, 'summary', side_effect=AssertionError('Unavailable store read')): + with self.assertRaises(project_session.ProjectSessionError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'projection_unavailable') + + def test_other_family_publication_requires_reopen(self): + changed = copy.deepcopy(self.records) + changed[-1]['document']['bytes'] = 8 + self.store.commit(changed, self.store.summary()['checkpoint'], 'other', 'c'*64) + with self.assertRaises(project_session.ProjectSessionError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'stale_project') + + def test_same_length_unselected_corruption_with_restored_mtime_is_rejected(self): + inventory = json.loads((self.root/'.tuldok/project.json').read_bytes())['inventory'] + entry = next(r for r in inventory if r['id'] == 'f'*64) + path = self.root/'.tuldok'/entry['path'] + before, info = path.read_bytes(), path.stat() + changed = before.replace(b'"bytes": 7', b'"bytes": 8') + self.assertNotEqual(changed, before) + self.assertEqual(len(changed), len(before)) + path.chmod(0o600) + path.write_bytes(changed) + os.utime(path, ns=(info.st_atime_ns, info.st_mtime_ns)) + with self.assertRaises(ProjectStoreError) as caught: + self.owner._documents() + self.assertEqual(caught.exception.code, 'stale') + + def test_family_can_be_empty_and_returned_snapshot_contract_stays_complete(self): + records = [r for r in self.records if r['family'] not in (project_measurements.FAMILY,'project_operation')] + validate_project_records(records, project_id=PROJECT, artifacts=None) + self.session._snapshot = self.store.commit(records, self.store.summary()['checkpoint'], 'empty', 'd'*64) + self.assertEqual(self.owner._documents(), {}) + snapshot = self.session.snapshot + self.assertEqual(snapshot, self.store.snapshot()) + snapshot['records'].clear() + self.assertEqual(self.session.snapshot, self.store.snapshot()) + + +if __name__ == '__main__': + unittest.main()