From c64b3a9d02a01f92ea1756ca995cc7a99a423750 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 12:26:05 -0700 Subject: [PATCH 01/15] feat(project): inspect retained scientific fields in native projects Share the existing numeric field inspector between workbench and native project readers. Pin native slices to the selected project checkpoint and source content, and clear cancelled or superseded reads without importing execution authority. --- agent_api.py | 2 + app.py | 2 + docs/contracts/scientific-fields.md | 36 ++ project_data.py | 8 + project_dispatch.py | 4 +- static/project-files.html | 3 + static/project-files.js | 524 +++++++++++++++------------- static/scientific-fields.js | 86 +---- static/scientific-inspector.css | 2 + static/scientific-inspector.js | 84 +++++ static/workbench.css | 2 - static/workbench.html | 6 +- tests/browser_scientific_fields.cjs | 2 +- tests/test_project_files_clients.py | 12 + tests/test_project_files_m2_http.py | 33 +- tests/test_project_sources.py | 24 ++ tuldok_client.py | 4 + 17 files changed, 502 insertions(+), 332 deletions(-) create mode 100644 static/scientific-inspector.css create mode 100644 static/scientific-inspector.js diff --git a/agent_api.py b/agent_api.py index 818a38b2..2da06bbd 100644 --- a/agent_api.py +++ b/agent_api.py @@ -868,6 +868,8 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non # Actual data and its authored histories. Inline source imports retain the # existing decoded-byte bound; compound sources use their existing parsers. _project_file_operation('data_query', 'GET', '/data', domain='data.query') +_project_file_operation('data_scientific', 'GET', '/data/{record_id}/scientific', + {'content_hash': SHA}, domain='data.scientific') _project_file_operation('data_get', 'GET', '/data/{record_id}', {'record_id': ID}) _project_file_operation('data_availability', 'GET', '/data/{record_id}/availability', {'record_id': ID}) for _action, _suffix in [('data_history', 'history'), ('data_responses', 'responses'), diff --git a/app.py b/app.py index c7b7d25e..5bc42e31 100644 --- a/app.py +++ b/app.py @@ -899,6 +899,8 @@ def _get(self): '/pointclouds.js': ('pointclouds.js', 'text/javascript'), '/volume-editor.js': ('volume-editor.js', 'text/javascript'), '/scientific-fields.js': ('scientific-fields.js', 'text/javascript'), + '/scientific-inspector.js': ('scientific-inspector.js', 'text/javascript'), + '/scientific-inspector.css': ('scientific-inspector.css', 'text/css'), '/audio-editor.js': ('audio-editor.js', 'text/javascript'), '/audio-editor.css': ('audio-editor.css', 'text/css'), '/meshes.css': ('meshes.css', 'text/css'), diff --git a/docs/contracts/scientific-fields.md b/docs/contracts/scientific-fields.md index b176215a..cf31a01a 100644 --- a/docs/contracts/scientific-fields.md +++ b/docs/contracts/scientific-fields.md @@ -158,3 +158,39 @@ and restart, rollback custody, changed/FIFO source refusal, and real HTTP project import and slice inspection. Native producer observations and independent NumPy qualification are recorded separately; these tests are not physics, training, or model acceptance. + +## Native external-project inspection + +`project_files.data_scientific/1` reads the same bounded numeric slice through +the native project session. It requires the current `binding`, canonical +`checkpoint`, `record_id` and exact `content_hash`; `frame`, `field`, `axis` and +`index` retain the existing inspector's selection grammar. ProjectData checks +current inspect authority, the project session and source identity before using +ScientificAssets. The scientific owner verifies retained artifact bytes and +metadata. Stale bindings/checkpoints, changed content, missing/corrupt artifacts +and revoked credentials refuse; no source-path or cached-array fallback runs. +The read changes no project records and grants no execution authority. + +The native project explorer mounts the same `ScientificFieldInspector` used by +the workbench. That module owns slice validation, numeric probing, display-only +grayscale rendering and cancellation. Each page supplies its authoritative +selection predicate and slice reader; the workbench separately owns its source +download and review-note editor. The native project reader uses negotiated +project operations rather than the legacy workbench URLs. Native project +annotation stays with its existing data-note owner. + +Changing the selected record, project/checkpoint or session clears the native +view and aborts pending reads. A superseded reply cannot repaint it. Page teardown +also aborts reads. Hidden or loading canvases do not display old pixels. Field +units, producer stage, original clock and grid coordinates remain explicit; +opening a scientific data record does not qualify its physical meaning or turn +an authored diagnostic into an accepted model result. + +`test_project_sources` covers exact content and cold projection reads without +the original source root. `test_project_files_m2_http` covers real selected +transport, stale/refused reads and artifact corruption. The existing +`browser_scientific_fields.cjs` exercises the shared module through the original +workbench with its browser sandbox enabled. Lab qualification additionally +inspects unchanged retained pressure fields through native project operations +and independent relocated cold Open. Scalar Run comparisons and Experiment +progression remain separately owned interfaces. diff --git a/project_data.py b/project_data.py index 69c86914..4a48dd38 100644 --- a/project_data.py +++ b/project_data.py @@ -1168,6 +1168,14 @@ def query(self, options=None): return self._read('workbench.query', lambda: self.workbench.query(options or {})) def get(self, record_id): return self._read('workbench.get', lambda: self.workbench.get(record_id), record_id) + def scientific(self, record_id, content_hash, **selection): + sha(content_hash) + def inspect(): + record = self.workbench.get(record_id) + if record['content_hash'] != content_hash: + fail('Scientific source differs from the selected content hash.', 'conflict', 409) + return self.workbench.scientific.inspect(record_id, **selection) + return self._read('scientific_fields.inspect', inspect, record_id) def history(self, record_id): return self._read('workbench.history', lambda: self.workbench.history(record_id), record_id) def responses(self, record_id): diff --git a/project_dispatch.py b/project_dispatch.py index d12eb3f9..a437405b 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -12,7 +12,7 @@ _BOUND_FIELDS = frozenset(('binding', 'checkpoint', 'expected_checkpoint', 'request_key')) -_DATA_ACTIONS = frozenset(('data_query', 'data_get', 'data_availability', 'data_history', +_DATA_ACTIONS = frozenset(('data_query', 'data_get', 'data_scientific', 'data_availability', 'data_history', 'data_responses', 'data_response_history', 'data_preferences', 'data_preference_history', 'data_import_text', 'data_import_image', 'data_import_audio', 'data_import_mesh', 'data_import_pointcloud', 'data_import_volume', 'data_import_sequence', 'data_import_scientific', @@ -104,6 +104,8 @@ def _data(projection, action, a): data = projection.data if action == 'data_query': return data.query(_body(a)) if action == 'data_get': return data.get(a['record_id']) + if action == 'data_scientific': return data.scientific(a['record_id'], a['content_hash'], + **{key: a[key] for key in ('frame', 'field', 'axis', 'index') if key in a}) if action == 'data_availability': return data.availability(a['record_id']) if action == 'data_history': return _page(data.history(a['record_id']), a, default=1) if action == 'data_responses': return _page(data.responses(a['record_id'])['responses'], a) diff --git a/static/project-files.html b/static/project-files.html index 59a847a0..4738a772 100644 --- a/static/project-files.html +++ b/static/project-files.html @@ -7,7 +7,9 @@ + + @@ -173,6 +175,7 @@

Project data and scientific history

Choose a saved record

+
diff --git a/static/project-files.js b/static/project-files.js index 4357888c..3f25cec3 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -35,6 +35,7 @@ "project_files.data_response_save": "fa5b92afe529520fe8206d629bd88085a68f4f9b399cd40dd05fd3948d3592f0", "project_files.data_responses": "3e0c009244cad04fca4356cf3647bfd2b42cfc0d6a2ac31bb8de25cdb3c60ec8", "project_files.data_rights": "ead16cca68b03533daa81bf06148e1918ce40dbc823ee060bd9908581f788653", + "project_files.data_scientific": "c954a680440fdb822a0f895f7e14b90e05dca82535478b1271dfb33cfbfdada9", "project_files.data_source_delete": "457618e618cca92a71b668ffb9f68b0db87e51c417c2e5f41dcde76d71a23985", "project_files.data_source_save": "0d50c45d445b2ce1014a696c3bc95965f0b548ded939f4a650f985f34e1ce849", "project_files.definitions": "3516fcf8764382780c7da699218a05ff1f811634656cc81722ef547f21159698", @@ -165,623 +166,628 @@ { "project_files.artifact_resolve": { "method": "POST", - "path": "/api/workspace/project-files/artifacts/resolve", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/artifacts/resolve" }, "project_files.artifact_status": { "method": "POST", - "path": "/api/workspace/project-files/artifacts/status", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/artifacts/status" }, "project_files.artifacts": { "method": "GET", - "path": "/api/workspace/project-files/artifacts", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/artifacts" }, "project_files.candidate": { "method": "GET", - "path": "/api/workspace/project-files/candidates/{candidate_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/candidates/{candidate_id}" }, "project_files.candidate_select": { "method": "POST", - "path": "/api/workspace/project-files/studies/{study_id}/candidates", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies/{study_id}/candidates" }, "project_files.data_annotate": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}" }, "project_files.data_availability": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/availability", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/availability" }, "project_files.data_get": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}" }, "project_files.data_history": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/history" }, "project_files.data_import_audio": { "method": "POST", - "path": "/api/workspace/project-files/data/import-audio", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-audio" }, "project_files.data_import_image": { "method": "POST", - "path": "/api/workspace/project-files/data/import-image", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-image" }, "project_files.data_import_mesh": { "method": "POST", - "path": "/api/workspace/project-files/data/import-mesh", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-mesh" }, "project_files.data_import_pointcloud": { "method": "POST", - "path": "/api/workspace/project-files/data/import-pointcloud", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-pointcloud" }, "project_files.data_import_scientific": { "method": "POST", - "path": "/api/workspace/project-files/data/import-scientific", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-scientific" }, "project_files.data_import_sequence": { "method": "POST", - "path": "/api/workspace/project-files/data/import-sequence", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-sequence" }, "project_files.data_import_text": { "method": "POST", - "path": "/api/workspace/project-files/data/import-text", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-text" }, "project_files.data_import_volume": { "method": "POST", - "path": "/api/workspace/project-files/data/import-volume", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/import-volume" }, "project_files.data_preference_delete": { "method": "POST", - "path": "/api/workspace/project-files/preferences/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/preferences/delete" }, "project_files.data_preference_history": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/preference-history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/preference-history" }, "project_files.data_preference_save": { "method": "POST", - "path": "/api/workspace/project-files/preferences", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/preferences" }, "project_files.data_preferences": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/preferences", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/preferences" }, "project_files.data_query": { "method": "GET", - "path": "/api/workspace/project-files/data", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data" }, "project_files.data_response_history": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/response-history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/response-history" }, "project_files.data_response_save": { "method": "POST", - "path": "/api/workspace/project-files/responses", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/responses" }, "project_files.data_responses": { "method": "GET", - "path": "/api/workspace/project-files/data/{record_id}/responses", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/responses" }, "project_files.data_rights": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}/rights", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}/rights" + }, + "project_files.data_scientific": { + "method": "GET", + "mutation": false, + "path": "/api/workspace/project-files/data/{record_id}/scientific" }, "project_files.data_source_delete": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}/source/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}/source/delete" }, "project_files.data_source_save": { "method": "POST", - "path": "/api/workspace/project-files/data/{record_id}/source", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/data/{record_id}/source" }, "project_files.definitions": { "method": "GET", - "path": "/api/workspace/project-files/definitions", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/definitions" }, "project_files.dependencies": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}/dependencies", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/dependencies" }, "project_files.evidence": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/evidence", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/evidence" }, "project_files.execution_approve_external": { "method": "POST", - "path": "/api/workspace/project-files/execution/approvals", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/approvals" }, "project_files.execution_cancel": { "method": "POST", - "path": "/api/workspace/project-files/execution/runs/{run_id}/cancel", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/runs/{run_id}/cancel" }, "project_files.execution_generation_resume": { "method": "POST", - "path": "/api/workspace/project-files/execution/generation/{producer_id}/resume", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/generation/{producer_id}/resume" }, "project_files.execution_grounded_review": { "method": "POST", - "path": "/api/workspace/project-files/execution/grounded/{producer_id}/review", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/grounded/{producer_id}/review" }, "project_files.execution_health": { "method": "GET", - "path": "/api/workspace/project-files/execution", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/execution" }, "project_files.execution_preflight_external": { "method": "POST", - "path": "/api/workspace/project-files/execution/preflight", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/execution/preflight" }, "project_files.execution_producer_cancel": { "method": "POST", - "path": "/api/workspace/project-files/execution/producers/{producer_id}/cancel", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/producers/{producer_id}/cancel" }, "project_files.execution_producer_decide": { "method": "POST", - "path": "/api/workspace/project-files/execution/producers/{producer_id}/decide", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/producers/{producer_id}/decide" }, "project_files.execution_producer_health": { "method": "GET", - "path": "/api/workspace/project-files/execution/producers/health", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/execution/producers/health" }, "project_files.execution_producer_start": { "method": "POST", - "path": "/api/workspace/project-files/execution/producers", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/producers" }, "project_files.execution_reconcile_current": { "method": "POST", - "path": "/api/workspace/project-files/execution/reconcile-current", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/reconcile-current" }, "project_files.execution_study_abandon_preparation": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/abandon", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/abandon" }, "project_files.execution_study_admit": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/runs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/runs" }, "project_files.execution_study_approve": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/approvals", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/approvals" }, "project_files.execution_study_continue": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/continue", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/continue" }, "project_files.execution_study_prepare_package": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/packages", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/packages" }, "project_files.execution_study_prepare_verifier": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/verifiers", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/verifiers" }, "project_files.execution_study_reconcile_preparation": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/reconcile", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/reconcile" }, "project_files.execution_study_stop": { "method": "POST", - "path": "/api/workspace/project-files/execution/studies/{study_id}/stop", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/studies/{study_id}/stop" }, "project_files.execution_submit_external": { "method": "POST", - "path": "/api/workspace/project-files/execution/runs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/runs" }, "project_files.expose_holdout": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/disclose", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/runs/{run_id}/disclose" }, "project_files.file_adopt": { "method": "POST", - "path": "/api/workspace/project-files/files/adopt", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/files/adopt" }, "project_files.file_adopt_deletion": { "method": "POST", - "path": "/api/workspace/project-files/files/adopt-deletion", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/files/adopt-deletion" }, "project_files.file_get": { "method": "GET", - "path": "/api/workspace/project-files/files/{file_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/files/{file_id}" }, "project_files.file_history": { "method": "GET", - "path": "/api/workspace/project-files/files/{file_id}/history", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/files/{file_id}/history" }, "project_files.files_inspect": { "method": "GET", - "path": "/api/workspace/project-files/files", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/files" }, "project_files.fork_info": { "method": "GET", - "path": "/api/workspace/project-files/fork", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/fork" }, "project_files.graph_clone": { "method": "POST", - "path": "/api/workspace/project-files/graphs/{graph_id}/clone", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs/{graph_id}/clone" }, "project_files.graph_create": { "method": "POST", - "path": "/api/workspace/project-files/graphs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs" }, "project_files.graph_create_external": { "method": "POST", - "path": "/api/workspace/project-files/graphs/external", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs/external" }, "project_files.graph_get": { "method": "GET", - "path": "/api/workspace/project-files/graphs/{graph_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/graphs/{graph_id}" }, "project_files.graph_save": { "method": "POST", - "path": "/api/workspace/project-files/graphs/{graph_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/graphs/{graph_id}" }, "project_files.graphs": { "method": "GET", - "path": "/api/workspace/project-files/graphs", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/graphs" }, "project_files.lineage": { "method": "GET", - "path": "/api/workspace/project-files/lineage", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/lineage" }, "project_files.lineage_record": { "method": "POST", - "path": "/api/workspace/project-files/lineage", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/lineage" }, "project_files.material_intent": { "method": "GET", - "path": "/api/workspace/project-files/material-intents/{family}/{definition_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/material-intents/{family}/{definition_id}" }, "project_files.material_intent_capabilities": { "method": "GET", - "path": "/api/workspace/project-files/material-intents/capabilities", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/material-intents/capabilities" }, "project_files.material_intents": { "method": "GET", - "path": "/api/workspace/project-files/material-intents", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/material-intents" }, "project_files.materials": { "method": "GET", - "path": "/api/workspace/project-files/materials", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials" }, "project_files.model_create": { "method": "POST", - "path": "/api/workspace/project-files/models", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/models" }, "project_files.model_get": { "method": "GET", - "path": "/api/workspace/project-files/models/{model_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/models/{model_id}" }, "project_files.model_material": { "method": "POST", - "path": "/api/workspace/project-files/materials/model", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/model" }, "project_files.model_publish": { "method": "POST", - "path": "/api/workspace/project-files/models/{model_id}/publish", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/models/{model_id}/publish" }, "project_files.model_save": { "method": "POST", - "path": "/api/workspace/project-files/models/{model_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/models/{model_id}" }, "project_files.models": { "method": "GET", - "path": "/api/workspace/project-files/models", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/models" }, "project_files.native_preflight": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/preflight", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/preflight" }, "project_files.native_reconcile": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/phases/{phase_id}/reconcile", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/phases/{phase_id}/reconcile" }, "project_files.native_retry": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/runs/{run_id}/retry", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/runs/{run_id}/retry" }, "project_files.native_submit": { "method": "POST", - "path": "/api/workspace/project-files/execution/native/runs", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/execution/native/runs" }, "project_files.package_state": { "method": "GET", - "path": "/api/workspace/project-files/candidates/{candidate_id}/package-state", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/candidates/{candidate_id}/package-state" }, "project_files.policies": { "method": "GET", - "path": "/api/workspace/project-files/policies", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/policies" }, "project_files.policy": { "method": "POST", - "path": "/api/workspace/project-files/policies/get", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/policies/get" }, "project_files.producer": { "method": "GET", - "path": "/api/workspace/project-files/producers/{origin_catalog_id}/{producer_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/producers/{origin_catalog_id}/{producer_id}" }, "project_files.producers": { "method": "GET", - "path": "/api/workspace/project-files/producers", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/producers" }, "project_files.publication_result": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}/publication-result", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/publication-result" }, "project_files.recipe_create": { "method": "POST", - "path": "/api/workspace/project-files/recipes", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/recipes" }, "project_files.recipe_get": { "method": "GET", - "path": "/api/workspace/project-files/recipes/{recipe_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/recipes/{recipe_id}" }, "project_files.recipe_material": { "method": "POST", - "path": "/api/workspace/project-files/materials/recipe", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/recipe" }, "project_files.recipe_prepare": { "method": "POST", - "path": "/api/workspace/project-files/recipes/{graph_id}/prepare", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/recipes/{graph_id}/prepare" }, "project_files.recipe_save": { "method": "POST", - "path": "/api/workspace/project-files/recipes/{recipe_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/recipes/{recipe_id}" }, "project_files.recipes": { "method": "GET", - "path": "/api/workspace/project-files/recipes", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/recipes" }, "project_files.release_freeze": { "method": "POST", - "path": "/api/workspace/project-files/releases/freeze", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/releases/freeze" }, "project_files.release_preview": { "method": "POST", - "path": "/api/workspace/project-files/releases/preview", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/releases/preview" }, "project_files.retained_analysis": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/analysis", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/analysis" }, "project_files.retained_analysis_export": { "method": "POST", - "path": "/api/workspace/project-files/runs/{run_id}/analysis/export", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/analysis/export" }, "project_files.run": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}" }, "project_files.run_document_segment": { "method": "GET", - "path": "/api/workspace/project-files/runs/{run_id}/document-segment", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs/{run_id}/document-segment" }, "project_files.runs": { "method": "GET", - "path": "/api/workspace/project-files/runs", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runs" }, "project_files.runtime_abandon": { "method": "POST", - "path": "/api/workspace/project-files/runtime/abandon", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/runtime/abandon" }, "project_files.runtime_capture": { "method": "POST", - "path": "/api/workspace/project-files/runtime/capture", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/runtime/capture" }, "project_files.runtime_inspect": { "method": "GET", - "path": "/api/workspace/project-files/runtime/inspect", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runtime/inspect" }, "project_files.runtime_material": { "method": "POST", - "path": "/api/workspace/project-files/materials/runtime", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/runtime" }, "project_files.runtime_sources": { "method": "GET", - "path": "/api/workspace/project-files/runtime/sources", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runtime/sources" }, "project_files.runtime_status": { "method": "GET", - "path": "/api/workspace/project-files/runtime/status", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/runtime/status" }, "project_files.save_as": { "method": "POST", - "path": "/api/workspace/project-files/save-as", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/save-as" }, "project_files.search_create": { "method": "POST", - "path": "/api/workspace/project-files/searches", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/searches" }, "project_files.search_delete": { "method": "POST", - "path": "/api/workspace/project-files/searches/{search_id}/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/searches/{search_id}/delete" }, "project_files.search_rename": { "method": "POST", - "path": "/api/workspace/project-files/searches/{search_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/searches/{search_id}" }, "project_files.searches": { "method": "GET", - "path": "/api/workspace/project-files/searches", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/searches" }, "project_files.selection_create": { "method": "POST", - "path": "/api/workspace/project-files/selections", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/selections" }, "project_files.selection_delete": { "method": "POST", - "path": "/api/workspace/project-files/selections/{selection_id}/delete", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/selections/{selection_id}/delete" }, "project_files.selection_get": { "method": "GET", - "path": "/api/workspace/project-files/selections/{selection_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/selections/{selection_id}" }, "project_files.selection_rename": { "method": "POST", - "path": "/api/workspace/project-files/selections/{selection_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/selections/{selection_id}" }, "project_files.selections": { "method": "GET", - "path": "/api/workspace/project-files/selections", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/selections" }, "project_files.snapshot": { "method": "POST", - "path": "/api/workspace/project-files/materials/snapshot", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/materials/snapshot" }, "project_files.source_references": { "method": "GET", - "path": "/api/workspace/project-files/sources/{project_id}/references", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/sources/{project_id}/references" }, "project_files.source_register": { "method": "POST", - "path": "/api/workspace/project-files/sources/register", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/sources/register" }, "project_files.studies": { "method": "GET", - "path": "/api/workspace/project-files/studies", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/studies" }, "project_files.study": { "method": "GET", - "path": "/api/workspace/project-files/studies/{study_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/studies/{study_id}" }, "project_files.study_create": { "method": "POST", - "path": "/api/workspace/project-files/studies", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies" }, "project_files.study_save": { "method": "POST", - "path": "/api/workspace/project-files/studies/{study_id}", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies/{study_id}" }, "project_files.trial": { "method": "GET", - "path": "/api/workspace/project-files/trials/{trial_id}", - "mutation": false + "mutation": false, + "path": "/api/workspace/project-files/trials/{trial_id}" }, "project_files.trial_prepare": { "method": "POST", - "path": "/api/workspace/project-files/studies/{study_id}/trials", - "mutation": true + "mutation": true, + "path": "/api/workspace/project-files/studies/{study_id}/trials" } } /* PROJECT_FILES_ROUTES_END */; @@ -801,6 +807,18 @@ const m2HistoryActions=new Set(['data_history','data_response_history','data_preference_history']); const dirty=()=>importDirty||annotationDirty||m2Dirty; const checkpoint=()=>state?.project?.checkpoint??null; + let scientificEpoch=null; + const projectScientific=window.ScientificFieldInspector({ + root:$('scientific-inspection'), + isCurrent:record=>scientificEpoch===epoch&&!blocked&&state?.state==='open'&&explored?.category==='data_query'&&explored.id===record.id&&explored.value.content_hash===record.content_hash&&explored.checkpoint===checkpoint(), + async readSlice(record,selection,signal){ + const result=await m2Request('data_scientific',{record_id:record.id,content_hash:record.content_hash,...selection},epoch,{signal}); + if(result===null)throw new DOMException('Project selection changed.','AbortError'); + return result; + } + }); + function clearScientific(){scientificEpoch=null;projectScientific.clear();} + window.addEventListener('pagehide',clearScientific); const exact=(value,keys)=>value!==null&&typeof value==='object'&&!Array.isArray(value)&&Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); const str=(value,max,empty=false)=>typeof value==='string'&&Array.from(value).length<=max&&(empty||/\S/u.test(value)); const id=value=>typeof value==='string'&&/^[a-f0-9]{32}$/.test(value); @@ -1055,6 +1073,7 @@ } const m2Allowed=action=>!busy&&!blocked&&state?.state==='open'&&selectedContracts.has('project_files.'+action); function m2Controls(){ + if(blocked||state?.state!=='open'||scientificEpoch!==null&&(scientificEpoch!==epoch||explored?.checkpoint!==checkpoint()))clearScientific(); const action=$('project-explore-kind').value; $('project-explore-load').disabled=!m2Allowed(action); $('project-explore-more').disabled=!m2Allowed(action); @@ -1074,6 +1093,7 @@ for(const button of $('project-explore-list').querySelectorAll('button'))button.disabled=busy||blocked; } function clearExploration(){ + clearScientific(); explored=null;exploreOffset=0;exploreTotal=0;historyOffset=0;historyTotal=0; $('project-explore-list').replaceChildren();$('project-detail-facts').replaceChildren();$('project-detail-history-list').replaceChildren(); $('project-detail-title').textContent='Choose a saved record';$('project-explore-summary').textContent=''; @@ -1125,7 +1145,7 @@ for(const item of page.items)if(!exact(item,['family','source_id','source_revision','destination_id'])||!safeText(item.family,80)||!safeText(item.source_id,512)||!(item.source_revision===null||positive(item.source_revision))||!safeText(item.destination_id,512))invalid(); } } - async function m2Request(action,arguments_,ticket){ + async function m2Request(action,arguments_,ticket,{signal}={}){ const operation='project_files.'+action,definition=m2Operations[operation]; if(!definition||!selectedContracts.has(operation))throw Error('This operation is unavailable for the selected project.'); const binding=state.binding,cp=checkpoint(),project=state.project.project_id; @@ -1133,7 +1153,7 @@ let path=definition.path; for(const [name,value] of Object.entries(values))path=path.replace('{'+name+'}',encodeURIComponent(value)); const remaining=Object.fromEntries(Object.entries(values).filter(([name])=>!definition.path.includes('{'+name+'}'))); - const options={method:definition.method,cache:'no-store',headers:{'Tuldok-Operation-Contract':selectedHeader(operation)}}; + const options={method:definition.method,cache:'no-store',signal,headers:{'Tuldok-Operation-Contract':selectedHeader(operation)}}; if(definition.method==='GET')path+='?'+new URLSearchParams(remaining); else{ if(definition.mutation&&!['execution_reconcile_current','execution_study_reconcile_preparation','native_reconcile'].includes(action)){ @@ -1160,7 +1180,7 @@ } let response,value; try{response=await window.TuldokAuth.fetch(path,options);value=await response.json();} - catch(_){const failure=Error(definition.mutation?'Acknowledgement unavailable; this change may already be saved. Keep your edits and refresh project status.':'Saved history is unavailable. Refresh project status.');failure.code='transport';throw failure;} + catch(problem){if(problem.name==='AbortError')throw problem;const failure=Error(definition.mutation?'Acknowledgement unavailable; this change may already be saved. Keep your edits and refresh project status.':'Saved history is unavailable. Refresh project status.');failure.code='transport';throw failure;} if(ticket!==epoch)return null; if(!response.ok){ const failure=Error(str(value?.error,1000)?value.error:'Unsupported project error response.');failure.code=value?.code??'transport';failure.status=response.status; @@ -1224,6 +1244,7 @@ if(category==='models')$('project-model-publish').hidden=false; if(entry.saveAction&&origin===state.project.project_id){$('project-metadata-form').hidden=false;$('project-metadata-name').value=entry.value.name??'';} if(category==='data_query'){ + if(entry.value.kind==='field_sequence'){scientificEpoch=ticket;projectScientific.show(entry.value);} $('project-detail-history').hidden=false;$('project-detail-availability').hidden=false;$('project-rights-form').hidden=false;$('project-selection-form').hidden=false; $('project-rights-note').value=entry.value.provenance?.rights_note_correction?.note??entry.value.provenance?.rights??'unknown'; const key=entry.value.task?.endsWith('_classification')?'label':entry.value.task==='image_caption'?'caption':entry.value.task==='text_corpus'||['sequence','mesh','pointcloud','volume','field_sequence'].includes(entry.value.kind)?'note':null; @@ -1233,6 +1254,7 @@ controls();$('project-detail-title').focus(); } function clearExplorationDetail(){ + clearScientific(); explored=null;historyOffset=0;historyTotal=0;$('project-detail-facts').replaceChildren();$('project-detail-history-list').replaceChildren(); for(const name of ['project-metadata-form','project-data-note-form','project-rights-form','project-selection-form','project-detail-history','project-detail-history-more','project-detail-availability','project-run-evidence','project-run-document','project-run-disclose','project-disclosure-label','project-model-publish','project-artifact-status','project-artifact-resolve-form'])$(name).hidden=true; } diff --git a/static/scientific-fields.js b/static/scientific-fields.js index 0e4477e8..68f27f3b 100644 --- a/static/scientific-fields.js +++ b/static/scientific-fields.js @@ -1,76 +1,16 @@ 'use strict'; -let scientificGeneration=0,scientificController=null,scientificSlice=null; -function scientificShape(record,field) { - const shape=[...record.field_sequence.manifest.grid.shape_xyz]; - if(field.centering.startsWith('face_'))shape['xyz'.indexOf(field.centering.slice(5))]++; - return shape.reverse(); -} -function decodeScientificSlice(value,record,request) { - const manifest=record.field_sequence.manifest,field=manifest.fields.find(item=>item.id===request.field),shape=scientificShape(record,field); - const dimensions={z:shape[0],y:shape[1],x:shape[2]},[row,column]={z:['y','x'],y:['z','x'],x:['z','y']}[request.axis]; - const vector=v=>Array.isArray(v)&&v.length===3&&v.every(n=>typeof n==='number'&&Number.isFinite(n)); - if(!value||value.id!==record.id||value.content_hash!==record.content_hash||value.frame!==request.frame||value.field!==request.field|| - value.axis!==request.axis||value.index!==request.index||value.time!==manifest.frames[request.frame].time||value.time_units!==manifest.time_units|| - value.units!==field.units||value.stage!==field.stage||value.centering!==field.centering||value.dtype!==field.dtype|| - !value.grid||value.grid.units!==manifest.grid.units||!Array.isArray(value.shape_zyx)||value.shape_zyx.length!==3||!shape.every((n,i)=>value.shape_zyx[i]===n)||value.row_axis!==row||value.column_axis!==column|| - !['physical_origin','physical_row_step','physical_column_step'].every(key=>vector(value[key]))|| - !Array.isArray(value.values)||value.values.length!==dimensions[row]||!value.values.every(values=>Array.isArray(values)&&values.length===dimensions[column]&&values.every(n=>typeof n==='number'&&Number.isFinite(n))))throw Error('Invalid scientific field slice response.'); - return value; -} -function scientificProbe() { - if(!scientificSlice)return; - const row=Number($('scientific-row').value),column=Number($('scientific-column').value),value=scientificSlice; - if(!Number.isInteger(row)||!Number.isInteger(column)||row<0||column<0||row>=value.values.length||column>=value.values[0].length){$('scientific-probe').textContent='Choose an in-bounds row and column.';return;} - const physical=value.physical_origin.map((n,i)=>n+row*value.physical_row_step[i]+column*value.physical_column_step[i]); - $('scientific-probe').textContent=`Row ${row}, column ${column}: ${Object.is(value.values[row][column],-0)?'-0':String(value.values[row][column])} ${value.units} · physical (${physical.join(', ')}) ${value.grid.units}.`; -} -function drawScientificSlice(value) { - scientificSlice=value; - const canvas=$('scientific-canvas'),rows=value.values.length,columns=value.values[0].length; - canvas.width=columns;canvas.height=rows; - let low=Infinity,high=-Infinity;for(const row of value.values)for(const number of row){low=Math.min(low,number);high=Math.max(high,number);} - const scale=Math.max(Math.abs(low),Math.abs(high)),range=scale?high/scale-low/scale:0; - const ctx=canvas.getContext('2d'),pixels=ctx.createImageData(columns,rows); - for(let row=0;row({value:String(i),label:`${i} · ${frame.time} ${manifest.time_units}`}))],['scientific-field',manifest.fields.map(field=>({value:field.id,label:field.label}))]]){ - $(id).replaceChildren();for(const item of items){const option=document.createElement('option');option.value=item.value;option.textContent=item.label;$(id).append(option);} - } - $('scientific-axis').value='z';$('scientific-index').value='0'; - const load=async reset=>{ - scientificController?.abort();const controller=new AbortController();scientificController=controller;scientificSlice=null; - const field=manifest.fields.find(item=>item.id===$('scientific-field').value),shape=scientificShape(record,field),axis=$('scientific-axis').value; - $('scientific-index').max=String(shape['zyx'.indexOf(axis)]-1);if(reset||Number($('scientific-index').value)>Number($('scientific-index').max))$('scientific-index').value='0'; - const request={frame:Number($('scientific-frame').value),field:field.id,axis,index:Number($('scientific-index').value)}; - const owns=()=>generation===scientificGeneration&&scientificController===controller&¤t?.id===record.id&¤t.content_hash===record.content_hash; - $('scientific-canvas').hidden=true;$('scientific-status').textContent='Loading exact numeric slice…';$('scientific-probe').textContent='';$('scientific-coordinates').textContent=''; - try{ - const response=await (globalThis.TuldokAuth?.fetch || fetch)('/api/workbench/records/'+record.id+'/scientific?'+new URLSearchParams(request),{signal:controller.signal}),raw=await response.json(); - if(!response.ok)throw Error(raw.error||'Scientific slice unavailable.'); - const value=decodeScientificSlice(raw,record,request);if(owns())drawScientificSlice(value); - }catch(error){if(owns()&&error.name!=='AbortError')$('scientific-status').textContent=error.message;} - finally{if(scientificController===controller)scientificController=null;} - }; - for(const id of ['scientific-frame','scientific-field','scientific-axis','scientific-index']){ - $(id).oninput=event=>event.stopPropagation(); - $(id).onchange=event=>{event.stopPropagation();load(id==='scientific-field'||id==='scientific-axis');}; +// Workbench transport and editor ownership for the shared field inspector. +const scientificInspector=window.ScientificFieldInspector({ + root:document.getElementById('scientific-inspection'), + isCurrent:record=>current?.id===record.id&¤t.content_hash===record.content_hash, + downloadURL:record=>'/api/workbench/asset/'+record.id, + async readSlice(record,selection,signal){ + const response=await (globalThis.TuldokAuth?.fetch||fetch)('/api/workbench/records/'+record.id+'/scientific?'+new URLSearchParams(selection),{signal}); + const value=await response.json();if(!response.ok)throw Error(value.error||'Scientific slice unavailable.');return value; } - for(const id of ['scientific-row','scientific-column'])$(id).oninput=event=>{event.stopPropagation();scientificProbe();}; - load(true); +}); +function scientificShown(record){ + scientificInspector.show(record); + if(record.kind==='field_sequence')$('scientific-note').value=record.annotation?.note||''; } -window.addEventListener('pagehide',()=>{++scientificGeneration;scientificController?.abort();scientificController=null;}); +window.addEventListener('pagehide',()=>scientificInspector.clear()); diff --git a/static/scientific-inspector.css b/static/scientific-inspector.css new file mode 100644 index 00000000..f8cc9799 --- /dev/null +++ b/static/scientific-inspector.css @@ -0,0 +1,2 @@ +.scientific-controls{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:8px}.scientific-controls label{min-width:0}#scientific-canvas{display:block;max-width:100%;width:420px;image-rendering:pixelated;background:#eee}#scientific-status,#scientific-coordinates,#scientific-probe{font-size:12px;overflow-wrap:anywhere} +#scientific-canvas[hidden]{display:none} diff --git a/static/scientific-inspector.js b/static/scientific-inspector.js new file mode 100644 index 00000000..e0427888 --- /dev/null +++ b/static/scientific-inspector.js @@ -0,0 +1,84 @@ +'use strict'; +function scientificShape(record,field) { + const shape=[...record.field_sequence.manifest.grid.shape_xyz]; + if(field.centering.startsWith('face_'))shape['xyz'.indexOf(field.centering.slice(5))]++; + return shape.reverse(); +} +function decodeScientificSlice(value,record,request) { + const manifest=record.field_sequence.manifest,field=manifest.fields.find(item=>item.id===request.field),shape=scientificShape(record,field); + const dimensions={z:shape[0],y:shape[1],x:shape[2]},[row,column]={z:['y','x'],y:['z','x'],x:['z','y']}[request.axis]; + const vector=v=>Array.isArray(v)&&v.length===3&&v.every(n=>typeof n==='number'&&Number.isFinite(n)); + if(!value||value.id!==record.id||value.content_hash!==record.content_hash||value.frame!==request.frame||value.field!==request.field|| + value.axis!==request.axis||value.index!==request.index||value.time!==manifest.frames[request.frame].time||value.time_units!==manifest.time_units|| + value.units!==field.units||value.stage!==field.stage||value.centering!==field.centering||value.dtype!==field.dtype|| + !value.grid||value.grid.units!==manifest.grid.units||!Array.isArray(value.shape_zyx)||value.shape_zyx.length!==3||!shape.every((n,i)=>value.shape_zyx[i]===n)||value.row_axis!==row||value.column_axis!==column|| + !['physical_origin','physical_row_step','physical_column_step'].every(key=>vector(value[key]))|| + !Array.isArray(value.values)||value.values.length!==dimensions[row]||!value.values.every(values=>Array.isArray(values)&&values.length===dimensions[column]&&values.every(n=>typeof n==='number'&&Number.isFinite(n))))throw Error('Invalid scientific field slice response.'); + return value; +} + +window.ScientificFieldInspector=function({root,isCurrent,readSlice,downloadURL}){ + const $=id=>id==='scientific-inspection'?root:root.querySelector('#'+id); + let scientificGeneration=0,scientificController=null,scientificSlice=null; + const content=document.createElement('div'); + content.innerHTML="Download original scientific bundle

Native fields, geometry and declared provenance
"; + root.prepend(content); +function scientificProbe() { + if(!scientificSlice)return; + const row=Number($('scientific-row').value),column=Number($('scientific-column').value),value=scientificSlice; + if(!Number.isInteger(row)||!Number.isInteger(column)||row<0||column<0||row>=value.values.length||column>=value.values[0].length){$('scientific-probe').textContent='Choose an in-bounds row and column.';return;} + const physical=value.physical_origin.map((n,i)=>n+row*value.physical_row_step[i]+column*value.physical_column_step[i]); + $('scientific-probe').textContent=`Row ${row}, column ${column}: ${Object.is(value.values[row][column],-0)?'-0':String(value.values[row][column])} ${value.units} · physical (${physical.join(', ')}) ${value.grid.units}.`; +} +function drawScientificSlice(value) { + scientificSlice=value; + const canvas=$('scientific-canvas'),rows=value.values.length,columns=value.values[0].length; + canvas.width=columns;canvas.height=rows; + let low=Infinity,high=-Infinity;for(const row of value.values)for(const number of row){low=Math.min(low,number);high=Math.max(high,number);} + const scale=Math.max(Math.abs(low),Math.abs(high)),range=scale?high/scale-low/scale:0; + const ctx=canvas.getContext('2d'),pixels=ctx.createImageData(columns,rows); + for(let row=0;row({value:String(i),label:`${i} · ${frame.time} ${manifest.time_units}`}))],['scientific-field',manifest.fields.map(field=>({value:field.id,label:field.label}))]]){ + $(id).replaceChildren();for(const item of items){const option=document.createElement('option');option.value=item.value;option.textContent=item.label;$(id).append(option);} + } + $('scientific-axis').value='z';$('scientific-index').value='0'; + const load=async reset=>{ + scientificController?.abort();const controller=new AbortController();scientificController=controller;scientificSlice=null; + const field=manifest.fields.find(item=>item.id===$('scientific-field').value),shape=scientificShape(record,field),axis=$('scientific-axis').value; + $('scientific-index').max=String(shape['zyx'.indexOf(axis)]-1);if(reset||Number($('scientific-index').value)>Number($('scientific-index').max))$('scientific-index').value='0'; + const request={frame:Number($('scientific-frame').value),field:field.id,axis,index:Number($('scientific-index').value)}; + const owns=()=>generation===scientificGeneration&&scientificController===controller&&isCurrent(record); + $('scientific-canvas').hidden=true;$('scientific-status').textContent='Loading exact numeric slice…';$('scientific-probe').textContent='';$('scientific-coordinates').textContent=''; + try{ + const raw=await readSlice(record,request,controller.signal); + const value=decodeScientificSlice(raw,record,request);if(owns())drawScientificSlice(value); + }catch(error){if(owns()&&error.name!=='AbortError')$('scientific-status').textContent=error.message;} + finally{if(scientificController===controller)scientificController=null;} + }; + for(const id of ['scientific-frame','scientific-field','scientific-axis','scientific-index']){ + $(id).oninput=event=>event.stopPropagation(); + $(id).onchange=event=>{event.stopPropagation();load(id==='scientific-field'||id==='scientific-axis');}; + } + for(const id of ['scientific-row','scientific-column'])$(id).oninput=event=>{event.stopPropagation();scientificProbe();}; + load(true); +} + + function clear(){++scientificGeneration;scientificController?.abort();scientificController=null;scientificSlice=null;root.hidden=true;$('scientific-canvas').hidden=true;$('scientific-status').textContent='';$('scientific-probe').textContent='';$('scientific-coordinates').textContent='';} + return {show,clear}; +}; diff --git a/static/workbench.css b/static/workbench.css index aab05af2..7afb5fc5 100644 --- a/static/workbench.css +++ b/static/workbench.css @@ -33,5 +33,3 @@ #mesh-projections{display:flex;flex-wrap:wrap;gap:.7rem}#mesh-projections figure{flex:1 1 180px;margin:0;min-width:0}#mesh-projections svg{display:block;width:100%;max-height:240px;background:#f4f6fa;border:1px solid #ccd4df}#mesh-projections polygon{fill:none;stroke:#28598b;stroke-width:1.2}#mesh-metadata{max-width:100%;overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere} #pumas-gateway-panel {margin-block:1rem;} #pumas-gateway-metadata {max-height:24rem;overflow:auto;} - -.scientific-controls{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:8px}.scientific-controls label{min-width:0}#scientific-canvas{display:block;max-width:100%;width:420px;image-rendering:pixelated;background:#eee}#scientific-status,#scientific-coordinates,#scientific-probe{font-size:12px;overflow-wrap:anywhere} diff --git a/static/workbench.html b/static/workbench.html index e00c8ec2..5fa0887a 100644 --- a/static/workbench.html +++ b/static/workbench.html @@ -1,8 +1,8 @@ -Tuldok · Dataset workbench +Tuldok · Dataset workbench
Dataset workspaces · M1 preview · ← Corner studio

Dataset workbench

Real assets, explicit targets, reproducible releases.

Development workspace

Loading collection…

Collection

Exact, case-sensitive values; leave blank for any. For values with line breaks, choose JSON strings and enter double-quoted values: "first\nsecond" for LF, \r for CR and \r\n for CRLF. Escape literal backslashes as \\ and quotes as \". Groups match IDs stored on the record, not all related sources. Rights notes are metadata, not permission decisions. Missing notes appear as unknown. Filtered results are dynamic; saved fixed membership and release selection stay separate.

Saved searches · dynamic results

Save the entered search and filters. Open explicitly to search the current collection; new matching records can appear. Fixed selections, editor drafts and review decisions stay separate. Up to 100 searches. Search text is single-line; exact metadata values may contain line breaks.

Saved fixed selections

Save the chosen records at their exact revisions. Searches and filters stay separate; saving or opening never approves annotations.

    Curation diagnostics

    Inspect current metadata in all applied collection results or exact selected IDs. Selection revision pairs stay fixed. Counts describe metadata, not permission or model quality. Missing sources means deleted source records; file integrity is checked during release.

    Refresh to inspect current metadata.

      -

      Annotate & review

      Find Pumas for annotation

      Inspect a local gateway before choosing it for a proposal. Build features describe the compiled program; they do not prove model readiness. Gateway details are advertised, without authentication of library ownership. Existing manual gateway URLs remain available.

      Scan reads local advertisements. Choosing a gateway starts no model request.

      Advertised identity and observation hash

      Open a record to inspect its source and define its target.

      +

      Annotate & review

      Find Pumas for annotation

      Inspect a local gateway before choosing it for a proposal. Build features describe the compiled program; they do not prove model readiness. Gateway details are advertised, without authentication of library ownership. Existing manual gateway URLs remain available.

      Scan reads local advertisements. Choosing a gateway starts no model request.

      Advertised identity and observation hash

      Open a record to inspect its source and define its target.

      +
      Import a frozen caption corpus

      Choose an expanded Tuldok image-caption v1 release folder with manifest.json and train/val/test metadata and PNGs.

      Captions become drafts for your review. Existing split assignments and protected families are retained. Source identity is recorded as origin evidence; existing records are never updated.

      Ready to import native caption records.

        Successful rows remain imported. Stop prevents the next admission. Batch progress stays in this page; inspect the collection after reload.

        Generate offline

        Known targets from authored scene or template state. These recipes demonstrate reproducible production; they do not establish real-world model quality. Pumas image generation remains in the corner studio.

        Grounded model candidates

        Typed v1 sends finite requests. Text messages only; seed is unavailable. Generation runs until completion, failure or cancellation. Capability inspection has a separate bounded discovery budget.

        Open a human-reviewed text-classification source. Generate class-preserving rewrite proposals using your Pumas gateway; the captured source text is sent to the selected server.

        Schema and exact source quotes are checked; semantic correctness is not. Admitting a candidate creates a draft that still needs annotation review before export. Requested seeds do not guarantee reproducibility across models.

        Freeze selected release

        0 selected

        Only reviewed targets can be released. Related sources stay together, including unselected lineage bridges. Whole groups can prevent exact ratios.

        Select records to preview a release.

        Freeze selected answers

        Answer selection is separate from record selection and collection filters. Select individual answer revisions; edits and newly added answers never replace these fixed pairs.

        0 answers selected

          Whole source families stay together. Percentages weight examples, including multiple answers to one prompt. Draft or stale selections block export. The complete archive is bounded to 40 MiB.

          Select exact reviewed answers to preview.

          Freeze selected preference pairs

          Selections retain exact stored judgment, prompt and answer revisions. Reviewed ties and abstentions are excluded with reasons. Stale, draft or conflicting current directional judgments block export. Refresh/rejudge deliberately after edits.

            Whole source families stay together. Percentages weight emitted pairs; ties and abstentions do not weight splits. 5,000 judgments and 40 MiB per release.

            diff --git a/tests/browser_scientific_fields.cjs b/tests/browser_scientific_fields.cjs index 85d42245..7e6e37f4 100644 --- a/tests/browser_scientific_fields.cjs +++ b/tests/browser_scientific_fields.cjs @@ -32,7 +32,7 @@ const serverArgs=port=>['-u','app.py','--port',String(port),'--data',path.join(t const port=await until(()=>{if(server.exitCode!==null)throw Error('Project server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];}); const base='http://127.0.0.1:'+port; await fixtureAuth.register('http://127.0.0.1:'+port,path.join(temporary,'data'),root); - const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,900','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:['ignore','ignore','ignore'],env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); + const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,900','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:['ignore','ignore','ignore'],env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); const debugPort=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')); const tabs=await(await fetch('http://127.0.0.1:'+debugPort+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); ws=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;}); diff --git a/tests/test_project_files_clients.py b/tests/test_project_files_clients.py index 65e57a1c..e692dc45 100644 --- a/tests/test_project_files_clients.py +++ b/tests/test_project_files_clients.py @@ -72,6 +72,18 @@ def request(method, path, body=None, **kwargs): class M2Contracts(unittest.TestCase): + def test_scientific_slice_request_requires_project_checkpoint_and_exact_content(self): + arguments=dict(binding=BINDING,checkpoint=CHECKPOINT,record_id=RECORD, + content_hash=OTHER,frame=1,field='pressure',axis='z',index=2) + method,path,body=api.project_request('project_files.data_scientific',arguments) + self.assertEqual(method,'GET');self.assertIsNone(body) + self.assertTrue(path.startswith('/api/workspace/project-files/data/'+RECORD+'/scientific?')) + for key in ('binding','checkpoint','record_id','content_hash'): + with self.subTest(missing=key),self.assertRaises(api.AgentAPIError): + api.validate_arguments('project_files.data_scientific',{k:v for k,v in arguments.items() if k!=key}) + with self.assertRaises(api.AgentAPIError): + api.validate_arguments('project_files.data_scientific',dict(arguments,execute=True)) + def test_fixed_consumers_cover_all_m2_operations_and_preserve_m1(self): sdk = ProjectFilesClient(Mock()) cli = project_cli.parser() diff --git a/tests/test_project_files_m2_http.py b/tests/test_project_files_m2_http.py index 1234fc13..6a92ed91 100644 --- a/tests/test_project_files_m2_http.py +++ b/tests/test_project_files_m2_http.py @@ -34,7 +34,9 @@ SOURCE_FILES = ('app.py','agent_api.py','access.py','project_workspace.py','project_dispatch.py', 'project_projection.py','project_data.py','project_sources.py','project_material.py', 'project_history.py','project_producers.py','project_session.py','project_store.py', - 'static/project-files.js','static/project-files.html','tuldok_client.py','project_cli.py','mcp_stdio.py') + 'static/project-files.js','static/project-files.html','tuldok_client.py','project_cli.py','mcp_stdio.py', + 'static/scientific-inspector.js','static/scientific-inspector.css','static/scientific-fields.js', + 'static/workbench.html','static/workbench.css') def subject_hashes(): @@ -265,6 +267,35 @@ def test_unpinned_scientific_reference_uses_current_observed_hash(self): document={'name':'Unpinned but explicitly measured source','groups':['m2-unpinned'],'rights':'Owned authored data'}) self.assertEqual(row['source_sha256'],item['observed_sha256']) + def test_native_scientific_slice_refuses_stale_selection_and_changed_artifact(self): + self.create();_manifest,raw=authored_source(self.external) + registered=self.call('source_register',root='fixture',manifest_path='source-project.json') + snapshot={key:registered[key] for key in ('origin_catalog_id','project_id','manifest_sha256')} + row=self.call('data_import_scientific',snapshot=snapshot,reference_id='fields', + expected_sha256=hashlib.sha256(raw).hexdigest(), + document={'name':'Native slice source','groups':['m2-slice'],'rights':'Owned fixture'}) + arguments=self.arguments('data_scientific',record_id=row['id'],content_hash=row['content_hash'], + frame=1,field='scalar',axis='z',index=2) + before=m1.canonical_bytes(self.external/'alpha') + status,value=self.transport('data_scientific',arguments) + self.assertEqual(status,200,value) + self.assertEqual(value['value']['values'][1][2],1212+2**-45) + for change in ({'checkpoint':'0'*64},{'binding':'0'*32},{'content_hash':'0'*64}): + status,error=self.transport('data_scientific',dict(arguments,**change)) + self.assertEqual(status,409,error) + with self.owner.bound(self.workspace): + credential=self.workspace.access.issue({'label':'Revoked slice inspection','scopes':['inspect']}) + self.workspace.access.revoke(credential['identity']['credential_id']) + self.assertEqual(self.transport('data_scientific',arguments,token=credential['token'])[0],401) + self.assertEqual(m1.canonical_bytes(self.external/'alpha'),before) + path=self.external/'alpha/artifacts/sha256'/row['content_hash'] + self.assertEqual(path.read_bytes(),raw) + path.chmod(0o600) # Deliberate external corruption of this disposable fixture. + path.write_bytes(b'x'*len(raw)) + self.assertNotEqual(self.transport('data_scientific',arguments)[0],200) + path.unlink() + self.assertNotEqual(self.transport('data_scientific',arguments)[0],200) + @unittest.skipUnless(importlib.util.find_spec('mcp'),'Requires pinned real MCP optional dependencies') def test_real_sdk_cli_mcp_roles_lost_reply_and_stale_binding(self): import anyio diff --git a/tests/test_project_sources.py b/tests/test_project_sources.py index ca7b3556..5156e3f0 100644 --- a/tests/test_project_sources.py +++ b/tests/test_project_sources.py @@ -179,6 +179,30 @@ def test_real_scientific_import_uses_one_data_session_and_cold_source_access_sta with self.assertRaises(WorkbenchError):adapter.import_scientific(forged,projection.session.snapshot['checkpoint'],'foreign-unbound') self.assertEqual(self.store.snapshot(),canonical) + def test_scientific_slice_read_pins_content_and_survives_cold_open_without_source_root(self): + body=self.import_body(self.register()) + row=self.adapter.import_scientific(body,self.session.snapshot['checkpoint'],'slice-source') + before=self.store.snapshot() + expected=self.projection.data.scientific(row['id'],row['content_hash'],frame=1,field='scalar',axis='z',index=2) + self.assertEqual(expected['values'][1][2],1212+2**-45) + projection,_adapter=self.project('cold-slice-without-roots',{}) + self.assertEqual(projection.data.scientific(row['id'],row['content_hash'],frame=1,field='scalar',axis='z',index=2),expected) + for changes in ({'frame':2},{'index':4},{'field':'absent'}): + selection=dict(frame=0,field='scalar',axis='z',index=0) + selection.update(changes) + with self.subTest(changes=changes),self.assertRaises(WorkbenchError): + projection.data.scientific(row['id'],row['content_hash'],**selection) + with self.assertRaises(WorkbenchError) as caught: + projection.data.scientific(row['id'],'0'*64) + self.assertEqual(caught.exception.code,'conflict') + token=self.access.issue(dict(label='Revoked inspection',scopes=['inspect'])) + with self.access.bind(token['token']): + with self.access.bind(self.token): + self.access.revoke(token['identity']['credential_id']) + with self.assertRaises(AccessError): + projection.data.scientific(row['id'],row['content_hash']) + self.assertEqual(self.store.snapshot(),before) + def test_deferred_sql_failure_retains_snapshot_receipt_and_explicit_cold_replay_rebinds(self): self.db.execute("CREATE FUNCTION fixture_refuse_source() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'controlled source SQL failure' USING ERRCODE='23514'; END; $$") self.db.execute('CREATE CONSTRAINT TRIGGER fixture_source_failure AFTER INSERT ON workspace_projects DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION fixture_refuse_source()') diff --git a/tuldok_client.py b/tuldok_client.py index d18ba547..11efd24d 100644 --- a/tuldok_client.py +++ b/tuldok_client.py @@ -278,6 +278,10 @@ def data_get(self, **arguments): """data get; exact selected project checkpoint, existing domain validation, no execution.""" return self._call('data_get', arguments) + def data_scientific(self, **arguments): + """Inspect an exact scientific field slice in the selected project.""" + return self._call('data_scientific', arguments) + def data_history(self, **arguments): """data history; exact selected project checkpoint, existing domain validation, no execution.""" return self._call('data_history', arguments) From 261b9c7debd3d57d6a60785e005cf318f0c75769 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 13:15:54 -0700 Subject: [PATCH 02/15] fix(project): validate extraction content before source stop Report native composition failures during the live read-only preview. Keep the stopped-source capability as an independent extraction requirement so content readiness never grants cessation authority. --- docs/runbooks/external-project-extraction.md | 6 ++++++ project_transfer.py | 2 +- tests/test_project_transfer.py | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/runbooks/external-project-extraction.md b/docs/runbooks/external-project-extraction.md index 0684abe0..682a689b 100644 --- a/docs/runbooks/external-project-extraction.md +++ b/docs/runbooks/external-project-extraction.md @@ -38,6 +38,12 @@ The report binds the exact selection, source schema and snapshot, backup and external inventory. Extraction requires the same report digest and selection. An unresolved or incomplete report is not successful empty-project creation. +When the selected content and unique payloads are available, Preview validates +the captured native domain composition while the source is still live. Domain +refusals appear before an owner has to stop the source to discover them. This +does not satisfy cessation: the independent `cessation_proof_unavailable` +blocker remains until the existing trusted owner completes Stop and capture. + ## Stop through the existing owner The explicit stop operation belongs to the currently opened legacy controller. diff --git a/project_transfer.py b/project_transfer.py index c7aee05b..1507e71e 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -501,10 +501,10 @@ def _preview(self,selection): for value in captured['working_files']: closed(value,{'kind','path','sha256','bytes','text'},'observed working file') if ('project_file',file_id(source.db.dataset_id,value['kind'],value['path'])) in required:file_observations.append(copy.deepcopy(value)) - if not isinstance(source,StoppedSourceLease):blockers.append(dict(family='source',id=source.db.dataset_id,code='cessation_proof_unavailable',detail='The current trusted owner must explicitly stop and capture this source.')) if not blockers: try:_validate_candidate(records,source.db.dataset_id,source.artifacts) except WorkbenchError as error:blockers.append(dict(family='closure',id=source.db.dataset_id,code=getattr(error,'code','invalid'),detail=str(error))) + if not isinstance(source,StoppedSourceLease):blockers.append(dict(family='source',id=source.db.dataset_id,code='cessation_proof_unavailable',detail='The current trusted owner must explicitly stop and capture this source.')) report=dict(format='project_transfer_report_v1',source=dict(catalog_id=source.db.dataset_id,schema_version=source.db.schema_version, catalog_signature=source.db._signature(),snapshot_sha256=source.fingerprint, backup_receipt_sha256=digest(canonical(source.backup_receipt)) if source.backup_receipt is not None else None, diff --git a/tests/test_project_transfer.py b/tests/test_project_transfer.py index 7f088140..b3bf4848 100644 --- a/tests/test_project_transfer.py +++ b/tests/test_project_transfer.py @@ -79,6 +79,23 @@ def test_actual_stop_backup_is_private_verified_and_source_unchanged(self): with self.assertRaises(WorkbenchError): with self.dataset._project_request():pass with self.assertRaises(WorkbenchError):self.dataset.stop_for_project_extraction(self.parent_fd,'retry-backup') + + def test_live_preview_reports_domain_refusal_before_source_stop(self): + from project_projection import ProjectProjection + record=self.text();self.selection['roots']['records']=[record['id']] + before=tree_manifest(self.dataset.path) + # Controlled downstream domain refusal: callers must see it while the + # source is still live, without stopping just to discover the failure. + with preview_source(self.dataset) as source,patch.object(ProjectProjection,'validate_records', + side_effect=WorkbenchError('Selected retained composition is incompatible.','invalid',409)): + report=ProjectTransfer(source).preview(self.selection) + self.assertFalse(report['can_extract']) + self.assertIn(dict(family='closure',id=self.dataset.db.dataset_id,code='invalid', + detail='Selected retained composition is incompatible.'),report['blockers']) + self.assertTrue(any(row['code']=='cessation_proof_unavailable' for row in report['blockers'])) + self.assertFalse(self.dataset._extraction_stopping) + self.assertFalse(self.dataset._closed) + self.assertEqual(tree_manifest(self.dataset.path),before) def test_json_bool_and_fake_controller_guards_never_stop(self): for guard in (True,{},SimpleNamespace(require=lambda _:None)): with self.assertRaises(WorkbenchError) as caught: From d054f5b13bff3896044ec54fd1befeea42be61d0 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 14:37:53 -0700 Subject: [PATCH 03/15] fix(recipes): preserve streamed input and historical host contracts --- capture_payloads.py | 97 ++++++++++++++++++++ model_definitions.py | 22 ++++- pipeline.py | 35 +++++--- projects.py | 23 +++-- recipe_contract.py | 48 +++++++--- recipe_process.py | 91 ++++++++++++++++--- tests/test_streamed_payloads.py | 136 ++++++++++++++++++++++++++++ tests/test_streamed_recipe.py | 151 ++++++++++++++++++++++++++++++++ 8 files changed, 564 insertions(+), 39 deletions(-) create mode 100644 capture_payloads.py create mode 100644 tests/test_streamed_payloads.py create mode 100644 tests/test_streamed_recipe.py diff --git a/capture_payloads.py b/capture_payloads.py new file mode 100644 index 00000000..de6c26b2 --- /dev/null +++ b/capture_payloads.py @@ -0,0 +1,97 @@ +"""Bounded-memory, descriptor-backed payloads for explicitly streamed recipes. + +These objects never serialize a live path into execution material. Publication +copies and hashes the selected bytes into the existing owned inventory. +""" +import hashlib +import os +import stat + +from workbench import WorkbenchError + +CHUNK_BYTES = 1024 * 1024 +FORMAT = 'streamed_project_inputs_v1' + + +def identity(info): + return (info.st_dev, info.st_ino, info.st_mode, info.st_nlink, info.st_uid, + info.st_gid, info.st_size, info.st_mtime_ns, info.st_ctime_ns) + + +class FilePayload: + def __init__(self, descriptor, maximum, *, expected_sha256=None, guard=None): + self.fd = os.dup(descriptor) + self.guard = guard + try: + info = os.fstat(self.fd) + if not stat.S_ISREG(info.st_mode) or info.st_size > maximum: + raise WorkbenchError('Capture must be a bounded regular file.', 'too_large', 400) + self.size = info.st_size + self.original = identity(info) + self.guard_identity = guard() if guard else None + self.expected_sha256 = expected_sha256 + self.expected_sha256 = self._transfer() + except BaseException: + self.close() + raise + + def __len__(self): + return self.size + + def close(self): + if getattr(self, 'fd', None) is not None: + os.close(self.fd) + self.fd = None + + def __del__(self): + self.close() + + def _check(self): + try: + if (self.fd is None or identity(os.fstat(self.fd)) != self.original + or (self.guard and self.guard() != self.guard_identity)): + raise WorkbenchError('Selected capture identity changed.', 'conflict', 409) + except OSError as error: + raise WorkbenchError('Selected capture path unavailable.', 'unavailable', 409) from error + + def _transfer(self, destination=None): + self._check() + digest = hashlib.sha256() + offset = 0 + while offset < self.size: + chunk = os.pread(self.fd, min(CHUNK_BYTES, self.size - offset), offset) + if not chunk: + raise WorkbenchError('Selected capture was truncated.', 'conflict', 409) + digest.update(chunk) + if destination is not None: + destination.write(chunk) + offset += len(chunk) + self._check() + measured = digest.hexdigest() + if self.expected_sha256 is not None and measured != self.expected_sha256: + raise WorkbenchError('Selected capture digest changed.', 'conflict', 409) + return measured + + def digest(self): + return self._transfer() + + def copy_to(self, handle): + self._transfer(handle) + + def __eq__(self, other): + if isinstance(other, FilePayload): + return len(self) == len(other) and self.digest() == other.digest() + if isinstance(other, bytes): + return len(self) == len(other) and self.digest() == hashlib.sha256(other).hexdigest() + return NotImplemented + + +def payload_digest(raw): + return raw.digest() if isinstance(raw, FilePayload) else hashlib.sha256(raw).hexdigest() + + +def copy_payload(raw, destination): + if isinstance(raw, FilePayload): + raw.copy_to(destination) + else: + destination.write(raw) diff --git a/model_definitions.py b/model_definitions.py index dabfe61f..c69c0281 100644 --- a/model_definitions.py +++ b/model_definitions.py @@ -78,7 +78,8 @@ def reference(value, kind, catalog_id): def blob(raw, name, media_type='application/json'): - return {'name':name,'sha256':hashlib.sha256(raw).hexdigest(),'bytes':len(raw),'media_type':media_type} + from capture_payloads import payload_digest + return {'name':name,'sha256':payload_digest(raw),'bytes':len(raw),'media_type':media_type} def blob_identity(value): @@ -109,7 +110,7 @@ def owned_inventory(root, material_sha256, names): if root_fd is not None:os.close(root_fd) -def read_owned_blob(root, material_sha256, item, maximum, *, with_identity=False): +def read_owned_blob(root, material_sha256, item, maximum, *, with_identity=False, stream=False): """Read a flat owner inventory without following links or trusting row sizes.""" closed(item, ('name','sha256','bytes','media_type')) natural(item['bytes'],'Blob bytes',0,maximum) @@ -126,6 +127,23 @@ def read_owned_blob(root, material_sha256, item, maximum, *, with_identity=False before=os.fstat(source.fileno()) if not stat.S_ISREG(before.st_mode) or before.st_size!=item['bytes'] or before.st_nlink!=1: raise WorkbenchError('Retained blob is not an owned bounded regular file.','unavailable',409) + if stream: + from capture_payloads import FilePayload + def guard(): + parent = os.open(root, os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW) + selected = None + try: + selected = os.open(material_sha256, os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW, dir_fd=parent) + leaf = blob_identity(os.stat(item['name'], dir_fd=selected, follow_symlinks=False)) + if leaf != blob_identity(before): + raise WorkbenchError('Retained blob path changed.', 'unavailable', 409) + directory_id = lambda info: (info.st_dev,info.st_ino,info.st_mode,info.st_uid,info.st_gid) + return (directory_id(os.fstat(parent)), directory_id(os.fstat(selected)), leaf) + finally: + if selected is not None: os.close(selected) + os.close(parent) + raw=FilePayload(source.fileno(), maximum, expected_sha256=item['sha256'], guard=guard) + return (raw,blob_identity(before)) if with_identity else raw raw=source.read(item['bytes']+1);after=os.fstat(source.fileno()) if blob_identity(before)!=blob_identity(after) or len(raw)!=item['bytes'] or hashlib.sha256(raw).hexdigest()!=item['sha256']: raise WorkbenchError('Retained blob identity differs.','unavailable',409) diff --git a/pipeline.py b/pipeline.py index 82bd30ba..d9c9894c 100644 --- a/pipeline.py +++ b/pipeline.py @@ -338,10 +338,15 @@ def decode_recipe_material(value, *, catalog_id): for item in material['data']:reference(item,'release',catalog_id) envelope=material['resource_envelope'];closed(envelope,('attempt_slots','admitted_wall_seconds','retained_bytes','peak_staging_bytes')) for number in envelope.values():natural(number,'Resource envelope',1) + # This pure decoder owns descriptor bounds. The authenticated recipe bytes + # own the narrower capture policy before any large capture is read/copied. + capture_limit = (min(envelope['retained_bytes'], recipe_contract.MAX_CONFIGURED_CAPTURE_BYTES) + if material['trainer']['id'] == 'external_recipe_v1' + else recipe_contract.CPU_CAPTURE_BYTES) inventory=[material['recipe_document'],material['resolved_profile'],*material['captures']] names=[];total=0 for item in inventory: - closed(item,('name','sha256','bytes','media_type'));natural(item['bytes'],'Capture bytes',0,recipe_contract.CPU_CAPTURE_BYTES) + closed(item,('name','sha256','bytes','media_type'));natural(item['bytes'],'Capture bytes',0,capture_limit) if (type(item['name']) is not str or not re.fullmatch(r'(?:recipe\.json|profile\.json|capture-\d{4}\.bin)',item['name']) or type(item['sha256']) is not str or not HASH.fullmatch(item['sha256']) or item['media_type']!=('application/octet-stream' if item['name'].startswith('capture-') else 'application/json')): @@ -349,7 +354,7 @@ def decode_recipe_material(value, *, catalog_id): if item['name'] in ('recipe.json','profile.json') and item['bytes']>2*1024**2: raise WorkbenchError('Recipe/profile document exceeds its bound.','unavailable',409) names.append(item['name']);total+=item['bytes'] - if len(set(names))!=len(names) or total>recipe_contract.CPU_CAPTURE_BYTES+2*1024**2: + if len(set(names))!=len(names) or total>capture_limit+2*1024**2: raise WorkbenchError('Recipe inventory exceeds its bound.','unavailable',409) if material['recipe_document']['name']!='recipe.json' or material['resolved_profile']['name']!='profile.json': raise WorkbenchError('Recipe document names differ.','unavailable',409) @@ -1215,7 +1220,7 @@ def prepare_builtin_material(self, graph_id, revision, model_ref, *, reconcile_c try: for name,raw in payloads.items(): with (staging/name).open('xb') as handle: - handle.write(raw);handle.flush();os.fchmod(handle.fileno(),0o400);os.fsync(handle.fileno()) + recipe_contract.copy_payload(raw,handle);handle.flush();os.fchmod(handle.fileno(),0o400);os.fsync(handle.fileno()) _sync_directory(staging) return {'document':document,'staging':staging,'graph':graph,'orphan':None,'reconcile_capture':False} except BaseException: @@ -1240,7 +1245,7 @@ def _prepare_builtin_recipe_material_current(self, value): raise WorkbenchError('Retained built-in execution changed.', 'stale', 409) captured={item['path']:read_owned_blob(self.material_path,sha,item['blob'],recipe_contract.CPU_CAPTURE_BYTES) for item in snapshot['capture_paths']} expected=self.definition_identity(graph['recipe'])['definition_files'] - if {name:hashlib.sha256(raw).hexdigest() for name,raw in captured.items()}!=expected: + if {name:recipe_contract.file_digest(raw) for name,raw in captured.items()}!=expected: raise WorkbenchError('Retained adapter source differs.', 'stale', 409) return {'kind':'builtin','graph':graph,'validation':validation, 'release':self.inspect_release(validation['release']['id'],private=True,recipe=graph['recipe']), @@ -1274,7 +1279,7 @@ def prepare_external_material(self, graph_id, revision, model_ref, *, reconcile_ for path,raw in captured.items(): if path.startswith('source/'): original=source_map.get(path[len('source/'):]) - if original is None or original['sha256']!=hashlib.sha256(raw).hexdigest() or original['bytes']!=len(raw): + if original is None or original['sha256']!=recipe_contract.file_digest(raw) or original['bytes']!=len(raw): raise WorkbenchError('Model and execution source inventories differ.','stale',409) payloads={};paths=[] for number,(path,raw) in enumerate(sorted(captured.items())): @@ -1330,18 +1335,22 @@ def definition(kind,name,value): 'captures':[item['blob'] for item in paths],'resource_envelope':envelope} material=checked_json(material);sha=digest(material) document={'ref':{'kind':'recipe_material','catalog_id':self.db.dataset_id,'id':sha,'sha256':sha},'material':material} + if 'capture_policy' in preflight['material']['recipe']: + needed = envelope['peak_staging_bytes'] + 1024**3 + if shutil.disk_usage(self.material_path).free < needed: + raise WorkbenchError('Insufficient disk for retained capture and execution staging.', 'too_large', 409) staging=Path(tempfile.mkdtemp(prefix='.recipe-',dir=self.material_path)) try: for name,raw in payloads.items(): with (staging/name).open('xb') as handle: - handle.write(raw);handle.flush();os.fchmod(handle.fileno(),0o400);os.fsync(handle.fileno()) + recipe_contract.copy_payload(raw,handle);handle.flush();os.fchmod(handle.fileno(),0o400);os.fsync(handle.fileno()) _sync_directory(staging) if self.get(graph_id)!=graph: raise WorkbenchError('Recipe changed during capture.','stale',409) orphan=None;target=self.material_path/sha if reconcile_capture and target.exists(): - files={name:read_owned_blob(self.material_path,sha,blob(raw,name),recipe_contract.CPU_CAPTURE_BYTES, - with_identity=True)[1] for name,raw in payloads.items()} + files={name:read_owned_blob(self.material_path,sha,blob(raw,name),recipe_contract.capture_limits(preflight['material']['recipe'])[2], + with_identity=True,stream='capture_policy' in preflight['material']['recipe'])[1] for name,raw in payloads.items()} orphan=owned_inventory(self.material_path,sha,payloads) if orphan['files']!=files:raise WorkbenchError('Orphan recipe changed after hashing.','unavailable',409) return {'document':document,'staging':staging,'graph':graph,'orphan':orphan,'reconcile_capture':reconcile_capture} @@ -1400,6 +1409,8 @@ def _recipe_material_record(self, ref): def recipe_material(self, ref, *, verify=False): from model_definitions import read_owned_blob,retained_json,ModelDefinitions value=self._recipe_material_record(ref);material=value['material'] + capture_limit = recipe_contract.CPU_CAPTURE_BYTES + streamed = False if material['trainer']['id'] in ('external_recipe_v1',recipe_contract.EVIDENCE_FORMAT): # Current disclosure of retained captures follows all Run inputs, # independently from their historical integrity and exact grant. @@ -1415,12 +1426,14 @@ def recipe_material(self, ref, *, verify=False): or preflight['material']['graph_execution_sha256']!=execution_digest(graph)): raise WorkbenchError('Retained input material binding differs.','unavailable',409) recipe_contract.validate_material(preflight['material']) + capture_limit = recipe_contract.capture_limits(preflight['material']['recipe'])[2] + streamed = 'capture_policy' in preflight['material']['recipe'] from candidate_packages import authorize_material_inputs authorize_material_inputs(self.dataset.runs,preflight['material']) if evidence:self._retained_extension_static(value) if verify: inventory=[material['recipe_document'],material['resolved_profile'],*material['captures']] - for item in inventory:read_owned_blob(self.material_path,ref['sha256'],item,recipe_contract.CPU_CAPTURE_BYTES) + for item in inventory:read_owned_blob(self.material_path,ref['sha256'],item,capture_limit,stream=streamed) (getattr(self.dataset,'models',None) or ModelDefinitions(self.dataset)).material(material['model'],verify=True) return value @@ -1462,7 +1475,9 @@ def _prepare_recipe_material(self, ref, *, _verified=None): closed(item,('path','blob')) if item['path'] in retained or item['blob'] not in material['captures']: raise WorkbenchError('Capture mapping differs.','unavailable',409) - retained[item['path']]=read_owned_blob(self.material_path,ref['sha256'],item['blob'],recipe_contract.CPU_CAPTURE_BYTES) + retained[item['path']]=read_owned_blob(self.material_path,ref['sha256'],item['blob'], + recipe_contract.capture_limits(preflight['material']['recipe'])[2], + stream='capture_policy' in preflight['material']['recipe']) if recipe_contract.is_bound_evidence(preflight['material']['recipe']): recipe_contract.revalidate_bound_material(self.dataset.projects,preflight['material'],retained, getattr(self.dataset,'external_runtime_profile',None),releases=self.dataset.releases) diff --git a/projects.py b/projects.py index 78869541..4a55f852 100644 --- a/projects.py +++ b/projects.py @@ -447,16 +447,16 @@ def read_snapshot_recipe_sources(self, project_id, sources): return self._read_recipe_sources(project_id, sources, extensions=True) @authorized('inspect', 'projects.read_data_reference') - def read_data_reference(self, project_id, reference_id, expected_sha256, limit): + def read_data_reference(self, project_id, reference_id, expected_sha256, limit, *, stream=False): """Capture a selected saved reference; source owners interpret its bytes.""" saved = self._saved(project_id, extensions=True) return self._read_data_reference(project_id, reference_id, expected_sha256, limit, - extensions=saved['manifest']['schema_version'] == 2) + extensions=saved['manifest']['schema_version'] == 2, stream=stream) - def _read_data_reference(self, project_id, reference_id, expected_sha256, limit, *, extensions=False): + def _read_data_reference(self, project_id, reference_id, expected_sha256, limit, *, extensions=False, stream=False): if not isinstance(expected_sha256, str) or not SHA.fullmatch(expected_sha256): raise WorkbenchError('Import requires the observed source SHA256.') - if type(limit) is not int or not 0 < limit <= DATA_LIMIT: + if type(stream) is not bool or type(limit) is not int or not 0 < limit <= (2**53 - 1 if stream else DATA_LIMIT): raise WorkbenchError('Invalid bounded data read limit.') saved = self._snapshot_saved(project_id)[0] if extensions else self._saved(project_id) if not self._bound(saved): @@ -465,13 +465,24 @@ def _read_data_reference(self, project_id, reference_id, expected_sha256, limit, if ref is None: raise WorkbenchError('Data reference not found.', 'not_found', 404) try: - if extensions: + if stream: + from capture_payloads import FilePayload + before = self._snapshot_identity(saved, ref['path']) + def guard(): + return self._snapshot_identity(saved, ref['path']) + with self._file(saved['root'], ref['path']) as (fd, _info): + raw = FilePayload(fd, limit, expected_sha256=expected_sha256, guard=guard) + if before != guard(): + raw.close() + raise WorkbenchError('Snapshot path changed during capture.', 'conflict', 409) + elif extensions: raw = self._read_snapshot_file(saved, ref['path'], limit) else: raw = self._read(saved['root'], ref['path'], limit) except OSError: raise WorkbenchError('Data reference cannot be safely read.', 'unavailable', 409) from None - digest = hashlib.sha256(raw).hexdigest() + from capture_payloads import payload_digest + digest = payload_digest(raw) if digest != expected_sha256 or ref.get('sha256', digest) != digest: raise WorkbenchError('Data changed or differs from its declared SHA256. Inspect again before importing.', 'conflict', 409) if extensions: diff --git a/recipe_contract.py b/recipe_contract.py index 351d3730..2c1f85fc 100644 --- a/recipe_contract.py +++ b/recipe_contract.py @@ -10,6 +10,7 @@ from projects import closed, decode, relative_path from recipe_process import CUDA_WALL_SECONDS_LIMIT from workbench import WorkbenchError, encode +from capture_payloads import FORMAT as STREAMED_CAPTURE, payload_digest, copy_payload NAME = re.compile(r'[A-Za-z][A-Za-z0-9_-]{0,63}') SHA = re.compile(r'[a-f0-9]{64}') @@ -18,6 +19,7 @@ CPU_INPUT_BYTES = 256 * 1024 * 1024 CPU_CAPTURE_BYTES = 320 * 1024 * 1024 MAX_FILES = 128 +MAX_CONFIGURED_CAPTURE_BYTES = 2**53 - 1 RUNTIME_LIMIT = 512 * 1024 * 1024 TYPES = ('input', 'execute', 'output') PORTS = {'input': (None, 'data'), 'execute': ('data', 'artifacts'), 'output': ('artifacts', None)} @@ -32,7 +34,7 @@ INPUT_EVIDENCE = 'runtime-bootstrap/input-evidence.json' CPU_DESCRIPTOR = 'runtime-bootstrap/snapshot.json' CPU_BOOTSTRAP = 'runtime-bootstrap/recipe_process.py' -HOST_NAMES = ('recipe_contract.py', 'projects.py', 'pipeline.py', 'recipe_process.py') +HOST_NAMES = ('recipe_contract.py', 'projects.py', 'pipeline.py', 'recipe_process.py', 'capture_payloads.py', 'model_definitions.py') EVIDENCE_HOST_NAMES = (*HOST_NAMES, 'extension_evidence.py', 'field_evidence.py', 'scientific_fields.py', 'scientific_release.py') INDEXED_POLICY_FORMAT = 'extension_field_policy_indexed_v1' DIAGNOSTIC_HOST_NAMES = ('tuldok_extension_sdk/__init__.py', 'tuldok_extension_sdk/evidence.py', 'tuldok_extension_sdk/diagnostics.py') @@ -82,7 +84,7 @@ def digest(value): def file_digest(raw): - return hashlib.sha256(raw).hexdigest() + return payload_digest(raw) def bounded_int(value, low, high, label): @@ -91,6 +93,24 @@ def bounded_int(value, low, high, label): return value +def capture_limits(definition): + """Explicit per-recipe byte budgets; omission preserves historical defaults.""" + policy = definition.get('capture_policy') + if 'capture_policy' not in definition: + managed = definition['runtime'] in (CPU_PROFILE, TORCH_PROFILE) + return (40*1024**2, CPU_INPUT_BYTES if managed else MAX_CAPTURE_BYTES, + CPU_CAPTURE_BYTES if managed else MAX_CAPTURE_BYTES) + if definition['runtime'] != TORCH_PROFILE or is_evidence(definition): + raise WorkbenchError('Streamed inputs require an explicit managed Torch opaque recipe.', 'unsupported', 400) + closed(policy, ('format', 'max_input_file_bytes', 'max_input_bytes', 'max_capture_bytes')) + if policy['format'] != STREAMED_CAPTURE: + raise WorkbenchError('Unsupported input capture policy.') + total = bounded_int(policy['max_capture_bytes'], 1, MAX_CONFIGURED_CAPTURE_BYTES, 'Capture budget') + inputs = bounded_int(policy['max_input_bytes'], 1, total, 'Input aggregate budget') + per_file = bounded_int(policy['max_input_file_bytes'], 1, inputs, 'Input file budget') + return per_file, inputs, total + + def named(value): if not isinstance(value, str) or not NAME.fullmatch(value): raise WorkbenchError('Recipe names must use bounded letters, digits, underscore or hyphen.') @@ -102,7 +122,7 @@ def decode_definition(raw): raise WorkbenchError('Recipe definition exceeds 64 KiB.') value = decode(raw) closed(value, ('schema_version', 'format', 'id', 'name', 'version', 'runtime', 'sources', 'entrypoint', - 'arguments', 'parameters', 'inputs', 'outputs', 'environment', 'resources'), ('native_sources','runtime_snapshot_sha256','runtime_device','model_evidence','extension_selection')) + 'arguments', 'parameters', 'inputs', 'outputs', 'environment', 'resources'), ('native_sources','runtime_snapshot_sha256','runtime_device','model_evidence','extension_selection','capture_policy')) evidence = is_evidence(value) bound = is_bound_evidence(value) if bound != ('extension_selection' in value): @@ -134,6 +154,7 @@ def decode_definition(raw): raise WorkbenchError('Managed Torch recipes require an explicit CPU or CUDA device.') elif 'runtime_device' in value: raise WorkbenchError('Only managed Torch recipes select a device.') + input_file_limit, _input_limit, _capture_limit = capture_limits(value) sources = value['sources'] if not isinstance(sources, list) or not 1 <= len(sources) <= 64: raise WorkbenchError('Declare 1–64 source files.') @@ -174,7 +195,7 @@ def decode_definition(raw): raise WorkbenchError('Declare at most 32 named inputs.') for key, rule in value['inputs'].items(): named(key); closed(rule, ('max_bytes',)) - bounded_int(rule['max_bytes'], 1, 40 * 1024 * 1024, 'Input bytes') + bounded_int(rule['max_bytes'], 1, input_file_limit, 'Input bytes') argument_limit = 56 if value['runtime'] == CPU_PROFILE else 55 if value['runtime'] == TORCH_PROFILE else 59 if not isinstance(value['arguments'], list) or len(value['arguments']) > argument_limit: raise WorkbenchError('Recipe arguments exceed the runtime argument bound.') @@ -742,7 +763,8 @@ def prepare(projects, graph, execution_sha256, profile=None, *, releases=None, r else: closed(binding, ('reference_id', 'expected_sha256')) reader = projects.read_snapshot_data_reference if bound else projects.read_data_reference - raw, receipt = reader(config['project_id'], binding['reference_id'], binding['expected_sha256'], definition['inputs'][name]['max_bytes']) + options = {'stream': True} if 'capture_policy' in definition else {} + raw, receipt = reader(config['project_id'], binding['reference_id'], binding['expected_sha256'], definition['inputs'][name]['max_bytes'], **options) captured['input/' + name] = raw; inputs[name] = receipt if canonical: prefix={'catalog_id':projects.db.dataset_id,'recipe':definition,'inputs':inputs, @@ -750,8 +772,7 @@ def prepare(projects, graph, execution_sha256, profile=None, *, releases=None, r validate_canonical_captures(prefix,captured) if {'input_evidence': True} in definition['arguments']: captured[INPUT_EVIDENCE] = encode(inputs).encode() - capture_limit = CPU_CAPTURE_BYTES if definition['runtime'] in (CPU_PROFILE,TORCH_PROFILE) else MAX_CAPTURE_BYTES - input_limit = CPU_INPUT_BYTES if definition['runtime'] in (CPU_PROFILE,TORCH_PROFILE) else MAX_CAPTURE_BYTES + _file_limit, input_limit, capture_limit = capture_limits(definition) if sum(len(raw) for path,raw in captured.items() if path.startswith('input/')) > input_limit: raise WorkbenchError('Captured inputs exceed the runtime profile byte bound.') if len(captured) > MAX_FILES or sum(map(len,captured.values())) > capture_limit: @@ -936,8 +957,7 @@ def receipt(value): raise WorkbenchError('Frozen input receipt and captured size differ.') if path in generated and item['bytes'] != len(generated[path]): raise WorkbenchError('Generated native policy size differs.') if item['sha256'] != expected[path]: raise WorkbenchError('Captured digest contradicts source/input identity.') - capture_limit = CPU_CAPTURE_BYTES if definition['runtime'] in (CPU_PROFILE,TORCH_PROFILE) else MAX_CAPTURE_BYTES - input_limit = CPU_INPUT_BYTES if definition['runtime'] in (CPU_PROFILE,TORCH_PROFILE) else MAX_CAPTURE_BYTES + _file_limit, input_limit, capture_limit = capture_limits(definition) if total > capture_limit or input_total > input_limit: raise WorkbenchError('Captured material exceeds runtime profile byte bound.') runtime = material['runtime'] @@ -958,7 +978,12 @@ def receipt(value): host_names = CANONICAL_EVIDENCE_HOST_NAMES if is_canonical_evidence(definition) else BOUND_EVIDENCE_HOST_NAMES if bound else EVIDENCE_HOST_NAMES if evidence else HOST_NAMES if is_indexed_evidence(definition): host_names = (*host_names, *DIAGNOSTIC_HOST_NAMES) if definition['runtime'] == TORCH_PROFILE: host_names = (*host_names, *TORCH_HOST_NAMES) - if not isinstance(material['host_files'],dict) or set(material['host_files']) != set(host_names): raise WorkbenchError('Invalid retained host closure.') + allowed_names = [set(host_names)] + if 'capture_policy' not in definition: + # Both complete inventories were emitted by supported historical hosts. + allowed_names.append(set(host_names) - {'capture_payloads.py', 'model_definitions.py'}) + if not isinstance(material['host_files'],dict) or set(material['host_files']) not in allowed_names: + raise WorkbenchError('Invalid retained host closure.') for value in material['host_files'].values(): sha(value) return material @@ -997,6 +1022,7 @@ def spec_for_material(material, staging, *, context=None): if 'native_policies' in material or 'cpu_snapshot' in material: spec['native_loader_policy'] = NATIVE_POLICY if 'cpu_snapshot' in material: spec['cpu_snapshot'] = material['cpu_snapshot'] if 'torch_runtime' in material: spec['torch_runtime'] = material['torch_runtime'] + if 'capture_policy' in definition: spec['capture_policy'] = dict(definition['capture_policy']) if 'profile' in definition['resources']: spec['resource_profile'] = definition['resources']['profile'] if is_evidence(definition): spec['schema_version']=2 @@ -1037,7 +1063,7 @@ def capture(preflight, captured, staging): for relative, raw in captured.items(): path = staging/relative; path.parent.mkdir(mode=0o700,parents=True,exist_ok=True) with path.open('xb') as handle: - handle.write(raw); handle.flush(); os.fchmod(handle.fileno(),0o400); os.fsync(handle.fileno()) + copy_payload(raw, handle); handle.flush(); os.fchmod(handle.fileno(),0o400); os.fsync(handle.fileno()) for directory, _dirs, _files in os.walk(staging, topdown=False): _sync_directory(directory) _sync_directory(staging.parent) diff --git a/recipe_process.py b/recipe_process.py index a476f84e..49694f11 100644 --- a/recipe_process.py +++ b/recipe_process.py @@ -166,16 +166,37 @@ def absolute(value): TORCH_VERIFY_SECONDS = 60 +def capture_policy(spec): + policy = spec.get('capture_policy', spec.get('recipe', {}).get('capture_policy')) + if 'capture_policy' not in spec and 'capture_policy' not in spec.get('recipe', {}): + return None + expected = {'format', 'max_input_file_bytes', 'max_input_bytes', 'max_capture_bytes'} + if (type(policy) is not dict or set(policy) != expected + or policy['format'] != 'streamed_project_inputs_v1' + or any(type(policy[n]) is not int for n in expected - {'format'}) + or not 1 <= policy['max_input_file_bytes'] <= policy['max_input_bytes'] <= policy['max_capture_bytes'] <= 2**53-1): + raise ProcessError('Invalid streamed capture budget.') + return policy + + +def streamed_verification_seconds(spec, minimum): + policy = capture_policy(spec) + if policy is None: + return minimum + # Record conservative disk-I/O allowance separately from training wall time. + return minimum + (policy['max_capture_bytes'] + 64*1024**2 - 1) // (64*1024**2) + + def preparation_seconds(spec): """Custody readiness budget; independent of the recipe's child wall time.""" if 'torch_runtime' in spec: - return TORCH_READY_SECONDS + return streamed_verification_seconds(spec, TORCH_READY_SECONDS) return CPU_READY_SECONDS if 'cpu_snapshot' in spec else READY_SECONDS def verification_seconds(spec): if 'torch_runtime' in spec: - return TORCH_VERIFY_SECONDS + return streamed_verification_seconds(spec, TORCH_VERIFY_SECONDS) return CPU_READY_SECONDS if 'cpu_snapshot' in spec else 0 @@ -434,7 +455,7 @@ def _validate_spec(value, *, current): if type(value) is not dict or type(value.get('schema_version')) is not int or value['schema_version'] not in (1,2): raise ProcessError('Unsupported custody spec.') if value['schema_version']==2:keys.add('model_evidence_context') - if set(value)-{'native_loader_policy','cpu_snapshot','torch_runtime','resource_profile'}!=keys: + if set(value)-{'native_loader_policy','cpu_snapshot','torch_runtime','resource_profile','capture_policy'}!=keys: raise ProcessError('Unsupported custody spec.') if not isinstance(value['attempt_id'], str) or not ID.fullmatch(value['attempt_id']): raise ProcessError('Invalid attempt identity.') @@ -463,6 +484,9 @@ def _validate_spec(value, *, current): runtime = value['runtime_files'] if type(runtime) is not list or len(runtime)>10000: raise ProcessError('Invalid runtime inventory.') + policy = capture_policy(value) + if policy is not None and ('torch_runtime' not in value or value['schema_version'] != 1): + raise ProcessError('Streamed inputs require managed Torch opaque execution.') files = value['captured_files'] if type(files) is not list or len(files) > 128: raise ProcessError('Invalid captured file inventory.') @@ -473,7 +497,9 @@ def _validate_spec(value, *, current): if item['path'] in seen or type(item['sha256']) is not str or not HASH.fullmatch(item['sha256']): raise ProcessError('Duplicate capture or invalid hash.') seen.add(item['path']) - if type(item['bytes']) is not int or not 0 <= item['bytes'] <= CAPTURE_FILE_LIMIT: + item_limit = (policy['max_input_file_bytes'] if policy is not None and item in files + and Path(item['path']).parent == Path(value['cwd'])/'input' else CAPTURE_FILE_LIMIT) + if type(item['bytes']) is not int or not 0 <= item['bytes'] <= item_limit: raise ProcessError('Invalid captured file size.') if 'cpu_snapshot' in value: descriptor = validate_cpu_snapshot_descriptor(value['cpu_snapshot']) @@ -518,7 +544,7 @@ def _validate_spec(value, *, current): capture_limit = 64 * 1024 * 1024 if 'cpu_snapshot' in value or 'torch_runtime' in value: # The complete descriptor/bootstrap contract above qualifies this profile. - capture_limit = 320 * 1024 * 1024 + capture_limit = policy['max_capture_bytes'] if policy else 320 * 1024 * 1024 input_root = Path(value['cwd']) / 'input' input_bytes = 0 for item in files: @@ -526,10 +552,10 @@ def _validate_spec(value, *, current): if relative is None or len(relative.parts) < 2 or relative.parts[0] not in ('source', 'input', 'native-policy', 'runtime-bootstrap'): raise ProcessError('CPU capture must use its contained source/input/runtime layout.') if relative.parts[0] == 'input': - if Path(item['path']).parent != input_root or item['bytes'] > 40 * 1024 * 1024: + if Path(item['path']).parent != input_root or item['bytes'] > (policy['max_input_file_bytes'] if policy else 40 * 1024 * 1024): raise ProcessError('CPU input must be a bounded named input file.') input_bytes += item['bytes'] - if input_bytes > 256 * 1024 * 1024: + if input_bytes > (policy['max_input_bytes'] if policy else 256 * 1024 * 1024): raise ProcessError('CPU input aggregate exceeds its byte bound.') if sum(item['bytes'] for item in files) > capture_limit or sum(item['bytes'] for item in runtime) > 512 * 1024 * 1024: raise ProcessError('Runtime/source closure exceeds its byte bound.') @@ -613,6 +639,48 @@ def verify_model_context(spec): if handle is not None:os.close(handle) +def verify_streamed_capture(item, *, checkpoint=None): + """Hash without materializing a large captured file, retaining path custody.""" + descriptors = [] + before_dirs = [] + directory = os.open('/', os.O_RDONLY|os.O_DIRECTORY) + descriptors.append(directory) + parts = Path(item['path']).parts[1:] + directory_identity = lambda x: (x.st_dev, x.st_ino, x.st_mode, x.st_uid, x.st_gid) + file_identity = lambda x: (x.st_dev, x.st_ino, x.st_mode, x.st_uid, x.st_gid, + x.st_nlink, x.st_size, x.st_mtime_ns, x.st_ctime_ns) + try: + for part in parts[:-1]: + child = os.open(part, os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW, dir_fd=directory) + before_dirs.append((directory, part, directory_identity(os.fstat(child)))) + descriptors.append(child) + directory = child + descriptor = os.open(parts[-1], os.O_RDONLY|os.O_NOFOLLOW|os.O_NONBLOCK, dir_fd=directory) + descriptors.append(descriptor) + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or before.st_size != item['bytes']: + raise ProcessError('Captured input size or type changed.', 'stale') + measured = hashlib.sha256() + size = 0 + while size < item['bytes']: + if checkpoint is not None: checkpoint() + chunk = os.read(descriptor, min(1024*1024, item['bytes'] - size)) + if not chunk: raise ProcessError('Captured input truncated.', 'stale') + measured.update(chunk) + size += len(chunk) + if (file_identity(before) != file_identity(os.fstat(descriptor)) + or file_identity(before) != file_identity(os.stat(parts[-1], dir_fd=directory, follow_symlinks=False)) + or measured.hexdigest() != item['sha256']): + raise ProcessError('Captured input/source identity changed.', 'stale') + for parent, name, expected in before_dirs: + if directory_identity(os.stat(name, dir_fd=parent, follow_symlinks=False)) != expected: + raise ProcessError('Captured parent path changed.', 'stale') + except OSError as error: + raise ProcessError('Captured input path unavailable.', 'stale') from error + finally: + for descriptor in reversed(descriptors): os.close(descriptor) + + def verify_captures(spec, *, checkpoint=None, include_cpu=True, include_torch=True, on_torch_inventory=None): verify_model_context(spec) @@ -622,9 +690,12 @@ def verify_captures(spec, *, checkpoint=None, include_cpu=True, include_torch=Tr verify_native_loader_policy(spec['trusted_install_owner_uid']) for item in spec['captured_files']+spec['runtime_files']: if checkpoint is not None: checkpoint() - raw = read_regular(item['path'], item['bytes']) - if len(raw) != item['bytes'] or digest(raw) != item['sha256']: - raise ProcessError('Captured input/source identity changed.', 'stale') + if 'capture_policy' in spec: + verify_streamed_capture(item, checkpoint=checkpoint) + else: + raw = read_regular(item['path'], item['bytes']) + if len(raw) != item['bytes'] or digest(raw) != item['sha256']: + raise ProcessError('Captured input/source identity changed.', 'stale') if 'torch_runtime' in spec: owner = _torch_owner() try: diff --git a/tests/test_streamed_payloads.py b/tests/test_streamed_payloads.py new file mode 100644 index 00000000..8d6c9874 --- /dev/null +++ b/tests/test_streamed_payloads.py @@ -0,0 +1,136 @@ +"""Large-file custody, bounded reads and mutation controls; no model execution.""" +import hashlib +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from capture_payloads import FilePayload, CHUNK_BYTES, copy_payload, payload_digest +from model_definitions import blob, read_owned_blob +from projects import Projects +import recipe_process +from workbench import WorkbenchError + + +class StreamedPayloadTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.path = self.root/'input.bin' + digest = hashlib.sha256() + chunk = bytes(range(256))*4096 + with self.path.open('wb') as handle: + for _ in range(41): + handle.write(chunk) + digest.update(chunk) + self.expected = digest.hexdigest() + + def payload(self): + with self.path.open('rb') as handle: + raw = FilePayload(handle.fileno(), self.path.stat().st_size, expected_sha256=self.expected) + self.addCleanup(raw.close) + return raw + + def test_hash_and_copy_above_old_file_cap_are_bounded_and_exact(self): + read = os.pread + sizes = [] + def observed(fd, size, offset): + sizes.append(size) + return read(fd, size, offset) + with patch('capture_payloads.os.pread', side_effect=observed): + raw = self.payload() + self.assertEqual(payload_digest(raw), self.expected) + with (self.root/'copy.bin').open('wb') as destination: + copy_payload(raw, destination) + self.assertGreater(len(sizes), 3) + self.assertLessEqual(max(sizes), CHUNK_BYTES) + item = blob(raw, 'capture-0000.bin') + self.assertEqual(item['bytes'], self.path.stat().st_size) + recipe_process.verify_streamed_capture(dict(path=str(self.root/'copy.bin'), + bytes=item['bytes'], sha256=self.expected)) + + def test_same_size_rewrite_with_restored_mtime_is_rejected(self): + raw = self.payload() + before = self.path.stat() + with self.path.open('r+b') as handle: + handle.write(b'changed') + os.utime(self.path, ns=(before.st_atime_ns, before.st_mtime_ns)) + with self.assertRaises(WorkbenchError): payload_digest(raw) + + def test_truncation_and_wrong_hash_are_rejected(self): + raw = self.payload() + with self.path.open('r+b') as handle: handle.truncate(1) + with self.assertRaises(WorkbenchError): payload_digest(raw) + with self.path.open('rb') as handle, self.assertRaises(WorkbenchError): + FilePayload(handle.fileno(), 100, expected_sha256='0'*64) + + def test_project_capture_retains_path_and_allows_unrelated_publication(self): + projects = Projects(SimpleNamespace(access=None, db=None, lock=None), {'fixture':self.root}) + self.addCleanup(projects.close) + saved = dict(root='fixture', root_identity=projects.roots['fixture'][1], + manifest_sha256='a'*64, manifest=dict(data=[dict(id='sample', path='input.bin', sha256=self.expected)])) + with patch.object(projects, '_snapshot_saved', return_value=(saved,b'')): + raw, receipt = projects._read_data_reference('b'*32, 'sample', self.expected, + self.path.stat().st_size, extensions=True, stream=True) + self.addCleanup(raw.close) + self.assertIsInstance(raw, FilePayload) + self.assertEqual(receipt['measured_sha256'], self.expected) + (self.root/'independent').mkdir() + self.assertEqual(raw.digest(), self.expected) + self.path.rename(self.root/'previous.bin') + self.path.symlink_to(self.root/'previous.bin') + with self.assertRaises(WorkbenchError): raw.digest() + + def test_owned_blob_streaming_and_replaced_file_are_checked(self): + owner = self.root/'materials' + directory = owner/('a'*64) + directory.mkdir(parents=True) + target = directory/'capture-0000.bin' + self.path.rename(target) + item = dict(name=target.name, bytes=target.stat().st_size, sha256=self.expected, + media_type='application/octet-stream') + raw = read_owned_blob(owner,'a'*64,item,item['bytes'],stream=True) + self.addCleanup(raw.close) + self.assertIsInstance(raw,FilePayload) + (owner/('b'*64)).mkdir() + self.assertEqual(raw.digest(),self.expected) + target.rename(directory/'previous.bin') + target.write_bytes(b'x') + with self.assertRaises(WorkbenchError): raw.digest() + + def test_process_verification_rejects_symlink_parent_and_changed_content(self): + item = dict(path=str(self.path),bytes=self.path.stat().st_size,sha256=self.expected) + recipe_process.verify_streamed_capture(item) + alias = self.root/'alias' + alias.symlink_to(self.root, target_is_directory=True) + with self.assertRaises(recipe_process.ProcessError): + recipe_process.verify_streamed_capture(dict(item,path=str(alias/'input.bin'))) + with self.path.open('r+b') as handle: handle.write(b'changed') + with self.assertRaises(recipe_process.ProcessError): recipe_process.verify_streamed_capture(item) + + def test_process_verification_honors_cancellation_within_large_file(self): + calls = [] + class Cancelled(Exception): pass + def checkpoint(): + calls.append(True) + if len(calls)==2: raise Cancelled() + with self.assertRaises(Cancelled): + recipe_process.verify_streamed_capture(dict(path=str(self.path), + bytes=self.path.stat().st_size,sha256=self.expected),checkpoint=checkpoint) + self.assertEqual(len(calls),2) + + def test_small_owned_shards_are_also_file_backed_in_streamed_mode(self): + owner=self.root/'small-materials';directory=owner/('a'*64) + directory.mkdir(parents=True) + target=directory/'capture-0000.bin';target.write_bytes(b'small shard') + item=blob(b'small shard',target.name) + raw=read_owned_blob(owner,'a'*64,item,1024,stream=True) + self.addCleanup(raw.close) + self.assertIsInstance(raw,FilePayload) + self.assertEqual(payload_digest(raw),item['sha256']) + + +if __name__=='__main__': unittest.main() diff --git a/tests/test_streamed_recipe.py b/tests/test_streamed_recipe.py new file mode 100644 index 00000000..c3c49d2c --- /dev/null +++ b/tests/test_streamed_recipe.py @@ -0,0 +1,151 @@ +"""Real Project/material/staging integration above both old byte bounds.""" +import copy +import hashlib +import json +from pathlib import Path +import unittest +from unittest.mock import patch + +from capture_payloads import FilePayload +from model_definitions import ModelDefinitions +import recipe_contract as contract +import recipe_process +import test_recipe_contract as fixtures +import test_torch_runtime as torch_fixtures +import torch_runtime +from workbench import WorkbenchError + + +def sha_file(path): + h=hashlib.sha256() + with path.open('rb') as source: + while chunk:=source.read(1024**2): h.update(chunk) + return h.hexdigest() + + +class StreamedRecipeTests(unittest.TestCase): + def setUp(self): + fixtures.RecipePipelineTests.setUp(self) + torch_fixtures.InstalledAssets.setUp(self) + manifest=torch_runtime.inventory(self.config) + self.runtime_raw=torch_runtime.encoded(manifest) + identity=hashlib.sha256(self.runtime_raw).hexdigest() + self.descriptor=dict(schema_version=1,profile=torch_runtime.PROFILE, + catalog_id=self.dataset.db.dataset_id,manifest_sha256=identity, + manifest_bytes=len(self.runtime_raw),file_count=len(manifest['files']),total_bytes=manifest['total_bytes']) + self.value.update(runtime=contract.TORCH_PROFILE,runtime_device='cuda',runtime_snapshot_sha256=identity, + capture_policy=dict(format='streamed_project_inputs_v1',max_input_file_bytes=300*1024**2, + max_input_bytes=300*1024**2,max_capture_bytes=400*1024**2)) + self.value['inputs']['sample']['max_bytes']=300*1024**2 + + def graph(self, *, large=False): + if large: + self.value['capture_policy'].update(max_input_file_bytes=400*1024**2, + max_input_bytes=400*1024**2,max_capture_bytes=512*1024**2) + self.value['inputs']['sample']['max_bytes']=400*1024**2 + with (self.project/'data.bin').open('wb') as handle: handle.truncate(350*1024**2) + self.recipe_raw=json.dumps(self.value).encode() + (self.project/'recipe.json').write_bytes(self.recipe_raw) + self.body['expected_sha256']=hashlib.sha256(self.recipe_raw).hexdigest() + self.body['inputs']['sample']['expected_sha256']=sha_file(self.project/'data.bin') + return self.dataset.pipelines.create_external(self.body) + + def runtime(self): + return patch('torch_runtimes.TorchRuntimes.get',return_value=(self.descriptor,self.runtime_raw)) + + def test_large_project_retained_material_and_complete_execution_spec(self): + graph=self.graph(large=True) + with self.runtime(),patch('subprocess.Popen',side_effect=AssertionError('Capture launched code')): + preflight,captured=contract.prepare(self.dataset.projects,graph,__import__('pipeline').execution_digest(graph)) + self.assertIsInstance(captured['input/sample'],FilePayload) + model_owner=ModelDefinitions(self.dataset) + draft=model_owner.create(dict(name='Stream capture fixture',kind='captured_source', + task=dict(kind='task',catalog_id=self.dataset.db.dataset_id,id='neutral.bytes',revision=1,sha256='a'*64), + specification=dict(format='capture_fixture_v1'),source_selector={k:self.body[k] for k in ('project_id','recipe_id','expected_sha256')}, + preprocessing_policy=dict(kind='identity'),output_policy=dict(kind='exact_bytes'))) + model=model_owner.publish(draft['id'],dict(revision=1)) + prepared=self.dataset.pipelines.prepare_external_material(graph['id'],1,model['ref']) + with self.dataset.lock,self.dataset.db.transaction(): + retained=self.dataset.pipelines._retain_material(prepared) + self.assertEqual(self.dataset.pipelines.recipe_material(retained['ref'],verify=True),retained) + stage=self.root/'large-stage' + execution=self.dataset.pipelines.capture_recipe_material(retained['ref'],stage) + spec=dict(execution['spec'],attempt_id='b'*32,request_sha256='c'*64) + recipe_process.validate_spec(spec) + recipe_process.verify_captures(spec,include_torch=False) + self.assertEqual(sha_file(stage/'input/sample'),self.body['inputs']['sample']['expected_sha256']) + self.assertEqual(spec['capture_policy'],self.value['capture_policy']) + self.assertIn('capture_payloads.py',preflight['material']['host_files']) + self.assertGreater(recipe_process.preparation_seconds(spec),60) + self.assertGreater(recipe_process.verification_seconds(spec),60) + self.assertEqual(self.dataset.runs.list(),[]) + bad=copy.deepcopy(spec);bad.pop('capture_policy') + with self.assertRaises(recipe_process.ProcessError):recipe_process.validate_spec(bad) + (stage/'input/sample').chmod(0o600) + with (stage/'input/sample').open('r+b') as handle:handle.write(b'changed') + with self.assertRaises(recipe_process.ProcessError):recipe_process.verify_captures(spec,include_torch=False) + + def test_configured_budgets_and_policy_identity_are_enforced(self): + for mutate in ( + lambda v:v.update(runtime='system_python'), + lambda v:v['capture_policy'].update(max_input_bytes=True), + lambda v:v['capture_policy'].update(max_capture_bytes=-1), + lambda v:v['capture_policy'].update(max_input_file_bytes=500*1024**2), + lambda v:v['capture_policy'].update(format='unknown'), + lambda v:v.update(capture_policy=None), + lambda v:v['capture_policy'].update(unrecorded=True), + ): + value=copy.deepcopy(self.value);mutate(value) + with self.subTest(value=value),self.assertRaises(WorkbenchError): + contract.decode_definition(json.dumps(value).encode()) + self.value['capture_policy'].update(max_input_file_bytes=5,max_input_bytes=5) + self.value['inputs']['sample']['max_bytes']=5 + graph=self.graph() + with self.runtime(),self.assertRaises(WorkbenchError):self.dataset.pipelines.preflight_external(graph['id'],1) + + def test_explicit_total_budget_failure_and_legacy_defaults(self): + old=copy.deepcopy(self.value);old.pop('capture_policy') + with self.assertRaises(WorkbenchError):contract.decode_definition(json.dumps(old).encode()) + old['inputs']['sample']['max_bytes']=40*1024**2 + contract.decode_definition(json.dumps(old).encode()) + self.assertEqual(contract.capture_limits(old),(40*1024**2,256*1024**2,320*1024**2)) + self.value['capture_policy'].update(max_input_file_bytes=12,max_input_bytes=12,max_capture_bytes=12) + self.value['inputs']['sample']['max_bytes']=12 + graph=self.graph() + with self.runtime(),self.assertRaises(WorkbenchError):self.dataset.pipelines.preflight_external(graph['id'],1) + + def test_old_host_inventory_remains_cold_readable(self): + self.value.pop('capture_policy');self.value['inputs']['sample']['max_bytes']=1024 + graph=self.graph() + with self.runtime():material=self.dataset.pipelines.preflight_external(graph['id'],1)['material'] + material['host_files'].pop('capture_payloads.py') + material['host_files'].pop('model_definitions.py') + with patch.object(Path,'read_bytes',side_effect=AssertionError('Cold validation read installed files')): + contract.validate_material(material) + + def test_host_inventory_requires_one_complete_supported_layout(self): + self.value.pop('capture_policy');self.value['inputs']['sample']['max_bytes']=1024 + graph=self.graph() + with self.runtime():material=self.dataset.pipelines.preflight_external(graph['id'],1)['material'] + partial=copy.deepcopy(material) + partial['host_files'].pop('capture_payloads.py') + with self.assertRaises(WorkbenchError) as refused: + contract.validate_material(partial) + self.assertEqual(str(refused.exception),'Invalid retained host closure.') + unknown=copy.deepcopy(material) + unknown['host_files']['unregistered_host.py']='a'*64 + with self.assertRaises(WorkbenchError) as refused: + contract.validate_material(unknown) + self.assertEqual(str(refused.exception),'Invalid retained host closure.') + + def test_streamed_policy_cannot_use_the_legacy_host_layout(self): + graph=self.graph() + with self.runtime():material=self.dataset.pipelines.preflight_external(graph['id'],1)['material'] + material['host_files'].pop('capture_payloads.py') + material['host_files'].pop('model_definitions.py') + with self.assertRaises(WorkbenchError) as refused: + contract.validate_material(material) + self.assertEqual(str(refused.exception),'Invalid retained host closure.') + + +if __name__=='__main__':unittest.main() From f74842ad818972482867230846333e55d3ca7366 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 15:27:49 -0700 Subject: [PATCH 04/15] Allow larger project indexes and reject overflow before staging --- docs/contracts/project-files.md | 5 +++++ project_store.py | 21 ++++++++++++++++----- tests/test_project_store.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/docs/contracts/project-files.md b/docs/contracts/project-files.md index d78bfce8..ae88676a 100644 --- a/docs/contracts/project-files.md +++ b/docs/contracts/project-files.md @@ -31,6 +31,11 @@ checkpoint: project UUID, display name, membership roots, sorted record inventor request receipts and a SHA256 checkpoint. Its digest covers canonical bytes of the document with `checkpoint` omitted. The store supports only its declared format and family; malformed and future versions fail before projection writes. +The canonical checkpoint index is limited to 64 MiB, separately from the +100,000-record, 40 MiB-per-record and 256 MiB-total-record limits. Commit checks +the index size before staging immutable records. This format is unchanged; +older installations with the former 16 MiB index limit cannot open larger +checkpoints and must be upgraded before consuming them. Each inventory entry identifies a rooted immutable file `.tuldok/records///.json`, its format, namespace, diff --git a/project_store.py b/project_store.py index c72bebc5..9c4bd108 100644 --- a/project_store.py +++ b/project_store.py @@ -24,7 +24,9 @@ FORMAT = 'tuldok_project_v1' GITIGNORE = b'/artifacts/\n/.tuldok-local/\n/.venv/\n/__pycache__/\n/PGDATA/\n/output/\n' MAX_RECORD_BYTES = 40 * 1024 * 1024 -MAX_MANIFEST_BYTES = 16 * 1024 * 1024 +# A retained scientific runtime can declare tens of thousands of small files. +# Their index is bounded separately from the actual record/payload byte budgets. +MAX_MANIFEST_BYTES = 64 * 1024 * 1024 # Coordinated internal codecs; adding a family requires its owning decoder. SUPPORTED_FORMATS = { 'workbench_text': 'workbench_text_v1', @@ -115,6 +117,13 @@ def _encode(value): allow_nan=False) + '\n').encode('utf-8') +def _manifest_raw(manifest): + raw = _encode(manifest) + if len(raw) > MAX_MANIFEST_BYTES: + _fail('Project checkpoint exceeds its byte limit.') + return raw + + def _pairs(pairs): result = {} for key, value in pairs: @@ -720,9 +729,7 @@ def _record_file(self, family, record_id, digest, raw=None, sync=False): os.close(fd) def _publish(self, manifest, creating=False): - raw = _encode(manifest) - if len(raw) > MAX_MANIFEST_BYTES: - _fail('Project checkpoint exceeds its byte limit.') + raw = _manifest_raw(manifest) temporary = '.checkpoint-' + uuid.uuid4().hex + '.tmp' replaced = False try: @@ -794,7 +801,6 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256): inventory = [] for family, identity, document, raw in prepared: digest = _sha(raw) - self._record_file(family, identity, digest, raw) inventory.append(dict(family=family, id=identity, format=document['format'], namespace=current['project_id'], path=f'records/{family}/{identity}/{digest}.json', bytes=len(raw), sha256=digest)) @@ -804,6 +810,11 @@ def commit(self, records, expected_checkpoint, request_key, request_sha256): request_sha256=request_sha256, records_sha256=records_digest)) del manifest['checkpoint'] manifest['checkpoint'] = _sha(_encode(manifest)) + # Refuse before record staging. The final publication also + # rechecks this bound after the immutable files are durable. + _manifest_raw(manifest) + for family, identity, document, raw in prepared: + self._record_file(family, identity, _sha(raw), raw) self._publish(manifest) self._manifest = manifest return self._snapshot(manifest, normalized) diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 9c1c6271..98e52ad9 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -272,6 +272,34 @@ def test_closed_document_envelopes_and_supported_family_format(self): self.assert_code('unsupported', self.store.commit, [value], checkpoint, 'bad', body('bad')) self.assert_code('invalid', self.store.commit, [record(), record()], checkpoint, 'bad', body('bad')) + def test_large_checkpoint_index_publishes_and_cold_opens(self): + # The migrated lab's small record declarations produced a 17.99 MB + # index, even though individual records and their total were in bounds. + # Valid receipts exercise the same index publication/read boundary + # without tens of thousands of fsyncs in a focused regression fixture. + manifest = self.manifest() + manifest['receipts'].extend( + dict(request_key='operation.' + str(index).zfill(100), + request_sha256=body(index), records_sha256=sha(canonical([]))) + for index in range(52_000)) + manifest.pop('checkpoint') + manifest['checkpoint'] = sha(canonical(manifest)) + self.assertGreater(len(canonical(manifest)), 16 * 1024 * 1024) + self.store._publish(manifest) + reopened = self.reopen() + self.assertEqual(reopened['checkpoint'], manifest['checkpoint']) + self.assertEqual(reopened['records'], []) + self.assertEqual(self.manifest()['receipts'], manifest['receipts']) + + def test_oversized_checkpoint_refuses_before_staging_record_files(self): + before = self.bytes_tree() + # The existing checkpoint fits, but a new record's index would not. + maximum = len((self.root / '.tuldok/project.json').read_bytes()) + 10 + with patch.object(store_module, 'MAX_MANIFEST_BYTES', maximum): + error = self.assert_code('invalid', self.commit) + self.assertEqual(str(error), 'Project checkpoint exceeds its byte limit.') + self.assertEqual(self.bytes_tree(), before) + def test_future_record_format_rejected_on_open(self): self.commit() self.store.close() From d9467b2fc9550cb978196ef02b8032350ef9942d Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 15:45:35 -0700 Subject: [PATCH 05/15] Deduplicate verified bytes within private catalog backups --- backup_metadata.py | 60 ++++++++++++++++++++++- docs/contracts/project-files.md | 11 +++++ project_transfer.py | 2 +- tests/test_backup_storage.py | 83 ++++++++++++++++++++++++++++++++ tests/test_metadata_migration.py | 11 +++++ tests/test_project_transfer.py | 12 +++++ 6 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 tests/test_backup_storage.py diff --git a/backup_metadata.py b/backup_metadata.py index 0407bfcc..17745495 100644 --- a/backup_metadata.py +++ b/backup_metadata.py @@ -112,10 +112,61 @@ def tree_modes(path, expected): return result -def copy_tree(source, target, expected): +def _linked_backup_copy(source, target, anchor, entry, mode): + """Verify both contents before sharing an inode inside a private backup. + + The caller owns the fresh target tree exclusively. No link ever names an + original source file; all anchors were independently copied into this tree. + """ + def verified(path): + before = path.lstat() + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + opened = os.fstat(descriptor) + if (not stat.S_ISREG(opened.st_mode) or before != opened + or stat.S_IMODE(opened.st_mode) != mode): + raise StorageTransferError('Backup copy source identity or permissions changed.') + result = hashlib.sha256() + for chunk in iter(lambda: os.read(descriptor, 1024 * 1024), b''): + result.update(chunk) + after = os.fstat(descriptor) + fields = lambda info: (info.st_dev, info.st_ino, info.st_mode, info.st_size, + info.st_mtime_ns, info.st_ctime_ns, info.st_nlink) + if (fields(opened) != fields(after) or fields(path.lstat()) != fields(after) + or after.st_size != entry['bytes'] or result.hexdigest() != entry['sha256']): + raise StorageTransferError('Backup copy source bytes or binding changed.') + return descriptor, after + except BaseException: + os.close(descriptor) + raise + + source_fd, original = verified(source) + anchor_fd = None + try: + anchor_fd, copied = verified(anchor) + if (original.st_dev, original.st_ino) == (copied.st_dev, copied.st_ino): + raise StorageTransferError('Backup anchor must be independent of the original source.') + os.link(anchor, target, follow_symlinks=False) + linked = target.lstat() + if (not stat.S_ISREG(linked.st_mode) or + (linked.st_dev, linked.st_ino) != (copied.st_dev, copied.st_ino)): + raise StorageTransferError('Backup anchor changed during link publication.') + # The already-durable copied inode is unchanged; linking changes only + # its link count/ctime. The containing directories are synced by copy_tree. + os.fsync(anchor_fd) + finally: + os.close(source_fd) + if anchor_fd is not None: + os.close(anchor_fd) + + +def copy_tree(source, target, expected, *, deduplicate=False): """Preserve bytes and permissions, refusing changed or contradictory staging.""" source, target = Path(source), Path(target) + if deduplicate and target.exists(): + raise StorageTransferError('Deduplicated backup requires an absent destination tree.') modes = tree_modes(source, expected) + copies = {} target.mkdir(parents=True, exist_ok=True) for name, mode in modes.items(): directory = target / name @@ -135,10 +186,13 @@ def copy_tree(source, target, expected): target_file.mkdir(parents=True, exist_ok=True) else: target_file.parent.mkdir(parents=True, exist_ok=True) + content_key = (entry['sha256'], entry['bytes'], modes[name]) if target_file.exists(): if (not target_file.is_file() or file_hash(target_file) != entry['sha256'] or stat.S_IMODE(target_file.stat().st_mode) != modes[name]): raise StorageTransferError('Staged artifact bytes or permissions differ: ' + name) + elif deduplicate and content_key in copies: + _linked_backup_copy(source_file, target_file, copies[content_key], entry, modes[name]) else: # Never widen read-only staging permissions to force a retry. if not target_file.parent.stat().st_mode & stat.S_IWUSR: @@ -169,6 +223,8 @@ def copy_tree(source, target, expected): os.rename(temporary, target_file) finally: temporary.unlink(missing_ok=True) + if deduplicate: + copies.setdefault(content_key, target_file) if tree_modes(source, expected) != modes: raise StorageTransferError('Source permissions changed during copy.') directories = [name for name, entry in expected.items() if entry['kind'] == 'directory'] @@ -340,7 +396,7 @@ def backup_catalog(path, backup, *, dsn=None, quiesced=False): before = tree_manifest(path) tables = read_catalog(metadata) sequences = read_sequences(metadata) - copy_tree(path, temporary / 'catalog', before) + copy_tree(path, temporary / 'catalog', before, deduplicate=True) if (before != tree_manifest(path) or before != tree_manifest(temporary / 'catalog') or tree_modes(path, before) != tree_modes(temporary / 'catalog', before)): raise StorageTransferError('Artifacts changed during backup. Keep all writers stopped.') diff --git a/docs/contracts/project-files.md b/docs/contracts/project-files.md index ae88676a..7959d37d 100644 --- a/docs/contracts/project-files.md +++ b/docs/contracts/project-files.md @@ -37,6 +37,17 @@ the index size before staging immutable records. This format is unchanged; older installations with the former 16 MiB index limit cannot open larger checkpoints and must be upgraded before consuming them. +Verified stopped-source backups and ordinary catalog backups share identical +file contents only inside their fresh private backup tree, and only when byte +count, SHA256 and POSIX permission mode all match. The first copy has an +independent inode; no original source file is hard-linked into the backup. +Both source and copied content are verified before a duplicate link is made, +and the complete tree/hash/mode checks still run before backup publication. +The existing backup format remains a complete ordinary-file tree. Restore +copies every path into an independent file, preserving bytes and modes without +introducing writable aliases into the restored catalog. Backup trees are +immutable custody and must not be edited as working catalogs. + Each inventory entry identifies a rooted immutable file `.tuldok/records///.json`, its format, namespace, byte count and SHA256. Original `workbench_text_v1` preserves original source, current diff --git a/project_transfer.py b/project_transfer.py index 1507e71e..932d0963 100644 --- a/project_transfer.py +++ b/project_transfer.py @@ -297,7 +297,7 @@ def _verified_backup(source,parent_fd,directory): path=Path('/proc/self/fd')/str(parent_fd)/stage tables,sequences,fingerprint=_snapshot(source) before=tree_manifest(source.root_path) - copy_tree(source.root_path,path/'catalog',before) + copy_tree(source.root_path,path/'catalog',before,deduplicate=True) metadata=dict(format_version=1,schema_version=source.db.schema_version,dataset_id=source.db.dataset_id,tables=tables,sequences=sequences) from backup_metadata import write_json,file_hash write_json(path/'metadata.json',metadata) diff --git a/tests/test_backup_storage.py b/tests/test_backup_storage.py new file mode 100644 index 00000000..87f721b2 --- /dev/null +++ b/tests/test_backup_storage.py @@ -0,0 +1,83 @@ +"""Independent original files, compact backup storage and independent restores.""" +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from backup_metadata import StorageTransferError, copy_tree, tree_manifest, tree_modes + + +class BackupStorageTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root / 'source' + self.source.mkdir() + for name in ('first.bin', 'nested/second.bin', 'other-mode.bin'): + path = self.source / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'Exact captured training data\x00\xff' * 100) + path.chmod(0o444 if name == 'other-mode.bin' else 0o644) + (self.source / 'empty-directory').mkdir() + self.expected = tree_manifest(self.source) + + def test_backup_shares_only_own_equal_bytes_and_modes_and_restores_independently(self): + before = {p: (p.stat().st_ino, p.stat().st_ctime_ns, p.stat().st_nlink) + for p in self.source.rglob('*') if p.is_file()} + backup = self.root / 'backup' + copy_tree(self.source, backup, self.expected, deduplicate=True) + self.assertEqual(tree_manifest(backup), self.expected) + self.assertEqual(tree_modes(backup, self.expected), tree_modes(self.source, self.expected)) + first, second = backup/'first.bin', backup/'nested/second.bin' + self.assertEqual(first.stat().st_ino, second.stat().st_ino) + self.assertNotEqual(first.stat().st_ino, (backup/'other-mode.bin').stat().st_ino) + self.assertEqual(before, {p: (p.stat().st_ino, p.stat().st_ctime_ns, p.stat().st_nlink) + for p in before}) + self.assertTrue(all((backup/p.relative_to(self.source)).stat().st_ino != p.stat().st_ino + for p in before)) + restored = self.root / 'restored' + copy_tree(backup, restored, self.expected) + self.assertEqual(tree_manifest(restored), self.expected) + self.assertEqual(tree_modes(restored, self.expected), tree_modes(self.source, self.expected)) + files = [p for p in restored.rglob('*') if p.is_file()] + self.assertEqual(len({p.stat().st_ino for p in files}), len(files)) + self.assertTrue(all(p.stat().st_nlink == 1 for p in files)) + (restored/'first.bin').write_bytes(b'Later independent working file') + self.assertEqual((restored/'nested/second.bin').read_bytes(), second.read_bytes()) + self.assertEqual(tree_manifest(self.source), self.expected) + self.assertEqual(tree_manifest(backup), self.expected) + + def test_changed_duplicate_source_is_not_replaced_with_anchor_bytes(self): + path = self.source / 'nested/second.bin' + path.write_bytes(b'Wrong source data with a declared original hash') + with self.assertRaisesRegex(StorageTransferError, 'source bytes or binding changed'): + copy_tree(self.source, self.root/'backup', self.expected, deduplicate=True) + self.assertFalse((self.root/'backup/nested/second.bin').exists()) + + def test_changed_anchor_during_link_is_rejected_without_following_it(self): + sentinel = self.root/'outside.bin' + sentinel.write_bytes(b'Untouched outside file') + link = os.link + def replace_anchor(source, destination, **kwargs): + Path(source).unlink() + Path(source).symlink_to(sentinel) + return link(source, destination, **kwargs) + with patch('backup_metadata.os.link', side_effect=replace_anchor): + with self.assertRaisesRegex(StorageTransferError, 'anchor changed during link publication'): + copy_tree(self.source, self.root/'backup', self.expected, deduplicate=True) + self.assertEqual(sentinel.read_bytes(), b'Untouched outside file') + self.assertEqual(tree_manifest(self.source), self.expected) + + def test_existing_target_cannot_be_used_as_an_external_anchor(self): + target = self.root/'backup' + target.mkdir() + (target/'first.bin').write_bytes(b'Untrusted existing data') + with self.assertRaisesRegex(StorageTransferError, 'absent destination tree'): + copy_tree(self.source, target, self.expected, deduplicate=True) + self.assertEqual((target/'first.bin').read_bytes(), b'Untrusted existing data') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_metadata_migration.py b/tests/test_metadata_migration.py index ab154455..6040f62c 100644 --- a/tests/test_metadata_migration.py +++ b/tests/test_metadata_migration.py @@ -423,6 +423,8 @@ def cli(module, arguments, environment=None): def test_complete_backup_restore_preserves_identity_metadata_artifacts_and_retries(self): fixture(self.root / 'source', all_tables=True) + for name in ('duplicate-a.bin', 'duplicate-b.bin'): + (self.root / 'source' / name).write_bytes(b'Identical captured payload\x00\xff') self.migrate() before = tree_manifest(self.root / 'dest') metadata = Metadata(self.root / 'dest', create=False) @@ -431,6 +433,11 @@ def test_complete_backup_restore_preserves_identity_metadata_artifacts_and_retri sequences = read_sequences(metadata) metadata.close() receipt = backup_catalog(self.root / 'dest', self.root / 'pg-backup', quiesced=True) + compact = self.root / 'pg-backup/catalog' + self.assertEqual((compact/'duplicate-a.bin').stat().st_ino, + (compact/'duplicate-b.bin').stat().st_ino) + self.assertNotEqual((compact/'duplicate-a.bin').stat().st_ino, + (self.root/'dest/duplicate-a.bin').stat().st_ino) metadata = Metadata(self.root / 'dest', create=False) try: # Existing catalogs are not valid restore destinations, even with the same identity. @@ -448,6 +455,10 @@ def test_complete_backup_restore_preserves_identity_metadata_artifacts_and_retri finally: restored.close() self.assertEqual(tree_manifest(self.root / 'restored', ('.restore-receipt-' + digest(receipt) + '.json',)), before) + self.assertNotEqual((self.root/'restored/duplicate-a.bin').stat().st_ino, + (self.root/'restored/duplicate-b.bin').stat().st_ino) + self.assertTrue(all((self.root/'restored'/name).stat().st_nlink==1 + for name in ('duplicate-a.bin','duplicate-b.bin'))) self.assertEqual(restore_catalog(self.root / 'pg-backup', self.root / 'restored'), result) self.assertEqual(tree_manifest(self.root / 'dest'), before) # Tampering any artifact makes the complete backup unusable, before writes. diff --git a/tests/test_project_transfer.py b/tests/test_project_transfer.py index b3bf4848..efb639b4 100644 --- a/tests/test_project_transfer.py +++ b/tests/test_project_transfer.py @@ -80,6 +80,18 @@ def test_actual_stop_backup_is_private_verified_and_source_unchanged(self): with self.dataset._project_request():pass with self.assertRaises(WorkbenchError):self.dataset.stop_for_project_extraction(self.parent_fd,'retry-backup') + def test_native_stop_backup_deduplicates_copies_without_linking_originals(self): + original=[self.dataset.path/name for name in ('duplicate-a.bin','duplicate-b.bin')] + for path in original:path.write_bytes(b'Exact original payload' * 1000) + before=[(p.stat().st_ino,p.stat().st_ctime_ns,p.stat().st_nlink) for p in original] + source=self.stop() + copied=[self.root/'verified-backup/catalog'/p.name for p in original] + self.assertEqual(copied[0].stat().st_ino,copied[1].stat().st_ino) + self.assertTrue(all(p.stat().st_ino!=c.stat().st_ino for p,c in zip(original,copied))) + self.assertEqual(before,[(p.stat().st_ino,p.stat().st_ctime_ns,p.stat().st_nlink) for p in original]) + self.assertEqual(tree_manifest(self.root/'verified-backup/catalog'),source.backup_receipt['files']) + _assert_unchanged(source) + def test_live_preview_reports_domain_refusal_before_source_stop(self): from project_projection import ProjectProjection record=self.text();self.selection['roots']['records']=[record['id']] From 6213cda0375779d758ee68e75c1ea851f9f093f8 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 16:15:22 -0700 Subject: [PATCH 06/15] Validate checkpoint Unicode with native string operations --- project_store.py | 7 +++++-- tests/test_project_store.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/project_store.py b/project_store.py index 9c4bd108..7ec6cd76 100644 --- a/project_store.py +++ b/project_store.py @@ -96,8 +96,11 @@ def _json_value(value, depth=0): if type(value) is float and math.isfinite(value): return if type(value) is str: - if any(0xD800 <= ord(c) <= 0xDFFF for c in value): - _fail('Project JSON contains an invalid Unicode surrogate.') + if not value.isascii(): + try: + value.encode('utf-8') + except UnicodeEncodeError: + _fail('Project JSON contains an invalid Unicode surrogate.') return if type(value) is list: for item in value: diff --git a/tests/test_project_store.py b/tests/test_project_store.py index 98e52ad9..0f65339d 100644 --- a/tests/test_project_store.py +++ b/tests/test_project_store.py @@ -272,6 +272,24 @@ def test_closed_document_envelopes_and_supported_family_format(self): self.assert_code('unsupported', self.store.commit, [value], checkpoint, 'bad', body('bad')) self.assert_code('invalid', self.store.commit, [record(), record()], checkpoint, 'bad', body('bad')) + def test_every_unicode_scalar_is_valid_and_every_surrogate_is_refused(self): + # Unicode scalar values exclude exactly D800..DFFF. Check the complete + # finite domain rather than a copied sample of the implementation. + for value in range(0x110000): + text = chr(value) + if 0xD800 <= value <= 0xDFFF: + for content in (text, 'ASCII prefix' + text, 'λ' + text + 'suffix'): + error = self.assert_code('invalid', store_module._json_value, content) + self.assertEqual(str(error), 'Project JSON contains an invalid Unicode surrogate.') + else: + store_module._json_value(text) + + def test_unicode_validation_preserves_exact_canonical_bytes(self): + for value in ('ASCII' * 20000, '\x00\n\r\t\\"', 'é e\u0301 漢字 😀', + '\ud7ff\ue000\ufffe\uffff\U0010ffff'): + document = {'key': [value, {'λ': value}]} + self.assertEqual(store_module._encode(document), canonical(document)) + def test_large_checkpoint_index_publishes_and_cold_opens(self): # The migrated lab's small record declarations produced a 17.99 MB # index, even though individual records and their total were in bounds. From e9b2819ab5a12dd122eacf1820f1be704fc326be Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 17:37:14 -0700 Subject: [PATCH 07/15] Group historical artifact presence reads under a verified root Validate the canonical closure before and after each presence page while preserving anchored file checks and unverified availability semantics. Reject mid-read corruption, checkpoint changes and unsafe aliases before returning observations. Validation: 104 native artifact/store/history/workspace/transfer tests pass on disposable PostgreSQL. The actual 40-payload history page preserves every observation with two closure scans instead of forty; full native browser acceptance remains a separate check. --- project_artifacts.py | 39 ++++++++++++++++++++++++------ project_history.py | 6 +++-- project_store.py | 8 ++++-- tests/test_project_artifacts.py | 43 +++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 12 deletions(-) diff --git a/project_artifacts.py b/project_artifacts.py index ce691bd6..743ba5ce 100644 --- a/project_artifacts.py +++ b/project_artifacts.py @@ -207,14 +207,37 @@ def inspect(self, declaration): self._ensure_open() try: with self.store.root_directory() as root: - try: - with self._source(root, declaration) as source: - observed = _info(source, rooted=True).st_size - state = 'unverified' - except _Missing: - observed, state = None, 'missing' - return dict(state=state, expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], - observed_sha256=None, observed_bytes=observed) + return self._inspect_at(root, declaration) + except OSError as error: + _io(error) + + def _inspect_at(self, root, declaration): + try: + with self._source(root, declaration) as source: + observed = _info(source, rooted=True).st_size + state = 'unverified' + except _Missing: + observed, state = None, 'missing' + return dict(state=state, expected_sha256=declaration['sha256'], expected_bytes=declaration['bytes'], + observed_sha256=None, observed_bytes=observed) + + def inspect_many(self, declarations): + """Observe a finite owner-supplied page under one verified root scope. + + Each file still receives the same anchored ordinary-file checks as + inspect. Validate the canonical closure before AND after the group; + metadata corruption or a checkpoint change cannot produce a result. + Presence remains unverified and never claims a payload hash was read. + """ + if type(declarations) is not list: + _fail('Artifact inspection pages must be explicit finite lists.') + declarations = [validate_declaration(value) for value in declarations] + with self._mutex: + self._ensure_open() + try: + with self.store.root_directory(verify_after=True) as root: + result = [self._inspect_at(root, value) for value in declarations] + return result except OSError as error: _io(error) diff --git a/project_history.py b/project_history.py index 6388052c..989142d0 100644 --- a/project_history.py +++ b/project_history.py @@ -781,11 +781,13 @@ def run(self,run_id,*,origin_catalog_id=None,offset=None,limit=None): 'disclosures_total':len(disclosures), 'disclosures_next_offset':disclosure_end if disclosure_end Date: Fri, 23 Oct 2026 18:03:56 -0700 Subject: [PATCH 08/15] Separate experiment graph presentation from Workbench workflows Share bounded layout, explicit parent/baseline edges and escaped node presentation through a display-only component. Workbench retains Trial projection, authorization, async selection, journals and branch/admission behavior. Validation: four Node controller/graph fixture programs and actual Chromium DOM/CSS with synthetic transport pass. Tab/Enter and pointer selection preserve focus and unknown ancestry for an inspect-only actor. Native project history integration remains the next host; no backend or full-portfolio qualification is claimed. --- app.py | 1 + static/experiment-graph.js | 50 ++++++++++++++++++++ static/experiment-lineage.js | 34 +++++-------- static/workspaces.html | 1 + tests/browser_model_development.cjs | 2 +- tests/browser_workspace_recovery.cjs | 36 ++++++++++++++ tests/test_experiment_graph.cjs | 31 ++++++++++++ tests/test_experiment_lineage_controller.cjs | 8 +--- tests/test_workspace_external_controller.cjs | 2 +- tests/test_workspace_publication_lineage.cjs | 2 +- 10 files changed, 135 insertions(+), 32 deletions(-) create mode 100644 static/experiment-graph.js create mode 100644 tests/test_experiment_graph.cjs diff --git a/app.py b/app.py index 5bc42e31..d105d15c 100644 --- a/app.py +++ b/app.py @@ -907,6 +907,7 @@ def _get(self): '/workspaces': ('workspaces.html', 'text/html'), '/workspaces.css': ('workspaces.css', 'text/css'), '/workspaces.js': ('workspaces.js', 'text/javascript'), + '/experiment-graph.js': ('experiment-graph.js', 'text/javascript'), '/experiment-lineage.js': ('experiment-lineage.js', 'text/javascript'), '/experiment-lineage.css': ('experiment-lineage.css', 'text/css'), '/workspace-real.js': ('workspace-real.js', 'text/javascript'), diff --git a/static/experiment-graph.js b/static/experiment-graph.js new file mode 100644 index 00000000..ee785618 --- /dev/null +++ b/static/experiment-graph.js @@ -0,0 +1,50 @@ +'use strict'; +// Shared presentation of declared relationships. Callers own provenance, +// authorization, asynchronous reads, selection and the corresponding details. +(() => { + const escape=value=>String(value).replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); + function text(value,limit) {return typeof value==='string'&&value.length>0&&value.length<=limit;} + function validate(nodes) { + if(!Array.isArray(nodes)||nodes.length>100)throw Error('The experiment graph holds at most 100 entries.'); + const ids=new Set(); + for(const node of nodes) { + if(!node||!text(node.id,256)||ids.has(node.id)||!text(node.title,500)|| + typeof node.recordedAt!=='string'||node.recordedAt.length>80|| + !Array.isArray(node.lines)||node.lines.length>5||node.lines.some(line=>!text(line,2000))|| + [node.parent,node.baseline].some(ref=>ref!==null&&!text(ref,256)))throw Error('Invalid experiment graph entry.'); + ids.add(node.id); + } + } + function layout(nodes) { + validate(nodes); + const byId=new Map(nodes.map(node=>[node.id,node])),depth=new Map(); + function level(id,path=new Set()) { + if(depth.has(id))return depth.get(id); + if(path.has(id))throw Error('Derivation contains a cycle.'); + path=new Set(path);path.add(id); + const parent=byId.get(id)?.parent; + const value=parent&&byId.has(parent)?level(parent,path)+1:0; + depth.set(id,value);return value; + } + const rows=new Map(),positions=new Map(); + for(const node of [...nodes].sort((a,b)=>a.recordedAt.localeCompare(b.recordedAt)||a.id.localeCompare(b.id))) { + const column=level(node.id),row=rows.get(column)||0;rows.set(column,row+1); + positions.set(node.id,{x:24+column*300,y:24+row*180}); + } + return {positions,width:Math.max(600,...[...positions.values()].map(p=>p.x+276)),height:Math.max(240,...[...positions.values()].map(p=>p.y+160))}; + } + function render(nodes,{selected='',zoom=1}={}) { + if(typeof selected!=='string'||selected.length>256||typeof zoom!=='number'||!Number.isFinite(zoom)||zoom<0.6||zoom>1.6)throw Error('Invalid experiment graph selection or zoom.'); + const graph=layout(nodes),edges=[]; + for(const node of nodes)for(const kind of ['parent','baseline']) { + const from=graph.positions.get(node[kind]),to=graph.positions.get(node.id); + if(from&&to)edges.push(``); + } + const buttons=nodes.map(node=>{ + const p=graph.positions.get(node.id); + return ``; + }).join(''); + return {...graph,html:`
            ${buttons}
            `}; + } + window.WorkspaceExperimentGraph=Object.freeze({layout,render}); +})(); diff --git a/static/experiment-lineage.js b/static/experiment-lineage.js index a4f807f6..1df3bcc2 100644 --- a/static/experiment-lineage.js +++ b/static/experiment-lineage.js @@ -4,23 +4,15 @@ // Generated from agent_api.contract; checked by the lineage contract regression. Object.assign(window.WorkspaceModelContracts, {"studies.branch_trial": "76b6415bce7e03aef825e0a64fac690f6f2e46bcbb54fd3423979c87449b2af5", "studies.lineage": "33ee4e76a92b1481c2e6fca72b571b2f363ca18d9098acf065d025c2e27963f9", "studies.lineage_trial": "f3ec663dfa413eeeaf5ab017c28d02d0ec4f14a724c0067d07592110bf7e694a", "studies.prepare_branch": "7f4e4d8b92d86fc6f5dedf363c4ad4dbc125df914af9a18450aab0c894a13b43", "studies.record_lineage": "f8315af375e0667d4dd55e6d7decf8155ef70eaed2c19b60d7256010da9d26a4"}); const enc=encodeURIComponent, short=value=>String(value||'').slice(0,12); - function layout(nodes) { - const byId=new Map(nodes.map(node=>[node.trial.id,node])), depth=new Map(); - function level(id,path=new Set()) { - if(depth.has(id))return depth.get(id); - if(path.has(id))throw Error('Derivation contains a cycle.'); - path=new Set(path);path.add(id); - const parent=byId.get(id)?.lineage?.intent.parent?.trial_id; - const value=parent&&byId.has(parent)?level(parent,path)+1:0;depth.set(id,value);return value; - } - const rows=new Map(), positions=new Map(); - for(const node of [...nodes].sort((a,b)=>(a.lineage?.recorded_at||'').localeCompare(b.lineage?.recorded_at||'')||a.trial.id.localeCompare(b.trial.id))) { - const column=level(node.trial.id),row=rows.get(column)||0;rows.set(column,row+1); - positions.set(node.trial.id,{x:24+column*300,y:24+row*180}); - } - return {positions,width:Math.max(600,...[...positions.values()].map(p=>p.x+276)),height:Math.max(240,...[...positions.values()].map(p=>p.y+160))}; + function graphEntry(node) { + const intent=node.lineage?.intent,run=node.runs[0]; + return {id:node.trial.id,title:node.study.name,recordedAt:node.lineage?.recorded_at||'', + parent:intent?.parent?.trial_id??null,baseline:intent?.baseline?.trial_id??null, + lines:[short(node.trial.id)+' · '+(intent?.ancestry||'ancestry unknown'), + intent?.change_summary||'Intent not recorded', + (run?.state||'prepared')+' · '+node.attempt_count+' attempts', + ...(intent?['Target: '+intent.target.metric_id+' '+intent.target.direction]:[])]}; } - window.WorkspaceLineageLayout=layout; window.WorkspaceExperimentLineage=function({api,announce,getIntent,getView,getGraph,isDirty,openRecipe,openRun,compare}) { const esc=value=>window.workspaceEscape(value), identity=()=>window.TuldokAuth?.identity(); let root=null,catalog=null,nodes=new Map(),revision=null,next=null,selected='',filter='',appliedFilter='',journalScope=null,message='',failed=false, @@ -111,13 +103,9 @@ function render(){ if(!root||getView()!=='pipeline')return; const focus=root.contains(document.activeElement)?document.activeElement.id:null,scroll=root.querySelector('.lineage-viewport'),position=scroll?{left:scroll.scrollLeft,top:scroll.scrollTop}:null; - let graph;try{graph=layout([...nodes.values()]);}catch(error){graph={positions:new Map(),width:600,height:240};message=error.message;failed=true;} - const edges=[]; - for(const node of nodes.values())for(const kind of ['parent','baseline']){ - const target=node.lineage?.intent[kind]?.trial_id,from=graph.positions.get(target),to=graph.positions.get(node.trial.id); - if(from&&to)edges.push(``); - } - root.innerHTML=`
            EXPERIMENTS · ACROSS STUDIES

            Experiment progression

            Follow declared derivation, comparison baselines and actual outcomes. Solid edges show derivation; dashed edges show baselines. Execution retries remain inside their Trial.

            ${esc(message)}

            ${Math.round(zoom*100)}%
            ${[...nodes.values()].map(node=>{const p=graph.positions.get(node.trial.id);if(!p)return '';const intent=node.lineage?.intent,run=node.runs[0];return ``;}).join('')}

            ${nodes.size} trials loaded${appliedFilter?' · applied cohort: '+esc(appliedFilter):' · all cohorts'}${next?' · more pages available':''}. ${related.length} related identities are outside this loaded window. Layout order does not imply chronology.

            ${next?``:''}${related.map(item=>``).join('')}
            ${detail()}
            ${jobHTML()}
            `; + let graph;try{graph=window.WorkspaceExperimentGraph.render([...nodes.values()].map(graphEntry),{selected,zoom});} + catch(error){graph=window.WorkspaceExperimentGraph.render([],{zoom});message=error.message;failed=true;} + root.innerHTML=`
            EXPERIMENTS · ACROSS STUDIES

            Experiment progression

            Follow declared derivation, comparison baselines and actual outcomes. Solid edges show derivation; dashed edges show baselines. Execution retries remain inside their Trial.

            ${esc(message)}

            ${Math.round(zoom*100)}%
            ${graph.html}

            ${nodes.size} trials loaded${appliedFilter?' · applied cohort: '+esc(appliedFilter):' · all cohorts'}${next?' · more pages available':''}. ${related.length} related identities are outside this loaded window. Layout order does not imply chronology.

            ${next?``:''}${related.map(item=>``).join('')}
            ${detail()}
            ${jobHTML()}
            `; bind();if(position){const viewport=root.querySelector('.lineage-viewport');viewport.scrollLeft=position.left;viewport.scrollTop=position.top;} if(focus)root.querySelector('#'+CSS.escape(focus))?.focus({preventScroll:true}); } diff --git a/static/workspaces.html b/static/workspaces.html index b9bd23ee..1ff76169 100644 --- a/static/workspaces.html +++ b/static/workspaces.html @@ -144,6 +144,7 @@

            Unsaved fixture edits

            + diff --git a/tests/browser_model_development.cjs b/tests/browser_model_development.cjs index 6f907fa4..2ecce76f 100644 --- a/tests/browser_model_development.cjs +++ b/tests/browser_model_development.cjs @@ -16,7 +16,7 @@ const root = path.resolve(__dirname, ".."), report = qaDirectory(root, "model-development"), temporary = fs.mkdtempSync(path.join(os.tmpdir(), "tuldok-model-development-")), children = []; -const sourceHashes=Object.fromEntries(['static/workspace-service.js','static/workspace-real.js','static/experiment-lineage.js','static/experiment-lineage.css','static/workspaces.css','static/workspaces.html','experiment_lineage.py','agent_api.py','app.py','experiments.py','pipeline.py','runs.py','metadata.py','upgrade_metadata.py','tests/browser_model_development.cjs'].map(file=>[file,require('node:crypto').createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex')])); +const sourceHashes=Object.fromEntries(['static/workspace-service.js','static/workspace-real.js','static/experiment-graph.js','static/experiment-lineage.js','static/experiment-lineage.css','static/workspaces.css','static/workspaces.html','experiment_lineage.py','agent_api.py','app.py','experiments.py','pipeline.py','runs.py','metadata.py','upgrade_metadata.py','tests/browser_model_development.cjs'].map(file=>[file,require('node:crypto').createHash('sha256').update(fs.readFileSync(path.join(root,file))).digest('hex')])); const errors = [], posts = [], captures = [], diff --git a/tests/browser_workspace_recovery.cjs b/tests/browser_workspace_recovery.cjs index 0ae67dcc..f54bf637 100644 --- a/tests/browser_workspace_recovery.cjs +++ b/tests/browser_workspace_recovery.cjs @@ -34,6 +34,7 @@ else if(route.includes('/predictions')){if(route.includes(b.id)&&route.endsWith( else value={};return {ok:status===200,status,json:async()=>structuredClone(value)}};`); await ev(fs.readFileSync(path.resolve(__dirname,'../static/workspace-external.js'),'utf8')); await ev(fs.readFileSync(path.resolve(__dirname,'../static/workspace-service.js'),'utf8')); +await ev(fs.readFileSync(path.resolve(__dirname,'../static/experiment-graph.js'),'utf8')); await ev(fs.readFileSync(path.resolve(__dirname,'../static/experiment-lineage.js'),'utf8')); await ev(fs.readFileSync(path.resolve(__dirname,'../static/workspace-phase-timings.js'),'utf8')); await ev(fs.readFileSync(path.resolve(__dirname,'../static/workspace-scientific-comparison.js'),'utf8')); @@ -67,4 +68,39 @@ const damaged=await ev(`({runButtons:document.querySelectorAll('[data-real-run]' await ev(`document.querySelector('[data-real-run="'+a.id+'"]').click()`);await pause(150); if(!await ev(`document.getElementById('real-run-'+a.id).getAttribute('aria-current')==='true' && document.getElementById('real-run-state').textContent==='completed'`))throw Error('Healthy run not selectable after damaged detail'); console.log('PASS: damaged selected run retains list/warning, no metrics, and healthy run remains selectable.'); + +// Exercise the shared graph with the actual Workbench controller and real DOM. +// Only the service transport is synthetic; these are no training/history claims. +await ev(`real.show(null);document.body.replaceChildren(); +window.graphCalls=[];window.graphHost=document.createElement('main');document.body.append(graphHost); +window.graphStyle=document.createElement('style');graphStyle.textContent=${JSON.stringify(fs.readFileSync(path.resolve(__dirname,'../static/experiment-lineage.css'),'utf8'))};document.head.append(graphStyle); +window.rootTrial='1'.repeat(32);window.childTrial='2'.repeat(32);window.graphCatalog='3'.repeat(32); +window.TuldokAuth={identity:()=>({kind:'agent',actor_id:'6'.repeat(32),credential_id:'7'.repeat(32),scopes:['inspect']})}; +window.baseGraphNode={trial:{id:rootTrial,material:{catalog_id:graphCatalog,sha256:'a'.repeat(64)}},study:{id:'4'.repeat(32),name:'Saved Study '},recipe:{id:'5'.repeat(32),revision:1},lineage:null,attempt_count:0,runs:[],runs_truncated:false}; +window.childGraphNode=structuredClone(baseGraphNode);childGraphNode.trial.id=childTrial; +childGraphNode.lineage={recorded_at:'2026-10-23',timing:'retrospective',intent:{ancestry:'derived',parent:{catalog_id:graphCatalog,trial_id:rootTrial,material_sha256:'a'.repeat(64)},baseline:{catalog_id:graphCatalog,trial_id:rootTrial,material_sha256:'a'.repeat(64)},cohort:'Graph rendering fixture',hypothesis:'Preserve declared relationships',change_summary:'Synthetic graph presentation fixture',target:{metric_id:'loss',direction:'minimize',threshold:null,acceptance:'Rendering only',metric_definition_sha256:null}}}; +window.graphUI=WorkspaceExperimentLineage({api:async(route,body)=>{graphCalls.push({route,body});return {revision:'b'.repeat(64),nodes:[baseGraphNode,childGraphNode],related:[],next_cursor:null};},announce(){},getIntent:()=>0,getView:()=> 'pipeline',getGraph:()=>null,isDirty:()=>false,openRecipe:async()=>{},openRun:async()=>{},compare:async()=>{}}); +graphUI.mount(graphHost,{dataset_id:graphCatalog});`); +await pause(100); +const initialGraph=await ev(`({nodes:graphHost.querySelectorAll('[data-lineage-node]').length,parents:graphHost.querySelectorAll('path.parent').length,baselines:graphHost.querySelectorAll('path.baseline').length,images:graphHost.querySelectorAll('img').length,writes:graphCalls.filter(v=>v.body!==undefined).length})`); +if(initialGraph.nodes!==2||initialGraph.parents!==1||initialGraph.baselines!==1||initialGraph.images||initialGraph.writes)throw Error('Shared graph changed declared ancestry, escaped names or read-only transport'); +await ev(`graphHost.querySelector('#lineage-zoom-reset').focus()`); +for(let i=0;i<3;i++){ + await send('Input.dispatchKeyEvent',{type:'keyDown',key:'Tab',code:'Tab',windowsVirtualKeyCode:9}); + await send('Input.dispatchKeyEvent',{type:'keyUp',key:'Tab',code:'Tab',windowsVirtualKeyCode:9}); +} +const keyboardTarget=await ev(`({id:document.activeElement.dataset.lineageNode,visible:document.activeElement.matches(':focus-visible'),outline:getComputedStyle(document.activeElement).outlineWidth})`); +if(keyboardTarget.id!==await ev('childTrial')||!keyboardTarget.visible||parseFloat(keyboardTarget.outline)<3)throw Error('Graph entries are not reachable with visible keyboard focus'); +await send('Input.dispatchKeyEvent',{type:'keyDown',key:'Enter',code:'Enter',windowsVirtualKeyCode:13,text:'\r'}); +await send('Input.dispatchKeyEvent',{type:'keyUp',key:'Enter',code:'Enter',windowsVirtualKeyCode:13}); +const activatedGraph=await ev(`({selected:graphHost.querySelector('[aria-pressed="true"]')?.dataset.lineageNode,focused:document.activeElement.dataset.lineageNode,details:graphHost.querySelector('#lineage-detail').innerText,branch:!!graphHost.querySelector('#lineage-branch')})`); +if(activatedGraph.selected!==await ev('childTrial')||activatedGraph.focused!==activatedGraph.selected||!activatedGraph.details.includes('Preserve declared relationships')||activatedGraph.branch)throw Error('Keyboard selection lost graph focus, exact details or inspect-only behavior'); +await ev(`graphHost.querySelector('[data-lineage-node="'+rootTrial+'"]').scrollIntoView({block:'center'})`); +const point=await ev(`(()=>{const r=graphHost.querySelector('[data-lineage-node="'+rootTrial+'"]').getBoundingClientRect();return {x:r.left+r.width/2,y:r.top+r.height/2};})()`); +await send('Input.dispatchMouseEvent',{type:'mousePressed',...point,button:'left',clickCount:1}); +await send('Input.dispatchMouseEvent',{type:'mouseReleased',...point,button:'left',clickCount:1}); +if(!await ev(`graphHost.querySelector('[aria-pressed="true"]')?.dataset.lineageNode===rootTrial && graphHost.querySelector('#lineage-detail').innerText.includes('Historical ancestry and intent are unknown')`))throw Error('Pointer selection fabricated root ancestry or lost detail'); +console.log(JSON.stringify({initialGraph,keyboardTarget,activatedSelected:activatedGraph.selected})); +console.log('PASS: shared graph real DOM/CSS, declared parent versus baseline, escaped names, Tab/Enter selection with visible retained focus, pointer selection, unknown ancestry and inspect-only behavior. Synthetic transport; no backend or native-project acceptance claim.'); + })().catch(e=>{console.error(e);process.exitCode=1}).finally(async()=>{clearTimeout(deadline);if(ws)ws.close();if(browser)browser.kill();if(server)server.close();await pause(150);fs.rmSync(dir,{recursive:true,force:true})}); diff --git a/tests/test_experiment_graph.cjs b/tests/test_experiment_graph.cjs new file mode 100644 index 00000000..572adcc9 --- /dev/null +++ b/tests/test_experiment_graph.cjs @@ -0,0 +1,31 @@ +'use strict'; +// Real graph layout/rendering decisions; DOM focus and activation need browser evidence. +const assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'),vm=require('node:vm'); +const window={}; +vm.runInNewContext(fs.readFileSync(path.join(__dirname,'../static/experiment-graph.js'),'utf8'),{window}); +const graph=window.WorkspaceExperimentGraph; +const node=(id,extra={})=>({id,title:'Recorded experiment '+id,recordedAt:'',parent:null,baseline:null,lines:['Ancestry unknown'],...extra}); +const a=node('a'),b=node('b',{parent:'a'}),c=node('c',{parent:'b'}); +const chain=graph.layout([c,a,b]); +assert.ok(chain.positions.get('a').xgraph.layout([node('a',{parent:'c'}),b,c]),/cycle/); +assert.throws(()=>graph.layout([a,a]),/Invalid/); +assert.throws(()=>graph.layout(Array.from({length:101},(_,i)=>node(String(i)))),/100/); +assert.throws(()=>graph.render([a],{zoom:Infinity}),/zoom/); +assert.throws(()=>graph.render([node('x',{lines:['x'.repeat(2001)]})]),/Invalid/); +const roots=graph.render([a,node('d',{recordedAt:'later'})]); +assert.doesNotMatch(roots.html,/',lines:['']})],{selected:'x" onclick="bad',zoom:0.6}); +assert.doesNotMatch(escaped.html,/fs.readFileSync(require('node:path').join(__dirname,'../static',file),'utf8')).join('\n'); const deferred=()=>{let resolve,reject;const promise=new Promise((yes,no)=>{resolve=yes;reject=no;});return {promise,resolve,reject};}; const tick=()=>new Promise(resolve=>setImmediate(resolve)); const catalog='c'.repeat(32),trial=id=>({trial:{id,material:{catalog_id:catalog,sha256:'a'.repeat(64)}},study:{id:'s',name:'Finite study'},recipe:{id:'g',revision:1},lineage:null,attempt_count:0,runs:[]}); @@ -16,10 +16,6 @@ function fixture(saved=null){ return {window,root,ui,calls,history,setHandler:value=>handler=value,mount(){elements.clear();ui.mount(root,{dataset_id:catalog});},navigate(){intent++;ui.invalidate();},session(){actor={...actor,credential_id:'session2'};ui.sessionChanged();},other(){actor={...actor,actor_id:'other',credential_id:'otherSession'};ui.sessionChanged();},click(id){const button=root.querySelector('#'+id);assert.ok(button,id+' exists');button.click();},submit(){root.querySelector('#lineage-filter-form').submit({preventDefault(){}});}}; } (async()=>{ - const f=fixture(),layout=f.window.WorkspaceLineageLayout; - const a=trial('a'),b=trial('b'),c=trial('c');b.lineage={recorded_at:'2',intent:{parent:{trial_id:'a'}}};c.lineage={recorded_at:'3',intent:{parent:{trial_id:'b'}}}; - const positioned=layout([c,a,b]);assert.ok(positioned.positions.get('a').xlayout([a,b,c]),/cycle/); for(const change of ['navigate','session','other']){ const g=fixture(),held=deferred();g.setHandler(()=>held.promise);g.mount();g[change]();held.resolve({revision:'r',nodes:[trial('secret')],related:[],next_cursor:null});await tick(); assert.ok(!g.root.innerHTML.includes('data-lineage-node="secret"'),'late page after '+change+' must not publish'); @@ -39,5 +35,5 @@ function fixture(saved=null){ function workflowAdmission(){return {request_key:'original-admission',revision:2,admission_epoch:1,trial_id:'child'};} const filters=fixture();let pages=0;filters.setHandler(route=>Promise.resolve({revision:'r',nodes:[trial('page'+(++pages))],related:[],next_cursor:pages<3?'cursor':null}));filters.mount();await tick();filters.root.querySelector('#lineage-filter').input({target:{value:'cohort-a'}});filters.submit();await tick();filters.root.querySelector('#lineage-filter').input({target:{value:'cohort-b'}});filters.click('lineage-more');await tick(); assert.match(filters.calls.at(-1).route,/cohort=cohort-a/);assert.doesNotMatch(filters.calls.at(-1).route,/cohort-b/);assert.match(filters.root.innerHTML,/applied cohort: cohort-a/); - console.log('PASS lineage layout, cycles, read fences, busy ownership, actor-isolated writes, approval acknowledgement, remount retention and applied cohort pagination'); + console.log('PASS lineage read fences, busy ownership, actor-isolated writes, approval acknowledgement, remount retention and applied cohort pagination'); })().catch(error=>{console.error(error);process.exitCode=1;}); diff --git a/tests/test_workspace_external_controller.cjs b/tests/test_workspace_external_controller.cjs index 58555e5b..5319185f 100644 --- a/tests/test_workspace_external_controller.cjs +++ b/tests/test_workspace_external_controller.cjs @@ -165,7 +165,7 @@ function fixture(){ const context={window,TuldokAuth:auth,history:{state:{},replaceState(){}},structuredClone,crypto,clearTimeout,setTimeout,performance,AbortController,document:{activeElement:null},CSS:{escape:String}}; vm.runInNewContext(source,context);context.WorkspaceExternal=window.WorkspaceExternal; vm.runInNewContext(fs.readFileSync(require('node:path').join(__dirname,'../static/workspace-service.js'),'utf8'),context);context.WorkspaceModel=window.WorkspaceModel; - vm.runInNewContext(fs.readFileSync(require('node:path').join(__dirname,'../static/experiment-lineage.js'),'utf8'),context); + for(const file of ['experiment-graph.js','experiment-lineage.js'])vm.runInNewContext(fs.readFileSync(require('node:path').join(__dirname,'../static',file),'utf8'),context); for(const file of ['workspace-phase-timings.js','workspace-scientific-comparison.js'])vm.runInNewContext(fs.readFileSync(require('node:path').join(__dirname,'../static',file),'utf8'),context); vm.runInNewContext(fs.readFileSync(require('node:path').join(__dirname,'../static/workspace-real.js'),'utf8'),context); const workspace=window.WorkspaceReal({pipelineRoot,runsRoot,announce(){}}),loading=workspace.show('runs');await requested.promise; diff --git a/tests/test_workspace_publication_lineage.cjs b/tests/test_workspace_publication_lineage.cjs index a51d9db3..21041291 100644 --- a/tests/test_workspace_publication_lineage.cjs +++ b/tests/test_workspace_publication_lineage.cjs @@ -19,7 +19,7 @@ function fixture(){ const camel=v=>v.replace(/-([a-z])/g,(_,c)=>c.toUpperCase()); function make(key,values){if(!elements.has(key))elements.set(key,{...values,addEventListener(type,fn){this[type]=fn;},focus(){document.activeElement=this;}});return elements.get(key);} const context={window,history,document,crypto:crypto.webcrypto,TextEncoder,structuredClone,URLSearchParams,CSS:{escape:v=>v},Map,Set,Event}; - for(const file of ['workspace-external.js','experiment-lineage.js'])vm.runInNewContext(fs.readFileSync(path.join(__dirname,'../static',file),'utf8'),context); + for(const file of ['workspace-external.js','experiment-graph.js','experiment-lineage.js'])vm.runInNewContext(fs.readFileSync(path.join(__dirname,'../static',file),'utf8'),context); const ui=window.WorkspaceExperimentLineage({api:async(route,body,version)=>{calls.push({route,body,version});return handler(route,body,version);},announce(){},getIntent:()=>intent,getView:()=> 'pipeline',getGraph:()=>null,isDirty:()=>false,openRecipe:async()=>{},openRun:async(id,current)=>{if(current())opened.push(id);},compare:async()=>{}}); return {root,ui,calls,opened,setHandler:fn=>handler=fn,mount(){ui.mount(root,{dataset_id:catalog});},click:selector=>{const button=root.querySelector(selector);assert.ok(button,selector);return button.onclick?button.onclick():button.click();},navigate(){intent++;ui.invalidate();},session(){actor={...actor,credential_id:'two'};ui.sessionChanged();},other(){actor={...actor,actor_id:'9'.repeat(32),credential_id:'two'};ui.sessionChanged();}}; } From 5636510c99d1215a400f20a98d6a70c7dd0c96b1 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 18:25:33 -0700 Subject: [PATCH 09/15] Expose original project history through native Experiment progression Add revision-bound read-only origin discovery and Trial graph projection with exact Study/material/lineage references and saved Run statuses. Keep metric disclosure in the existing result owners. Reuse the shared graph in a native host with source/cohort paging, original-identity navigation, keyboard focus and late-response fences. Validation: 51 real native SQL, contract, SDK/MCP and HTTP tests pass, including actual credential revocation during reads; nine Chromium DOM lifecycle checks pass with controlled transport. Actual imported-project browser and independent cold reopening remain the next qualification gate. No live service change or full portfolio qualification. --- agent_api.py | 12 + app.py | 1 + project_dispatch.py | 4 +- project_history.py | 49 ++ static/project-files.html | 12 +- static/project-files.js | 534 ++++++++++++---------- static/project-progression.js | 117 +++++ tests/test_project_files_m2_http.py | 26 ++ tests/test_project_history.py | 49 ++ tests/test_workspace_project_files.cjs | 3 + tests/test_workspace_project_files_m4.cjs | 3 + tuldok_client.py | 8 + 12 files changed, 560 insertions(+), 258 deletions(-) create mode 100644 static/project-progression.js diff --git a/agent_api.py b/agent_api.py index 2da06bbd..d7b29b83 100644 --- a/agent_api.py +++ b/agent_api.py @@ -991,6 +991,18 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non _project_file_operation('trial_prepare', 'POST', '/studies/{study_id}/trials', { 'study_id': ID, 'revision': SAFE_POS, 'material': MATERIAL_REF, 'replicate': SAFE_NAT}, mutation=True) _project_file_operation('lineage_record', 'POST', '/lineage', {'trial': _LINEAGE_REF, 'intent': _LINEAGE_INTENT}, mutation=True) +_PROJECT_PROGRESSION_ORIGIN = obj({'origin_catalog_id':ID,'study_count':SAFE_NAT,'trial_count':SAFE_NAT}) +_PROJECT_PROGRESSION_RUN = obj({key:value for key,value in _LINEAGE_RUN['properties'].items() if key!='validation'}) +_PROJECT_PROGRESSION_NODE = obj({'trial':_TRIAL,'lineage':{'oneOf':[_LINEAGE_RECORD,_NULL]}, + 'study':obj({'id':ID,'name':text(120)}),'attempt_count':SAFE_NAT, + 'runs':array(_PROJECT_PROGRESSION_RUN,5),'runs_truncated':{'type':'boolean'}}) +_project_file_operation('progression_origins','GET','/progression/origins',_PROJECT_FILES_PAGE_INPUT,required=[], + output=obj({'items':array(_PROJECT_PROGRESSION_ORIGIN,40),'offset':SAFE_NAT,'limit':integer(1,40),'total':SAFE_NAT})) +_project_file_operation('progression','GET','/progression', + {'origin_catalog_id':ID,'offset':SAFE_NAT,'limit':integer(1,20),'cohort':text(120)},required=['origin_catalog_id'], + output=obj({'origin_catalog_id':ID,'evidence':const('saved_metadata_only'),'historical':const(True), + 'execution_available':const(False),'nodes':array(_PROJECT_PROGRESSION_NODE,20),'related':array(_LINEAGE_REF,40), + 'offset':SAFE_NAT,'limit':integer(1,20),'total':SAFE_NAT,'cohort':{'oneOf':[text(120),_NULL]}})) _project_file_operation('candidate_select', 'POST', '/studies/{study_id}/candidates', domain='studies.select_candidate', mutation=True) _project_file_operation('evidence', 'POST', '/runs/{run_id}/evidence', { diff --git a/app.py b/app.py index d105d15c..9f14ad20 100644 --- a/app.py +++ b/app.py @@ -908,6 +908,7 @@ def _get(self): '/workspaces.css': ('workspaces.css', 'text/css'), '/workspaces.js': ('workspaces.js', 'text/javascript'), '/experiment-graph.js': ('experiment-graph.js', 'text/javascript'), + '/project-progression.js': ('project-progression.js', 'text/javascript'), '/experiment-lineage.js': ('experiment-lineage.js', 'text/javascript'), '/experiment-lineage.css': ('experiment-lineage.css', 'text/css'), '/workspace-real.js': ('workspace-real.js', 'text/javascript'), diff --git a/project_dispatch.py b/project_dispatch.py index a437405b..49ce82c5 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -25,7 +25,7 @@ 'recipe_get', 'recipe_create', 'recipe_save', 'recipe_prepare', 'model_material', 'recipe_material', 'runtime_material', 'snapshot', 'materials')) _HISTORY_ACTIONS = frozenset(('studies', 'study', 'study_create', 'study_save', 'trial', 'trial_prepare', - 'lineage', 'lineage_record', 'run', 'runs', 'candidate', 'candidate_select', 'evidence', + 'lineage', 'lineage_record', 'progression_origins', 'progression', 'run', 'runs', 'candidate', 'candidate_select', 'evidence', 'retained_analysis', 'retained_analysis_export', 'policies', 'policy', 'dependencies', 'publication_result', 'package_state', 'expose_holdout', 'run_document_segment')) _OTHER_ACTIONS = frozenset(('artifacts', 'artifact_status', 'artifact_resolve', 'producers', 'producer', 'source_register', 'source_references')) @@ -268,6 +268,8 @@ def _history(projection, action, a): origin_catalog_id=origin, expected_sha256=a['expected_sha256'], offset=a['offset'], limit=a['limit']) if action == 'candidate': return history.candidate(a['candidate_id'], origin_catalog_id=origin, verify=a.get('verify', False)) if action == 'lineage': return _page(history.lineage(origin_catalog_id=origin)['records'], a) + if action == 'progression_origins': return history.progression_origins(offset=a.get('offset',0),limit=a.get('limit',40)) + if action == 'progression': return history.progression(origin,offset=a.get('offset',0),limit=a.get('limit',20),cohort=a.get('cohort')) if action == 'policies': return _page(history.policies(origin_catalog_id=origin)['policies'], a) if action == 'policy': return history.policy(a['ref']) if action == 'runs': diff --git a/project_history.py b/project_history.py index 989142d0..6180850c 100644 --- a/project_history.py +++ b/project_history.py @@ -764,6 +764,55 @@ def lineage(self,*,origin_catalog_id=None): origin=self._origin(origin_catalog_id) return {'records':[copy.deepcopy(doc) for (namespace,_),doc in sorted(self._facts().trials.items()) if namespace==origin]} + def progression_origins(self,*,offset=0,limit=40): + """Discover original namespaces without inventing current project ancestry.""" + self._authorize('studies.lineage') + natural(offset,'Origin offset');natural(limit,'Origin page limit',1,40) + facts=self._facts();origins={origin for origin,_ in facts.studies}|{origin for origin,_ in facts.trials} + items=[{'origin_catalog_id':origin, + 'study_count':sum(namespace==origin for namespace,_ in facts.studies), + 'trial_count':sum(namespace==origin for namespace,_ in facts.trials)} for origin in sorted(origins)] + self._authorize('studies.lineage') + return {'items':items[offset:offset+limit],'offset':offset,'limit':limit,'total':len(items)} + + def progression(self,origin_catalog_id,*,offset=0,limit=20,cohort=None): + """Bounded original Trial graph, with saved Run status only. + + Numeric evidence stays behind its existing disclosure/query owners. + All references come from the already validated immutable history; + this projection reads no payload bytes or execution/recovery owner. + """ + self._authorize('studies.lineage');identifier(origin_catalog_id,'Origin catalog') + natural(offset,'Progression offset');natural(limit,'Progression page limit',1,20) + if cohort is not None:text(cohort,'Recorded cohort',120) + facts=self._facts();origin=origin_catalog_id + if not any(namespace==origin for namespace,_ in facts.studies): + _invalid('Historical origin is unavailable.','not_found') + documents=[doc for (namespace,_),doc in sorted(facts.trials.items()) if namespace==origin + and (cohort is None or doc['lineage'] is not None and doc['lineage']['intent']['cohort']==cohort)] + nodes=[] + for doc in documents[offset:offset+limit]: + trial=doc['trial'];study=facts.studies[(origin,trial['study_id'])]['study'] + attempts=[facts.runs[(origin,value['run_id'])]['run'] + for (namespace,_),value in facts.reservations.items() + if namespace==origin and value['trial_id']==trial['id']] + attempts.sort(key=lambda value:(value['created_at'],value['id']),reverse=True) + runs=[{key:copy.deepcopy(run.get(key)) for key in + ('id','state','parent_run_id','created_at','finished_at')} for run in attempts[:5]] + nodes.append({'trial':copy.deepcopy(trial),'lineage':copy.deepcopy(doc['lineage']), + 'study':{'id':study['id'],'name':study['name']}, + 'attempt_count':len(attempts),'runs':runs,'runs_truncated':len(attempts)>5}) + visible={node['trial']['id'] for node in nodes};related={} + for node in nodes: + for kind in ('parent','baseline'): + ref=(node['lineage'] or {}).get('intent',{}).get(kind) + if ref is not None and (ref['catalog_id']!=origin or ref['trial_id'] not in visible): + related[(ref['catalog_id'],ref['trial_id'])]=copy.deepcopy(ref) + self._authorize('studies.lineage') + return {'origin_catalog_id':origin,'evidence':'saved_metadata_only','historical':True, + 'execution_available':False,'nodes':nodes,'related':[related[key] for key in sorted(related)], + 'offset':offset,'limit':limit,'total':len(documents),'cohort':cohort} + def run(self,run_id,*,origin_catalog_id=None,offset=None,limit=None): self._authorize('runs.get',run_id) facts=self._facts();doc=self._record(facts.runs,run_id,origin_catalog_id) diff --git a/static/project-files.html b/static/project-files.html index 4738a772..b59af9e1 100644 --- a/static/project-files.html +++ b/static/project-files.html @@ -3,22 +3,25 @@ - Text projects · Tuldok + Projects · Tuldok + + +
            -

            Tuldok / external projects

            Text project workspace

            -

            Create or open a folder under a configured local root. Submitted text and annotations are saved before acknowledgement.

            +

            Tuldok / external projects

            Project workspace

            +

            Create or open a project folder to work with data, models and experiment history.

            -

            Reading project status…

            +

            Reading project status…

            @@ -160,6 +163,7 @@

            Saved annotation history

            +

            Project data and scientific history

            Inspect saved records, models, Studies, Runs and their exact artifact availability. Opening a project never starts computation or restores permission to disclose held-out data.

            diff --git a/static/project-files.js b/static/project-files.js index 3f25cec3..8f168dc0 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -99,6 +99,8 @@ "project_files.policy": "5bdb86600f2ec387fde721b22fdd0afa3ed47660126847c2908c1bb9de0421cb", "project_files.producer": "6187a9eb1e32cb3bcf13d283492b1ad909c32753aa5ed97fbc12a9708523d9e3", "project_files.producers": "65b2f4dff355c3bf5a023bf4f9dc116e302eda85ad90e6afaa2f21f0aa0667b5", + "project_files.progression": "4b70964ce2509f3526b2915cb5bd1ec086f7f553d82d8cf43c87d524cc3db7f3", + "project_files.progression_origins": "380b9ae9931faf5f161bcbd0e689dd93525be22ae2d9b00a524ce48d377d8c95", "project_files.publication_result": "10b4a5f3d4f37343322327024ddbedf5b6b69672c785e208ec48534d7b40c867", "project_files.query": "9a8b0d3a99adf5d7d53b62261dfac83e8b3ea44c10cb2f6c7a9c3d79422fef08", "project_files.recipe_create": "53a6760f03675d4d8a8d3a0ffd5b56689975667bbc136b1edf8a206307c18651", @@ -166,628 +168,638 @@ { "project_files.artifact_resolve": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/artifacts/resolve" + "path": "/api/workspace/project-files/artifacts/resolve", + "mutation": true }, "project_files.artifact_status": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/artifacts/status" + "path": "/api/workspace/project-files/artifacts/status", + "mutation": false }, "project_files.artifacts": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/artifacts" + "path": "/api/workspace/project-files/artifacts", + "mutation": false }, "project_files.candidate": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/candidates/{candidate_id}" + "path": "/api/workspace/project-files/candidates/{candidate_id}", + "mutation": false }, "project_files.candidate_select": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/studies/{study_id}/candidates" + "path": "/api/workspace/project-files/studies/{study_id}/candidates", + "mutation": true }, "project_files.data_annotate": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/{record_id}" + "path": "/api/workspace/project-files/data/{record_id}", + "mutation": true }, "project_files.data_availability": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/availability" + "path": "/api/workspace/project-files/data/{record_id}/availability", + "mutation": false }, "project_files.data_get": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}" + "path": "/api/workspace/project-files/data/{record_id}", + "mutation": false }, "project_files.data_history": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/history" + "path": "/api/workspace/project-files/data/{record_id}/history", + "mutation": false }, "project_files.data_import_audio": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-audio" + "path": "/api/workspace/project-files/data/import-audio", + "mutation": true }, "project_files.data_import_image": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-image" + "path": "/api/workspace/project-files/data/import-image", + "mutation": true }, "project_files.data_import_mesh": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-mesh" + "path": "/api/workspace/project-files/data/import-mesh", + "mutation": true }, "project_files.data_import_pointcloud": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-pointcloud" + "path": "/api/workspace/project-files/data/import-pointcloud", + "mutation": true }, "project_files.data_import_scientific": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-scientific" + "path": "/api/workspace/project-files/data/import-scientific", + "mutation": true }, "project_files.data_import_sequence": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-sequence" + "path": "/api/workspace/project-files/data/import-sequence", + "mutation": true }, "project_files.data_import_text": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-text" + "path": "/api/workspace/project-files/data/import-text", + "mutation": true }, "project_files.data_import_volume": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/import-volume" + "path": "/api/workspace/project-files/data/import-volume", + "mutation": true }, "project_files.data_preference_delete": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/preferences/delete" + "path": "/api/workspace/project-files/preferences/delete", + "mutation": true }, "project_files.data_preference_history": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/preference-history" + "path": "/api/workspace/project-files/data/{record_id}/preference-history", + "mutation": false }, "project_files.data_preference_save": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/preferences" + "path": "/api/workspace/project-files/preferences", + "mutation": true }, "project_files.data_preferences": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/preferences" + "path": "/api/workspace/project-files/data/{record_id}/preferences", + "mutation": false }, "project_files.data_query": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data" + "path": "/api/workspace/project-files/data", + "mutation": false }, "project_files.data_response_history": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/response-history" + "path": "/api/workspace/project-files/data/{record_id}/response-history", + "mutation": false }, "project_files.data_response_save": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/responses" + "path": "/api/workspace/project-files/responses", + "mutation": true }, "project_files.data_responses": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/responses" + "path": "/api/workspace/project-files/data/{record_id}/responses", + "mutation": false }, "project_files.data_rights": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/{record_id}/rights" + "path": "/api/workspace/project-files/data/{record_id}/rights", + "mutation": true }, "project_files.data_scientific": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/data/{record_id}/scientific" + "path": "/api/workspace/project-files/data/{record_id}/scientific", + "mutation": false }, "project_files.data_source_delete": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/{record_id}/source/delete" + "path": "/api/workspace/project-files/data/{record_id}/source/delete", + "mutation": true }, "project_files.data_source_save": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/data/{record_id}/source" + "path": "/api/workspace/project-files/data/{record_id}/source", + "mutation": true }, "project_files.definitions": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/definitions" + "path": "/api/workspace/project-files/definitions", + "mutation": false }, "project_files.dependencies": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/dependencies" + "path": "/api/workspace/project-files/runs/{run_id}/dependencies", + "mutation": false }, "project_files.evidence": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/evidence" + "path": "/api/workspace/project-files/runs/{run_id}/evidence", + "mutation": false }, "project_files.execution_approve_external": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/approvals" + "path": "/api/workspace/project-files/execution/approvals", + "mutation": true }, "project_files.execution_cancel": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/runs/{run_id}/cancel" + "path": "/api/workspace/project-files/execution/runs/{run_id}/cancel", + "mutation": true }, "project_files.execution_generation_resume": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/generation/{producer_id}/resume" + "path": "/api/workspace/project-files/execution/generation/{producer_id}/resume", + "mutation": true }, "project_files.execution_grounded_review": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/grounded/{producer_id}/review" + "path": "/api/workspace/project-files/execution/grounded/{producer_id}/review", + "mutation": true }, "project_files.execution_health": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/execution" + "path": "/api/workspace/project-files/execution", + "mutation": false }, "project_files.execution_preflight_external": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/execution/preflight" + "path": "/api/workspace/project-files/execution/preflight", + "mutation": false }, "project_files.execution_producer_cancel": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/producers/{producer_id}/cancel" + "path": "/api/workspace/project-files/execution/producers/{producer_id}/cancel", + "mutation": true }, "project_files.execution_producer_decide": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/producers/{producer_id}/decide" + "path": "/api/workspace/project-files/execution/producers/{producer_id}/decide", + "mutation": true }, "project_files.execution_producer_health": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/execution/producers/health" + "path": "/api/workspace/project-files/execution/producers/health", + "mutation": false }, "project_files.execution_producer_start": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/producers" + "path": "/api/workspace/project-files/execution/producers", + "mutation": true }, "project_files.execution_reconcile_current": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/reconcile-current" + "path": "/api/workspace/project-files/execution/reconcile-current", + "mutation": true }, "project_files.execution_study_abandon_preparation": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/abandon" + "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/abandon", + "mutation": true }, "project_files.execution_study_admit": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/runs" + "path": "/api/workspace/project-files/execution/studies/{study_id}/runs", + "mutation": true }, "project_files.execution_study_approve": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/approvals" + "path": "/api/workspace/project-files/execution/studies/{study_id}/approvals", + "mutation": true }, "project_files.execution_study_continue": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/continue" + "path": "/api/workspace/project-files/execution/studies/{study_id}/continue", + "mutation": true }, "project_files.execution_study_prepare_package": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/packages" + "path": "/api/workspace/project-files/execution/studies/{study_id}/packages", + "mutation": true }, "project_files.execution_study_prepare_verifier": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/verifiers" + "path": "/api/workspace/project-files/execution/studies/{study_id}/verifiers", + "mutation": true }, "project_files.execution_study_reconcile_preparation": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/reconcile" + "path": "/api/workspace/project-files/execution/studies/{study_id}/preparations/reconcile", + "mutation": true }, "project_files.execution_study_stop": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/studies/{study_id}/stop" + "path": "/api/workspace/project-files/execution/studies/{study_id}/stop", + "mutation": true }, "project_files.execution_submit_external": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/runs" + "path": "/api/workspace/project-files/execution/runs", + "mutation": true }, "project_files.expose_holdout": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/runs/{run_id}/disclose" + "path": "/api/workspace/project-files/runs/{run_id}/disclose", + "mutation": true }, "project_files.file_adopt": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/files/adopt" + "path": "/api/workspace/project-files/files/adopt", + "mutation": true }, "project_files.file_adopt_deletion": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/files/adopt-deletion" + "path": "/api/workspace/project-files/files/adopt-deletion", + "mutation": true }, "project_files.file_get": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/files/{file_id}" + "path": "/api/workspace/project-files/files/{file_id}", + "mutation": false }, "project_files.file_history": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/files/{file_id}/history" + "path": "/api/workspace/project-files/files/{file_id}/history", + "mutation": false }, "project_files.files_inspect": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/files" + "path": "/api/workspace/project-files/files", + "mutation": false }, "project_files.fork_info": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/fork" + "path": "/api/workspace/project-files/fork", + "mutation": false }, "project_files.graph_clone": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/graphs/{graph_id}/clone" + "path": "/api/workspace/project-files/graphs/{graph_id}/clone", + "mutation": true }, "project_files.graph_create": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/graphs" + "path": "/api/workspace/project-files/graphs", + "mutation": true }, "project_files.graph_create_external": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/graphs/external" + "path": "/api/workspace/project-files/graphs/external", + "mutation": true }, "project_files.graph_get": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/graphs/{graph_id}" + "path": "/api/workspace/project-files/graphs/{graph_id}", + "mutation": false }, "project_files.graph_save": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/graphs/{graph_id}" + "path": "/api/workspace/project-files/graphs/{graph_id}", + "mutation": true }, "project_files.graphs": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/graphs" + "path": "/api/workspace/project-files/graphs", + "mutation": false }, "project_files.lineage": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/lineage" + "path": "/api/workspace/project-files/lineage", + "mutation": false }, "project_files.lineage_record": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/lineage" + "path": "/api/workspace/project-files/lineage", + "mutation": true }, "project_files.material_intent": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/material-intents/{family}/{definition_id}" + "path": "/api/workspace/project-files/material-intents/{family}/{definition_id}", + "mutation": false }, "project_files.material_intent_capabilities": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/material-intents/capabilities" + "path": "/api/workspace/project-files/material-intents/capabilities", + "mutation": false }, "project_files.material_intents": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/material-intents" + "path": "/api/workspace/project-files/material-intents", + "mutation": false }, "project_files.materials": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/materials" + "path": "/api/workspace/project-files/materials", + "mutation": false }, "project_files.model_create": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/models" + "path": "/api/workspace/project-files/models", + "mutation": true }, "project_files.model_get": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/models/{model_id}" + "path": "/api/workspace/project-files/models/{model_id}", + "mutation": false }, "project_files.model_material": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/materials/model" + "path": "/api/workspace/project-files/materials/model", + "mutation": false }, "project_files.model_publish": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/models/{model_id}/publish" + "path": "/api/workspace/project-files/models/{model_id}/publish", + "mutation": true }, "project_files.model_save": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/models/{model_id}" + "path": "/api/workspace/project-files/models/{model_id}", + "mutation": true }, "project_files.models": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/models" + "path": "/api/workspace/project-files/models", + "mutation": false }, "project_files.native_preflight": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/native/preflight" + "path": "/api/workspace/project-files/execution/native/preflight", + "mutation": true }, "project_files.native_reconcile": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/native/phases/{phase_id}/reconcile" + "path": "/api/workspace/project-files/execution/native/phases/{phase_id}/reconcile", + "mutation": true }, "project_files.native_retry": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/native/runs/{run_id}/retry" + "path": "/api/workspace/project-files/execution/native/runs/{run_id}/retry", + "mutation": true }, "project_files.native_submit": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/execution/native/runs" + "path": "/api/workspace/project-files/execution/native/runs", + "mutation": true }, "project_files.package_state": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/candidates/{candidate_id}/package-state" + "path": "/api/workspace/project-files/candidates/{candidate_id}/package-state", + "mutation": false }, "project_files.policies": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/policies" + "path": "/api/workspace/project-files/policies", + "mutation": false }, "project_files.policy": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/policies/get" + "path": "/api/workspace/project-files/policies/get", + "mutation": false }, "project_files.producer": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/producers/{origin_catalog_id}/{producer_id}" + "path": "/api/workspace/project-files/producers/{origin_catalog_id}/{producer_id}", + "mutation": false }, "project_files.producers": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/producers" + "path": "/api/workspace/project-files/producers", + "mutation": false + }, + "project_files.progression": { + "method": "GET", + "path": "/api/workspace/project-files/progression", + "mutation": false + }, + "project_files.progression_origins": { + "method": "GET", + "path": "/api/workspace/project-files/progression/origins", + "mutation": false }, "project_files.publication_result": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/publication-result" + "path": "/api/workspace/project-files/runs/{run_id}/publication-result", + "mutation": false }, "project_files.recipe_create": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/recipes" + "path": "/api/workspace/project-files/recipes", + "mutation": true }, "project_files.recipe_get": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/recipes/{recipe_id}" + "path": "/api/workspace/project-files/recipes/{recipe_id}", + "mutation": false }, "project_files.recipe_material": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/materials/recipe" + "path": "/api/workspace/project-files/materials/recipe", + "mutation": false }, "project_files.recipe_prepare": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/recipes/{graph_id}/prepare" + "path": "/api/workspace/project-files/recipes/{graph_id}/prepare", + "mutation": true }, "project_files.recipe_save": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/recipes/{recipe_id}" + "path": "/api/workspace/project-files/recipes/{recipe_id}", + "mutation": true }, "project_files.recipes": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/recipes" + "path": "/api/workspace/project-files/recipes", + "mutation": false }, "project_files.release_freeze": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/releases/freeze" + "path": "/api/workspace/project-files/releases/freeze", + "mutation": true }, "project_files.release_preview": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/releases/preview" + "path": "/api/workspace/project-files/releases/preview", + "mutation": false }, "project_files.retained_analysis": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/analysis" + "path": "/api/workspace/project-files/runs/{run_id}/analysis", + "mutation": false }, "project_files.retained_analysis_export": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/analysis/export" + "path": "/api/workspace/project-files/runs/{run_id}/analysis/export", + "mutation": false }, "project_files.run": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}" + "path": "/api/workspace/project-files/runs/{run_id}", + "mutation": false }, "project_files.run_document_segment": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs/{run_id}/document-segment" + "path": "/api/workspace/project-files/runs/{run_id}/document-segment", + "mutation": false }, "project_files.runs": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runs" + "path": "/api/workspace/project-files/runs", + "mutation": false }, "project_files.runtime_abandon": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/runtime/abandon" + "path": "/api/workspace/project-files/runtime/abandon", + "mutation": true }, "project_files.runtime_capture": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/runtime/capture" + "path": "/api/workspace/project-files/runtime/capture", + "mutation": true }, "project_files.runtime_inspect": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runtime/inspect" + "path": "/api/workspace/project-files/runtime/inspect", + "mutation": false }, "project_files.runtime_material": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/materials/runtime" + "path": "/api/workspace/project-files/materials/runtime", + "mutation": false }, "project_files.runtime_sources": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runtime/sources" + "path": "/api/workspace/project-files/runtime/sources", + "mutation": false }, "project_files.runtime_status": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/runtime/status" + "path": "/api/workspace/project-files/runtime/status", + "mutation": false }, "project_files.save_as": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/save-as" + "path": "/api/workspace/project-files/save-as", + "mutation": true }, "project_files.search_create": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/searches" + "path": "/api/workspace/project-files/searches", + "mutation": true }, "project_files.search_delete": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/searches/{search_id}/delete" + "path": "/api/workspace/project-files/searches/{search_id}/delete", + "mutation": true }, "project_files.search_rename": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/searches/{search_id}" + "path": "/api/workspace/project-files/searches/{search_id}", + "mutation": true }, "project_files.searches": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/searches" + "path": "/api/workspace/project-files/searches", + "mutation": false }, "project_files.selection_create": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/selections" + "path": "/api/workspace/project-files/selections", + "mutation": true }, "project_files.selection_delete": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/selections/{selection_id}/delete" + "path": "/api/workspace/project-files/selections/{selection_id}/delete", + "mutation": true }, "project_files.selection_get": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/selections/{selection_id}" + "path": "/api/workspace/project-files/selections/{selection_id}", + "mutation": false }, "project_files.selection_rename": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/selections/{selection_id}" + "path": "/api/workspace/project-files/selections/{selection_id}", + "mutation": true }, "project_files.selections": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/selections" + "path": "/api/workspace/project-files/selections", + "mutation": false }, "project_files.snapshot": { "method": "POST", - "mutation": false, - "path": "/api/workspace/project-files/materials/snapshot" + "path": "/api/workspace/project-files/materials/snapshot", + "mutation": false }, "project_files.source_references": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/sources/{project_id}/references" + "path": "/api/workspace/project-files/sources/{project_id}/references", + "mutation": false }, "project_files.source_register": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/sources/register" + "path": "/api/workspace/project-files/sources/register", + "mutation": true }, "project_files.studies": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/studies" + "path": "/api/workspace/project-files/studies", + "mutation": false }, "project_files.study": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/studies/{study_id}" + "path": "/api/workspace/project-files/studies/{study_id}", + "mutation": false }, "project_files.study_create": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/studies" + "path": "/api/workspace/project-files/studies", + "mutation": true }, "project_files.study_save": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/studies/{study_id}" + "path": "/api/workspace/project-files/studies/{study_id}", + "mutation": true }, "project_files.trial": { "method": "GET", - "mutation": false, - "path": "/api/workspace/project-files/trials/{trial_id}" + "path": "/api/workspace/project-files/trials/{trial_id}", + "mutation": false }, "project_files.trial_prepare": { "method": "POST", - "mutation": true, - "path": "/api/workspace/project-files/studies/{study_id}/trials" + "path": "/api/workspace/project-files/studies/{study_id}/trials", + "mutation": true } } /* PROJECT_FILES_ROUTES_END */; @@ -818,6 +830,20 @@ } }); function clearScientific(){scientificEpoch=null;projectScientific.clear();} + const projectProgression=window.ProjectExperimentProgression({ + root:$('project-progression'), + async read(action,args,signal){ + if(!m2Allowed(action))throw Error('History inspection is unavailable for the selected project.'); + const value=await m2Request(action,args,epoch,{signal}); + if(value===null)throw new DOMException('Project selection changed.','AbortError'); + return value; + }, + inspectStudy:(origin,id)=>run(ticket=>inspectItem('studies',{id},origin,ticket),{clean:true}), + inspectTrial:(origin,id)=>run(ticket=>inspectItem('trials',{id},origin,ticket),{clean:true}), + inspectRun:(origin,id)=>run(ticket=>inspectItem('runs',{id},origin,ticket),{clean:true}), + onContextFocusLost:()=>$('project-files-status').focus() + }); + window.addEventListener('pagehide',()=>projectProgression.clear()); window.addEventListener('pagehide',clearScientific); const exact=(value,keys)=>value!==null&&typeof value==='object'&&!Array.isArray(value)&&Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); const str=(value,max,empty=false)=>typeof value==='string'&&Array.from(value).length<=max&&(empty||/\S/u.test(value)); @@ -1073,6 +1099,8 @@ } const m2Allowed=action=>!busy&&!blocked&&state?.state==='open'&&selectedContracts.has('project_files.'+action); function m2Controls(){ + projectProgression.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), + project_id:state?.project?.project_id,enabled:m2Allowed('progression')&&m2Allowed('progression_origins')}); if(blocked||state?.state!=='open'||scientificEpoch!==null&&(scientificEpoch!==epoch||explored?.checkpoint!==checkpoint()))clearScientific(); const action=$('project-explore-kind').value; $('project-explore-load').disabled=!m2Allowed(action); @@ -1233,12 +1261,12 @@ async function inspectItem(category,item,origin,ticket){ const fact=item.study??item.run??item.current??item.value??item; const entry={category,origin,id:fact.id??item.id,value:fact,checkpoint:checkpoint()}; - const request={data_query:['data_get','record_id'],models:['model_get','model_id'],graphs:['graph_get','graph_id'],recipes:['recipe_get','recipe_id'],studies:['study','study_id'],runs:['run','run_id'],selections:['selection_get','selection_id']}[category]; + const request={data_query:['data_get','record_id'],models:['model_get','model_id'],graphs:['graph_get','graph_id'],recipes:['recipe_get','recipe_id'],studies:['study','study_id'],trials:['trial','trial_id'],runs:['run','run_id'],selections:['selection_get','selection_id']}[category]; let detail=fact; if(category==='producers'){detail=await m2Request('producer',{origin_catalog_id:origin,producer_id:entry.id,family:$('project-producer-family').value},ticket);if(!detail)return;} - if(request){const args={[request[1]]:entry.id};if(['studies','runs'].includes(category))args.origin_catalog_id=origin;detail=await m2Request(request[0],args,ticket);if(!detail)return;} + if(request){const args={[request[1]]:entry.id};if(['studies','trials','runs'].includes(category))args.origin_catalog_id=origin;detail=await m2Request(request[0],args,ticket);if(!detail)return;} clearExplorationDetail();entry.value=detail.study??detail;entry.saveAction={models:'model_save',graphs:'graph_save',recipes:'recipe_save',studies:'study_save'}[category];explored=entry; - $('project-detail-title').textContent=entry.value.name??`${category==='runs'?'Historical Run':'Saved record'} ${entry.id??''}`; + $('project-detail-title').textContent=entry.value.name??`${category==='runs'?'Historical Run':category==='trials'?'Historical Trial':'Saved record'} ${entry.id??''}`; $('project-detail-facts').append(projectValue(detail)); if(category==='artifacts'){entry.artifactSha=detail.sha256;$('project-artifact-status').hidden=false;artifactResolutionForm(detail.availability?.state);} if(category==='models')$('project-model-publish').hidden=false; diff --git a/static/project-progression.js b/static/project-progression.js new file mode 100644 index 00000000..ae2fa97e --- /dev/null +++ b/static/project-progression.js @@ -0,0 +1,117 @@ +'use strict'; +// The native host owns binding/checkpoint/credential changes and exact reads. +// This controller owns source choice, its finite graph window and pending reads. +(() => { + const id=value=>typeof value==='string'&&/^[a-f0-9]{32}$/.test(value); + const natural=value=>Number.isSafeInteger(value)&&value>=0; + const short=value=>String(value).slice(0,12); + const key=(origin,trial)=>origin+'.'+trial; + const element=(tag,text)=>{const e=document.createElement(tag);if(text!==undefined)e.textContent=text;return e;}; + window.ProjectExperimentProgression=function({root,read,inspectStudy,inspectTrial,inspectRun,onContextFocusLost=()=>{}}) { + root.innerHTML='

            Experiment progression

            Explore recorded relationships across saved Studies. Solid edges show derivation; dashed edges show comparison baselines. Run states are saved outcomes.

            '; + const $=name=>root.querySelector('#project-progression-'+name); + let context=null,generation=0,pending=null,busy=false,origins=[],originsNext=0,originsTotal=0; + let nodes=new Map(),origin='',appliedCohort=null,next=0,total=0,related=new Map(),selected='',zoom=1; + function clearGraph(){nodes.clear();related.clear();origin='';appliedCohort=null;next=0;total=0;selected='';$('graph').replaceChildren();$('detail').replaceChildren();$('summary').textContent='';} + function controls(){ + const enabled=!!context?.enabled; + $('origins').disabled=!enabled||busy;$('origins-more').disabled=!enabled||busy; + $('origins-more').hidden=originsNext>=originsTotal; + $('origin').disabled=!enabled||busy||!origins.length;$('cohort').disabled=!enabled||busy; + $('load').disabled=!enabled||busy||!$('origin').value; + $('more').hidden=next>=total;$('more').disabled=!enabled||busy||nodes.size>=100; + for(const name of ['zoom-out','zoom-in','zoom-reset'])$(name).disabled=!enabled||busy||!nodes.size; + for(const button of root.querySelectorAll('[data-lineage-node],#project-progression-detail button'))button.disabled=!enabled||busy; + root.setAttribute('aria-busy',String(busy)); + } + function reset(){ + ++generation;pending?.abort();pending=null;busy=false;clearGraph();origins=[];originsNext=0;originsTotal=0; + const option=element('option','Choose a history source');option.value='';$('origin').replaceChildren(option); + $('cohort').value='';$('status').textContent='';controls(); + } + async function perform(work){ + if(!context?.enabled||busy)return; + const ticket=++generation,authority=context.key;pending=new AbortController();const signal=pending.signal;busy=true;controls(); + const active=()=>ticket===generation&&context?.key===authority&&!signal.aborted; + try{await work(active,signal);}catch(error){if(active()){const ownedFocus=root.contains(document.activeElement);reset();$('status').textContent=error.message;if(ownedFocus)$('origins').focus();}} + finally{if(active()){busy=false;pending=null;controls();}} + } + function page(value,offset,limit,items){ + if(!value||value.offset!==offset||value.limit!==limit||!natural(value.total)||!Array.isArray(items)||items.length!==Math.min(limit,Math.max(0,value.total-offset)))throw Error('History page differs from its selected continuation.'); + } + async function loadOrigins(append=false){ + return perform(async(active,signal)=>{ + const offset=append?originsNext:0,value=await read('progression_origins',{offset,limit:40},signal);if(!active())return; + page(value,offset,40,value.items); + const seen=new Set(append?origins.map(item=>item.origin_catalog_id):[]); + for(const item of value.items){if(!id(item.origin_catalog_id)||seen.has(item.origin_catalog_id)||!natural(item.study_count)||!natural(item.trial_count))throw Error('Invalid history source.');seen.add(item.origin_catalog_id);} + if(!append){clearGraph();origins=[];const option=element('option','Choose a history source');option.value='';$('origin').replaceChildren(option);} + origins.push(...value.items);originsNext=offset+value.items.length;originsTotal=value.total; + for(const item of value.items){const option=element('option',(item.origin_catalog_id===context.project_id?'This project':'Imported catalog '+short(item.origin_catalog_id))+' · '+item.study_count+' Studies · '+item.trial_count+' Trials');option.value=item.origin_catalog_id;$('origin').append(option);} + $('status').textContent=origins.length?'Choose a history source to load its Trials.':'No recorded Study history in this project.'; + if(!append)$('origin').focus(); + }); + } + function validNode(node,source){ + const trial=node?.trial,lineage=node?.lineage; + if(!id(trial?.id)||!id(trial?.study_id)||trial.study_id!==node.study?.id||typeof node.study.name!=='string'|| + trial.material?.catalog_id!==source||!natural(node.attempt_count)||!Array.isArray(node.runs)||node.runs.length>5||node.runs.length>node.attempt_count||node.runs_truncated!==(node.attempt_count>node.runs.length)|| + node.runs.some(run=>!id(run.id)||typeof run.state!=='string'||Object.hasOwn(run,'validation')))throw Error('Invalid saved Trial overview.'); + if(lineage!==null&&(lineage?.trial?.catalog_id!==source||lineage.trial.trial_id!==trial.id||lineage.trial.material_sha256!==trial.material.sha256))throw Error('Trial ancestry belongs to different material.'); + } + async function loadGraph(append=false){ + const requestedOrigin=append?origin:$('origin').value,cohort=append?appliedCohort:($('cohort').value||null),offset=append?next:0; + if(!id(requestedOrigin)||append&&nodes.size>=100)return; + return perform(async(active,signal)=>{ + const args={origin_catalog_id:requestedOrigin,offset,limit:20,...(cohort===null?{}:{cohort})}; + const value=await read('progression',args,signal);if(!active())return; + page(value,offset,20,value.nodes); + if(value.origin_catalog_id!==requestedOrigin||value.evidence!=='saved_metadata_only'||value.historical!==true||value.execution_available!==false||value.cohort!==cohort||!Array.isArray(value.related)||value.related.length>40)throw Error('History response differs from the selected source.'); + const seen=new Set(append?nodes.keys():[]); + for(const node of value.nodes){validNode(node,requestedOrigin);const identity=key(requestedOrigin,node.trial.id);if(seen.has(identity))throw Error('Duplicate Trial in history page.');seen.add(identity);} + for(const ref of value.related)if(!id(ref.catalog_id)||!id(ref.trial_id)||typeof ref.material_sha256!=='string'||!/^[a-f0-9]{64}$/.test(ref.material_sha256))throw Error('Invalid related Trial identity.'); + if(seen.size>100)throw Error('The graph window holds 100 Trials; choose a recorded cohort for another window.'); + if(!append){clearGraph();origin=requestedOrigin;appliedCohort=cohort;} + for(const node of value.nodes)nodes.set(key(origin,node.trial.id),structuredClone(node)); + for(const ref of value.related)related.set(key(ref.catalog_id,ref.trial_id),structuredClone(ref)); + next=offset+value.nodes.length;total=value.total;render(); + $('status').textContent=nodes.size?'Loaded '+nodes.size+' recorded Trials.':'No Trials match this source and recorded cohort.'; + }); + } + function detail(){ + const node=nodes.get(selected);$('detail').replaceChildren();if(!node)return; + const add=(tag,text)=>{const e=element(tag,text);$('detail').append(e);return e;}; + add('h3','Trial '+short(node.trial.id));add('p',node.study.name); + if(node.lineage){const intent=node.lineage.intent;add('p',intent.hypothesis);add('p',intent.change_summary);} + else add('p','Historical ancestry and intent are unknown. No relationships are inferred.'); + for(const kind of ['parent','baseline']){ + const ref=node.lineage?.intent[kind],label=kind==='parent'?'Parent':'Comparison baseline'; + if(ref){const button=add('button',label+' Trial '+short(ref.trial_id));button.type='button';button.onclick=()=>inspectTrial(ref.catalog_id,ref.trial_id);} + else add('p',label+': none declared.'); + } + for(const [label,inspect,identity] of [['Inspect Study',inspectStudy,node.study.id],['Inspect Trial',inspectTrial,node.trial.id]]){const button=add('button',label);button.type='button';button.onclick=()=>inspect(origin,identity);} + add('h4','Saved Run outcomes'); + for(const run of node.runs){const button=add('button','Inspect Run '+short(run.id)+' · saved '+run.state);button.type='button';button.onclick=()=>inspectRun(origin,run.id);} + if(!node.attempt_count)add('p','No recorded admitted attempt.'); + if(node.runs_truncated)add('p','Showing five of '+node.attempt_count+' recorded attempts.'); + const facts=add('details'),summary=element('summary','Exact original references and recorded intent');facts.append(summary,element('pre',JSON.stringify({origin_catalog_id:origin,trial:node.trial,lineage:node.lineage},null,2))); + } + function render(){ + const focused=$('graph').contains(document.activeElement)?document.activeElement.id:null; + const graph=window.WorkspaceExperimentGraph.render([...nodes.values()].map(node=>({id:key(origin,node.trial.id),title:node.study.name,recordedAt:node.lineage?.recorded_at||'', + parent:node.lineage?.intent.parent?key(node.lineage.intent.parent.catalog_id,node.lineage.intent.parent.trial_id):null, + baseline:node.lineage?.intent.baseline?key(node.lineage.intent.baseline.catalog_id,node.lineage.intent.baseline.trial_id):null, + lines:[short(node.trial.id)+' · '+(node.lineage?.intent.ancestry||'ancestry unknown'),node.lineage?.intent.change_summary||'Intent not recorded',(node.runs[0]?.state||'prepared')+' · '+node.attempt_count+' recorded attempts']})),{selected,zoom}); + $('graph').innerHTML=graph.html; + for(const button of $('graph').querySelectorAll('[data-lineage-node]')){button.disabled=busy;button.onclick=()=>{selected=button.dataset.lineageNode;render();$('status').textContent='Selected Trial '+short(nodes.get(selected).trial.id)+'.';};} + const outside=[...related.keys()].filter(identity=>!nodes.has(identity)).length; + $('summary').textContent=nodes.size+' of '+total+' Trials loaded · '+(appliedCohort===null?'all recorded cohorts':'applied cohort: '+appliedCohort)+' · '+outside+' related identities outside this graph window.'; + detail();if(focused)$('graph').querySelector('#'+CSS.escape(focused))?.focus({preventScroll:true}); + } + $('origins').onclick=()=>loadOrigins();$('origins-more').onclick=()=>loadOrigins(true); + $('form').onsubmit=event=>{event.preventDefault();loadGraph();};$('more').onclick=()=>loadGraph(true); + $('origin').onchange=()=>{++generation;pending?.abort();pending=null;busy=false;clearGraph();$('status').textContent='Load the selected history source.';controls();}; + for(const [name,delta] of [['zoom-in',0.2],['zoom-out',-0.2],['zoom-reset',null]])$(name).onclick=()=>{zoom=delta===null?1:Math.max(0.6,Math.min(1.6,zoom+delta));render();}; + return {setContext(value){if(value.key!==context?.key){const ownedFocus=root.contains(document.activeElement);context=value;reset();if(ownedFocus)onContextFocusLost();}else context=value;root.hidden=!value.enabled;controls();},clear(){context=null;reset();root.hidden=true;}}; + }; +})(); diff --git a/tests/test_project_files_m2_http.py b/tests/test_project_files_m2_http.py index 6a92ed91..63db21a4 100644 --- a/tests/test_project_files_m2_http.py +++ b/tests/test_project_files_m2_http.py @@ -138,6 +138,32 @@ def config(self,name,token): 'catalog_id':self.owner.catalog_id,'token_file':str(secret),'timeout':30}));config.chmod(0o600) return config + def test_progression_public_reads_pin_origin_scope_and_checkpoint(self): + from test_project_history import policy + self.create();old_checkpoint=self.state()['project']['checkpoint'] + self.call('study_create',name='Saved progression fixture', + limits={'attempt_slots':3,'admitted_wall_seconds':100,'retained_bytes':1000000,'peak_staging_bytes':1000000}, + policy=policy()) + origin=self.state()['project']['project_id'] + with self.owner.bound(self.workspace): + inspector=self.workspace.access.issue({'label':'Progression inspector','scopes':['inspect']}) + before=m1.canonical_bytes(self.external/'alpha') + origins=self.call('progression_origins',token=inspector['token']) + self.assertEqual(origins['items'],[{'origin_catalog_id':origin,'study_count':1,'trial_count':0}]) + result=self.call('progression',token=inspector['token'],origin_catalog_id=origin,limit=1) + self.assertEqual(result['nodes'],[]);self.assertEqual(result['related'],[]) + self.assertEqual(result['evidence'],'saved_metadata_only') + self.assertFalse(result['execution_available']) + with self.owner.bound(self.workspace): + self.workspace.access.revoke(inspector['identity']['credential_id']) + for action,args in [('progression_origins',{}),('progression',{'origin_catalog_id':origin})]: + arguments=self.arguments(action,**args) + self.assertEqual(self.transport(action,arguments,token=inspector['token'])[0],401) + self.assertEqual(self.transport(action,dict(arguments,checkpoint=old_checkpoint))[0],409) + status,error=self.transport('progression',self.arguments('progression',origin_catalog_id='f'*32)) + self.assertEqual(status,409);self.assertEqual(error['code'],'not_found') + self.assertEqual(m1.canonical_bytes(self.external/'alpha'),before) + def test_public_mixed_authoring_git_clone_and_cold_reconstruction(self): self.create();image,raw=self.image() image=self.call('data_annotate',record_id=image['id'],revision=image['revision'],source_revision=image['source_revision'], diff --git a/tests/test_project_history.py b/tests/test_project_history.py index 61da134c..c505f663 100644 --- a/tests/test_project_history.py +++ b/tests/test_project_history.py @@ -293,6 +293,55 @@ def start(self): self.host,self.owner,self.session=host,owner,session self.history=ProjectHistory(session,materials=Materials()) + def test_progression_uses_original_namespaces_exact_edges_and_bounded_pages(self): + parent='a'*32;child='b'*32 + ref={'catalog_id':ORIGIN,'trial_id':parent,'material_sha256':MATERIAL} + records=[study('3'*32,trials=[parent]),study('4'*32,trials=[child]), + trial(parent,'3'*32),trial(child,'4'*32,parent=ref,baseline=ref)] + before=self.store.snapshot() + captured=self.store.commit(records,before['checkpoint'],'fixture-progress',digest(records)) + self.session.adopt(captured);before=self.session.snapshot + with self.owner.bound(self.host),patch('runs.Runs',side_effect=AssertionError('No live Run owner')): + origins=self.history.progression_origins(limit=1) + self.assertEqual(origins,{'items':[{'origin_catalog_id':ORIGIN,'study_count':2,'trial_count':2}], + 'total':1,'offset':0,'limit':1}) + self.assertEqual(self.history.progression_origins(offset=1)['items'],[]) + first=self.history.progression(ORIGIN,limit=1) + self.assertEqual(first['nodes'][0]['trial'],records[2]['document']['trial']) + self.assertIsNone(first['nodes'][0]['lineage']) + self.assertEqual(first['related'],[]) + second=self.history.progression(ORIGIN,offset=1,limit=1) + self.assertEqual(second['nodes'][0]['lineage'],records[3]['document']['lineage']) + self.assertEqual(second['related'],[ref]) + self.assertEqual(second['nodes'][0]['attempt_count'],0) + self.assertFalse(second['nodes'][0]['runs_truncated']) + self.assertFalse(second['execution_available']);self.assertTrue(second['historical']) + self.assertEqual(second['evidence'],'saved_metadata_only') + both=self.history.progression(ORIGIN) + self.assertEqual(both['related'],[]) + filtered=self.history.progression(ORIGIN,cohort='cross-study') + self.assertEqual([v['trial']['id'] for v in filtered['nodes']],[child]) + self.assertEqual(self.history.progression(ORIGIN,cohort='not recorded')['nodes'],[]) + for kwargs in ({'limit':21},{'offset':True},{'cohort':''}): + with self.assertRaises(WorkbenchError):self.history.progression(ORIGIN,**kwargs) + with self.assertRaisesRegex(WorkbenchError,'origin'): + self.history.progression(self.session.snapshot['project_id']) + self.assertEqual(self.session.snapshot,before) + from access import AccessError + original_facts=self.history._facts + for operation in (self.history.progression_origins,lambda:self.history.progression(ORIGIN)): + with self.owner.bound(self.host): + inspector=self.host.access.issue({'label':'Revoked during progression read','scopes':['inspect']}) + def revoke_after_read(): + facts=original_facts() + with self.owner.bound(self.host): + self.host.access.revoke(inspector['identity']['credential_id']) + return facts + with self.host.access.bind(inspector['token']),patch.object(self.history,'_facts',side_effect=revoke_after_read): + with self.assertRaises(AccessError):operation() + for table in ('model_trials','model_studies','workspace_runs','access_admissions'): + self.assertEqual(self.host.db.execute('SELECT count(*) FROM '+table).fetchone()[0],0) + def test_metadata_edits_cold_reopen_keep_history_without_active_authority(self): with self.owner.bound(self.host): before=self.session.snapshot['checkpoint'] diff --git a/tests/test_workspace_project_files.cjs b/tests/test_workspace_project_files.cjs index ee2e9d80..0fefa259 100644 --- a/tests/test_workspace_project_files.cjs +++ b/tests/test_workspace_project_files.cjs @@ -34,6 +34,9 @@ function mount(initial=[reply(state()),reply(records())],capabilityValue=capabil const context={document:{getElementById:id=>{assert.ok(nodes.has(id),'Unknown node '+id);return nodes.get(id);},createElement:tag=>new Node(tag)},URLSearchParams,TextEncoder,crypto,console,window:null}; context.window=context;context.TuldokAuth={canReview:()=>true,fetch:(url,options)=>{if(url==='/api/agent/capabilities'){discoveries.push({url,options});return Promise.resolve(reply(capabilityValue));}const header=options.headers?.['Tuldok-Operation-Contract'];assert.ok(header,'Every project call selects an exact contract');const [operation,version,digest]=header.split('/');assert.equal(version,'1');assert.equal(digest,pins[operation]);requests.push({url,options});assert.ok(answers.length,'Unexpected fetch '+url);return Promise.resolve(answers.shift());}}; context.addEventListener=(name,handler)=>events.set(name,handler); + // View modules have separate real-browser evidence; this fixture owns native record/control lifetimes. + context.ScientificFieldInspector=()=>({clear(){},show(){}}); + context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,events,discoveries,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, diff --git a/tests/test_workspace_project_files_m4.cjs b/tests/test_workspace_project_files_m4.cjs index c1a8cdec..77448d3f 100644 --- a/tests/test_workspace_project_files_m4.cjs +++ b/tests/test_workspace_project_files_m4.cjs @@ -39,6 +39,9 @@ function mount(capabilityValue=capabilities){ const request={url,options};requests.push(request);assert.ok(answers.length,'Unexpected fetch '+url); const next=answers.shift();return typeof next==='function'?next(request):next; }}; + // View modules have separate real-browser evidence; this fixture owns native record/control lifetimes. + context.ScientificFieldInspector=()=>({clear(){},show(){}}); + context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, diff --git a/tuldok_client.py b/tuldok_client.py index 11efd24d..c26e5489 100644 --- a/tuldok_client.py +++ b/tuldok_client.py @@ -394,6 +394,14 @@ def lineage_record(self, **arguments): """lineage record; exact selected project checkpoint, existing domain validation, no execution.""" return self._call('lineage_record', arguments) + def progression_origins(self, **arguments): + """Read original history namespaces at the selected checkpoint.""" + return self._call('progression_origins', arguments) + + def progression(self, **arguments): + """Read a bounded saved Trial graph without execution or metric disclosure.""" + return self._call('progression', arguments) + def model_create(self, **arguments): """model create; exact selected project checkpoint, existing domain validation, no execution.""" return self._call('model_create', arguments) From b9df1f69ef0abec929c98224597748183b5bdf6a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 19:22:55 -0700 Subject: [PATCH 10/15] Define bounded source-bound retained scalar measurements Preserve complete subject/case/metric matrices, declared dimension ownership and explicit missing/failed results. Verify every measured binary64 value against retained JSON bytes and pin metric/scope definitions. Queries perform fixed typed selections and maximum/count projections without producer execution or scientific acceptance claims. Validation: nine decision tests pass, including malformed sources, exact tiny/signed-zero values, incomplete matrices, typed selectors and bounds. A real C106/C107 packet preserves 1024 values and256 unavailable timing entries. Native project admission, current-access lifecycle and chart rendering remain separate qualification gates. --- retained_measurements.py | 340 ++++++++++++++++++++++++++++ tests/test_retained_measurements.py | 151 ++++++++++++ 2 files changed, 491 insertions(+) create mode 100644 retained_measurements.py create mode 100644 tests/test_retained_measurements.py diff --git a/retained_measurements.py b/retained_measurements.py new file mode 100644 index 00000000..97c5032e --- /dev/null +++ b/retained_measurements.py @@ -0,0 +1,340 @@ +"""Finite retained scalar tables, exact source bindings and explicit missingness. + +This owner validates data representation and performs fixed selections/reductions. +Producers own metric definitions, comparison scope and scientific interpretation. +It reads no paths, runs no producer and grants no evidence/disclosure authority. +""" +import base64 +import binascii +import copy +from decimal import Decimal +import hashlib +import json +import math +import re +import struct + +from workbench import WorkbenchError + + +FORMAT = 'retained_measurements_v1' +MAX_BYTES = 32 * 1024 * 1024 +MAX_SOURCE_BYTES = 8 * 1024 * 1024 +MAX_SOURCE_TOTAL = 16 * 1024 * 1024 +MAX_RESPONSE_BYTES = 256 * 1024 +MAX_ROWS = 100000 +STATES = ('measured', 'failed', 'unsupported', 'unavailable') +ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$') +SHA = re.compile(r'^[a-f0-9]{64}$') + + +def fail(message, code='invalid', status=400): + raise WorkbenchError('Retained measurements: ' + message, code, status) + + +def exact(value, fields): + if type(value) is not dict or set(value) != set(fields): + fail('Object fields differ from schema 1.') + + +def text(value, maximum=240): + if (type(value) is not str or not value.strip() or len(value) > maximum + or any(ord(c) < 32 or 127 <= ord(c) <= 159 for c in value)): + fail('Bounded nonempty display text required.') + try: + value.encode('utf-8') + except UnicodeError: + fail('Text must contain valid Unicode scalars.') + return value + + +def identifier(value): + if type(value) is not str or not ID.fullmatch(value): + fail('Invalid declared identity.') + return value + + +def number(value): + return (type(value) is int and abs(value) <= 2**53-1 + or type(value) is float and math.isfinite(value)) + + +def encode(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, allow_nan=False).encode('utf-8') + + +def digest(value): + return hashlib.sha256(encode(value)).hexdigest() + + +def decode(raw): + def pairs(items): + result = {} + for key, value in items: + if key in result: + fail('Duplicate JSON key.') + result[key] = value + return result + def floating(value): + result = float(value) + if not math.isfinite(result) or result == 0 and Decimal(value) != 0: + fail('JSON number exceeds finite binary64 range.') + return result + def constant(_value): + fail('Nonfinite JSON constants are unsupported.') + try: + value = json.loads(raw.decode('utf-8'), object_pairs_hook=pairs, + parse_float=floating, parse_constant=constant) + pending = [(value, 0)] + while pending: + item, depth = pending.pop() + if depth > 24: + fail('JSON nesting exceeds the supported bound.', 'too_large', 413) + if type(item) is dict: + pending.extend((v, depth+1) for v in item.values()) + for key in item: + key.encode('utf-8') + elif type(item) is list: + pending.extend((v, depth+1) for v in item) + elif type(item) is str: + item.encode('utf-8') + return value + except (ValueError, UnicodeError, RecursionError): + fail('Unique-key finite UTF-8 JSON required.') + + +def collection(values, maximum): + if type(values) is not list or not values: + fail('Nonempty declared collection required.') + if len(values) > maximum: + fail('Declared collection exceeds its supported bound.', 'too_large', 413) + result = {} + for value in values: + if type(value) is not dict: + fail('Declared entries must be objects.') + identity = identifier(value.get('id')) + if identity in result: + fail('Duplicate declared identity.') + result[identity] = value + return result + + +def cell(kind, value): + if value is None: + return True + if kind == 'string': + return type(value) is str and len(value) <= 256 + if kind == 'boolean': + return type(value) is bool + if kind == 'integer': + return type(value) is int and abs(value) <= 2**53-1 + return kind == 'number' and number(value) + + +class MeasurementTable: + """Own an immutable input snapshot; returned projections are independent.""" + def __init__(self, raw): + if type(raw) is not bytes or not 0 < len(raw) <= MAX_BYTES: + fail('A bounded complete byte snapshot is required.', 'too_large', 413) + self.sha256 = hashlib.sha256(raw).hexdigest() + doc = decode(raw) + exact(doc, ('format', 'schema_version', 'title', 'scope', 'dimensions', 'subjects', + 'cases', 'metrics', 'observations', 'sources')) + if doc['format'] != FORMAT or type(doc['schema_version']) is not int or doc['schema_version'] != 1: + fail('Unsupported measurement schema.', 'unsupported', 409) + text(doc['title']) + self._doc = doc + self._sources = collection(doc['sources'], 256) + self._json = {} + total = 0 + for identity, source in self._sources.items(): + exact(source, ('id', 'label', 'kind', 'bytes', 'sha256', 'data_base64')) + text(source['label']) + if source['kind'] not in ('json', 'opaque'): + fail('Unsupported retained source kind.', 'unsupported', 409) + if type(source['bytes']) is not int or not 0 < source['bytes'] <= MAX_SOURCE_BYTES: + fail('Source byte declaration exceeds its bound.', 'too_large', 413) + total += source['bytes'] + if total > MAX_SOURCE_TOTAL: + fail('Source collection exceeds its decoded byte bound.', 'too_large', 413) + if type(source['sha256']) is not str or not SHA.fullmatch(source['sha256']): + fail('Source requires an exact SHA256.') + value = source['data_base64'] + if type(value) is not str or len(value) != 4*((source['bytes']+2)//3): + fail('Source encoding differs from its byte declaration.') + try: + payload = base64.b64decode(value, validate=True) + except (ValueError, binascii.Error): + fail('Invalid source base64.') + if (len(payload) != source['bytes'] or hashlib.sha256(payload).hexdigest() != source['sha256'] + or base64.b64encode(payload).decode('ascii') != value): + fail('Retained source bytes differ from their declaration.') + if source['kind'] == 'json': + self._json[identity] = decode(payload) + scope = doc['scope'] + exact(scope, ('id', 'label', 'population', 'definition')) + identifier(scope['id']);text(scope['label']) + if scope['population'] not in ('training', 'consumed', 'independent', 'reserved', 'unknown'): + fail('Unsupported declared population.', 'unsupported', 409) + self._definition(scope, ('id', 'label', 'population')) + if type(doc['dimensions']) is not list or len(doc['dimensions']) > 32: + fail('Invalid dimension collection.') + self._dimensions = collection(doc['dimensions'], 32) if doc['dimensions'] else {} + for dimension in self._dimensions.values(): + exact(dimension, ('id', 'label', 'type', 'units', 'scope')) + text(dimension['label']) + if dimension['scope'] not in ('subject', 'case', 'observation'): + fail('Unsupported dimension owner.', 'unsupported', 409) + if dimension['type'] not in ('string', 'number', 'integer', 'boolean'): + fail('Unsupported dimension type.', 'unsupported', 409) + if dimension['units'] is not None: + text(dimension['units'], 120) + self._subjects = collection(doc['subjects'], MAX_ROWS) + self._cases = collection(doc['cases'], 4096) + self._metrics = collection(doc['metrics'], 32) + expected = len(self._subjects)*len(self._cases)*len(self._metrics) + if expected > MAX_ROWS: + fail('Complete subject/case/metric matrix exceeds its row bound.', 'too_large', 413) + for subject in self._subjects.values(): + exact(subject, ('id', 'label', 'dimensions', 'reference'));text(subject['label']);self._reference(subject['reference']) + self._dimension_values('subject',subject['dimensions']) + for case in self._cases.values(): + exact(case, ('id', 'dimensions', 'reference')) + self._dimension_values('case',case['dimensions']) + self._reference(case['reference']) + for metric in self._metrics.values(): + exact(metric, ('id', 'label', 'units', 'definition'));text(metric['label']);text(metric['units'], 120) + self._definition(metric, ('id', 'label', 'units')) + observations = doc['observations'] + if type(observations) is not list or len(observations) != expected: + fail('Every subject/case/metric needs one explicit result, including missing or failed results.') + self._observations = collection(observations, MAX_ROWS) + pairs = set() + for observation in observations: + exact(observation, ('id', 'subject_id', 'case_id', 'metric_id', 'state', 'value', 'reason', 'dimensions', 'reference')) + self._dimension_values('observation',observation['dimensions']) + identity = observation['subject_id'], observation['case_id'], observation['metric_id'] + for value in identity: + identifier(value) + if (identity in pairs or identity[0] not in self._subjects or identity[1] not in self._cases + or identity[2] not in self._metrics): + fail('Duplicate or unresolved measurement identity.') + pairs.add(identity) + if observation['state'] not in STATES: + fail('Unsupported observation state.', 'unsupported', 409) + source_value = self._reference(observation['reference']) + if observation['state'] == 'measured': + value = observation['value'] + if (not number(value) or not number(source_value) or observation['reason'] is not None + or struct.pack('!d', float(value)) != struct.pack('!d', float(source_value))): + fail('Measured scalar differs from its exact retained source value.') + else: + if observation['value'] is not None: + fail('Unmeasured results require an explicit null value.') + text(observation['reason'], 1000) + + def _dimension_values(self,scope,values): + columns={k:d for k,d in self._dimensions.items() if d['scope']==scope} + if type(values) is not dict or set(values)!=set(columns): + fail('Dimension values differ from their declared owner and columns.') + if any(not cell(columns[k]['type'],v) for k,v in values.items()): + fail('Dimension value differs from its declared type.') + + def _row_dimensions(self,row): + return {**self._subjects[row['subject_id']]['dimensions'], + **self._cases[row['case_id']]['dimensions'],**row['dimensions']} + + def _reference(self, reference, *, hashed=False): + exact(reference, ('source_id', 'pointer', 'sha256') if hashed else ('source_id', 'pointer')) + source = self._sources.get(identifier(reference['source_id'])) + if source is None: + fail('Source reference is unresolved.') + pointer = reference['pointer'] + if pointer is None and not hashed: + return None + if (type(pointer) is not str or len(pointer) > 512 or pointer and not pointer.startswith('/') + or re.search(r'~(?:[^01]|$)', pointer) or source['kind'] != 'json'): + fail('A bounded existing JSON pointer is required.') + value = self._json[source['id']] + try: + for part in pointer[1:].split('/') if pointer else (): + part = part.replace('~1', '/').replace('~0', '~') + if type(value) is dict: + value = value[part] + elif type(value) is list and re.fullmatch(r'0|[1-9][0-9]*', part): + value = value[int(part)] + else: + fail('Source pointer does not identify a retained value.') + except (KeyError, IndexError): + fail('Source pointer does not identify a retained value.') + if hashed and reference['sha256'] != digest(value): + fail('Declared definition digest differs from its retained source.') + return value + + def _definition(self, descriptor, fields): + value = self._reference(descriptor['definition'], hashed=True) + if type(value) is not dict or len(encode(value)) > 16384 or any(value.get(k) != descriptor[k] for k in fields): + fail('Definition metadata differs from its exact retained document.') + return value + + def metadata(self): + return copy.deepcopy({'format': FORMAT, 'packet_sha256': self.sha256, 'title': self._doc['title'], + 'scope': self._doc['scope'], 'scope_definition':self._definition(self._doc['scope'],('id','label','population')), + 'dimensions': self._doc['dimensions'], 'metrics': self._doc['metrics'], + 'subject_count':len(self._subjects),'case_count': len(self._cases), + 'observation_count': len(self._observations),'source_count':len(self._sources)}) + + def subjects(self, *, offset=0, limit=40): + if type(offset) is not int or not 0<=offset<=2**53-1 or type(limit) is not int or not 1<=limit<=40: + fail('Invalid subject page bounds.') + return copy.deepcopy({'items':list(self._subjects.values())[offset:offset+limit], + 'offset':offset,'limit':limit,'total':len(self._subjects)}) + + def query(self, metric_id, *, subject_ids=None, filters=None, offset=0, limit=50): + identifier(metric_id) + if metric_id not in self._metrics: + fail('Metric is unavailable.', 'not_found', 404) + selected = list(self._subjects) if subject_ids is None else subject_ids + if (type(selected) is not list or not selected or len(selected)>MAX_ROWS or any(type(v) is not str for v in selected) + or len(set(selected)) != len(selected) or any(v not in self._subjects for v in selected)): + fail('Choose existing unique subjects.') + filters = {} if filters is None else filters + if (type(filters) is not dict or any(k not in self._dimensions or not cell(self._dimensions[k]['type'],v) + for k,v in filters.items())): + fail('Filters must use declared dimensions and typed scalar values.') + if type(offset) is not int or not 0<=offset<=2**53-1 or type(limit) is not int or not 1<=limit<=100: + fail('Page bounds must be explicit finite integers.') + rows = [v for v in self._doc['observations'] if v['metric_id']==metric_id + and v['subject_id'] in selected + and all(self._row_dimensions(v)[k]==wanted for k,wanted in filters.items())] + summaries = [] + for identity in selected: + own = [v for v in rows if v['subject_id']==identity] + counts = {state:sum(v['state']==state for v in own) for state in STATES} + measured = [v for v in own if v['state']=='measured'] + maximum = max(measured, key=lambda v:v['value']) if measured else None + summaries.append({'subject_id':identity,'count':len(own),'states':counts, + 'maximum':None if maximum is None else {k:maximum[k] for k in ('id','case_id','value')}}) + result = {'format':'retained_measurement_query_v1','packet_sha256':self.sha256, + 'scope':copy.deepcopy(self._doc['scope']),'metric':copy.deepcopy(self._metrics[metric_id]), + 'scope_definition':copy.deepcopy(self._definition(self._doc['scope'],('id','label','population'))), + 'definition':copy.deepcopy(self._definition(self._metrics[metric_id],('id','label','units'))), + 'filters':copy.deepcopy(filters),'subject_ids':list(selected),'summary':summaries, + 'rows':[dict(copy.deepcopy(v),dimensions=copy.deepcopy(self._row_dimensions(v))) + for v in rows[offset:offset+limit]], + 'total':len(rows),'offset':offset,'limit':limit,'sampling':'none'} + if len(encode(result)) > MAX_RESPONSE_BYTES: + fail('Selected response exceeds its bound; choose a smaller page.', 'too_large', 413) + return result + + def inspect(self, observation_id): + identifier(observation_id) + observation = self._observations.get(observation_id) + if observation is None: + fail('Observation is unavailable.', 'not_found', 404) + reference = observation['reference'];source = self._sources[reference['source_id']] + return copy.deepcopy({'packet_sha256':self.sha256,'observation':observation, + 'case':self._cases[observation['case_id']],'subject':self._subjects[observation['subject_id']], + 'metric':self._metrics[observation['metric_id']],'scope':self._doc['scope'], + 'scope_definition':self._definition(self._doc['scope'],('id','label','population')), + 'source':{k:v for k,v in source.items() if k!='data_base64'}}) diff --git a/tests/test_retained_measurements.py b/tests/test_retained_measurements.py new file mode 100644 index 00000000..73da3da7 --- /dev/null +++ b/tests/test_retained_measurements.py @@ -0,0 +1,151 @@ +"""Source-bound scalar, complete-denominator and closed-query decisions.""" +import base64 +import copy +import hashlib +import json +from pathlib import Path +import struct +import sys +import unittest + +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from retained_measurements import MeasurementTable, decode, digest, encode +from workbench import WorkbenchError + + +def source(identity,value,kind='json'): + raw=encode(value) if kind=='json' else value + return {'id':identity,'label':identity,'kind':kind,'bytes':len(raw), + 'sha256':hashlib.sha256(raw).hexdigest(),'data_base64':base64.b64encode(raw).decode('ascii')} + + +def ref(pointer,hashed=None): + result={'source_id':'context','pointer':pointer} + if hashed is not None:result['sha256']=digest(hashed) + return result + + +def fixture(): + scope={'id':'consumed','label':'Retained consumed diagnostics','population':'consumed', + 'task':'Fixture measurements, not scientific acceptance'} + metrics=[{'id':'error','label':'Declared error ratio','units':'1','description':'Exact retained scalar'}, + {'id':'latency','label':'Complete-call latency','units':'ms','description':'No qualified timing available'}] + context={'scope':scope,'metrics':metrics,'values':[1e-17,2e-15,0.01,0.02],'latency_claim':False} + doc={'format':'retained_measurements_v1','schema_version':1,'title':'Finite comparison fixture', + 'scope':{k:scope[k] for k in ('id','label','population')}, + 'dimensions':[{'id':'cpm','label':'Cells per metre','type':'number','units':'cells/m','scope':'case'}, + {'id':'grid_x','label':'Domain X cells','type':'integer','units':'cells','scope':'case'}, + {'id':'device','label':'Device','type':'string','units':None,'scope':'subject'}, + {'id':'passes','label':'Actual work','type':'integer','units':'passes','scope':'observation'}], + 'subjects':[{'id':i,'label':i,'dimensions':{'device':'GPU' if i=='a' else 'CPU'},'reference':ref('')} for i in ('a','b')], + 'cases':[{'id':'case1','dimensions':{'cpm':16,'grid_x':8},'reference':ref('')}, + {'id':'case2','dimensions':{'cpm':8,'grid_x':16},'reference':ref('')}], + 'metrics':[{**{k:m[k] for k in ('id','label','units')},'definition':ref('/metrics/'+str(i),m)} for i,m in enumerate(metrics)], + 'observations':[],'sources':[source('context',context),source('failure',b'Original failed timing attempt','opaque')]} + doc['scope']['definition']=ref('/scope',scope) + index=0 + for subject in ('a','b'): + for case in ('case1','case2'): + doc['observations'].append({'id':subject+'-'+case+'-error','subject_id':subject,'case_id':case,'metric_id':'error', + 'state':'measured','value':context['values'][index],'reason':None,'dimensions':{'passes':1 if subject=='a' else 2},'reference':ref('/values/'+str(index))}) + failed=subject=='b' and case=='case2' + doc['observations'].append({'id':subject+'-'+case+'-latency','subject_id':subject,'case_id':case,'metric_id':'latency', + 'state':'failed' if failed else 'unavailable','value':None, + 'reason':'Retained failure' if failed else 'No qualified timing', + 'dimensions':{'passes':1 if subject=='a' else 2}, + 'reference':{'source_id':'failure','pointer':None} if failed else ref('/latency_claim')}) + index+=1 + return doc + + +class RetainedMeasurementTests(unittest.TestCase): + def test_exact_tiny_values_density_and_grid_filters_and_detached_results(self): + doc=fixture();table=MeasurementTable(encode(doc));query=table.query('error') + self.assertEqual(query['total'],4);self.assertEqual(query['sampling'],'none') + self.assertEqual([r['maximum']['value'] for r in query['summary']],[2e-15,0.02]) + row=table.inspect('a-case1-error') + self.assertEqual(struct.pack('!d',row['observation']['value']),struct.pack('!d',1e-17)) + self.assertEqual(table.query('error',filters={'cpm':16})['rows'][0]['dimensions']['grid_x'],8) + self.assertEqual(table.query('error',filters={'grid_x':16})['rows'][0]['dimensions']['cpm'],8) + self.assertEqual(table.query('error',filters={'device':'CPU'})['total'],2) + self.assertEqual(table.query('error',filters={'passes':1})['summary'][1]['count'],0) + query['rows'][0]['value']=999;row['subject']['label']='wrong' + self.assertEqual(table.inspect('a-case1-error')['observation']['value'],1e-17) + self.assertEqual(table.subjects()['items'][0]['label'],'a') + self.assertEqual(table.subjects(offset=1,limit=1)['items'][0]['id'],'b') + self.assertEqual(table.subjects(offset=2)['items'],[]) + self.assertEqual(table.metadata()['subject_count'],2) + + def test_missing_and_failed_timings_are_explicit_not_zero_or_dropped(self): + table=MeasurementTable(encode(fixture()));query=table.query('latency',offset=1,limit=2) + self.assertEqual(query['total'],4);self.assertEqual(len(query['rows']),2) + self.assertTrue(all(v['maximum'] is None and v['count']==2 for v in query['summary'])) + self.assertEqual(query['summary'][1]['states'],{'measured':0,'failed':1,'unsupported':0,'unavailable':1}) + self.assertEqual(table.inspect('b-case2-latency')['source']['kind'],'opaque') + + def test_missing_duplicate_and_unresolved_matrix_members_are_refused(self): + for mutation in ('missing','duplicate','unknown','malformed'): + doc=fixture() + if mutation=='missing':doc['observations'].pop() + if mutation=='duplicate':doc['observations'][-1]=copy.deepcopy(doc['observations'][0]);doc['observations'][-1]['id']='new-id' + if mutation=='unknown':doc['observations'][0]['case_id']='outside' + if mutation=='malformed':doc['observations'][0]['case_id']=[] + with self.subTest(mutation=mutation),self.assertRaises(WorkbenchError):MeasurementTable(encode(doc)) + + def test_measured_value_source_hash_and_definition_changes_are_refused(self): + for mutation in ('value','hash','definition','units','pointer','source-type','boolean','huge-integer'): + doc=fixture() + if mutation=='value':doc['observations'][0]['value']=0.5 + if mutation=='hash':doc['sources'][0]['sha256']='f'*64 + if mutation=='definition':doc['metrics'][0]['definition']['sha256']='f'*64 + if mutation=='units':doc['metrics'][0]['units']='s' + if mutation=='pointer':doc['observations'][0]['reference']['pointer']='/values/099' + if mutation=='source-type':doc['observations'][0]['reference']['source_id']=[] + if mutation=='boolean':doc['observations'][0]['value']=True + if mutation=='huge-integer':doc['observations'][0]['value']=10**400 + with self.subTest(mutation=mutation),self.assertRaises(WorkbenchError):MeasurementTable(encode(doc)) + + def test_null_status_reason_and_dimension_contracts_are_not_inferred(self): + for mutation in ('null-measured','non-null-missing','empty-reason','boolean-dimension','extra-dimension'): + doc=fixture() + if mutation=='null-measured':doc['observations'][0]['value']=None + if mutation=='non-null-missing':doc['observations'][1]['value']=0 + if mutation=='empty-reason':doc['observations'][1]['reason']='' + if mutation=='boolean-dimension':doc['cases'][0]['dimensions']['grid_x']=True + if mutation=='extra-dimension':doc['cases'][0]['dimensions']['unexpected']=1 + with self.subTest(mutation=mutation),self.assertRaises(WorkbenchError):MeasurementTable(encode(doc)) + + def test_nonfinite_underflow_duplicate_keys_and_unicode_are_refused(self): + for raw in (b'{"a":1,"a":2}',b'{"a":NaN}',b'{"a":1e999}',b'{"a":1e-999}',b'{"a":"\\ud800"}'): + with self.subTest(raw=raw),self.assertRaises(WorkbenchError):decode(raw) + self.assertEqual(struct.pack('!d',decode(b'{"a":5e-324}')['a']),struct.pack('!d',5e-324)) + + def test_query_rejects_unknown_selectors_and_never_executes_expressions(self): + table=MeasurementTable(encode(fixture())) + for args in ({'metric_id':[]},{'metric_id':'outside'},{'metric_id':'error','filters':{'grid_x':True}}, + {'metric_id':'error','filters':{'expression':'a > 0'}},{'metric_id':'error','subject_ids':['a','a']}, + {'metric_id':'error','offset':True},{'metric_id':'error','limit':101}): + with self.subTest(args=args),self.assertRaises(WorkbenchError):table.query(**args) + with self.assertRaises(WorkbenchError):table.inspect([]) + + def test_source_scalar_sign_and_pointer_escape_are_exact(self): + doc=fixture();original=json.loads(base64.b64decode(doc['sources'][0]['data_base64'])) + original['a/b~c']=-0.0;doc['sources'][0]=source('context',original) + doc['observations'][0]['value']=-0.0;doc['observations'][0]['reference']=ref('/a~1b~0c') + table=MeasurementTable(encode(doc)) + self.assertEqual(struct.pack('!d',table.inspect('a-case1-error')['observation']['value']),struct.pack('!d',-0.0)) + doc['observations'][0]['value']=0.0 + with self.assertRaisesRegex(WorkbenchError,'scalar'):MeasurementTable(encode(doc)) + + def test_source_and_complete_matrix_limits_refuse_before_expansion(self): + doc=fixture();doc['sources'][0]['bytes']=8*1024*1024+1 + with self.assertRaises(WorkbenchError) as error:MeasurementTable(encode(doc)) + self.assertEqual(error.exception.status,413) + doc=fixture();base=doc['subjects'][0] + doc['subjects']=[dict(copy.deepcopy(base),id='s'+str(i)) for i in range(128)] + base=doc['cases'][0];doc['cases']=[dict(copy.deepcopy(base),id='c'+str(i)) for i in range(1000)] + with self.assertRaises(WorkbenchError) as error:MeasurementTable(encode(doc)) + self.assertEqual(error.exception.status,413) + + +if __name__=='__main__':unittest.main() From 77e05fe7dfe6d6da10c7baa189d524a3e3ea0697 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 19:29:09 -0700 Subject: [PATCH 11/15] Expose bounded exact measurement source details Return the original case reference document and source scalar for observation inspection, with explicit opaque/oversized states. Keep raw source bytes and current authority outside the pure query interface. --- retained_measurements.py | 10 ++++++++++ tests/test_retained_measurements.py | 3 +++ 2 files changed, 13 insertions(+) diff --git a/retained_measurements.py b/retained_measurements.py index 97c5032e..11188802 100644 --- a/retained_measurements.py +++ b/retained_measurements.py @@ -337,4 +337,14 @@ def inspect(self, observation_id): 'case':self._cases[observation['case_id']],'subject':self._subjects[observation['subject_id']], 'metric':self._metrics[observation['metric_id']],'scope':self._doc['scope'], 'scope_definition':self._definition(self._doc['scope'],('id','label','population')), + 'case_evidence':self._inspection_reference(self._cases[observation['case_id']]['reference']), + 'scalar_evidence':self._inspection_reference(reference), 'source':{k:v for k,v in source.items() if k!='data_base64'}}) + + def _inspection_reference(self,reference): + value=self._reference(reference) + state='opaque' if reference['pointer'] is None else 'included' + if state=='included' and len(encode(value))>16384: + state='exceeds_inspection_bound';value=None + return {'reference':copy.deepcopy(reference),'source_sha256':self._sources[reference['source_id']]['sha256'], + 'state':state,'value':value} diff --git a/tests/test_retained_measurements.py b/tests/test_retained_measurements.py index 73da3da7..a56a19d6 100644 --- a/tests/test_retained_measurements.py +++ b/tests/test_retained_measurements.py @@ -65,6 +65,8 @@ def test_exact_tiny_values_density_and_grid_filters_and_detached_results(self): self.assertEqual([r['maximum']['value'] for r in query['summary']],[2e-15,0.02]) row=table.inspect('a-case1-error') self.assertEqual(struct.pack('!d',row['observation']['value']),struct.pack('!d',1e-17)) + self.assertEqual(row['scalar_evidence']['value'],1e-17) + self.assertEqual(row['case_evidence']['state'],'included') self.assertEqual(table.query('error',filters={'cpm':16})['rows'][0]['dimensions']['grid_x'],8) self.assertEqual(table.query('error',filters={'grid_x':16})['rows'][0]['dimensions']['cpm'],8) self.assertEqual(table.query('error',filters={'device':'CPU'})['total'],2) @@ -82,6 +84,7 @@ def test_missing_and_failed_timings_are_explicit_not_zero_or_dropped(self): self.assertTrue(all(v['maximum'] is None and v['count']==2 for v in query['summary'])) self.assertEqual(query['summary'][1]['states'],{'measured':0,'failed':1,'unsupported':0,'unavailable':1}) self.assertEqual(table.inspect('b-case2-latency')['source']['kind'],'opaque') + self.assertEqual(table.inspect('b-case2-latency')['scalar_evidence']['state'],'opaque') def test_missing_duplicate_and_unresolved_matrix_members_are_refused(self): for mutation in ('missing','duplicate','unknown','malformed'): From acb5feb77037a9e4b83a90041a0b453d7754eb32 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 19:49:19 -0700 Subject: [PATCH 12/15] Retain exact scalar comparisons in native projects --- project_fork_composition.py | 8 + project_measurements.py | 200 +++++++++++++++++++++++++ project_projection.py | 8 +- project_sources.py | 43 ++++-- project_store.py | 1 + tests/test_project_measurements.py | 226 +++++++++++++++++++++++++++++ 6 files changed, 476 insertions(+), 10 deletions(-) create mode 100644 project_measurements.py create mode 100644 tests/test_project_measurements.py diff --git a/project_fork_composition.py b/project_fork_composition.py index 6c72643f..7551d665 100644 --- a/project_fork_composition.py +++ b/project_fork_composition.py @@ -98,6 +98,14 @@ def mark(sha, record, role, name, size=None, *, policy='unique'): for edge in record['document']['payloads']: mark(edge['blob']['sha256'],record,edge['role'],edge['name'],edge['blob']['bytes']) + # Scalar sources are immutable original evidence, including when the + # same bytes are also declared by a regenerable runtime dependency. + from project_measurements import decode_records as measurement_records, FAMILY as MEASUREMENT_FAMILY + for locator, doc in measurement_records(records).items(): + blob = doc['blob'] + mark(blob['sha256'], dict(family=MEASUREMENT_FAMILY, id=locator), + 'measurement_packet', 'measurements.json', blob['bytes']) + # Existing pure getter covers the four producer families, entry history # payload maps, and variation prompt-batch request/response observations. from project_producers import FAMILIES as PRODUCER_FAMILIES, _referenced_artifacts diff --git a/project_measurements.py b/project_measurements.py new file mode 100644 index 00000000..b97bca56 --- /dev/null +++ b/project_measurements.py @@ -0,0 +1,200 @@ +"""Native custody and current access for immutable retained scalar packets. + +The scalar owner interprets the packet. ProjectArtifacts owns bytes and +ProjectSession owns publication/receipts. Retained source paths are facts only; +reads after Open need no current source registration or producer execution. +""" +import copy +from datetime import datetime + +from project_artifacts import validate_declaration +from project_material import decode_records as material_records +from projects import ID, NAME, SHA, relative_path +from retained_measurements import MeasurementTable, MAX_BYTES, MAX_RESPONSE_BYTES, encode, text +from workbench import WorkbenchError, timestamp + + +FAMILY = 'measurement_source' +FORMAT = 'project_measurement_source_v1' +FIELDS = {'format', 'origin_catalog_id', 'id', 'title', 'blob', 'source', 'actor_id', 'imported_at'} + + +def _closed(value, fields): + if type(value) is not dict or set(value) != set(fields): + raise WorkbenchError('Invalid native measurement source fields.') + + +def _identity(value, pattern): + if type(value) is not str or not pattern.fullmatch(value): + raise WorkbenchError('Invalid native measurement source identity.') + return value + + +def declaration(blob): + return validate_declaration(dict(format='project_artifact_v1', **blob)) + + +def _document(value): + _closed(value, FIELDS) + if value['format'] != FORMAT: + raise WorkbenchError('Unsupported native measurement source format.', 'unsupported', 409) + for key in ('origin_catalog_id', 'actor_id'): + _identity(value[key], ID) + _identity(value['id'], SHA) + text(value['title']) + _closed(value['blob'], ('sha256', 'bytes')) + blob = declaration(value['blob']) + if not 0 < blob['bytes'] <= MAX_BYTES or blob['sha256'] != value['id']: + raise WorkbenchError('Measurement packet identity or byte bound differs.') + source = value['source'] + _closed(source, ('origin_catalog_id', 'project_id', 'manifest_sha256', 'reference_id', 'path', 'measured_sha256')) + for key in ('origin_catalog_id', 'project_id'): + _identity(source[key], ID) + for key in ('manifest_sha256', 'measured_sha256'): + _identity(source[key], SHA) + _identity(source['reference_id'], NAME) + relative_path(source['path']) + if source['measured_sha256'] != blob['sha256']: + raise WorkbenchError('Measurement source capture hash differs from its packet.') + try: + if type(value['imported_at']) is not str or len(value['imported_at']) > 80: + raise ValueError() + if datetime.fromisoformat(value['imported_at']).utcoffset() is None: + raise ValueError() + except ValueError: + raise WorkbenchError('Measurement import requires an attributed timestamp.') from None + return value + + +def decode_records(records, *, artifacts=None): + """Validate candidate closure, including exact available packet bytes. + + Missing payloads remain discoverable as missing. They never yield numerical + results. Present payloads must pass custody and scalar-source validation. + """ + declarations = {item['id']: item['document'] for item in records if item['family'] == 'project_artifact'} + snapshots = material_records(records) + result = {} + for item in records: + if item['family'] != FAMILY: + continue + doc = _document(item['document']) + identity = doc['origin_catalog_id'] + '.' + doc['id'] + if item['id'] != identity or identity in result: + raise WorkbenchError('Duplicate or inconsistent native measurement identity.') + if declarations.get(doc['blob']['sha256']) != declaration(doc['blob']): + raise WorkbenchError('Measurement artifact declaration is missing or inconsistent.') + source = doc['source'] + snapshot = snapshots.get(('project_snapshot', source['origin_catalog_id'] + '.' + source['manifest_sha256'])) + if snapshot is None or snapshot['value']['id'] != source['project_id']: + raise WorkbenchError('Measurement source snapshot is missing or inconsistent.') + reference = next((r for r in snapshot['value']['manifest']['data'] if r['id'] == source['reference_id']), None) + if (reference is None or reference['path'] != source['path'] + or reference.get('sha256', source['measured_sha256']) != source['measured_sha256']): + raise WorkbenchError('Measurement capture differs from its retained source reference.') + if artifacts is not None and artifacts.inspect(declaration(doc['blob']))['state'] != 'missing': + table = MeasurementTable(artifacts.read_blob(doc['blob'], maximum_bytes=MAX_BYTES)) + if table.metadata()['title'] != doc['title']: + raise WorkbenchError('Measurement title differs from the exact retained packet.') + result[identity] = copy.deepcopy(doc) + return result + + +def validate_operation_result(operation, result, records=None): + if operation != 'measurements.import': + return False + _document(result) + if records is not None: + identity = result['origin_catalog_id'] + '.' + result['id'] + if decode_records(records).get(identity) != result: + raise WorkbenchError('Measurement receipt has no exact retained source.') + return True + + +class ProjectMeasurements: + def __init__(self, session): + self.session, self.artifacts = session, session.artifacts + self.access, self.db = session.access, session.db + + def _documents(self): + return decode_records(self.session.snapshot['records']) + + def import_packet(self, raw, source, expected_checkpoint, key): + """Called only after SourceAdapter's explicit current rooted capture.""" + with self.access.lock, self.session.lock, self.db.lock: + actor = self.access.require('edit', 'measurements.import') + self.session.check() + table = MeasurementTable(raw) + blob = dict(sha256=table.sha256, bytes=len(raw)) + origin = self.session.snapshot['project_id'] + request = dict(blob=blob, source=copy.deepcopy(source)) + + def invoke(): + prior = self._documents().get(origin + '.' + table.sha256) + if prior is not None: + if prior['source'] != source: + raise WorkbenchError('This packet already has different retained capture facts.', 'conflict', 409) + return prior + self.artifacts.publish_bytes(raw, declaration(blob), maximum_bytes=MAX_BYTES) + return dict(format=FORMAT, origin_catalog_id=origin, id=table.sha256, + title=table.metadata()['title'], blob=blob, source=copy.deepcopy(source), + actor_id=actor.actor_id, imported_at=timestamp()) + + def capture(result, before): + return dict(records=[dict(family='project_artifact', id=table.sha256, document=declaration(blob)), + dict(family=FAMILY, id=origin + '.' + table.sha256, document=result)], result=result) + + return self.session.mutate('measurements.import', table.sha256, request, expected_checkpoint, key, invoke, capture) + + def sources(self, *, offset=0, limit=40): + if type(offset) is not int or not 0 <= offset <= 2**53-1 or type(limit) is not int or not 1 <= limit <= 40: + raise WorkbenchError('Invalid measurement source page.') + with self.access.lock, self.session.lock, self.db.lock: + self.access.require('inspect', 'measurements.sources') + self.session.check() + docs = sorted(self._documents().values(), key=lambda d: (d['origin_catalog_id'], d['id'])) + items = [dict(doc, availability=self.artifacts.inspect(declaration(doc['blob']))) + for doc in docs[offset:offset+limit]] + self.session.check() + self.access.require('inspect', 'measurements.sources') + return self._bounded(dict(items=items, total=len(docs), offset=offset, limit=limit)) + + @staticmethod + def _bounded(value): + if len(encode(value)) > MAX_RESPONSE_BYTES: + raise WorkbenchError('Measurement response exceeds its bound; select a smaller page.', 'too_large', 413) + return value + + def _read(self, origin_catalog_id, packet_sha256, action, select): + _identity(origin_catalog_id, ID) + _identity(packet_sha256, SHA) + target = origin_catalog_id + '.' + packet_sha256 + with self.access.lock, self.session.lock, self.db.lock: + self.access.require('inspect', action, target) + self.session.check() + doc = self._documents().get(target) + if doc is None: + raise WorkbenchError('Measurement source is unavailable.', 'not_found', 404) + raw = self.artifacts.read_blob(doc['blob'], maximum_bytes=MAX_BYTES) + table = MeasurementTable(raw) + if table.metadata()['title'] != doc['title']: + raise WorkbenchError('Measurement title differs from the exact retained packet.') + value = self._bounded(dict(source=doc, value=select(table))) + self.session.check() + self.access.require('inspect', action, target) + return value + + def metadata(self, origin_catalog_id, packet_sha256): + return self._read(origin_catalog_id, packet_sha256, 'measurements.metadata', lambda table: table.metadata()) + + def subjects(self, origin_catalog_id, packet_sha256, *, offset=0, limit=40): + return self._read(origin_catalog_id, packet_sha256, 'measurements.subjects', + lambda table: table.subjects(offset=offset, limit=limit)) + + def query(self, origin_catalog_id, packet_sha256, metric_id, **selection): + return self._read(origin_catalog_id, packet_sha256, 'measurements.query', + lambda table: table.query(metric_id, **selection)) + + def inspect(self, origin_catalog_id, packet_sha256, observation_id): + return self._read(origin_catalog_id, packet_sha256, 'measurements.inspect', + lambda table: table.inspect(observation_id)) diff --git a/project_projection.py b/project_projection.py index b3f05b29..e8a81a40 100644 --- a/project_projection.py +++ b/project_projection.py @@ -31,6 +31,9 @@ from project_native_execution import FAMILY as NATIVE_FAMILY, decode_records as decode_native_records from project_native_preparation import ProjectNativePreparation, validate_operation_result as validate_preparation_result from project_sources import SourceAdapter, validate_operation_result as validate_source_result +from project_measurements import ProjectMeasurements, FAMILY as MEASUREMENT_FAMILY +from project_measurements import decode_records as decode_measurement_records +from project_measurements import validate_operation_result as validate_measurement_result from workbench import WorkbenchError from workbench import timestamp @@ -76,6 +79,7 @@ def __init__(self, store, db, access, roots, *, custody=None, restore=True, runt self.producers = ProjectProducers(self.session) self.data = ProjectData(self.session, generation_entries=self.producers.generation_jobs) self.materials = ProjectMaterial(self.session) + self.measurements = ProjectMeasurements(self.session) self.history = ProjectHistory(self.session, materials=self.materials, authorize_disclosure=self.authorize_disclosure) if restore: @@ -125,7 +129,7 @@ def execute(self, *args, **kwargs): def validate_records(self, records, *, project_id=None): project_id = self.project_id if project_id is None else project_id - claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_extraction', NATIVE_FAMILY} | set(DATA_FAMILIES) | set( + claimed = {'workbench_text', 'project_artifact', 'project_operation', 'project_file', 'project_fork', 'project_extraction', NATIVE_FAMILY, MEASUREMENT_FAMILY} | set(DATA_FAMILIES) | set( MATERIAL_FAMILIES) | set(HISTORY_FAMILIES) | set(PRODUCER_FAMILIES) identities = set() declarations = {} @@ -159,12 +163,14 @@ def validate_records(self, records, *, project_id=None): artifacts=self.artifacts, check=lambda: None) decode_producer_records(records, artifacts=self.artifacts) decode_history_records(records, materials=ProjectMaterial(candidate), artifacts=self.artifacts) + decode_measurement_records(records, artifacts=self.artifacts) for receipt in operation_documents(records): operation, result = receipt['operation'], receipt['result'] if (validate_data_result(operation, result, records) or validate_material_result(operation, result, records) or validate_history_result(operation, result, records) or validate_source_result(operation, result, records) + or validate_measurement_result(operation, result, records) or validate_execution_result(operation, result, records) or validate_producer_result(operation, receipt['target'], result, records) or validate_file_result(operation, result, records) diff --git a/project_sources.py b/project_sources.py index 8aef085f..08939eba 100644 --- a/project_sources.py +++ b/project_sources.py @@ -184,15 +184,40 @@ def import_scientific(self,body,expected_checkpoint,key): with self.access.lock,self.session.lock,self.db.lock: self.access.require('edit','source.import_scientific',snapshot['project_id']) self.session.check() - historical=self.materials.snapshot_ref(snapshot['project_id'],snapshot['manifest_sha256'], - catalog_id=snapshot['origin_catalog_id']) - current,_raw=self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) - if current['manifest']!=historical['manifest']: - raise WorkbenchError('Current source differs from the selected retained snapshot.','stale',409) - raw,source=self.sources.read_snapshot_data_reference(snapshot['project_id'],body['reference_id'], - body['expected_sha256'],DATA_LIMIT) - if source['manifest_sha256']!=snapshot['manifest_sha256']: - raise WorkbenchError('Current source snapshot changed during capture.','stale',409) + raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],DATA_LIMIT) # ProjectData owns the one canonical mutation and native parser; # current rooted reads above never become an imported capability. return self.data.import_scientific(raw,copy.deepcopy(body['document']),source,expected_checkpoint,key) + + def _capture_reference(self,snapshot,reference_id,expected_sha256,maximum_bytes): + """One rooted capture policy shared by the two fixed native consumers. + + Caller holds current access/session/database locks and authorizes the + action before any source read. Returned paths remain historical facts. + """ + historical=self.materials.snapshot_ref(snapshot['project_id'],snapshot['manifest_sha256'], + catalog_id=snapshot['origin_catalog_id']) + current,_raw=self.sources.read_snapshot_manifest(snapshot['project_id'],snapshot['manifest_sha256']) + if current['manifest']!=historical['manifest']: + raise WorkbenchError('Current source differs from the selected retained snapshot.','stale',409) + raw,source=self.sources.read_snapshot_data_reference(snapshot['project_id'],reference_id, + expected_sha256,maximum_bytes) + if source['manifest_sha256']!=snapshot['manifest_sha256']: + raise WorkbenchError('Current source snapshot changed during capture.','stale',409) + self.session.check() + return raw,source + + def import_measurements(self,body,expected_checkpoint,key): + self._mutation_identity(expected_checkpoint,key) + _closed(body,('snapshot','reference_id','expected_sha256')) + snapshot=self._snapshot_reference(body['snapshot']) + _identity(body['reference_id'],NAME,'data reference') + _identity(body['expected_sha256'],SHA,'data SHA256') + from retained_measurements import MAX_BYTES + with self.access.lock,self.session.lock,self.db.lock: + self.access.require('edit','source.import_measurements',snapshot['project_id']) + self.session.check() + raw,source=self._capture_reference(snapshot,body['reference_id'],body['expected_sha256'],MAX_BYTES) + source=dict(source,origin_catalog_id=snapshot['origin_catalog_id']) + self.access.require('edit','source.import_measurements',snapshot['project_id']) + return self.projection.measurements.import_packet(raw,source,expected_checkpoint,key) diff --git a/project_store.py b/project_store.py index 11e3c09d..0531b8ea 100644 --- a/project_store.py +++ b/project_store.py @@ -43,6 +43,7 @@ 'model_material': 'retained_model_material_v1', 'recipe_material': 'retained_recipe_material_v1', 'project_snapshot': 'retained_project_snapshot_v1', + 'measurement_source': 'project_measurement_source_v1', 'runtime_material': 'retained_runtime_material_v1', 'study_history': 'project_study_history_v1', 'trial_history': 'project_trial_history_v1', diff --git a/tests/test_project_measurements.py b/tests/test_project_measurements.py new file mode 100644 index 00000000..22af6480 --- /dev/null +++ b/tests/test_project_measurements.py @@ -0,0 +1,226 @@ +"""Native scalar custody, publication, current access and independent Open. + +Only fixture JSON is imported. No neural or simulation runtime is involved. +""" +import copy +from functools import partial +import hashlib +import json +import os +from pathlib import Path +import shutil +import tempfile +import unittest +from unittest.mock import patch +import uuid + +from access import Access, AccessError +from metadata import Metadata, connection_info +from project_fork import active_records, publish_fork +from project_fork_composition import build_candidate +from project_measurements import FAMILY, validate_operation_result +from project_projection import ProjectProjection +from project_store import ProjectStore, DIR_FLAGS +from retained_measurements import MeasurementTable, encode +from test_retained_measurements import fixture +from workbench import WorkbenchError, timestamp + + +@unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL') +class ProjectMeasurementTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix='tuldok-measurements-') + self.root = Path(self.tmp.name) + self.addCleanup(self.tmp.cleanup) + self.resources, self.schemas = [], [] + self.addCleanup(self.cleanup) + self.host = self.metadata('host') + self.access = Access(self.host, self.host.path) + self.owner_token = self.access.exchange(self.access._local_owner_token())['token'] + self.enterContext(self.access.bind(self.owner_token)) + self.source_root = self.root / 'source' + self.source_root.mkdir() + self.raw = encode(fixture()) + self.digest = hashlib.sha256(self.raw).hexdigest() + (self.source_root / 'measurements.json').write_bytes(self.raw) + self.manifest = dict(schema_version=1, id='a'*32, name='Scalar source fixture', + data=[dict(id='measurements', label='Exact scalar fixture', path='measurements.json', sha256=self.digest)], + recipes=[], evidence=[], requirements=[]) + (self.source_root / 'source.json').write_text(json.dumps(self.manifest)) + self.store = ProjectStore.create(self.root / 'project', 'Native scalar fixture', 'new-scalars', 'a'*64) + self.resources.append(self.store) + self.projection = self.project(self.store, 'projection', {'fixture': self.source_root}) + self.session = self.projection.session + self.measurements = self.projection.measurements + self.registration = self.projection.source_adapter.register( + dict(root='fixture', manifest_path='source.json'), self.checkpoint, 'register-source') + self.body = dict(snapshot={k:self.registration[k] for k in ('origin_catalog_id', 'project_id', 'manifest_sha256')}, + reference_id='measurements', expected_sha256=self.digest) + self.origin = self.session.snapshot['project_id'] + + @property + def checkpoint(self): + return self.session.snapshot['checkpoint'] + + def metadata(self, name, *, project_id=None): + kwargs = dict(dataset_id=project_id, projection_id=uuid.uuid4().hex) if project_id else {} + db = Metadata(self.root / name, **kwargs) + self.resources.append(db) + self.schemas.append(db.schema) + return db + + def project(self, store, name, roots): + db = self.metadata(name, project_id=store.snapshot()['project_id']) + projection = ProjectProjection(store, db, self.access, roots) + self.resources.append(projection) + return projection + + def cleanup(self): + import psycopg + from psycopg import sql + for resource in reversed(self.resources): + resource.close() + with psycopg.connect(connection_info(), autocommit=True) as db: + for schema in self.schemas: + db.execute(sql.SQL('DROP SCHEMA IF EXISTS {} CASCADE').format(sql.Identifier(schema))) + + def admit(self, *, body=None, key='import-measurements', checkpoint=None): + return self.projection.source_adapter.import_measurements( + body or self.body, self.checkpoint if checkpoint is None else checkpoint, key) + + def test_native_import_retains_one_exact_packet_and_no_fabricated_history(self): + with patch('subprocess.Popen', side_effect=AssertionError('No producer process')): + result = self.admit() + query = self.measurements.query(self.origin, self.digest, 'error')['value'] + records = self.store.snapshot()['records'] + self.assertEqual(len([r for r in records if r['family'] == FAMILY]), 1) + self.assertEqual({r['family'] for r in records}, + {'project_snapshot', 'project_artifact', 'project_operation', FAMILY}) + self.assertEqual(self.projection.artifacts.read_blob(result['blob']), self.raw) + self.assertTrue(validate_operation_result('measurements.import', result, records)) + self.assertEqual(result['actor_id'], self.access._context().actor_id) + self.assertEqual(query, MeasurementTable(self.raw).query('error')) + self.assertEqual([s['maximum']['value'] for s in query['summary']], [2e-15, 0.02]) + missing = self.measurements.query(self.origin, self.digest, 'latency')['value'] + self.assertTrue(all(s['maximum'] is None for s in missing['summary'])) + self.assertEqual(missing['summary'][1]['states']['failed'], 1) + self.assertEqual(self.measurements.sources()['items'][0]['availability']['state'], 'unverified') + row = self.measurements.inspect(self.origin, self.digest, 'a-case1-error')['value'] + self.assertEqual(row['scalar_evidence']['value'], 1e-17) + self.assertEqual(self.measurements.subjects(self.origin, self.digest, offset=1, limit=1)['value']['items'][0]['id'], 'b') + + def test_replay_is_exact_and_stale_or_changed_requests_do_not_publish(self): + original_cp = self.checkpoint + result = self.admit() + canonical = self.store.snapshot() + self.assertEqual(self.admit(checkpoint=original_cp), result) + self.assertEqual(self.store.snapshot(), canonical) + with self.assertRaises(WorkbenchError) as error: + self.admit(key='stale-import', checkpoint=original_cp) + self.assertEqual(error.exception.code, 'stale_project') + different = copy.deepcopy(self.body) + different['expected_sha256'] = 'f'*64 + with self.assertRaises(WorkbenchError): + self.admit(body=different) + self.assertEqual(self.store.snapshot(), canonical) + + def test_permission_precedes_source_or_artifact_reads(self): + self.admit() + inspector = self.access.issue(dict(label='Scalar inspector', scopes=['inspect'])) + with self.access.bind(inspector['token']): + self.assertEqual(self.measurements.metadata(self.origin, self.digest)['value']['case_count'], 2) + with patch.object(self.projection.sources, 'read_snapshot_manifest', side_effect=AssertionError('Denied before source I/O')): + with self.assertRaises(AccessError): + self.admit(key='not-an-editor') + with self.access.bind(inspector['token']): + with self.access.bind(self.owner_token): + self.access.revoke(inspector['identity']['credential_id']) + with patch.object(self.projection.artifacts, 'read_blob', side_effect=AssertionError('Denied before artifact I/O')): + with self.assertRaises(AccessError): + self.measurements.query(self.origin, self.digest, 'error') + + def test_current_revocation_during_selection_prevents_return(self): + self.admit() + inspector = self.access.issue(dict(label='Revoked during selection', scopes=['inspect'])) + original = MeasurementTable.query + def revoke(table, *args, **kwargs): + value = original(table, *args, **kwargs) + with self.access.bind(self.owner_token): + self.access.revoke(inspector['identity']['credential_id']) + return value + with self.access.bind(inspector['token']), patch.object(MeasurementTable, 'query', revoke): + with self.assertRaises(AccessError): + self.measurements.query(self.origin, self.digest, 'error') + + def test_cold_independent_copy_reads_without_restoring_source_bindings(self): + result = self.admit() + expected = self.measurements.query(self.origin, self.digest, 'error', filters={'cpm':16}) + copied = self.root / 'relocated-project' + shutil.copytree(self.root / 'project', copied) + for path in (self.root / 'project').rglob('*'): + if path.is_file(): + self.assertNotEqual(path.stat().st_ino, (copied / path.relative_to(self.root / 'project')).stat().st_ino) + store = ProjectStore.open(copied) + self.resources.append(store) + cold = self.project(store, 'cold-projection', {}) + shutil.rmtree(self.source_root) + self.assertEqual(cold.db.execute('SELECT count(*) FROM workspace_projects').fetchone()[0], 0) + self.assertEqual(cold.measurements.query(self.origin, self.digest, 'error', filters={'cpm':16}), expected) + self.assertEqual(cold.artifacts.read_blob(result['blob']), self.raw) + self.assertEqual(cold.measurements.metadata(self.origin, self.digest)['value']['observation_count'], 8) + + def test_fork_preserves_origin_scoped_packet_without_fabricating_new_capture(self): + result = self.admit() + before = self.store.snapshot() + root_fd = os.open(self.root, DIR_FLAGS) + try: + publish_fork(self.store, self.projection.artifacts, root_fd, 'fork', root_path=self.root, + name='Independent scalar fork', expected_checkpoint=before['checkpoint'], request_key='fork-scalars', + quiescent=True, build_candidate=partial(build_candidate, created_at=timestamp(), + id_factory=lambda _family, _source_id, _revision: uuid.uuid4().hex), + validate_candidate=lambda records, identity:self.projection.validate_records(records, project_id=identity)) + finally: + os.close(root_fd) + store = ProjectStore.open(self.root / 'fork') + self.resources.append(store) + cold = self.project(store, 'fork-projection', {}) + self.assertNotEqual(store.snapshot()['project_id'], self.origin) + records = active_records(store.snapshot()['records'], project_id=store.snapshot()['project_id']) + self.assertEqual(next(r['document'] for r in records if r['family'] == FAMILY), result) + self.assertEqual(cold.measurements.inspect(self.origin, self.digest, 'a-case1-error')['value']['observation']['value'], 1e-17) + + def test_candidate_cross_reference_receipt_and_title_forgery_are_refused(self): + result = self.admit() + canonical = self.store.snapshot() + for mutation in ('declaration', 'snapshot', 'source-path', 'title', 'receipt'): + records = copy.deepcopy(canonical['records']) + doc = next(r['document'] for r in records if r['family'] == FAMILY) + if mutation == 'declaration': + records = [r for r in records if not (r['family'] == 'project_artifact' and r['id'] == self.digest)] + if mutation == 'snapshot': + records = [r for r in records if r['family'] != 'project_snapshot'] + if mutation == 'source-path': + doc['source']['path'] = 'other.json' + if mutation == 'title': + doc['title'] = 'Different retained title' + if mutation == 'receipt': + receipt = next(r['document'] for r in records if r['family'] == 'project_operation' and r['document']['operation'] == 'measurements.import') + receipt['result']['title'] = 'Forged result' + with self.subTest(mutation=mutation), self.assertRaises(WorkbenchError): + self.projection.validate_records(records) + self.assertEqual(self.store.snapshot(), canonical) + result['title'] = 'Detached caller mutation' + self.assertNotEqual(self.measurements.sources()['items'][0]['title'], result['title']) + + def test_changed_source_packet_fails_before_canonical_import(self): + before = self.store.snapshot() + changed = fixture() + changed['observations'][0]['value'] = 0.5 + (self.source_root / 'measurements.json').write_bytes(encode(changed)) + with self.assertRaises(WorkbenchError): + self.admit() + self.assertEqual(self.store.snapshot(), before) + + +if __name__ == '__main__': + unittest.main() From ce4cb1b58c407d78cbaa235d7968f7a5102735ad Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 19:57:14 -0700 Subject: [PATCH 13/15] Expose bounded native measurement selections and source inspection --- agent_api.py | 112 ++++++++++++++++++++- project_dispatch.py | 18 +++- static/project-files.js | 38 ++++++- tests/test_project_files_m2_http.py | 61 +++++++++++ tests/test_project_measurements_clients.py | 88 ++++++++++++++++ tuldok_client.py | 18 ++++ 6 files changed, 331 insertions(+), 4 deletions(-) create mode 100644 tests/test_project_measurements_clients.py diff --git a/agent_api.py b/agent_api.py index d7b29b83..4bcd9ce6 100644 --- a/agent_api.py +++ b/agent_api.py @@ -885,6 +885,73 @@ def _project_file_operation(action, method, suffix, fields=None, *, required=Non 'snapshot': obj({'origin_catalog_id': ID, 'project_id': ID, 'manifest_sha256': SHA}), 'reference_id': REF, 'expected_sha256': SHA, 'document': obj(OPTIONAL_META, required=['groups'])}, mutation=True) +# Retained scalar packets use their own fixed native owner. Definitions and +# original JSON evidence stay producer-authored data, never executable specs. +_MEASUREMENT_ID = {'type':'string','pattern':'^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$'} +_MEASUREMENT_SCALAR = {'type':['string','number','boolean','null']} +_MEASUREMENT_DIMENSIONS = {'type':'object','maxProperties':32,'additionalProperties':_MEASUREMENT_SCALAR} +_MEASUREMENT_REF = obj({'source_id':_MEASUREMENT_ID,'pointer':{'oneOf':[text(512,True),_NULL]}}) +_MEASUREMENT_DEFINITION = obj(dict(_MEASUREMENT_REF['properties'],sha256=SHA)) +_MEASUREMENT_SCOPE = obj({'id':_MEASUREMENT_ID,'label':text(240), + 'population':{'enum':['training','consumed','independent','reserved','unknown']},'definition':_MEASUREMENT_DEFINITION}) +_MEASUREMENT_METRIC = obj({'id':_MEASUREMENT_ID,'label':text(240), + 'units':text(120),'definition':_MEASUREMENT_DEFINITION}) +_MEASUREMENT_SUBJECT = obj({'id':_MEASUREMENT_ID,'label':text(240), + 'dimensions':_MEASUREMENT_DIMENSIONS,'reference':_MEASUREMENT_REF}) +_MEASUREMENT_CASE = obj({'id':_MEASUREMENT_ID,'dimensions':_MEASUREMENT_DIMENSIONS,'reference':_MEASUREMENT_REF}) +_MEASUREMENT_STATE = {'enum':['measured','failed','unsupported','unavailable']} +_MEASUREMENT_OBSERVATION = obj({'id':_MEASUREMENT_ID,'subject_id':_MEASUREMENT_ID, + 'case_id':_MEASUREMENT_ID,'metric_id':_MEASUREMENT_ID,'state':_MEASUREMENT_STATE, + 'value':{'type':['number','null']},'reason':{'oneOf':[text(1000),_NULL]}, + 'dimensions':_MEASUREMENT_DIMENSIONS,'reference':_MEASUREMENT_REF}) +_MEASUREMENT_CAPTURE = obj({'origin_catalog_id':ID,'project_id':ID,'manifest_sha256':SHA, + 'reference_id':REF,'path':text(512),'measured_sha256':SHA}) +_MEASUREMENT_SOURCE = obj({'format':const('project_measurement_source_v1'),'origin_catalog_id':ID, + 'id':SHA,'title':text(240),'blob':obj({'sha256':SHA,'bytes':integer(1,32*1024*1024)}), + 'source':_MEASUREMENT_CAPTURE,'actor_id':ID,'imported_at':text(80)}) +_MEASUREMENT_AVAILABILITY = obj({'state':{'enum':['unverified','missing']}, + 'expected_sha256':SHA,'expected_bytes':SAFE_NAT,'observed_sha256':const(None), + 'observed_bytes':{'oneOf':[SAFE_NAT,_NULL]}}) +_MEASUREMENT_SOURCE_ITEM = obj(dict(_MEASUREMENT_SOURCE['properties'],availability=_MEASUREMENT_AVAILABILITY)) +_MEASUREMENT_HEADER = obj({'format':const('retained_measurements_v1'),'packet_sha256':SHA,'title':text(240), + 'scope':_MEASUREMENT_SCOPE,'scope_definition':{'type':'object'}, + 'dimensions':array(obj({'id':_MEASUREMENT_ID,'label':text(240),'type':{'enum':['string','number','integer','boolean']}, + 'units':{'oneOf':[text(120),_NULL]},'scope':{'enum':['subject','case','observation']}}),32), + 'metrics':array(_MEASUREMENT_METRIC,32),'subject_count':integer(1,100000),'case_count':integer(1,4096), + 'observation_count':integer(1,100000),'source_count':integer(1,256)}) +_MEASUREMENT_SUMMARY = obj({'subject_id':_MEASUREMENT_ID,'count':SAFE_NAT, + 'states':obj({state:SAFE_NAT for state in ('measured','failed','unsupported','unavailable')}), + 'maximum':{'oneOf':[obj({'id':_MEASUREMENT_ID,'case_id':_MEASUREMENT_ID,'value':{'type':'number'}}),_NULL]}}) +_MEASUREMENT_QUERY = obj({'format':const('retained_measurement_query_v1'),'packet_sha256':SHA, + 'scope':_MEASUREMENT_SCOPE,'metric':_MEASUREMENT_METRIC,'scope_definition':{'type':'object'},'definition':{'type':'object'}, + 'filters':_MEASUREMENT_DIMENSIONS,'subject_ids':array(_MEASUREMENT_ID,100000,1), + 'summary':array(_MEASUREMENT_SUMMARY,100000),'rows':array(_MEASUREMENT_OBSERVATION,100), + 'total':SAFE_NAT,'offset':SAFE_NAT,'limit':integer(1,100),'sampling':const('none')}) +_MEASUREMENT_EVIDENCE = obj({'reference':_MEASUREMENT_REF,'source_sha256':SHA, + 'state':{'enum':['included','opaque','exceeds_inspection_bound']},'value':{}}) +_MEASUREMENT_INSPECTION = obj({'packet_sha256':SHA,'observation':_MEASUREMENT_OBSERVATION, + 'case':_MEASUREMENT_CASE,'subject':_MEASUREMENT_SUBJECT,'metric':_MEASUREMENT_METRIC, + 'scope':_MEASUREMENT_SCOPE,'scope_definition':{'type':'object'},'case_evidence':_MEASUREMENT_EVIDENCE, + 'scalar_evidence':_MEASUREMENT_EVIDENCE,'source':obj({'id':_MEASUREMENT_ID,'label':text(240), + 'kind':{'enum':['json','opaque']},'bytes':integer(1,8*1024*1024),'sha256':SHA})}) +_MEASUREMENT_SELECTION = {'origin_catalog_id':ID,'packet_sha256':SHA} +_MEASUREMENT_BOUND = lambda value:obj({'source':_MEASUREMENT_SOURCE,'value':value}) +_project_file_operation('measurement_import','POST','/measurements/import', + {'snapshot':obj({'origin_catalog_id':ID,'project_id':ID,'manifest_sha256':SHA}), + 'reference_id':REF,'expected_sha256':SHA},mutation=True,output=_MEASUREMENT_SOURCE) +_project_file_operation('measurement_sources','GET','/measurements',_PROJECT_FILES_PAGE_INPUT,required=[], + output=obj({'items':array(_MEASUREMENT_SOURCE_ITEM,40),'offset':SAFE_NAT,'limit':integer(1,40),'total':SAFE_NAT})) +_project_file_operation('measurement_metadata','GET','/measurements/{packet_sha256}',_MEASUREMENT_SELECTION, + output=_MEASUREMENT_BOUND(_MEASUREMENT_HEADER)) +_project_file_operation('measurement_subjects','GET','/measurements/{packet_sha256}/subjects', + dict(_MEASUREMENT_SELECTION,**_PROJECT_FILES_PAGE_INPUT),required=list(_MEASUREMENT_SELECTION), + output=_MEASUREMENT_BOUND(obj({'items':array(_MEASUREMENT_SUBJECT,40),'offset':SAFE_NAT,'limit':integer(1,40),'total':SAFE_NAT}))) +_project_file_operation('measurement_query','POST','/measurements/{packet_sha256}/query', + dict(_MEASUREMENT_SELECTION,metric_id=_MEASUREMENT_ID,subject_ids=array(_MEASUREMENT_ID,100000,1), + filters=_MEASUREMENT_DIMENSIONS,offset=SAFE_NAT,limit=integer(1,100)), + required=[*list(_MEASUREMENT_SELECTION),'metric_id'],output=_MEASUREMENT_BOUND(_MEASUREMENT_QUERY)) +_project_file_operation('measurement_inspect','GET','/measurements/{packet_sha256}/observations/{observation_id}', + dict(_MEASUREMENT_SELECTION,observation_id=_MEASUREMENT_ID),output=_MEASUREMENT_BOUND(_MEASUREMENT_INSPECTION)) _project_file_operation('data_annotate', 'POST', '/data/{record_id}', { 'record_id': ID, 'revision': SAFE_POS, 'source_revision': SAFE_POS, 'task': {'enum': list(ANNOTATIONS)}, 'annotation': JSON_OBJECT, @@ -1689,8 +1756,6 @@ def contract(name, version=1): def capabilities(catalog_id, profile='catalog'): if type(catalog_id) is not str or not re.fullmatch('[a-f0-9]{32}', catalog_id): raise AgentAPIError('Invalid capability catalog identity.', 'registry_invalid', 503) - if len(_CONTRACTS) > MAX_OPERATIONS: - raise AgentAPIError('Operation registry exceeds its capability bound.', 'registry_invalid', 503) if profile not in ('catalog','project'): raise AgentAPIError('Unsupported capability profile.','registry_invalid',503) operations = [] @@ -1698,6 +1763,11 @@ def capabilities(catalog_id, profile='catalog'): if (profile=='catalog' and name.startswith('project_files.') or profile=='project' and not name.startswith(('project_files.','project_transfer.')) and name not in ('context.get','identity.get')): continue + # The protocol bound applies to this advertised profile, just as the + # receiver checks its one manifest. Disjoint catalog/project contracts + # are not all transmitted together. Keep the installed 256-op limit. + if len(operations) >= MAX_OPERATIONS: + raise AgentAPIError('Operation profile exceeds its capability bound.', 'registry_invalid', 503) if not 1 <= len(versions) <= MAX_CONTRACTS: raise AgentAPIError('Contract version count exceeds its bound.', 'registry_invalid', 503) operations.append({'id': name, 'contracts': [contract(name, version) for version in sorted(versions)], @@ -2762,6 +2832,8 @@ def call(self, name, arguments, *, observer=None, observation=None): or len(segment) != min(page['limit'], page['bytes'] - page['offset']) or page['next_offset'] != (end if end < page['bytes'] else None)): raise AgentAPIError('Retained Run document segment differs from its exact selection.', 'transport', 502) + if action.startswith('measurement_'): + _validate_measurement_selection(action,page,arguments,value['project_id']) target_key = {'data_get': 'record_id', 'model_get': 'model_id', 'graph_get': 'graph_id', 'recipe_get': 'recipe_id', 'trial': 'trial_id'}.get(op['operation_id'].split('.')[1]) if target_key and page.get('id') != arguments[target_key]: @@ -2949,6 +3021,42 @@ def call(self,name,arguments,**kwargs): return result +def _validate_measurement_selection(action, page, arguments, project_id): + """Correlate fixed scalar replies; scientific validity remains upstream.""" + def refuse(): + raise AgentAPIError('Retained measurements differ from the selected source or continuation.', 'transport', 502) + if action=='measurement_sources': + offset,limit=arguments.get('offset',0),arguments.get('limit',40) + keys=[(d['origin_catalog_id'],d['id']) for d in page['items']] + if (page['offset']!=offset or page['limit']!=limit or keys!=sorted(set(keys)) + or len(keys)!=min(limit,max(0,page['total']-offset))):refuse() + return + if action=='measurement_import': + source=page['source'] + if (page['origin_catalog_id']!=project_id or page['id']!=arguments['expected_sha256'] + or page['blob']['sha256']!=page['id'] or source['measured_sha256']!=page['id'] + or source['reference_id']!=arguments['reference_id'] + or any(source[k]!=v for k,v in arguments['snapshot'].items())):refuse() + return + source,result=page['source'],page['value'] + if (source['origin_catalog_id']!=arguments['origin_catalog_id'] or source['id']!=arguments['packet_sha256'] + or source['blob']['sha256']!=source['id'] or source['source']['measured_sha256']!=source['id']):refuse() + if action!='measurement_subjects' and result['packet_sha256']!=source['id']:refuse() + if action=='measurement_inspect' and result['observation']['id']!=arguments['observation_id']:refuse() + if action=='measurement_metadata' and result['title']!=source['title']:refuse() + if action in ('measurement_query','measurement_subjects'): + offset,limit=arguments.get('offset',0),arguments.get('limit',50 if action=='measurement_query' else 40) + rows=result['rows' if action=='measurement_query' else 'items'] + if (result['offset']!=offset or result['limit']!=limit + or len(rows)!=min(limit,max(0,result['total']-offset))):refuse() + if action=='measurement_query': + subjects=result['subject_ids'] + if (result['metric']['id']!=arguments['metric_id'] or result['filters']!=arguments.get('filters',{}) + or 'subject_ids' in arguments and subjects!=arguments['subject_ids'] + or len(subjects)!=len(set(subjects)) or [s['subject_id'] for s in result['summary']]!=subjects + or any(r['metric_id']!=arguments['metric_id'] or r['subject_id'] not in subjects for r in result['rows'])):refuse() + + class PublicationOwnerClient(AgentClient): """Explicit owner publication actions only; ordinary AgentClient stays agent-only.""" OPERATIONS = frozenset(('runs.publication_status', 'runs.recover_publication', 'runs.cancel_publication', diff --git a/project_dispatch.py b/project_dispatch.py index 49ce82c5..bdbdc250 100644 --- a/project_dispatch.py +++ b/project_dispatch.py @@ -36,9 +36,11 @@ 'execution_producer_decide', 'execution_producer_cancel', 'execution_producer_health', 'execution_study_prepare_package','execution_study_prepare_verifier','execution_study_abandon_preparation','execution_study_reconcile_preparation')) _RUNTIME_ACTIONS=frozenset(('runtime_sources','runtime_inspect','runtime_capture','runtime_status','runtime_abandon')) +_MEASUREMENT_ACTIONS=frozenset(('measurement_import','measurement_sources','measurement_metadata', + 'measurement_subjects','measurement_query','measurement_inspect')) _M4_ACTIONS=frozenset(('files_inspect','file_get','file_history','file_adopt','file_adopt_deletion','material_intents','material_intent','material_intent_capabilities','fork_info')) SUPPORTED_OPERATIONS = frozenset('project_files.' + action for action in - _DATA_ACTIONS | _MATERIAL_ACTIONS | _HISTORY_ACTIONS | _OTHER_ACTIONS | _EXECUTION_ACTIONS | _M4_ACTIONS | _RUNTIME_ACTIONS) + _DATA_ACTIONS | _MATERIAL_ACTIONS | _HISTORY_ACTIONS | _OTHER_ACTIONS | _EXECUTION_ACTIONS | _M4_ACTIONS | _RUNTIME_ACTIONS | _MEASUREMENT_ACTIONS) def _body(arguments, *omit): @@ -401,6 +403,7 @@ def dispatch(projection, name, arguments): raise WorkbenchError('Unsupported external-project operation.', 'unsupported', 409) action = name.split('.')[1] projection.session.access.require('inspect', name) + if action in _MEASUREMENT_ACTIONS:return _measurements(projection,action,arguments) if action in _RUNTIME_ACTIONS:return _runtime(projection,action,arguments) if action in _M4_ACTIONS:return _m4(projection,action,arguments) if action in _EXECUTION_ACTIONS: return _execution(projection, action, arguments) @@ -428,3 +431,16 @@ def dispatch(projection, name, arguments): projection.session.access.require('edit', 'project_files.source_register', owner_only=True) return projection.source_adapter.register(_body(arguments), arguments['expected_checkpoint'], arguments['request_key']) raise AssertionError('Closed project dispatch is incomplete.') + + +def _measurements(projection,action,a): + owner=projection.measurements + if action=='measurement_import': + return projection.source_adapter.import_measurements(_body(a),a['expected_checkpoint'],a['request_key']) + if action=='measurement_sources':return owner.sources(offset=a.get('offset',0),limit=a.get('limit',40)) + origin,packet=a['origin_catalog_id'],a['packet_sha256'] + if action=='measurement_metadata':return owner.metadata(origin,packet) + if action=='measurement_subjects':return owner.subjects(origin,packet,offset=a.get('offset',0),limit=a.get('limit',40)) + if action=='measurement_query':return owner.query(origin,packet,**_body(a,'origin_catalog_id','packet_sha256')) + if action=='measurement_inspect':return owner.inspect(origin,packet,a['observation_id']) + raise AssertionError('Closed measurement dispatch is incomplete.') diff --git a/static/project-files.js b/static/project-files.js index 8f168dc0..fc148da5 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -142,7 +142,13 @@ "project_files.study_create": "8d2aac7c073a6a7b3ebb41227b5b466cf03606625404f93a59b325038a4ad7a1", "project_files.study_save": "5bf3fb814588fd7d9c35836e9d964a64481b3cc4169caf76cad6ce776ff0fe15", "project_files.trial": "79afa4bed6d0907f3d4116cc04cb784e94e5a24ab89725ec24f5d7bce70e869f", - "project_files.trial_prepare": "e1f6973536fd4c356c4a92fc674d6766f7df0d8a113b8ee71bb05d0adc40b04f" + "project_files.trial_prepare": "e1f6973536fd4c356c4a92fc674d6766f7df0d8a113b8ee71bb05d0adc40b04f", + "project_files.measurement_import": "c8e500396138fc60a31fc916c441bb95454f410231a6508609a7ebba5cd31f18", + "project_files.measurement_sources": "7805e9c5a6b2e2ab566ca134c18697116c89e08a1991b5192b0b5c152a34041d", + "project_files.measurement_metadata": "5a92ed5259f8876a3c5b8fd1886a9e0f44f0aea8e53d7f63902ac765ddac0de7", + "project_files.measurement_subjects": "37bd3f717c7158e34b9c221bfa1dbce79490b9d55248803d13dab2ea17b35ea5", + "project_files.measurement_query": "65b1456a6870dd6d4b7499669777a2c66157142ccc27cac1864a995664220e1c", + "project_files.measurement_inspect": "fcbe9846ca49c3da4713b54a3cb4a8237189d8eea5799f275d33992ef268914f" } /* PROJECT_FILES_CONTRACTS_END */; // Generated fixed operation routes from agent_api; remote schemas are never executed. @@ -800,6 +806,36 @@ "method": "POST", "path": "/api/workspace/project-files/studies/{study_id}/trials", "mutation": true + }, + "project_files.measurement_import": { + "method": "POST", + "path": "/api/workspace/project-files/measurements/import", + "mutation": true + }, + "project_files.measurement_inspect": { + "method": "GET", + "path": "/api/workspace/project-files/measurements/{packet_sha256}/observations/{observation_id}", + "mutation": false + }, + "project_files.measurement_metadata": { + "method": "GET", + "path": "/api/workspace/project-files/measurements/{packet_sha256}", + "mutation": false + }, + "project_files.measurement_query": { + "method": "POST", + "path": "/api/workspace/project-files/measurements/{packet_sha256}/query", + "mutation": false + }, + "project_files.measurement_sources": { + "method": "GET", + "path": "/api/workspace/project-files/measurements", + "mutation": false + }, + "project_files.measurement_subjects": { + "method": "GET", + "path": "/api/workspace/project-files/measurements/{packet_sha256}/subjects", + "mutation": false } } /* PROJECT_FILES_ROUTES_END */; diff --git a/tests/test_project_files_m2_http.py b/tests/test_project_files_m2_http.py index 63db21a4..ec3e431e 100644 --- a/tests/test_project_files_m2_http.py +++ b/tests/test_project_files_m2_http.py @@ -37,6 +37,7 @@ 'static/project-files.js','static/project-files.html','tuldok_client.py','project_cli.py','mcp_stdio.py', 'static/scientific-inspector.js','static/scientific-inspector.css','static/scientific-fields.js', 'static/workbench.html','static/workbench.css') +SOURCE_FILES+=('project_measurements.py','retained_measurements.py','project_fork_composition.py') def subject_hashes(): @@ -385,4 +386,64 @@ async def exercise(): self.assertEqual(status,409,error);self.assertEqual(m1.canonical_bytes(self.external/'adapters'),committed) + def test_retained_measurement_import_query_permissions_and_independent_cold_copy(self): + from retained_measurements import MeasurementTable,encode + from test_retained_measurements import fixture + self.create() + raw=encode(fixture());digest=hashlib.sha256(raw).hexdigest() + (self.external/'measurements.json').write_bytes(raw) + manifest=dict(schema_version=1,id=uuid.uuid4().hex,name='Exact native scalar source', + data=[dict(id='measurements',label='Retained scalars',path='measurements.json',sha256=digest)], + recipes=[],evidence=[],requirements=[]) + (self.external/'measurement-source.json').write_text(json.dumps(manifest)) + registered=self.call('source_register',root='fixture',manifest_path='measurement-source.json') + snapshot={k:registered[k] for k in ('origin_catalog_id','project_id','manifest_sha256')} + import_args=self.arguments('measurement_import',snapshot=snapshot,reference_id='measurements',expected_sha256=digest) + status,admitted=self.transport('measurement_import',import_args) + self.assertEqual(status,200,admitted) + origin=admitted['project_id'];selection=dict(origin_catalog_id=origin,packet_sha256=digest) + canonical=m1.canonical_bytes(self.external/'alpha') + self.assertEqual(self.transport('measurement_import',import_args)[1],admitted) + self.assertEqual(m1.canonical_bytes(self.external/'alpha'),canonical) + metadata=self.call('measurement_metadata',**selection) + self.assertEqual(metadata['value'],MeasurementTable(raw).metadata()) + self.assertEqual(self.call('measurement_sources')['items'][0]['id'],digest) + self.assertEqual(self.call('measurement_subjects',**selection,offset=1,limit=1)['value']['items'][0]['id'],'b') + query_args=self.arguments('measurement_query',**selection,metric_id='error',filters={'cpm':16}) + expected=self.call('measurement_query',**selection,metric_id='error',filters={'cpm':16}) + self.assertEqual(expected['value'],MeasurementTable(raw).query('error',filters={'cpm':16})) + row=self.call('measurement_inspect',**selection,observation_id='a-case1-error') + self.assertEqual(row['value']['scalar_evidence']['value'],1e-17) + timing=self.call('measurement_query',**selection,metric_id='latency')['value'] + self.assertTrue(all(s['maximum'] is None for s in timing['summary'])) + status,error=self.transport('measurement_query',dict(query_args,checkpoint='0'*64)) + self.assertEqual(status,409,error) + status,error=self.transport('measurement_query',dict(query_args,filters={'expression':'never execute'})) + self.assertEqual(status,400,error) + with self.owner.bound(self.workspace): + inspector=self.workspace.access.issue(dict(label='Scalar HTTP inspector',scopes=['inspect'])) + status,value=self.transport('measurement_query',query_args,token=inspector['token']) + self.assertEqual(status,200,value) + denied=self.arguments('measurement_import',snapshot=snapshot,reference_id='measurements',expected_sha256=digest) + self.assertEqual(self.transport('measurement_import',denied,token=inspector['token'])[0],403) + with self.owner.bound(self.workspace): + self.workspace.access.revoke(inspector['identity']['credential_id']) + self.assertEqual(self.transport('measurement_query',query_args,token=inspector['token'])[0],401) + self.assertEqual(m1.canonical_bytes(self.external/'alpha'),canonical) + self.action('save');self.action('close') + coldroot=self.path/'scalar-cold';coldroot.mkdir() + shutil.copytree(self.external/'alpha',coldroot/'copied') + self.assertEqual(m1.canonical_bytes(coldroot/'copied'),canonical) + for name in canonical: + self.assertNotEqual((self.external/'alpha'/name).stat().st_ino,(coldroot/'copied'/name).stat().st_ino) + self.select_host(self.path/'scalar-cold-control',{'fixture':coldroot}) + self.action('open',root='fixture',directory='copied') + self.assertEqual(self.workspace._projection.db.execute('SELECT count(*) FROM workspace_projects').fetchone()[0],0) + self.assertEqual(self.call('measurement_query',**selection,metric_id='error',filters={'cpm':16}),expected) + self.assertEqual(self.call('measurement_inspect',**selection,observation_id='a-case1-error'),row) + self.assertEqual(self.transport('measurement_query',query_args)[0],409) + self.action('save');self.action('close') + self.assertEqual(m1.canonical_bytes(coldroot/'copied'),canonical) + + if __name__=='__main__':unittest.main() diff --git a/tests/test_project_measurements_clients.py b/tests/test_project_measurements_clients.py new file mode 100644 index 00000000..bc415313 --- /dev/null +++ b/tests/test_project_measurements_clients.py @@ -0,0 +1,88 @@ +"""Closed scalar selections and exact thin-client response correlation.""" +import copy +import unittest +from unittest.mock import patch + +import agent_api as api +from retained_measurements import MeasurementTable, encode +from test_project_files_clients import BINDING, PROJECT, CHECKPOINT, OTHER, bound, client +from test_retained_measurements import fixture +from tuldok_client import ProjectFilesClient + + +class MeasurementContracts(unittest.TestCase): + def setUp(self): + self.table=MeasurementTable(encode(fixture())) + self.source=dict(format='project_measurement_source_v1',origin_catalog_id=PROJECT, + id=self.table.sha256,title=self.table.metadata()['title'],blob=dict(sha256=self.table.sha256,bytes=len(encode(fixture()))), + source=dict(origin_catalog_id=PROJECT,project_id='a'*32,manifest_sha256=OTHER, + reference_id='measurements',path='measurements.json',measured_sha256=self.table.sha256), + actor_id='d'*32,imported_at='2026-10-23T00:00:00+00:00') + self.args=dict(binding=BINDING,checkpoint=CHECKPOINT,origin_catalog_id=PROJECT,packet_sha256=self.table.sha256) + + def sdk(self,value): + return ProjectFilesClient(client({'status':200,'result':dict(binding=BINDING,project_id=PROJECT, + checkpoint=CHECKPOINT,value=value)})) + + def test_capability_bound_applies_to_the_actual_advertised_profile(self): + sizes={profile:len(api.capabilities(PROJECT,profile=profile)['operations']) for profile in ('catalog','project')} + maximum=max(sizes.values()) + with patch.object(api,'MAX_OPERATIONS',maximum): + for profile in sizes: + manifest=api.capabilities(PROJECT,profile=profile) + self.assertLess(len(api.encoded(manifest)),api.CAPABILITIES_LIMIT) + self.assertEqual(len(manifest['operations']),sizes[profile]) + with patch.object(api,'MAX_OPERATIONS',maximum-1),self.assertRaises(api.AgentAPIError) as caught: + api.capabilities(PROJECT,profile=max(sizes,key=sizes.get)) + self.assertEqual(caught.exception.code,'registry_invalid') + + def test_typed_read_routes_pin_packet_selection_and_never_accept_execution(self): + arguments=dict(self.args,metric_id='error',filters={'cpm':16},subject_ids=['a'],limit=2) + method,path,body=api.project_request('project_files.measurement_query',arguments) + self.assertEqual(method,'POST') + self.assertEqual(body['filters'],{'cpm':16}) + self.assertIn(self.table.sha256,path) + self.assertFalse(api.operation('project_files.measurement_query')['side_effects']) + for change in ({'script':'never.py'},{'execute':True},{'packet_sha256':'not-a-hash'},{'limit':101}): + with self.subTest(change=change),self.assertRaises(api.AgentAPIError): + api.validate_arguments('project_files.measurement_query',dict(arguments,**change)) + for missing in ('origin_catalog_id','packet_sha256','binding','checkpoint','metric_id'): + with self.subTest(missing=missing),self.assertRaises(api.AgentAPIError): + api.validate_arguments('project_files.measurement_query',{k:v for k,v in arguments.items() if k!=missing}) + + def test_exact_query_and_inspection_reply_is_accepted_and_changed_selection_refused(self): + query=self.table.query('error',subject_ids=['a'],filters={'cpm':16},limit=2) + value=dict(source=self.source,value=query) + args=dict(self.args,metric_id='error',subject_ids=['a'],filters={'cpm':16},limit=2) + self.assertEqual(self.sdk(value).measurement_query(**args)['value'],value) + for key,replacement in (('packet_sha256',OTHER),('filters',{}),('offset',1),('subject_ids',['b'])): + changed=copy.deepcopy(value);changed['value'][key]=replacement + with self.subTest(key=key),self.assertRaises(api.AgentAPIError): + self.sdk(changed).measurement_query(**args) + changed=copy.deepcopy(value);changed['source']['origin_catalog_id']='f'*32 + with self.assertRaises(api.AgentAPIError):self.sdk(changed).measurement_query(**args) + row=dict(source=self.source,value=self.table.inspect('a-case1-error')) + self.assertEqual(self.sdk(row).measurement_inspect(**self.args,observation_id='a-case1-error')['value'],row) + with self.assertRaises(api.AgentAPIError): + self.sdk(row).measurement_inspect(**self.args,observation_id='a-case2-error') + + def test_header_subject_and_source_pages_and_import_keep_exact_identity(self): + header=dict(source=self.source,value=self.table.metadata()) + self.assertEqual(self.sdk(header).measurement_metadata(**self.args)['value'],header) + header['value']['title']='Different retained title' + with self.assertRaises(api.AgentAPIError):self.sdk(header).measurement_metadata(**self.args) + subjects=dict(source=self.source,value=self.table.subjects(offset=1,limit=1)) + self.assertEqual(self.sdk(subjects).measurement_subjects(**self.args,offset=1,limit=1)['value'],subjects) + with self.assertRaises(api.AgentAPIError):self.sdk(subjects).measurement_subjects(**self.args,offset=0,limit=1) + availability=dict(state='unverified',expected_sha256=self.table.sha256,expected_bytes=self.source['blob']['bytes'], + observed_sha256=None,observed_bytes=self.source['blob']['bytes']) + page=dict(items=[dict(self.source,availability=availability)],offset=0,limit=40,total=1) + self.assertEqual(self.sdk(page).measurement_sources(binding=BINDING,checkpoint=CHECKPOINT)['value'],page) + request=dict(bound(),snapshot={k:self.source['source'][k] for k in ('origin_catalog_id','project_id','manifest_sha256')}, + reference_id='measurements',expected_sha256=self.table.sha256) + self.assertEqual(self.sdk(self.source).measurement_import(**request)['value'],self.source) + with self.assertRaises(api.AgentAPIError): + self.sdk(dict(self.source,id=OTHER)).measurement_import(**request) + + +if __name__=='__main__':unittest.main() diff --git a/tuldok_client.py b/tuldok_client.py index c26e5489..76d09cc5 100644 --- a/tuldok_client.py +++ b/tuldok_client.py @@ -402,6 +402,24 @@ def progression(self, **arguments): """Read a bounded saved Trial graph without execution or metric disclosure.""" return self._call('progression', arguments) + def measurement_import(self, **arguments): + return self._call('measurement_import', arguments) + + def measurement_sources(self, **arguments): + return self._call('measurement_sources', arguments) + + def measurement_metadata(self, **arguments): + return self._call('measurement_metadata', arguments) + + def measurement_subjects(self, **arguments): + return self._call('measurement_subjects', arguments) + + def measurement_query(self, **arguments): + return self._call('measurement_query', arguments) + + def measurement_inspect(self, **arguments): + return self._call('measurement_inspect', arguments) + def model_create(self, **arguments): """model create; exact selected project checkpoint, existing domain validation, no execution.""" return self._call('model_create', arguments) From 916f8483e84985c75357b69fdd8dc6c78f24bb43 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 20:25:03 -0700 Subject: [PATCH 14/15] Render exact retained scalar comparisons in native projects --- app.py | 4 + static/project-files.html | 8 + static/project-files.js | 12 + static/project-measurements.css | 14 ++ static/project-measurements.js | 266 +++++++++++++++++++++ static/retained-analysis.js | 51 +--- static/retained-chart.js | 66 +++++ static/retained-measurement-maxima.vl.json | 1 + static/workspaces.html | 1 + tests/retained_analysis_browser_fixture.py | 4 +- tests/test_project_files_m2_http.py | 2 + tests/test_workspace_project_files.cjs | 1 + tests/test_workspace_project_files_m4.cjs | 1 + 13 files changed, 391 insertions(+), 40 deletions(-) create mode 100644 static/project-measurements.css create mode 100644 static/project-measurements.js create mode 100644 static/retained-chart.js create mode 100644 static/retained-measurement-maxima.vl.json diff --git a/app.py b/app.py index 9f14ad20..b7a59bb2 100644 --- a/app.py +++ b/app.py @@ -909,6 +909,8 @@ def _get(self): '/workspaces.js': ('workspaces.js', 'text/javascript'), '/experiment-graph.js': ('experiment-graph.js', 'text/javascript'), '/project-progression.js': ('project-progression.js', 'text/javascript'), + '/project-measurements.js': ('project-measurements.js', 'text/javascript'), + '/project-measurements.css': ('project-measurements.css', 'text/css'), '/experiment-lineage.js': ('experiment-lineage.js', 'text/javascript'), '/experiment-lineage.css': ('experiment-lineage.css', 'text/css'), '/workspace-real.js': ('workspace-real.js', 'text/javascript'), @@ -916,7 +918,9 @@ def _get(self): '/workspace-phase-timings.js': ('workspace-phase-timings.js', 'text/javascript'), '/workspace-service.js': ('workspace-service.js', 'text/javascript'), '/static/retained-analysis.js': ('retained-analysis.js', 'text/javascript'), + '/static/retained-chart.js': ('retained-chart.js', 'text/javascript'), '/static/retained-checkpoint-curve.vl.json': ('retained-checkpoint-curve.vl.json', 'application/json'), + '/static/retained-measurement-maxima.vl.json': ('retained-measurement-maxima.vl.json', 'application/json'), '/static/vendor/retained-analysis/vega.min.js': ('vendor/retained-analysis/vega.min.js', 'text/javascript'), '/static/vendor/retained-analysis/vega-lite.min.js': ('vendor/retained-analysis/vega-lite.min.js', 'text/javascript'), '/static/vendor/retained-analysis/vega-embed.min.js': ('vendor/retained-analysis/vega-embed.min.js', 'text/javascript'), diff --git a/static/project-files.html b/static/project-files.html index b59af9e1..de9c6f1a 100644 --- a/static/project-files.html +++ b/static/project-files.html @@ -9,10 +9,17 @@ + + + + + + + @@ -164,6 +171,7 @@

            Saved annotation history

            +

            Project data and scientific history

            Inspect saved records, models, Studies, Runs and their exact artifact availability. Opening a project never starts computation or restores permission to disclose held-out data.

            diff --git a/static/project-files.js b/static/project-files.js index fc148da5..708658ed 100644 --- a/static/project-files.js +++ b/static/project-files.js @@ -880,6 +880,16 @@ onContextFocusLost:()=>$('project-files-status').focus() }); window.addEventListener('pagehide',()=>projectProgression.clear()); + const projectMeasurements=window.ProjectMeasurements({ + root:$('project-measurements'), + async read(action,args,signal){ + if(!m2Allowed(action))throw Error('Measurement inspection is unavailable for the selected project.'); + const value=await m2Request(action,args,epoch,{signal}); + if(value===null)throw new DOMException('Project selection changed.','AbortError'); + return value; + },onContextFocusLost:()=>$('project-files-status').focus() + }); + window.addEventListener('pagehide',()=>projectMeasurements.clear()); window.addEventListener('pagehide',clearScientific); const exact=(value,keys)=>value!==null&&typeof value==='object'&&!Array.isArray(value)&&Object.keys(value).length===keys.length&&keys.every(key=>Object.hasOwn(value,key)); const str=(value,max,empty=false)=>typeof value==='string'&&Array.from(value).length<=max&&(empty||/\S/u.test(value)); @@ -1137,6 +1147,8 @@ function m2Controls(){ projectProgression.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), project_id:state?.project?.project_id,enabled:m2Allowed('progression')&&m2Allowed('progression_origins')}); + projectMeasurements.setContext({key:JSON.stringify([state?.binding,state?.project?.project_id,checkpoint(),window.TuldokAuth?.identity?.()?.credential_id,epoch]), + enabled:['measurement_sources','measurement_metadata','measurement_subjects','measurement_query','measurement_inspect'].every(m2Allowed)}); if(blocked||state?.state!=='open'||scientificEpoch!==null&&(scientificEpoch!==epoch||explored?.checkpoint!==checkpoint()))clearScientific(); const action=$('project-explore-kind').value; $('project-explore-load').disabled=!m2Allowed(action); diff --git a/static/project-measurements.css b/static/project-measurements.css new file mode 100644 index 00000000..d1ba5ff3 --- /dev/null +++ b/static/project-measurements.css @@ -0,0 +1,14 @@ +#project-measurements{overflow-wrap:anywhere;min-width:0} +#project-measurements section{margin-top:1rem;padding:1rem} +#project-measurements table{width:100%;border-collapse:collapse;margin:1rem 0} +#project-measurements caption{text-align:left;font-weight:600;padding:.5rem 0} +#project-measurements th,#project-measurements td{border-bottom:1px solid #c6d0c7;padding:.6rem;text-align:left;vertical-align:top} +#project-measurements pre{white-space:pre-wrap;max-height:28rem;overflow:auto;font-size:.9rem} +#project-measurements dl{margin:.5rem 0}#project-measurements dt{font-weight:600}#project-measurements dd{margin:0 0 .5rem} +#project-measurements details{margin:.75rem 0}#project-measurements summary{cursor:pointer;font-weight:600} +#project-measurements-filters{display:grid;grid-template-columns:repeat(auto-fit,minmax(240px,1fr));gap:1rem;margin:1rem 0} +#project-measurements-filters fieldset{border:1px solid #c6d0c7;padding:.75rem} +#project-measurements-chart{min-height:5rem;min-width:0;width:100%} +#project-measurements-chart .vega-embed{width:100%} +#project-measurements-maxima,#project-measurements-rows{overflow:auto;min-width:0} +@media(max-width:760px){#project-measurements-filters{grid-template-columns:1fr}#project-measurements table{min-width:34rem}} diff --git a/static/project-measurements.js b/static/project-measurements.js new file mode 100644 index 00000000..7f99a043 --- /dev/null +++ b/static/project-measurements.js @@ -0,0 +1,266 @@ +/* Native scalar selection/inspection. The host owns current access and reads. */ +(() => { + 'use strict'; + const STATES = ['measured', 'failed', 'unsupported', 'unavailable']; + const hash = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); + const identity = value => typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(value); + const natural = value => Number.isSafeInteger(value) && value >= 0; + const finite = value => typeof value === 'number' && Number.isFinite(value); + const display = value => value === null ? 'Unknown' : Object.is(value, -0) ? '-0' : String(value); + const node = (tag, text) => { const value = document.createElement(tag); if (text !== undefined) value.textContent = text; return value; }; + const button = (text, invoke) => { const value = node('button', text); value.type = 'button'; value.onclick = invoke; return value; }; + const sourceKey = source => source.origin_catalog_id + '.' + source.id; + const spec = {path: '/static/retained-measurement-maxima.vl.json', sha256: 'c9440bbe90d215afb15d47a5f12c5aefcb20dc1d3ddba19d72fe5ba3adb1cd92'}; + function page(value, offset, limit, items) { + if (!value || value.offset !== offset || value.limit !== limit || !natural(value.total) || !Array.isArray(items) + || items.length !== Math.min(limit, Math.max(0, value.total - offset))) throw Error('Measurement page differs from the selected continuation.'); + } + function jsonDetails(title, value, open = false) { + const details = node('details'); details.open = open; + details.append(node('summary', title), node('pre', JSON.stringify(value, null, 2))); return details; + } + window.ProjectMeasurements = function({root, read, onContextFocusLost = () => {}}) { + root.innerHTML = '

            Retained measurements

            Compare exact saved scalars and inspect their original cases. Maxima use every matching case; failed and unavailable observations remain counted.

            '; + const $ = name => root.querySelector('#project-measurements-' + name); + let context = null, generation = 0, plotGeneration = 0, pending, busy = false, pendingFocus = false; + let sources = [], sourceOffset = 0, sourceTotal = 0, selected = null, descriptor = null, metadata = null; + let subjects = [], subjectOffset = 0, subjectTotal = 0, filters = new Map(), applied = null, query = null; + const renderer = window.RetainedChart.mount($('chart'), spec, {onError: error => { + const ownsFocus = root.contains(document.activeElement); clearResults(); $('status').textContent = error.message; controls(); + if (ownsFocus) $('compare').focus(); + }}); + function clearInspection() { $('inspection').replaceChildren(); $('inspection').hidden = true; } + function clearResults() { + ++plotGeneration; renderer.clear(); query = null; applied = null; + $('results').hidden = true; $('maxima').replaceChildren(); $('rows').replaceChildren(); clearInspection(); + } + function controls() { + for (const control of root.querySelectorAll('input,select,button')) control.disabled = !context?.enabled || busy; + $('sources-prev').hidden = sourceOffset === 0; $('sources-next').hidden = sourceOffset + sources.length >= sourceTotal; + $('source').disabled ||= !sources.length; $('load').disabled ||= !$('source').value; + $('subjects-prev').hidden = subjectOffset === 0; $('subjects-next').hidden = subjectOffset + subjects.length >= subjectTotal; + $('compare').disabled ||= !metadata || !$('subjects').selectedOptions.length; + $('prev').disabled ||= !query || query.offset === 0; + $('next').disabled ||= !query || query.offset + query.rows.length >= query.total; + root.setAttribute('aria-busy', String(busy)); + } + function clearSource() { + selected = descriptor = metadata = null; subjects = []; subjectOffset = subjectTotal = 0; filters.clear(); + $('form').hidden = true; $('context').replaceChildren(); $('filters').replaceChildren(); + $('metric').replaceChildren(); $('subjects').replaceChildren(); clearResults(); + } + function reset() { + ++generation; pending?.abort(); pending = undefined; busy = false; pendingFocus = false; + clearSource(); sources = []; sourceOffset = sourceTotal = 0; + $('source').replaceChildren(); $('source-page').textContent = ''; $('status').textContent = ''; controls(); + } + const ownsFocus = () => root.contains(document.activeElement) + || busy && pendingFocus && document.activeElement === document.body; + async function perform(work) { + if (!context?.enabled || busy) return; + const ticket = ++generation, authority = context.key; + pending = new AbortController(); const signal = pending.signal; + const active = () => ticket === generation && context?.key === authority && context.enabled && !signal.aborted; + pendingFocus = root.contains(document.activeElement); busy = true; controls(); + let focus; + try { focus = await work(active, signal, authority); } + catch (error) { + if (active()) { + const ownedFocus = ownsFocus(); reset(); $('status').textContent = error.message; + if (ownedFocus) $('sources').focus(); + } + } finally { if (active()) { busy = false; pending = undefined; pendingFocus = false; controls(); focus?.focus(); } } + } + async function loadSources(offset = 0) { + clearSource(); + return perform(async(active, signal) => { + const value = await read('measurement_sources', {offset, limit: 40}, signal); if (!active()) return; + page(value, offset, 40, value.items); + const seen = new Set(); + for (const item of value.items) { + const key = sourceKey(item); + if (!hash(item.id) || !/^[a-f0-9]{32}$/.test(item.origin_catalog_id) || seen.has(key) + || item.blob?.sha256 !== item.id || typeof item.title !== 'string') throw Error('Invalid retained measurement source.'); + seen.add(key); + } + sources = value.items; sourceOffset = offset; sourceTotal = value.total; $('source').replaceChildren(); + for (const item of sources) { + const option = node('option', item.title + ' · ' + item.id.slice(0, 12) + (item.availability?.state === 'missing' ? ' · missing packet' : '')); + option.value = sourceKey(item); $('source').append(option); + } + $('source-page').textContent = `${sources.length ? offset + 1 : 0}–${offset + sources.length} of ${sourceTotal} sources`; + $('status').textContent = sources.length ? 'Choose a source and load its exact retained metadata.' : 'No retained measurement sources in this project.'; + return $('source'); + }); + } + function checkRead(value, source = selected) { + if (!value || value.source?.origin_catalog_id !== source?.origin_catalog_id || value.source?.id !== source?.id + || value.source.blob?.sha256 !== source.id || value.source.source?.measured_sha256 !== source.id + || descriptor && (value.source.blob.bytes !== descriptor.blob.bytes || value.source.title !== descriptor.title) + || new TextEncoder().encode(JSON.stringify(value)).length > 256 * 1024) throw Error('Measurement reply differs from the selected packet.'); + return value.value; + } + const selection = () => ({origin_catalog_id: selected.origin_catalog_id, packet_sha256: selected.id}); + async function subjectPage(offset, active, signal) { + const value = checkRead(await read('measurement_subjects', {...selection(), offset, limit: 40}, signal)); if (!active()) return; + page(value, offset, 40, value.items); + if (value.total !== metadata.subject_count) throw Error('Subject page differs from the retained packet population.'); + if (value.items.some(item => !identity(item.id) || typeof item.label !== 'string') + || new Set(value.items.map(item => item.id)).size !== value.items.length) throw Error('Invalid retained subjects.'); + subjects = value.items; subjectOffset = offset; subjectTotal = value.total; $('subjects').replaceChildren(); + for (const item of subjects) { const option = node('option', item.label + ' · ' + item.id); option.value = item.id; option.selected = true; $('subjects').append(option); } + $('subject-page').textContent = `Subjects ${subjects.length ? offset + 1 : 0}–${offset + subjects.length} of ${subjectTotal}. Only explicitly selected subjects on this page will be compared.`; + return $('subjects'); + } + function dimensions() { + filters.clear(); $('filters').replaceChildren(); + for (const [index, dimension] of metadata.dimensions.entries()) { + const field = node('fieldset'), legend = node('legend', `${dimension.label}${dimension.units ? ' · ' + dimension.units : ''} (${dimension.scope})`); + const mode = node('select'); mode.id = 'project-measurements-filter-mode-' + index; + for (const [value, label] of [['any','Any recorded value'],['value','Exact value'],['unknown','Unknown only']]) { const option = node('option', label); option.value = value; mode.append(option); } + const label = node('label', 'Match'); label.htmlFor = mode.id; label.append(mode); + let input; + if (dimension.type === 'boolean') { input = node('select'); for (const value of ['true','false']) { const option = node('option', value); option.value = value; input.append(option); } } + else { input = node('input'); input.type = dimension.type === 'string' ? 'text' : 'number'; input.maxLength = 256; input.step = dimension.type === 'integer' ? '1' : 'any'; } + input.id = 'project-measurements-filter-value-' + index; + const valueLabel = node('label', 'Exact recorded value'); valueLabel.htmlFor = input.id; valueLabel.append(input); + field.append(legend, label, valueLabel); $('filters').append(field); filters.set(dimension.id, {dimension, mode, input}); + mode.onchange = invalidate; input.oninput = invalidate; + } + } + function filterValues() { + const values = {}; + for (const [id, {dimension, mode, input}] of filters) { + if (mode.value === 'any') continue; + if (mode.value === 'unknown') { values[id] = null; continue; } + if (mode.value !== 'value' || !input.value.length) throw Error('Enter an exact filter value or choose Any recorded value.'); + const value = dimension.type === 'boolean' ? input.value === 'true' : dimension.type === 'string' ? input.value : Number(input.value); + if ((dimension.type === 'number' && !finite(value)) || (dimension.type === 'integer' && !Number.isSafeInteger(value))) throw Error('Filter values must match the declared numeric type.'); + values[id] = value; + } + return values; + } + function invalidate() { + ++generation; pending?.abort(); pending = undefined; busy = false; clearResults(); + $('status').textContent = 'Selection edited. Compare metric to apply it.'; controls(); + } + function attributes(title, values) { + const details = node('details'); details.append(node('summary', title)); + const list = node('dl'); + for (const [key, value] of Object.entries(values)) { + const dimension = metadata.dimensions.find(item => item.id === key); + list.append(node('dt', dimension?.label ?? key), node('dd', display(value) + (dimension?.units ? ' ' + dimension.units : ''))); + } + details.append(list); return details; + } + function table(caption, headings) { + const result = node('table'), row = node('tr'), head = node('thead'), body = node('tbody'); + for (const text of headings) { const cell = node('th', text); cell.scope = 'col'; row.append(cell); } + head.append(row); result.append(node('caption', caption), head, body); return {result, body}; + } + async function inspect(observation_id) { + clearInspection(); + return perform(async(active, signal) => { + const value = checkRead(await read('measurement_inspect', {...selection(), observation_id}, signal)); if (!active()) return; + if (value.packet_sha256 !== selected.id || value.observation?.id !== observation_id + || value.observation.metric_id !== applied?.metric_id || !applied.subject_ids.includes(value.observation.subject_id)) throw Error('Observation differs from the selected comparison.'); + const heading = node('h3', value.subject.label + ' · ' + value.case.id); heading.tabIndex = -1; + const observation = value.observation; + $('inspection').append(heading, node('p', `${value.metric.label} · ${value.metric.units} · ${observation.state} · ${observation.value === null ? observation.reason : display(observation.value)}`), + attributes('Case attributes, including grid density and domain cell counts', value.case.dimensions), + attributes('Subject execution attributes', value.subject.dimensions), attributes('Observation work attributes', observation.dimensions), + jsonDetails('Original case evidence', value.case_evidence, true), jsonDetails('Original scalar evidence', value.scalar_evidence, true), + jsonDetails('Exact observation, metric, source and scope', value)); + $('inspection').hidden = false; heading.focus(); $('status').textContent = 'Exact retained observation and original source evidence.'; + }); + } + async function showQuery(value, request, active, signal, authority) { + if (value.packet_sha256 !== selected.id || value.metric?.id !== request.metric_id + || JSON.stringify(value.subject_ids) !== JSON.stringify(request.subject_ids) + || !value.filters || Object.keys(value.filters).length !== Object.keys(request.filters).length + || Object.entries(request.filters).some(([key, wanted]) => !Object.hasOwn(value.filters, key) || value.filters[key] !== wanted) + || !Array.isArray(value.summary) || value.summary.length !== request.subject_ids.length || value.sampling !== 'none') throw Error('Measurement comparison differs from its requested selection.'); + page(value, request.offset, request.limit, value.rows); + const maxima = table('Maximum over all matching cases, including every observation status', ['Subject / execution attributes','Maximum','Measured','Failed','Unsupported','Unavailable']); + const points = []; + for (const [index, summary] of value.summary.entries()) { + if (summary.subject_id !== request.subject_ids[index] || !natural(summary.count) || STATES.some(state => !natural(summary.states?.[state])) + || STATES.reduce((sum, state) => sum + summary.states[state], 0) !== summary.count + || (summary.maximum === null) !== (summary.states.measured === 0)) throw Error('Invalid complete measurement summary.'); + const subject = subjects.find(item => item.id === summary.subject_id); if (!subject) throw Error('Subject is outside the selected page.'); + const row = node('tr'), label = node('td'); label.append(node('strong', subject.label), attributes('Execution attributes', subject.dimensions)); + const maximum = node('td'); + if (summary.maximum === null) maximum.textContent = 'No measured value'; + else { + const point = summary.maximum; + if (!finite(point.value) || !identity(point.id) || !identity(point.case_id)) throw Error('Invalid retained maximum.'); + maximum.append(button(display(point.value) + ' · inspect maximum for ' + subject.label, () => void inspect(point.id))); + points.push({subject_id:subject.id, value:point.value, observation_id:point.id}); + } + row.append(label, maximum, ...STATES.map(state => node('td', String(summary.states[state])))); maxima.body.append(row); + } + const observations = table('Exact observations on this page', ['Subject','Case','Value / status','Inspection']); + for (const observation of value.rows) { + if (!request.subject_ids.includes(observation.subject_id) || observation.metric_id !== request.metric_id || !identity(observation.id) + || !STATES.includes(observation.state) || (observation.state === 'measured' ? !finite(observation.value) : observation.value !== null)) throw Error('Invalid retained observation row.'); + const subject = subjects.find(item => item.id === observation.subject_id), row = node('tr'), detail = node('td'); + detail.append(button('Inspect ' + observation.id, () => void inspect(observation.id)), attributes('Case and execution attributes', observation.dimensions)); + row.append(node('td', subject.label), node('td', observation.case_id), node('td', observation.value === null ? observation.state + ': ' + observation.reason : display(observation.value)), detail); observations.body.append(row); + } + query = value; applied = structuredClone(request); $('results').hidden = false; + $('heading').textContent = value.metric.label + ' · ' + value.metric.units; + $('counts').textContent = `${request.subject_ids.length} selected subjects of ${subjectTotal}; ${value.total} matching observations. Showing ${value.rows.length ? value.offset + 1 : 0}–${value.offset + value.rows.length}. Maxima use every matching case, independent of this page.`; + $('maxima').replaceChildren(maxima.result); $('rows').replaceChildren(observations.result); + const plotTicket = ++plotGeneration; + if (points.length) await renderer.render({maxima:points}, {signal, + isCurrent: () => context?.enabled && context.key === authority && plotTicket === plotGeneration, + onSignal:{measurement_select: selection => { const id = selection?.observation_id?.[0]; if (points.some(point => point.observation_id === id)) void inspect(id); }}}); + else { renderer.clear(); $('chart').append(node('p', 'No measured values for this selection. Missing and failed observations remain counted below.')); } + if (active()) { $('heading').focus(); $('status').textContent = 'Exact retained comparison. Metric and scope definitions are available above.'; } + } + function compare(offset = 0, saved = null) { + return perform(async(active, signal, authority) => { + const request = saved ? {...saved, offset} : {metric_id:$('metric').value, + subject_ids:[...$('subjects').selectedOptions].map(option => option.value), filters:filterValues(), offset, limit:50}; + if (!request.subject_ids.length) throw Error('Choose at least one subject.'); + clearResults(); + const value = checkRead(await read('measurement_query', {...selection(), ...request}, signal)); if (!active()) return; + await showQuery(value, request, active, signal, authority); + }); + } + $('sources').onclick = () => void loadSources(); + $('sources-prev').onclick = () => void loadSources(Math.max(0, sourceOffset - 40)); + $('sources-next').onclick = () => void loadSources(sourceOffset + 40); + $('source').onchange = () => { invalidate(); clearSource(); controls(); }; + $('source-form').onsubmit = event => { + event.preventDefault(); + void perform(async(active, signal) => { + const chosen = sources.find(source => sourceKey(source) === $('source').value); clearSource(); selected = chosen; + if (!selected) throw Error('Choose a retained measurement source.'); + const response = await read('measurement_metadata', selection(), signal); if (!active()) return; + metadata = checkRead(response); descriptor = response.source; + if (metadata.packet_sha256 !== selected.id || metadata.title !== selected.title || !Array.isArray(metadata.metrics) || metadata.metrics.length > 32 + || !Array.isArray(metadata.dimensions) || metadata.dimensions.length > 32) throw Error('Invalid exact measurement metadata.'); + $('context').append(node('h3', metadata.title), node('p', metadata.scope.label + ' · population: ' + metadata.scope.population), + node('p', typeof metadata.scope_definition.task === 'string' ? metadata.scope_definition.task : 'Interpret results using the retained scope definition.'), + jsonDetails('Exact scope, metric definitions and packet source', {metadata, source:descriptor})); + for (const metric of metadata.metrics) { const option = node('option', metric.label + ' · ' + metric.units); option.value = metric.id; $('metric').append(option); } + dimensions(); await subjectPage(0, active, signal); if (!active()) return; + $('form').hidden = false; $('status').textContent = 'Choose a metric, subjects and optional exact attribute filters.'; return $('metric'); + }); + }; + $('metric').onchange = invalidate; $('subjects').onchange = invalidate; + for (const [name, direction] of [['subjects-prev',-1],['subjects-next',1]]) $(name).onclick = () => { + clearResults(); void perform((active, signal) => subjectPage(subjectOffset + direction * 40, active, signal)); + }; + $('form').onsubmit = event => { event.preventDefault(); void compare(); }; + $('prev').onclick = () => void compare(Math.max(0, query.offset - query.limit), applied); + $('next').onclick = () => void compare(query.offset + query.limit, applied); + controls(); + return {setContext(value) { + if (value.key !== context?.key || context?.enabled && !value.enabled) { const ownedFocus = ownsFocus(); context = value; reset(); if (ownedFocus) onContextFocusLost(); } + else context = value; + root.hidden = !value.enabled; controls(); + }, clear() { context = null; reset(); root.hidden = true; }}; + }; +})(); diff --git a/static/retained-analysis.js b/static/retained-analysis.js index 11475136..06274428 100644 --- a/static/retained-analysis.js +++ b/static/retained-analysis.js @@ -64,30 +64,10 @@ for (const [key, value] of Object.entries(attributes)) element.setAttribute(key, value); return element; } - async function specification(signal) { - const reply = await fetch(SPEC_PATH, {signal, credentials: 'same-origin', cache: 'no-cache'}); - if (!reply.ok) throw Error('Retained chart specification is unavailable.'); - const reader = reply.body.getReader(); - let raw = new Uint8Array(0); - while (true) { - const {value, done} = await reader.read(); - if (done) break; - if (raw.byteLength + value.byteLength > 16384) { - await reader.cancel(); - throw Error('Retained chart specification exceeds its budget.'); - } - const next = new Uint8Array(raw.byteLength + value.byteLength); - next.set(raw); next.set(value, raw.byteLength); raw = next; - } - const hash = [...new Uint8Array(await crypto.subtle.digest('SHA-256', raw))].map(byte => byte.toString(16).padStart(2, '0')).join(''); - if (hash !== SPEC_SHA256) throw Error('Retained chart specification identity differs.'); - return JSON.parse(new TextDecoder().decode(raw)); - } function mount(root, {load, onSelection = () => {}}) { - let disposed = false, generation = 0, pending, view, observer, current; + let disposed = false, generation = 0, pending, renderer, current; function clear() { - view?.finalize(); view = undefined; - observer?.disconnect(); observer = undefined; + renderer?.dispose(); renderer = undefined; current = undefined; root.replaceChildren(); } @@ -100,7 +80,7 @@ root.append(node('p', 'Loading exact retained result…', {role: 'status'})); const active = () => !disposed && ticket === generation && !signal.aborted; try { - const [received, base] = await Promise.all([load(request, {signal, authorityKey, export: false}), specification(signal)]); + const received = await load(request, {signal, authorityKey, export: false}); if (!active()) return; const data = validate(received, request); clear(); @@ -157,21 +137,16 @@ }); const details = node('details'); details.append(node('summary', 'Exact retained provenance'), node('pre', JSON.stringify(data.provenance, null, 2))); root.append(heading, status, chart, buttons, filterLabel, support, table, navigation, exportButton, details); - if (typeof window.vegaEmbed !== 'function' || typeof window.vega?.expressionInterpreter !== 'object') throw Error('Pinned chart assets are unavailable.'); - base.datasets = {curve: data.curve.map(point => ({...point, selected: point.checkpoint_sha256 === request.checkpoint_sha256}))}; - // Data, images and links cannot leave the controlled inline dataset. - const deny = () => Promise.reject(Error('Chart URL loading is disabled.')); - const embedded = await window.vegaEmbed(chart, base, {actions: false, defaultStyle: false, tooltip: false, mode: 'vega-lite', renderer: 'svg', ast: true, expr: window.vega.expressionInterpreter, loader: {load: deny, sanitize: deny}}); - if (!active()) { embedded.finalize(); return; } - view = embedded.view; - view.addSignalListener('checkpoint_select', (_name, value) => { - const sha = value?.checkpoint_sha256?.[0]; - if (data.curve.some(point => point.checkpoint_sha256 === sha)) choose(sha); + renderer = window.RetainedChart.mount(chart, {path: SPEC_PATH, sha256: SPEC_SHA256}, {onError: () => { + if (active()) { clear(); root.append(node('p', 'Retained chart resize failed.', {role: 'alert'})); } + }}); + await renderer.render({curve: data.curve.map(point => ({...point, selected: point.checkpoint_sha256 === request.checkpoint_sha256}))}, { + signal, isCurrent: active, onSignal: {checkpoint_select: value => { + const sha = value?.checkpoint_sha256?.[0]; + if (data.curve.some(point => point.checkpoint_sha256 === sha)) choose(sha); + }} }); - observer = new ResizeObserver(() => { - if (active()) void view?.resize().runAsync().catch(() => { if (active()) { clear(); root.append(node('p', 'Retained chart resize failed.', {role: 'alert'})); } }); - }); - observer.observe(chart); + if (!active()) return; current = {request, authorityKey}; } catch (error) { if (active()) { @@ -182,7 +157,7 @@ } return { update, - resize() { return view?.resize().runAsync(); }, + resize() { return renderer?.resize(); }, selection() { return current ? structuredClone(current.request) : null; }, dispose() { disposed = true; ++generation; pending?.abort(); clear(); root.removeAttribute('aria-busy'); } }; diff --git a/static/retained-chart.js b/static/retained-chart.js new file mode 100644 index 00000000..9394062f --- /dev/null +++ b/static/retained-chart.js @@ -0,0 +1,66 @@ +/* Shared pinned Vega renderer. Domain hosts own data, selection and authority. */ +(() => { + 'use strict'; + async function specification({path, sha256}, signal) { + if (!/^\/static\/[a-z0-9-]+\.vl\.json$/.test(path) || !/^[a-f0-9]{64}$/.test(sha256)) throw Error('Invalid pinned chart identity.'); + const reply = await fetch(path, {signal, credentials: 'same-origin', cache: 'no-cache'}); + if (!reply.ok) throw Error('Retained chart specification is unavailable.'); + const reader = reply.body.getReader(); + let raw = new Uint8Array(0); + try { + while (true) { + const {value, done} = await reader.read(); + if (done) break; + if (raw.byteLength + value.byteLength > 16384) throw Error('Retained chart specification exceeds its budget.'); + const next = new Uint8Array(raw.byteLength + value.byteLength); + next.set(raw); next.set(value, raw.byteLength); raw = next; + } + } finally { await reader.cancel(); } + const hash = [...new Uint8Array(await crypto.subtle.digest('SHA-256', raw))].map(byte => byte.toString(16).padStart(2, '0')).join(''); + if (hash !== sha256) throw Error('Retained chart specification identity differs.'); + return JSON.parse(new TextDecoder().decode(raw)); + } + function mount(root, pinned, {onError = error => { root.textContent = error.message; }} = {}) { + let disposed = false, generation = 0, pending, view, observer, detach; + function clear() { + ++generation; pending?.abort(); pending = undefined; + detach?.(); detach = undefined; + observer?.disconnect(); observer = undefined; + view?.finalize(); view = undefined; + root.replaceChildren(); root.removeAttribute('aria-busy'); + } + async function render(datasets, {signal, onSignal = {}, isCurrent = () => true} = {}) { + clear(); + if (disposed || signal?.aborted || !isCurrent()) return false; + if (new TextEncoder().encode(JSON.stringify(datasets)).length > 256 * 1024) throw Error('Retained chart data exceeds its budget.'); + const ticket = generation; + pending = new AbortController(); const ownSignal = pending.signal; + signal?.addEventListener('abort', clear, {once: true}); + detach = () => signal?.removeEventListener('abort', clear); + const active = () => !disposed && ticket === generation && !ownSignal.aborted && !signal?.aborted && isCurrent(); + root.setAttribute('aria-busy', 'true'); + try { + const base = await specification(pinned, ownSignal); + if (!active()) return false; + if (typeof window.vegaEmbed !== 'function' || typeof window.vega?.expressionInterpreter !== 'object') throw Error('Pinned chart assets are unavailable.'); + base.datasets = structuredClone(datasets); + const deny = () => Promise.reject(Error('Chart URL loading is disabled.')); + const embedded = await window.vegaEmbed(root, base, {actions: false, defaultStyle: false, tooltip: false, + mode: 'vega-lite', renderer: 'svg', ast: true, expr: window.vega.expressionInterpreter, loader: {load: deny, sanitize: deny}}); + if (!active()) { embedded.finalize(); return false; } + view = embedded.view; + for (const [name, select] of Object.entries(onSignal)) view.addSignalListener(name, (_name, value) => { if (active()) select(value); }); + observer = new ResizeObserver(() => { + if (active()) void view?.resize().runAsync().catch(error => { if (active()) { clear(); onError(error); } }); + }); + observer.observe(root); + return true; + } catch (error) { + if (!active()) return false; + clear(); throw error; + } finally { if (active()) root.setAttribute('aria-busy', 'false'); } + } + return {render, clear, resize: () => view?.resize().runAsync(), dispose() { disposed = true; clear(); }}; + } + window.RetainedChart = Object.freeze({mount}); +})(); diff --git a/static/retained-measurement-maxima.vl.json b/static/retained-measurement-maxima.vl.json new file mode 100644 index 00000000..42035e08 --- /dev/null +++ b/static/retained-measurement-maxima.vl.json @@ -0,0 +1 @@ +{"$schema":"https://vega.github.io/schema/vega-lite/v5.20.1.json","description":"Maximum retained scalar for each selected subject over all matching cases. Exact values, missing counts and keyboard inspection follow the chart.","width":"container","height":{"step":30},"data":{"name":"maxima"},"mark":{"type":"point","filled":true,"size":100,"color":"#245d8a"},"encoding":{"x":{"field":"value","type":"quantitative","title":"Maximum value (selected metric units)","scale":{"zero":false},"axis":{"format":".3~e"}},"y":{"field":"subject_id","type":"nominal","title":"Subject","sort":null}},"params":[{"name":"measurement_select","select":{"type":"point","fields":["observation_id"],"on":"click","clear":false,"toggle":false}}]} diff --git a/static/workspaces.html b/static/workspaces.html index 1ff76169..f9db1279 100644 --- a/static/workspaces.html +++ b/static/workspaces.html @@ -138,6 +138,7 @@

            Unsaved fixture edits

            + diff --git a/tests/retained_analysis_browser_fixture.py b/tests/retained_analysis_browser_fixture.py index 9fd08603..67ce6810 100644 --- a/tests/retained_analysis_browser_fixture.py +++ b/tests/retained_analysis_browser_fixture.py @@ -4,7 +4,7 @@ import sys ROOT = Path(__file__).resolve().parents[1] -HTML = b'''Retained analysis renderer fixture
            ''' +HTML = b'''Retained analysis renderer fixture
            ''' class Handler(SimpleHTTPRequestHandler): @@ -22,7 +22,7 @@ def do_GET(self): self.end_headers() self.wfile.write(HTML) elif self.path.startswith('/static/vendor/retained-analysis/') or self.path in ( - '/static/retained-analysis.js', '/static/retained-checkpoint-curve.vl.json', + '/static/retained-chart.js', '/static/retained-analysis.js', '/static/retained-checkpoint-curve.vl.json', '/tests/fixtures/retained-analysis-browser.js', '/tests/fixtures/retained-analysis-browser.css'): super().do_GET() else: diff --git a/tests/test_project_files_m2_http.py b/tests/test_project_files_m2_http.py index ec3e431e..82492847 100644 --- a/tests/test_project_files_m2_http.py +++ b/tests/test_project_files_m2_http.py @@ -38,6 +38,8 @@ 'static/scientific-inspector.js','static/scientific-inspector.css','static/scientific-fields.js', 'static/workbench.html','static/workbench.css') SOURCE_FILES+=('project_measurements.py','retained_measurements.py','project_fork_composition.py') +SOURCE_FILES+=('static/retained-chart.js','static/project-measurements.js','static/project-measurements.css', + 'static/retained-measurement-maxima.vl.json') def subject_hashes(): diff --git a/tests/test_workspace_project_files.cjs b/tests/test_workspace_project_files.cjs index 0fefa259..31339b01 100644 --- a/tests/test_workspace_project_files.cjs +++ b/tests/test_workspace_project_files.cjs @@ -37,6 +37,7 @@ function mount(initial=[reply(state()),reply(records())],capabilityValue=capabil // View modules have separate real-browser evidence; this fixture owns native record/control lifetimes. context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); + context.ProjectMeasurements=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,events,discoveries,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, diff --git a/tests/test_workspace_project_files_m4.cjs b/tests/test_workspace_project_files_m4.cjs index 77448d3f..dbdb85b7 100644 --- a/tests/test_workspace_project_files_m4.cjs +++ b/tests/test_workspace_project_files_m4.cjs @@ -42,6 +42,7 @@ function mount(capabilityValue=capabilities){ // View modules have separate real-browser evidence; this fixture owns native record/control lifetimes. context.ScientificFieldInspector=()=>({clear(){},show(){}}); context.ProjectExperimentProgression=()=>({setContext(){},clear(){}}); + context.ProjectMeasurements=()=>({setContext(){},clear(){}}); vm.createContext(context);vm.runInContext(source,context); return {nodes,requests,answers,focused:()=>focused, input(id,value){const node=nodes.get(id);node.value=value;node.listeners.get('input')?.({target:node});}, From a5b04e9c6e27e6776291d8584d3dc44b4ada4978 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Fri, 23 Oct 2026 21:38:27 -0700 Subject: [PATCH 15/15] Preserve verified historical runtime dependencies when forking projects --- project_fork_composition.py | 10 +++-- project_history.py | 31 +++++++++++++++ tests/test_project_fork_composition.py | 53 ++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/project_fork_composition.py b/project_fork_composition.py index 7551d665..ee13410a 100644 --- a/project_fork_composition.py +++ b/project_fork_composition.py @@ -71,21 +71,23 @@ def mark(sha, record, role, name, size=None, *, policy='unique'): mark(edge['blob']['sha256'],record,edge['role'],edge['name'],edge['blob']['bytes'], policy='regenerable' if runtime_file else 'unique') - # Run outputs, exact execution captures, publication proofs and measured - # host module bytes remain unique regardless of a weaker edge label. - from project_history import declarations as run_declarations + # The history owner distinguishes recorded runtime dependencies from + # outputs/proofs/host sources; a weaker payload label alone grants none. + from project_history import declarations as run_declarations, fork_preservation for record in records: if record['family'] not in ('run_history', 'run_observation'): continue doc = (record['document']['observation'] if record['family'] == 'run_observation' else record['document']) by_sha = {edge['blob']['sha256']:edge['name'] for edge in doc['payloads']} publication = doc['publication'] + preservation = fork_preservation(doc) if publication is not None: for name in ('source','result'): if publication[name] is not None: by_sha[publication[name]['sha256']] = 'publication.'+name for declaration in run_declarations(dict(family='run_history', id=record['id'], document=doc)): blob = declaration['document'] - mark(blob['sha256'],record,'run_payload',by_sha[blob['sha256']],blob['bytes']) + mark(blob['sha256'],record,'run_payload',by_sha[blob['sha256']],blob['bytes'], + policy=preservation[blob['sha256']]) run = doc['run'] if run.get('execution_kind') == 'external_recipe': for name, sha in run['material']['host_files'].items(): diff --git a/project_history.py b/project_history.py index 6180850c..2fec6803 100644 --- a/project_history.py +++ b/project_history.py @@ -289,6 +289,37 @@ def declarations(record): return list(values.values()) +def fork_preservation(document): + """Copy obligations for an already decoded historical Run document. + + A payload label alone cannot weaken custody. Only an explicitly regenerable + historical runtime payload with the exact hash/count in this Run's recorded + runtime inventory may remain absent. Outputs, publication proofs and measured + host sources override that classification when they share the same bytes. + This projects copy policy; it does not rewrite the original retained facts. + """ + run=document['run'];runtime=set();protected={item['sha256'] for item in run.get('artifacts',[])} + if run.get('execution_kind')=='external_recipe': + material=run['material'] + for item in material.get('runtime',{}).get('files',{}).values(): + _sha(item['sha256']);natural(item['bytes'],'Historical runtime file bytes') + runtime.add((item['sha256'],item['bytes'])) + protected.update(material['host_files'].values()) + protected.update(item['sha256'] for item in material.get('captures',{}).values()) + result={} + for edge in document['payloads']: + blob=edge['blob'];digest=blob['sha256'] + regenerative=(edge.get('role')=='historical_runtime' and edge.get('preservation')=='regenerable' + and (blob['sha256'],blob['bytes']) in runtime) + policy='regenerable' if regenerative else 'unique' + if result.get(digest)!='unique':result[digest]=policy + publication=document['publication'] + if publication is not None: + protected.update(publication[name]['sha256'] for name in ('source','result') if publication[name] is not None) + for digest in protected:result[digest]='unique' + return result + + class HistoryFacts: def __init__(self): self.studies={};self.trials={};self.runs={};self.candidates={};self.policies={};self.reservations={};self.observations={} diff --git a/tests/test_project_fork_composition.py b/tests/test_project_fork_composition.py index 34c7fb23..b5fcf831 100644 --- a/tests/test_project_fork_composition.py +++ b/tests/test_project_fork_composition.py @@ -140,6 +140,59 @@ def test_missing_unique_host_hash_is_explicit_without_invented_size(self): self.assertEqual(caught.exception.code,'missing_unique') self.assertEqual(caught.exception.missing_unique,[dict(family='run_history',id=record['id'],role='host_source',name='recipe_process.py',sha256=missing,bytes=None)]) + def test_imported_historical_runtime_keeps_its_verified_regenerable_disposition(self): + blob=dict(sha256='a'*64,bytes=12) + edge=dict(name='runtime-files/0',blob=blob,population='unknown',role='historical_runtime', + preservation='regenerable',required_for=['scientific_inspection','execution_proof']) + doc=dict(payloads=[edge],publication=None,run=dict(execution_kind='external_recipe',material=dict( + runtime=dict(files={'/historical/runtime/dependency.py':copy.deepcopy(blob)}),host_files={}))) + record=dict(family='run_history',id=self.catalog+'.'+uuid.uuid4().hex,document=doc) + policy=self.policies([declaration(blob['sha256'],blob['bytes']),record]) + self.assertEqual(policy[blob['sha256']],'regenerable') + + def test_historical_runtime_requires_its_exact_recorded_inventory_and_role(self): + blob=dict(sha256='a'*64,bytes=12) + for mutation in ('missing-inventory','different-hash','different-bytes','different-role','explicit-unique'): + edge=dict(name='runtime-files/0',blob=copy.deepcopy(blob),population='unknown',role='historical_runtime', + preservation='regenerable',required_for=['scientific_inspection','execution_proof']) + runtime=copy.deepcopy(blob) + if mutation=='different-hash':runtime['sha256']='b'*64 + if mutation=='different-bytes':runtime['bytes']+=1 + if mutation=='different-role':edge['role']='checkpoint' + if mutation=='explicit-unique':edge['preservation']='unique' + doc=dict(payloads=[edge],publication=None,run=dict(execution_kind='external_recipe',material=dict( + runtime=dict(files={} if mutation=='missing-inventory' else {'/historical/file':runtime}),host_files={}))) + record=dict(family='run_history',id=self.catalog+'.'+uuid.uuid4().hex,document=doc) + with self.subTest(mutation=mutation): + self.assertEqual(self.policies([declaration(blob['sha256'],blob['bytes']),record])[blob['sha256']],'unique') + + def test_historical_runtime_alias_cannot_weaken_outputs_captures_proofs_or_host_sources(self): + blob=dict(sha256='a'*64,bytes=12) + for use in ('output','capture','publication','host','other-payload-first','other-payload-last'): + edge=dict(name='runtime-files/0',blob=copy.deepcopy(blob),population='unknown',role='historical_runtime', + preservation='regenerable',required_for=['scientific_inspection','execution_proof']) + doc=dict(payloads=[edge],publication=None,run=dict(execution_kind='external_recipe',artifacts=[],material=dict( + runtime=dict(files={'/historical/file':copy.deepcopy(blob)}),host_files={},captures={}))) + if use=='output':doc['run']['artifacts']=[dict(name=edge['name'],**blob)] + if use=='capture':doc['run']['material']['captures']={'capture.txt':copy.deepcopy(blob)} + if use=='publication':doc['publication']=dict(source=copy.deepcopy(blob),result=None) + if use=='host':doc['run']['material']['host_files']={'owned.py':blob['sha256']} + if use.startswith('other-payload'): + other=dict(copy.deepcopy(edge),name='weights.bin',role='checkpoint') + doc['payloads'].insert(0 if use.endswith('first') else 1,other) + record=dict(family='run_history',id=self.catalog+'.'+uuid.uuid4().hex,document=doc) + with self.subTest(use=use): + self.assertEqual(self.policies([declaration(blob['sha256'],blob['bytes']),record])[blob['sha256']],'unique') + + def test_retained_run_observation_preserves_the_same_historical_runtime_policy(self): + blob=dict(sha256='a'*64,bytes=12) + edge=dict(name='runtime-files/0',blob=blob,population='unknown',role='historical_runtime', + preservation='regenerable',required_for=['scientific_inspection','execution_proof']) + doc=dict(payloads=[edge],publication=None,run=dict(execution_kind='external_recipe',material=dict( + runtime=dict(files={'/historical/file':copy.deepcopy(blob)}),host_files={}))) + record=dict(family='run_observation',id='b'*64,document=dict(observation=doc)) + self.assertEqual(self.policies([declaration(blob['sha256'],blob['bytes']),record])[blob['sha256']],'regenerable') + def test_native_measured_bytes_win_over_regenerable_runtime_alias_and_missing_declaration_blocks(self): sha=hashlib.sha256(b'').hexdigest() records,_,_=runtime_records(self.catalog,shared_sha=sha,shared_bytes=0)