diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d8f3d880..51529b6b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,6 +49,10 @@ jobs: - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_pointclouds.py - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_volume_assets.py - run: python -m unittest discover -s tests + - run: node --check static/workspace-projects.js + - run: node tests/browser_workspace_project.cjs + env: + BROWSER: /usr/bin/google-chrome - run: node --check static/workbench.js - run: node --check static/audio-editor.js - run: node --check static/saved-selections.js diff --git a/README.md b/README.md index 3ff53395..ebeef7c4 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ current record; saving/reviewing and freezing a new release leaves old results unchanged. Viewing test results records durable exposure; reused records or pixels cannot be presented as fresh holdout evidence. -Fresh catalogs use schema 4. Existing PostgreSQL schema-1, schema-2 or schema-3 catalogs require an +Fresh catalogs use schema 5. Existing PostgreSQL schema-1 through schema-4 catalogs require an explicit stopped-writer backup and upgrade, never an automatic startup change: ```sh @@ -852,4 +852,8 @@ fixtures and do not download model weights or call paid inference providers. ### Point-cloud and volume workflows -In Data, import `points.ply` with `points.json`, or `volume.npy` with `volume.json`. The bounded profiles are [ordered ASCII PLY points](docs/contracts/point-cloud-data.md) and [uint8 NPY1 volumes](docs/contracts/volume-data.md). Inspect native points or index slices/physical coordinates, save human review, select whole assets and freeze a canonical release. In Pipeline, create and edit the corresponding validation/export recipe; Runs retains its integrity report and the unchanged release. Classification releases also offer validation/export without installing a trainer. Existing catalogs require the explicit schema4 upgrade described in the storage runbook; the application never silently migrates them. [Accepted workspace scope, evidence and limitations](docs/plans/dataset-centered-workspaces/reports/final-20261013.md). +In Data, import `points.ply` with `points.json`, or `volume.npy` with `volume.json`. The bounded profiles are [ordered ASCII PLY points](docs/contracts/point-cloud-data.md) and [uint8 NPY1 volumes](docs/contracts/volume-data.md). Inspect native points or index slices/physical coordinates, save human review, select whole assets and freeze a canonical release. In Pipeline, create and edit the corresponding validation/export recipe; Runs retains its integrity report and the unchanged release. Classification releases also offer validation/export without installing a trainer. Existing catalogs require the explicit schema5 upgrade described in the storage runbook; the application never silently migrates them. [Accepted workspace scope, evidence and limitations](docs/plans/dataset-centered-workspaces/reports/final-20261013.md). + +## Open a local project + +Start with `python app.py --data ./catalog --project-root research=/absolute/research`, then select **Open project** in `/workspaces`. A [project manifest](docs/contracts/projects.md) references files beneath that explicitly selected root. Data shows sources and missing requirements, Pipeline shows recipe references, and Runs shows external evidence. This stage is **inspection only**: no project code, model loading, runtime probing or training executes. Existing catalog workflows remain separate. Opening registers a snapshot in Tuldok; it does not modify the opened project. diff --git a/app.py b/app.py index 02329e03..31bf8584 100644 --- a/app.py +++ b/app.py @@ -26,10 +26,12 @@ import meshes import spatial_assets import pipeline +import projects import runs import base64 import hashlib import io +import ipaddress import json import math import shutil @@ -133,7 +135,7 @@ def validate_annotation(body): class Dataset: - def __init__(self, path, *, dsn=None): + def __init__(self, path, *, dsn=None, project_roots=None): self.path = Path(path).resolve() self.path.mkdir(parents=True, exist_ok=True) (self.path / 'images').mkdir(exist_ok=True) @@ -142,6 +144,7 @@ def __init__(self, path, *, dsn=None): self.db = Metadata(self.path, dsn=dsn) self.lock = self.db.lock try: + self.projects = projects.Projects(self, project_roots) self.generation_jobs = synthetic.Jobs(self) with self.lock: self._recover_deletions() @@ -158,6 +161,8 @@ def __init__(self, path, *, dsn=None): self.pipelines = pipeline.Pipelines(self) self.runs = runs.Runs(self, self.pipelines) except BaseException: + if hasattr(self, 'projects'): + self.projects.close() self.db.close() raise @@ -167,6 +172,7 @@ def close(self): self.caption_proposals.close() self.grounded.close() self.generation_jobs.close() + self.projects.close() self.db.close() def rows(self): @@ -402,11 +408,26 @@ def reply(self, data, status=200, content_type='application/json'): except (BrokenPipeError, ConnectionResetError): pass + def project_access(self): + # This opt-in file inspection surface is local-only until actor authentication. + host = urlsplit('http://' + self.headers.get('Host', '')).hostname + if (host not in ('localhost', '127.0.0.1', '::1') or + not ipaddress.ip_address(self.client_address[0]).is_loopback): + raise workbench.WorkbenchError('Project inspection requires a local loopback client and host.', 'forbidden', 403) + def do_GET(self): path = urlsplit(self.path).path try: + if path == '/api/workspace/projects' or path.startswith('/api/workspace/projects/'): + self.project_access() if path.startswith('/api/workspace/'): parts = path.removeprefix('/api/workspace/').split('/') + if parts == ['projects']: + return self.reply(dataset.projects.list()) + if len(parts) == 2 and parts[0] == 'projects': + return self.reply(dataset.projects.get(parts[1])) + if len(parts) == 4 and parts[0] == 'projects' and parts[2] == 'evidence': + return self.reply(dataset.projects.evidence(parts[1], parts[3])) if parts == ['context']: return self.reply({'version': 1, 'mode': 'real', 'dataset_id': dataset.db.dataset_id, 'recipe': 'image_classification_v1', 'execution': 'optional_local_cpu'}) @@ -533,6 +554,7 @@ def do_GET(self): assets = {'/native-detection-import.js': ('native_detection_import.js', 'text/javascript'), '/saved-searches.js': ('saved-searches.js', 'text/javascript'), '/sequence-batch.js': ('sequence-batch.js', 'text/javascript'), '/pumas-typed.js': ('pumas-typed.js', 'text/javascript'), '/pumas-gateways.js': ('pumas-gateways.js', 'text/javascript'), '/meshes.js': ('meshes.js', 'text/javascript'), '/sequences.js': ('sequences.js', 'text/javascript'), '/text-classification-proposals.js': ('text-classification-proposals.js', 'text/javascript'), '/caption-proposals.js': ('caption-proposals.js', 'text/javascript'), '/curation.js': ('curation.js', 'text/javascript'), '/caption-import.js': ('caption_import.js', 'text/javascript'), '/bulk-import.js': ('bulk_import.js', 'text/javascript'), '/saved-selections.js': ('saved-selections.js', 'text/javascript'), '/workbench': ('workbench.html', 'text/html'), '/workbench.js': ('workbench.js', 'text/javascript'), '/workbench.css': ('workbench.css', 'text/css'), '/': ('index.html', 'text/html'), '/app.js': ('app.js', 'text/javascript'), '/style.css': ('style.css', 'text/css'), '/rights-note.js': ('rights-note.js', 'text/javascript'), '/preferences.js': ('preferences.js', 'text/javascript'), '/instruction-responses.js': ('instruction-responses.js', 'text/javascript'), '/native-text-import.js': ('native_text_import.js', 'text/javascript')} # Opt-in workspace shell keeps fixture and real service boundaries distinct. assets.update({ + '/workspace-projects.js': ('workspace-projects.js', 'text/javascript'), '/pointclouds.js': ('pointclouds.js', 'text/javascript'), '/volume-editor.js': ('volume-editor.js', 'text/javascript'), '/audio-editor.js': ('audio-editor.js', 'text/javascript'), @@ -573,6 +595,8 @@ def do_POST(self): raise ValueError('Send JSON.') length = int(self.headers.get('Content-Length', '0')) path = urlsplit(self.path).path + if path == '/api/workspace/projects' or path.startswith('/api/workspace/projects/'): + self.project_access() limit = native_detection_import.MAX_PREPARE_REQUEST if path == '/api/workbench/native-detection-import/prepare' else native_detection_import.MAX_REQUEST if path == '/api/workbench/native-detection-import/row' else saved_searches.MAX_REQUEST if path == '/api/workbench/searches' or path.startswith('/api/workbench/searches/') else meshes.MAX_REQUEST if path == '/api/workbench/mesh-import' else rheon_sequences.MAX_REQUEST if path in ('/api/workbench/sequence-import', '/api/workbench/sequence-import-item') else MAX_BODY if path in ('/api/workbench/pointcloud-import', '/api/workbench/volume-import'): limit = spatial_assets.MAX_REQUEST @@ -581,12 +605,14 @@ def do_POST(self): if not 0 < length <= limit: raise ValueError('Request is too large or empty.') raw_body = self.rfile.read(length) - body = spatial_assets.parse_request(raw_body) if path in ('/api/workbench/pointcloud-import', '/api/workbench/volume-import') else native_detection_import.parse_request(raw_body) if path in ('/api/workbench/native-detection-import/prepare', '/api/workbench/native-detection-import/row') else saved_searches.parse_request(raw_body) if path == '/api/workbench/searches' or path.startswith('/api/workbench/searches/') else sequence_batch_import.parse_request(raw_body) if path in ('/api/workbench/sequence-import', '/api/workbench/sequence-import-item') else meshes.parse_json(raw_body) if path == '/api/workbench/mesh-import' else json.loads(raw_body) + body = projects.decode(raw_body) if path == '/api/workspace/projects' else spatial_assets.parse_request(raw_body) if path in ('/api/workbench/pointcloud-import', '/api/workbench/volume-import') else native_detection_import.parse_request(raw_body) if path in ('/api/workbench/native-detection-import/prepare', '/api/workbench/native-detection-import/row') else saved_searches.parse_request(raw_body) if path == '/api/workbench/searches' or path.startswith('/api/workbench/searches/') else sequence_batch_import.parse_request(raw_body) if path in ('/api/workbench/sequence-import', '/api/workbench/sequence-import-item') else meshes.parse_json(raw_body) if path == '/api/workbench/mesh-import' else json.loads(raw_body) if not isinstance(body, dict): raise ValueError('Expected an object.') path = urlsplit(self.path).path if path.startswith('/api/workspace/'): parts = path.removeprefix('/api/workspace/').split('/') + if parts == ['projects']: + return self.reply(dataset.projects.open(body), 201) if parts == ['graphs']: return self.reply(dataset.pipelines.create(body), 201) if len(parts) == 2 and parts[0] == 'graphs': @@ -742,10 +768,18 @@ def main(): parser.add_argument('--data', type=Path, default=ROOT / 'data') parser.add_argument('--host', default='127.0.0.1') parser.add_argument('--port', type=int, default=8091) + parser.add_argument('--project-root', action='append', default=[], metavar='ALIAS=PATH', + help='Allow read-only project inspection beneath a local directory; repeat for multiple roots.') args = parser.parse_args() + roots = {} + for binding in args.project_root: + alias, separator, location = binding.partition('=') + if not separator or not location or alias in roots: + parser.error('Each project root needs a unique ALIAS=PATH.') + roots[alias] = location try: - dataset = Dataset(args.data) - except StorageError as error: + dataset = Dataset(args.data, project_roots=roots) + except (StorageError, workbench.WorkbenchError) as error: parser.exit(2, str(error) + "\n") server = ThreadingHTTPServer((args.host, args.port), make_handler(dataset)) print('Tuldok: http://' + args.host + ':' + str(server.server_port), flush=True) diff --git a/docs/contracts/projects.md b/docs/contracts/projects.md new file mode 100644 index 00000000..47c0be29 --- /dev/null +++ b/docs/contracts/projects.md @@ -0,0 +1,119 @@ +# Project inspection v1 + +Owner: `projects.py:Projects`; PostgreSQL schema5 owns immutable registration +snapshots. HTTP and the workspace UI project the same owner. This first stage +inspects references only: project files are not imported as Data records, +recipes are not executable graphs, and external evidence is not a Tuldok Run. + +## Opening and reference schema + +Configure selected directories locally when starting the application: + +```sh +python app.py --data ./catalog --project-root research=/absolute/research +``` + +Then choose **Open project** in `/workspaces`, select `research` and enter the +manifest's relative path. Root aliases use 1–64 letters/digits/dot/dash/underscore +and start with a letter or digit. The server does not accept browser-supplied +absolute root paths. Project endpoints require loopback client addresses and +loopback Host names; existing broader application authentication is unchanged. +Only expose directories whose referenced files local application clients may +read. Project opening grants no actor/execution authority. + +An external manifest can live beside an existing source checkout, so opening +does not require adding files to that checkout. All file references are relative +to the configured root, **not** the manifest's directory: + +```json +{ + "schema_version": 1, + "id": "8bd3793c538d4a0bba1d1e0c675a86d3", + "name": "Pressure experiment", + "data": [{"id": "capture", "label": "Trajectory capture", "path": "lab/runs/capture.npz"}], + "recipes": [{"id": "driver", "label": "Experiment driver", "path": "lab/scripts/trajectory_experiment.py"}], + "evidence": [{"id": "freeze", "label": "Frozen protocol", "path": "lab/experiments/trajectory-v2/freeze.json"}], + "requirements": [{"id": "torch", "label": "Compatible PyTorch environment"}] +} +``` + +Every top-level field shown is required; unknown fields/versions are refused. +Use a new 32-character lowercase hex ID per independent project snapshot. +Names are at most120 characters. Each section has at most64 entries with unique +IDs (same alias syntax) within that section and labels of at most160 characters. +Data/recipe/evidence entries require `id,label,path` and optionally a lowercase +64-character `sha256`. Requirements require `id,label` and optionally `path`. +Requirements are declarations: file presence does not establish interpreter, +package, device, version or scientific compatibility. No subprocess probe, +module import, checkpoint deserialization or dependency installation occurs. + +Manifest input is at most64KiB of strict UTF-8 JSON. Duplicate keys, nonfinite +numbers, lone surrogates, unsupported fields and unsafe paths fail before +registration. Paths are at most512 characters and16 components; absolute paths, +backslashes, empty/dot/parent components and control characters are refused. +The registered snapshot includes the parsed manifest and SHA256 of its original +bytes, selected root alias and root device/inode identity. Reopening identical +input returns that registration. A changed manifest or root for the same ID +returns409; there is no replacement/delete/edit operation in this stage. + +## Filesystem and evidence boundary + +Startup pins open directory handles for configured roots. Inspection walks +directory descriptors with no-follow flags, refusing symlinks at every component, +including final files. Only regular files are read; FIFOs/devices/directories +cannot block the file reader. Root selection itself is a trusted local startup +action and resolves the configured directory. All project access is read-only; +opening writes only Tuldok's catalog registration. + +Inspection hashes at most8MiB per referenced file and64MiB total per detail +request. Larger regular files are reported present with unverified contents. +Bounded reads check size and modification identity before/after and refuse +observed changes. Presence, measured byte hash, equality to a declared hash and +scientific verification are distinct. `verified` means only equality to the +manifest's declared SHA256, not source authenticity or scientific correctness. + +Text evidence previews permit only `.json`, `.txt` and `.md`, at most64KiB of +UTF-8 without binary control characters. The response is JSON text, never active +HTML/Markdown. An expected hash mismatch refuses preview. Other files may be +listed/hashed but cannot be parsed or downloaded through this first-stage API. + +## HTTP operations and observation + +- `GET /api/workspace/projects` returns configured alias names and registered + snapshot summaries. No absolute root path or credential is returned. +- `POST /api/workspace/projects` accepts exactly `{root,manifest_path}` and + registers a valid immutable snapshot, returning detail. A lost reply can be + reconciled by listing; repeating identical opening is idempotent. +- `GET /api/workspace/projects/ID` inspects the saved references. It reports + manifest `unchanged`, `changed`, `missing` or `unavailable`; a changed source + never replaces the saved references implicitly. +- `GET /api/workspace/projects/ID/evidence/REFERENCE_ID` returns bounded external + text plus its hash and `scientific_verification: not_performed`. + +Detail is always `mode: inspection`, `execution: unsupported`. Referenced file +states are `available`, `missing`, `unavailable`, with separate integrity +`verified`, `mismatch`, `unverified`. Requirements may be `present_unqualified`, +`missing`, `unavailable`, `unverified`. Paths with missing files leave the rest +of the project usable. Removing/rebinding a configured root leaves the snapshot +visible but unavailable; matching alias alone cannot silently redirect reads. + +Browser source selection remembers `project:ID` and Data/Pipeline/Runs navigation +without selecting the current catalog as that project's data. Switching projects +does not move catalog records or alter existing dirty editors. External project +mode cannot submit training. Its three panels show file references, recipe +references and external evidence respectively. + +## Persistence and compatibility + +Schema5 adds `workspace_projects(id TEXT PRIMARY KEY, document TEXT NOT NULL)`. +Schema1–4 definitions and stored bytes are retained exactly; use the explicit +quiesced [upgrade/backup procedure](workspace-storage.md). Complete backup +preserves registration rows, but does not copy external project roots. Back up +external research files separately. Restoring on a different filesystem leaves +root identity unavailable; explicit rebinding/migration is future work, not a +silent reassignment. Existing catalog-only startup requires no project roots. + +Schema version, project manifest version and later scientific/recipe versions +have independent owners. Runtime requirements are declared file references in +v1; later qualified execution must use its own admitted runtime/permission +contract rather than treating presence as capability. diff --git a/docs/contracts/workspace-storage.md b/docs/contracts/workspace-storage.md index e17a9a51..c216c8e7 100644 --- a/docs/contracts/workspace-storage.md +++ b/docs/contracts/workspace-storage.md @@ -40,8 +40,8 @@ upgrade. ## Upgrade an existing catalog -Fresh catalogs use schema 4. Ordinary application startup **does not migrate** -an existing schema-1, schema-2 or schema-3 catalog. It refuses with instructions to run the +Fresh catalogs use schema 5. Ordinary application startup **does not migrate** +an existing schema-1 through schema-4 catalog. It refuses with instructions to run the explicit upgrade. Stop every application, provider/training worker and artifact writer; keep them stopped throughout backup, upgrade and verification. The catalog advisory lock coordinates cooperating commands, but cannot fence an old binary @@ -57,7 +57,7 @@ python app.py --data ./dataset The backup destination must be new. Keep the complete backup and its receipt. The accepted transitions have stable identities `workspace-metadata-1-to-2`, -`audio-metadata-2-to-3` and `spatial-metadata-3-to-4`. The command applies the ordered steps needed by the +`audio-metadata-2-to-3`, `spatial-metadata-3-to-4` and `project-metadata-4-to-5`. The command applies the ordered steps needed by the recognized source, with the complete chain in one transaction. It verifies: 1. The manifest, catalog identity, version ledger and exact retained definition. @@ -74,26 +74,26 @@ recognized source, with the complete chain in one transaction. It verifies: All DDL and version ledger updates commit in one transaction under the existing dataset advisory lock. A failed second step rolls back the first step as well; the old kind constraint is never authoritatively absent. Only the known version-1, -version-2 or version-3 definitions are eligible. Unknown versions, altered constraints, +version-2, version-3 or version-4 definitions are eligible. Unknown versions, altered constraints, indexes, tables or triggers, malformed ledgers, missing catalogs and contradictory identities fail closed. No command refreshes a signature to bless unsupported drift. The manifest remains version 1; manifest and SQL schema versions are different contracts. The command prints the dataset identity, final migration name -(`spatial-metadata-3-to-4`), final schema version, definition/structure hashes and +(`project-metadata-4-to-5`), final schema version, definition/structure hashes and `upgraded` or `already_current`. A repeat on a -known schema-4 catalog verifies its structure and leaves rows/files unchanged. +known schema-5 catalog verifies its structure and leaves rows/files unchanged. Run one upgrade/transfer command at a time. A command failure rolls back its transaction. A lost commit acknowledgement is explicitly an unknown outcome: keep writers stopped, reopen, and rerun this same command to verify whether the -catalog is still the known source version or already known schema 4. Never +catalog is still the known source version or already known schema 5. Never manually change the ledger, delete a table or replay a training request to resolve it. ## Backup, restore and legacy imports The backup container remains `format_version: 1` and explicitly records -`schema_version: 1`, `2`, `3` or `4`. `backup_metadata.py` accepts each recognized schema. +`schema_version: 1`, `2`, `3`, `4` or `5`. `backup_metadata.py` accepts each recognized schema. Restore recreates **the exact backed-up schema version**, identity, rows, sequence states and ordinary files in an absent catalog/new directory. A retained schema-1, schema-2 or schema-3 backup remains restorable; upgrade that restored copy explicitly @@ -106,11 +106,11 @@ specified in [local storage and recovery](local-storage.md). `Metadata(..., schema_version=1)`, `schema_version=2`, `schema_version=3` and `schema_version=None` are tooling entry points for exact old-version restoration and recognized-version inspection. -Normal Dataset and Workbench callers omit that argument and require schema 4. +Normal Dataset and Workbench callers omit that argument and require schema 5. Version selection never authorizes an unsupported definition or structural drift. Legacy SQLite preflight still recognizes only the original 19 owner tables. -A new import creates schema 4 and leaves the three workspace tables empty. +A new import creates schema 5 and leaves workspace/project tables empty. A retained schema-1 or schema-2 migration receipt remains byte-for-byte authoritative: an interrupted import continues at its receipt's version, and a completed import can be reconciled after an explicit upgrade only while its original rows/files still @@ -119,7 +119,7 @@ never rewritten to imply that later workspace or audio writes belonged to the ol There is no automatic downgrade. Before any new writes, retain and restore the complete pre-upgrade backup to a separate destination/database if returning to -the older binary is necessary. After new writes, retain the schema-4 catalog and +the older binary is necessary. After new writes, retain the schema-5 catalog and artifacts and reconcile/export them under an explicit recovery decision; an old backup cannot represent those later writes. @@ -146,3 +146,7 @@ verifies unchanged graph, record, completed run, report and release bytes after explicit upgrade, schema4 backup/restore, and schema3 backup/restore followed by explicit upgrade. Both backup trees remain unchanged. CI checks out that exact legacy revision and retains the qualification report with QA artifacts. + +## Project registration (schema5) + +Schema5 adds only `workspace_projects(id TEXT PRIMARY KEY, document TEXT NOT NULL)` for immutable project manifest snapshots. The schema4 definition remains exactly `52242c56f7b88144157b66d93b151d818cecf0b2219226d4aff5734e8a1adcef`. Schema5 backup/restore includes project rows but does not copy external project files or grant a new filesystem binding. See [project inspection](projects.md). Tests retain exact v4 backups and prove the explicit v4→5 transition, transactional rollback and restored project snapshots alongside old domain rows. The retained-run qualifier now targets schema5 while preserving its actual historical schema3 source and backup. diff --git a/docs/plans/generic-scientific-projects/execution-ledger.md b/docs/plans/generic-scientific-projects/execution-ledger.md index df0a172e..93301920 100644 --- a/docs/plans/generic-scientific-projects/execution-ledger.md +++ b/docs/plans/generic-scientific-projects/execution-ledger.md @@ -21,3 +21,15 @@ Verification results and independent review disposition are appended after compl ## 2026-10-13 — publication and implementation admission User approved the presented plan and staged implementation (Sentinel_d7dcd8c1780481919cb7f76d7475632e). Operation: continue `docs/plans/generic-scientific-projects/plan.md`. Publish plan first, then P1 on a separate branch; no merge/deploy. Remote PR inventory still shows PR38–43 as open drafts; refreshed main remains the baseline pending inspection. Preserve their history and qualify this stack independently. + +## 2026-10-13 — P1 implementation and local qualification + +Plan published first as PR44 at fd3f6a93b9c7f2d7d8056298a02d3f9b362e9b3c. P1 branch feature/generic-project-opening-20261013 descends from it. Root owns project service/API/docs; mesh_ui owned four UI files; volume_assets owned schema5/upgrade and bounded storage tests; recipe_design owned focused project tests; point_assets owned the browser gate; mesh_review independently reviewed source. Shared services were integrated serially. + +Implemented immutable generic project registration, explicit root binding, bounded read-only source/evidence inspection, missing requirements, three workspace navigation and retained dataset-only behavior. Schema5 adds one project table with exact old schema definitions retained. Local-only client/Host checks protect the opt-in filesystem surface before P3 actor policy. Opening never executes source code, probes a runtime or loads a model. + +649 full Python tests pass (412.564s, two optional-reader skips); independent point/volume reader suites pass20 tests with no skips. Affected schema/migration/worker subset71 passes, focused project11 passes independently, existing workspace browser18 checks passes, generic and actual-lab-copy project browsers pass offline with12 transport canaries each. Retained historical schema3 Run and exact schema4 backups survive explicit schema5 evolution. See [P1 evidence](reports/p1-verification.md). No runtime type-checker or Standards Engine certificate is claimed. + +Independent review fixes: reject lone surrogate input; return bytes matching actual hashed read; explain immutable changed-manifest snapshots honestly; clean root handles on malformed internal configuration. Final source review reports no remaining blocker. Hosted exact-head checks remain pending until publication. + +Next-stage research found no raw lab capture in the checkout/releases and no native toolchain initially configured. Evaluation NPZ omits RHS; training NPZ omits corrected fields. A separately isolated, pinned native two-frame capture is being prepared as the smallest P2 contract observation; no training or model execution. It is new evidence, not reconstruction of historical artifacts. diff --git a/docs/plans/generic-scientific-projects/plan.md b/docs/plans/generic-scientific-projects/plan.md index 5ca5f860..567a91ca 100644 --- a/docs/plans/generic-scientific-projects/plan.md +++ b/docs/plans/generic-scientific-projects/plan.md @@ -1,12 +1,12 @@ # Plan: Generic scientific projects and agent operations -**Plan status:** `Planned` +**Plan status:** `Active` -**Current phase:** Plan publication authorized; P1 implementation follows on a separate feature branch. +**Current phase:** P1 locally verified on feature/generic-project-opening-20261013; final hosted CI pending. Plan published as PR44. -**Next slice:** P1 — open a generic project and inspect referenced evidence without executing code. +**Next slice:** P1 — qualify the published project-opening implementation through final CI before P2 admission. -**Acceptance status:** `pending` +**Acceptance status:** `partial` **Execution ledger:** [execution-ledger.md](execution-ledger.md) @@ -32,7 +32,7 @@ The user approved publishing this plan and staged implementation on 2026-10-13. | ID | Observable criterion | Kind | Environment | Mode | Status | Evidence | | --- | --- | --- | --- | --- | --- | --- | -| G1 | Open/reopen a local project, inspect available/missing artifacts and runtime requirements, navigate all three workspaces without executing or modifying repository code | user-workflow | representative | automated | pending | P1 browser and subprocess/filesystem observation | +| G1 | Open/reopen a local project, inspect available/missing artifacts and runtime requirements, navigate all three workspaces without executing or modifying repository code | user-workflow | representative | automated | satisfied | [P1 observed workflow](reports/p1-verification.md) | | G2 | Scientific fields preserve exact float bytes, dimensions, staggered placement, units, time and trajectory identity through import, freeze, download and cold read | contract | representative | automated | pending | P2 independent NumPy reader and actual lab samples | | G3 | An optional Rheon adapter exposes bounded generation controls and produces real generic scientific data; removing it leaves generic project/data use intact | integration | required-real | automated | pending | P4 actual native generator plus no-adapter startup | | G4 | Saved graph executes existing lab fitting and autonomous evaluation scripts; verified checkpoint, full-field comparisons and physics diagnostics survive restart with complete provenance | system | required-real | automated | pending | P5 lab verifier, numerical field comparison and new execution receipts | @@ -166,13 +166,13 @@ All statuses below describe proposed work under the planning-only boundary. Befo **Goal:** Runnable compact project selector/header and existing three-workspace navigation, with honest external evidence and missing-dependency states. -**Allowed write set:** proposed `projects.py`, `docs/contracts/projects.md`, `tests/test_projects.py`, `tests/browser_workspace_project.cjs`; existing `app.py`, `metadata.py`, `static/workspace-real.js`, `static/workspaces.js`, `static/workspaces.css`, `static/workspaces.html`, `static/workspace-service.js`, workspace contract tests; this plan directory. Project registration in existing metadata requires explicit upgrade/restore coverage; do not introduce sidecar database authority. +**Allowed write set:** `README.md`, `docs/contracts/workspace-storage.md`, `upgrade_metadata.py`, `tests/test_spatial_storage.py`, `tests/qualify_spatial_retained_runs.py`, `.github/workflows/tests.yml`, `static/workspace-projects.js`; proposed `projects.py`, `docs/contracts/projects.md`, `tests/test_projects.py`, `tests/browser_workspace_project.cjs`; existing `app.py`, `metadata.py`, `static/workspace-real.js`, `static/workspaces.js`, `static/workspaces.css`, `static/workspaces.html`, `static/workspace-service.js`, workspace contract tests; this plan directory. Project registration in existing metadata requires explicit upgrade/restore coverage; do not introduce sidecar database authority. **Tasks:** Define manifest/reference boundaries; add read-only validation/inspection and explicit registration/opening; display project context and externally supplied receipt metadata as unverified until checked; preserve ordinary dataset-only entry; handle missing runtime/artifacts without execution. **Acceptance gate:** G1; real desktop/narrow/keyboard walkthrough; no spawned project code, no source mutations, traversal/symlink rejection, cross-project isolation and restart; existing workspace navigation regressions pass. First slice does not parse arbitrary checkpoints or claim scientific results verified. -**Status:** `Planned` +**Status:** `Verifying` ### P2: Lossless scientific data and retained compatibility @@ -271,6 +271,6 @@ Plan publication uses `plan/generic-scientific-projects-20261013` from PR43 in t ## Final Acceptance -- Acceptance status: `pending` +- Acceptance status: `partial` - Deferred follow-ups: broader ASR, arbitrary scientific models/physics, distributed execution and GPU performance reproduction; each requires a separate concrete consumer/qualification decision. -- Final status: `Planned` +- Final status: `Active` diff --git a/docs/plans/generic-scientific-projects/reports/p1-verification.md b/docs/plans/generic-scientific-projects/reports/p1-verification.md new file mode 100644 index 00000000..5cd52913 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p1-verification.md @@ -0,0 +1,78 @@ +# P1 project opening verification + +## Delivered behavior + +Projects can be opened from explicitly configured local roots and inspected in +Data/Pipeline/Runs. Saved manifests are immutable PostgreSQL snapshots. Referenced +data, recipe files, external evidence and missing requirements remain distinct +from imported records, executable graphs and completed Tuldok Runs. No project +code, dependency probe or checkpoint deserializer is invoked. + +The implementation adds schema5 with an explicit quiesced upgrade. Schema1–4 +definition hashes remain exact. External roots are read-only, descriptor-bound +and local-client-only; every path component refuses symlinks, unsafe traversal +and special files. Hash inspection and text previews have explicit bounds. +Opening the same ID with different source/root returns a conflict. Removing a +root leaves the durable snapshot visible but unavailable. + +## Verification scope + +- 11 real PostgreSQL/HTTP project tests pass, independently rerun by the source + reviewer. They cover registration/reopen, unchanged source bytes/no execution, + immutable conflicts, changed manifests, absent/rebound roots, unsafe paths, + symlinks/FIFOs, malformed/deep/duplicate JSON, invalid Unicode, file/hash + budgets, evidence refusal and hostile Host rejection. +- 71 affected schema, migration, rollback, storage and worker tests pass. + The actual retained schema3 completed mesh Run survives explicit upgrade and + backup/restore with exact graph/run/report/release bytes. See the + [retained-run receipt](p1/retained-runs.json). +- Existing workspace browser regression passes all18 checks, retaining fixture + dirty guards, catalog editing/review, context restoration and responsive focus. +- New [generic browser receipt](p1/generic/receipt.json) passes its four workflow + checks. The [actual-lab inspection receipt](p1/actual-lab/receipt.json) passes + five, using exact copies of five actual files from lab revision + `e8a90bfef6b2955c502008ccc62443e29dd9f2ec`. Both verify two-project isolation, + browser/server restart, no graph/run/record submissions, and unchanged files. + Both run with the existing loopback-only transport injection and twelve + external-denial/local-roundtrip canaries. This proves the named ordinary libc + transport boundary, not an OS sandbox or denial of arbitrary syscall code. +- Python compilation, JavaScript syntax and scoped whitespace checks pass. + The repository uses runtime contract checks and these existing static checks; + no new language type-checking framework or certification is claimed. +- Adopted plan-contract validation passes. Source-based standards review covers + ownership/contracts, persistence/migration, filesystem security, lifecycle and + browser accessibility. Standards Engine is unavailable; no Engine certificate. + +Final local discovery passed649 tests in412.564s, with two optional independent-reader skips. Both reader suites then passed separately under the installed qualification interpreter:11 point-cloud and9 volume tests, zero skips. Hosted CI is pending publication; local results do not imply hosted success. +Local browser receipts identify the precommit tree as dirty honestly; the +[implementation file hashes](p1/source-files.json) bind their reviewed material +content. Documentation/publication changes do not imply those receipts came +from a clean committed checkout. + +## Visual and independent review + +The source reviewer `/root/mesh_review` found no remaining blocker after correcting +file-size/hash consistency, invalid Unicode handling and misleading changed- +manifest wording. It independently ran the11 project tests. Root reviewed the +desktop and narrow screenshots; project reference cards, status labels and +external text remain readable without horizontal overflow. Browser checks use +keyboard form submission and navigation; external markup is rendered as text. + +- [Actual source Data desktop](p1/actual-lab/project-data-1440.jpg) +- [Actual source Data narrow](p1/actual-lab/project-data-390.jpg) +- [Recipe references narrow](p1/actual-lab/project-pipeline-390.jpg) +- [External evidence desktop](p1/actual-lab/project-evidence-1440.jpg) +- [External evidence narrow](p1/actual-lab/project-evidence-390.jpg) + +## Limits and next prerequisite + +This qualifies project inspection, not execution or scientific data import. The +lab's checkpoint is only byte-inspected; its large result file is hashed but its +oversize text preview is honestly refused. Existing published scientific results +are external evidence, not reverified results. No new training or simulation was +performed. Runtime requirements remain declared/unqualified. + +P2 requires representative actual lab field captures before fixing the scientific +schema. Tracked model weights, freeze/results JSON and synthetic UI fixtures do +not supply those arrays. Later permissions/external recipes remain P3–P6 work; +this stage's local filesystem gate does not pretend to implement actor roles. diff --git a/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-data-1440.jpg b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-data-1440.jpg new file mode 100644 index 00000000..70b1da89 Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-data-1440.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-data-390.jpg b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-data-390.jpg new file mode 100644 index 00000000..8b7a46d9 Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-data-390.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-evidence-1440.jpg b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-evidence-1440.jpg new file mode 100644 index 00000000..8b1674c4 Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-evidence-1440.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-evidence-390.jpg b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-evidence-390.jpg new file mode 100644 index 00000000..914e7dee Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-evidence-390.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-pipeline-390.jpg b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-pipeline-390.jpg new file mode 100644 index 00000000..0b53a057 Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/project-pipeline-390.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/actual-lab/receipt.json b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/receipt.json new file mode 100644 index 00000000..dc12f80b --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p1/actual-lab/receipt.json @@ -0,0 +1,91 @@ +{ + "source_commit": "fd3f6a93b9c7f2d7d8056298a02d3f9b362e9b3c", + "dirty": true, + "command": "node tests/browser_workspace_project.cjs --lab-root ", + "qualification": "Inspection of exact copies of actual lab files only; no new training, execution or scientific verification", + "actual_lab_source": { + "commit": "e8a90bfef6b2955c502008ccc62443e29dd9f2ec", + "files": { + "experiments/trajectory-v2/freeze.json": "7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba", + "experiments/trajectory-v2/model.pt": "25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9", + "scripts/trajectory_experiment.py": "e1fd053f693730d39ded6a7ee50fd6e562bef37b2171bb7ed26acb770605e2f6", + "README.md": "cac636f10eebf862201eb472e4a592408091a0ead45c05a2d17681e96d05a16b", + "experiments/trajectory-v2/result.json": "abe2b6fa3e63ac38fd9ab3e397f682204a351d894f73b08fe21a7931cca5bb76" + } + }, + "project_file_hashes": { + "alpha.project.json": "3d16d9076dc38f2b05f586fd2b8e6a2e3b8a7bd2c2dee577c3d7e68bc12861e0", + "beta/data.txt": "37db6957d5b32141f37051d6ea915e32e7d84638a31abd5af8e8c0d180ac937e", + "beta/report.md": "1bbc8e85704e222c29f19d8a3edd0e0118b66969a08fc3568de5577b18fade38", + "beta.project.json": "a79cc0671db902a8c3b78f433e81fd213bfacb8a95928cbc933e7bacdb8533b3", + "copied/README.md": "cac636f10eebf862201eb472e4a592408091a0ead45c05a2d17681e96d05a16b", + "copied/experiments/trajectory-v2/freeze.json": "7132acb88fc8a0725c33f4d1d3534a2732c72a4e260b014538a6cf32c95b0aba", + "copied/experiments/trajectory-v2/model.pt": "25533a2fbaa5ab0f5fb574dc6307a2b8d6a686deb7db867d0d03338d4d70d5a9", + "copied/experiments/trajectory-v2/result.json": "abe2b6fa3e63ac38fd9ab3e397f682204a351d894f73b08fe21a7931cca5bb76", + "copied/scripts/trajectory_experiment.py": "e1fd053f693730d39ded6a7ee50fd6e562bef37b2171bb7ed26acb770605e2f6" + }, + "checks": [ + "Project opens through keyboard form; data hashes and missing/present-unqualified/unprobed requirements remain distinct", + "Actual copied 4MiB result is hash-inspected but bounded text preview is refused honestly", + "Pipeline exposes nonexecutable references; Runs renders exact external evidence as text without a completion claim", + "Two projects with shared reference ID stay isolated; switching clears previous external evidence", + "Browser reload and server restart retain project selection/catalog; all source/copy bytes unchanged and no execution submitted" + ], + "captures": [ + { + "file": "project-data-1440.jpg", + "width": 1440, + "height": 900, + "selector": "#project-data-root", + "qualification": "Copied actual source inspection only; no execution or scientific verification" + }, + { + "file": "project-data-390.jpg", + "width": 390, + "height": 900, + "selector": "#project-data-root", + "qualification": "Copied actual source inspection only; no execution or scientific verification" + }, + { + "file": "project-pipeline-390.jpg", + "width": 390, + "height": 900, + "selector": "#project-pipeline-root", + "qualification": "Copied actual source inspection only; no execution or scientific verification" + }, + { + "file": "project-evidence-1440.jpg", + "width": 1440, + "height": 900, + "selector": "#project-runs-root .project-evidence-output", + "qualification": "Copied actual source inspection only; no execution or scientific verification" + }, + { + "file": "project-evidence-390.jpg", + "width": 390, + "height": 900, + "selector": "#project-runs-root .project-evidence-output", + "qualification": "Copied actual source inspection only; no execution or scientific verification" + } + ], + "offline_canary": { + "qualification": "ordinary libc loopback-only transport", + "checks": [ + "connect denied 192.0.2.1", + "sendto denied 192.0.2.1", + "connect denied 2001:db8::1", + "sendto denied 2001:db8::1", + "resolution denied 192.0.2.1", + "resolution denied 2001:db8::1", + "resolution denied external-test.invalid", + "tcp local 127.0.0.1", + "udp local 127.0.0.1", + "tcp local ::1", + "udp local ::1", + "unix local" + ] + }, + "execution_submitted": false, + "source_mutation": false, + "browser": "Chrome/151.0.7922.173" +} \ No newline at end of file diff --git a/docs/plans/generic-scientific-projects/reports/p1/generic/project-data-1440.jpg b/docs/plans/generic-scientific-projects/reports/p1/generic/project-data-1440.jpg new file mode 100644 index 00000000..b1b1fedf Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/generic/project-data-1440.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/generic/project-data-390.jpg b/docs/plans/generic-scientific-projects/reports/p1/generic/project-data-390.jpg new file mode 100644 index 00000000..d680a0e5 Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/generic/project-data-390.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/generic/project-evidence-1440.jpg b/docs/plans/generic-scientific-projects/reports/p1/generic/project-evidence-1440.jpg new file mode 100644 index 00000000..9f2bfdf7 Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/generic/project-evidence-1440.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/generic/project-evidence-390.jpg b/docs/plans/generic-scientific-projects/reports/p1/generic/project-evidence-390.jpg new file mode 100644 index 00000000..3fbdddaa Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/generic/project-evidence-390.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/generic/project-pipeline-390.jpg b/docs/plans/generic-scientific-projects/reports/p1/generic/project-pipeline-390.jpg new file mode 100644 index 00000000..f2f91e4f Binary files /dev/null and b/docs/plans/generic-scientific-projects/reports/p1/generic/project-pipeline-390.jpg differ diff --git a/docs/plans/generic-scientific-projects/reports/p1/generic/receipt.json b/docs/plans/generic-scientific-projects/reports/p1/generic/receipt.json new file mode 100644 index 00000000..eac82b49 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p1/generic/receipt.json @@ -0,0 +1,79 @@ +{ + "source_commit": "fd3f6a93b9c7f2d7d8056298a02d3f9b362e9b3c", + "dirty": true, + "command": "node tests/browser_workspace_project.cjs", + "qualification": "Generic synthetic project UI/lifecycle only", + "actual_lab_source": null, + "project_file_hashes": { + "alpha/data.json": "b4ce3cdbbc38a800a5c94cb5a19c63a20d66f7d66e2725ecd36b8e7c2f879a70", + "alpha/recipe.py": "acc0fb68b3a717a405510704c11f511272a5ba0f07663c9a7332611d4a865855", + "alpha/report.md": "eb5a076586c59eabfffed64cd4447ac0cb6800d17094666c17b7f5a96516f6e5", + "alpha.project.json": "dc7b3206f1a9a4a710f2bf181baa7f5b693a7dc1446bded9bd00a507f1152149", + "beta/data.txt": "37db6957d5b32141f37051d6ea915e32e7d84638a31abd5af8e8c0d180ac937e", + "beta/report.md": "1bbc8e85704e222c29f19d8a3edd0e0118b66969a08fc3568de5577b18fade38", + "beta.project.json": "a79cc0671db902a8c3b78f433e81fd213bfacb8a95928cbc933e7bacdb8533b3" + }, + "checks": [ + "Project opens through keyboard form; data hashes and missing/present-unqualified/unprobed requirements remain distinct", + "Pipeline exposes nonexecutable references; Runs renders exact external evidence as text without a completion claim", + "Two projects with shared reference ID stay isolated; switching clears previous external evidence", + "Browser reload and server restart retain project selection/catalog; all source/copy bytes unchanged and no execution submitted" + ], + "captures": [ + { + "file": "project-data-1440.jpg", + "width": 1440, + "height": 900, + "selector": "#project-data-root", + "qualification": "Synthetic project UI/lifecycle fixture only" + }, + { + "file": "project-data-390.jpg", + "width": 390, + "height": 900, + "selector": "#project-data-root", + "qualification": "Synthetic project UI/lifecycle fixture only" + }, + { + "file": "project-pipeline-390.jpg", + "width": 390, + "height": 900, + "selector": "#project-pipeline-root", + "qualification": "Synthetic project UI/lifecycle fixture only" + }, + { + "file": "project-evidence-1440.jpg", + "width": 1440, + "height": 900, + "selector": "#project-runs-root .project-evidence-output", + "qualification": "Synthetic project UI/lifecycle fixture only" + }, + { + "file": "project-evidence-390.jpg", + "width": 390, + "height": 900, + "selector": "#project-runs-root .project-evidence-output", + "qualification": "Synthetic project UI/lifecycle fixture only" + } + ], + "offline_canary": { + "qualification": "ordinary libc loopback-only transport", + "checks": [ + "connect denied 192.0.2.1", + "sendto denied 192.0.2.1", + "connect denied 2001:db8::1", + "sendto denied 2001:db8::1", + "resolution denied 192.0.2.1", + "resolution denied 2001:db8::1", + "resolution denied external-test.invalid", + "tcp local 127.0.0.1", + "udp local 127.0.0.1", + "tcp local ::1", + "udp local ::1", + "unix local" + ] + }, + "execution_submitted": false, + "source_mutation": false, + "browser": "Chrome/151.0.7922.173" +} \ No newline at end of file diff --git a/docs/plans/generic-scientific-projects/reports/p1/retained-runs.json b/docs/plans/generic-scientific-projects/reports/p1/retained-runs.json new file mode 100644 index 00000000..4098d2e3 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p1/retained-runs.json @@ -0,0 +1,36 @@ +{ + "legacy_commit": "f1ecf7888a8dfa5b67ac8b087034226d533c682e", + "legacy_schema": 3, + "execution": "real mesh integrity, no training", + "upgrade": { + "schema_version": 5, + "state": "completed", + "run_id": "05e3fc9df8cd4f46b89200021dd3ad3f", + "artifacts": { + "validation-report.json": "eabc051de18a220e0557a23ea9b06f4c4ea2db7b85a8e7bb7c1498ea26f2fda0", + "artifact-manifest.json": "3177b5474f0521e0c3bb29b2ff86de7d8905977eb5ebdbc8d3850b1c1bfad892", + "release.zip": "9d149085679effb1b966d9ea96a5b253616310e7263c648cd64a830538d62e13" + } + }, + "current_schema_restore": { + "schema_version": 5, + "state": "completed", + "run_id": "05e3fc9df8cd4f46b89200021dd3ad3f", + "artifacts": { + "validation-report.json": "eabc051de18a220e0557a23ea9b06f4c4ea2db7b85a8e7bb7c1498ea26f2fda0", + "artifact-manifest.json": "3177b5474f0521e0c3bb29b2ff86de7d8905977eb5ebdbc8d3850b1c1bfad892", + "release.zip": "9d149085679effb1b966d9ea96a5b253616310e7263c648cd64a830538d62e13" + } + }, + "schema3_restore_then_upgrade": { + "schema_version": 5, + "state": "completed", + "run_id": "05e3fc9df8cd4f46b89200021dd3ad3f", + "artifacts": { + "validation-report.json": "eabc051de18a220e0557a23ea9b06f4c4ea2db7b85a8e7bb7c1498ea26f2fda0", + "artifact-manifest.json": "3177b5474f0521e0c3bb29b2ff86de7d8905977eb5ebdbc8d3850b1c1bfad892", + "release.zip": "9d149085679effb1b966d9ea96a5b253616310e7263c648cd64a830538d62e13" + } + }, + "original_backups_unchanged": true +} diff --git a/docs/plans/generic-scientific-projects/reports/p1/source-files.json b/docs/plans/generic-scientific-projects/reports/p1/source-files.json new file mode 100644 index 00000000..9d193dc8 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p1/source-files.json @@ -0,0 +1,15 @@ +{ + "projects.py": "0206fc4b39c5d0b754f40504496f30a3c876bc4dc9dc200b41dcea55445d0f1b", + "app.py": "51a484d33ff2d6d197442b18e0472d9eb83823e5d645a0b34ff486b62cfb000e", + "metadata.py": "074aff9bc4db7917d0764faf86c7794b0e7781d0522cbf502c25860212ac8985", + "upgrade_metadata.py": "62250ff2ce17f9a4e180f2795e78ac80a5b31d240c793cd0cb47dbf8adf90c63", + "static/workspace-projects.js": "29cd6447c2d67467c3139c5994ae117edb7099df0bd2b1eeeae33d607531152c", + "static/workspaces.js": "1ff5b05332792fe396f947653f4eb024ad1aef0cb4eb9892b77b22980d6fa30a", + "static/workspaces.html": "43a62a80c4031b959210611692004ba94f1e5393b0f2f21df1bd77555e51b93b", + "static/workspaces.css": "a4912a5702edb54e7a7f7cb8ca0c9f6d587974c6e2b6167466076d75de9b32b3", + "tests/test_projects.py": "5e8684683d43c75fc693d1ccfc6dc5d9ae94302d5a74c12d26dec9ca0f9c9264", + "tests/test_workspace_metadata.py": "6935818e9d2c9d35f675559997eb51e2c00862b034a99d3d85b032e0929a30ab", + "tests/test_spatial_storage.py": "5f06bc189f9b49ea470b46334978b19d464f5f9aba588505bde8302de62d58a7", + "tests/qualify_spatial_retained_runs.py": "3df98866035b5d02e51d048bdc4a80a31b4418ee32be74e7e806a9b19c75b2ab", + "tests/browser_workspace_project.cjs": "7e1b04e157d035e36491f1fb785a4a82d9c4a249c58260f10d186fbe8c800de0" +} diff --git a/metadata.py b/metadata.py index 19e79620..db1de0ac 100644 --- a/metadata.py +++ b/metadata.py @@ -19,8 +19,8 @@ from psycopg import sql from psycopg.conninfo import conninfo_to_dict -SCHEMA_VERSION = 4 -SUPPORTED_SCHEMA_VERSIONS = (1, 2, 3, 4) +SCHEMA_VERSION = 5 +SUPPORTED_SCHEMA_VERSIONS = (1, 2, 3, 4, 5) MANIFEST = '.tuldok.json' CATALOG_VERSION_DDL = '''CREATE TABLE catalog_version ( version INTEGER PRIMARY KEY, dataset_id TEXT NOT NULL, @@ -415,7 +415,7 @@ def close(self): def tables_for_version(version): if type(version) is not int or version not in SUPPORTED_SCHEMA_VERSIONS: raise StorageError('Unsupported PostgreSQL catalog version.') - return {1: TABLES_V1, 2: TABLES_V2, 3: TABLES_V3, 4: TABLES}[version] + return {1: TABLES_V1, 2: TABLES_V2, 3: TABLES_V3, 4: TABLES_V4, 5: TABLES}[version] def definition_hash(version=SCHEMA_VERSION): @@ -516,8 +516,15 @@ def definition_hash(version=SCHEMA_VERSION): id TEXT PRIMARY KEY, bundle BYTEA NOT NULL, metadata_json TEXT NOT NULL, bundle_bytes INTEGER NOT NULL CHECK(bundle_bytes > 0))""", } -TABLES = { +TABLES_V4 = { **TABLES_V3, **SPATIAL_TABLES, 'workbench_records': TABLES_V3['workbench_records'].replace( "'sequence','mesh','audio'", "'sequence','mesh','audio','pointcloud','volume'"), } + +# Retained schema 4 remains exact for old catalogs, receipts and explicit upgrades. +PROJECT_TABLES = { + 'workspace_projects': '''CREATE TABLE workspace_projects ( + id TEXT PRIMARY KEY, document TEXT NOT NULL)''', +} +TABLES = {**TABLES_V4, **PROJECT_TABLES} diff --git a/projects.py b/projects.py new file mode 100644 index 00000000..fe594728 --- /dev/null +++ b/projects.py @@ -0,0 +1,277 @@ +"""Read-only project inspection; catalog registration never executes project code.""" +from contextlib import contextmanager +import hashlib +import json +import os +from pathlib import Path +import re +import stat + +from workbench import WorkbenchError, encode + +ID = re.compile(r'[a-f0-9]{32}') +NAME = re.compile(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,63}') +SHA = re.compile(r'[a-f0-9]{64}') +MANIFEST_LIMIT = 64 * 1024 +FILE_LIMIT = 8 * 1024 * 1024 +INSPECTION_LIMIT = 64 * 1024 * 1024 +TEXT_LIMIT = 64 * 1024 +GROUPS = ('data', 'recipes', 'evidence', 'requirements') + + +def closed(value, required, optional=()): + if not isinstance(value, dict) or not set(required) <= set(value) or set(value) - set(required) - set(optional): + raise WorkbenchError('Invalid project fields.') + + +def text(value, label, limit=160): + if (not isinstance(value, str) or not value.strip() or len(value) > limit + or any(ord(c) < 32 or 0xD800 <= ord(c) <= 0xDFFF for c in value)): + raise WorkbenchError(label + ' must be bounded, nonempty text without control characters.') + return value + + +def relative_path(value): + text(value, 'Project-relative path', 512) + if '\\' in value or value.startswith('/') or any(p in ('', '.', '..') for p in value.split('/')): + raise WorkbenchError('Use a relative path with no traversal or empty components.') + if len(value.split('/')) > 16: + raise WorkbenchError('Project path is too deep.') + return value + + +def decode(raw): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate field') + result[key] = value + return result + def constant(_value): + raise ValueError('Nonfinite value') + try: + return json.loads(raw.decode('utf-8'), object_pairs_hook=pairs, parse_constant=constant) + except (ValueError, UnicodeError, RecursionError): + raise WorkbenchError('Project JSON must be bounded UTF-8 with unique fields and finite values.') from None + + +def manifest(value): + closed(value, ('schema_version', 'id', 'name', *GROUPS)) + if type(value['schema_version']) is not int or value['schema_version'] != 1: + raise WorkbenchError('Only project schema 1 is supported.', 'unsupported', 400) + if not isinstance(value['id'], str) or not ID.fullmatch(value['id']): + raise WorkbenchError('Project ID must be 32 lowercase hexadecimal characters.') + text(value['name'], 'Project name', 120) + for group in GROUPS: + entries = value[group] + if not isinstance(entries, list) or len(entries) > 64: + raise WorkbenchError('Each project section permits at most 64 references.') + seen = set() + for entry in entries: + closed(entry, ('id', 'label') if group == 'requirements' else ('id', 'label', 'path'), + ('path',) if group == 'requirements' else ('sha256',)) + key = entry['id'] + if not isinstance(key, str) or not NAME.fullmatch(key) or key in seen: + raise WorkbenchError('Reference IDs must be bounded and unique within their section.') + seen.add(key) + text(entry['label'], 'Reference label') + if 'path' in entry: + relative_path(entry['path']) + if 'sha256' in entry and (not isinstance(entry['sha256'], str) or not SHA.fullmatch(entry['sha256'])): + raise WorkbenchError('Expected SHA256 must be lowercase hexadecimal.') + return value + + +class Projects: + """One catalog's immutable project snapshots and explicitly configured root handles.""" + + def __init__(self, dataset, roots=None): + self.db, self.lock = dataset.db, dataset.lock + self.roots = {} + try: + for alias, location in (roots or {}).items(): + if not isinstance(alias, str) or not NAME.fullmatch(alias): + raise WorkbenchError('Project root aliases use 1–64 letters, digits, dot, dash or underscore.') + path = Path(location).expanduser().resolve(strict=True) + # Root selection is a local startup action, not a browser-supplied filesystem path. + fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + info = os.fstat(fd) + self.roots[alias] = (fd, [info.st_dev, info.st_ino]) + except (OSError, ValueError, TypeError, WorkbenchError): + self.close() + raise WorkbenchError('Cannot open configured project roots; check aliases and local directories.') from None + + def close(self): + for fd, _identity in self.roots.values(): + os.close(fd) + self.roots.clear() + + @contextmanager + def _file(self, root, path): + """Walk open directory handles; refuse symlinks, special files and path races.""" + parts = relative_path(path).split('/') + entry = self.roots.get(root) + if entry is None: + raise WorkbenchError('Project root is not configured on this server.', 'unavailable', 409) + descriptors = [] + directory = entry[0] + try: + for part in parts[:-1]: + directory = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory) + descriptors.append(directory) + fd = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory) + descriptors.append(fd) + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode): + raise WorkbenchError('Only regular project files can be inspected.', 'unsupported', 400) + yield fd, info + finally: + for fd in reversed(descriptors): + os.close(fd) + + def _read(self, root, path, limit): + with self._file(root, path) as (fd, before): + if before.st_size > limit: + raise WorkbenchError('File exceeds this inspection limit.', 'too_large', 400) + chunks, remaining = [], limit + 1 + while remaining: + chunk = os.read(fd, min(remaining, 65536)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + after = os.fstat(fd) + raw = b''.join(chunks) + if len(raw) > limit: + raise WorkbenchError('File exceeds this inspection limit.', 'too_large', 400) + if ((before.st_size, before.st_mtime_ns, before.st_ctime_ns) != + (after.st_size, after.st_mtime_ns, after.st_ctime_ns) or len(raw) != after.st_size): + raise WorkbenchError('Project file changed during inspection. Retry.', 'conflict', 409) + return raw + + def _saved(self, project_id): + if not isinstance(project_id, str) or not ID.fullmatch(project_id): + raise WorkbenchError('Invalid project ID.') + with self.lock: + row = self.db.execute('SELECT document FROM workspace_projects WHERE id=%s', (project_id,)).fetchone() + if row is None: + raise WorkbenchError('Project not found.', 'not_found', 404) + result = json.loads(row[0]) + return result + + @staticmethod + def _summary(saved): + return {key: saved[key] for key in ('id', 'name', 'root', 'manifest_path', 'manifest_sha256')} + + def list(self): + with self.lock: + rows = self.db.execute('SELECT document FROM workspace_projects ORDER BY id').fetchall() + return {'mode': 'inspection', 'roots': sorted(self.roots), + 'projects': [self._summary(json.loads(row[0])) for row in rows]} + + def open(self, body): + closed(body, ('root', 'manifest_path')) + root, path = body['root'], relative_path(body['manifest_path']) + if not isinstance(root, str) or root not in self.roots: + raise WorkbenchError('Choose a configured project root.', 'unavailable', 409) + try: + raw = self._read(root, path, MANIFEST_LIMIT) + except OSError: + raise WorkbenchError('Project manifest is missing or cannot be safely read.', 'unavailable', 400) from None + value = manifest(decode(raw)) + saved = {'id': value['id'], 'name': value['name'], 'root': root, + 'root_identity': self.roots[root][1], 'manifest_path': path, + 'manifest_sha256': hashlib.sha256(raw).hexdigest(), 'manifest': value} + document = encode(saved) + with self.lock, self.db.transaction(): + existing = self.db.execute('SELECT document FROM workspace_projects WHERE id=%s', (value['id'],)).fetchone() + if existing and existing[0] != document: + raise WorkbenchError('This project ID already names a different snapshot or root. Keep the original or use a new project ID.', 'conflict', 409) + if not existing: + self.db.execute('INSERT INTO workspace_projects VALUES (%s,%s)', (value['id'], document)) + return self.get(value['id']) + + def _bound(self, saved): + entry = self.roots.get(saved['root']) + return bool(entry and entry[1] == saved['root_identity']) + + def _inspect(self, root, ref, budget): + result = {**ref, 'status': 'available', 'integrity': 'unverified'} + try: + with self._file(root, ref['path']) as (_fd, info): + result['bytes'] = info.st_size + if info.st_size > FILE_LIMIT or info.st_size > budget[0]: + result['reason'] = 'Present; content exceeds the bounded hash inspection budget.' + return result + raw = self._read(root, ref['path'], min(FILE_LIMIT, budget[0])) + budget[0] -= len(raw) + result['bytes'] = len(raw) + result['observed_sha256'] = hashlib.sha256(raw).hexdigest() + if 'sha256' in ref: + result['integrity'] = 'verified' if ref['sha256'] == result['observed_sha256'] else 'mismatch' + else: + result['reason'] = 'No expected hash declared; scientific meaning is not verified.' + except FileNotFoundError: + result.update(status='missing', reason='Referenced file is missing.') + except (OSError, WorkbenchError): + result.update(status='unavailable', reason='File cannot be safely inspected or changed during inspection.') + return result + + def get(self, project_id): + saved = self._saved(project_id) + result = {**self._summary(saved), 'mode': 'inspection', 'execution': 'unsupported', + 'manifest_status': 'unavailable'} + bound = self._bound(saved) + if bound: + try: + raw = self._read(saved['root'], saved['manifest_path'], MANIFEST_LIMIT) + result['manifest_status'] = 'unchanged' if hashlib.sha256(raw).hexdigest() == saved['manifest_sha256'] else 'changed' + except FileNotFoundError: + result['manifest_status'] = 'missing' + except (OSError, WorkbenchError): + pass + budget = [INSPECTION_LIMIT] + for group in GROUPS: + result[group] = [] + for ref in saved['manifest'][group]: + if not bound: + item = {**ref, 'status': 'unavailable', 'integrity': 'unverified', + 'reason': 'Original project root is not configured; snapshot retained.'} + elif 'path' not in ref: + item = {**ref, 'status': 'unverified', 'reason': 'Declared requirement; no runtime probe is executed.'} + elif group == 'requirements': + item = {**ref, 'status': 'present_unqualified', 'reason': 'File presence does not establish runtime compatibility.'} + try: + with self._file(saved['root'], ref['path']): + pass + except FileNotFoundError: + item.update(status='missing', reason='Required file is missing.') + except (OSError, WorkbenchError): + item.update(status='unavailable', reason='Required file cannot be safely inspected.') + else: + item = self._inspect(saved['root'], ref, budget) + result[group].append(item) + return result + + def evidence(self, project_id, reference_id): + saved = self._saved(project_id) + if not self._bound(saved): + raise WorkbenchError('Original project root is not configured.', 'unavailable', 409) + ref = next((entry for entry in saved['manifest']['evidence'] if entry['id'] == reference_id), None) + if ref is None: + raise WorkbenchError('Evidence reference not found.', 'not_found', 404) + if Path(ref['path']).suffix.lower() not in ('.json', '.txt', '.md'): + raise WorkbenchError('Only bounded plain text, Markdown and JSON evidence can be previewed.', 'unsupported', 400) + try: + raw = self._read(saved['root'], ref['path'], TEXT_LIMIT) + value = raw.decode('utf-8') + except (OSError, UnicodeError): + raise WorkbenchError('Evidence is unavailable or is not UTF-8 text.', 'unavailable', 409) from None + digest = hashlib.sha256(raw).hexdigest() + if ref.get('sha256', digest) != digest: + raise WorkbenchError('Evidence differs from its declared SHA256.', 'conflict', 409) + if any(ord(c) < 32 and c not in '\n\r\t' for c in value): + raise WorkbenchError('Evidence contains non-text control characters.', 'unsupported', 400) + return {'id': ref['id'], 'label': ref['label'], 'text': value, 'sha256': digest, + 'external': True, 'scientific_verification': 'not_performed'} diff --git a/static/workspace-projects.js b/static/workspace-projects.js new file mode 100644 index 00000000..690616bf --- /dev/null +++ b/static/workspace-projects.js @@ -0,0 +1,113 @@ +'use strict'; +// Projects project server-owned references. Opening never admits an execution. +window.WorkspaceProjects = function ({select, roots, announce, switchSource, contextChanged}) { + const $=id=>document.getElementById(id), esc=value=>window.workspaceEscape(value); + let active=null, view=null, detail=null, generation=0, controller=null, evidenceController=null; + let entries=[], aliases=[], listGeneration=0, opening=false, openGeneration=0; + const currentSource=()=>select.value; + async function api(path, options={}) { + const response=await fetch('/api/workspace/projects'+path,options), value=await response.json(); + if(!response.ok)throw Error(value.error||'Project inspection unavailable.'); + return value; + } + function populate() { + const selected=currentSource(); + for(const option of [...select.options])if(option.value.startsWith('project:'))option.remove(); + for(const entry of entries){const option=document.createElement('option');option.value='project:'+entry.id;option.textContent=entry.name+' · project';select.append(option);} + if(selected.startsWith('project:')&&!entries.some(entry=>'project:'+entry.id===selected)){ + const option=document.createElement('option');option.value=selected;option.textContent='Remembered project';select.append(option); + } + select.value=selected; + const previous=$('project-root').value;$('project-root').replaceChildren(); + for(const alias of aliases){const option=document.createElement('option');option.value=alias;option.textContent=alias;$('project-root').append(option);} + if(aliases.includes(previous))$('project-root').value=previous; + $('project-open-submit').disabled=opening||!aliases.length; + contextChanged(); + } + async function refreshList() { + const epoch=++listGeneration; + try { + const value=await api('');if(epoch!==listGeneration)return; + if(value.mode!=='inspection'||!Array.isArray(value.roots)||!value.roots.every(alias=>typeof alias==='string')||!Array.isArray(value.projects))throw Error('Unsupported project catalog.'); + aliases=value.roots;entries=value.projects;populate(); + $('project-open-status').textContent=aliases.length?'Choose a configured root and its relative manifest path.':'No project roots configured by this local server.'; + }catch(error){if(epoch===listGeneration)$('project-open-status').textContent=error.message;} + } + function reference(item, category) { + return `
  • ${esc(item.label)}${esc(item.status)} · ${esc(item.integrity)}

    ${esc(item.path)}

    ${item.reason?`

    ${esc(item.reason)}

    `:''}${item.bytes!==undefined?`${esc(item.bytes)} bytes`:''}${category==='evidence'&&item.status==='available'?``:''}
    Reference identity
    ${esc(JSON.stringify({id:item.id,expected_sha256:item.sha256,observed_sha256:item.observed_sha256},null,2))}
  • `; + } + function render() { + if(!active||!view)return; + const root=roots[view]; + if(!detail){root.innerHTML='

    Loading project references…

    ';return;} + const changed=detail.manifest_status!=='unchanged'; + const items=view==='data'?detail.data:view==='pipeline'?detail.recipes:detail.evidence; + const title=view==='data'?'Referenced data':view==='pipeline'?'Referenced recipes':'External evidence'; + const note=view==='data'?'References are inspected in place; they are not imported catalog records.':view==='pipeline'?'Referenced recipes are not executable in project inspection mode.':'External evidence is not a Tuldok-completed run or scientific qualification.'; + root.innerHTML=`
    PROJECT · INSPECTION ONLY

    ${esc(detail.name)}

    ${esc(detail.root)} / ${esc(detail.manifest_path)}

    ${changed?`

    Manifest ${esc(detail.manifest_status)}. Saved references remain unchanged; a different manifest requires a new project ID.

    `:''}

    ${title}

    ${note}

    ${items.length?``:`

    No ${title.toLowerCase()} declared.

    `}
    ${view==='data'?`

    Runtime requirements

    Presence does not qualify a runtime.

    ${detail.requirements.length?``:'

    No requirements declared.

    '}
    `:''}
    Saved manifest identity

    ${esc(detail.manifest_sha256)}

    Execution: unsupported

    `; + root.querySelector('.project-refresh').onclick=()=>loadDetail(active,true); + for(const button of root.querySelectorAll('[data-project-evidence]'))button.onclick=()=>loadEvidence(button.dataset.projectEvidence); + contextChanged(); + } + async function loadDetail(id, refresh=false) { + const epoch=++generation;controller?.abort();evidenceController?.abort();evidenceController=null; + const request=new AbortController();controller=request; + if(refresh)detail=null;render(); + try { + const value=await api('/'+encodeURIComponent(id),{signal:request.signal}); + if(epoch!==generation||id!==active)return; + if(value.id!==id||value.mode!=='inspection'||value.execution!=='unsupported'||!['data','recipes','evidence','requirements'].every(key=>Array.isArray(value[key])))throw Error('Unsupported project inspection response.'); + detail=value;render(); + }catch(error){ + if(epoch===generation&&id===active&&error.name!=='AbortError'){ + roots[view].innerHTML=``; + roots[view].querySelector('button').onclick=()=>loadDetail(id,true); + } + }finally{if(controller===request)controller=null;} + } + async function loadEvidence(id) { + evidenceController?.abort();const request=new AbortController();evidenceController=request; + const project=active,epoch=generation,root=roots[view],output=root.querySelector('.project-evidence-output'); + output.textContent='Loading external evidence…'; + try{ + const value=await api('/'+encodeURIComponent(project)+'/evidence/'+encodeURIComponent(id),{signal:request.signal}); + if(evidenceController!==request||epoch!==generation||project!==active||view!=='runs')return; + if(value.id!==id||value.external!==true||typeof value.text!=='string')throw Error('Unsupported external evidence response.'); + output.innerHTML=`

    ${esc(value.label)} · external evidence

    Referenced source text; not a Tuldok run or scientific validation.

    SHA256 ${esc(value.sha256)}

    ${esc(value.text)}
    `; + output.querySelector('h3').focus({preventScroll:true}); + }catch(error){if(evidenceController===request&&epoch===generation&&error.name!=='AbortError')output.textContent=error.message;} + finally{if(evidenceController===request)evidenceController=null;} + } + $('open-project').onclick=()=>{$('project-open-dialog').showModal();$('project-manifest').focus();refreshList();}; + $('project-open-close').onclick=()=>{openGeneration++;$('project-open-dialog').close();$('open-project').focus();}; + $('project-open-dialog').addEventListener('cancel',()=>{openGeneration++;}); + $('project-roots-refresh').onclick=refreshList; + $('project-open-form').onsubmit=async event=>{ + event.preventDefault();if(opening)return; + const root=$('project-root').value,manifest_path=$('project-manifest').value; + if(!aliases.includes(root))return; + const epoch=++openGeneration;opening=true;$('project-open-submit').disabled=true;$('project-open-status').textContent='Inspecting project manifest…'; + try{ + const value=await api('',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({root,manifest_path})}); + if(epoch!==openGeneration||!$('project-open-dialog').open)return; + if(typeof value.id!=='string'||value.mode!=='inspection')throw Error('Unsupported opened project response.'); + listGeneration++;entries=entries.filter(entry=>entry.id!==value.id);entries.push(value);populate(); + $('project-open-dialog').close();switchSource('project:'+value.id); + }catch(error){if(epoch===openGeneration)$('project-open-status').textContent=error.message+' Refresh saved projects before repeating an opening whose reply was lost.';} + finally{opening=false;$('project-open-submit').disabled=!aliases.length;} + }; + window.addEventListener('pagehide',()=>{generation++;listGeneration++;openGeneration++;controller?.abort();evidenceController?.abort();}); + return { + initialize:refreshList, + label:()=>detail?.name||entries.find(entry=>'project:'+entry.id===currentSource())?.name||'Project', + show(source,next){ + const id=source.startsWith('project:')?source.slice(8):null; + for(const [name,root] of Object.entries(roots))root.hidden=!id||name!==next; + if(id===active&&next===view)return; + const changed=id!==active;active=id;view=next;generation++;controller?.abort();evidenceController?.abort();controller=null;evidenceController=null; + if(changed)detail=null; + if(!id)return; + if(detail)render();else loadDetail(id); + }, + }; +}; diff --git a/static/workspaces.css b/static/workspaces.css index f5b32c41..5283194f 100644 --- a/static/workspaces.css +++ b/static/workspaces.css @@ -1021,3 +1021,20 @@ details > p { .real-inspector>label { margin-top:0; } .real-inspector>h2,.real-inspector>p { margin:0; } @media(max-width:430px) { .real-inspector { grid-template-columns:1fr; } } + +/* Project inspection projects source references without execution controls. */ +#project-open-dialog{width:min(520px,calc(100vw - 32px));max-height:90vh;overflow:auto} +#project-open-dialog label{display:block;margin:12px 0} +#project-open-dialog input,#project-open-dialog select{width:100%;max-width:100%} +.project-header{display:flex;justify-content:space-between;align-items:center;gap:12px} +.project-header>div{min-width:0}.project-header p{margin:6px 0;overflow-wrap:anywhere} +.project-header h2{margin:4px 0}.project-header button{flex-shrink:0} +[id^="project-"][id$="-root"]>.panel{margin-bottom:12px} +.project-references{list-style:none;padding:0;margin:12px 0} +.project-references>li{padding:12px 0;border-top:1px solid var(--line);overflow-wrap:anywhere} +.project-reference>div{display:flex;gap:8px;align-items:center;justify-content:space-between;flex-wrap:wrap} +.project-reference small{display:block;margin:5px 0}.project-reference button{margin:8px 0} +.project-references .badge{margin-left:8px}.project-path{font-size:12px;overflow-wrap:anywhere} +.project-evidence-output pre{max-height:480px;overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere} +.project-identity{margin-top:12px} +@media(max-width:600px){.project-header{align-items:start;flex-direction:column}.project-reference>div{align-items:start}.project-references .badge{margin-left:0}} diff --git a/static/workspaces.html b/static/workspaces.html index d58f6ec0..0e894e8c 100644 --- a/static/workspaces.html +++ b/static/workspaces.html @@ -21,13 +21,14 @@
    - DATASETCONTEXT
    +
    Loading workspace…
    Local preview
    @@ -50,6 +51,7 @@

    Data

    Inspect records, keep their evidence, review deliberately.

    +
    + + +

    Open project

    +

    Inspect a manifest beneath a configured local root.

    +
    + + +
    +
    +

    Loading configured roots…

    +

    Unsaved fixture edits

    @@ -111,6 +125,7 @@

    Unsaved fixture edits

    + diff --git a/static/workspaces.js b/static/workspaces.js index ecdddbfe..63d02dc8 100644 --- a/static/workspaces.js +++ b/static/workspaces.js @@ -78,7 +78,7 @@ window.workspaceEscape = (value) => // dataset identity. Live references are re-read and revision-checked on reload. const remembered = history.state?.workspace; if (remembered?.version === 1) { - if (["fixtures", "catalog"].includes(remembered.source)) + if (["fixtures", "catalog"].includes(remembered.source) || (typeof remembered.source === "string" && remembered.source.startsWith("project:") && remembered.source.length > 8)) source = remembered.source; if (typeof remembered.active === "string") active = remembered.active; if (Array.isArray(remembered.selected)) @@ -87,6 +87,12 @@ window.workspaceEscape = (value) => ); catalogContext = remembered.catalog || null; } + if (source.startsWith("project:")) { + const option=document.createElement("option");option.value=source;option.textContent="Remembered project";$("dataset").append(option);$("dataset").value=source; + } + const projects = WorkspaceProjects({select:$("dataset"), + roots:{data:$("project-data-root"),pipeline:$("project-pipeline-root"),runs:$("project-runs-root")}, + announce,switchSource,contextChanged:context}); let restoringCatalog = false, catalogRestoreNotice = null; function remember() { @@ -126,6 +132,8 @@ window.workspaceEscape = (value) => if (source === "fixtures") $("context").textContent = `fixture-dataset-v1 · ${selected.size} selected · ${draftDirty ? "unsaved fixture draft" : "authored CC0 examples"} · release fixture-release-v1`; + else if (source.startsWith("project:")) + $("context").textContent = projects.label() + " · inspection only · external references"; else $("context").textContent = catalogContext ? `${catalogContext.selection.length} exact revisions selected · ${catalogContext.record?.name || "No active record"} · ${catalogContext.dirty ? "unsaved edits" : "saved state"}${real.identity() ? " · catalog " + real.identity().slice(0, 12) : " · current catalog handle"}` @@ -162,6 +170,7 @@ window.workspaceEscape = (value) => $("catalog-frame").src = "/workbench?workspace=1"; return; } + if (source.startsWith("project:")) return; if (!fixtures || !store) return; const record = fixtures.records.find((r) => r.id === active); if (!record) { @@ -243,6 +252,7 @@ window.workspaceEscape = (value) => } invalidateCorrection(); source = next; + if (source.startsWith("project:")) $("dataset").disabled = false; $("dataset").value = next; renderData(); showView(); @@ -285,8 +295,11 @@ window.workspaceEscape = (value) => $("real-pipeline-root").hidden = source !== "catalog"; $("real-runs-root").hidden = source !== "catalog"; real.show(source === "catalog" ? next : null); + projects.show(source, next); if (next === "runs" && source === "fixtures") runs?.render(); - if (next !== "data" && source === "catalog") + if (source.startsWith("project:")) + announce("Project inspection · " + next + ". Execution is unsupported."); + else if (next !== "data" && source === "catalog") announce("Current catalog · real " + next + ". Source edits remain mounted in Data."); else announce( @@ -463,6 +476,7 @@ window.workspaceEscape = (value) => $("workspace-error").textContent = error.message; $("workspace-error").hidden = false; $("retry-load").hidden = false; + if (source.startsWith("project:")) { $("dataset").disabled=false;renderData();showView(); } announce( "Workspace fixture loading failed. The classic workbench remains available.", true, @@ -470,5 +484,6 @@ window.workspaceEscape = (value) => } } $("retry-load").onclick = load; + projects.initialize(); load(); })(); diff --git a/tests/browser_workspace_project.cjs b/tests/browser_workspace_project.cjs new file mode 100644 index 00000000..156033fb --- /dev/null +++ b/tests/browser_workspace_project.cjs @@ -0,0 +1,102 @@ +// Real read-only project UI. Synthetic lifecycle fixture or copies of actual lab files; +// neither mode executes project code or qualifies externally recorded science/training. +'use strict'; +const assert=require('node:assert/strict'), fs=require('node:fs'), os=require('node:os'), path=require('node:path'), crypto=require('node:crypto'); +const {spawn,execFileSync}=require('node:child_process'); +const {pageLoadTracker}=require('./browser_page_load.cjs'); +const {waitForDebugger}=require('./browser_startup.cjs'); +const {qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); +const root=path.resolve(__dirname,'..'), labIndex=process.argv.indexOf('--lab-root'); +assert.ok(labIndex<0||process.argv[labIndex+1],'--lab-root requires an existing checkout'); +const labRoot=labIndex<0?null:path.resolve(process.argv[labIndex+1]); +const report=qaDirectory(root,labRoot?'workspace-project-lab':'workspace-project'); +const temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-project-browser-')),projectRoot=path.join(temporary,'projects'); +fs.mkdirSync(projectRoot); +const sha=raw=>crypto.createHash('sha256').update(raw).digest('hex'); +const sourceHashes={},captures=[],checks=[],requests=[],errors=[],children=[],tracker=pageLoadTracker(); +const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms)); +let ws,send,evaluate,serverErrors=''; +async function until(fn,message='Timed out'){for(let i=0;i<600;i++){const value=await fn();if(value)return value;await pause(100);}throw Error(message);} +function launch(command,args,options){const child=spawn(command,args,options);children.push(child);return child;} +function write(relative,raw){const destination=path.join(projectRoot,relative);fs.mkdirSync(path.dirname(destination),{recursive:true});fs.writeFileSync(destination,raw);return relative;} +function reference(id,label,relative){return {id,label,path:relative,sha256:sha(fs.readFileSync(path.join(projectRoot,relative)))};} +function copied(relative){const raw=fs.readFileSync(path.join(labRoot,relative));sourceHashes[relative]=sha(raw);return write('copied/'+relative,raw);} +function snapshot(directory){const result={};function walk(current){for(const entry of fs.readdirSync(current,{withFileTypes:true})){const next=path.join(current,entry.name);if(entry.isDirectory())walk(next);else{assert.ok(entry.isFile(),'Fixture has no symlinks/special files');result[path.relative(directory,next)]=sha(fs.readFileSync(next));}}}walk(directory);return result;} +const first={schema_version:1,id:'a'.repeat(32),name:labRoot?'Copied actual lab · inspection only':'Authored project Alpha',data:[],recipes:[],evidence:[],requirements:[]}; +if(labRoot){ + first.data.push(reference('freeze','Copied actual trajectory freeze',copied('experiments/trajectory-v2/freeze.json'))); + first.data.push(reference('checkpoint','Copied external checkpoint · not loaded',copied('experiments/trajectory-v2/model.pt'))); + first.recipes.push(reference('recipe','Copied actual recipe · not executable',copied('scripts/trajectory_experiment.py'))); + first.evidence.push(reference('report','Copied actual README · external evidence',copied('README.md'))); + first.evidence.push(reference('large-result','Copied actual result · bounded preview',copied('experiments/trajectory-v2/result.json'))); +}else{ + first.data.push(reference('sample','Authored sample data',write('alpha/data.json',JSON.stringify({source:'synthetic project fixture',values:[1,2,3]})))); + first.recipes.push(reference('recipe','Authored recipe · execution trap',write('alpha/recipe.py',"from pathlib import Path\nPath(__file__).with_name('EXECUTED').write_text('must never run')\n"))); + first.evidence.push(reference('report','Alpha external evidence',write('alpha/report.md','# External Alpha evidence\nAuthored UI fixture, not a completed Tuldok run.\n\nCafé 😀 山\n'))); +} +first.data.push({id:'missing-data',label:'Unavailable external dataset',path:'not-installed/data.json'}); +first.requirements=[{id:'present',label:'Recipe file present; runtime unqualified',path:first.recipes[0].path}, + {id:'missing',label:'Missing optional runtime',path:'not-installed/python'}, + {id:'declared',label:'Unprobed GPU requirement'}]; +const second={schema_version:1,id:'b'.repeat(32),name:'Authored project Beta',data:[reference('beta','Only Beta data',write('beta/data.txt','Private-to-Beta synthetic fixture data\n'))],recipes:[],evidence:[reference('report','Beta external evidence',write('beta/report.md','Beta evidence belongs only to the second project.\n'))],requirements:[]}; +write('alpha.project.json',JSON.stringify(first,null,2));write('beta.project.json',JSON.stringify(second,null,2)); +const before=snapshot(projectRoot), expectedEvidence=fs.readFileSync(path.join(projectRoot,first.evidence[0].path),'utf8'); +const serverArgs=port=>['-u','app.py','--port',String(port),'--data',path.join(temporary,'data'),'--project-root','sample='+projectRoot]; +(async()=>{ + let server=launch('python3',serverArgs(0),{cwd:root,stdio:['ignore','pipe','pipe']}); + let output='';server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); + const port=await until(()=>{if(server.exitCode!==null)throw Error('Project server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];}); + const base='http://127.0.0.1:'+port; + const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,900','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:['ignore','ignore','ignore'],env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); + const debugPort=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')); + const tabs=await(await fetch('http://127.0.0.1:'+debugPort+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); + ws=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;}); + let next=0;const pending=new Map(); + send=(method,params={})=>new Promise((resolve,reject)=>{const id=++next,timer=setTimeout(()=>{pending.delete(id);reject(Error('CDP timeout '+method));},20000);pending.set(id,{resolve:value=>{clearTimeout(timer);resolve(value);},reject:error=>{clearTimeout(timer);reject(error);}});ws.send(JSON.stringify({id,method,params}));}); + ws.onmessage=event=>{const message=JSON.parse(event.data);tracker.observe(message);if(message.id){const task=pending.get(message.id);pending.delete(message.id);if(task)message.error?task.reject(message.error):task.resolve(message.result);}if(message.method==='Runtime.exceptionThrown')errors.push(message.params.exceptionDetails);if(message.method==='Network.requestWillBeSent')requests.push({url:message.params.request.url,method:message.params.request.method});}; + evaluate=async expression=>{const result=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; + const element=selector=>`document.querySelector(${JSON.stringify(selector)})`,text=selector=>evaluate(`${element(selector)}?.textContent`); + async function click(selector){const box=await until(()=>evaluate(`(async()=>{const e=${element(selector)};if(!e||e.disabled||!e.getClientRects().length)return null;e.scrollIntoView({block:'center'});await new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));const b=e.getBoundingClientRect(),x=b.x+b.width/2,y=b.y+b.height/2,hit=document.elementFromPoint(x,y);return hit&&(hit===e||e.contains(hit))?{x,y}:null})()`),'Click target unavailable '+selector);for(const type of ['mousePressed','mouseReleased'])await send('Input.dispatchMouseEvent',{type,button:'left',clickCount:1,...box});} + async function key(key,code=key,modifiers=0){for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Escape:27,Tab:9,a:65}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});} + async function fill(selector,value){await click(selector);await key('a','KeyA',2);await send('Input.insertText',{text:value});} + const select=(selector,value)=>evaluate(`(()=>{const e=${element(selector)};e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));})()`); + async function nav(view){await evaluate(`document.getElementById('nav-${view}').focus()`);await key('Enter');await until(()=>evaluate(`!document.getElementById('view-${view}').hidden && !document.getElementById('project-${view}-root').hidden`));await until(()=>text(`#project-${view}-root`).then(value=>value?.includes('PROJECT · INSPECTION ONLY')));} + async function capture(name,width,selector){await send('Emulation.setDeviceMetricsOverride',{width,height:900,deviceScaleFactor:1,mobile:false});await evaluate(`${element(selector)}.scrollIntoView({block:'start'})`);await pause(80);assert.ok(await evaluate('document.documentElement.scrollWidth<=innerWidth+1'),'Project viewport overflow '+name);const file=name+'.jpg';fs.writeFileSync(path.join(report,file),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64'));captures.push({file,width,height:900,selector,qualification:labRoot?'Copied actual source inspection only; no execution or scientific verification':'Synthetic project UI/lifecycle fixture only'});} + const check=value=>{checks.push(value);console.log('PASS',value);}; + async function reload(){const previous=(await send('Page.getFrameTree')).frameTree.frame;await send('Page.reload');await until(()=>tracker.reloaded(previous),'Main document reload not observed');} + async function openProject(manifest,name,id){await click('#open-project');await until(()=>evaluate('document.getElementById("project-open-dialog").open && !document.getElementById("project-open-submit").disabled'));await select('#project-root','sample');await fill('#project-manifest',manifest);await evaluate('document.getElementById("project-open-submit").focus()');await key('Enter');await until(()=>evaluate('document.getElementById("dataset").value==='+JSON.stringify('project:'+id)));await nav('data');assert.match(await text('#project-data-root'),new RegExp(name.replace(/[.*+?^${}()|[\]\\]/g,'\\$&')));} + await send('Page.enable');await send('Page.setLifecycleEventsEnabled',{enabled:true});await send('Runtime.enable');await send('Network.enable'); + const offlineCanary=process.env.LD_PRELOAD?JSON.parse(execFileSync('python3',['-c',"import sys;sys.path.insert(0,'tests');from test_workspace_offline import CANARY;exec(CANARY)"],{cwd:root,encoding:'utf8'})):null; + await send('Page.navigate',{url:base+'/workspaces'});await until(()=>evaluate('!!document.getElementById("open-project")')); + await openProject('alpha.project.json',first.name,first.id); + const details=await(await fetch(base+'/api/workspace/projects/'+first.id)).json();assert.equal(details.execution,'unsupported');assert.equal(details.manifest_status,'unchanged'); + assert.equal(details.data.find(item=>item.id==='missing-data').status,'missing');assert.equal(details.requirements.find(item=>item.id==='present').status,'present_unqualified');assert.equal(details.requirements.find(item=>item.id==='missing').status,'missing');assert.equal(details.requirements.find(item=>item.id==='declared').status,'unverified'); + assert.match(await text('#project-data-root'),/present_unqualified/);assert.match(await text('#project-data-root'),/Missing optional runtime/); + assert.ok(details.data.filter(item=>item.id!=='missing-data').every(item=>item.integrity==='verified')); + await capture('project-data-1440',1440,'#project-data-root');await capture('project-data-390',390,'#project-data-root'); + check('Project opens through keyboard form; data hashes and missing/present-unqualified/unprobed requirements remain distinct'); + await nav('pipeline');assert.match(await text('#project-pipeline-root'),/not executable/);assert.equal(await evaluate('document.querySelectorAll("#project-pipeline-root form").length'),0); + await capture('project-pipeline-390',390,'#project-pipeline-root'); + await nav('runs');assert.match(await text('#project-runs-root'),/not a Tuldok-completed run or scientific qualification/); + await click('#project-runs-root [data-project-evidence="report"]');await until(()=>evaluate('!!document.querySelector("#project-runs-root .project-evidence-output pre")')); + assert.equal(await text('#project-runs-root .project-evidence-output pre'),expectedEvidence);assert.equal(await evaluate('window.projectExecuted===true'),false); + assert.match(await text('#project-runs-root .project-evidence-output'),/not a Tuldok run or scientific validation/); + await capture('project-evidence-1440',1440,'#project-runs-root .project-evidence-output');await capture('project-evidence-390',390,'#project-runs-root .project-evidence-output'); + if(labRoot){await click('#project-runs-root [data-project-evidence="large-result"]');await until(()=>text('#project-runs-root .project-evidence-output').then(value=>value?.includes('inspection limit')));check('Actual copied 4MiB result is hash-inspected but bounded text preview is refused honestly');} + check('Pipeline exposes nonexecutable references; Runs renders exact external evidence as text without a completion claim'); + await openProject('beta.project.json',second.name,second.id);assert.match(await text('#project-data-root'),/Only Beta data/);assert.doesNotMatch(await text('#project-data-root'),/Unavailable external dataset|Copied actual trajectory freeze|Authored sample data/); + await nav('runs');await click('#project-runs-root [data-project-evidence="report"]');await until(()=>text('#project-runs-root .project-evidence-output pre').then(value=>value?.startsWith('Beta evidence'))); + await select('#dataset','project:'+first.id);await until(()=>text('#project-runs-root').then(value=>value?.includes(first.name)));assert.equal(await evaluate('document.querySelector("#project-runs-root .project-evidence-output pre")'),null);assert.doesNotMatch(await text('#project-runs-root'),/Beta evidence belongs/); + await click('#project-runs-root [data-project-evidence="report"]');await until(()=>text('#project-runs-root .project-evidence-output pre').then(value=>value===expectedEvidence)); + check('Two projects with shared reference ID stay isolated; switching clears previous external evidence'); + await reload();await until(()=>evaluate('document.getElementById("dataset")?.value==='+JSON.stringify('project:'+first.id)));await until(()=>text('#project-runs-root').then(value=>value?.includes(first.name)));assert.equal(await evaluate('document.getElementById("view-runs").hidden'),false); + server.kill('SIGINT');await until(()=>server.exitCode!==null,'Project server did not drain');server=launch('python3',serverArgs(port),{cwd:root,stdio:['ignore','pipe','pipe']});let reopened='';server.stdout.on('data',chunk=>reopened+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk);await until(()=>reopened.includes('Tuldok:')); + await reload();await until(()=>text('#project-runs-root').then(value=>value?.includes(first.name)));await click('#project-runs-root [data-project-evidence="report"]');await until(()=>text('#project-runs-root .project-evidence-output pre').then(value=>value===expectedEvidence)); + assert.equal((await(await fetch(base+'/api/workspace/projects')).json()).projects.length,2);assert.deepEqual(await(await fetch(base+'/api/workspace/runs')).json(),[]); + assert.deepEqual(snapshot(projectRoot),before);if(labRoot)for(const [relative,digest] of Object.entries(sourceHashes))assert.equal(sha(fs.readFileSync(path.join(labRoot,relative))),digest); + assert.ok(!requests.some(item=>item.method==='POST'&&/\/api\/workspace\/(runs|graphs)|\/api\/workbench\//.test(item.url)),'Project browsing must not submit run, graph or record writes'); + assert.deepEqual(errors,[]);assert.ok(!serverErrors.includes('Traceback'),serverErrors); + check('Browser reload and server restart retain project selection/catalog; all source/copy bytes unchanged and no execution submitted'); + fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({source_commit:execFileSync('git',['rev-parse','HEAD'],{cwd:root,encoding:'utf8'}).trim(),dirty:!!execFileSync('git',['status','--porcelain'],{cwd:root,encoding:'utf8'}).trim(),command:'node tests/browser_workspace_project.cjs'+(labRoot?' --lab-root ':''),qualification:labRoot?'Inspection of exact copies of actual lab files only; no new training, execution or scientific verification':'Generic synthetic project UI/lifecycle only',actual_lab_source:labRoot?{commit:execFileSync('git',['rev-parse','HEAD'],{cwd:labRoot,encoding:'utf8'}).trim(),files:sourceHashes}:null,project_file_hashes:before,checks,captures,offline_canary:offlineCanary,execution_submitted:false,source_mutation:false,browser:(await send('Browser.getVersion')).product},null,2)); + console.log('PASS read-only project browser:',report); +})().catch(async error=>{console.error(error);console.error('Application stderr:',serverErrors);if(evaluate)try{console.error(await evaluate('JSON.stringify({url:location.href,body:document.body.innerText.slice(-9000)})'));}catch{}process.exitCode=1;}).finally(()=>{ws?.close();for(const child of children.reverse())child.kill('SIGTERM');}); diff --git a/tests/qualify_spatial_retained_runs.py b/tests/qualify_spatial_retained_runs.py index c696b565..33ad81d6 100644 --- a/tests/qualify_spatial_retained_runs.py +++ b/tests/qualify_spatial_retained_runs.py @@ -1,4 +1,4 @@ -"""Qualify real old-source schema3 results through explicit schema4 evolution. +"""Qualify real old-source schema3 results through explicit current-schema evolution. Uses disposable catalogs only. The legacy checkout must be the exact pinned, unmodified source; no synthetic downgrade or mocked execution is accepted. @@ -18,7 +18,7 @@ from psycopg import sql from app import Dataset from backup_metadata import backup_catalog, restore_catalog, tree_manifest -from metadata import Metadata, StorageError, connection_info +from metadata import SCHEMA_VERSION, Metadata, StorageError, connection_info from upgrade_metadata import upgrade_catalog LEGACY_COMMIT = 'f1ecf7888a8dfa5b67ac8b087034226d533c682e' @@ -70,7 +70,7 @@ def drop_fixture_schema(identity): def verify(path, receipt, artifacts): dataset = Dataset(path) try: - assert dataset.db.schema_version == 4 + assert dataset.db.schema_version == SCHEMA_VERSION assert dataset.db.dataset_id == receipt['dataset_id'] assert dataset.pipelines.get(receipt['graph']['id']) == receipt['graph'] assert dataset.runs.get(receipt['run']['id']) == receipt['run'] @@ -78,7 +78,7 @@ def verify(path, receipt, artifacts): for name, expected in artifacts.items(): observed, _ = dataset.runs.artifact(receipt['run']['id'], name) assert observed == expected, name - return {'schema_version': 4, 'state': 'completed', 'run_id': receipt['run']['id'], + return {'schema_version': SCHEMA_VERSION, 'state': 'completed', 'run_id': receipt['run']['id'], 'artifacts': {name: fingerprint(raw) for name, raw in artifacts.items()}} finally: dataset.close() @@ -114,16 +114,16 @@ def qualify(legacy_root): else: unexpected.close() raise AssertionError('Current application silently accepted schema3.') - assert upgrade_catalog(source, quiesced=True)['schema_version'] == 4 + assert upgrade_catalog(source, quiesced=True)['schema_version'] == SCHEMA_VERSION assert tree_manifest(source) == before upgraded = verify(source, receipt, artifacts) - backup4 = root / 'backup4' - assert backup_catalog(source, backup4, quiesced=True)['schema_version'] == 4 - backup4_files = tree_manifest(backup4) + backup_current = root / 'backup_current' + assert backup_catalog(source, backup_current, quiesced=True)['schema_version'] == SCHEMA_VERSION + backup_current_files = tree_manifest(backup_current) drop_fixture_schema(identity) - restored4 = root / 'restored4' - restore_catalog(backup4, restored4) - restored = verify(restored4, receipt, artifacts) + restored_current = root / 'restored_current' + restore_catalog(backup_current, restored_current) + restored = verify(restored_current, receipt, artifacts) drop_fixture_schema(identity) restored3 = root / 'restored3' restore_catalog(backup3, restored3) @@ -132,9 +132,9 @@ def qualify(legacy_root): upgrade_catalog(restored3, quiesced=True) restored_legacy = verify(restored3, receipt, artifacts) assert tree_manifest(backup3) == backup3_files - assert tree_manifest(backup4) == backup4_files + assert tree_manifest(backup_current) == backup_current_files return {'legacy_commit': head, 'legacy_schema': 3, 'execution': 'real mesh integrity, no training', - 'upgrade': upgraded, 'schema4_restore': restored, 'schema3_restore_then_upgrade': restored_legacy, + 'upgrade': upgraded, 'current_schema_restore': restored, 'schema3_restore_then_upgrade': restored_legacy, 'original_backups_unchanged': True} finally: # Recover identity even if the child failed after creating its catalog. diff --git a/tests/test_projects.py b/tests/test_projects.py new file mode 100644 index 00000000..23a128d4 --- /dev/null +++ b/tests/test_projects.py @@ -0,0 +1,271 @@ +"""Real project registration and read-only filesystem/HTTP inspection boundaries.""" +import copy +import hashlib +from http.server import ThreadingHTTPServer +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +import unittest +from unittest.mock import patch +import urllib.error +import urllib.request +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from app import Dataset, make_handler +import projects +from workbench import WorkbenchError + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +class ProjectsTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) + self.external = self.path / 'external' + self.external.mkdir() + self.files = {'data.bin': b'native-data', 'recipe.py': b'raise RuntimeError("Project source must never execute")\n', + 'note.md': b'# Evidence\nDeclared observations only.\n', 'runtime.txt': b'Unqualified environment declaration\n'} + for name, raw in self.files.items(): + (self.external / name).write_bytes(raw) + self.value = {'schema_version': 1, 'id': uuid.uuid4().hex, 'name': 'Local fixture', + 'data': [{'id': 'source', 'label': 'Native source', 'path': 'data.bin', 'sha256': sha(self.files['data.bin'])}], + 'recipes': [{'id': 'script', 'label': 'Unexecuted recipe', 'path': 'recipe.py'}], + 'evidence': [{'id': 'note', 'label': 'Observations', 'path': 'note.md', 'sha256': sha(self.files['note.md'])}], + 'requirements': [{'id': 'python', 'label': 'Declared Python'}, + {'id': 'runtime', 'label': 'Runtime file', 'path': 'runtime.txt'}]} + self.write_manifest() + self.dataset = Dataset(self.path / 'catalog', project_roots={'fixture': self.external}) + self.addCleanup(lambda: self.dataset.close()) + self.owner = self.dataset.projects + + def write_manifest(self, value=None, raw=None, name='project.json'): + (self.external / name).write_bytes(raw if raw is not None else json.dumps(value or self.value).encode()) + + def open(self, **changes): + return self.owner.open(dict({'root': 'fixture', 'manifest_path': 'project.json'}, **changes)) + + def snapshot(self): + return {path.name: path.read_bytes() for path in self.external.iterdir() if path.is_file() and not path.is_symlink()} + + def reopen(self, roots): + self.dataset.close() + self.dataset = Dataset(self.path / 'catalog', project_roots=roots) + self.owner = self.dataset.projects + + def test_real_registration_reads_hashes_and_never_executes_or_mutates_sources(self): + before = self.snapshot() + with patch.object(subprocess, 'Popen', side_effect=AssertionError('Inspection cannot execute project code')): + result = self.open() + self.assertEqual(self.open(), result) + evidence = self.owner.evidence(self.value['id'], 'note') + self.assertEqual(self.snapshot(), before) + self.assertEqual(result['mode'], 'inspection') + self.assertEqual(result['execution'], 'unsupported') + self.assertEqual(result['manifest_status'], 'unchanged') + self.assertEqual(result['data'][0]['integrity'], 'verified') + self.assertEqual(result['recipes'][0]['integrity'], 'unverified') + self.assertEqual(result['requirements'][0]['status'], 'unverified') + self.assertEqual(result['requirements'][1]['status'], 'present_unqualified') + self.assertEqual(evidence['text'], self.files['note.md'].decode()) + self.assertEqual(evidence['scientific_verification'], 'not_performed') + self.assertEqual(len(self.owner.list()['projects']), 1) + self.reopen({'fixture': self.external}) + self.assertEqual(self.owner.get(self.value['id']), result) + + def test_changed_manifest_flags_original_snapshot_and_id_conflicts(self): + opened = self.open() + changed = copy.deepcopy(self.value) + changed['name'] = 'Changed owner file' + changed['data'] = [] + self.write_manifest(changed) + current = self.owner.get(self.value['id']) + self.assertEqual(current['manifest_status'], 'changed') + self.assertEqual(current['name'], opened['name']) + self.assertEqual(current['data'], opened['data']) + with self.assertRaises(WorkbenchError) as caught: + self.open() + self.assertEqual(caught.exception.code, 'conflict') + self.assertEqual(len(self.owner.list()['projects']), 1) + self.write_manifest(name='elsewhere.json') + with self.assertRaises(WorkbenchError): + self.open(manifest_path='elsewhere.json') + (self.external / 'project.json').unlink() + self.assertEqual(self.owner.get(self.value['id'])['manifest_status'], 'missing') + + def test_absent_and_rebound_roots_keep_snapshot_without_new_root_reads(self): + self.open() + self.reopen({}) + result = self.owner.get(self.value['id']) + self.assertEqual(result['manifest_status'], 'unavailable') + self.assertEqual(result['data'][0]['status'], 'unavailable') + self.assertEqual(self.owner.list()['roots'], []) + with self.assertRaises(WorkbenchError): + self.owner.evidence(self.value['id'], 'note') + replacement = self.path / 'different' + replacement.mkdir() + for name, raw in self.snapshot().items(): + (replacement / name).write_bytes(raw) + self.reopen({'fixture': replacement}) + self.assertEqual(self.owner.get(self.value['id'])['data'][0]['status'], 'unavailable') + with self.assertRaises(WorkbenchError) as caught: + self.open() + self.assertEqual(caught.exception.code, 'conflict') + + def test_manifest_traversal_symlinks_directories_and_fifo_refused_without_registration(self): + outside = self.path / 'outside.json' + outside.write_text(json.dumps(self.value)) + (self.external / 'link.json').symlink_to(outside) + (self.external / 'directory').mkdir() + (self.external / 'linkdir').symlink_to(self.path, target_is_directory=True) + os.mkfifo(self.external / 'fifo.json') + for path in ('../outside.json', '/absolute.json', 'directory/../project.json', './project.json', + 'directory//file', 'directory\\file', '/'.join(['deep']*17), 'link.json', 'directory', 'fifo.json', 'linkdir/outside.json'): + with self.subTest(path=path), self.assertRaises(WorkbenchError): + self.open(manifest_path=path) + self.assertEqual(self.owner.list()['projects'], []) + + def test_unsafe_reference_files_and_missing_entries_are_reported_without_following(self): + (self.external / 'link.md').symlink_to(self.path / 'outside.md') + (self.path / 'outside.md').write_text('outside secret') + os.mkfifo(self.external / 'pipe.md') + (self.external / 'folder.md').mkdir() + self.value['evidence'] = [{'id': str(index), 'label': name, 'path': name} + for index, name in enumerate(('link.md', 'pipe.md', 'folder.md', 'missing.md'))] + self.write_manifest() + result = self.open() + self.assertEqual([entry['status'] for entry in result['evidence']], ['unavailable','unavailable','unavailable','missing']) + for entry in self.value['evidence']: + with self.assertRaises(WorkbenchError): + self.owner.evidence(self.value['id'], entry['id']) + + def test_malformed_manifest_types_json_and_unknown_fields_are_atomic(self): + invalid = [b'{"schema_version":1,"schema_version":1}', b'\xff', b'{"name":NaN}', b'['*1200+b']'*1200, b'{}'] + for mutation in ( + {'schema_version': True}, {'schema_version': 2}, {'id': 'bad'}, {'name': ''}, {'name': '\ud800'}, + {'data': {}}, {'data': [None]}, {'data': [{'id':'x','label':'x','path':'../bad'}]}, + {'data': [{'id':'x','label':'x','path':'data.bin','sha256':'not-a-hash'}]}, + {'requirements': [{'id':'a','label':'x','sha256':'0'*64}]}, {'extra': 1}, + ): + invalid.append(json.dumps(dict(self.value, **mutation)).encode()) + duplicated = copy.deepcopy(self.value) + duplicated['data'] *= 2 + invalid.append(json.dumps(duplicated).encode()) + for raw in invalid: + self.write_manifest(raw=raw) + with self.subTest(raw=raw[:80]), self.assertRaises(WorkbenchError): + self.open() + self.assertEqual(self.owner.list()['projects'], []) + + def test_evidence_hash_text_suffix_and_size_fail_closed(self): + self.open() + for raw in (b'changed', b'\xff', b'\x00binary', b'x'*(projects.TEXT_LIMIT+1)): + (self.external / 'note.md').write_bytes(raw) + with self.subTest(size=len(raw)), self.assertRaises(WorkbenchError): + self.owner.evidence(self.value['id'], 'note') + with self.assertRaises(WorkbenchError): + self.owner.evidence(self.value['id'], 'script') + self.value['id'] = uuid.uuid4().hex + self.value['evidence'] = [{'id':'code','label':'Code','path':'recipe.py'}] + self.write_manifest() + self.open() + with self.assertRaises(WorkbenchError) as caught: + self.owner.evidence(self.value['id'], 'code') + self.assertEqual(caught.exception.code, 'unsupported') + + def test_unhashed_evidence_controls_remain_unavailable_and_changed_sources_are_explicit(self): + self.value['evidence'][0].pop('sha256') + self.write_manifest() + self.open() + for raw in (b'\x00binary', b'\x1b[31mterminal'): + (self.external / 'note.md').write_bytes(raw) + with self.assertRaises(WorkbenchError) as caught: + self.owner.evidence(self.value['id'], 'note') + self.assertEqual(caught.exception.code, 'unsupported') + (self.external / 'data.bin').write_bytes(b'changed native bytes') + inspected = self.owner.get(self.value['id'])['data'][0] + self.assertEqual(inspected['status'], 'available') + self.assertEqual(inspected['integrity'], 'mismatch') + self.assertEqual(inspected['sha256'], sha(self.files['data.bin'])) + self.assertEqual(inspected['observed_sha256'], sha(b'changed native bytes')) + (self.external / 'data.bin').unlink() + self.assertEqual(self.owner.get(self.value['id'])['data'][0]['status'], 'missing') + + def test_file_manifest_reference_and_total_inspection_budgets(self): + self.write_manifest(raw=b' ' * (projects.MANIFEST_LIMIT+1)) + with self.assertRaises(WorkbenchError) as caught: + self.open() + self.assertEqual(caught.exception.code, 'too_large') + self.value['data'] = [{'id':f'a{i}','label':'One','path':'data.bin'} for i in range(65)] + self.write_manifest() + with self.assertRaises(WorkbenchError): + self.open() + with (self.external / 'large.bin').open('wb') as handle: + handle.truncate(projects.FILE_LIMIT+1) + self.value['data'] = [{'id':'large','label':'Large','path':'large.bin','sha256':'0'*64}, + {'id':'small','label':'Small','path':'data.bin','sha256':sha(self.files['data.bin'])}] + self.write_manifest() + result = self.open() + self.assertEqual(result['data'][0]['integrity'], 'unverified') + self.assertNotIn('observed_sha256', result['data'][0]) + self.assertEqual(result['data'][1]['integrity'], 'verified') + with patch.object(projects, 'INSPECTION_LIMIT', len(self.files['data.bin'])): + result = self.owner.get(self.value['id']) + self.assertEqual(result['data'][1]['integrity'], 'verified') + self.assertNotIn('observed_sha256', result['recipes'][0]) + self.assertNotIn('observed_sha256', result['evidence'][0]) + + def test_request_misuse_invalid_ids_and_configuration_are_typed(self): + for body in (None, [], {}, {'root': '/tmp','manifest_path':'project.json'}, + {'root': {},'manifest_path':'project.json'}, {'root':'fixture','manifest_path':True}, + {'root':'fixture','manifest_path':'project.json','execute':True}): + with self.subTest(body=body), self.assertRaises(WorkbenchError): + self.owner.open(body) + for value in ('bad', '../project', True, uuid.uuid4().hex): + with self.assertRaises(WorkbenchError): + self.owner.get(value) + for roots in ({'invalid/alias':self.external}, {'valid':self.path/'absent'}): + with self.assertRaises(WorkbenchError): + projects.Projects(self.dataset, roots) + + def test_real_http_registration_detail_evidence_and_no_execution_route(self): + server = ThreadingHTTPServer(('127.0.0.1',0), make_handler(self.dataset)) + thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + self.addCleanup(server.server_close) + self.addCleanup(lambda: (server.shutdown(), thread.join())) + base = 'http://127.0.0.1:' + str(server.server_port) + '/api/workspace/projects' + def request(suffix='', body=None, raw=None): + raw = json.dumps(body).encode() if body is not None else raw + with urllib.request.urlopen(urllib.request.Request(base+suffix, data=raw, + headers={'Content-Type':'application/json'} if raw is not None else {})) as response: + return response.status, json.load(response) + self.assertEqual(request()[1]['roots'], ['fixture']) + status, opened = request(body={'root':'fixture','manifest_path':'project.json'}) + self.assertIn(status, (200,201)) + project = '/' + self.value['id'] + self.assertEqual(request(project)[1], opened) + self.assertEqual(request(project+'/evidence/note')[1]['text'], self.files['note.md'].decode()) + for suffix, payload in (('', None), (project, None), (project+'/evidence/note', None), + ('', json.dumps({'root':'fixture','manifest_path':'project.json'}).encode())): + hostile = urllib.request.Request(base+suffix, data=payload, + headers={'Host':'attacker.example', 'Content-Type':'application/json'}) + with self.subTest(host='attacker.example', suffix=suffix, method=hostile.get_method()), self.assertRaises(urllib.error.HTTPError) as caught: + urllib.request.urlopen(hostile) + self.assertEqual(caught.exception.code, 403) + self.assertEqual(json.loads(caught.exception.read())['code'], 'forbidden') + self.assertEqual(len(self.owner.list()['projects']), 1) + for suffix, raw in (('', b'{"root":"fixture","root":"fixture","manifest_path":"project.json"}'), + ('', b'{"x":'+b'['*1200+b'0'+b']'*1200+b'}'), + ('', b'{"root":"fixture","manifest_path":"../outside.json"}'), + (project+'/execute', b'{}')): + with self.subTest(suffix=suffix), self.assertRaises(urllib.error.HTTPError) as caught: + request(suffix, raw=raw) + self.assertIn(caught.exception.code, (400,404,405)) diff --git a/tests/test_spatial_storage.py b/tests/test_spatial_storage.py index c92878fc..e5eed255 100644 --- a/tests/test_spatial_storage.py +++ b/tests/test_spatial_storage.py @@ -1,4 +1,4 @@ -"""Real PostgreSQL schema4, exact spatial bundles and stopped-writer evolution.""" +"""Real PostgreSQL spatial storage, exact bundles and stopped-writer evolution.""" import base64 import copy import io @@ -14,7 +14,7 @@ from app import Dataset, make_handler from backup_metadata import backup_catalog, restore_catalog, read_catalog, summaries, tree_manifest -from metadata import Metadata, StorageError, definition_hash, TABLES_V3, SPATIAL_TABLES +from metadata import SCHEMA_VERSION, Metadata, StorageError, definition_hash, TABLES_V3, SPATIAL_TABLES from upgrade_metadata import upgrade_catalog, UPGRADES import spatial_assets import test_workspace_metadata @@ -50,7 +50,7 @@ def review(self, dataset, row): def test_real_both_subtypes_review_inspection_export_and_reopen(self): dataset = self.dataset() - self.assertEqual(dataset.db.schema_version, 4) + self.assertEqual(dataset.db.schema_version, SCHEMA_VERSION) rows = [] bundles = {} for kind in ('pointcloud', 'volume'): @@ -106,7 +106,7 @@ def test_schema3_requires_explicit_upgrade_and_preserves_old_evidence(self): with self.assertRaises(StorageError): upgrade_catalog(path) result = upgrade_catalog(path, quiesced=True) - self.assertEqual(result['migration'], 'spatial-metadata-3-to-4') + self.assertEqual(result['migration'], 'project-metadata-4-to-5') current = Metadata(path, create=False) self.addCleanup(current.close) after = read_catalog(current) @@ -131,7 +131,7 @@ def test_failed_schema4_transition_retains_exact_schema3(self): self.assertEqual(retained.execute('SELECT * FROM catalog_version').fetchall(), before) self.assertEqual(set(read_catalog(retained)), set(TABLES_V3)) - def test_schema4_backup_restore_retains_original_bytes_review_history_and_release(self): + def test_current_schema_backup_restore_retains_original_bytes_review_history_and_release(self): dataset = self.dataset() rows = [self.review(dataset, spatial_assets.admit(dataset.workbench, kind, body(kind))) for kind in ('pointcloud','volume')] request = {'items': [{key: row[key] for key in ('id', 'revision', 'source_revision')} for row in rows], @@ -145,7 +145,7 @@ def test_schema4_backup_restore_retains_original_bytes_review_history_and_releas dataset.close() backup = self.root / 'backup' receipt = backup_catalog(path, backup, quiesced=True) - self.assertEqual(receipt['schema_version'], 4) + self.assertEqual(receipt['schema_version'], SCHEMA_VERSION) self.assertTrue(set(SPATIAL_TABLES) <= set(receipt['tables'])) before = tree_manifest(backup) destination = self.root / 'restored' diff --git a/tests/test_workspace_metadata.py b/tests/test_workspace_metadata.py index a225388c..201f487b 100644 --- a/tests/test_workspace_metadata.py +++ b/tests/test_workspace_metadata.py @@ -20,7 +20,7 @@ read_sequences, restore_catalog, summaries, table_state, tree_manifest) from metadata import (CommitOutcomeUnknown, Metadata, SCHEMA_VERSION, StorageError, - TABLES_V1, TABLES_V2, WORKSPACE_TABLES, connection_info, + TABLES_V1, TABLES_V2, TABLES_V3, TABLES_V4, PROJECT_TABLES, WORKSPACE_TABLES, connection_info, definition_hash) from migrate_sqlite import migrate_sqlite, read_legacy from upgrade_metadata import UPGRADES, upgrade_catalog @@ -33,7 +33,7 @@ def test_retained_v1_definition_is_exactly_the_qualified_19_table_contract(self) self.assertEqual(len(TABLES_V1), 19) self.assertEqual(definition_hash(1), '859a04c5ee618faae689885bee79054d5e92098e746d058fd2bcb599da4c80db') - self.assertEqual(SCHEMA_VERSION, 4) + self.assertEqual(SCHEMA_VERSION, 5) self.assertEqual(set(WORKSPACE_TABLES), {'workspace_graphs', 'workspace_runs', 'workspace_exposures'}) @@ -45,6 +45,15 @@ def test_retained_v2_definition_is_exactly_the_qualified_22_table_contract(self) self.assertEqual([UPGRADES[version][0] for version in (1, 2)], ['workspace-metadata-1-to-2', 'audio-metadata-2-to-3']) + def test_retained_v3_v4_definitions_and_additive_v5_project_table(self): + self.assertEqual(len(TABLES_V3), 22) + self.assertEqual(definition_hash(3), '54fade5b56207edf7c9a33bfa9187f85b3e42ea779869314e7ab0fa8bdc242a4') + self.assertEqual(len(TABLES_V4), 24) + self.assertEqual(definition_hash(4), '52242c56f7b88144157b66d93b151d818cecf0b2219226d4aff5734e8a1adcef') + self.assertEqual(set(PROJECT_TABLES), {'workspace_projects'}) + self.assertEqual(UPGRADES[4], ('project-metadata-4-to-5', tuple(PROJECT_TABLES.values()))) + self.assertNotEqual(definition_hash(5), definition_hash(4)) + @unittest.skipUnless(os.environ.get('TULDOK_DATABASE_URL'), 'Requires disposable PostgreSQL TULDOK_DATABASE_URL') class WorkspaceMetadata(unittest.TestCase): @@ -116,7 +125,7 @@ def test_default_startup_refuses_v1_until_explicit_upgrade_preserving_history_id self.assertEqual(read_sequences(reopened), sequences) self.assertTrue(all(current[name]['count'] == 0 for name in WORKSPACE_TABLES)) self.assertEqual(reopened.dataset_id, identity) - self.assertEqual(reopened.schema_version, 4) + self.assertEqual(reopened.schema_version, 5) from app import Dataset dataset = Dataset(self.root / 'catalog') try: @@ -184,8 +193,8 @@ def test_v2_upgrade_preserves_populated_catalog_and_only_then_admits_audio(self) with self.assertRaisesRegex(StorageError, 'Stop all'): upgrade_catalog(metadata.path) result = upgrade_catalog(metadata.path, quiesced=True) - self.assertEqual(result['migration'], 'spatial-metadata-3-to-4') - self.assertEqual(result['schema_version'], 4) + self.assertEqual(result['migration'], 'project-metadata-4-to-5') + self.assertEqual(result['schema_version'], 5) current = Metadata(metadata.path, create=False) self.addCleanup(current.close) self.assertEqual(current.dataset_id, metadata.dataset_id) @@ -363,7 +372,7 @@ def transaction(*inner_args, **inner_kwargs): upgrade_catalog(metadata.path, quiesced=True) reopened = Metadata(metadata.path, create=False) self.addCleanup(reopened.close) - self.assertEqual(reopened.schema_version, 4) + self.assertEqual(reopened.schema_version, 5) self.assertEqual(upgrade_catalog(metadata.path, quiesced=True)['status'], 'already_current') def test_v1_backup_restores_exact_v1_and_can_explicitly_upgrade_with_backup_unchanged(self): @@ -495,6 +504,101 @@ def test_retained_v2_import_resumes_exact_receipt_and_reconciles_upgrade_until_a with self.assertRaisesRegex(ValueError, 'later writes'): migrate_sqlite(source, destination, backup, quiesced=True) + def test_v4_to_v5_preserves_history_and_only_explicitly_admits_projects(self): + metadata = self.historical_catalog(version=4) + self.workspace_rows(metadata) + self.audio_row(metadata) + rows = summaries(read_catalog(metadata)) + files = tree_manifest(metadata.path) + identity = metadata.dataset_id + with self.assertRaisesRegex(StorageError, 'explicit upgrade'): + Metadata(metadata.path, create=False) + with self.assertRaisesRegex(StorageError, 'Stop all'): + upgrade_catalog(metadata.path) + result = upgrade_catalog(metadata.path, quiesced=True) + self.assertEqual((result['schema_version'], result['migration']), (5, 'project-metadata-4-to-5')) + current = Metadata(metadata.path, create=False) + self.addCleanup(current.close) + self.assertEqual(current.dataset_id, identity) + self.assertEqual(summaries({name: value for name, value in read_catalog(current).items() if name in rows}), rows) + self.assertEqual(tree_manifest(current.path), files) + self.assertEqual(current.execute('SELECT count(*) FROM workspace_projects').fetchone()[0], 0) + self.assertEqual(upgrade_catalog(current.path, quiesced=True), dict(result, status='already_current')) + + def test_project_upgrade_failure_rolls_back_schema_history_and_ledger(self): + for source in (1, 4): + for stage in ('create', 'proof', 'ledger'): + with self.subTest(source=source, stage=stage): + metadata = self.catalog('project-rollback-' + str(source) + '-' + stage, source) + before = summaries(read_catalog(metadata)) + signature = metadata._signature() + ledger = metadata.execute('SELECT * FROM catalog_version').fetchall() + execute, proof = Metadata.execute, Metadata.expected_signature + def fail(owner, statement, parameters=None): + result = execute(owner, statement, parameters) + if stage == 'create' and statement == UPGRADES[4][1][0]: + raise OSError('injected after project table creation') + if stage == 'ledger' and isinstance(statement, str) and statement.startswith('UPDATE catalog_version') and parameters[0] == 5: + raise OSError('injected after project ledger write') + return result + def fail_proof(owner, target): + return 'wrong' if stage == 'proof' and target == 5 else proof(owner, target) + with patch.object(Metadata, 'execute', fail), patch.object(Metadata, 'expected_signature', fail_proof): + with self.assertRaises((StorageError, OSError)): + upgrade_catalog(metadata.path, quiesced=True) + self.assertEqual(metadata._signature(), signature) + self.assertEqual(metadata.execute('SELECT * FROM catalog_version').fetchall(), ledger) + self.assertEqual(summaries(read_catalog(metadata)), before) + metadata.validate_schema() + + def test_retained_v4_backup_restores_exact_old_schema_then_requires_upgrade(self): + metadata = self.historical_catalog(version=4) + self.workspace_rows(metadata) + before = summaries(read_catalog(metadata)) + backup = self.root / 'backup-v4' + receipt = backup_catalog(metadata.path, backup, quiesced=True) + self.assertEqual(receipt['schema_version'], 4) + self.assertEqual(set(receipt['tables']), set(TABLES_V4)) + backup_files = tree_manifest(backup) + self.drop(metadata) + restored = self.root / 'restored-v4' + restore_catalog(backup, restored) + with self.assertRaisesRegex(StorageError, 'explicit upgrade'): + Metadata(restored, create=False) + old = Metadata(restored, create=False, schema_version=4) + self.addCleanup(old.close) + self.assertEqual(summaries(read_catalog(old)), before) + old.close() + self.assertEqual(upgrade_catalog(restored, quiesced=True)['schema_version'], 5) + self.assertEqual(tree_manifest(backup), backup_files) + + def test_v5_backup_restore_preserves_exact_project_snapshot_and_history(self): + metadata = self.historical_catalog(version=5) + self.workspace_rows(metadata) + document = ' { "id":"project-1", "root_alias":"local", "source":"project.json", "sha256":"snapshot", "manifest":{"name":"原本 e\u0301"} } ' + with metadata.transaction(): + metadata.execute('INSERT INTO workspace_projects VALUES (%s,%s)', ('project-1', document)) + with self.assertRaises(psycopg.errors.UniqueViolation), metadata.transaction(): + metadata.execute('INSERT INTO workspace_projects VALUES (%s,%s)', ('project-1', '{}')) + rows = summaries(read_catalog(metadata)) + backup = self.root / 'backup-v5' + receipt = backup_catalog(metadata.path, backup, quiesced=True) + self.assertEqual(receipt['schema_version'], 5) + self.assertEqual(receipt['tables']['workspace_projects']['count'], 1) + backup_files = tree_manifest(backup) + self.drop(metadata) + restored = self.root / 'restored-v5' + restore_catalog(backup, restored) + current = Metadata(restored, create=False) + self.addCleanup(current.close) + self.assertEqual(summaries(read_catalog(current)), rows) + self.assertEqual(current.execute('SELECT document FROM workspace_projects').fetchone()[0], document) + restore_catalog(backup, restored) + current.execute("UPDATE workspace_projects SET document='{}' WHERE id='project-1'") + with self.assertRaisesRegex(ValueError, 'later writes'): + restore_catalog(backup, restored) + self.assertEqual(tree_manifest(backup), backup_files) + def test_upgrade_cli_reopens_catalog_and_never_runs_on_ordinary_startup(self): metadata = self.catalog() command = [sys.executable, str(Path(__file__).resolve().parents[1] / 'upgrade_metadata.py'), str(metadata.path)] diff --git a/upgrade_metadata.py b/upgrade_metadata.py index 98b01a97..4c580b6c 100644 --- a/upgrade_metadata.py +++ b/upgrade_metadata.py @@ -1,15 +1,16 @@ -"""Explicit quiesced PostgreSQL schema 1, 2 or 3 to 4 upgrade, without data rewriting.""" +"""Explicit quiesced PostgreSQL schema 1–4 to 5 upgrade, without data rewriting.""" import argparse import json import psycopg -from metadata import (Metadata, SCHEMA_VERSION, StorageError, WORKSPACE_TABLES, SPATIAL_TABLES, +from metadata import (Metadata, SCHEMA_VERSION, StorageError, WORKSPACE_TABLES, SPATIAL_TABLES, PROJECT_TABLES, definition_hash) # Accepted ordered transitions, with stable identities. The complete chain is # one transaction: no intermediate version is published if a later step fails. UPGRADES = { + 4: ('project-metadata-4-to-5', tuple(PROJECT_TABLES.values())), 1: ('workspace-metadata-1-to-2', tuple(WORKSPACE_TABLES.values())), 3: ('spatial-metadata-3-to-4', ( 'ALTER TABLE workbench_records DROP CONSTRAINT workbench_records_kind_check', @@ -28,7 +29,7 @@ def upgrade_catalog(path, *, dsn=None, quiesced=False): Stop every old/new app and artifact writer before calling. The advisory lock coordinates cooperating commands; it is not a fence against a live old app. - Re-entry verifies current schema 4, including its independently known shape, + Re-entry verifies current schema 5, including its independently known shape, and performs no writes to domain rows, artifacts or the dataset manifest. """ if not quiesced: