diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 51529b6b..e6eaaa63 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,7 @@ jobs: --health-retries 12 env: TULDOK_TRAINING_PYTHON: ${{ github.workspace }}/build/workspace-training/bin/python + TULDOK_NATIVE_READER_PYTHON: ${{ github.workspace }}/build/workspace-training/bin/python TULDOK_DATABASE_URL: postgresql://tuldok_test:isolated-ci-fixture@127.0.0.1:5432/tuldok_test steps: - uses: actions/checkout@v4 @@ -49,6 +50,11 @@ jobs: - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_pointclouds.py - run: build/workspace-training/bin/python -m unittest discover -s tests -p test_volume_assets.py - run: python -m unittest discover -s tests + - run: node --check static/scientific-fields.js + - run: build/workspace-training/bin/python tests/qualify_scientific_fields.py --output build/qa/scientific-neutral.zip --report build/qa/scientific-neutral.json + - run: node tests/browser_scientific_fields.cjs + env: + BROWSER: /usr/bin/google-chrome - run: node --check static/workspace-projects.js - run: node tests/browser_workspace_project.cjs env: diff --git a/README.md b/README.md index ebeef7c4..d787dba9 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ current record; saving/reviewing and freezing a new release leaves old results unchanged. Viewing test results records durable exposure; reused records or pixels cannot be presented as fresh holdout evidence. -Fresh catalogs use schema 5. Existing PostgreSQL schema-1 through schema-4 catalogs require an +Fresh catalogs use schema 6. Existing PostgreSQL schema-1 through schema-5 catalogs require an explicit stopped-writer backup and upgrade, never an automatic startup change: ```sh @@ -852,8 +852,12 @@ fixtures and do not download model weights or call paid inference providers. ### Point-cloud and volume workflows -In Data, import `points.ply` with `points.json`, or `volume.npy` with `volume.json`. The bounded profiles are [ordered ASCII PLY points](docs/contracts/point-cloud-data.md) and [uint8 NPY1 volumes](docs/contracts/volume-data.md). Inspect native points or index slices/physical coordinates, save human review, select whole assets and freeze a canonical release. In Pipeline, create and edit the corresponding validation/export recipe; Runs retains its integrity report and the unchanged release. Classification releases also offer validation/export without installing a trainer. Existing catalogs require the explicit schema5 upgrade described in the storage runbook; the application never silently migrates them. [Accepted workspace scope, evidence and limitations](docs/plans/dataset-centered-workspaces/reports/final-20261013.md). +In Data, import `points.ply` with `points.json`, or `volume.npy` with `volume.json`. The bounded profiles are [ordered ASCII PLY points](docs/contracts/point-cloud-data.md) and [uint8 NPY1 volumes](docs/contracts/volume-data.md). Inspect native points or index slices/physical coordinates, save human review, select whole assets and freeze a canonical release. In Pipeline, create and edit the corresponding validation/export recipe; Runs retains its integrity report and the unchanged release. Classification releases also offer validation/export without installing a trainer. Existing catalogs require the explicit schema6 upgrade described in the storage runbook; the application never silently migrates them. [Accepted workspace scope, evidence and limitations](docs/plans/dataset-centered-workspaces/reports/final-20261013.md). ## Open a local project -Start with `python app.py --data ./catalog --project-root research=/absolute/research`, then select **Open project** in `/workspaces`. A [project manifest](docs/contracts/projects.md) references files beneath that explicitly selected root. Data shows sources and missing requirements, Pipeline shows recipe references, and Runs shows external evidence. This stage is **inspection only**: no project code, model loading, runtime probing or training executes. Existing catalog workflows remain separate. Opening registers a snapshot in Tuldok; it does not modify the opened project. +Start with `python app.py --data ./catalog --project-root research=/absolute/research`, then select **Open project** in `/workspaces`. A [project manifest](docs/contracts/projects.md) references files beneath that explicitly selected root. Data shows sources and missing requirements, Pipeline shows recipe references, and Runs shows external evidence. Opening performs inspection: no project code, model loading, runtime probing or training executes. Existing catalog workflows remain separate. Opening registers a snapshot in Tuldok; it does not modify the opened project. + +## Scientific field sequences + +For a conforming [scientific field bundle](docs/contracts/scientific-fields.md), use **Import fields** on its project Data reference. Inspect frames, fields and index slices with physical coordinates, units and producer stage labels; save whole-trajectory review and freeze a canonical release. Original bundle and floating-point bytes remain unchanged. First-observed-state and declared-family grouping does not certify independent initial conditions. The cold reader `python scientific_release.py RELEASE.zip` verifies frozen custody; generation and training require later execution milestones. diff --git a/app.py b/app.py index 31bf8584..42780131 100644 --- a/app.py +++ b/app.py @@ -480,6 +480,13 @@ def do_GET(self): if path == '/api/workbench/recipes': return self.reply(dataset_recipes.RECIPES) if path.startswith('/api/workbench/records/'): + if path.endswith('/scientific'): + options = parse_qs(urlsplit(self.path).query, keep_blank_values=True) + if set(options) - {'frame', 'field', 'axis', 'index'} or any(len(values) != 1 for values in options.values()): + raise workbench.WorkbenchError('Choose one frame, field, axis and slice index.') + return self.reply(dataset.workbench.scientific.inspect(path.split('/')[-2], + frame=int(options.get('frame', ['0'])[0]), field=options.get('field', [None])[0], + axis=options.get('axis', ['z'])[0], index=int(options.get('index', ['0'])[0]))) if path.endswith('/preferences'): return self.reply(dataset.workbench.preferences.list(path.split('/')[-2])) if path.endswith('/responses'): @@ -557,6 +564,7 @@ def do_GET(self): '/workspace-projects.js': ('workspace-projects.js', 'text/javascript'), '/pointclouds.js': ('pointclouds.js', 'text/javascript'), '/volume-editor.js': ('volume-editor.js', 'text/javascript'), + '/scientific-fields.js': ('scientific-fields.js', 'text/javascript'), '/audio-editor.js': ('audio-editor.js', 'text/javascript'), '/audio-editor.css': ('audio-editor.css', 'text/css'), '/meshes.css': ('meshes.css', 'text/css'), @@ -605,7 +613,7 @@ def do_POST(self): if not 0 < length <= limit: raise ValueError('Request is too large or empty.') raw_body = self.rfile.read(length) - body = projects.decode(raw_body) if path == '/api/workspace/projects' else spatial_assets.parse_request(raw_body) if path in ('/api/workbench/pointcloud-import', '/api/workbench/volume-import') else native_detection_import.parse_request(raw_body) if path in ('/api/workbench/native-detection-import/prepare', '/api/workbench/native-detection-import/row') else saved_searches.parse_request(raw_body) if path == '/api/workbench/searches' or path.startswith('/api/workbench/searches/') else sequence_batch_import.parse_request(raw_body) if path in ('/api/workbench/sequence-import', '/api/workbench/sequence-import-item') else meshes.parse_json(raw_body) if path == '/api/workbench/mesh-import' else json.loads(raw_body) + body = projects.decode(raw_body) if path == '/api/workspace/projects' or path.startswith('/api/workspace/projects/') else spatial_assets.parse_request(raw_body) if path in ('/api/workbench/pointcloud-import', '/api/workbench/volume-import') else native_detection_import.parse_request(raw_body) if path in ('/api/workbench/native-detection-import/prepare', '/api/workbench/native-detection-import/row') else saved_searches.parse_request(raw_body) if path == '/api/workbench/searches' or path.startswith('/api/workbench/searches/') else sequence_batch_import.parse_request(raw_body) if path in ('/api/workbench/sequence-import', '/api/workbench/sequence-import-item') else meshes.parse_json(raw_body) if path == '/api/workbench/mesh-import' else json.loads(raw_body) if not isinstance(body, dict): raise ValueError('Expected an object.') path = urlsplit(self.path).path @@ -613,6 +621,12 @@ def do_POST(self): parts = path.removeprefix('/api/workspace/').split('/') if parts == ['projects']: return self.reply(dataset.projects.open(body), 201) + if len(parts) == 4 and parts[0] == 'projects' and parts[2] == 'import-data': + projects.closed(body, ('expected_sha256',), ('name', 'rights', 'groups', 'parents')) + raw, origin = dataset.projects.read_data_reference(parts[1], parts[3], + body['expected_sha256'], projects.DATA_LIMIT) + admission = {key: value for key, value in body.items() if key != 'expected_sha256'} + return self.reply(dataset.workbench.scientific.admit(raw, admission, origin), 201) if parts == ['graphs']: return self.reply(dataset.pipelines.create(body), 201) if len(parts) == 2 and parts[0] == 'graphs': diff --git a/audio_assets.py b/audio_assets.py index 6398dc3f..c36b06e5 100644 --- a/audio_assets.py +++ b/audio_assets.py @@ -2,7 +2,6 @@ import base64 import hashlib import io -import os import struct import uuid import wave @@ -92,21 +91,13 @@ def validate_annotation(value, record): return {'segments': result} -def _sync_directory(path): - descriptor = os.open(path, os.O_RDONLY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - - class AudioAssets: def __init__(self, workbench): self.workbench = workbench self.path = workbench.dataset.path / 'audio' def admit(self, body): - from metadata import CommitOutcomeUnknown + from immutable_assets import publish_file if type(body) is not dict or set(body) - {'wav', 'name', 'rights', 'groups', 'parents'}: invalid('import expects wav, name, rights, groups and optional parents.') encoded = body.get('wav') @@ -132,35 +123,17 @@ def admit(self, body): record_id, created = uuid.uuid4().hex, timestamp() folder = self.path / record_id w = self.workbench - def discard(): - if folder.exists(): - (folder / 'source.wav').unlink(missing_ok=True) - folder.rmdir() - try: - with w.lock, w.db.transaction(): - for parent in parents: - w._get(parent) - if w.db.execute("SELECT 1 FROM workbench_records WHERE kind='audio' AND content_hash=%s", (digest,)).fetchone(): - invalid('this original WAV already exists; open its existing record.', 'conflict', 409) - self.path.mkdir(exist_ok=True) - folder.mkdir() - # Successful savepoints pass custody to their outer transaction. - w.db.on_rollback(discard) - with (folder / 'source.wav').open('xb') as target: - target.write(raw); target.flush(); os.fsync(target.fileno()) - _sync_directory(folder); _sync_directory(self.path); _sync_directory(self.path.parent) + with w.lock, w.db.transaction(): + for parent in parents: + w._get(parent) + if w.db.execute("SELECT 1 FROM workbench_records WHERE kind='audio' AND content_hash=%s", (digest,)).fetchone(): + invalid('this original WAV already exists; open its existing record.', 'conflict', 409) + with publish_file(w.db, folder, 'source.wav', raw): w.db.execute('INSERT INTO workbench_records VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', (record_id, 'audio', name, None, None, digest, None, encode(groups), encode(parents), encode(origin), TASK, None, 'draft', 1, created, created)) record = w._get(record_id) w._history(record) - except CommitOutcomeUnknown: - # A lost PostgreSQL commit acknowledgement may already have published this reference. - # Retain its bytes; the existing catalog reconciliation owns the next action. - raise - except Exception: - discard() - raise return record def metadata(self, record_id, provenance): diff --git a/canonical_release.py b/canonical_release.py index 1f9a8e38..4ec3ee8b 100644 --- a/canonical_release.py +++ b/canonical_release.py @@ -69,7 +69,7 @@ def archive(raw, maximum_entries, maximum_bytes): def validate_bytes(raw, expected_sha256, *, kind, validate_row): """Return custody and subtype summaries for one supported whole-asset kind.""" - if kind not in ('mesh', 'sequence', 'pointcloud', 'volume'): + if kind not in ('mesh', 'sequence', 'pointcloud', 'volume', 'field_sequence'): raise ValueError('Unsupported canonical release reader.') require(type(raw) is bytes and 0 < len(raw) <= MAX_RELEASE_BYTES, 'select a release of at most 64 MiB.') digest = sha(raw) @@ -106,11 +106,12 @@ def validate_bytes(raw, expected_sha256, *, kind, validate_row): and set(report['actual_counts']) <= set(SPLITS) and all(type(n) is int and n >= 0 for n in report['actual_counts'].values()) and {split: report['actual_counts'].get(split, 0) for split in SPLITS} == counts, 'split report counts disagree with records.') + row_limit = MAX_METADATA_BYTES if kind == 'field_sequence' else MAX_ROW_BYTES for split in SPLITS: raw_rows = release.read(split + '/records.jsonl') require(not raw_rows or raw_rows.endswith(b'\n'), 'split JSONL must end with a newline.') lines = raw_rows.splitlines() - require(all(0 < len(line) <= MAX_ROW_BYTES for line in lines), 'split JSONL row exceeds 256 KiB or is empty.') + require(all(0 < len(line) <= row_limit for line in lines), 'split JSONL row exceeds its subtype metadata bound or is empty.') require([encode(decode(line)) for line in lines] == [encode(row) for row in rows if row['split'] == split], 'split JSONL differs from frozen manifest records.') coco = decode(release.read(split + '/coco.json')) require(type(coco) is dict and set(coco) == {'info', 'licenses', 'images', 'annotations', 'categories'} diff --git a/caption_import.py b/caption_import.py index 3bd5eef4..bbbc8baa 100644 --- a/caption_import.py +++ b/caption_import.py @@ -62,7 +62,7 @@ def snapshot(row): invalid('Invalid protected source snapshot fields.') if not isinstance(row['id'], str) or not IDENTIFIER.fullmatch(row['id']): invalid('Invalid origin source ID.') - if row['kind'] not in ('image', 'text', 'sequence', 'mesh', 'pointcloud', 'volume') or type(row['revision']) is not int or row['revision'] < 1: + if row['kind'] not in ('image', 'text', 'sequence', 'mesh', 'pointcloud', 'volume', 'field_sequence') or type(row['revision']) is not int or row['revision'] < 1: invalid('Invalid origin kind or revision.') if type(row['source_available']) is not bool or row['source_lineage_known'] is not True: invalid('Protected source lineage must be known, including deleted members.') @@ -89,7 +89,7 @@ def snapshot(row): invalid('Invalid retained book/session links.') if row['kind'] == 'text' and (row['book_id'] is not None or row['session_id'] is not None or not is_hash(row['content_hash']) or not is_hash(row['source_sha256']) or row['pixel_hash'] is not None): invalid('Invalid protected text source snapshot.') - if row['kind'] in ('sequence', 'mesh', 'pointcloud', 'volume') and (row['book_id'] is not None or row['session_id'] is not None + if row['kind'] in ('sequence', 'mesh', 'pointcloud', 'volume', 'field_sequence') and (row['book_id'] is not None or row['session_id'] is not None or not is_hash(row['content_hash']) or row['source_sha256'] != row['content_hash'] or row['pixel_hash'] is not None or type(row['source_revision']) is not int or row['source_revision'] != 1 or row['source_split'] != 'unassigned'): diff --git a/dataset_releases.py b/dataset_releases.py index 0be1ec72..7d4cf1f3 100644 --- a/dataset_releases.py +++ b/dataset_releases.py @@ -236,7 +236,7 @@ def block(error, record_id=None): raise WorkbenchError('Caption export requires available images with human-reviewed image-caption annotations.') elif not row['source_available'] or row['annotation'] is None or row['review'] == 'draft': raise WorkbenchError('Every selected record needs an available source and reviewed or programmatically verified annotation.') - if row['kind'] in ('sequence', 'mesh', 'pointcloud', 'volume') and row['review'] != 'human_reviewed': + if row['kind'] in ('sequence', 'mesh', 'pointcloud', 'volume', 'field_sequence') and row['review'] != 'human_reviewed': raise WorkbenchError('Sequence export requires human-reviewed whole trajectories.' if row['kind'] == 'sequence' else 'Spatial export requires human-reviewed whole asset records.') if row['kind'] == 'audio': audio_export.validate_record(row) @@ -312,6 +312,10 @@ def block(error, record_id=None): except WorkbenchError as error: block(error) if format_name == 'canonical_v1': + if any(row['kind'] == 'field_sequence' for row in rows): + if len(rows) > 64 or sum(len(encode(row).encode()) + 512 for row in rows) > 8 * 1024 * 1024 - 65536: + block(WorkbenchError('Scientific releases permit at most 64 records and 8 MiB of frozen metadata.')) + preview['warnings'].append('Scientific fields retain exact arrays, grid placement, time and producer stages. Whole trajectories stay indivisible; first observed states and declared families do not certify independent initial conditions.') if any(row['kind'] == 'audio' for row in rows): preview['warnings'].append(audio_export.CONTRACT) if any(row['kind'] == 'mesh' for row in rows): @@ -701,6 +705,8 @@ def _create(self, body): 'records': []} if any(row['kind'] in ('pointcloud', 'volume') for row in rows): manifest['coordinate_contract'] += ' Point clouds preserve ordered points and declared frame; volumes preserve uint8 voxel values and voxel-center affine; whole spatial assets are indivisible.' + if any(row['kind'] == 'field_sequence' for row in rows): + manifest['coordinate_contract'] += ' Scientific field sequences preserve exact floating values, C-order ZYX arrays, declared grid transform, cell/face offsets, times and producer stages; each whole trajectory is indivisible. First-observed-state identity is not proof of initial-condition independence.' vocabulary = sorted({target['label'] for row in rows if row['task'] == 'image_detection' for target in row['annotation']['boxes']}) categories = {label: i + 1 for i, label in enumerate(vocabulary)} coco = {split: {'info': {'description': 'Tuldok detection release', 'version': '1'}, 'licenses': [], 'images': [], 'annotations': [], 'categories': [{'id': i, 'name': label} for label, i in categories.items()]} for split in SPLITS} @@ -712,7 +718,7 @@ def _create(self, body): for index, row in enumerate(rows, 1): split = assignments[row['id']] asset, _ = workbench.asset(row['id']) - filename = 'assets/' + row['id'] + {'image': '.png', 'text': '.txt', 'sequence': '.zip', 'mesh': '.zip', 'audio': '.wav', 'pointcloud': '.zip', 'volume': '.zip'}[row['kind']] + filename = 'assets/' + row['id'] + {'image': '.png', 'text': '.txt', 'sequence': '.zip', 'mesh': '.zip', 'audio': '.wav', 'pointcloud': '.zip', 'volume': '.zip', 'field_sequence': '.zip'}[row['kind']] digest = archive_asset(archive, asset, filename, row['content_hash']) record = dict(row, split=split, asset=filename, asset_sha256=digest) if row['kind'] == 'audio': @@ -727,6 +733,8 @@ def _create(self, body): else: text_rows[split].append(record) manifest['vocabulary'] = vocabulary + if any(row['kind'] == 'field_sequence' for row in rows) and len(encode(manifest).encode()) > 8 * 1024 * 1024: + raise WorkbenchError('Scientific release metadata exceeds 8 MiB.') archive.writestr(zipfile.ZipInfo('manifest.json'), encode(manifest)) for split in SPLITS: archive.writestr(zipfile.ZipInfo(split + '/coco.json'), encode(coco[split])) diff --git a/docs/contracts/projects.md b/docs/contracts/projects.md index 47c0be29..3baf6b05 100644 --- a/docs/contracts/projects.md +++ b/docs/contracts/projects.md @@ -1,9 +1,7 @@ # Project inspection v1 Owner: `projects.py:Projects`; PostgreSQL schema5 owns immutable registration -snapshots. HTTP and the workspace UI project the same owner. This first stage -inspects references only: project files are not imported as Data records, -recipes are not executable graphs, and external evidence is not a Tuldok Run. +snapshots. HTTP and the workspace UI project the same owner. Opening inspects references without importing records. A separate explicit action may import a conforming scientific field bundle; recipe references remain non-executable and external evidence is not a Tuldok Run. ## Opening and reference schema @@ -65,7 +63,7 @@ cannot block the file reader. Root selection itself is a trusted local startup action and resolves the configured directory. All project access is read-only; opening writes only Tuldok's catalog registration. -Inspection hashes at most8MiB per referenced file and64MiB total per detail +Inspection hashes at most40MiB per data reference,8MiB per other referenced file and64MiB total per detail request. Larger regular files are reported present with unverified contents. Bounded reads check size and modification identity before/after and refuse observed changes. Presence, measured byte hash, equality to a declared hash and @@ -117,3 +115,7 @@ Schema version, project manifest version and later scientific/recipe versions have independent owners. Runtime requirements are declared file references in v1; later qualified execution must use its own admitted runtime/permission contract rather than treating presence as capability. + +## Explicit scientific data import + +POST `/api/workspace/projects/{id}/import-data/{reference_id}` selects only a saved data reference. Body requires `expected_sha256` from the displayed inspection; optional `name`, `rights`, `groups`, `parents` describe the new catalog record. The same root binding and no-follow file walk apply. Both the selected observed hash and any manifest-declared hash must match captured bytes before publication; stale selections return409. The generic project owner captures bytes and source identity; the [scientific data owner](scientific-fields.md) validates their format and owns immutable catalog custody. Opening/getting never imports automatically. Import leaves source files and project registration unchanged. diff --git a/docs/contracts/scientific-fields.md b/docs/contracts/scientific-fields.md new file mode 100644 index 00000000..37dfc499 --- /dev/null +++ b/docs/contracts/scientific-fields.md @@ -0,0 +1,90 @@ +# Scientific field sequence custody (schema 1) + +`scientific_fields.py` owns a generic, lossless stored-field format and its local +artifact service. It does not import or execute producer code, authenticate +scientific claims, infer units, resample arrays, or run a simulation/model. +`parse_bundle(bytes)` validates the complete source and returns bounded metadata. +`ScientificAssets` publishes the exact original ZIP under +`field_sequence//source.zip`, with the PostgreSQL catalog retaining its +identity, provenance, review history, and protected groups. Shared +`immutable_assets.publish_file` owns filesystem publication and transaction rollback. +Unknown commit outcomes retain the artifact pending reconciliation. + +## Bundle + +The original ZIP is at most 40 MiB and contains only uncompressed regular files. +It has exactly one `manifest.json` (UTF-8 JSON, at most 1 MiB) and its declared +array/source members. Duplicate names/JSON keys, archive comments, traversal, +symlinks, extra members, compressed or encrypted entries, and trailing archive +bytes are rejected. Relative member paths have at most 160 characters and eight +components. Parsing never extracts files. Manifest nesting is bounded to 16. + +The manifest has exactly `format`, `schema_version`, `grid`, `time_units`, +`fields`, `frames`, `family`, `sources`, and `provenance`. Format is +`scientific_field_sequence_v1`, schema version is integer `1`. + +- `grid`: `shape_xyz` contains three cell counts (1–128); `origin_xyz` and positive + `spacing_xyz` contain finite values of absolute magnitude at most 1e12. + `orientation` is a row-major orthonormal 3×3 matrix (tolerance 1e-9), allowing + rotations and reflections. `units` is an explicit bounded string. +- `fields`: 1–16 unique entries with `id`, `label`, `dtype`, `centering`, `units`, + and `stage`. IDs are 1–64 ASCII token characters. Dtype is `/source.zip`. +The scientific domain owner validates its manifest, numerical samples and +annotation contract. Storage does not interpret physics or producer metadata. + Existing domain rows, other constraints, indexes, sequence high-water marks, stored JSON spelling, manifest identity and artifact bytes are unchanged by upgrade. ## Upgrade an existing catalog -Fresh catalogs use schema 5. Ordinary application startup **does not migrate** -an existing schema-1 through schema-4 catalog. It refuses with instructions to run the +Fresh catalogs use schema 6. Ordinary application startup **does not migrate** +an existing schema-1 through schema-5 catalog. It refuses with instructions to run the explicit upgrade. Stop every application, provider/training worker and artifact writer; keep them stopped throughout backup, upgrade and verification. The catalog advisory lock coordinates cooperating commands, but cannot fence an old binary @@ -57,7 +66,7 @@ python app.py --data ./dataset The backup destination must be new. Keep the complete backup and its receipt. The accepted transitions have stable identities `workspace-metadata-1-to-2`, -`audio-metadata-2-to-3`, `spatial-metadata-3-to-4` and `project-metadata-4-to-5`. The command applies the ordered steps needed by the +`audio-metadata-2-to-3`, `spatial-metadata-3-to-4`, `project-metadata-4-to-5` and `field-sequence-metadata-5-to-6`. The command applies the ordered steps needed by the recognized source, with the complete chain in one transaction. It verifies: 1. The manifest, catalog identity, version ledger and exact retained definition. @@ -74,26 +83,26 @@ recognized source, with the complete chain in one transaction. It verifies: All DDL and version ledger updates commit in one transaction under the existing dataset advisory lock. A failed second step rolls back the first step as well; the old kind constraint is never authoritatively absent. Only the known version-1, -version-2, version-3 or version-4 definitions are eligible. Unknown versions, altered constraints, +version-2, version-3, version-4 or version-5 definitions are eligible. Unknown versions, altered constraints, indexes, tables or triggers, malformed ledgers, missing catalogs and contradictory identities fail closed. No command refreshes a signature to bless unsupported drift. The manifest remains version 1; manifest and SQL schema versions are different contracts. The command prints the dataset identity, final migration name -(`project-metadata-4-to-5`), final schema version, definition/structure hashes and +(`field-sequence-metadata-5-to-6`), final schema version, definition/structure hashes and `upgraded` or `already_current`. A repeat on a -known schema-5 catalog verifies its structure and leaves rows/files unchanged. +known schema-6 catalog verifies its structure and leaves rows/files unchanged. Run one upgrade/transfer command at a time. A command failure rolls back its transaction. A lost commit acknowledgement is explicitly an unknown outcome: keep writers stopped, reopen, and rerun this same command to verify whether the -catalog is still the known source version or already known schema 5. Never +catalog is still the known source version or already known schema 6. Never manually change the ledger, delete a table or replay a training request to resolve it. ## Backup, restore and legacy imports The backup container remains `format_version: 1` and explicitly records -`schema_version: 1`, `2`, `3`, `4` or `5`. `backup_metadata.py` accepts each recognized schema. +`schema_version: 1`, `2`, `3`, `4`, `5` or `6`. `backup_metadata.py` accepts each recognized schema. Restore recreates **the exact backed-up schema version**, identity, rows, sequence states and ordinary files in an absent catalog/new directory. A retained schema-1, schema-2 or schema-3 backup remains restorable; upgrade that restored copy explicitly @@ -106,11 +115,11 @@ specified in [local storage and recovery](local-storage.md). `Metadata(..., schema_version=1)`, `schema_version=2`, `schema_version=3` and `schema_version=None` are tooling entry points for exact old-version restoration and recognized-version inspection. -Normal Dataset and Workbench callers omit that argument and require schema 5. +Normal Dataset and Workbench callers omit that argument and require schema 6. Version selection never authorizes an unsupported definition or structural drift. Legacy SQLite preflight still recognizes only the original 19 owner tables. -A new import creates schema 5 and leaves workspace/project tables empty. +A new import creates schema 6 and leaves workspace/project tables empty. A retained schema-1 or schema-2 migration receipt remains byte-for-byte authoritative: an interrupted import continues at its receipt's version, and a completed import can be reconciled after an explicit upgrade only while its original rows/files still @@ -119,7 +128,7 @@ never rewritten to imply that later workspace or audio writes belonged to the ol There is no automatic downgrade. Before any new writes, retain and restore the complete pre-upgrade backup to a separate destination/database if returning to -the older binary is necessary. After new writes, retain the schema-5 catalog and +the older binary is necessary. After new writes, retain the schema-6 catalog and artifacts and reconcile/export them under an explicit recovery decision; an old backup cannot represent those later writes. @@ -150,3 +159,37 @@ legacy revision and retains the qualification report with QA artifacts. ## Project registration (schema5) Schema5 adds only `workspace_projects(id TEXT PRIMARY KEY, document TEXT NOT NULL)` for immutable project manifest snapshots. The schema4 definition remains exactly `52242c56f7b88144157b66d93b151d818cecf0b2219226d4aff5734e8a1adcef`. Schema5 backup/restore includes project rows but does not copy external project files or grant a new filesystem binding. See [project inspection](projects.md). Tests retain exact v4 backups and prove the explicit v4→5 transition, transactional rollback and restored project snapshots alongside old domain rows. The retained-run qualifier now targets schema5 while preserving its actual historical schema3 source and backup. + +## Local single-file publication + +`immutable_assets.publish_file(db, folder, filename, raw)` is shared by audio +and scientific field sequences. Callers hold the Workbench lock, supply a +server-owned unique record directory and fixed filename, and publish records +and history inside the context. It joins an active metadata transaction or +owns one when called independently. It creates only the new record directory, +writes the original bytes using exclusive creation, and flushes/fsyncs the +file, record directory, modality directory and catalog directory before +allowing metadata publication. + +The rollback callback belongs to the metadata transaction. Successful nested +savepoints transfer cleanup custody to the outer transaction; a confirmed +rollback removes only the newly created source and record directory. An +existing destination collision is refused without deleting owner bytes. +Unknown commit acknowledgement retains source bytes for catalog reconciliation. +The context is a fixed local file operation, not an archive extractor or a +second persistence framework. Complete catalog backups retain these ordinary +files alongside exact project, record and history rows. + +Selected immutable assets share a 40 MiB synchronous bound: embedded bundle +bytes plus audio source bytes plus field-sequence source bytes. Individually +valid sources cannot evade the combined bound by mixing modalities. +`tests/test_audio_assets.py` proves sync ordering, rollback/savepoint custody, +unknown-commit retention, collision preservation and the combined bound; +existing audio import/history/backup tests remain compatibility evidence. + +`tests/test_spatial_storage.py` also imports the producer-neutral scientific +bundle, saves human review and history, then backs up and restores schema 6. +It compares exact source ZIP bytes, metadata rows, history and unchanged backup +files after reopen. The retained schema-3 real Run qualification now verifies +upgrade and restore into schema 6 without changing any saved result bytes. +These are storage and numerical-custody claims, not physics qualification. diff --git a/docs/plans/generic-scientific-projects/execution-ledger.md b/docs/plans/generic-scientific-projects/execution-ledger.md index 93301920..629a843c 100644 --- a/docs/plans/generic-scientific-projects/execution-ledger.md +++ b/docs/plans/generic-scientific-projects/execution-ledger.md @@ -33,3 +33,17 @@ Implemented immutable generic project registration, explicit root binding, bound Independent review fixes: reject lone surrogate input; return bytes matching actual hashed read; explain immutable changed-manifest snapshots honestly; clean root handles on malformed internal configuration. Final source review reports no remaining blocker. Hosted exact-head checks remain pending until publication. Next-stage research found no raw lab capture in the checkout/releases and no native toolchain initially configured. Evaluation NPZ omits RHS; training NPZ omits corrected fields. A separately isolated, pinned native two-frame capture is being prepared as the smallest P2 contract observation; no training or model execution. It is new evidence, not reconstruction of historical artifacts. + +## 2026-10-13 — P1 hosted qualification and P2 admission + +P1 published as draft PR45 at 46384fca; exact-head UI14/14 and lifecycle122/122 hosted steps passed. Updated PR evidence through REST after the installed gh edit command hit GitHub deprecated classic-project GraphQL fields. No merge. Created feature/scientific-field-sequences-20261013 for P2. Actual native two-frame capture and independent cold verification now resolve the sample-availability prerequisite; algorithm/model qualification remains pending. Root prepared bounded contract and write set; independent design review precedes implementation. + +## 2026-10-13 — P2 implementation and independent qualification + +Root owns project capture/API, Workbench and canonical release composition. recipe_design owns scientific parser/service/contract; volume_assets owns schema6/sharedfilepublication/storage; mesh_ui owns UI; point_assets owns independent converter/browser; milestone_audit owns cold release reader; mesh_review independently reviews product. Exact write set expanded for existing canonical snapshot kind and scoped metadata bound before dependent edits. + +Implemented typed CZYX float sequence custody, pinned observation import, stage/time/grid-aware viewer, whole-trajectory review/release, cold reader, explicit5→6 upgrade and old+new restore. Native and rotated neutral offline browser gates pass with independent NumPy slices/original arrays, restart and held-refresh clean-selection race. Review fixes bind duplicated frozen provenance and protect newer UI intent. Server-only file helper import was moved into audio admission after broad tests exposed the pinned audio worker dependency regression. + +Final stable-source suite is in progress; see [P2 verification](reports/p2-verification.md) for completed evidence and boundaries. No training/physics acceptance, merge or deployment. Read-only P3 preparation found existing Runs assumes childless trainers and cgroups are read-only; later external execution requires explicit descendant ownership qualification. P5 installedTorch2.8CPU differs from declaredPumas2.14 target, so actual runtime compatibility remains unqualified. + +P2 final stable source:696 tests passed in566.842s, three optional-reader skips independently covered by20 spatial and6 scientific-release tests without skips. Final source hash inventory unchanged. Plan validator/local links/Python-JavaScript syntax/whitespace pass. Ready to commit/push draft; hosted exact-head CI remains acceptance prerequisite before P3 implementation. diff --git a/docs/plans/generic-scientific-projects/plan.md b/docs/plans/generic-scientific-projects/plan.md index 567a91ca..457cfc84 100644 --- a/docs/plans/generic-scientific-projects/plan.md +++ b/docs/plans/generic-scientific-projects/plan.md @@ -2,9 +2,9 @@ **Plan status:** `Active` -**Current phase:** P1 locally verified on feature/generic-project-opening-20261013; final hosted CI pending. Plan published as PR44. +**Current phase:** P1 qualified at published commit 46384fca; P2 implemented and undergoing final qualification on feature/scientific-field-sequences-20261013. -**Next slice:** P1 — qualify the published project-opening implementation through final CI before P2 admission. +**Next slice:** P2 — lossless scientific field sequences, source custody, review and freeze. **Acceptance status:** `partial` @@ -18,7 +18,7 @@ Open an existing scientific repository as a Tuldok project, acquire lossless dat Use `MrScripty/rheon-surrogate-lab` as the concrete complete acceptance workflow. Wrap its proven generation, modal pressure fitting and autonomous rollout evaluation scripts before considering algorithm replacement. This plan does not promise unrestricted scientific software execution or arbitrary 3D model support. -The user approved publishing this plan and staged implementation on 2026-10-13. Commit/push and draft PR creation are authorized; merge and deployment are not. Start P1 after plan publication and progress to later stages only after their prerequisite gates qualify. All product milestone gates remain pending. +The user approved publishing this plan and staged implementation on 2026-10-13. Commit/push and draft PR creation are authorized; merge and deployment are not. Start P1 after plan publication and progress to later stages only after their prerequisite gates qualify. G1 is qualified; remaining product gates require their staged evidence. ## Roadmap continuation @@ -33,7 +33,7 @@ The user approved publishing this plan and staged implementation on 2026-10-13. | ID | Observable criterion | Kind | Environment | Mode | Status | Evidence | | --- | --- | --- | --- | --- | --- | --- | | G1 | Open/reopen a local project, inspect available/missing artifacts and runtime requirements, navigate all three workspaces without executing or modifying repository code | user-workflow | representative | automated | satisfied | [P1 observed workflow](reports/p1-verification.md) | -| G2 | Scientific fields preserve exact float bytes, dimensions, staggered placement, units, time and trajectory identity through import, freeze, download and cold read | contract | representative | automated | pending | P2 independent NumPy reader and actual lab samples | +| G2 | Scientific fields preserve exact float bytes, dimensions, staggered placement, units, time and trajectory identity through import, freeze, download and cold read | contract | representative | automated | satisfied | [Actual native and neutral custody workflows](reports/p2-verification.md) | | G3 | An optional Rheon adapter exposes bounded generation controls and produces real generic scientific data; removing it leaves generic project/data use intact | integration | required-real | automated | pending | P4 actual native generator plus no-adapter startup | | G4 | Saved graph executes existing lab fitting and autonomous evaluation scripts; verified checkpoint, full-field comparisons and physics diagnostics survive restart with complete provenance | system | required-real | automated | pending | P5 lab verifier, numerical field comparison and new execution receipts | | G5 | Agent and UI clients use the same operations; stale requests, forged human review, unauthorized execution, path escape and approval reuse fail before side effects | contract | representative | automated | pending | P3/P6 public-interface denial and parity tests | @@ -114,7 +114,7 @@ Enforce at the service boundary reached by every transport, not only at MCP. Aud | Assumption or claim | Decision affected | Owner | Executable procedure | Required environment | Status/evidence | | --- | --- | --- | --- | --- | --- | | Generic project inspection requires no lab runtime | P1 independence | Project owner | Open external manifest with missing runtime/data; observe no children or source writes | Local app/PostgreSQL/browser | Source-supported; product test pending | -| Full lab arrays fit the proposed contract | P2 schema and limits | Data owner | Read representative actual NPZ captures in a separate process; inventory dtype/shape/centering/time and round-trip exact arrays | Actual lab capture bytes + NumPy | Missing bulk captures may block positive acceptance | +| Full lab arrays fit the proposed contract | P2 schema and limits | Data owner | Read representative actual NPZ captures in a separate process; inventory dtype/shape/centering/time and round-trip exact arrays | Actual lab capture bytes + NumPy | Actual two-frame native capture acquired; see P2 admission report | | Native process can be controlled by Runs | P3 execution boundary | Runs owner | Start bounded native probe, cancel during request/output, close pipe, kill supervisor, reconcile descendants | Linux, compatible Rust/Rheon binary | Not executed in planning | | CPU reproduction is available | P5 acceptance runtime | Lab owner | Inspect installed interpreter/package versions; load hash-verified checkpoint in isolated worker; execute documented CPU control | PyTorch runtime + native build + required inputs | Not qualified; do not auto-install | | Prior performance numbers transfer | No design depends on this | Qualification owner | Separate matched-hardware benchmark with recorded protocol | Recorded GPU/CPU hardware | Not claimed | @@ -160,7 +160,7 @@ The lab verifier expects its own frozen schemas and paths. Keep lab-native artif ## Milestones -All statuses below describe proposed work under the planning-only boundary. Before starting each milestone, confirm its exact write set against current source, route relevant language/security/persistence standards, and record semantic changes to this plan. +Statuses below track the approved staged implementation. Before starting each milestone, confirm its exact write set against current source, route relevant language/security/persistence standards, and record semantic changes to this plan. ### P1: Open and inspect a project @@ -172,19 +172,21 @@ All statuses below describe proposed work under the planning-only boundary. Befo **Acceptance gate:** G1; real desktop/narrow/keyboard walkthrough; no spawned project code, no source mutations, traversal/symlink rejection, cross-project isolation and restart; existing workspace navigation regressions pass. First slice does not parse arbitrary checkpoints or claim scientific results verified. -**Status:** `Verifying` +**Status:** `Accepted` ### P2: Lossless scientific data and retained compatibility **Goal:** Inspect a real lab field/frame sequence in Data, review/freeze it and download an exactly recoverable generic release. -**Allowed write set:** proposed `scientific_fields.py`, `scientific_release.py`, `docs/contracts/scientific-fields.md`, `tests/test_scientific_fields.py`, `tests/test_scientific_release.py`, `static/scientific-fields.js`; existing immutable asset, Workbench, release, metadata/upgrade/backup owners, `app.py`, workspace UI and affected tests; scoped compatibility modules identified before edits. +**Allowed write set:** `docs/contracts/projects.md`, `canonical_release.py`, `caption_import.py`, `tests/test_workspace_metadata.py`, `projects.py`, `audio_assets.py`, `immutable_assets.py`, `workbench.py`, `dataset_releases.py`, `metadata.py`, `upgrade_metadata.py`, `README.md`, `docs/contracts/workspace-storage.md`, `.github/workflows/tests.yml`, `tests/qualify_scientific_fields.py`, `tests/browser_scientific_fields.cjs`, `tests/test_projects.py`, `tests/test_audio_assets.py`, `tests/test_spatial_storage.py`, `tests/qualify_spatial_retained_runs.py`, this plan directory; proposed `scientific_fields.py`, `scientific_release.py`, `docs/contracts/scientific-fields.md`, `tests/test_scientific_fields.py`, `tests/test_scientific_release.py`, `static/scientific-fields.js`; existing immutable asset, Workbench, release, metadata/upgrade/backup owners, `app.py`, workspace UI and affected tests; scoped compatibility modules identified before edits. + +**Contract admission:** [P2 bounded contract and native evidence](reports/p2-admission.md). No source implementation precedes independent design review. **Tasks:** Acquire actual sample evidence; fix bounds/layout contract; import via bounded file/stream interface; frame/field/slice viewer with units and sampling disclosure; preserve whole-trajectory families; explicit conversion links; qualify retained v1/v2 readers and schema upgrade. **Acceptance gate:** G2 and data portion of G7; independent cold read, exact bytes, geometry/time checks, malformed/oversize inputs, restore and retained-history tests. Actual sample unavailability leaves G2 pending while negative fixtures can proceed. -**Status:** `Planned` +**Status:** `Verifying` ### P3: Authorized external recipe execution @@ -263,7 +265,7 @@ All statuses below describe proposed work under the planning-only boundary. Befo ## Concurrent Work -Current planning is serial. After contract decisions, independent UI, adapter and read-only review work may proceed with disjoint assigned write sets. Core data schemas, access policy, metadata, Runs, shared tests and this plan have one serial integration owner. Do not spawn competing implementations of those owners. +P2 contracts and integration have one serial root owner. After contract decisions, independent UI, adapter and read-only review work may proceed with disjoint assigned write sets. Core data schemas, access policy, metadata, Runs, shared tests and this plan have one serial integration owner. Do not spawn competing implementations of those owners. ## Repository Isolation diff --git a/docs/plans/generic-scientific-projects/reports/p2-admission.md b/docs/plans/generic-scientific-projects/reports/p2-admission.md new file mode 100644 index 00000000..cf97d7ef --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p2-admission.md @@ -0,0 +1,32 @@ +# P2 admission and bounded data contract + +P1 exact head `46384fca492f72a3588cc79309a703d54cd3675b` passed [UI CI](https://github.com/MrScripty/Tuldok/actions/runs/38416791909) (14 steps) and [lifecycle CI](https://github.com/MrScripty/Tuldok/actions/runs/38416792077) (122 steps). PR45 remains a draft; no merge. Main was fetched again before branching. + +Actual evidence: isolated lab e8a90bf and Rheon 2209012, Rust 1.92, two 32³ Jacobi seed100 native frames. Wire SHA256 `abe671cc217b2a3e6b7f23a7506a8406a35e6ec3358a6d6da3f690c865bfc684`, 2,122,648 bytes. Native binary SHA256 `4e4e49f0e3febf5b11556c2a1daac3dc684e9fb9a9c3c742329d804a4913ceff`. Independent standard-library cold verification matched every slice/hash, divergence and normal-wall velocity; tracer integral agrees within declared numerical tolerance. The capture is new native evidence, not historical model acceptance. + +RHS and pressure are little-endian float64; corrected MAC velocities and advected tracer are float32. Arrays use X-fastest storage. Different algorithm stages require explicit labels. Physical units are source-derived only when supported, otherwise declared unspecified. + +## Candidate contract for independent review + +P2 candidate contract — new field_sequence asset, task field_sequence_fields. +Dependency: P1 exact-head CI must pass before source edits. Actual native sample exists in /workspace/scratch/tuldok-science-probe/capture-32-jacobi-100-two-frames. + +One immutable local bundle.zip (ZIP_STORED only, <=40MiB), with manifest.json (strict UTF8 <=1MiB) and exactly declared NPY members/source-provenance files, no directory entries/symlinks/duplicates/unreferenced entries/extraction. Names relative <=160chars, <=8components, no backslash/dot/parent. Core never imports a producer package. +Manifest exact keys: format='scientific_field_sequence_v1', schema_version=1, grid, time_units, fields, frames, family, sources, provenance. +grid exact {shape_xyz:[nx,ny,nz],origin_xyz:[x,y,z],spacing_xyz:[dx,dy,dz],orientation:[[...],[...],[...]],units:string}. Grid dims1..128; originfinite abs<=1e12; spacingpositive finite<=1e12. orientation is row-major orthonormal matrix mapping gridXYZ (after spacing) to worldXYZ; rotation/reflection allowed (dot tolerances1e-9). Field array C-order ZYX shapes computed from grid and centering. No deformation/shear/unstructured grids. +fields list1..16; each {id,label,dtype,centering,units,stage}; id safe token<=64, unique; label<=160, units<=80, stage<=160; all nonempty validUTF8 no control. dtype exactly ' view field/frame/axis/slice and physicalcoord/units/stage -> wholeasset note/humanreview -> existing preview/freeze -> download originalbundle+canonical release. No new graph execution in P2. Frozen reader independently revalidates source metadata, protects families. Source unavailable/corrupt refuses save/release, no silentrepair. +Suggested public source API ScientificAssets.admit(raw,body,source), metadata(record_id,provenance)->(meta,available), asset(record)->(verified_bytes,'application/zip'), inspect(record_id,frame=0,field=,axis='z',index=0)->bounded exact numeric slice and metadata. Tests negative contract, source bytes, cold NumPy, lifecyclefault/restore, actual capture and producer-neutral fixture, project root attacks, browser imports/review/freeze/download. Workbench remains record/review owner, Releases split/archive owner. + +UI integration: explicit import opens the imported catalog record through existing Workbench context. Project reference inspection remains available; it does not silently remap ownership. The scientific viewer uses frame/field/axis/slice controls, with unit and producer-stage labels. Existing record review and release controls remain canonical. + +Review refinements (binding): first observed state is not the trajectory initial condition. Fingerprint grid geometry plus sorted centering/dtype/payload-SHA tuples for first stored frame, excluding labels/field IDs/time spelling; declared family stays separate. No independence claim. Cell sample coordinate = origin + orientation @ (spacing * (XYZindex + 0.5)); face axis has offset0 on that axis and0.5 on the others. NPY header AST must reject duplicate keys and unexpected nested forms before literal decoding/shape products. Import body requires expected_sha256 from displayed observation; public data accessor checks it against actual bytes even if saved reference has no declared SHA. Stale observation refuses409 before catalog writes. + +Independent design review completed by mesh_review: all six findings incorporated before their affected implementation. File publication retains outer rollback and unknown-commit semantics. Scientific asset publication/serving uses the same bounded verified byte object, not verify-then-reopen paths. Data-reference hash inspection permits40MiB per data file within existing64MiB total budget; other reference kinds remain8MiB. + +Frozen scientific metadata may contain the bounded manifest twice (record summary and immutable provenance). The existing canonical reader now admits scientific JSONL rows up to its existing8MiB metadata limit; older subtype256KiB row bounds remain unchanged. Scientific release admission permits≤64 selected records and bounds final manifest to8MiB, with an early conservative preview guard. Independent reader compares both provenance and record summaries to source bytes. diff --git a/docs/plans/generic-scientific-projects/reports/p2-verification.md b/docs/plans/generic-scientific-projects/reports/p2-verification.md new file mode 100644 index 00000000..a9c8cc1a --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p2-verification.md @@ -0,0 +1,26 @@ +# P2 lossless scientific fields qualification + +Scope: generic scientific data custody and user inspection/review/freeze. No generator, training, physics/model accuracy or actor-authentication acceptance is claimed. Source branch: `feature/scientific-field-sequences-20261013`, based on P1 `46384fca`. [Bounded admission decisions](p2-admission.md) and [source hashes](p2/source-files.json) identify the local reviewed implementation. Hosted exact-head CI follows publication. + +## Implemented behavior + +Explicit import captures a saved project reference under its pinned root and requires the displayed observed SHA. A generic stored ZIP/NPY contract preserves exact float32/float64 bits, grid geometry, cell/MAC-face placement, time strings and producer stages. Original bundles live as ordinary immutable files; PostgreSQL schema6 owns catalog/review history. Schema1–5 identities are retained; upgrade is explicit and quiesced. Shared file publication preserves rollback and uncertain-commit custody without introducing a second store. + +Data offers frame/field/axis/slice controls, physical coordinates and an exact numeric probe. Human whole-trajectory review uses the existing record and release owners. Canonical download and cold reader preserve source identity, repeated provenance and protected source/first-observed-state/declared-family groups. These groups are not proof of independent initial conditions. + +## Observed workflow + +- [Producer-neutral receipt](p2/neutral/receipt.json): rotated3×2×4 geometry, four field centerings and two frames;12 HTTP slices independently checked with NumPy. Five browser gates include stale-hash409 with no catalog write, review/freeze, cold original-byte read, restart and delayed import/newer clean-selection protection. +- [Native receipt](p2/native/receipt.json): actual two-frame32³ native capture, six fields;18 slices and12 arrays independently checked, with five original evidence files retained unchanged. Same five browser gates pass. Native unit declarations are explicitly unspecified. +- [Native observation](p2/native-observation/receipt.json), [decoded wire](p2/native-observation/decoded.json), [independent cold observation](p2/native-observation/cold-verification.json): pinned source/compiler/binary/raw identities and field semantics. Original scratch data remain at `/workspace/scratch/tuldok-science-probe`; reports do not claim the ignored historical lab packet was recovered. +- [Retained historical Run](p2/retained-runs.json): real schema3 Run upgraded to6, backed up/restored with exact old artifacts. New scientific source, review and history backup/restore is exercised by the storage suite. +- Desktop and390px screenshots: [neutral](p2/neutral/scientific-inspector-390.jpg), [native](p2/native/scientific-inspector-390.jpg). Root inspected narrow native output; selectors, original-value probe, geometry and stage remain readable. + +## Verification and corrections + +- Storage/publication/migration suite67 passed, including scientific restore and retained versions. Focused project17 passed. Scientific parser/service7 passed, including real HTTP import/inspection. Cold release6 passed with an external NumPy process and large legal metadata; independent spatial reader20 passed without skips. +- Existing workspace browser18 checks passed after final UI race fix (`run-8jQjNK`). Changed Python/JavaScript syntax and whitespace checks pass. Adopted standards plan-contract validator passes; no Standards Engine certificate or static type-checker claim. +- Independent source review:38 baseline checks and updated provenance-tampering test pass; no remaining product finding. Fixed contradictory frozen provenance and delayed import replacing a newer clean selection; browser regression covers the latter. +- Initial broad run exposed four audio worker failures caused by a new top-level server-only publication-helper import. Moving that import into admission restores the pinned worker dependency set; real audio training passes.33 integrity tests refused deliberately changed source identities while implementation was still being edited. Final stable-source full-suite result is recorded below when complete. + +Full stable-source suite:696 tests passed in566.842s, three optional-reader skips. Separate qualified-reader runs pass20 spatial tests and6 scientific-release tests with no skips. Workbench controller, page-load and startup checks pass. Hosted exact-head CI: pending publication. P3/P4/P5 execution/actor/model work remains unimplemented by this milestone. diff --git a/docs/plans/generic-scientific-projects/reports/p2/native-observation/cold-verification.json b/docs/plans/generic-scientific-projects/reports/p2/native-observation/cold-verification.json new file mode 100644 index 00000000..f91c247a --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p2/native-observation/cold-verification.json @@ -0,0 +1,27 @@ +{ + "scope": "Saved artifact custody and independent full-grid numerical check; no replay or benchmark", + "receipt_sha256": "332aa36d8ce02585be77fbd74f8a7dfcfe9a297a3054b5bb67461ee3556d1e40", + "verifier_sha256": "98cd90583743bf09cecacf587505f6ba679aebdece2009fd603cfd83dd44554e", + "checks": [ + { + "frame": 1, + "native_divergence_matches": true, + "actual_divergence_max": 2.9802322387695312e-08, + "normal_wall_velocity_max": 0.0, + "native_tracer_integral_within_tolerance": true, + "tracer_integral": 0.00034499217481302226, + "tracer_integral_abs_difference": 5.366800753803247e-18 + }, + { + "frame": 2, + "native_divergence_matches": true, + "actual_divergence_max": 5.960464477539063e-08, + "normal_wall_velocity_max": 0.0, + "native_tracer_integral_within_tolerance": true, + "tracer_integral": 0.0006897932466234993, + "tracer_integral_abs_difference": 6.5052130349130266e-18 + } + ], + "all_artifact_hashes_pass": true, + "all_slices_match_original_wire": true +} diff --git a/docs/plans/generic-scientific-projects/reports/p2/native-observation/decoded.json b/docs/plans/generic-scientific-projects/reports/p2/native-observation/decoded.json new file mode 100644 index 00000000..887c6701 --- /dev/null +++ b/docs/plans/generic-scientific-projects/reports/p2/native-observation/decoded.json @@ -0,0 +1,337 @@ +{ + "ready": { + "actual_divergence_limit": 1e-05, + "density": 1.25, + "dt": 0.016666666666666666, + "face_lengths": [ + 33792, + 33792, + 33792 + ], + "method": "jacobi", + "n": 32, + "native_relative_residual": 1e-08, + "schema": "rheon-trajectory-v1", + "seed": 100, + "spacing": [ + 0.03125, + 0.03125, + 0.03125 + ] + }, + "frames": [ + { + "report": { + "actual_divergence_max": 2.9802322387695312e-08, + "advection_ms": 36.578454, + "correction_ms": 3.6205, + "courant": 0.012171820551156998, + "frame": 1, + "iterations": 296, + "pressure_ms": 199.245475, + "relative_residual": 9.047536548105893e-09, + "time": 0.016666666666666666, + "total_ms": 250.967202, + "tracer_integral": 0.0003449921748130169, + "tracer_ms": 11.522012 + }, + "fields": { + "rhs": { + "file": "frame-01-rhs.bin", + "wire_offset": 585, + "bytes": 262144, + "sha256": "3cc118b9c270d56a77ece2d19735fe1774860e88b850db419a6211d12cb234ed", + "dtype": " MAX_SELECTED_ASSET_BYTES: raise WorkbenchError('Selected immutable assets exceed the synchronous 40 MiB bound.') +def _sync_directory(path): + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +@contextmanager +def publish_file(db, folder, filename, raw): + """Publish one trusted local asset path with its metadata transaction. + + Callers supply a server-owned unique folder and fixed filename. Successful + savepoints transfer rollback custody to the outer transaction; an uncertain + commit retains bytes for catalog reconciliation. This never extracts paths. + """ + from metadata import CommitOutcomeUnknown + created = False + path = folder / filename + def discard(): + if created and folder.exists(): + path.unlink(missing_ok=True) + folder.rmdir() + try: + with (nullcontext(db) if db.in_transaction else db.transaction()): + folder.parent.mkdir(exist_ok=True) + folder.mkdir() + created = True + db.on_rollback(discard) + with path.open('xb') as target: + target.write(raw) + target.flush() + os.fsync(target.fileno()) + _sync_directory(folder) + _sync_directory(folder.parent) + _sync_directory(folder.parent.parent) + yield path + except CommitOutcomeUnknown: + raise + except BaseException: + discard() + raise + + def migrate_records(db): """PostgreSQL schema evolution belongs to the versioned metadata owner.""" db.validate_schema() diff --git a/metadata.py b/metadata.py index db1de0ac..8ea6b988 100644 --- a/metadata.py +++ b/metadata.py @@ -19,8 +19,8 @@ from psycopg import sql from psycopg.conninfo import conninfo_to_dict -SCHEMA_VERSION = 5 -SUPPORTED_SCHEMA_VERSIONS = (1, 2, 3, 4, 5) +SCHEMA_VERSION = 6 +SUPPORTED_SCHEMA_VERSIONS = (1, 2, 3, 4, 5, 6) MANIFEST = '.tuldok.json' CATALOG_VERSION_DDL = '''CREATE TABLE catalog_version ( version INTEGER PRIMARY KEY, dataset_id TEXT NOT NULL, @@ -415,7 +415,7 @@ def close(self): def tables_for_version(version): if type(version) is not int or version not in SUPPORTED_SCHEMA_VERSIONS: raise StorageError('Unsupported PostgreSQL catalog version.') - return {1: TABLES_V1, 2: TABLES_V2, 3: TABLES_V3, 4: TABLES_V4, 5: TABLES}[version] + return {1: TABLES_V1, 2: TABLES_V2, 3: TABLES_V3, 4: TABLES_V4, 5: TABLES_V5, 6: TABLES}[version] def definition_hash(version=SCHEMA_VERSION): @@ -527,4 +527,10 @@ def definition_hash(version=SCHEMA_VERSION): 'workspace_projects': '''CREATE TABLE workspace_projects ( id TEXT PRIMARY KEY, document TEXT NOT NULL)''', } -TABLES = {**TABLES_V4, **PROJECT_TABLES} +TABLES_V5 = {**TABLES_V4, **PROJECT_TABLES} +# The scientific source lives in the existing local artifact tree, not a new table. +TABLES = { + **TABLES_V5, + 'workbench_records': TABLES_V5['workbench_records'].replace( + "'pointcloud','volume'", "'pointcloud','volume','field_sequence'"), +} diff --git a/projects.py b/projects.py index fe594728..a9130d6c 100644 --- a/projects.py +++ b/projects.py @@ -14,6 +14,7 @@ SHA = re.compile(r'[a-f0-9]{64}') MANIFEST_LIMIT = 64 * 1024 FILE_LIMIT = 8 * 1024 * 1024 +DATA_LIMIT = 40 * 1024 * 1024 INSPECTION_LIMIT = 64 * 1024 * 1024 TEXT_LIMIT = 64 * 1024 GROUPS = ('data', 'recipes', 'evidence', 'requirements') @@ -196,15 +197,15 @@ def _bound(self, saved): entry = self.roots.get(saved['root']) return bool(entry and entry[1] == saved['root_identity']) - def _inspect(self, root, ref, budget): + def _inspect(self, root, ref, budget, limit=FILE_LIMIT): result = {**ref, 'status': 'available', 'integrity': 'unverified'} try: with self._file(root, ref['path']) as (_fd, info): result['bytes'] = info.st_size - if info.st_size > FILE_LIMIT or info.st_size > budget[0]: + if info.st_size > limit or info.st_size > budget[0]: result['reason'] = 'Present; content exceeds the bounded hash inspection budget.' return result - raw = self._read(root, ref['path'], min(FILE_LIMIT, budget[0])) + raw = self._read(root, ref['path'], min(limit, budget[0])) budget[0] -= len(raw) result['bytes'] = len(raw) result['observed_sha256'] = hashlib.sha256(raw).hexdigest() @@ -250,10 +251,32 @@ def get(self, project_id): except (OSError, WorkbenchError): item.update(status='unavailable', reason='Required file cannot be safely inspected.') else: - item = self._inspect(saved['root'], ref, budget) + item = self._inspect(saved['root'], ref, budget, DATA_LIMIT if group == 'data' else FILE_LIMIT) result[group].append(item) return result + def read_data_reference(self, project_id, reference_id, expected_sha256, limit): + """Capture a selected saved reference; source owners interpret its bytes.""" + if not isinstance(expected_sha256, str) or not SHA.fullmatch(expected_sha256): + raise WorkbenchError('Import requires the observed source SHA256.') + if type(limit) is not int or not 0 < limit <= DATA_LIMIT: + raise WorkbenchError('Invalid bounded data read limit.') + saved = self._saved(project_id) + if not self._bound(saved): + raise WorkbenchError('Original project root is not configured.', 'unavailable', 409) + ref = next((entry for entry in saved['manifest']['data'] if entry['id'] == reference_id), None) + if ref is None: + raise WorkbenchError('Data reference not found.', 'not_found', 404) + try: + raw = self._read(saved['root'], ref['path'], limit) + except OSError: + raise WorkbenchError('Data reference cannot be safely read.', 'unavailable', 409) from None + digest = hashlib.sha256(raw).hexdigest() + if digest != expected_sha256 or ref.get('sha256', digest) != digest: + raise WorkbenchError('Data changed or differs from its declared SHA256. Inspect again before importing.', 'conflict', 409) + return raw, {'project_id': project_id, 'manifest_sha256': saved['manifest_sha256'], + 'reference_id': reference_id, 'path': ref['path'], 'measured_sha256': digest} + def evidence(self, project_id, reference_id): saved = self._saved(project_id) if not self._bound(saved): diff --git a/scientific_fields.py b/scientific_fields.py new file mode 100644 index 00000000..1fc20a34 --- /dev/null +++ b/scientific_fields.py @@ -0,0 +1,401 @@ +"""Lossless bounded structured-grid fields; no producer execution or physics policy.""" +import ast +from decimal import Decimal, InvalidOperation +import hashlib +import io +import json +import math +import os +from pathlib import Path +import re +import stat +import struct +import uuid +import zipfile + +from workbench import WorkbenchError, encode, strings, text_value, timestamp + +FORMAT = 'scientific_field_sequence_v1' +TASK = 'field_sequence_fields' +BUNDLE_LIMIT = 40 * 1024 * 1024 +MAX_BUNDLE_BYTES = BUNDLE_LIMIT +MANIFEST_LIMIT = 1024 * 1024 +HEADER_LIMIT = 4096 +TOKEN = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$') +HASH = re.compile(r'^[a-f0-9]{64}$') +TIME = re.compile(r'^[+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$') +CENTERINGS = ('cell', 'face_x', 'face_y', 'face_z') +SCOPE = 'Whole stored field sequence; first observed state and declared family do not establish independent trajectories or scientific validity.' + + +def invalid(message, code='invalid', status=400): + raise WorkbenchError('Scientific fields: ' + message, code, status) + + +def exact(value, keys, label): + if type(value) is not dict or set(value) != set(keys): + invalid(label + ' fields differ from schema 1.') + + +def text(value, maximum, label): + if (type(value) is not str or not value.strip() or len(value) > maximum or + any(ord(char) < 32 or 127 <= ord(char) <= 159 or 0xD800 <= ord(char) <= 0xDFFF for char in value)): + invalid(label + ' must be bounded nonempty Unicode text without controls.') + return value + + +def path_name(value): + text(value, 160, 'Member path') + if ('\\' in value or value.startswith('/') or len(value.split('/')) > 8 + or any(part in ('', '.', '..') for part in value.split('/'))): + invalid('Member paths must be relative without traversal or empty components.') + return value + + +def decode(raw): + def pairs(items): + result = {} + for key, value in items: + if key in result: + invalid('Duplicate JSON key.') + result[key] = value + return result + def constant(_value): + invalid('Nonfinite JSON constants are unsupported.') + try: + value = json.loads(raw.decode('utf-8'), object_pairs_hook=pairs, parse_constant=constant) + pending = [(value, 0)] + while pending: + item, depth = pending.pop() + if depth > 16: + invalid('Manifest nesting exceeds its schema bound.') + if type(item) is dict: + pending.extend((child, depth + 1) for child in item.values()) + elif type(item) is list: + pending.extend((child, depth + 1) for child in item) + return value + except (ValueError, UnicodeError, RecursionError): + invalid('Manifest must be bounded UTF-8 JSON with unique fields and finite values.') + + +def finite(value, *, positive=False): + return (type(value) in (int, float) and abs(value) <= 1e12 and math.isfinite(value) + and (not positive or value > 0)) + + +def grid_contract(grid): + exact(grid, ('shape_xyz', 'origin_xyz', 'spacing_xyz', 'orientation', 'units'), 'Grid') + shape = grid['shape_xyz'] + if type(shape) is not list or len(shape) != 3 or any(type(n) is not int or not 1 <= n <= 128 for n in shape): + invalid('Grid dimensions must be three integers from 1 to 128.') + for key in ('origin_xyz', 'spacing_xyz'): + values = grid[key] + if type(values) is not list or len(values) != 3 or any(not finite(n, positive=key=='spacing_xyz') for n in values): + invalid('Grid origin/spacing must be bounded finite XYZ values with positive spacing.') + orientation = grid['orientation'] + if (type(orientation) is not list or len(orientation) != 3 or + any(type(row) is not list or len(row) != 3 or any(not finite(n) for n in row) for row in orientation)): + invalid('Orientation must be a finite 3 by 3 matrix.') + for left in range(3): + for right in range(3): + dot = math.fsum(orientation[row][left] * orientation[row][right] for row in range(3)) + if abs(dot - (1 if left == right else 0)) > 1e-9: + invalid('Orientation must be orthonormal; shear and deformation are unsupported.') + text(grid['units'], 80, 'Grid units') + + +def shape_for(grid, field): + xyz = list(grid['shape_xyz']) + if field['centering'] != 'cell': + xyz['xyz'.index(field['centering'][-1])] += 1 + return list(reversed(xyz)) + + +def npy_payload(raw, field, grid): + """Validate literal headers before computing shapes; retain every payload bit.""" + if len(raw) < 10 or raw[:8] != b'\x93NUMPY\x01\x00': + invalid('Only NPY 1.0 members are supported.', 'unsupported') + size = struct.unpack_from(' len(raw) or end % 16 or raw[end-1:end] != b'\n': + invalid('NPY header length, alignment or terminator is invalid.') + try: + tree = ast.parse(raw[10:end].decode('ascii').strip(), mode='eval').body + if not isinstance(tree, ast.Dict) or len(tree.keys) != 3: + invalid('NPY header requires exactly three literal fields.') + keys = [key.value if isinstance(key, ast.Constant) and type(key.value) is str else None for key in tree.keys] + if set(keys) != {'descr', 'fortran_order', 'shape'}: + invalid('NPY header fields must be unique descr, fortran_order and shape.') + nodes = dict(zip(keys, tree.values)) + if (not isinstance(nodes['descr'], ast.Constant) or type(nodes['descr'].value) is not str or + not isinstance(nodes['fortran_order'], ast.Constant) or nodes['fortran_order'].value is not False or + not isinstance(nodes['shape'], ast.Tuple) or len(nodes['shape'].elts) != 3 or + any(not isinstance(node, ast.Constant) or type(node.value) is not int for node in nodes['shape'].elts)): + invalid('NPY values must be literal dtype, false C-order flag and three integer dimensions.') + dtype = nodes['descr'].value + shape = [node.value for node in nodes['shape'].elts] + except (SyntaxError, ValueError, UnicodeError, RecursionError): + invalid('Malformed or unsupported NPY literal header.') + if dtype != field['dtype'] or shape != shape_for(grid, field): + invalid('NPY dtype or shape disagrees with declared field placement.') + width = 4 if dtype == ' Decimal('1e12') or (previous is not None and time <= previous): + invalid('Frame times must be finite, bounded and strictly increasing.') + except InvalidOperation: + invalid('Invalid decimal frame time.') + previous = time + arrays = frame['arrays'] + if type(arrays) is not list or len(arrays) != len(fields): + invalid('Every frame requires exactly one array per field.') + seen = set() + for array in arrays: + exact(array, ('field','path','bytes','sha256'), 'Array') + if type(array['field']) is not str or array['field'] not in by_id or array['field'] in seen: + invalid('Frame field references must be complete and unique.') + seen.add(array['field']); refs.append(array) + exact(value['family'], ('namespace','id'), 'Family') + for name in ('namespace','id'): + text(value['family'][name], 160, 'Declared family ' + name) + exact(value['provenance'], ('producer','source_id'), 'Provenance') + text(value['provenance']['producer'], 160, 'Producer') + text(value['provenance']['source_id'], 256, 'Source identity') + sources = value['sources'] + if type(sources) is not list or len(sources) > 16: + invalid('At most 16 opaque original source files are supported.') + for source in sources: + exact(source, ('path','bytes','sha256','description'), 'Original source') + text(source['description'], 160, 'Original source description') + refs.append(source) + paths = {'manifest.json'} + for ref in refs: + name = path_name(ref['path']) + if name in paths: + invalid('Every array/source path must be unique.') + paths.add(name) + if type(ref['bytes']) is not int or not 0 < ref['bytes'] <= BUNDLE_LIMIT: + invalid('Declared file size must be a bounded positive integer.') + if type(ref['sha256']) is not str or not HASH.fullmatch(ref['sha256']): + invalid('Declared file hash must be lowercase SHA256.') + return by_id, refs + + +def parse_bundle(raw): + if type(raw) is not bytes or not 0 < len(raw) <= BUNDLE_LIMIT: + invalid('Select an original bundle of at most 40 MiB.', 'too_large') + try: + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + entries = archive.infolist() + names = archive.namelist() + if (not entries or len(entries) > 4113 or len(names) != len(set(names)) or + raw[:4] != b'PK\x03\x04' or raw[-22:-18] != b'PK\x05\x06' or archive.comment): + invalid('Bundle must be a bounded ZIP without duplicate entries, preamble, comment or trailing bytes.') + for item in entries: + path_name(item.filename) + if (item.compress_type != zipfile.ZIP_STORED or item.flag_bits & 1 or item.file_size != item.compress_size or + stat.S_IFMT(item.external_attr >> 16) not in (0, stat.S_IFREG)): + invalid('Only stored, unencrypted regular ZIP files are supported.') + if sum(item.file_size for item in entries) > BUNDLE_LIMIT or 'manifest.json' not in names: + invalid('Bundle expanded size or manifest is invalid.') + if not 0 < archive.getinfo('manifest.json').file_size <= MANIFEST_LIMIT: + invalid('Manifest exceeds its 1 MiB bound.') + value = decode(archive.read('manifest.json')) + fields, refs = manifest_contract(value) + if set(names) != {'manifest.json'} | {ref['path'] for ref in refs}: + invalid('Bundle files differ from the exact declared inventory.') + for ref in refs: + if archive.getinfo(ref['path']).file_size != ref['bytes']: + invalid('Declared file size differs from the archived member.') + contents = archive.read(ref['path']) + if hashlib.sha256(contents).hexdigest() != ref['sha256']: + invalid('Archived bytes differ from the declared SHA256.') + first_state = [] + for index, frame in enumerate(value['frames']): + for array in frame['arrays']: + field = fields[array['field']] + payload, _shape = npy_payload(archive.read(array['path']), field, value['grid']) + if index == 0: + first_state.append((field['centering'], field['dtype'], hashlib.sha256(payload).hexdigest())) + # Numeric JSON spellings and signed zero cannot separate identical grids. + geometry = dict(value['grid']) + for key in ('origin_xyz','spacing_xyz'): + geometry[key] = [float(n) if n else 0.0 for n in geometry[key]] + geometry['orientation'] = [[float(n) if n else 0.0 for n in row] for row in geometry['orientation']] + state_sha = hashlib.sha256(encode({'grid':geometry,'arrays':sorted(first_state)}).encode()).hexdigest() + source_sha = hashlib.sha256(raw).hexdigest() + family_sha = hashlib.sha256(encode(value['family']).encode()).hexdigest() + return {'schema_version':1,'format':FORMAT,'manifest':value,'frame_count':len(value['frames']), + 'field_count':len(value['fields']),'source_bytes':len(raw),'first_observed_state_sha256':state_sha, + 'protected_groups':['field-sequence-source:'+source_sha,'field-sequence-state:'+state_sha, + 'field-sequence-family:'+family_sha], 'scope':SCOPE} + except (zipfile.BadZipFile, OSError, KeyError, RuntimeError, OverflowError, RecursionError, struct.error) as error: + invalid('Malformed scientific bundle: ' + type(error).__name__) + + +class ScientificAssets: + def __init__(self, workbench): + self.workbench = workbench + self.path = workbench.dataset.path / 'field_sequence' + + def admit(self, raw, body, source): + from immutable_assets import publish_file + if type(body) is not dict or set(body) - {'name','rights','groups','parents'}: + invalid('Unexpected import metadata fields.') + meta = parse_bundle(raw) + name = text_value(body.get('name', 'Scientific field sequence'), 'Name') + rights = text_value(body.get('rights', 'unknown'), 'Rights / permission note', 1000) + groups = strings(body.get('groups', []), 'Protected groups', maximum=27) + parents = strings(body.get('parents', []), 'Parent IDs') + groups = list(dict.fromkeys(groups + meta['protected_groups'])) + exact(source, ('project_id','manifest_sha256','reference_id','path','measured_sha256'), 'Project source') + if type(source['project_id']) is not str or not re.fullmatch('[a-f0-9]{32}', source['project_id']): + invalid('Invalid project source ID.') + for key in ('manifest_sha256','measured_sha256'): + if type(source[key]) is not str or not HASH.fullmatch(source[key]): + invalid('Invalid project source hash.') + text(source['reference_id'], 64, 'Project reference') + text(source['path'], 512, 'Project source path') + digest = hashlib.sha256(raw).hexdigest() + if source['measured_sha256'] != digest: + invalid('Captured project bytes differ from their source receipt.', 'conflict', 409) + origin = {'method':'project_data_import','rights':rights,'field_sequence':meta, + 'source_sha256':digest,'project_source':dict(source)} + record_id, created = uuid.uuid4().hex, timestamp() + w = self.workbench + with w.lock, w.db.transaction(): + for parent in parents: + w._get(parent) + if w.db.execute("SELECT 1 FROM workbench_records WHERE kind='field_sequence' AND content_hash=%s", (digest,)).fetchone(): + invalid('This exact scientific bundle already exists.', 'conflict', 409) + with publish_file(w.db, self.path / record_id, 'source.zip', raw): + w.db.execute('INSERT INTO workbench_records VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', + (record_id,'field_sequence',name,None,None,digest,None,encode(groups),encode(parents), + encode(origin),TASK,None,'draft',1,created,created)) + record = w._get(record_id) + w._history(record) + return record + + def _capture(self, record_id): + if type(record_id) is not str or not re.fullmatch('[a-f0-9]{32}', record_id): + invalid('Invalid scientific record identity.') + try: + path = self.path / record_id / 'source.zip' + if path.is_symlink() or path.parent.is_symlink(): + invalid('Stored bundle must be a regular owned file.', 'unavailable', 409) + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + with os.fdopen(descriptor, 'rb') as handle: + before = os.fstat(handle.fileno()) + if not stat.S_ISREG(before.st_mode) or not 0 < before.st_size <= BUNDLE_LIMIT: + invalid('Stored bundle must be a bounded regular file.', 'unavailable', 409) + raw = handle.read(BUNDLE_LIMIT + 1) + after = os.fstat(handle.fileno()) + if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns): + invalid('Stored bundle changed during capture.', 'conflict', 409) + if not 0 < len(raw) <= BUNDLE_LIMIT: + invalid('Stored bundle exceeds its byte bound.', 'unavailable', 409) + return raw + except OSError: + invalid('Original scientific bundle is unavailable.', 'unavailable', 404) + + def metadata(self, record_id, provenance): + meta = provenance.get('field_sequence') + if type(meta) is not dict or meta.get('schema_version') != 1 or meta.get('format') != FORMAT: + invalid('Stored scientific metadata is unavailable.', 'unavailable', 409) + try: + raw = self._capture(record_id) + available = hashlib.sha256(raw).hexdigest() == provenance.get('source_sha256') and len(raw) == meta.get('source_bytes') + except WorkbenchError: + available = False + return dict(meta), available + + def asset(self, record): + raw = self._capture(record['id']) + if hashlib.sha256(raw).hexdigest() != record['content_hash'] or encode(parse_bundle(raw)) != encode(record['field_sequence']): + invalid('Stored scientific bytes or metadata changed outside Tuldok.', 'conflict', 409) + return raw, 'application/zip' + + def inspect(self, record_id, frame=0, field=None, axis='z', index=0): + w = self.workbench + with w.lock: + record = w._get(record_id) + if record['kind'] != 'field_sequence': + invalid('Inspection requires a scientific field sequence.') + raw, _ = self.asset(record) + meta = record['field_sequence']['manifest'] + if type(frame) is not int or not 0 <= frame < len(meta['frames']): + invalid('Choose an existing frame index.') + field = meta['fields'][0]['id'] if field is None else field + definition = next((item for item in meta['fields'] if item['id'] == field), None) + if definition is None or axis not in ('x','y','z') or type(index) is not int: + invalid('Choose an existing field, axis and integer slice index.') + array = next(item for item in meta['frames'][frame]['arrays'] if item['field'] == field) + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + payload, shape = npy_payload(archive.read(array['path']), definition, meta['grid']) + xyz = list(reversed(shape)); fixed = 'xyz'.index(axis) + if not 0 <= index < xyz[fixed]: + invalid('Slice index lies outside the field dimensions.') + row_axis, column_axis = {'z':('y','x'),'y':('z','x'),'x':('z','y')}[axis] + row_index, column_index = 'xyz'.index(row_axis), 'xyz'.index(column_axis) + width = 4 if definition['dtype'] == 'item.id===request.field),shape=scientificShape(record,field); + const dimensions={z:shape[0],y:shape[1],x:shape[2]},[row,column]={z:['y','x'],y:['z','x'],x:['z','y']}[request.axis]; + const vector=v=>Array.isArray(v)&&v.length===3&&v.every(n=>typeof n==='number'&&Number.isFinite(n)); + if(!value||value.id!==record.id||value.content_hash!==record.content_hash||value.frame!==request.frame||value.field!==request.field|| + value.axis!==request.axis||value.index!==request.index||value.time!==manifest.frames[request.frame].time||value.time_units!==manifest.time_units|| + value.units!==field.units||value.stage!==field.stage||value.centering!==field.centering||value.dtype!==field.dtype|| + !value.grid||value.grid.units!==manifest.grid.units||!Array.isArray(value.shape_zyx)||value.shape_zyx.length!==3||!shape.every((n,i)=>value.shape_zyx[i]===n)||value.row_axis!==row||value.column_axis!==column|| + !['physical_origin','physical_row_step','physical_column_step'].every(key=>vector(value[key]))|| + !Array.isArray(value.values)||value.values.length!==dimensions[row]||!value.values.every(values=>Array.isArray(values)&&values.length===dimensions[column]&&values.every(n=>typeof n==='number'&&Number.isFinite(n))))throw Error('Invalid scientific field slice response.'); + return value; +} +function scientificProbe() { + if(!scientificSlice)return; + const row=Number($('scientific-row').value),column=Number($('scientific-column').value),value=scientificSlice; + if(!Number.isInteger(row)||!Number.isInteger(column)||row<0||column<0||row>=value.values.length||column>=value.values[0].length){$('scientific-probe').textContent='Choose an in-bounds row and column.';return;} + const physical=value.physical_origin.map((n,i)=>n+row*value.physical_row_step[i]+column*value.physical_column_step[i]); + $('scientific-probe').textContent=`Row ${row}, column ${column}: ${Object.is(value.values[row][column],-0)?'-0':String(value.values[row][column])} ${value.units} · physical (${physical.join(', ')}) ${value.grid.units}.`; +} +function drawScientificSlice(value) { + scientificSlice=value; + const canvas=$('scientific-canvas'),rows=value.values.length,columns=value.values[0].length; + canvas.width=columns;canvas.height=rows; + let low=Infinity,high=-Infinity;for(const row of value.values)for(const number of row){low=Math.min(low,number);high=Math.max(high,number);} + const scale=Math.max(Math.abs(low),Math.abs(high)),range=scale?high/scale-low/scale:0; + const ctx=canvas.getContext('2d'),pixels=ctx.createImageData(columns,rows); + for(let row=0;row({value:String(i),label:`${i} · ${frame.time} ${manifest.time_units}`}))],['scientific-field',manifest.fields.map(field=>({value:field.id,label:field.label}))]]){ + $(id).replaceChildren();for(const item of items){const option=document.createElement('option');option.value=item.value;option.textContent=item.label;$(id).append(option);} + } + $('scientific-axis').value='z';$('scientific-index').value='0'; + const load=async reset=>{ + scientificController?.abort();const controller=new AbortController();scientificController=controller;scientificSlice=null; + const field=manifest.fields.find(item=>item.id===$('scientific-field').value),shape=scientificShape(record,field),axis=$('scientific-axis').value; + $('scientific-index').max=String(shape['zyx'.indexOf(axis)]-1);if(reset||Number($('scientific-index').value)>Number($('scientific-index').max))$('scientific-index').value='0'; + const request={frame:Number($('scientific-frame').value),field:field.id,axis,index:Number($('scientific-index').value)}; + const owns=()=>generation===scientificGeneration&&scientificController===controller&¤t?.id===record.id&¤t.content_hash===record.content_hash; + $('scientific-canvas').hidden=true;$('scientific-status').textContent='Loading exact numeric slice…';$('scientific-probe').textContent='';$('scientific-coordinates').textContent=''; + try{ + const response=await fetch('/api/workbench/records/'+record.id+'/scientific?'+new URLSearchParams(request),{signal:controller.signal}),raw=await response.json(); + if(!response.ok)throw Error(raw.error||'Scientific slice unavailable.'); + const value=decodeScientificSlice(raw,record,request);if(owns())drawScientificSlice(value); + }catch(error){if(owns()&&error.name!=='AbortError')$('scientific-status').textContent=error.message;} + finally{if(scientificController===controller)scientificController=null;} + }; + for(const id of ['scientific-frame','scientific-field','scientific-axis','scientific-index']){ + $(id).oninput=event=>event.stopPropagation(); + $(id).onchange=event=>{event.stopPropagation();load(id==='scientific-field'||id==='scientific-axis');}; + } + for(const id of ['scientific-row','scientific-column'])$(id).oninput=event=>{event.stopPropagation();scientificProbe();}; + load(true); +} +window.addEventListener('pagehide',()=>{++scientificGeneration;scientificController?.abort();scientificController=null;}); diff --git a/static/workbench.css b/static/workbench.css index 7afb5fc5..aab05af2 100644 --- a/static/workbench.css +++ b/static/workbench.css @@ -33,3 +33,5 @@ #mesh-projections{display:flex;flex-wrap:wrap;gap:.7rem}#mesh-projections figure{flex:1 1 180px;margin:0;min-width:0}#mesh-projections svg{display:block;width:100%;max-height:240px;background:#f4f6fa;border:1px solid #ccd4df}#mesh-projections polygon{fill:none;stroke:#28598b;stroke-width:1.2}#mesh-metadata{max-width:100%;overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere} #pumas-gateway-panel {margin-block:1rem;} #pumas-gateway-metadata {max-height:24rem;overflow:auto;} + +.scientific-controls{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:8px}.scientific-controls label{min-width:0}#scientific-canvas{display:block;max-width:100%;width:420px;image-rendering:pixelated;background:#eee}#scientific-status,#scientific-coordinates,#scientific-probe{font-size:12px;overflow-wrap:anywhere} diff --git a/static/workbench.html b/static/workbench.html index b63783f1..36a696db 100644 --- a/static/workbench.html +++ b/static/workbench.html @@ -1,8 +1,8 @@ Tuldok · Dataset workbench
Dataset workspaces · M1 preview · ← Corner studio

Dataset workbench

Real assets, explicit targets, reproducible releases.

Development workspace

Loading collection…

-

Collection

Exact, case-sensitive values; leave blank for any. For values with line breaks, choose JSON strings and enter double-quoted values: "first\nsecond" for LF, \r for CR and \r\n for CRLF. Escape literal backslashes as \\ and quotes as \". Groups match IDs stored on the record, not all related sources. Rights notes are metadata, not permission decisions. Missing notes appear as unknown. Filtered results are dynamic; saved fixed membership and release selection stay separate.

Saved searches · dynamic results

Save the entered search and filters. Open explicitly to search the current collection; new matching records can appear. Fixed selections, editor drafts and review decisions stay separate. Up to 100 searches. Search text is single-line; exact metadata values may contain line breaks.

Saved fixed selections

Save the chosen records at their exact revisions. Searches and filters stay separate; saving or opening never approves annotations.

    Curation diagnostics

    Inspect current metadata in all applied collection results or exact selected IDs. Selection revision pairs stay fixed. Counts describe metadata, not permission or model quality. Missing sources means deleted source records; file integrity is checked during release.

    Refresh to inspect current metadata.

      -

      Annotate & review

      Find Pumas for annotation

      Inspect a local gateway before choosing it for a proposal. Build features describe the compiled program; they do not prove model readiness. Gateway details are advertised, without authentication of library ownership. Existing manual gateway URLs remain available.

      Scan reads local advertisements. Choosing a gateway starts no model request.

      Advertised identity and observation hash

      Open a record to inspect its source and define its target.

      +

      Collection

      Exact, case-sensitive values; leave blank for any. For values with line breaks, choose JSON strings and enter double-quoted values: "first\nsecond" for LF, \r for CR and \r\n for CRLF. Escape literal backslashes as \\ and quotes as \". Groups match IDs stored on the record, not all related sources. Rights notes are metadata, not permission decisions. Missing notes appear as unknown. Filtered results are dynamic; saved fixed membership and release selection stay separate.

      Saved searches · dynamic results

      Save the entered search and filters. Open explicitly to search the current collection; new matching records can appear. Fixed selections, editor drafts and review decisions stay separate. Up to 100 searches. Search text is single-line; exact metadata values may contain line breaks.

      Saved fixed selections

      Save the chosen records at their exact revisions. Searches and filters stay separate; saving or opening never approves annotations.

        Curation diagnostics

        Inspect current metadata in all applied collection results or exact selected IDs. Selection revision pairs stay fixed. Counts describe metadata, not permission or model quality. Missing sources means deleted source records; file integrity is checked during release.

        Refresh to inspect current metadata.

          +

          Annotate & review

          Find Pumas for annotation

          Inspect a local gateway before choosing it for a proposal. Build features describe the compiled program; they do not prove model readiness. Gateway details are advertised, without authentication of library ownership. Existing manual gateway URLs remain available.

          Scan reads local advertisements. Choosing a gateway starts no model request.

          Advertised identity and observation hash

          Open a record to inspect its source and define its target.

          +
          Import a frozen caption corpus

          Choose an expanded Tuldok image-caption v1 release folder with manifest.json and train/val/test metadata and PNGs.

          Captions become drafts for your review. Existing split assignments and protected families are retained. Source identity is recorded as origin evidence; existing records are never updated.

          Ready to import native caption records.

            Successful rows remain imported. Stop prevents the next admission. Batch progress stays in this page; inspect the collection after reload.

            Generate offline

            Known targets from authored scene or template state. These recipes demonstrate reproducible production; they do not establish real-world model quality. Pumas image generation remains in the corner studio.

            Grounded model candidates

            Typed v1 sends finite requests. Text messages only; seed is unavailable. Generation runs until completion, failure or cancellation. Capability inspection has a separate bounded discovery budget.

            Open a human-reviewed text-classification source. Generate class-preserving rewrite proposals using your Pumas gateway; the captured source text is sent to the selected server.

            Schema and exact source quotes are checked; semantic correctness is not. Admitting a candidate creates a draft that still needs annotation review before export. Requested seeds do not guarantee reproducibility across models.

            Freeze selected release

            0 selected

            Only reviewed targets can be released. Related sources stay together, including unselected lineage bridges. Whole groups can prevent exact ratios.

            Select records to preview a release.

            Freeze selected answers

            Answer selection is separate from record selection and collection filters. Select individual answer revisions; edits and newly added answers never replace these fixed pairs.

            0 answers selected

              Whole source families stay together. Percentages weight examples, including multiple answers to one prompt. Draft or stale selections block export. The complete archive is bounded to 40 MiB.

              Select exact reviewed answers to preview.

              Freeze selected preference pairs

              Selections retain exact stored judgment, prompt and answer revisions. Reviewed ties and abstentions are excluded with reasons. Stale, draft or conflicting current directional judgments block export. Refresh/rejudge deliberately after edits.

                Whole source families stay together. Percentages weight emitted pairs; ties and abstentions do not weight splits. 5,000 judgments and 40 MiB per release.

                diff --git a/static/workbench.js b/static/workbench.js index badcc1dd..640f5bdc 100644 --- a/static/workbench.js +++ b/static/workbench.js @@ -95,7 +95,7 @@ function showRecord(record) { if(record.kind === 'image') $('asset-image').src = '/api/workbench/asset/' + record.id + '?revision=' + record.source_revision; $('asset-text').textContent = record.text || ''; $('task').replaceChildren(); - for(const task of record.kind === 'image' ? ['image_detection','image_classification','image_caption'] : record.kind === 'sequence' ? ['sequence_transport'] : record.kind === 'pointcloud' ? ['pointcloud_geometry'] : record.kind === 'volume' ? ['volume_grid'] : record.kind === 'mesh' ? ['mesh_geometry'] : record.kind === 'audio' ? ['audio_timed'] : ['text_classification','text_entities','text_corpus']) { + for(const task of record.kind === 'image' ? ['image_detection','image_classification','image_caption'] : record.kind === 'sequence' ? ['sequence_transport'] : record.kind === 'pointcloud' ? ['pointcloud_geometry'] : record.kind === 'field_sequence' ? ['field_sequence_fields'] : record.kind === 'volume' ? ['volume_grid'] : record.kind === 'mesh' ? ['mesh_geometry'] : record.kind === 'audio' ? ['audio_timed'] : ['text_classification','text_entities','text_corpus']) { const option = document.createElement('option'); option.value = task; option.textContent = task.replaceAll('_',' '); $('task').append(option); } $('corpus-note').value = record.task === 'text_corpus' ? record.annotation?.note || '' : ''; @@ -104,6 +104,7 @@ function showRecord(record) { if(typeof meshShown === 'function') meshShown(record); if(typeof pointcloudShown === 'function') pointcloudShown(record); if(typeof volumeShown === 'function') volumeShown(record); + if(typeof scientificShown === 'function') scientificShown(record); if(typeof audioShown === 'function') audioShown(record); targets = structuredClone(record.annotation?.boxes || record.annotation?.spans || []); $('groups').value = record.groups.join('\n'); $('record-review').value = record.review === 'human_reviewed' ? 'human_reviewed' : 'draft'; @@ -119,7 +120,7 @@ function showRecord(record) { function renderTargets() { const task = $('task').value; $('corpus-controls').hidden = task !== 'text_corpus'; - $('caption-controls').hidden = task !== 'image_caption'; $('label-control').hidden = task === 'text_corpus' || task === 'image_caption' || task === 'sequence_transport' || task === 'mesh_geometry' || task === 'pointcloud_geometry' || task === 'volume_grid' || task === 'audio_timed'; + $('caption-controls').hidden = task !== 'image_caption'; $('label-control').hidden = task === 'text_corpus' || task === 'image_caption' || task === 'sequence_transport' || task === 'mesh_geometry' || task === 'pointcloud_geometry' || task === 'field_sequence_fields' || task === 'volume_grid' || task === 'audio_timed'; $('classification-help').hidden = !task.endsWith('_classification'); $('detection-controls').hidden = task !== 'image_detection'; $('entity-controls').hidden = task !== 'text_entities'; $('targets').replaceChildren(); $('box-overlay').replaceChildren(); @@ -177,7 +178,7 @@ action('editor',async()=>{ if(typeof preferenceDirty !== 'undefined' && (preferenceDirty || preferenceBusy)) throw Error('Save or cancel the judgment edit before saving the annotation.'); const record=current, task=$('task').value, epoch=++editorEpoch, responseEpoch=responseIntentEpoch(); const selectionAtSave=selectionEpoch, pairAtSave=selected.get(record.id); - const annotation=task === 'audio_timed' ? audioAnnotation() : task === 'text_corpus' ? {note:$('corpus-note').value} : task === 'pointcloud_geometry' ? {note:$('pointcloud-note').value} : task === 'volume_grid' ? {note:$('volume-note').value} : task === 'mesh_geometry' ? {note:$('mesh-note').value} : task === 'sequence_transport' ? {note:$('sequence-note').value} : task === 'image_caption' ? {caption:$('caption').value} : task.endsWith('_classification')?{label:$('label').value}:{[task==='image_detection'?'boxes':'spans']:targets}; + const annotation=task === 'audio_timed' ? audioAnnotation() : task === 'text_corpus' ? {note:$('corpus-note').value} : task === 'pointcloud_geometry' ? {note:$('pointcloud-note').value} : task === 'field_sequence_fields' ? {note:$('scientific-note').value} : task === 'volume_grid' ? {note:$('volume-note').value} : task === 'mesh_geometry' ? {note:$('mesh-note').value} : task === 'sequence_transport' ? {note:$('sequence-note').value} : task === 'image_caption' ? {caption:$('caption').value} : task.endsWith('_classification')?{label:$('label').value}:{[task==='image_detection'?'boxes':'spans']:targets}; const saved=await api('records/'+record.id,{revision:record.revision,source_revision:record.source_revision,task,annotation,groups:$('groups').value.split('\n').map(x=>x.trim()).filter(Boolean),review:$('record-review').value}); // A later fixed-set open/reselection owns membership, even when IDs are unchanged. if(selectionAtSave===selectionEpoch && pairAtSave && selected.get(saved.id)===pairAtSave && diff --git a/static/workspace-projects.js b/static/workspace-projects.js index 690616bf..587f4922 100644 --- a/static/workspace-projects.js +++ b/static/workspace-projects.js @@ -1,8 +1,9 @@ 'use strict'; // Projects project server-owned references. Opening never admits an execution. -window.WorkspaceProjects = function ({select, roots, announce, switchSource, contextChanged}) { +window.WorkspaceProjects = function ({select, roots, announce, switchSource, contextChanged, openCatalogRecord}) { const $=id=>document.getElementById(id), esc=value=>window.workspaceEscape(value); let active=null, view=null, detail=null, generation=0, controller=null, evidenceController=null; + let importTarget=null, importing=false, importGeneration=0; let entries=[], aliases=[], listGeneration=0, opening=false, openGeneration=0; const currentSource=()=>select.value; async function api(path, options={}) { @@ -34,7 +35,7 @@ window.WorkspaceProjects = function ({select, roots, announce, switchSource, con }catch(error){if(epoch===listGeneration)$('project-open-status').textContent=error.message;} } function reference(item, category) { - return `
              1. ${esc(item.label)}${esc(item.status)} · ${esc(item.integrity)}

                ${esc(item.path)}

                ${item.reason?`

                ${esc(item.reason)}

                `:''}${item.bytes!==undefined?`${esc(item.bytes)} bytes`:''}${category==='evidence'&&item.status==='available'?``:''}
                Reference identity
                ${esc(JSON.stringify({id:item.id,expected_sha256:item.sha256,observed_sha256:item.observed_sha256},null,2))}
              2. `; + return `
              3. ${esc(item.label)}${esc(item.status)} · ${esc(item.integrity)}

                ${esc(item.path)}

                ${item.reason?`

                ${esc(item.reason)}

                `:''}${item.bytes!==undefined?`${esc(item.bytes)} bytes`:''}${category==='data'?``:''}${category==='evidence'&&item.status==='available'?``:''}
                Reference identity
                ${esc(JSON.stringify({id:item.id,expected_sha256:item.sha256,observed_sha256:item.observed_sha256},null,2))}
              4. `; } function render() { if(!active||!view)return; @@ -43,10 +44,11 @@ window.WorkspaceProjects = function ({select, roots, announce, switchSource, con const changed=detail.manifest_status!=='unchanged'; const items=view==='data'?detail.data:view==='pipeline'?detail.recipes:detail.evidence; const title=view==='data'?'Referenced data':view==='pipeline'?'Referenced recipes':'External evidence'; - const note=view==='data'?'References are inspected in place; they are not imported catalog records.':view==='pipeline'?'Referenced recipes are not executable in project inspection mode.':'External evidence is not a Tuldok-completed run or scientific qualification.'; + const note=view==='data'?'Import supported scientific fields explicitly into the catalog. Saved project references remain unchanged.':view==='pipeline'?'Referenced recipes are not executable in project inspection mode.':'External evidence is not a Tuldok-completed run or scientific qualification.'; root.innerHTML=`
                PROJECT · INSPECTION ONLY

                ${esc(detail.name)}

                ${esc(detail.root)} / ${esc(detail.manifest_path)}

                ${changed?`

                Manifest ${esc(detail.manifest_status)}. Saved references remain unchanged; a different manifest requires a new project ID.

                `:''}

                ${title}

                ${note}

                ${items.length?`
                  ${items.map(item=>reference(item,view==='runs'?'evidence':view)).join('')}
                `:`

                No ${title.toLowerCase()} declared.

                `}
                ${view==='data'?`

                Runtime requirements

                Presence does not qualify a runtime.

                ${detail.requirements.length?`
                  ${detail.requirements.map(item=>`
                • ${esc(item.label)}${esc(item.status)}${item.path?`

                  ${esc(item.path)}

                  `:''}${item.reason?`

                  ${esc(item.reason)}

                  `:''}
                • `).join('')}
                `:'

                No requirements declared.

                '}
                `:''}
                Saved manifest identity

                ${esc(detail.manifest_sha256)}

                Execution: unsupported

                `; root.querySelector('.project-refresh').onclick=()=>loadDetail(active,true); for(const button of root.querySelectorAll('[data-project-evidence]'))button.onclick=()=>loadEvidence(button.dataset.projectEvidence); + for(const button of root.querySelectorAll('[data-project-import]'))button.onclick=()=>openImport(button.dataset.projectImport); contextChanged(); } async function loadDetail(id, refresh=false) { @@ -78,6 +80,31 @@ window.WorkspaceProjects = function ({select, roots, announce, switchSource, con }catch(error){if(evidenceController===request&&epoch===generation&&error.name!=='AbortError')output.textContent=error.message;} finally{if(evidenceController===request)evidenceController=null;} } + function openImport(id) { + const item=detail?.data.find(ref=>ref.id===id); + if(importing||!item||item.status!=='available'||item.integrity==='mismatch'||!(/^[a-f0-9]{64}$/.test(item.observed_sha256||'')))return; + importTarget={project:active,ref:id,sha256:item.observed_sha256};importGeneration++; + $('project-import-name').value=item.label;$('project-import-rights').value='unknown';$('project-import-group').value=''; + $('project-import-identity').textContent=item.path+' · SHA256 '+item.observed_sha256; + $('project-import-status').textContent='Import original field bytes as a new catalog record. Source identity is checked again before import.'; + $('project-import-dialog').showModal();$('project-import-name').focus(); + } + $('project-import-close').onclick=()=>{importGeneration++;$('project-import-dialog').close();if(importing)announce('Import may still complete. Inspect the catalog before repeating it.');}; + $('project-import-dialog').addEventListener('cancel',()=>{importGeneration++;if(importing)announce('Import may still complete. Inspect the catalog before repeating it.');}); + $('project-import-form').onsubmit=async event=>{ + event.preventDefault();if(importing||!importTarget)return; + const target={...importTarget},epoch=++importGeneration,owner=generation; + const group=$('project-import-group').value.trim(); + const body={name:$('project-import-name').value,rights:$('project-import-rights').value,groups:group?[group]:[],parents:[],expected_sha256:target.sha256}; + importing=true;$('project-import-submit').disabled=true;$('project-import-status').textContent='Checking source identity and importing…'; + try{ + const record=await api('/'+encodeURIComponent(target.project)+'/import-data/'+encodeURIComponent(target.ref),{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)}); + if(epoch!==importGeneration||owner!==generation||active!==target.project||!$('project-import-dialog').open)return; + if(typeof record.id!=='string'||record.kind!=='field_sequence')throw Error('Unsupported imported field record response.'); + $('project-import-dialog').close();await openCatalogRecord(record.id); + }catch(error){if(epoch===importGeneration)$('project-import-status').textContent=error.message+' If a reply was lost, inspect the catalog before repeating the import.';} + finally{importing=false;$('project-import-submit').disabled=false;} + }; $('open-project').onclick=()=>{$('project-open-dialog').showModal();$('project-manifest').focus();refreshList();}; $('project-open-close').onclick=()=>{openGeneration++;$('project-open-dialog').close();$('open-project').focus();}; $('project-open-dialog').addEventListener('cancel',()=>{openGeneration++;}); @@ -96,7 +123,7 @@ window.WorkspaceProjects = function ({select, roots, announce, switchSource, con }catch(error){if(epoch===openGeneration)$('project-open-status').textContent=error.message+' Refresh saved projects before repeating an opening whose reply was lost.';} finally{opening=false;$('project-open-submit').disabled=!aliases.length;} }; - window.addEventListener('pagehide',()=>{generation++;listGeneration++;openGeneration++;controller?.abort();evidenceController?.abort();}); + window.addEventListener('pagehide',()=>{generation++;listGeneration++;openGeneration++;importGeneration++;controller?.abort();evidenceController?.abort();}); return { initialize:refreshList, label:()=>detail?.name||entries.find(entry=>'project:'+entry.id===currentSource())?.name||'Project', @@ -104,6 +131,7 @@ window.WorkspaceProjects = function ({select, roots, announce, switchSource, con const id=source.startsWith('project:')?source.slice(8):null; for(const [name,root] of Object.entries(roots))root.hidden=!id||name!==next; if(id===active&&next===view)return; + importGeneration++;$('project-import-dialog').close(); const changed=id!==active;active=id;view=next;generation++;controller?.abort();evidenceController?.abort();controller=null;evidenceController=null; if(changed)detail=null; if(!id)return; diff --git a/static/workspaces-data.js b/static/workspaces-data.js index 7d83862e..ed2d42cd 100644 --- a/static/workspaces-data.js +++ b/static/workspaces-data.js @@ -146,7 +146,14 @@ publish(); return { kind: "available", record: { id: record.id, revision: record.revision, source_revision: record.source_revision } }; } - window.TuldokWorkspaceData = { snapshot, restoreContext, openCurrentRecord, + async function openImportedRecord(id) { + const generation=++correctionGeneration,editor=editorEpoch,selectionIntent=selectionEpoch,responseIntent=responseIntentEpoch(); + await refresh(); + if(generation!==correctionGeneration||editor!==editorEpoch||selectionIntent!==selectionEpoch||responseIntent!==responseIntentEpoch()) + return {kind:"invalid",message:"Data changed while the imported record loaded. Your current work is retained."}; + return openCurrentRecord(id); + } + window.TuldokWorkspaceData = { snapshot, restoreContext, openCurrentRecord, openImportedRecord, cancelRecordOpen: () => { correctionGeneration++; } }; const publish = () => window.parent.postMessage( diff --git a/static/workspaces.html b/static/workspaces.html index 0e894e8c..af9bc12c 100644 --- a/static/workspaces.html +++ b/static/workspaces.html @@ -108,6 +108,17 @@

                Runs

                Loading configured roots…

                + +

                Import scientific fields

                +

                +
                + + + + +
                +

                +

                Unsaved fixture edits

                diff --git a/static/workspaces.js b/static/workspaces.js index 63d02dc8..1d3846fc 100644 --- a/static/workspaces.js +++ b/static/workspaces.js @@ -30,9 +30,10 @@ window.workspaceEscape = (value) => let store = null, pipeline = null, runs = null; + let catalogImportIntent=0; let workspaceIntent = 0, fragmentIntent = null, realCorrection = null; - function invalidateCorrection() { - workspaceIntent++; + function invalidateCorrection(navigation=true) { + workspaceIntent++;if(navigation)catalogImportIntent++; $("catalog-frame").contentWindow?.TuldokWorkspaceData?.cancelRecordOpen?.(); } function renderCorrection(record = catalogContext?.record) { @@ -92,7 +93,31 @@ window.workspaceEscape = (value) => } const projects = WorkspaceProjects({select:$("dataset"), roots:{data:$("project-data-root"),pipeline:$("project-pipeline-root"),runs:$("project-runs-root")}, - announce,switchSource,contextChanged:context}); + announce,switchSource,contextChanged:context,openCatalogRecord}); + async function openCatalogRecord(id) { + try { + switchSource("catalog"); + location.hash="#data"; + await new Promise(resolve=>setTimeout(resolve,0)); + const importIntent=catalogImportIntent,deadline=Date.now()+10000; + while(importIntent===catalogImportIntent&&source==="catalog" && Date.now()setTimeout(resolve,50)); + } + announce(`Imported ${id}. Open it from the current catalog.`); + } catch(error) { + const message=`Imported ${id}. Catalog opening failed: ${error.message}`; + $("catalog-status").textContent=message;announce(message); + } + } let restoringCatalog = false, catalogRestoreNotice = null; function remember() { @@ -367,9 +392,10 @@ window.workspaceEscape = (value) => : "Current Data owner ready."); if (source === "catalog" && currentView === "data") context(); }); - let catalogLoadEpoch = 0; + let catalogLoadEpoch = 0, catalogReady = false; $("catalog-frame").addEventListener("load", async () => { - invalidateCorrection(); + invalidateCorrection(false); + catalogReady=false; const epoch = ++catalogLoadEpoch; const bridge = $("catalog-frame").contentWindow?.TuldokWorkspaceData; if (!bridge) { @@ -398,6 +424,7 @@ window.workspaceEscape = (value) => } } } + if(epoch===catalogLoadEpoch)catalogReady=true; }); async function load() { const epoch = ++loadEpoch; diff --git a/tests/browser_scientific_fields.cjs b/tests/browser_scientific_fields.cjs new file mode 100644 index 00000000..5e34c68c --- /dev/null +++ b/tests/browser_scientific_fields.cjs @@ -0,0 +1,144 @@ +// Real project import and scientific inspection/review/freeze, with independent NumPy cold custody. +'use strict'; +const assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'),crypto=require('node:crypto'); +const {spawn,execFileSync}=require('node:child_process'); +const {pageLoadTracker}=require('./browser_page_load.cjs'); +const {waitForDebugger}=require('./browser_startup.cjs'); +const {qaDirectory,screenshotOptions}=require('./qa_artifacts.cjs'); +const root=path.resolve(__dirname,'..'),captureIndex=process.argv.indexOf('--capture-root'); +assert.ok(captureIndex<0||process.argv[captureIndex+1],'--capture-root needs a directory'); +const captureRoot=captureIndex<0?null:path.resolve(process.argv[captureIndex+1]); +const report=qaDirectory(root,captureRoot?'scientific-fields-native':'scientific-fields-neutral'); +const temporary=fs.mkdtempSync(path.join(os.tmpdir(),'tuldok-fields-browser-')),projectRoot=path.join(temporary,'projects');fs.mkdirSync(projectRoot); +const sha=raw=>crypto.createHash('sha256').update(raw).digest('hex'); +const bundlePath=path.join(projectRoot,'fields.zip'); +const fixtureArgs=['tests/qualify_scientific_fields.py','--output',bundlePath,'--build-only',...(captureRoot?['--capture-root',captureRoot]:[])]; +execFileSync('python3',fixtureArgs,{cwd:root});const source=fs.readFileSync(bundlePath),sourceHash=sha(source); +const first={schema_version:1,id:'c'.repeat(32),name:captureRoot?'Native capture transport only':'Rotated noncubic authored fields',data:[{id:'fields',label:'Generic scientific bundle',path:'fields.zip'}],recipes:[],evidence:[],requirements:[]}; +const manifestRaw=JSON.stringify(first);fs.writeFileSync(path.join(projectRoot,'tuldok.project.json'),manifestRaw); +const inspections=[],captures=[],checks=[],requests=[],errors=[],children=[],tracker=pageLoadTracker();const pause=ms=>new Promise(resolve=>setTimeout(resolve,ms));let ws,send,evaluate,serverErrors=''; +async function until(fn,message='Timed out'){for(let i=0;i<600;i++){const value=await fn();if(value)return value;await pause(100);}throw Error(message);} +function launch(command,args,options){const child=spawn(command,args,options);children.push(child);return child;} +const serverArgs=port=>['-u','app.py','--port',String(port),'--data',path.join(temporary,'data'),'--project-root','sample='+projectRoot]; +(async()=>{ + let server=launch('python3',serverArgs(0),{cwd:root,stdio:['ignore','pipe','pipe']}); + let output='';server.stdout.on('data',chunk=>output+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk); + const port=await until(()=>{if(server.exitCode!==null)throw Error('Project server exited: '+serverErrors);return output.match(/127\.0\.0\.1:(\d+)/)?.[1];}); + const base='http://127.0.0.1:'+port; + const browser=launch(process.env.BROWSER||'/usr/bin/chromium',['--headless','--no-sandbox','--disable-gpu','--no-first-run','--no-default-browser-check','--remote-debugging-port=0','--window-size=1440,900','--user-data-dir='+path.join(temporary,'browser'),'about:blank'],{stdio:['ignore','ignore','ignore'],env:{...process.env,XDG_CONFIG_HOME:temporary,XDG_CACHE_HOME:temporary}}); + const debugPort=await waitForDebugger(browser,path.join(temporary,'browser','DevToolsActivePort')); + const tabs=await(await fetch('http://127.0.0.1:'+debugPort+'/json')).json(),target=tabs.find(tab=>tab.type==='page'&&tab.url==='about:blank');assert.ok(target); + ws=new WebSocket(target.webSocketDebuggerUrl);await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;}); + let next=0;const pending=new Map(); + send=(method,params={})=>new Promise((resolve,reject)=>{const id=++next,timer=setTimeout(()=>{pending.delete(id);reject(Error('CDP timeout '+method));},20000);pending.set(id,{resolve:value=>{clearTimeout(timer);resolve(value);},reject:error=>{clearTimeout(timer);reject(error);}});ws.send(JSON.stringify({id,method,params}));}); + ws.onmessage=event=>{const message=JSON.parse(event.data);tracker.observe(message);if(message.id){const task=pending.get(message.id);pending.delete(message.id);if(task)message.error?task.reject(message.error):task.resolve(message.result);}if(message.method==='Runtime.exceptionThrown')errors.push(message.params.exceptionDetails);if(message.method==='Network.requestWillBeSent')requests.push({url:message.params.request.url,method:message.params.request.method});}; + evaluate=async expression=>{const result=await send('Runtime.evaluate',{expression,awaitPromise:true,returnByValue:true});if(result.exceptionDetails)throw Error(JSON.stringify(result.exceptionDetails));return result.result.value;}; + const doc=frame=>frame?'document.getElementById("catalog-frame").contentDocument':'document'; + const element=(selector,frame=false)=>`${doc(frame)}.querySelector(${JSON.stringify(selector)})`,text=(selector,frame=false)=>evaluate(`${element(selector,frame)}?.textContent`); + async function click(selector,frame=false){ + const box=await until(()=>evaluate(`(async()=>{const e=${element(selector,frame)};if(!e||e.disabled||!e.getClientRects().length)return null;${frame?'document.getElementById("catalog-frame").scrollIntoView({block:"start"});':''}e.scrollIntoView({block:'center'});await new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));const b=e.getBoundingClientRect(),x=b.x+b.width/2,y=b.y+b.height/2,hit=${doc(frame)}.elementFromPoint(x,y),f=${frame?'document.getElementById("catalog-frame").getBoundingClientRect()':'{x:0,y:0}'};return hit&&(hit===e||e.contains(hit))?{x:x+f.x,y:y+f.y}:null})()`),'Click target unavailable '+selector); + for(const type of ['mousePressed','mouseReleased'])await send('Input.dispatchMouseEvent',{type,button:'left',clickCount:1,...box}); + } + async function key(key,code=key,modifiers=0){for(const type of ['keyDown','keyUp'])await send('Input.dispatchKeyEvent',{type,key,code,modifiers,windowsVirtualKeyCode:{Enter:13,Escape:27,Tab:9,a:65}[key]||0,...(type==='keyDown'&&key==='Enter'?{text:'\r',unmodifiedText:'\r'}:{})});} + async function fill(selector,value,frame=false){await click(selector,frame);await key('a','KeyA',2);await send('Input.insertText',{text:value});} + const select=(selector,value,frame=false)=>evaluate(`(()=>{const e=${element(selector,frame)};e.value=${JSON.stringify(value)};e.dispatchEvent(new Event('change',{bubbles:true}));})()`); + async function nav(view){await evaluate(`document.getElementById('nav-${view}').focus()`);await key('Enter');await until(()=>evaluate(`!document.getElementById('view-${view}').hidden && !document.getElementById('project-${view}-root').hidden`));await until(()=>text(`#project-${view}-root`).then(value=>value?.includes('PROJECT · INSPECTION ONLY')));} + async function capture(name,width){ + await send('Emulation.setDeviceMetricsOverride',{width,height:900,deviceScaleFactor:1,mobile:false}); + await evaluate(`(async()=>{const iframe=document.getElementById('catalog-frame');iframe.scrollIntoView({block:'start'});iframe.contentDocument.getElementById('scientific-inspection').scrollIntoView({block:'start'});await new Promise(r=>requestAnimationFrame(()=>requestAnimationFrame(r)));window.scrollBy(0,iframe.getBoundingClientRect().top);})()`); + await pause(100);assert.ok(await evaluate('document.documentElement.scrollWidth<=innerWidth+1'),'Outer overflow'); + assert.ok(await evaluate(`${doc(true)}.documentElement.scrollWidth<=${doc(true)}.defaultView.innerWidth+1`),'Inspector overflow'); + const file=name+'.jpg';fs.writeFileSync(path.join(report,file),Buffer.from((await send('Page.captureScreenshot',screenshotOptions)).data,'base64'));captures.push({file,width,height:900,qualification:'Transport, inspection and custody only; no physics or model qualification'}); + } + const check=value=>{checks.push(value);console.log('PASS',value);}; + async function reload(){const previous=(await send('Page.getFrameTree')).frameTree.frame;await send('Page.reload');await until(()=>tracker.reloaded(previous),'Main document reload not observed');} + async function openProject(manifest,name,id){await click('#open-project');await until(()=>evaluate('document.getElementById("project-open-dialog").open && !document.getElementById("project-open-submit").disabled'));await select('#project-root','sample');await fill('#project-manifest',manifest);await evaluate('document.getElementById("project-open-submit").focus()');await key('Enter');await until(()=>evaluate('document.getElementById("dataset").value==='+JSON.stringify('project:'+id)));await nav('data');assert.match(await text('#project-data-root'),new RegExp(name.replace(/[.*+?^${}()|[\]\\]/g,'\\$&')));} + await send('Page.enable');await send('Page.setLifecycleEventsEnabled',{enabled:true});await send('Runtime.enable');await send('Network.enable'); + const offlineCanary=process.env.LD_PRELOAD?JSON.parse(execFileSync('python3',['-c',"import sys;sys.path.insert(0,'tests');from test_workspace_offline import CANARY;exec(CANARY)"],{cwd:root,encoding:'utf8'})):null; + await send('Page.navigate',{url:base+'/workspaces'});await until(()=>evaluate('!!document.getElementById("open-project")')); + await openProject('tuldok.project.json',first.name,first.id); + assert.equal((await(await fetch(base+'/api/workbench/records')).json()).items.length,0,'Opening does not import'); + await click('[data-project-import="fields"]'); + await until(()=>evaluate('document.getElementById("project-import-dialog").open')); + assert.match(await text('#project-import-identity'),new RegExp(sourceHash)); + await fill('#project-import-name','Whole generic field sequence'); + await fill('#project-import-rights','Test qualification custody only; no science acceptance'); + // Alter only our disposable copy after UI observation; saved references omit a declared hash. + fs.writeFileSync(bundlePath,Buffer.concat([source,Buffer.from('changed')])); + await click('#project-import-submit');await until(()=>text('#project-import-status').then(value=>/changed|hash|stale/i.test(value))); + assert.equal((await(await fetch(base+'/api/workbench/records')).json()).items.length,0,'Stale displayed observation creates no record'); + fs.writeFileSync(bundlePath,source); + await click('#project-import-submit'); + await until(()=>evaluate(`${doc(true)}?.getElementById('scientific-inspection')?.hidden===false`)); + await until(()=>text('#scientific-status',true).then(value=>value?.includes('Producer-declared stage:'))); + const records=(await(await fetch(base+'/api/workbench/records')).json()).items;assert.equal(records.length,1);const row=records[0]; + assert.equal(row.kind,'field_sequence');assert.equal(row.content_hash,sourceHash);assert.equal(row.review,'draft'); + const original=Buffer.from(await(await fetch(base+'/api/workbench/asset/'+row.id)).arrayBuffer());assert.deepEqual(original,source); + check('Opening remains read-only; explicit import refuses stale observed hash before catalog writes, then imports exact bytes'); + const manifest=row.field_sequence.manifest; + for(const field of manifest.fields){ + await select('#scientific-field',field.id,true);await select('#scientific-frame','1',true); + for(const axis of ['x','y','z']){ + await select('#scientific-axis',axis,true); + await until(()=>text('#scientific-status',true).then(value=>value?.includes(field.stage)&&value.includes(manifest.frames[1].time))); + await until(()=>text('#scientific-coordinates',true).then(value=>value?.startsWith(axis.toUpperCase()+' index'))); + const rawSlice=await(await fetch(base+'/api/workbench/records/'+row.id+'/scientific?'+new URLSearchParams({frame:1,field:field.id,axis,index:0}))).text(),actual=JSON.parse(rawSlice); + inspections.push(rawSlice);assert.equal(actual.time,manifest.frames[1].time);assert.equal(actual.stage,field.stage);assert.equal(actual.dtype,field.dtype); + assert.equal(actual.values.length,{x:manifest.grid.shape_xyz[2]+Number(field.centering==='face_z'),y:manifest.grid.shape_xyz[2]+Number(field.centering==='face_z'),z:manifest.grid.shape_xyz[1]+Number(field.centering==='face_y')}[axis]); + assert.match(await text('#scientific-probe',true),new RegExp('physical')); + } + } + await select('#scientific-field',manifest.fields[0].id,true);await select('#scientific-frame','0',true);await select('#scientific-axis','z',true); + await until(()=>text('#scientific-status',true).then(value=>value?.includes(manifest.frames[0].time)&&value.includes(manifest.fields[0].stage))); + if(!captureRoot){assert.match(await text('#scientific-probe',true),/-0 K/);assert.match(await text('#scientific-coordinates',true),/9, -3.75, 8.5/);} + await capture('scientific-inspector-1440',1440);await capture('scientific-inspector-390',390); + check(captureRoot?'All native fields and X/Y/Z planes retain time, stage, dtype and declared geometry':'All authored fields and X/Y/Z planes retain exact time/stage/dtype; rotated noncubic origin and signed zero inspected'); + await send('Emulation.setDeviceMetricsOverride',{width:1440,height:900,deviceScaleFactor:1,mobile:false}); + await fill('#scientific-note','Whole-asset transport inspection. No physical validity, model or training acceptance.',true); + await select('#record-review','human_reviewed',true); + await evaluate(`${element('#editor button:not([type="button"])',true)}.focus()`);await key('Enter'); + await until(async()=>{const value=await(await fetch(base+'/api/workbench/records/'+row.id)).json();return value.review==='human_reviewed';}); + await until(()=>evaluate(`${doc(true)}.defaultView.TuldokWorkspaceData?.snapshot().record?.revision===2`)); + await until(()=>text('#records',true).then(value=>value?.includes('revision 2'))); + await click('#select-page',true);await click('.embed-toolbar button',true); + await evaluate(`${element('#release-form',true)}.closest('details').open=true`); + await fill('#train','100',true);await fill('#validation','0',true);await fill('#test','0',true); + await click('#preview-release',true);await until(()=>evaluate(`${element('#freeze-release',true)}?.disabled===false`)); + await click('#freeze-release',true);await until(()=>evaluate(`!!${doc(true)}.querySelector('#release-result a')`)); + const download=await evaluate(`${doc(true)}.querySelector('#release-result a').href`),releasePath=path.join(report,'scientific-release.zip'); + const response=await fetch(download);assert.equal(response.status,200);const frozen=Buffer.from(await response.arrayBuffer());fs.writeFileSync(releasePath,frozen); + const recoveredPath=path.join(report,'recovered-source.zip'); + const frozenSummary=JSON.parse(execFileSync('python3',['-c',"import json,sys,zipfile;from pathlib import Path;from scientific_release import validate_release;p=Path(sys.argv[1]);r=validate_release(p);z=zipfile.ZipFile(p);m=json.loads(z.read('manifest.json'));assert len(m['records'])==1;row=m['records'][0];assert row['review']=='human_reviewed';Path(sys.argv[2]).write_bytes(z.read(row['asset']));print(json.dumps(r))",releasePath,recoveredPath],{cwd:root,encoding:'utf8'})); + assert.deepEqual(fs.readFileSync(recoveredPath),source); + const inspectionsPath=path.join(report,'inspection-slices.json');fs.writeFileSync(inspectionsPath,'['+inspections.join(',')+']'); + const cold=JSON.parse(execFileSync(process.env.TULDOK_TRAINING_PYTHON||process.env.NUMPY_PYTHON||'/workspace/Tuldok/.venv-training/bin/python',['tests/qualify_scientific_fields.py','--output',recoveredPath,'--read-existing','--inspections',inspectionsPath,...(captureRoot?['--capture-root',captureRoot]:[])],{cwd:root,encoding:'utf8'})); + assert.equal(cold.model_or_training_acceptance,false);assert.equal(cold.physics_acceptance,false); + check('Wholeasset human inspection freezes canonical release; separate NumPy reader verifies exact recovered original payload and opaque sources'); + server.kill('SIGINT');await until(()=>server.exitCode!==null);server=launch('python3',serverArgs(port),{cwd:root,stdio:['ignore','pipe','pipe']});let reopened='';server.stdout.on('data',chunk=>reopened+=chunk);server.stderr.on('data',chunk=>serverErrors+=chunk);await until(()=>reopened.includes('Tuldok:')); + await reload();await until(()=>text('#scientific-status',true).then(value=>value?.includes('Producer-declared stage:'))); + assert.deepEqual(Buffer.from(await(await fetch(download)).arrayBuffer()),frozen);assert.deepEqual(fs.readFileSync(bundlePath),source);assert.equal(fs.readFileSync(path.join(projectRoot,'tuldok.project.json'),'utf8'),manifestRaw); + // A second actual import holds its collection reply while the user explicitly + // chooses an existing clean record. Imported-record navigation must lose intent. + const secondPath=path.join(temporary,'second.zip'); + execFileSync('python3',['-c',"import io,json,sys,zipfile;from pathlib import Path;z=zipfile.ZipFile(sys.argv[1]);m=json.loads(z.read('manifest.json'));m['provenance']['source_id']+='-race-fixture';o=io.BytesIO();w=zipfile.ZipFile(o,'w',zipfile.ZIP_STORED);w.writestr('manifest.json',json.dumps(m));[w.writestr(n,z.read(n)) for n in z.namelist() if n!='manifest.json'];w.close();Path(sys.argv[2]).write_bytes(o.getvalue())",bundlePath,secondPath],{cwd:root}); + fs.copyFileSync(secondPath,bundlePath); + await select('#dataset','project:'+first.id);await until(()=>text('#project-data-root').then(value=>value?.includes(first.name))); + await click('#project-data-root .project-refresh'); + await until(()=>evaluate(`document.querySelector('#project-data-root [data-project-import="fields"]')&&!document.querySelector('#project-data-root [data-project-import="fields"]').disabled`)); + await evaluate(`(()=>{const w=${doc(true)}.defaultView,original=w.fetch.bind(w);w.scientificHeld=false;w.fetch=async(...args)=>{const response=await original(...args);if(new URL(String(args[0]),w.location.href).pathname==='/api/workbench/records'&&!w.scientificHeld){w.scientificHeld=true;await new Promise(resolve=>{w.releaseScientific=resolve});w.fetch=original;}return response;};})()`); + await click('[data-project-import="fields"]');await until(()=>evaluate('document.getElementById("project-import-dialog").open')); + await fill('#project-import-name','Second field sequence for selection race');await click('#project-import-submit'); + await until(()=>evaluate(`${doc(true)}.defaultView.scientificHeld===true`)); + await click('#records .record button',true); + await until(()=>evaluate(`${doc(true)}.defaultView.TuldokWorkspaceData.snapshot().record?.id===${JSON.stringify(row.id)}`)); + await evaluate(`${doc(true)}.defaultView.releaseScientific()`); + await until(()=>text('#catalog-status').then(value=>value?.includes('Data changed while the imported record loaded'))); + assert.equal(await evaluate(`${doc(true)}.defaultView.TuldokWorkspaceData.snapshot().record.id`),row.id); + assert.equal((await(await fetch(base+'/api/workbench/records')).json()).items.length,2); + fs.writeFileSync(bundlePath,source);assert.deepEqual(fs.readFileSync(bundlePath),source); + check('Held import refresh preserves a newer explicit clean-record selection instead of opening the newly imported record'); + assert.deepEqual(await(await fetch(base+'/api/workspace/runs')).json(),[]);assert.deepEqual(errors,[]);assert.ok(!serverErrors.includes('Traceback'),serverErrors); + check('Restart retains reviewed field record and exact release; original project bytes unchanged and no execution submitted'); + fs.writeFileSync(path.join(report,'receipt.json'),JSON.stringify({source_commit:execFileSync('git',['rev-parse','HEAD'],{cwd:root,encoding:'utf8'}).trim(),dirty:!!execFileSync('git',['status','--porcelain'],{cwd:root,encoding:'utf8'}).trim(),command:'node tests/browser_scientific_fields.cjs'+(captureRoot?' --capture-root ':''),capture_receipt_sha256:captureRoot?sha(fs.readFileSync(path.join(captureRoot,'receipt.json'))):null,qualification:captureRoot?'Actual saved native capture converted without element conversion; transport/UI/custody only':'Producer-neutral authored rotated noncubic fields; transport/UI/custody only',physics_acceptance:false,model_or_training_acceptance:false,checks,captures,source_sha256:sourceHash,cold,frozenSummary,offline_canary:offlineCanary,browser:(await send('Browser.getVersion')).product},null,2)); + console.log('PASS scientific field browser:',report); +})().catch(async error=>{console.error(error);console.error('Application stderr:',serverErrors);if(evaluate)try{console.error(await evaluate('JSON.stringify({url:location.href,body:document.body.innerText.slice(-9000),frame:document.getElementById("catalog-frame")?.contentDocument?.body.innerText.slice(-5000)})'));}catch{}process.exitCode=1;}).finally(()=>{ws?.close();for(const child of children.reverse())child.kill('SIGTERM');}); diff --git a/tests/qualify_scientific_fields.py b/tests/qualify_scientific_fields.py new file mode 100644 index 00000000..a22de6cf --- /dev/null +++ b/tests/qualify_scientific_fields.py @@ -0,0 +1,241 @@ +"""Test-only generic bundle construction and independent NumPy cold qualification. + +Native mode wraps saved capture slices without converting element bytes. It never +launches a producer, loads a model, or asserts training/physics acceptance. +""" +import argparse +import hashlib +import io +import json +import math +from pathlib import Path +import struct +import zipfile + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def npy_payload(raw, dtype, shape_zyx): + """Write only an NPY1.0 header around already typed little-endian payload.""" + size = math.prod(shape_zyx) * (4 if dtype == 'f8'"), + raw[:-8]+struct.pack('