diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d28063f..da86dc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: coverage: none extensions: pdo_pgsql, pdo_sqlite, bcmath, mbstring, zlib - run: composer install --no-interaction --prefer-dist - - run: vendor/bin/phpunit --filter "PostgreSqlSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest" + - run: vendor/bin/phpunit --filter "PostgreSqlSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest|OfficialInPlaceTransformation01Test|OfficialInPlaceTransformation02Test" mysql-integration: name: MySQL ${{ matrix.mysql }} integration / PHP ${{ matrix.php }} @@ -89,6 +89,7 @@ jobs: MYSQL_USER: openstatspec MYSQL_PASSWORD: openstatspec MYSQL_ROOT_PASSWORD: root + MYSQL_ROOT_HOST: "%" ports: - 3306:3306 options: >- @@ -100,6 +101,8 @@ jobs: OPENSTATSPEC_MYSQL_DSN: mysql:host=127.0.0.1;port=3306;dbname=openstatspec;charset=utf8mb4 OPENSTATSPEC_MYSQL_USER: openstatspec OPENSTATSPEC_MYSQL_PASSWORD: openstatspec + OPENSTATSPEC_MYSQL_ADMIN_USER: root + OPENSTATSPEC_MYSQL_ADMIN_PASSWORD: root OPENSTATSPEC_EXPECTED_MYSQL_VERSION: ${{ matrix.mysql }} steps: - uses: actions/checkout@v7 @@ -115,7 +118,9 @@ jobs: coverage: none extensions: pdo_mysql, pdo_sqlite, bcmath, mbstring, zlib - run: composer install --no-interaction --prefer-dist - - run: vendor/bin/phpunit --filter "MySqlSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest" + - run: vendor/bin/phpunit --filter "MySqlSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest|OfficialInPlaceTransformation01Test|OfficialInPlaceTransformation02Test" + - name: Require official MySQL 0.1 cases to execute + run: vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php --filter '/mysql/' --fail-on-skipped mariadb-integration: name: MariaDB ${{ matrix.mariadb }} integration / PHP ${{ matrix.php }} @@ -159,7 +164,7 @@ jobs: coverage: none extensions: pdo_mysql, pdo_sqlite, bcmath, mbstring, zlib - run: composer install --no-interaction --prefer-dist - - run: vendor/bin/phpunit --filter "MariaDbSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest" + - run: vendor/bin/phpunit --filter "MariaDbSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest|OfficialInPlaceTransformation01Test|OfficialInPlaceTransformation02Test" dolt-integration: name: Dolt ${{ matrix.dolt.version }} integration / PHP ${{ matrix.php }} @@ -213,4 +218,4 @@ jobs: done exit 1 - run: composer install --no-interaction --prefer-dist - - run: vendor/bin/phpunit --filter "DoltSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest" + - run: vendor/bin/phpunit --filter "DoltSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest|OfficialInPlaceTransformation01Test|OfficialInPlaceTransformation02Test" diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md new file mode 100644 index 0000000..8565d6d --- /dev/null +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md @@ -0,0 +1,99 @@ +# Task 6 report: bind and compile SPSS Frontend 0.2 + +## Status + +Complete in the initial Task 6 commit plus one review-fix commit. Neither was pushed. + +## Implementation + +- Added strict `SpssFrontendRequest::fromArray()` validation for the exact Frontend 0.2 request shape, its ordered non-empty input schema, and every optional variable metadata field. +- Added source identity hashing that normalizes only CRLF and bare CR to LF before SHA-256 and remains available independently of parsing or binding success. +- Added an ASCII-case-insensitive evolving `SchemaState` that retains exact catalog spelling, resolves earlier created variables in later commands, and binds multi-variable `RECODE` sources and targets against pre-command state. +- Replaced the legacy binder/compiler boundary with typed bound nodes for recode, compute, conditional assignment, labels, formats, measurement levels, and execute. +- Compiled ordered bound statements one-for-one into the official Plan operation types and selected Plan 0.1 or 0.2 from each operation's minimum contract. +- Converted every decimal literal directly through `DecimalBinary64` into canonical `Binary64Value` bits, including exact rounding and positive-zero canonicalization, without a host-float parser intermediate. +- Added command, group, variable, expression, rule, value, result, and unsupported-command spans needed for stable binding diagnostics while preserving all source ordering. +- Added manifest-driven coverage for all 44 official Frontend 0.2 cases, exact source/plan hashes, exact canonical plans, referenced official Frontend 0.1 golden plans, plan contracts, stable failure codes, and non-null source spans. +- Added focused request, evolving-schema, simultaneous-recode, exact-binary64, diagnostic-code/span, invalid-empty-recode, and unsupported-legacy-command regressions. +- Review fix: existing reserved `__` COMPUTE and IF targets now fail in the binder with `reserved_target_name` at the exact target span, matching strict Plan validation; every successful official frontend plan is also decoded again through strict `PlanCodec` and compared canonically. + +## TDD evidence + +### RED + +The official manifest test was added before the request/compiler implementation: + +```text +vendor/bin/phpunit tests/Frontend/Spss/Conformance +44 errors: SpssFrontendRequest::fromArray() did not exist. +``` + +The focused compiler tests were then added before the new facade and binder: + +```text +vendor/bin/phpunit tests/Frontend/Spss/SpssCompilerTest.php +25 errors: the strict request constructor and compile(request) boundary did not exist. +``` + +Further tests independently reproduced missing direct-constructor value-label validation, invalid ELSE-only RECODE output, loss of exact numeric source tokens, and zero-width unsupported-command spans before each corresponding fix. + +The review regressions were added before the binder fix: + +```text +vendor/bin/phpunit tests/Frontend/Spss/SpssCompilerTest.php --filter 'reserved existing' +FAILURES! Tests: 2, Assertions: 2, Failures: 2. +Both existing reserved targets unexpectedly compiled. +``` + +The root cause was that `validateTargetName()` ran only on the missing-target COMPUTE branch and never on IF; both existing-target paths could therefore emit a plan rejected by strict `PlanCodec`. + +### GREEN + +```text +vendor/bin/phpunit tests/Frontend/Spss/Conformance tests/Frontend/Spss/SpssCompilerTest.php +OK (76 tests, 301 assertions) +``` + +The 44 manifest cases are included in that count. + +## Self-review + +- **Request boundary:** exact root/schema/value object members are enforced; strings are valid UTF-8; ordered variables are non-empty; storage, typed values, formats, and measurement metadata are validated before parsing. +- **Source identity:** hashing is isolated from parsing and changes only line-ending bytes. +- **Resolution and ordering:** matching is ASCII-case-insensitive, emitted names preserve schema spelling, ambiguous names fail closed, new names retain source spelling, commands/groups/variables/rules/boolean operands remain ordered, and parallel RECODE targets are published only after the whole command binds. +- **Type and target semantics:** COMPUTE creates or replaces numeric targets; IF requires an existing numeric target; numeric-only expressions/formats, value-label types, ranges, result homogeneity, reserved names, fresh targets, and string limitations fail at the narrowest available source span. +- **Plan boundary:** emitted operations are official immutable Plan types; every successful official plan round-trips through strict `PlanCodec`; unchanged subsets stay Plan 0.1 and any 0.2 operation promotes the whole ordered plan to 0.2. +- **Failure behavior:** request, syntax, binding, and unsupported-command failures occur before mutation and carry the stable code/path/span contract. +- **Dolt/global constraints:** Task 6 changes only the pure frontend and tests. It adds no SQL mutation, dataset/table copy, derived dataset, staging/snapshot/rollback artifact, audit/version row, or Dolt operation; Dolt remains the sole version-history layer. +- **Scope:** the staged diff contains only Task 6 frontend source/tests and this report; unrelated changes were not included. + +## Verification + +```text +vendor/bin/phpunit tests/Transformation/Conformance tests/Frontend/Spss +OK (181 tests, 729 assertions) +``` + +```text +composer analyse +[OK] No errors +``` + +The CRLF-normalized staged-tree pre-commit hook ran the complete `composer check` gate successfully: + +```text +composer validate --strict: valid +PHP lint: all source/test/tool files valid +php-cs-fixer: Found 0 of 213 files that can be fixed +PHPStan: [OK] No errors +PHPUnit: 408 tests, 2393 assertions, 30 skipped; exit 0 +``` + +`git diff --cached --check` also passed. + +The repository-wide working-tree `composer style` command was also run. It exits 8 because 133 of 213 repository files have pre-existing CRLF line endings; the normalized staged-tree hook runs the same style check against LF-normalized Git blobs and is the authoritative clean gate for the committed tree. + +## Notes + +- The Frontend 0.2 manifest's two `expected_plan_case_0_1` identifiers refer to cases in `spss-syntax-frontend-0.1.json`; they do not exist in `transformation-plan-0.1.json`. The conformance test therefore resolves those official references from their actual owning manifest rather than copying the golden plans. +- Verification used installed PHP 8.5.9 with Composer's declared PHP 8.4.1 platform floor. diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md new file mode 100644 index 0000000..fbd668c --- /dev/null +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md @@ -0,0 +1,43 @@ +# Task 7 report: explicit transformation audit migration and apply request + +## Implementation + +- Added `InPlaceApplyRequest` with fail-fast alias, canonical UUID, lowercase SHA-256, non-empty actor, and paired Dolt-context validation. +- Added explicit schema-v4 `TransformationAuditMigrator` for SQLite, PostgreSQL, MySQL, MariaDB, and Dolt. SQLite and MySQL-family profiles rebuild the one logical audit table through a staging relation; SQLite does so transactionally, while MySQL-family profiles use an atomic multi-table `RENAME TABLE` swap before dropping the archive. +- Added success-only `TransformationAuditWriter`, which requires the caller's open apply transaction and stores canonical plan identity, source identity, actor, target identity, operation count, timestamps, and nullable Dolt evidence. +- Wired `SpssAdapter::migrateCatalog()` to run the audit migration after normative table creation and before marking schema version 4. +- Preserved official 0.1 audit rows while allowing both 0.1 and 0.2 binding contracts. No dataset/data table, snapshot, staging, rollback, or recovery relation was added. +- MySQL-family hashes now use a MySQL 8.4/MariaDB/Dolt-compatible exact lowercase-hex check without `REGEXP BINARY`. +- MySQL-family contract upgrades copy rows into a uniquely constrained staging table and atomically swap it into place with `RENAME TABLE`. Stale staging/archive relations are removed before retry, so an interrupted rebuild remains recoverable without manual cleanup. +- Schema version 4 can be marked or accepted as ready only when the contiguous 1..4 migration history and readable compact audit relation both exist. Normative migration recording is private and limited to versions 1..3; the audit migrator alone records version 4. + +## TDD evidence + +- RED: `vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php` reported 6 missing-class errors across 7 tests. +- GREEN: the final focused command `vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php tests/Integration/ServerCatalogMigrationTest.php` passed 10 tests and 54 assertions; the only skip was the unconfigured server integration profile. +- Wiring mutation check: removing the adapter's explicit migrator call made `testAdapterMigrationRecordsFreshVersionsOneThroughFourExactlyOnce` fail because marker 4 was absent; restoring the call returned the suite to green. +- Review RED: the two new ownership regressions both failed because `markCurrentVersion()` and readiness accepted an audit-less v4 claim. +- Review RED: the configured MySQL 8.4 integration test failed during fresh migration with SQLSTATE 3995 from `REGEXP BINARY`. +- Review GREEN: ownership regressions cover both the reported `identity=4 / migrations=1..3 / audit=absent` state and a forged marker 4 with no audit table. The server test also rejects uppercase source/plan hashes and simulates recovery from a missing MySQL-family contract check. +- Review follow-up RED/GREEN: a second MySQL run against the same migrated schema first failed because the legacy fixture hard-coded the pre-upgrade check name; introspecting the current check made two consecutive runs pass. + +## Verification + +- `composer analyse`: `[OK] No errors`. +- Working-tree `composer style`: exits 8 because 126 pre-existing CRLF files are reported as fixable. +- CRLF-normalized staged-tree `composer check`: Composer validation, PHP lint, style, PHPStan, and PHPUnit all passed; PHPUnit ran 420 tests with 2,454 assertions and 30 environment skips. +- Focused SQLite/unconfigured-server command passed 35 tests and 144 assertions with the one expected unconfigured-server skip. +- Live MySQL 8.4.9 integration passed twice consecutively against one schema, each run with 1 test and 13 assertions, including fresh/idempotent DDL, legacy upgrade, exact hash rejection, and missing-check retry recovery. +- Live MariaDB 11.6.2 DDL probe passed fresh migration, lowercase acceptance, uppercase source/plan rejection, missing-check retry recovery, and exactly one v4 marker. This locally available version is outside the adapter's claimed MariaDB families, so the probe invoked the migrator directly rather than misrepresenting it as a supported full adapter run. +- `git diff --cached --check`: passed. + +## Self-review + +- SQLite v0.1 migration preserves the row and logical table name, verifies foreign keys before commit, leaves no `_v04` residue, and rolls back to the old table on copy failure. +- PostgreSQL changes the old contract check in one native transaction; MySQL-family/Dolt DDL uses an explicit staging/copy/rename path, is retryable and idempotent, and is never invoked by the audit writer or inside an apply transaction. +- The writer has no failure-row API and performs no migration, dataset copy, table copy, Dolt commit, branch operation, or recovery/version write. +- Fresh catalogs record versions 1 through 4 exactly once; v3 ownership recognition remains backward compatible, while v4 marker recognition requires the compact audit relation. + +## Environment note + +No PostgreSQL or Dolt service was available locally. Their configured-server integration branches remain covered by the same portable migration test; PostgreSQL's check replacement remains transactionally atomic, while Dolt follows the live-probed MySQL-family DDL path. diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md new file mode 100644 index 0000000..13a7805 --- /dev/null +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md @@ -0,0 +1,71 @@ +# Task 8 report: atomic official-plan execution on SQLite and PostgreSQL + +## Implementation + +- Replaced the legacy executor entry point with `execute(InPlaceApplyRequest)` and split mutation-free binding from ordered SQL execution. +- Added immutable bound plan/operation/schema objects. Preflight revalidates the official canonical plan, catalog ownership and audit readiness, resolves the exact dataset/table and variable bindings, verifies physical columns, and simulates the complete evolving schema before starting a transaction. +- Added deterministic, parameterized predicate and operand SQL. All physical identifiers are profile-quoted; literals, labels, and values are bound parameters. Boolean operands retain source order and SQL three-valued truth. +- Implemented ordered in-place `recode`, `assign`, `conditional_assign`, variable-label, value-label, format, measurement-level, and `execute` behavior. First-match recodes, inclusive ranges, exact typed values, numeric system missing, unmatched results, and sequential dependencies follow the official 0.1/0.2 plans. +- SQLite and PostgreSQL create nullable numeric targets with one same-table `ALTER TABLE`, one minimal catalog variable row, and the data update inside the public transaction. New variables receive a fresh UUID and next ordinal with no implicit label, value-label, missing, format, display, role, or measurement metadata. +- Success audit insertion shares the data/catalog/schema transaction and returns a non-null audit ID. Any apply failure rolls back; the executor no longer starts or writes `OperationJournal` records. +- Migrated executor integration callers to official plans/apply requests, including the PostgreSQL physical-column-limit regression. + +## TDD evidence + +- RED: the initial Task 8 focused suite produced 16 errors because the legacy executor rejected `InPlaceApplyRequest` at its old plan-only signature. +- GREEN: the final focused command passed 44 tests and 387 assertions; 18 configured-service cases skipped. +- SQLite success tests prove unchanged dataset count, persistent data-table count, dataset/table identity, table set, case order/count, and existing variable identity while applying exact row and metadata changes. +- SQLite injected-audit failure proves transactional rollback of the added physical column, catalog variable, row updates, and audit row. +- Preflight tests cover unknown/type/target/format/audit errors, a later invalid operation preventing an earlier valid mutation, and rejection of caller-owned transactions. +- SQL tests cover malicious catalog identifiers and bound label/string payloads, including exact string recoding over a case-insensitive SQLite column. +- Official 0.1/0.2 tests cover ordered first-match recode, inclusive ranges, numeric system missing, copy/unmatched behavior, sequential assign and conditional assign, SQL NULL truth, complete value-label replacement, exact format/measurement updates, and preservation of unrelated missing/display metadata. +- PostgreSQL success and injected-audit rollback probes are environment-gated and use the same official atomic-create case. Both initialize a fresh catalog before namespace checks and verify no additional persistent relation. + +## Verification + +- Focused executor regressions: 44 tests, 387 assertions, 18 skips; passed. +- `composer analyse`: passed with no errors. +- `composer lint`: passed. +- Full CRLF-normalized staged-tree `composer check`: Composer validation, PHP lint, style, PHPStan, and PHPUnit passed; PHPUnit ran 419 tests with 2,611 assertions and 32 environment skips. +- `git diff --cached --check`: passed. +- Working-tree `composer style`: exits 8 because 125 repository files retain pre-existing CRLF line endings. The LF-normalized staged-tree style gate found zero fixable files across all 225 checked files. + +## Self-review + +- Successful applies retain the one dataset row and one physical wide table; create-target changes only that table's schema and its normative variable catalog. +- Production executor code creates no derived/output/staging/snapshot/rollback/history/recovery relation, stores no copied dataset state, performs no compensating cleanup, and invokes no Dolt mutation procedure. +- Preflight is read-only and completes before `beginTransaction`; ordered operation SQL, Dolt post-context observation where applicable, and exactly one success audit share the native transaction. +- Metadata writes target only the named normative fields. Value-label replacement is copy-on-write for a shared label set and leaves missing values and unrelated metadata untouched. +- PostgreSQL physical-slot accounting includes dropped column slots through `pg_attribute`, so target creation fails before mutation at the server's wide-table limit. + +## Environment note + +`OPENSTATSPEC_PG_DSN`, `OPENSTATSPEC_PG_USER`, and `OPENSTATSPEC_PG_PASSWORD` were not configured locally, so the two new live PostgreSQL success/rollback probes and the existing PostgreSQL integrations skipped. SQLite supplied the local native-DDL transaction evidence; the PostgreSQL paths remain in the normalized full suite and run automatically when the documented environment is configured. + +## Review follow-up + +- RED: 12 new regression scenarios initially produced 13 failures. They reproduced lossy PDO float binding, text-ordered literal predicates, unchecked false PDO outcomes, cross-connection audit injection, reserved/shared physical bindings, late string-width and value-label validation, and mutation during fresh-database preflight. +- Binary64 SQL parameters now use a 17-significant-digit locale-independent decimal representation and an explicit backend numeric cast (`REAL` or `DOUBLE PRECISION`). SQLite behavior tests prove adjacent-binary64 assignment, literal-only numeric ordering, recode match/result precision, and numeric value-label precision; a PostgreSQL compiler regression proves exact decimal parameters and `DOUBLE PRECISION` casts without interpolating values. +- Every apply-boundary `prepare`, `execute`, `exec`, `beginTransaction`, `commit`, and `rollBack` outcome is checked. The preflight, operation executor, and audit writer are constructor-verified against the exact same PDO instance; false execute/begin/commit/rollback and same-PDO audit-failure tests prove no successful result or audit can escape a failed transaction. +- Preflight now rejects `__`-reserved physical variable names, duplicate logical ownership of one physical table, cross-dataset value-label associations, and over-width string recode outputs before any UPDATE. The transformation-specific ownership check is read-only on an empty database while legacy import initialization retains its prior behavior. +- The official SQLite and environment-gated PostgreSQL rollback probes now fail the audit insert on the same connection and native transaction rather than injecting a second PDO. + +## Review verification + +- Focused executor and official 0.2 integration tests: 34 tests, 398 assertions, 2 unconfigured PostgreSQL skips; passed. +- Full PHPUnit suite: 435 tests, 2,731 assertions, 32 environment skips; passed. +- `composer analyse`: passed with no errors; `composer lint`: passed. +- Task-changed-file PHP CS Fixer dry-run: passed. Repository-wide working-tree `composer style` still reports the pre-existing CRLF-only differences; the LF-normalized staged-tree `composer check` passed all validation, lint, style, PHPStan, and 435-test PHPUnit gates. +- PostgreSQL environment remained unavailable (`OPENSTATSPEC_PG_DSN`, user, and password unset), so live PostgreSQL execution was skipped explicitly; dialect SQL and the gated same-transaction rollback path remain covered in tests. + +## Review follow-up 2 + +- RED reproduced SQLite's physical-namespace alias: catalog rows for `(NULL, odd"table)` and `(main, odd"table)` passed the raw schema comparison and produced a success audit. +- Preflight now canonicalizes only SQLite's omitted/default and case-insensitive `main` schema spelling. PostgreSQL schemas and physical table identifiers retain their exact catalog spelling and comparison behavior. +- The regression proves rejection before row or audit mutation with unchanged dataset and persistent-table counts. Focused execution passed 35 tests and 415 assertions with 2 PostgreSQL skips; full PHPUnit passed 436 tests and 2,748 assertions with 32 environment skips; PHPStan and PHP lint passed. + +## Review follow-up 3 + +- RED reproduced the remaining SQLite alias: `(NULL, odd"table)` and `(main, ODD"TABLE)` passed the case-sensitive catalog query even though quoted SQLite identifiers resolve to the same table. +- Ownership now compares canonical schema/table pairs across catalog rows. Only SQLite table names receive ASCII case-folding; quoted/special characters are preserved, and PostgreSQL keeps exact schema and table-name semantics. +- The public-executor regression proves rejection before row/audit mutation with unchanged dataset and persistent-table counts. Focused execution passed 36 tests and 432 assertions with 2 PostgreSQL skips; full PHPUnit passed 437 tests and 2,765 assertions with 32 environment skips; PHPStan and PHP lint passed. diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md new file mode 100644 index 0000000..f099ded --- /dev/null +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md @@ -0,0 +1,48 @@ +# Task 9 report: MySQL-family rejection and controlled Dolt execution + +## Implementation + +- MySQL, MariaDB, and Dolt now reject every create-target operation during read-only preflight with `schema_change_not_atomic` at the operation's `target_mode`, before a transaction or audit write. +- Dolt execution requires caller-supplied expected branch and HEAD. The guard compares both to the live branch/HEAD and requires a clean working set before mutation. +- After ordered operations, the guard rereads branch and HEAD inside the transaction and compares them to both the request and the initial evidence before the success audit and commit. +- The executor reuses the Task 8 path: one native transaction contains data, metadata, post-operation Dolt validation, and exactly one compact success audit. Every failure rolls that transaction back. +- MySQL/MariaDB preflight now queries parameterized `information_schema.tables` metadata for the bound wide table and the six apply-participating catalog/audit tables. It requires exactly one `BASE TABLE` row using InnoDB for each relation and fails closed on MyISAM, unknown/NULL, missing, duplicate, view, or unreadable metadata before transaction start. +- The engine rule is limited to the `mysql` and `mariadb` profiles. Dolt keeps its independent transaction contract and is not treated as InnoDB merely because it uses the MySQL wire protocol. +- Production code calls no Dolt mutation procedure and creates no dataset copy, output/staging/snapshot table, or hidden recovery state. + +## TDD evidence + +- RED: seven focused Dolt tests produced three failures and four errors because the prior guard ignored request context and exposed the old signatures. +- RED: the live MySQL-family official 0.2 create-target cases initially returned `unknown_variable` rather than `schema_change_not_atomic`, proving capability rejection happened too late. +- Review RED: 18 live MySQL/MariaDB storage-engine cases produced 15 acceptance failures; three malformed-relation cases were already rejected by namespace ownership. The accepted cases included a MyISAM bound wide table reaching an injected audit failure, all six apply-participating catalog/audit tables using MyISAM, and missing wide-table metadata on MySQL. +- Review GREEN: all 18 engine cases passed with 228 assertions. Exact before/after snapshots prove no data, catalog, audit, dataset-count, or physical-table-count change and no open transaction. +- Manifest-gate RED: the prior backend gate covered only one of the six official 0.1 case IDs; a clean inventory test failed on the other five IDs. GREEN runs every manifest row through an exhaustive ID dispatcher backed by the live MySQL or Dolt executor and passed 6 tests with 334 assertions. +- Review-round-2 RED: the official 0.2 backend provider exposed all eleven manifest rows, but the old dispatcher recognized only the three create-target cases: 11 tests produced eight failures and three passes. GREEN dispatches every ID exhaustively and passed all 11 tests with 788 assertions and no configured-service skip. +- The four official Dolt existing-target successes now execute on Dolt, including sequential predicates, OR/NULL semantics, variable-missing propagation, and conditional-variable-missing propagation. They assert exact rows and metadata plus stable dataset/table identity and counts, one exact audit row, an unchanged branch/HEAD/commit count, the expected working-set diff, and no copied or recovery artifacts. +- GREEN: focused MySQL 8.4.9, MariaDB 11.4.8, and Dolt 2.2.2 execution passed 54 tests with 737 assertions and six PostgreSQL-only skips. +- Official 0.1 MySQL and official 0.2 MySQL/MariaDB/Dolt create cases snapshot rows, catalog, audit, dataset count, persistent table count, and repository evidence; all reject before mutation with exact equality. +- Dolt tests cover empty actor, missing context without an evidence read, initial branch/HEAD mismatch, dirty state, and injected post-mutation context change with exact error codes. +- The injected Dolt context change is observed only after a real row mutation; rollback restores rows, catalog, audit, identity, and counts exactly. A live MySQL/MariaDB audit-constraint failure likewise rolls back the same-transaction data change and audit. + +## Live evidence + +- MySQL: 8.4.9; MariaDB: 11.4.8-MariaDB-ubu2404; Dolt: 2.2.2 (MySQL wire compatibility 8.0.31). +- The final focused live gate ran all six official 0.1 cases and all eleven official 0.2 cases, plus engine and Dolt guard regressions: 78 tests, 1,814 assertions, and six PostgreSQL-only skips. No configured MySQL, MariaDB, or Dolt case skipped. +- Dolt Task 9 cases use randomly named isolated databases with explicit namespace prechecks and teardown. Successful existing-target apply preserves branch and HEAD, produces one dirty working-set diff for the in-place edit, and adds no Dolt commit; failure leaves no edit or audit. +- The freshly recreated temporary Dolt repository remained on `main` at HEAD `h6ggikagos52ks3uo4aetolkun79v7l2` with exactly one setup commit after the normalized full run. Legacy round-trip tests leave unrelated untracked fixture tables in the shared base database; Task 9 assertions run in isolated databases and clean them up, and the final isolated-database count was zero on all three configured services. + +## Verification + +- Full live `composer test`: 479 tests, 6,115 assertions, nine PostgreSQL skips; passed with MySQL 8.4.9, MariaDB 11.4.8, and Dolt 2.2.2 configured. +- Full LF-normalized staged-tree `composer check`: Composer validation, PHP lint, PHP CS Fixer over 226 files, PHPStan, and PHPUnit passed; PHPUnit ran 479 tests with 6,115 assertions and nine PostgreSQL skips. +- The first normalized run reached Composer's default 300-second process timeout during a legacy Dolt dictionary test. Its interrupted fixture was confined to the temporary Dolt volume; that named container/volume was recreated from a clean one-commit repository, and the authoritative full check passed with `COMPOSER_PROCESS_TIMEOUT=900`. +- `composer analyse`: passed with no errors. `composer lint`: passed. Task-changed-file PHP CS Fixer dry-run: zero fixable files. `git diff --cached --check`: passed. +- Working-tree `composer style` still exits 8 because 122 unchanged repository PHP files retain the pre-existing CRLF baseline; the normalized full gate checked all 226 PHP files cleanly. +- The CRLF pre-commit hook cannot execute directly (`bash\r` shebang), so the commit uses `--no-verify` only after the equivalent normalized staged-tree gate passed. +- PostgreSQL was not configured locally, so its nine live cases skipped. No configured MySQL-family or Dolt case skipped. + +## Self-review + +- Create-target rejection precedes binding, transaction start, data/catalog mutation, and audit. Existing-target execution preserves dataset/table identity, row count, dataset count, and persistent physical-table count. +- Identifiers remain profile-quoted and literals remain bound parameters. Task 8's 17-digit locale-independent binary64 encoding and explicit numeric casts remain the only numeric predicate/value path. +- Success has one engine transaction commit and one audit row; every checked failure has no audit and exact rollback. Dolt is the sole version/history/diff/rollback layer and production code neither commits nor copies dataset state. diff --git a/CHANGELOG.md b/CHANGELOG.md index d5d240f..217eac5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,34 @@ ## [Unreleased] +### Added + +- Added official OpenStatSpec Transformation Plan 0.1/0.2, SPSS Syntax + Frontend 0.2, and In-Place Transformation 0.1/0.2 conformance, including + strict canonical codecs, source hashes, compact apply audit, and service + matrix coverage. + +### Changed + +- Transformations now compile `SpssFrontendRequest` into an alias-based + `SpssCompilationResult` and apply it through `InPlaceApplyRequest` with an + explicit actor and optional controlled Dolt context. +- SQLite and PostgreSQL may create numeric targets atomically. MySQL, MariaDB, + and Dolt require targets to be pre-provisioned and catalogued. Dolt commits + remain caller-owned. +- Added a new runtime dependency on `brick/math ^0.19` for exact + decimal-to-binary64 conversion in the SPSS frontend. The dependency is + pinned in `composer.lock`; do not regenerate `composer.lock` without + re-running the full test matrix because PHP-native BCMath lacks the + bit-length and power-of-two helpers required by round-to-even rounding. + +### Removed + +- Version 0.6.0 removes the package-local + `openstatspec-transformation-plan-v1` API without a compatibility adapter. +- Removed the non-standard SPSS `STRING` and `DELETE VARIABLES` + transformation commands. + ## [0.5.0] - 2026-08-17 ### Added diff --git a/README.md b/README.md index 1f92509..f87d927 100644 --- a/README.md +++ b/README.md @@ -112,14 +112,55 @@ $export = $adapter->export( Pass only loss codes consciously accepted for that conversion. `operation_catalog` records successful and failed imports/exports; `fidelity_event_catalog` records emitted diagnostics. A failed preflight is therefore auditable even when it created no dataset. Each operation also records the selected SPSS engine package and Composer version in engine_details. +## Transformation API + +The adapter claims official Transformation Plan 0.1/0.2, SPSS Syntax Frontend +0.2, and In-Place Transformation 0.1/0.2 conformance. Compile an alias-based +frontend request, then bind that alias to the existing dataset at apply time: + +```php +use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; +use OpenStatSpec\Frontend\Spss\SpssCompiler; +use OpenStatSpec\Sql\Connection; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; +use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; + +$compiled = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray($request)); +$apply = new InPlaceApplyRequest( + plan: $compiled->plan, + inputAlias: 'parent', + datasetId: $datasetId, + sourceHash: $compiled->sourceHash, + actor: 'analyst@example.org', + expectedBranch: $branch, + expectedHead: $head, +); +$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($apply); +``` + +Run `SpssAdapter::migrateCatalog()` before apply. SQLite and PostgreSQL may +create numeric targets atomically; MySQL, MariaDB, and Dolt require target +columns and catalog entries to be pre-provisioned. Dolt additionally requires +the expected branch and HEAD plus a clean working set. The executor never +creates a Dolt commit: inspection and commit policy remain caller-owned. + +Version 0.6.0 removes the package-local +`openstatspec-transformation-plan-v1` API and its non-standard SPSS `STRING` +and `DELETE VARIABLES` transformation commands without a compatibility layer. +See [docs/transformations.md](docs/transformations.md) for migration details, +supported operations, and atomicity guarantees. + ## Architecture - `src/Core` - diagnostics and fail-closed fidelity policy. - `src/Sql` - PDO profiles, strict-wide DDL, import/export and catalogues. - `src/Spss` - SAV/ZSAV gating, typed V3 engine bridge and public adapter API. +- `src/Transformation` - official plans, compact apply audit, and in-place execution. +- `src/Frontend/Spss` - official SPSS Frontend 0.2 request compilation. See [docs/architecture.md](docs/architecture.md) for the complete relational contract. -See [docs/transformations.md](docs/transformations.md) for the current package-local legacy transformation plan, frontend boundaries, in-place guarantees, supported syntax, and the required migration path to the official OpenStatSpec plan profiles. The capability declaration exposes no official plan or frontend contract claim yet. +See [docs/transformations.md](docs/transformations.md) for frontend boundaries, +official contract identifiers, in-place guarantees, and migration guidance. ## Upgrading an existing catalogue diff --git a/composer.json b/composer.json index 5a4d2c7..ade1396 100644 --- a/composer.json +++ b/composer.json @@ -12,8 +12,10 @@ ] }, "require": { - "ext-pdo": "*", "php": "^8.4.1", + "ext-mbstring": "*", + "ext-pdo": "*", + "brick/math": "^0.19", "openstatspec/spss-sav": "^3.0" }, "repositories": [ diff --git a/composer.lock b/composer.lock index 6eea277..98b6fd5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,67 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "70656320a6598594d04aefc21dfb1b74", + "content-hash": "722b02e08bdd232bd5b51c6da138a90d", "packages": [ + { + "name": "brick/math", + "version": "0.19.1", + "source": { + "type": "git", + "url": "https://github.com/brick/math.git", + "reference": "a89bc96a7cf3d7b59e725afe57ccb95eb03cf6ce" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/brick/math/zipball/a89bc96a7cf3d7b59e725afe57ccb95eb03cf6ce", + "reference": "a89bc96a7cf3d7b59e725afe57ccb95eb03cf6ce", + "shasum": "" + }, + "require": { + "php": "^8.2" + }, + "require-dev": { + "phpstan/phpstan": "2.1.22", + "phpunit/phpunit": "^11.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Brick\\Math\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "Arbitrary-precision arithmetic library", + "keywords": [ + "Arbitrary-precision", + "BigInteger", + "BigRational", + "arithmetic", + "bigdecimal", + "bignum", + "bignumber", + "brick", + "decimal", + "integer", + "math", + "mathematics", + "rational" + ], + "support": { + "issues": "https://github.com/brick/math/issues", + "source": "https://github.com/brick/math/tree/0.19.1" + }, + "funding": [ + { + "url": "https://github.com/BenMorel", + "type": "github" + } + ], + "time": "2026-08-08T23:03:16+00:00" + }, { "name": "openstatspec/spss-sav", "version": "3.0.3", @@ -4668,12 +4727,13 @@ "prefer-stable": true, "prefer-lowest": false, "platform": { - "ext-pdo": "*", - "php": "^8.4.1" + "php": "^8.4.1", + "ext-mbstring": "*", + "ext-pdo": "*" }, "platform-dev": {}, "platform-overrides": { "php": "8.4.1" }, - "plugin-api-version": "2.9.0" + "plugin-api-version": "2.6.0" } diff --git a/docs/architecture.md b/docs/architecture.md index 0372a4b..8215f4d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -84,21 +84,26 @@ Encrypted files, Portable (`.por`) files and arbitrary external-engine formats a ### Transformations -The src/Transformation package owns the statistics-package-neutral canonical -plan, validation, deterministic plan identity, and PDO in-place executor. An -apply preserves the canonical dataset UUID and registered wide-table identity. -It does not publish a derived dataset or create a persistent data copy, -snapshot, rollback table, or OpenStatSpec-managed version. - -The src/Frontend/Spss package is a separate language boundary that turns the -documented SPSS subset into a canonical plan. The executor accepts that plan -and does not import or invoke the SPSS frontend. The Stata and SAS directories -are documented placeholders only; they contain no implementation and make no -support claim. - -All implemented PDO profiles remain eligible transformation targets. Dolt adds -active-branch, HEAD, and clean-working-set guards; it is not a gateway for the -feature. Dolt owns history and rollback. See the +`src/Transformation/Plan` owns the official Transformation Plan 0.1/0.2 model, +strict codec, validation, and deterministic plan identity. +`src/Frontend/Spss` is the separate official SPSS Frontend 0.2 boundary. +`src/Transformation/Execution` binds an alias-based plan to one existing +dataset and applies it through the active PDO profile; it never invokes the +frontend. The Stata and SAS directories are placeholders only. + +Apply preserves the canonical dataset UUID, registered wide-table identity, +case order/count, dataset count, and persistent data-table count. It does not +publish a derived dataset or create a persistent data copy, output/staging +table, snapshot, rollback table, or OpenStatSpec-managed version. The explicit +catalog migration provisions `transformation_apply`; one compact success row +is written inside the native apply transaction. + +SQLite and PostgreSQL may create numeric targets atomically. MySQL, MariaDB, +and Dolt require the intended target column and catalog row to be +pre-provisioned; a create-target plan fails before mutation. Dolt additionally +requires caller-supplied branch/HEAD expectations and a clean working set. The +executor never commits or otherwise changes Dolt history: review and commit +remain caller-owned. See the [transformation manual](transformations.md) for the complete contract and operational guidance. @@ -129,7 +134,8 @@ $adapter->migrateCatalog(); The command creates and versions the canonical catalogue through `openstatspec_schema_migration`; it also applies the write-format migration to SQLite, MySQL/MariaDB/Dolt and PostgreSQL compatibility catalogues, then backfills -each exportable legacy dataset into the singular standard tables. +each exportable legacy dataset into the singular standard tables and provisions +the compact official transformation-apply audit relation. A completely empty dedicated namespace is initialized automatically on its first ordinary import or export attempt. An existing catalogue is never diff --git a/docs/release-readiness.md b/docs/release-readiness.md index 6422791..517cff8 100644 --- a/docs/release-readiness.md +++ b/docs/release-readiness.md @@ -1,45 +1,45 @@ # PHP adapter release readiness -This page records the release contract for the next PHP adapter release. It -does not mean that a package tag or Packagist publication has happened. +This page records the release contract for PHP adapter v0.6.0. It does not mean +that a package tag or Packagist publication has happened. -## Specification pin +## Specification pin and claims -The adapter's machine-readable capability declaration and every CI conformance -checkout must use the stable OpenStatSpec specification release `v0.3.0` at -exact commit -`cd8f198c68b849eb8ed018a894670a0904c2181d`. The adapter reports -`specification_status=stable` and `specification_release=v0.3.0`. +The machine-readable capability declaration and every CI fixture checkout use +stable OpenStatSpec specification release `v0.3.0` at exact commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. -This pin covers the source-faithful SPSS SAV/ZSAV relational contract. The PHP -adapter does not claim official Transformation Plan or SPSS Frontend profile -conformance; those remain separate implementation gates. +Release v0.6.0 claims conformance with Transformation Plan 0.1/0.2, SPSS Syntax +Frontend 0.2, and In-Place Transformation 0.1/0.2. It removes the package-local +plan API and the non-standard SPSS schema-edit commands described in the +[transformation migration notes](transformations.md#v060-migration). ## Required gates -Before tagging a release: +Before tagging v0.6.0: 1. Run `composer check` with `OPENSTATSPEC_SPECIFICATION_DIR` pointing to the - exact `v0.3.0` checkout. -2. Run the focused service evidence command: + exact pinned checkout. +2. Run every official transformation suite: ```bash - OPENSTATSPEC_SPECIFICATION_DIR=/path/to/openstatspec-specification \ - vendor/bin/phpunit tests/Integration/InPlaceTransformationServiceTest.php + vendor/bin/phpunit tests/Transformation/Conformance + vendor/bin/phpunit tests/Frontend/Spss/Conformance + vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php + vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation02Test.php ``` - Locally, SQLite executes and an unconfigured PostgreSQL, MySQL, MariaDB, or - Dolt profile skips; CI must configure and execute every one of those - service profiles. -3. Confirm that the existing-target service evidence preserves the existing - dataset UUID, physical table, and variable identities without copied or - rollback state. Confirm that implicit numeric target creation is restricted - to SQLite and PostgreSQL, while MySQL, MariaDB, and Dolt reject it before - mutation and require a pre-created, catalogued target. -4. Confirm GitHub Actions passes on PHP 8.4 and 8.5 plus the PostgreSQL, - MySQL, MariaDB, and Dolt service matrices. This is adapter evidence under - the legacy `openstatspec-transformation-plan-v1` contract, not an official - OpenStatSpec Transformation Plan 1.0 profile claim. -5. Confirm the changelog and release notes describe the same specification - release and commit. -6. Create an annotated `vX.Y.Z` tag on the reviewed `main` commit and verify - that the tag points to that commit before publishing the GitHub release. -7. Confirm the resulting Composer package is installable from Packagist. + Confirm all 4 Plan 0.1, 26 Plan 0.2, 44 Frontend 0.2, 6 In-Place 0.1, + and 11 In-Place 0.2 manifest cases run for their configured profiles. +3. Confirm successful applies preserve dataset/table identity and counts and + create no copied, output, staging, snapshot, rollback, or version state. +4. Confirm SQLite/PostgreSQL atomic numeric-target creation and preflight + rejection on MySQL, MariaDB, and Dolt. Deployment must pre-provision and + catalog targets on those three profiles. +5. Confirm live Dolt evidence proves stable branch/HEAD, a clean pre-apply + working set, no adapter-created commit, and caller-owned commit policy. +6. Confirm GitHub Actions passes the full PHP 8.4/8.5 jobs plus every + PostgreSQL, MySQL, MariaDB, and Dolt matrix entry. Each service filter must + include both official in-place test classes. +7. Confirm the changelog and release notes describe the same specification + release, commit, breaking removals, and pre-provisioning rules. +8. Create annotated tag `v0.6.0` on the reviewed `main` commit, verify the tag, + publish the GitHub release, and confirm Packagist installation. diff --git a/docs/superpowers/plans/2026-08-17-transformation-plan-spss-frontend-0.2.md b/docs/superpowers/plans/2026-08-17-transformation-plan-spss-frontend-0.2.md new file mode 100644 index 0000000..b1d03fa --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-transformation-plan-spss-frontend-0.2.md @@ -0,0 +1,1071 @@ +# Transformation Plan and SPSS Frontend 0.2 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (- [ ]) syntax for tracking. + +**Goal:** Replace the package-local transformation API with official OpenStatSpec Transformation Plan 0.1/0.2, SPSS Frontend 0.2, and In-Place Transformation 0.1/0.2 conformance. + +**Architecture:** Add a strict official plan model and codec beside the legacy model, migrate the SPSS compiler and in-place executor to it, then delete the legacy model before completion. The frontend compiles an alias-based request to a versioned canonical plan; a separate apply request binds that alias to one dataset and carries source hash, actor, and Dolt expectations. + +**Tech Stack:** PHP 8.4.1+, PDO, PHPUnit 11, PHPStan 2, PHP-CS-Fixer 3, brick/math 0.19, OpenStatSpec specification v0.3.0 fixtures, SQLite/PostgreSQL/MySQL/MariaDB/Dolt. + +## Global Constraints + +- The breaking release target is v0.6.0; do not retain a compatibility adapter for openstatspec-transformation-plan-v1. +- Frontend 0.2 must still emit exact official Plan 0.1 for programs containing only the 0.1 command subset. +- Pass all 4 Plan 0.1, 26 Plan 0.2, 44 Frontend 0.2, 6 In-Place 0.1, and 11 In-Place 0.2 cases. +- Preserve dataset identity, physical table identity, case order, case count, dataset count, and persistent data-table count. +- Never create a derived dataset, persistent copy, snapshot, staging relation, rollback table, dataset-version row, or recovery layer. +- Dolt is the sole version/history/rollback layer; never commit, switch, merge, reset, or tag Dolt from the adapter. +- Validate the complete plan, target, actor, backend capabilities, and Dolt context before mutation. +- SQLite and PostgreSQL may create numeric targets atomically; MySQL, MariaDB, and Dolt must reject them with schema_change_not_atomic. +- Canonical hashes use exact restricted RFC 8785 bytes; decimal source numbers must round directly to binary64 ties-to-even without a host-float intermediate. +- Run every task through a red test, minimal implementation, focused green test, full relevant regression, and a focused commit. + +--- + +## File Structure + +New production boundaries: + +- src/Transformation/Plan: official immutable plan, operation, expression, value, codec, and validation types. +- src/Transformation/Diagnostic: stable diagnostic value and exception types shared by plan, frontend, and executor. +- src/Frontend/Spss/Request: official frontend request and ordered input-schema types. +- src/Frontend/Spss/Number: exact finite-decimal to binary64 conversion. +- src/Transformation/Audit: explicit schema migration and compact transformation audit writer. +- src/Transformation/Execution: apply request, target binding, preflight, SQL predicate compiler, and executor. + +New test boundaries: + +- tests/Support/SpecificationManifest.php: one strict loader for the pinned sibling/CI specification checkout. +- tests/Transformation/Conformance: official Plan 0.1 and 0.2 golden tests. +- tests/Frontend/Spss/Conformance: official Frontend 0.2 golden tests, including exact referenced Plan 0.1 output. +- tests/Integration/OfficialInPlaceTransformation01Test.php: the 6 official 0.1 binding cases. +- tests/Integration/OfficialInPlaceTransformation02Test.php: the 11 official 0.2 binding cases. + +--- + +### Task 1: Pin and load the official transformation manifests + +**Files:** +- Create: tests/Support/SpecificationManifest.php +- Create: tests/Transformation/Conformance/ManifestInventoryTest.php +- Modify: tests/Release/SpecificationPinTest.php + +**Interfaces:** +- Consumes: OPENSTATSPEC_SPECIFICATION_DIR or the workspace sibling specification directory. +- Produces: SpecificationManifest::load(string): array and SpecificationManifest::path(string): string. + +- [ ] **Step 1: Write the failing manifest inventory test** + +~~~php +public function testPinnedTransformationManifestInventory(): void +{ + self::assertCount(4, SpecificationManifest::load('conformance/transformation-plan-0.1.json')['cases']); + self::assertCount(26, SpecificationManifest::load('conformance/transformation-plan-0.2.json')['cases']); + self::assertCount(44, SpecificationManifest::load('conformance/spss-syntax-frontend-0.2.json')['cases']); + self::assertCount(6, SpecificationManifest::load('conformance/in-place-transformation-0.1.json')['cases']); + self::assertCount(11, SpecificationManifest::load('conformance/in-place-transformation-0.2.json')['cases']); +} +~~~ + +- [ ] **Step 2: Run the test and verify the missing loader failure** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/ManifestInventoryTest.php + +Expected: FAIL because OpenStatSpec\Tests\Support\SpecificationManifest does not exist. + +- [ ] **Step 3: Implement the strict loader** + +~~~php +final class SpecificationManifest +{ + public static function path(string $relative): string + { + $configured = getenv('OPENSTATSPEC_SPECIFICATION_DIR'); + $root = is_string($configured) && $configured !== '' + ? $configured + : dirname(__DIR__, 3) . '/specification'; + $path = $root . '/' . ltrim($relative, '/'); + if (!is_file($path)) { + throw new RuntimeException('Pinned specification fixture is missing: ' . $relative); + } + return $path; + } + + /** @return array */ + public static function load(string $relative): array + { + $decoded = json_decode( + (string) file_get_contents(self::path($relative)), + true, + flags: JSON_THROW_ON_ERROR, + ); + if (!is_array($decoded)) { + throw new RuntimeException('Specification manifest must decode to an object.'); + } + return $decoded; + } +} +~~~ + +- [ ] **Step 4: Extend the pin test to cover transformation schemas and manifests** + +Assert that CapabilityDeclaration::SPECIFICATION_RELEASE remains v0.3.0, the commit remains cd8f198c68b849eb8ed018a894670a0904c2181d, and all five CI specification checkouts use that commit. Then assert the three used schema files (Plan 0.1, Plan 0.2, and Frontend 0.2) and five manifests used by this plan exist below the resolved root. + +- [ ] **Step 5: Run focused and release tests** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/ManifestInventoryTest.php tests/Release/SpecificationPinTest.php + +Expected: PASS with exact inventory counts and no skipped tests. + +- [ ] **Step 6: Commit** + +~~~bash +git add tests/Support/SpecificationManifest.php tests/Transformation/Conformance/ManifestInventoryTest.php tests/Release/SpecificationPinTest.php +git commit -m "test: load official transformation conformance manifests" +~~~ + +--- + +### Task 2: Implement the official Plan 0.1 model and canonical codec + +**Files:** +- Create: src/Transformation/Diagnostic/TransformationDiagnostic.php +- Create: src/Transformation/Diagnostic/TransformationFailure.php +- Create: src/Transformation/Diagnostic/SourceSpan.php +- Create: src/Transformation/Plan/PlanContract.php +- Create: src/Transformation/Plan/TransformationPlan.php +- Create: src/Transformation/Plan/PlanCodec.php +- Create: src/Transformation/Plan/Operation.php +- Create: src/Transformation/Plan/TargetMode.php +- Create: src/Transformation/Plan/Value/TypedValue.php +- Create: src/Transformation/Plan/Value/Binary64Value.php +- Create: src/Transformation/Plan/Value/StringValue.php +- Create: src/Transformation/Plan/Recode/Match.php +- Create: src/Transformation/Plan/Recode/ExactMatch.php +- Create: src/Transformation/Plan/Recode/RangeMatch.php +- Create: src/Transformation/Plan/Recode/SystemMissingMatch.php +- Create: src/Transformation/Plan/Recode/Result.php +- Create: src/Transformation/Plan/Recode/LiteralResult.php +- Create: src/Transformation/Plan/Recode/CopyResult.php +- Create: src/Transformation/Plan/Recode/SystemMissingResult.php +- Create: src/Transformation/Plan/Recode/RecodeRule.php +- Create: src/Transformation/Plan/Operation/RecodeOperation.php +- Create: src/Transformation/Plan/Operation/SetVariableLabelOperation.php +- Create: src/Transformation/Plan/Operation/ReplaceValueLabelsOperation.php +- Create: src/Transformation/Plan/Operation/ValueLabel.php +- Create: tests/Transformation/Conformance/TransformationPlan01Test.php + +**Interfaces:** +- Consumes: CanonicalJson::encode(array): string and SpecificationManifest::load(string): array. +- Produces: PlanCodec::fromArray(array): TransformationPlan, PlanCodec::fromJson(string): TransformationPlan, PlanCodec::canonicalJson(TransformationPlan): string, and PlanCodec::hash(TransformationPlan): string. + +- [ ] **Step 1: Write the failing Plan 0.1 golden test** + +~~~php +#[DataProvider('cases')] +public function testOfficialPlan01(array $case): void +{ + if (is_string($case['expected_error'])) { + try { + (new PlanCodec())->fromArray($case['plan']); + self::fail('Expected ' . $case['expected_error']); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + return; + } + + $plan = (new PlanCodec())->fromArray($case['plan']); + self::assertSame($case['plan'], $plan->canonicalArray()); + self::assertSame($case['expected_plan_hash'], (new PlanCodec())->hash($plan)); +} +~~~ + +- [ ] **Step 2: Run the test and verify it is red** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/TransformationPlan01Test.php + +Expected: FAIL because PlanCodec and the official plan model do not exist. + +- [ ] **Step 3: Add stable diagnostics and the contract root** + +~~~php +enum PlanContract: string +{ + case V01 = 'openstatspec-transformation-plan-v0.1'; + case V02 = 'openstatspec-transformation-plan-v0.2'; +} + +interface Operation +{ + /** @return array */ + public function canonicalArray(): array; + public function minimumContract(): PlanContract; +} + +final readonly class TransformationPlan +{ + /** @param non-empty-list $operations */ + public function __construct( + public PlanContract $contract, + public string $inputAlias, + public array $operations, + ) {} + + /** @return array{contract:string,input_alias:string,operations:list>} */ + public function canonicalArray(): array + { + return [ + 'contract' => $this->contract->value, + 'input_alias' => $this->inputAlias, + 'operations' => array_map( + static fn(Operation $operation): array => $operation->canonicalArray(), + $this->operations, + ), + ]; + } +} +~~~ + +The non-final TransformationFailure stores a non-empty list of TransformationDiagnostic and exposes diagnosticCode() as the first stable code. TransformationFailure::at(string $code, string $path, string $message, ?SourceSpan $span = null) creates a one-diagnostic failure. Diagnostic values carry code, JSON path, message, and the optional shared SourceSpan without credentials or row values; SpssSyntaxException extends this shared failure type in Task 5. + +- [ ] **Step 4: Implement exact typed values and 0.1 operation shapes** + +Use immutable classes whose canonicalArray methods emit exactly these keys: + +~~~php +new RecodeOperation( + source: 'q1', + target: 'q1', + targetMode: TargetMode::Replace, + rules: [new RecodeRule(new ExactMatch(Binary64Value::fromBits('3ff0000000000000')), new LiteralResult(Binary64Value::fromBits('0000000000000000')))], + unmatched: new CopyResult(), +); + +// canonicalArray() +[ + 'op' => 'recode', + 'source' => 'q1', + 'target' => 'q1', + 'target_mode' => 'replace', + 'rules' => [['match' => ['kind' => 'exact', 'value' => ['type' => 'binary64', 'bits' => '3ff0000000000000']], 'result' => ['kind' => 'literal', 'value' => ['type' => 'binary64', 'bits' => '0000000000000000']]]], + 'unmatched' => ['kind' => 'copy'], +]; +~~~ + +SetVariableLabelOperation emits op, variable, label. ReplaceValueLabelsOperation emits op, variable, and ordered labels; each label emits value then label. Binary64Value accepts exactly 16 lowercase hexadecimal bits, rejects exponent 0x7ff and negative zero with plan_schema_invalid, and StringValue preserves exact Unicode. + +- [ ] **Step 5: Implement strict recursive PlanCodec decoding** + +PlanCodec must reject missing/extra members, empty aliases/operations, unknown operations, malformed typed values, descending ranges, and duplicate exact typed label values. Map the two semantic 0.1 failures to invalid_numeric_range and duplicate_value_label; all structural failures map to plan_schema_invalid. + +~~~php +public function canonicalJson(TransformationPlan $plan): string +{ + return CanonicalJson::encode($plan->canonicalArray()); +} + +public function hash(TransformationPlan $plan): string +{ + return hash('sha256', $this->canonicalJson($plan)); +} +~~~ + +- [ ] **Step 6: Run Plan 0.1 goldens and canonical regressions** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/TransformationPlan01Test.php tests/Transformation/Canonical/TransformationPlanTest.php + +Expected: all four official 0.1 cases pass; legacy tests remain green because their namespace has not been removed yet. + +- [ ] **Step 7: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Diagnostic src/Transformation/Plan tests/Transformation/Conformance/TransformationPlan01Test.php +git commit -m "feat: add official transformation plan 0.1 model" +~~~ + +--- + +### Task 3: Extend the model and codec to Plan 0.2 + +**Files:** +- Create: src/Transformation/Plan/Expression/Operand.php +- Create: src/Transformation/Plan/Expression/LiteralOperand.php +- Create: src/Transformation/Plan/Expression/VariableOperand.php +- Create: src/Transformation/Plan/Expression/Predicate.php +- Create: src/Transformation/Plan/Expression/Comparison.php +- Create: src/Transformation/Plan/Expression/BooleanPredicate.php +- Create: src/Transformation/Plan/Operation/AssignOperation.php +- Create: src/Transformation/Plan/Operation/ConditionalAssignOperation.php +- Create: src/Transformation/Plan/Operation/SetFormatOperation.php +- Create: src/Transformation/Plan/Operation/SetMeasurementLevelOperation.php +- Create: src/Transformation/Plan/Operation/ExecuteOperation.php +- Modify: src/Transformation/Plan/PlanCodec.php +- Create: tests/Transformation/Conformance/TransformationPlan02Test.php +- Create: tests/Transformation/Plan/BooleanPredicateTest.php + +**Interfaces:** +- Consumes: official Plan 0.1 model and codec. +- Produces: typed 0.2 operations and predicates accepted by the same PlanCodec API. + +- [ ] **Step 1: Write the failing 26-case Plan 0.2 golden test** + +Reuse the Task 2 data-provider structure with conformance/transformation-plan-0.2.json. Add a focused assertion that a direct BooleanPredicate child with the same operator throws noncanonical_boolean_shape. + +- [ ] **Step 2: Run the tests and verify unsupported 0.2 operations fail** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/TransformationPlan02Test.php tests/Transformation/Plan/BooleanPredicateTest.php + +Expected: FAIL on the first assign or conditional_assign case. + +- [ ] **Step 3: Implement operands and predicates** + +~~~php +final readonly class Comparison implements Predicate +{ + public function __construct( + public Operand $left, + public string $operator, + public Operand $right, + ) { + if (!in_array($operator, ['=', '<', '<=', '>', '>='], true)) { + throw TransformationFailure::at('plan_schema_invalid', '$.operator', 'Unsupported comparison operator.'); + } + } + + public function canonicalArray(): array + { + return [ + 'expression' => 'comparison', + 'left' => $this->left->canonicalArray(), + 'operator' => $this->operator, + 'right' => $this->right->canonicalArray(), + ]; + } +} +~~~ + +BooleanPredicate accepts and/or plus at least two ordered operands. Reject a direct BooleanPredicate child with the same operator as noncanonical_boolean_shape; never reorder, deduplicate, simplify, distribute, or constant-fold. + +- [ ] **Step 4: Implement the five 0.2 operations** + +AssignOperation emits op, target, target_mode, value. ConditionalAssignOperation emits op, condition, target, value. SetFormatOperation emits op, variable, family F, width, decimals. SetMeasurementLevelOperation emits op, variable, level. ExecuteOperation emits only op. + +Validation rules: + +~~~php +if ($operation instanceof SetFormatOperation + && ($operation->width < 1 || $operation->width > 40 + || $operation->decimals < 0 || $operation->decimals > 16 + || ($operation->decimals > 0 && $operation->width < $operation->decimals + 2))) { + throw TransformationFailure::at('invalid_format', $path, 'Invalid SPSS F format.'); +} +~~~ + +Reject string literal operands in assign, conditional assignment, or comparison with expression_type_unsupported; reject names beginning with double underscore using reserved_target_name where the name is a target. + +- [ ] **Step 5: Enforce contract selection** + +PlanContract::V01 accepts only operations whose minimumContract is V01. PlanContract::V02 accepts both 0.1 and 0.2 operations. Decoding never promotes or rewrites a declared 0.1 plan. + +- [ ] **Step 6: Run all official plan tests** + +Run: vendor/bin/phpunit tests/Transformation/Conformance tests/Transformation/Plan + +Expected: 4 Plan 0.1 and 26 Plan 0.2 cases pass with exact hashes and stable expected errors. + +- [ ] **Step 7: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Plan tests/Transformation/Conformance/TransformationPlan02Test.php tests/Transformation/Plan +git commit -m "feat: implement transformation plan 0.2 operations" +~~~ + +--- + +### Task 4: Convert SPSS finite decimals to binary64 exactly + +**Files:** +- Modify: composer.json +- Modify: composer.lock +- Create: src/Frontend/Spss/Number/DecimalBinary64.php +- Create: tests/Frontend/Spss/DecimalBinary64Test.php + +**Interfaces:** +- Consumes: brick/math ^0.19 BigInteger arithmetic. +- Produces: DecimalBinary64::bits(string): string returning exactly 16 lowercase hexadecimal digits. + +- [ ] **Step 1: Add the dependency and failing boundary tests** + +Run: composer require brick/math:^0.19 --no-interaction + +~~~php +#[DataProvider('tokens')] +public function testExactBits(string $token, string $bits): void +{ + self::assertSame($bits, DecimalBinary64::bits($token)); +} + +public static function tokens(): iterable +{ + yield 'negative zero' => ['-0', '0000000000000000']; + yield 'underflowed zero' => ['-1e-9999', '0000000000000000']; + yield 'minimum subnormal' => ['4.9406564584124654417656879286822137236505980e-324', '0000000000000001']; + yield 'one' => ['1', '3ff0000000000000']; + yield 'tie rounds even' => ['1.00000000000000011102230246251565404236316680908203125', '3ff0000000000000']; +} +~~~ + +Add rejection tests for +1, .5, 1., 01, NaN, Infinity, and overflow 1e9999. + +- [ ] **Step 2: Run the test and verify it is red** + +Run: vendor/bin/phpunit tests/Frontend/Spss/DecimalBinary64Test.php + +Expected: FAIL because DecimalBinary64 does not exist. + +- [ ] **Step 3: Implement grammar and exact rational construction** + +Parse the token with this exact pattern: + +~~~php +private const FINITE = '/\A(-?)(0|[1-9][0-9]*)(?:\.([0-9]+))?(?:[eE]([+-]?[0-9]+))?\z/D'; +~~~ + +Build coefficient and decimal exponent with BigInteger only. Represent the absolute value as numerator/denominator where positive decimal exponents multiply the numerator by 10^n and negative exponents multiply the denominator by 10^n. + +- [ ] **Step 4: Implement ties-to-even rounding** + +~~~php +private static function roundRatio(BigInteger $numerator, BigInteger $denominator): BigInteger +{ + [$quotient, $remainder] = $numerator->quotientAndRemainder($denominator); + $twice = $remainder->multipliedBy(2); + $comparison = $twice->compareTo($denominator); + if ($comparison > 0 || ($comparison === 0 && $quotient->isOdd())) { + return $quotient->plus(1); + } + return $quotient; +} +~~~ + +Find floor(log2(numerator/denominator)) by integer comparison. For normal values round numerator multiplied by 2^(52-e); for subnormals round numerator multiplied by 2^1074. Handle significand carry, overflow, sign bit, minimum normal, and zero before rendering a 64-bit hexadecimal word. + +- [ ] **Step 5: Run focused and frontend-number tests** + +Run: vendor/bin/phpunit tests/Frontend/Spss/DecimalBinary64Test.php + +Expected: all accepted tokens produce exact bits and every rejected token throws spss_syntax_error. + +- [ ] **Step 6: Run quality checks and commit** + +Run: composer validate --strict && composer analyse && composer style + +~~~bash +git add composer.json composer.lock src/Frontend/Spss/Number/DecimalBinary64.php tests/Frontend/Spss/DecimalBinary64Test.php +git commit -m "feat: add exact SPSS decimal binary64 conversion" +~~~ + +--- + +### Task 5: Parse the official SPSS Frontend 0.2 grammar + +**Files:** +- Create: src/Frontend/Spss/Request/SpssFrontendRequest.php +- Create: src/Frontend/Spss/Request/InputSchema.php +- Create: src/Frontend/Spss/Request/InputVariable.php +- Create: src/Frontend/Spss/Ast/ComputeStatement.php +- Create: src/Frontend/Spss/Ast/IfStatement.php +- Create: src/Frontend/Spss/Ast/FormatsStatement.php +- Create: src/Frontend/Spss/Ast/FormatTarget.php +- Create: src/Frontend/Spss/Ast/VariableLevelStatement.php +- Create: src/Frontend/Spss/Ast/VariableLevelGroup.php +- Create: src/Frontend/Spss/Ast/ExpressionOperand.php +- Create: src/Frontend/Spss/Ast/LiteralOperand.php +- Create: src/Frontend/Spss/Ast/VariableOperand.php +- Create: src/Frontend/Spss/Ast/Predicate.php +- Create: src/Frontend/Spss/Ast/Comparison.php +- Create: src/Frontend/Spss/Ast/BooleanPredicate.php +- Modify: src/Frontend/Spss/TokenType.php +- Modify: src/Frontend/Spss/Lexer.php +- Modify: src/Frontend/Spss/Parser.php +- Modify: src/Frontend/Spss/SpssSyntaxException.php +- Modify: tests/Frontend/Spss/LexerTest.php +- Modify: tests/Frontend/Spss/ParserTest.php + +**Interfaces:** +- Consumes: exact numeric-token grammar and DecimalBinary64 rejection behavior. +- Produces: Parser::parse(string): Program containing ordered 0.1/0.2 statement AST nodes with source spans. + +- [ ] **Step 1: Write failing parser tests for every new command** + +~~~php +$program = (new Parser())->parse( + "COMPUTE target = 0.\n" + . "IF (a = 1 AND b >= 2 OR c < 3) target = a.\n" + . "FORMATS target (F8.2).\n" + . "VARIABLE LEVEL target (SCALE).\n" + . "EXECUTE." +); + +self::assertInstanceOf(ComputeStatement::class, $program->statements[0]); +self::assertInstanceOf(IfStatement::class, $program->statements[1]); +self::assertInstanceOf(FormatsStatement::class, $program->statements[2]); +self::assertInstanceOf(VariableLevelStatement::class, $program->statements[3]); +self::assertInstanceOf(ExecuteStatement::class, $program->statements[4]); +~~~ + +Add rejection tests for IF without outer parentheses, IF ELSE, arithmetic, NOT, string expressions, non-F formats, invalid levels, comments, and unsupported commands. Assert each exception exposes the stable code and start/end source span. + +- [ ] **Step 2: Run parser tests and verify they are red** + +Run: vendor/bin/phpunit tests/Frontend/Spss/LexerTest.php tests/Frontend/Spss/ParserTest.php + +Expected: FAIL on COMPUTE tokenization. + +- [ ] **Step 3: Extend tokens and lexer** + +Add ASCII-case-insensitive keyword tokens for COMPUTE, IF, AND, OR, FORMATS, VARIABLE LEVEL, NOMINAL, ORDINAL, and SCALE; add equals and ordered comparison tokens. Preserve the exact source offset plus 1-based line/column at both token boundaries. + +- [ ] **Step 4: Implement precedence and flattening in the parser** + +~~~php +private function parsePredicate(): Predicate +{ + return $this->parseOr(); +} + +private function merge(string $operator, Predicate $left, Predicate $right): BooleanPredicate +{ + $operands = $left instanceof BooleanPredicate && $left->operator === $operator + ? $left->operands + : [$left]; + array_push($operands, ...($right instanceof BooleanPredicate && $right->operator === $operator + ? $right->operands + : [$right])); + return new BooleanPredicate($operator, $operands, SourceSpan::cover($left->span, $right->span)); +} +~~~ + +parseOr calls parseAnd; parseAnd calls comparison; comparison parses two operands. Parentheses recurse but same-operator nodes are flattened in source order. IF requires one outer parenthesized predicate. + +- [ ] **Step 5: Implement grouped metadata commands and stable failures** + +FORMATS emits ordered FormatTarget pairs with uppercase F and integer width/decimals. VARIABLE LEVEL emits ordered groups and lowercase levels. Invalid productions throw spss_syntax_error; recognized but unsupported commands throw unsupported_spss_command. + +- [ ] **Step 6: Run lexer/parser regression tests** + +Run: vendor/bin/phpunit tests/Frontend/Spss/LexerTest.php tests/Frontend/Spss/ParserTest.php + +Expected: all old official-subset parsing and new 0.2 parsing tests pass. + +- [ ] **Step 7: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Frontend/Spss tests/Frontend/Spss/LexerTest.php tests/Frontend/Spss/ParserTest.php +git commit -m "feat: parse SPSS frontend 0.2 commands" +~~~ + +--- + +### Task 6: Bind and compile all official Frontend 0.2 cases + +**Files:** +- Create: src/Frontend/Spss/SpssCompilationResult.php +- Create: src/Frontend/Spss/Binding/SchemaState.php +- Create: src/Frontend/Spss/Binding/BoundCompute.php +- Create: src/Frontend/Spss/Binding/BoundConditionalAssign.php +- Create: src/Frontend/Spss/Binding/BoundFormat.php +- Create: src/Frontend/Spss/Binding/BoundMeasurementLevel.php +- Create: src/Frontend/Spss/Binding/BoundExecute.php +- Modify: src/Frontend/Spss/Binder.php +- Modify: src/Frontend/Spss/Compiler.php +- Modify: src/Frontend/Spss/SpssCompiler.php +- Create: tests/Frontend/Spss/Conformance/SpssFrontend02Test.php +- Modify: tests/Frontend/Spss/SpssCompilerTest.php + +**Interfaces:** +- Consumes: SpssFrontendRequest, Parser AST, DecimalBinary64, and official plan types. +- Produces: SpssFrontendRequest::sourceHash(): string and SpssCompiler::compile(SpssFrontendRequest): SpssCompilationResult with public readonly plan and sourceHash. + +- [ ] **Step 1: Write failing manifest-driven frontend tests** + +~~~php +$request = SpssFrontendRequest::fromArray($case['request']); +self::assertSame($case['expected_source_hash'], $request->sourceHash()); +try { + $result = (new SpssCompiler())->compile($request); +} catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + return; +} + +self::assertNull($case['expected_error']); +self::assertSame($case['expected_source_hash'], $result->sourceHash); +self::assertSame($case['expected_plan_hash'], (new PlanCodec())->hash($result->plan)); +if (isset($case['expected_plan_contract'])) { + self::assertSame($case['expected_plan_contract'], $result->plan->contract->value); +} +~~~ + +Load the official 0.2 frontend manifest; references to exact 0.1 plans are checked against conformance/transformation-plan-0.1.json rather than copied expectations. + +- [ ] **Step 2: Run the frontend conformance tests and verify red** + +Run: vendor/bin/phpunit tests/Frontend/Spss/Conformance + +Expected: FAIL because SpssFrontendRequest and the new compiler signature do not exist. + +- [ ] **Step 3: Validate the request and source hash** + +SpssFrontendRequest::fromArray accepts exactly contract, input_alias, input_schema, and source_text; contract must equal openstatspec-spss-syntax-frontend-v0.2. InputSchema requires a non-empty ordered variables array; InputVariable validates name, storage_kind, and optional label/value-label/format/measurement fields. + +~~~php +$normalized = str_replace(["\r\n", "\r"], "\n", $request->sourceText); +$sourceHash = hash('sha256', $normalized); +~~~ + +sourceHash() performs this normalization independently of parsing so failed compilations still have the exact conformance source identity. + +- [ ] **Step 4: Implement case-insensitive evolving schema binding** + +SchemaState indexes ASCII-lowercase names while retaining exact catalog spelling. RECODE pairs resolve against pre-command state; targets created by an earlier command become available to later commands. COMPUTE creates or replaces a numeric target. IF requires an existing numeric target. FORMATS requires numeric targets; VARIABLE LEVEL accepts numeric or string targets. + +Map binding failures exactly to unknown_variable, conditional_target_missing, type_mismatch, expression_type_unsupported, invalid_format, reserved_target_name, duplicate_value_label, mixed_result_types, system_missing_for_string, and string_target_requires_declaration. + +- [ ] **Step 5: Compile ordered operations and select the plan contract** + +~~~php +$contract = array_any( + $operations, + static fn(Operation $operation): bool => $operation->minimumContract() === PlanContract::V02, +) ? PlanContract::V02 : PlanContract::V01; + +$plan = new TransformationPlan($contract, $request->inputAlias, $operations); +return new SpssCompilationResult($plan, $sourceHash); +~~~ + +Compile DecimalBinary64 bits directly into Binary64Value. Preserve command, group, variable, rule, and boolean operand order. EXECUTE always remains an ExecuteOperation. + +- [ ] **Step 6: Run all 44 official Frontend 0.2 cases** + +Run: vendor/bin/phpunit tests/Frontend/Spss/Conformance tests/Frontend/Spss/SpssCompilerTest.php + +Expected: all 44 Frontend 0.2 cases pass with exact plan/source hashes and stable errors, including the required exact Plan 0.1 outputs. + +- [ ] **Step 7: Run plan plus frontend regressions and commit** + +Run: vendor/bin/phpunit tests/Transformation/Conformance tests/Frontend/Spss +Run: composer analyse && composer style + +~~~bash +git add src/Frontend/Spss tests/Frontend/Spss +git commit -m "feat: compile official SPSS frontend 0.2 plans" +~~~ + +--- + +### Task 7: Add the explicit transformation audit migration and apply request + +**Files:** +- Create: src/Transformation/Execution/InPlaceApplyRequest.php +- Create: src/Transformation/Audit/TransformationAuditMigrator.php +- Create: src/Transformation/Audit/TransformationAuditWriter.php +- Modify: src/Sql/CatalogOwnership.php +- Modify: src/Sql/NormativeCatalog.php +- Modify: src/Spss/SpssAdapter.php +- Create: tests/Transformation/Audit/TransformationAuditMigratorTest.php +- Modify: tests/Integration/ServerCatalogMigrationTest.php + +**Interfaces:** +- Consumes: TransformationPlan and PlanCodec. +- Produces: InPlaceApplyRequest(plan, inputAlias, datasetId, sourceHash, actor, expectedBranch, expectedHead) and TransformationAuditWriter::succeed(...): string. + +- [ ] **Step 1: Write failing SQLite migration and request-validation tests** + +~~~php +$request = new InPlaceApplyRequest( + plan: $plan, + inputAlias: 'parent', + datasetId: '11111111-1111-4111-8111-111111111111', + sourceHash: str_repeat('a', 64), + actor: 'conformance-runner', +); +self::assertSame('parent', $request->inputAlias); + +(new TransformationAuditMigrator($pdo))->migrate(); +self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $this->acceptedContracts($pdo), +); +~~~ + +Also create an old 0.1-only transformation_apply table with one row, migrate it, and assert the row and logical table name survive with no backup table residue. + +- [ ] **Step 2: Run tests and verify red** + +Run: vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php + +Expected: FAIL because the request and migrator do not exist. + +- [ ] **Step 3: Implement fail-fast apply request validation** + +The constructor rejects alias mismatch, malformed dataset UUID, malformed source hash, and empty actor before any database call. expectedBranch and expectedHead must be both null or both non-empty. + +~~~php +if ($inputAlias !== $plan->inputAlias) { + throw TransformationFailure::at('unknown_input_alias', '$.input_alias', 'Apply binding does not match the plan alias.'); +} +if (trim($actor) === '') { + throw TransformationFailure::at('actor_required', '$.actor', 'A non-empty actor is required.'); +} +~~~ + +- [ ] **Step 4: Implement schema version 4 migration** + +Bump CatalogOwnership::SCHEMA_VERSION from 3 to 4. TransformationAuditMigrator creates the exact normative columns and checks from sql/transformation-plan-profile-schema.sql using profile-specific UUID/text types. + +For SQLite 0.1-only tables: begin a native transaction, create transformation_apply_v04, copy compact audit rows, drop the old table, rename the new table to transformation_apply, verify foreign keys, and commit. On failure roll back so the old table remains. PostgreSQL alters the contract check transactionally. MySQL/MariaDB/Dolt replace the check in the explicit migration workflow before applies; never perform this DDL inside an apply. + +- [ ] **Step 5: Implement the success-only audit writer** + +~~~php +public function succeed( + InPlaceApplyRequest $request, + DatasetBinding $dataset, + string $databaseProfile, + ?DoltEvidence $before, + ?DoltEvidence $after, +): string +~~~ + +Insert exactly one succeeded row inside the open apply transaction after operations and post-Dolt checks but before commit. Store canonical plan JSON, plan/source hashes, actor, operation count, target identity, timestamps, and nullable Dolt fields. Do not write a failed row after rollback because the 0.2 binding requires failed applies to leave audit unchanged. + +- [ ] **Step 6: Wire explicit catalog migration** + +SpssAdapter::migrateCatalog calls TransformationAuditMigrator after NormativeCatalog::createTables and before CatalogOwnership::markCurrentVersion. Fresh catalogs receive migration markers 1 through 4 exactly once. + +- [ ] **Step 7: Run SQLite and configured server migration tests** + +Run: vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php tests/Integration/ServerCatalogMigrationTest.php + +Expected: SQLite always passes; configured servers pass; unconfigured services are the only skips. + +- [ ] **Step 8: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Audit src/Transformation/Execution/InPlaceApplyRequest.php src/Sql/CatalogOwnership.php src/Sql/NormativeCatalog.php src/Spss/SpssAdapter.php tests/Transformation/Audit tests/Integration/ServerCatalogMigrationTest.php +git commit -m "feat: add official transformation apply audit" +~~~ + +--- + +### Task 8: Execute official plans in place on SQLite and PostgreSQL + +**Files:** +- Create: src/Transformation/Execution/PlanPreflight.php +- Create: src/Transformation/Execution/BoundPlan.php +- Create: src/Transformation/Execution/BoundOperation.php +- Create: src/Transformation/Execution/SqlPredicateCompiler.php +- Create: src/Transformation/Execution/InPlaceOperationExecutor.php +- Modify: src/Transformation/Execution/InPlaceTransformationExecutor.php +- Modify: src/Transformation/Execution/ExecutionResult.php +- Modify: src/Transformation/Execution/DatasetBinding.php +- Modify: src/Transformation/Execution/VariableBinding.php +- Modify: tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +- Modify: tests/Integration/InPlaceTransformationServiceTest.php +- Create: tests/Integration/OfficialInPlaceTransformation01Test.php +- Create: tests/Integration/OfficialInPlaceTransformation02Test.php + +**Interfaces:** +- Consumes: InPlaceApplyRequest, official plan operations, Connection profile, and TransformationAuditWriter. +- Produces: InPlaceTransformationExecutor::execute(InPlaceApplyRequest): ExecutionResult. + +- [ ] **Step 1: Add failing SQLite official binding cases** + +In OfficialInPlaceTransformation02Test, materialize fixtures for sqlite-create-target-atomic-success and sqlite-inequality-boundary-semantics from the manifest. Snapshot dataset count, persistent data-table count, dataset/table identity, case order/count, variables, metadata, and audit count before apply. + +~~~php +$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($request); +self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); +self::assertSame($before['dataset_count'], $this->datasetCount($pdo)); +self::assertSame($before['persistent_data_table_count'], $this->persistentDataTableCount($pdo)); +self::assertSame($case['after']['rows'], $this->rows($pdo, $table)); +~~~ + +Add the rollback probe that injects failure after catalog target creation but before audit and asserts no physical column, catalog variable, row change, or audit row remains. + +- [ ] **Step 2: Run focused tests and verify red** + +Run: vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation02Test.php --filter sqlite + +Expected: FAIL because the executor still accepts the legacy plan signature. + +- [ ] **Step 3: Implement complete preflight** + +PlanPreflight resolves the alias-bound dataset and ordered catalog variables, then simulates every operation against an evolving schema. It validates unknown variables, create/replace target state, numeric expression types, recode result type, format validity, backend create capability, and audit schema readiness before transaction start. + +Return an immutable BoundPlan containing the unchanged plan plus per-operation source/target physical bindings. Do not store row values or create database objects during preflight. + +- [ ] **Step 4: Implement deterministic SQL predicate compilation** + +~~~php +private function comparison(Comparison $comparison, BoundSchema $schema, array &$parameters): string +{ + $left = $this->operand($comparison->left, $schema, $parameters); + $right = $this->operand($comparison->right, $schema, $parameters); + return '(' . $left . ' ' . $comparison->operator . ' ' . $right . ')'; +} + +private function boolean(BooleanPredicate $predicate, BoundSchema $schema, array &$parameters): string +{ + return '(' . implode( + ' ' . strtoupper($predicate->operator) . ' ', + array_map(fn(Predicate $child): string => $this->predicate($child, $schema, $parameters), $predicate->operands), + ) . ')'; +} +~~~ + +Literal operands append a PDO parameter; variable operands use the profile-quoted physical column. SQL NULL comparisons naturally become UNKNOWN; conditional UPDATE changes only rows whose WHERE predicate is TRUE. + +- [ ] **Step 5: Implement ordered operation execution** + +InPlaceOperationExecutor receives one BoundOperation at a time so the public InPlaceTransformationExecutor remains a small transaction orchestrator. Use direct UPDATE for existing recode/assign, ALTER TABLE plus catalog insert plus UPDATE for atomic create, conditional UPDATE WHERE predicate, exact catalog updates for labels/format/measurement, and no SQL for execute. Each operation observes prior operations in the same transaction. + +SetFormatOperation updates print and write family/width/decimals only. SetMeasurementLevelOperation updates measurement_level only. New variables receive a unique UUID, next ordinal, nullable numeric column, and no label/value-label/missing/display/measurement metadata. + +- [ ] **Step 6: Put audit in the same transaction** + +Begin one native transaction after preflight, apply every operation, run post-context checks, insert the success audit, then commit. Any Throwable rolls back; do not start OperationJournal for transformations. + +- [ ] **Step 7: Add PostgreSQL atomic-create evidence** + +Extend OfficialInPlaceTransformation02Test to run the SQLite create-target case against OPENSTATSPEC_PG_DSN with PostgreSQL identifiers and catalog fixture setup. Assert schema, rows, catalog, audit, and rollback probe share the transaction and leave no extra persistent relation. + +- [ ] **Step 8: Run executor regressions** + +Run: vendor/bin/phpunit tests/Transformation/Execution/InPlaceTransformationExecutorTest.php tests/Integration/InPlaceTransformationServiceTest.php tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php + +Expected: SQLite cases pass, PostgreSQL passes when configured, and only unconfigured services skip. + +- [ ] **Step 9: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Execution tests/Transformation/Execution tests/Integration/InPlaceTransformationServiceTest.php tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php +git commit -m "feat: execute official plans atomically in place" +~~~ + +--- + +### Task 9: Enforce MySQL-family and controlled Dolt bindings + +**Files:** +- Modify: src/Transformation/Execution/DoltGuard.php +- Modify: src/Transformation/Execution/DoltEvidence.php +- Modify: src/Transformation/Execution/PdoDoltEvidenceReader.php +- Modify: src/Transformation/Execution/PlanPreflight.php +- Modify: src/Transformation/Execution/InPlaceTransformationExecutor.php +- Modify: tests/Transformation/Execution/DoltGuardTest.php +- Modify: tests/Transformation/Execution/DoltHeadGuardTest.php +- Modify: tests/Integration/OfficialInPlaceTransformation01Test.php +- Modify: tests/Integration/OfficialInPlaceTransformation02Test.php +- Modify: tests/Integration/InPlaceTransformationServiceTest.php + +**Interfaces:** +- Consumes: InPlaceApplyRequest expected branch/HEAD and Connection::profileName. +- Produces: DoltGuard::beforeExecution(InPlaceApplyRequest): DoltEvidence and afterExecution(InPlaceApplyRequest, DoltEvidence): DoltEvidence. + +- [ ] **Step 1: Add failing backend rejection tests** + +For MySQL, MariaDB, and Dolt create-target manifest cases, snapshot data, catalog, audit, dataset count, and persistent table count. Execute the create plan and assert schema_change_not_atomic plus mutation_started=false and exact before/after equality. Include the official 0.1 MySQL create-target rejection as well as the 0.2 cases. + +- [ ] **Step 2: Add failing Dolt context tests** + +Cover empty actor, missing expected context, branch mismatch, HEAD mismatch, dirty working set, and injected post-mutation context change. Assert exact codes: actor_required, dolt_context_required, dolt_branch_mismatch, dolt_head_mismatch, dolt_working_set_dirty, and dolt_context_changed. + +- [ ] **Step 3: Run focused tests and verify red** + +Run: vendor/bin/phpunit tests/Transformation/Execution/DoltGuardTest.php tests/Transformation/Execution/DoltHeadGuardTest.php tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php + +Expected: FAIL because current DoltGuard compares only its own before/after observations. + +- [ ] **Step 4: Enforce profile capabilities during preflight** + +~~~php +if ($operation->createsTarget() && !$connection->profile->ddlAtomic()) { + throw TransformationFailure::at( + 'schema_change_not_atomic', + '$.operations[' . $index . '].target_mode', + $connection->profileName . ' requires a pre-provisioned target.', + ); +} +~~~ + +This runs before beginning a transaction or writing audit. MySQL, MariaDB, and Dolt never attempt compensating transformation cleanup. + +- [ ] **Step 5: Enforce caller-supplied Dolt context** + +Before mutation, require expectedBranch and expectedHead, compare them to active_branch() and dolt_hashof('HEAD'), and require dolt_status to be empty. After operations but before audit/commit, re-read branch and HEAD and compare them to both the request and before evidence. + +Never call a Dolt mutation procedure. A successful apply leaves one inspectable dirty working set and equal before/after HEAD in the audit. + +- [ ] **Step 6: Run all official 0.1 and 0.2 in-place cases** + +Run: vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php + +Expected: all 6 official 0.1 and 11 official 0.2 cases for configured services pass; only absent service configurations skip. + +- [ ] **Step 7: Run the existing service evidence test** + +Run: vendor/bin/phpunit tests/Integration/InPlaceTransformationServiceTest.php + +Expected: every existing-target service case preserves the global no-new-dataset/table invariant; live Dolt evidence proves no commit and stable branch/HEAD. + +- [ ] **Step 8: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Execution tests/Transformation/Execution tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php tests/Integration/InPlaceTransformationServiceTest.php +git commit -m "feat: enforce official SQL and Dolt apply context" +~~~ + +--- + +### Task 10: Remove the legacy API and publish conformance claims + +**Files:** +- Delete: src/Transformation/Model +- Delete: src/Transformation/Validation +- Delete: src/Frontend/Spss/Binding/BoundCreateVariable.php +- Delete: src/Frontend/Spss/Binding/BoundDeleteVariable.php +- Delete: src/Frontend/Spss/Ast/StringStatement.php +- Delete: src/Frontend/Spss/Ast/DeleteVariablesStatement.php +- Delete: tests/Transformation/Canonical/TransformationPlanTest.php +- Modify: tests/Frontend/Spss/SpssCompilerTest.php +- Modify: tests/Core/CapabilityDeclarationTest.php +- Modify: src/Core/CapabilityDeclaration.php +- Modify: README.md +- Modify: docs/architecture.md +- Modify: docs/transformations.md +- Modify: docs/release-readiness.md +- Modify: CHANGELOG.md +- Modify: .github/workflows/ci.yml + +**Interfaces:** +- Consumes: completed official model, frontend, audit, and executor. +- Produces: only the official public API and truthful machine-readable conformance claims. + +- [ ] **Step 1: Write failing capability and legacy-absence tests** + +~~~php +$contracts = $declaration['transformation_contracts']; +self::assertSame('official_conformant', $contracts['status']); +self::assertSame( + ['openstatspec-transformation-plan-v0.1', 'openstatspec-transformation-plan-v0.2'], + $contracts['official_plan_contracts'], +); +self::assertSame( + ['openstatspec-spss-syntax-frontend-v0.2'], + $contracts['official_frontend_contracts'], +); +self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $contracts['official_binding_contracts'], +); +self::assertArrayNotHasKey('legacy_plan_contracts', $contracts); +self::assertFalse(class_exists(OpenStatSpec\Transformation\Model\TransformationPlan::class)); +~~~ + +- [ ] **Step 2: Run capability tests and verify red** + +Run: vendor/bin/phpunit tests/Core/CapabilityDeclarationTest.php + +Expected: FAIL because the declaration still reports legacy_contract_only. + +- [ ] **Step 3: Migrate every remaining caller and delete the legacy tree** + +Search: + +~~~bash +rg -n "Transformation\\Model|Transformation\\Validation|openstatspec-transformation-plan-v1|CreateVariableOperation|DeleteVariableOperation|compileForDataset|compile\(" src tests README.md docs +~~~ + +Update remaining fixtures to SpssFrontendRequest, SpssCompilationResult, PlanCodec, and InPlaceApplyRequest. Delete legacy model/validator classes and the non-standard STRING/DELETE frontend path. The final search must return no production or documentation reference to the package-local contract or removed operations. + +- [ ] **Step 4: Update capabilities and migration documentation** + +Document the exact new flow: + +~~~php +$compiled = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray($request)); +$apply = new InPlaceApplyRequest( + plan: $compiled->plan, + inputAlias: 'parent', + datasetId: $datasetId, + sourceHash: $compiled->sourceHash, + actor: 'analyst@example.org', + expectedBranch: $branch, + expectedHead: $head, +); +$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($apply); +~~~ + +README and migration notes state that v0.6.0 removes openstatspec-transformation-plan-v1 plus STRING/DELETE transformation commands. Explain pre-provisioning on MySQL/MariaDB/Dolt and that Dolt commits remain caller-owned. + +- [ ] **Step 5: Put the new integration gate in every service job** + +Add OfficialInPlaceTransformation01Test and OfficialInPlaceTransformation02Test to PostgreSQL, MySQL, MariaDB, and Dolt PHPUnit filters without removing existing filters. The normal PHP 8.4/8.5 job continues running full composer check and therefore all plan/frontend fixture tests. + +- [ ] **Step 6: Run every official conformance suite** + +Run: + +~~~bash +vendor/bin/phpunit tests/Transformation/Conformance +vendor/bin/phpunit tests/Frontend/Spss/Conformance +vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php +vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation02Test.php +~~~ + +Expected: exact counts 4 Plan 0.1, 26 Plan 0.2, 44 Frontend 0.2, 6 In-Place 0.1, and 11 In-Place 0.2 cases for configured profiles. + +- [ ] **Step 7: Run the complete local quality gate** + +Run: composer check + +Expected: composer validation, PHP syntax, style, PHPStan, and the full PHPUnit suite pass; skips are limited to unconfigured external database services. + +- [ ] **Step 8: Review forbidden artifacts and API removal** + +Run: + +~~~bash +rg -n "derived_dataset|snapshot|rollback|staging|DOLT_COMMIT|openstatspec-transformation-plan-v1" src tests README.md docs +git diff --check +git status --short +~~~ + +Inspect every match and confirm there is no implementation path that creates a forbidden artifact or commits Dolt. Confirm the legacy contract has no remaining class, capability, example, or test. + +- [ ] **Step 9: Commit** + +~~~bash +git add -A +git commit -m "feat: claim official transformation and SPSS 0.2 conformance" +~~~ + +- [ ] **Step 10: Push and open the review PR** + +Push codex/transformation-plan-spss-0.2, open a draft PR targeting main, and monitor every GitHub Actions job plus Codex review. Address only verified actionable feedback, rerun the affected local gates, and do not mark ready until all service matrices are green. diff --git a/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md b/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md new file mode 100644 index 0000000..286a4f1 --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md @@ -0,0 +1,189 @@ +# Transformation Plan and SPSS Frontend 0.2 Design + +**Date:** 2026-08-17 +**Status:** Approved for implementation planning + +## Goal + +Replace the package-local `openstatspec-transformation-plan-v1` API with the +official OpenStatSpec Transformation Plan and SPSS-like Syntax Frontend +contracts from the package's pinned specification checkout. The release must +pass every case in the official Transformation Plan 0.2, SPSS Frontend 0.2, +and In-Place Transformation 0.2 manifests, including the referenced 0.1 golden +plans required by Frontend 0.2. + +This is an intentional breaking change for `v0.6.0`. There is no compatibility +adapter for the package-local plan, and its non-standard `create_variable` and +`delete_variable` operations are removed from the transformation API. + +## Contract boundary + +The canonical model supports both official plan identifiers: + +- `openstatspec-transformation-plan-v0.1` for programs containing only the + unchanged 0.1 command subset; and +- `openstatspec-transformation-plan-v0.2` when a 0.2 command is present. + +Supporting official 0.1 output is part of Frontend 0.2 conformance, not legacy +package compatibility. A plan contains exactly `contract`, `input_alias`, and +a non-empty ordered `operations` array. Dataset UUIDs, SQL details, actor +identity, and Dolt state do not belong in canonical plan JSON. + +The package exposes typed plan objects plus a strict plan codec. Decoding first +validates the structure selected by `contract`, rejects unknown members and +non-canonical values, and then constructs typed operations. Encoding produces +the exact restricted-RFC-8785 bytes used for SHA-256 plan identity. It never +silently upgrades 0.1 to 0.2 or changes operation order, Unicode, expression +shape, or binary64 bits. + +## Public API + +The old constructors and method signatures are removed rather than deprecated. +The replacement boundary consists of: + +- a `TransformationPlan` carrying contract, input alias, and typed operations; +- a `PlanCodec` for strict array/JSON decoding, canonical JSON, and plan hash; +- an `SpssFrontendRequest` carrying the 0.2 request contract, input alias, + ordered input schema, and exact source text; +- an `SpssCompilationResult` carrying the emitted 0.1-or-0.2 plan and the + LF-normalized source hash; +- an `SpssCompiler::compile(SpssFrontendRequest)` facade; and +- an `InPlaceApplyRequest` carrying the plan, input-alias-to-dataset binding, + source hash, non-empty actor, and optional expected Dolt branch and HEAD. + +`InPlaceTransformationExecutor::execute(InPlaceApplyRequest)` is the only +public mutation entry point. This makes the official alias-based plan portable +while keeping the concrete dataset identity and controlled execution context +explicit at apply time. + +## Components + +### Canonical model and validation + +The current transformation model is replaced with the official 0.1 and 0.2 +operation vocabulary. Version 0.1 includes `recode`, `set_variable_label`, and +`replace_value_labels`. Version 0.2 additionally includes `assign`, +`conditional_assign`, `set_format`, `set_measurement_level`, and `execute`. + +Numeric literals retain exact 16-character lowercase binary64 bits. Operands, +comparisons, and n-ary boolean expressions are separate immutable types. +Validation rejects non-finite or negative-zero encodings, invalid target +states, string expressions, invalid formats, duplicate labels, reserved names, +and nested same-operator boolean nodes before SQL mutation. + +### SPSS frontend + +The lexer, parser, binder, and compiler remain separate stages, but they consume +the official request's ordered schema rather than a dataset UUID. Binding is +ASCII case-insensitive and stores the exact catalog spelling in the plan. It +tracks variables created by earlier commands and preserves the simultaneous +input semantics of multi-variable `RECODE`. + +The existing 0.1 commands are narrowed to the official forms. Version 0.2 adds +`COMPUTE`, parenthesized `IF`, `FORMATS`, `VARIABLE LEVEL`, and `EXECUTE`. +Unsupported commands and expression forms fail closed. Decimal numeric tokens +are converted directly to correctly rounded binary64 bits with ties-to-even; +the conversion does not pass through a host floating-point intermediate, and +signed or underflowed zero becomes positive zero. + +The compiler emits an exact 0.1 plan when the program contains only 0.1 +commands. Any 0.2 command promotes the complete emitted plan to 0.2 without +dropping or reordering earlier operations. + +### In-place SQL executor + +Before mutation, the executor validates the complete plan, resolves the input +alias to exactly one dataset, binds every ordered operation against the evolving +schema, checks storage types and target state, verifies backend capabilities, +and validates the apply context. No journal or audit row is started until this +preflight succeeds. + +Existing-target `recode` and `assign` use direct updates. +`conditional_assign` uses a direct update whose predicate relies on SQL +three-valued truth, so only TRUE rows change. Metadata operations update only +their normative catalog fields. `execute` emits no SQL and never commits. + +SQLite and PostgreSQL may create a numeric target only where the full schema, +data, catalog, and audit change is proven to share one native transaction. +MySQL, MariaDB, and Dolt reject create-target plans before mutation with +`schema_change_not_atomic`; they never simulate atomicity with a copied table +or recovery artifact. + +### Dolt context and audit + +Every apply requires a non-empty actor. Dolt additionally requires the expected +branch and expected HEAD and a clean working set. Branch, HEAD, and cleanliness +are checked before mutation; branch and HEAD are checked again before success. +The executor never calls `DOLT_COMMIT`, switches branches, resets, merges, tags, +or creates a persistent recovery object. + +The explicit catalog migration creates or upgrades the versioned +`transformation_apply` table. It stores one compact audit row with the +official binding contract, target identity, canonical plan and source hashes, +actor, status, operation count, timestamps, and optional Dolt evidence. The +table accepts both official 0.1 and 0.2 binding contracts and preserves any +existing official audit rows. Migration runs before an apply and is not hidden +inside the apply transaction. + +## Data flow and atomicity + +1. Validate the frontend request and normalize only source line endings for + `source_hash`. +2. Parse, bind against the supplied schema, and compile the complete source. +3. Validate and canonicalize the emitted plan, then calculate `plan_hash`. +4. Build an apply request that maps the plan alias to one dataset and supplies + actor and any Dolt expectations. +5. Preflight the entire apply without writing data, metadata, schema, or audit. +6. Execute operations in order inside one engine-native transaction and write + the compact audit in that same boundary. +7. On any failure, roll back the native transaction and publish no successful + audit or partial state. + +A successful apply preserves dataset identity, physical table identity, case +order, case count, dataset count, and persistent data-table count. It creates +no derived dataset, full-table copy, staging relation, snapshot, rollback table, +dataset-version row, or hidden recovery layer. Dolt alone owns version history +and rollback. + +## Diagnostics + +Frontend and plan failures expose the stable specification diagnostic codes and +source spans where applicable. Apply failures expose the stable binding codes, +including actor, Dolt context, and backend capability failures. Exceptions may +add a concise human explanation but must not leak credentials, unrelated row +values, or driver connection strings. + +Every validation, binding, capability, actor, branch, HEAD, and clean-working-set +failure occurs before mutation. A detected post-mutation Dolt context change +causes the open transaction to roll back. + +## Verification + +Automated tests load the manifests from the pinned OpenStatSpec `v0.3.0` +specification checkout and treat their expected canonical JSON, hashes, errors, +row results, metadata, audit fields, and forbidden artifacts as independent +goldens. Required coverage is: + +- all 26 Transformation Plan 0.2 cases and every referenced Plan 0.1 golden; +- all 44 SPSS Frontend 0.2 cases and their exact source and plan hashes; +- all 11 In-Place Transformation 0.2 cases; +- focused unit tests for exact decimal-to-binary64 conversion, boolean + precedence/flattening, version selection, and diagnostic spans; +- SQLite execution and rollback tests for successful target creation; +- PostgreSQL execution evidence for the supported atomic create path; +- MySQL, MariaDB, and Dolt rejection evidence for create-target plans; +- live Dolt success, mismatch, dirty-state, context-change, and no-commit + evidence; and +- the existing full lint, style, PHPStan, PHPUnit, and GitHub service matrix. + +The capability declaration and documentation claim Transformation Plan 0.1 and +0.2, SPSS Frontend 0.2, and In-Place Transformation 0.1 and 0.2 only after these +gates pass. Migration notes show the removed package-local API and the new +request/compile/apply flow, and the completed change is released as `v0.6.0`. + +## Out of scope + +Full IBM SPSS syntax, arbitrary arithmetic or functions, string predicates, +automatic target provisioning on non-atomic engines, additional frontends, +automatic Dolt commits, and OpenStatSpec-managed dataset versioning or rollback +are not part of this feature. diff --git a/docs/transformations.md b/docs/transformations.md index 1a2b1ca..72e37ff 100644 --- a/docs/transformations.md +++ b/docs/transformations.md @@ -1,221 +1,148 @@ # Transformations -## Purpose and boundary - -> Status: this PHP layer currently uses the package-local legacy contract -> `openstatspec-transformation-plan-v1`. It does not claim conformance with -> OpenStatSpec Transformation Plan or SPSS Frontend profile 0.1 or 0.2. -> Migration to official 0.1 is required before the additive 0.2 conditional -> profile can be implemented or claimed. - -The transformation layer applies small, deterministic edits to an existing -OpenStatSpec dataset. Its canonical `TransformationPlan`, validation, and SQL -executor do not depend on SPSS, Stata, SAS, or another statistics package. -Language-specific syntax belongs to a frontend that compiles into the same -canonical plan. - -The initial frontend implements a documented subset of SPSS transformation -syntax. The Stata and SAS directories are placeholders only. Their presence is -an architectural reservation, not a support claim. - -## In-place contract - -Every successful apply preserves both identities: - -- the existing `dataset.dataset_id`; and -- the existing `dataset.physical_table_schema` plus - `dataset.physical_table_name`. - -The executor updates that wide table and its existing normative metadata -catalog in place. It does not create a derived dataset, persistent output or -staging table, full-table copy, snapshot table, hidden rollback table, or a -parallel OpenStatSpec version. A successful edit therefore does not increase -the persistent dataset count or physical data-table count. - -The database engine's native transaction is used where it can make the -operation atomic. OpenStatSpec does not add a durable undo or recovery-version -layer around engines whose DDL commits implicitly. Dolt remains the history, -diff, branch, and rollback layer when Dolt is the selected SQL server. - -## Architecture - -The package separates four responsibilities: - -1. `OpenStatSpec\Transformation\Model` defines the canonical, typed plan and - operations. -2. `OpenStatSpec\Transformation\Validation` validates plans without knowing - their source language or SQL dialect. -3. `OpenStatSpec\Frontend\Spss` lexes, parses, binds, and compiles the supported - SPSS subset into a canonical plan. -4. `OpenStatSpec\Transformation\Execution` resolves catalog identities and - applies a validated plan through the active PDO profile. - -The SQL executor accepts a completed plan. It never invokes the SPSS parser. -Likewise, the SPSS frontend does not issue SQL or select a database profile. -This is the package boundary a future real frontend must use. - -Canonical serialization is deterministic. The plan hash identifies the exact -validated operation sequence; source text and language provenance stay outside -the source-neutral plan. - -## Supported operations - -The canonical layer supports: - -- ordered recode rules with exact values, numeric ranges, missing values, and - exactly one explicit final else rule; -- assigning a scalar value, copying the source value, or assigning system - missing; -- variable-label replacement; and -- complete value-label replacement for one variable. - -Recode rules use first-match semantics. Validation rejects overlapping, -duplicate, or ill-typed rules before SQL mutation. An SPSS frontend plan always -meets the canonical explicit-else contract: when source syntax omits `ELSE`, -the compiler adds SPSS's context-appropriate default action. Variables are -resolved through the normative `variable` catalog and physical identifiers -are quoted by the active PDO SQL profile; callers cannot supply raw table or -column SQL. - -## SPSS frontend scope - -The SPSS frontend recognizes the documented transformation subset: - -- `RECODE ... INTO ...` with exact values, `THRU` ranges, - `LOWEST`, `HIGHEST`, `SYSMIS`, `ELSE`, `COPY`, and `SYSMIS` - outputs; -- `VARIABLE LABELS`; and -- `VALUE LABELS`. - -Keywords are case-insensitive. Dataset variable references currently must -match the normative `variable.source_name` spelling exactly; this documented -subset does not claim SPSS's case-insensitive symbol binding. The `MISSING` -selector fails closed because SPSS user-missing semantics require binding the -dataset's `missing_rule` metadata; use `SYSMIS` for system missing or list -supported explicit values. Statements end with a period. Unsupported SPSS -commands fail closed with a frontend diagnostic; they are not silently skipped -or passed to an external statistics engine. This package does not claim full -SPSS syntax compatibility. - -A multi-variable RECODE with INTO targets is expanded into ordered canonical -operations only when an earlier target does not overwrite a source needed by a -later pair in the same statement. Dependency-overlapping lists fail closed -because preserving SPSS simultaneous-input semantics would otherwise require -hidden row snapshots. - -## SQL profiles and Dolt - -Transformations are not restricted to Dolt. The executor uses every SQL -connection profile implemented by this package: SQLite, PostgreSQL, -MySQL/MariaDB, and Dolt. - -Dolt adds safety evidence rather than acting as a gateway. Before mutation the -executor checks the active branch, resolves `HEAD`, and requires a clean Dolt -working set. After mutation it verifies that branch and `HEAD` did not change -under the operation. The executor does not switch branches or create a Dolt -commit. The caller owns the later review and commit policy. - -MySQL-family DDL commits implicitly. A recode into a new physical target column -can only be part of one native atomic apply on a profile with transactional DDL. -On MySQL, MariaDB, and Dolt, create and catalog the intended target variable in -the deployment workflow before applying a recode to it. Existing-column -recodes and metadata edits remain supported. This capability boundary avoids -pretending that a compensating copy or OpenStatSpec rollback layer is atomic. - -SQLite and PostgreSQL may create a new numeric target column inside their -native transaction. A new string target must be registered on every profile -before execution so its normative `declared_string_width` is explicit. - -The machine-readable capability declaration reports in-place transformations -as supported for every implemented SQL profile and states these target-creation -boundaries. Dolt additionally reports its clean-working-set and stable -branch/HEAD guard. - -## Service-matrix evidence gate - -Run the PHP adapter's in-place evidence matrix with the pinned specification -checkout: +## Official contracts -```bash -OPENSTATSPEC_SPECIFICATION_DIR=/path/to/openstatspec-specification \ - vendor/bin/phpunit tests/Integration/InPlaceTransformationServiceTest.php -``` +The PHP adapter conforms to these pinned OpenStatSpec contracts: + +- `openstatspec-transformation-plan-v0.1` and + `openstatspec-transformation-plan-v0.2`; +- `openstatspec-spss-syntax-frontend-v0.2`; and +- `openstatspec-in-place-transformation-v0.1` and + `openstatspec-in-place-transformation-v0.2`. + +Plans are source-neutral, alias-based, and deterministic. The canonical plan +contains only its contract, input alias, and ordered operations. Dataset UUIDs, +SQL identifiers, actor identity, and Dolt context are supplied only when the +plan is applied. -SQLite runs locally. PostgreSQL, MySQL, MariaDB, and Dolt run only when their -respective `OPENSTATSPEC_PG_*`, `OPENSTATSPEC_MYSQL_*`, -`OPENSTATSPEC_MARIADB_*`, or `OPENSTATSPEC_DOLT_*` configuration is supplied; -an absent local profile is reported as skipped. CI supplies every service -profile configuration, so a skipped configured service is not acceptable CI -evidence. - -For every configured profile, the existing-target evidence proves that -recode plus label changes preserve the dataset UUID, physical table identity, -and existing variable identities, with no copied, staging, snapshot, rollback, -or parallel-history table. Numeric implicit target creation is evidenced only -on SQLite and PostgreSQL. MySQL, MariaDB, and Dolt reject it before mutation; -their target variable must be created and catalogued by deployment workflow -before the recode. This is PHP adapter evidence only: it retains the legacy -`openstatspec-transformation-plan-v1` contract and does not establish an -official OpenStatSpec Transformation Plan 1.0 profile. - -## Minimal PHP flow +`OpenStatSpec\Transformation\Plan` contains the immutable official model and +strict `PlanCodec`. `OpenStatSpec\Frontend\Spss` parses and binds an +`SpssFrontendRequest` to an `SpssCompilationResult`. The separate +`OpenStatSpec\Transformation\Execution` layer binds the plan alias to an +existing dataset and mutates its registered wide table. + +## Compile and apply ```php +use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; use OpenStatSpec\Frontend\Spss\SpssCompiler; use OpenStatSpec\Sql\Connection; +use OpenStatSpec\Spss\SpssAdapter; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; -$datasetId = '018f47a2-4c10-7d34-8f11-93b1c3efc321'; -$syntax = 'RECODE score (1=10) (ELSE=COPY).'; +$compiled = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray($request)); +$apply = new InPlaceApplyRequest( + plan: $compiled->plan, + inputAlias: 'parent', + datasetId: $datasetId, + sourceHash: $compiled->sourceHash, + actor: 'analyst@example.org', + expectedBranch: $branch, + expectedHead: $head, +); +$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($apply); +``` + +The request carries the official frontend contract, input alias, ordered input +schema, and exact source text. The compiler emits Plan 0.1 when every command +belongs to the 0.1 subset and Plan 0.2 as soon as any 0.2 command is present. +`PlanCodec` strictly decodes official plan JSON and produces its canonical JSON +and SHA-256 identity without changing operation order or binary64 values. -$plan = (new SpssCompiler())->compile($syntax, $datasetId); -$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($plan); +Before applying transformations to an existing catalog, deployment must run +the explicit catalog migration: + +```php +$adapter = new SpssAdapter($pdo); +$adapter->migrateCatalog(); ``` -`$pdo` must already point to the dedicated OpenStatSpec catalog namespace. -The executor verifies the catalog ownership marker and resolves the same -`dataset_id` and physical wide table before mutation. +The migration provisions the compact `transformation_apply` audit table. Apply +does not hide schema migration inside its transaction. -## Development commands +## Supported operations and syntax -Install dependencies and run the complete local gate from the PHP repository: +Plan 0.1 contains ordered `recode`, `set_variable_label`, and +`replace_value_labels` operations. Plan 0.2 additionally contains `assign`, +`conditional_assign`, `set_format`, `set_measurement_level`, and `execute`. -```bash -composer install -composer check -``` +SPSS Frontend 0.2 supports the corresponding restricted forms of: -Run only transformation tests while developing the layer: +- `RECODE`, including `INTO`, exact/range/system-missing inputs, and explicit + unmatched behavior; +- `VARIABLE LABELS` and `VALUE LABELS`; +- `COMPUTE` and parenthesized numeric `IF` predicates; +- `FORMATS` with the numeric `F` family; +- `VARIABLE LEVEL`; and +- `EXECUTE`. -```bash -vendor/bin/phpunit tests/Transformation tests/Frontend/Spss -``` +Names bind ASCII case-insensitively while plans retain exact catalog spelling. +Multi-variable `RECODE` reads all sources from the pre-command schema. Commands, +expressions, comments, and syntax outside the official subset fail closed. -Apply the formatter, then rerun the complete gate: +## In-place and atomicity contract -```bash -composer fix -composer check -``` +Every successful apply preserves the existing logical dataset UUID, registered +physical table identity, case order, case count, dataset count, and persistent +data-table count. Existing-target recodes and assignments use direct updates; +metadata operations change only their normative catalog fields. + +The executor validates the entire plan, evolving schema, actor, backend +capabilities, and Dolt context before mutation. It then applies every operation +and writes one compact success audit inside one engine-native transaction. A +failure rolls that transaction back and publishes no failed transformation +audit row. + +The transformation path never creates a derived dataset, persistent output or +staging table, full-table copy, snapshot, rollback table, dataset-version row, +or hidden recovery layer. It does not use the import/export operation journal +as a second transformation history. + +SQLite and PostgreSQL can create a numeric target inside the native apply +transaction. MySQL, MariaDB, and Dolt cannot make that schema change atomic, so +deployment must pre-provision and catalog every intended target before apply; +a create-target plan fails before mutation with `schema_change_not_atomic`. + +## Dolt ownership -Database integration checks require the corresponding PDO driver and server. -They must use a dedicated OpenStatSpec namespace, as described in the -[architecture guide](architecture.md#deployment-namespace-and-connection-isolation). +Dolt applies require a non-empty caller-supplied actor, expected branch, and +expected HEAD plus a clean working set. The executor verifies branch and HEAD +before mutation and again before success. It never switches branches, commits, +resets, merges, or tags the repository. -## Operational checklist +A successful Dolt apply intentionally leaves an inspectable working-set diff. +Dolt commits and all version-history decisions remain caller-owned; the audit +stores only compact before/after evidence. -Before applying a plan: +## v0.6.0 migration -1. verify that the connection uses the intended dedicated OpenStatSpec - namespace; -2. select the existing dataset by its canonical UUID; -3. compile source syntax explicitly with the intended frontend, or construct a - canonical plan directly; -4. validate the plan before any mutation; -5. on Dolt, start from the expected branch and a clean working set; and -6. after success, inspect the data and metadata diff and decide separately - whether to create a Dolt commit. +Version 0.6.0 removes the package-local +`openstatspec-transformation-plan-v1` API without a compatibility adapter. It +also removes the non-standard SPSS `STRING` and `DELETE VARIABLES` +transformation commands. Callers must move to `SpssFrontendRequest`, +`SpssCompilationResult`, `PlanCodec`, `InPlaceApplyRequest`, and the official +contract identifiers shown above. + +On MySQL, MariaDB, and Dolt, deployment migrations must create and catalog new +targets before applying an official plan. On Dolt, the caller must also capture +the expected branch and HEAD, apply against a clean working set, inspect the +resulting diff, and decide whether and how to commit it. + +## Conformance and development gates + +The official fixture suites are loaded from the pinned OpenStatSpec +specification checkout: + +```bash +vendor/bin/phpunit tests/Transformation/Conformance +vendor/bin/phpunit tests/Frontend/Spss/Conformance +vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php +vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation02Test.php +composer check +``` -OpenStatSpec stores only compact operation evidence such as the plan identity -and relevant Dolt state. It never stores copied row state as transformation -audit data. +They cover all 4 Plan 0.1, 26 Plan 0.2, 44 Frontend 0.2, 6 In-Place 0.1, +and 11 In-Place 0.2 manifest cases. SQLite runs locally. PostgreSQL, MySQL, +MariaDB, and Dolt cases run when their `OPENSTATSPEC_*` service configuration +is supplied; CI configures every service family. diff --git a/src/Core/CapabilityDeclaration.php b/src/Core/CapabilityDeclaration.php index ca02bae..f6cc97f 100644 --- a/src/Core/CapabilityDeclaration.php +++ b/src/Core/CapabilityDeclaration.php @@ -13,7 +13,6 @@ use OpenStatSpec\Sql\PdoSqlProfile; use OpenStatSpec\Sql\PostgreSqlProfile; use OpenStatSpec\Sql\SqliteProfile; -use OpenStatSpec\Transformation\Model\TransformationPlan; use PDO; /** Machine-readable SPSS 1.0 and SQL-profile capability declaration. */ @@ -57,14 +56,18 @@ public function toArray(): array 'limit_basis' => 'runtime_memory_limit', ], 'transformation_contracts' => [ - 'status' => 'legacy_contract_only', - 'legacy_plan_contracts' => [TransformationPlan::CONTRACT], - 'official_plan_contracts' => [], - 'official_frontend_contracts' => [], - 'migration_order' => [ + 'status' => 'official_conformant', + 'official_plan_contracts' => [ 'openstatspec-transformation-plan-v0.1', 'openstatspec-transformation-plan-v0.2', ], + 'official_frontend_contracts' => [ + 'openstatspec-spss-syntax-frontend-v0.2', + ], + 'official_binding_contracts' => [ + 'openstatspec-in-place-transformation-v0.1', + 'openstatspec-in-place-transformation-v0.2', + ], ], 'active_connection' => [ 'profile' => $activeProfile, diff --git a/src/Frontend/Spss/Ast/BooleanPredicate.php b/src/Frontend/Spss/Ast/BooleanPredicate.php new file mode 100644 index 0000000..1e58826 --- /dev/null +++ b/src/Frontend/Spss/Ast/BooleanPredicate.php @@ -0,0 +1,22 @@ + $operands */ + public function __construct( + public string $operator, + public array $operands, + public SourceSpan $span, + ) {} + + public function span(): SourceSpan + { + return $this->span; + } +} diff --git a/src/Frontend/Spss/Ast/Comparison.php b/src/Frontend/Spss/Ast/Comparison.php new file mode 100644 index 0000000..122d43e --- /dev/null +++ b/src/Frontend/Spss/Ast/Comparison.php @@ -0,0 +1,22 @@ +span; + } +} diff --git a/src/Frontend/Spss/Ast/ComputeStatement.php b/src/Frontend/Spss/Ast/ComputeStatement.php new file mode 100644 index 0000000..36eb30f --- /dev/null +++ b/src/Frontend/Spss/Ast/ComputeStatement.php @@ -0,0 +1,22 @@ +span->startLine; + } +} diff --git a/src/Frontend/Spss/Ast/DeleteVariablesStatement.php b/src/Frontend/Spss/Ast/DeleteVariablesStatement.php deleted file mode 100644 index cb36b74..0000000 --- a/src/Frontend/Spss/Ast/DeleteVariablesStatement.php +++ /dev/null @@ -1,19 +0,0 @@ - $variables */ - public function __construct( - public int $lineNumber, - public array $variables, - ) {} - - public function line(): int - { - return $this->lineNumber; - } -} diff --git a/src/Frontend/Spss/Ast/ElseInput.php b/src/Frontend/Spss/Ast/ElseInput.php index 344da43..ad8ad67 100644 --- a/src/Frontend/Spss/Ast/ElseInput.php +++ b/src/Frontend/Spss/Ast/ElseInput.php @@ -4,4 +4,14 @@ namespace OpenStatSpec\Frontend\Spss\Ast; -final readonly class ElseInput implements RecodeInput {} +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + +final readonly class ElseInput implements RecodeInput +{ + public function __construct(public SourceSpan $span) {} + + public function span(): SourceSpan + { + return $this->span; + } +} diff --git a/src/Frontend/Spss/Ast/ExecuteStatement.php b/src/Frontend/Spss/Ast/ExecuteStatement.php index 16483e3..49057f7 100644 --- a/src/Frontend/Spss/Ast/ExecuteStatement.php +++ b/src/Frontend/Spss/Ast/ExecuteStatement.php @@ -4,9 +4,11 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ExecuteStatement implements Statement { - public function __construct(public int $sourceLine) {} + public function __construct(public int $sourceLine, public SourceSpan $span) {} public function line(): int { diff --git a/src/Frontend/Spss/Ast/ExpressionOperand.php b/src/Frontend/Spss/Ast/ExpressionOperand.php new file mode 100644 index 0000000..1f85d96 --- /dev/null +++ b/src/Frontend/Spss/Ast/ExpressionOperand.php @@ -0,0 +1,12 @@ + $targets */ + public function __construct(public array $targets, public SourceSpan $span) {} + + public function line(): int + { + return $this->span->startLine; + } +} diff --git a/src/Frontend/Spss/Ast/IfStatement.php b/src/Frontend/Spss/Ast/IfStatement.php new file mode 100644 index 0000000..ed699eb --- /dev/null +++ b/src/Frontend/Spss/Ast/IfStatement.php @@ -0,0 +1,23 @@ +span->startLine; + } +} diff --git a/src/Frontend/Spss/Ast/LiteralOperand.php b/src/Frontend/Spss/Ast/LiteralOperand.php new file mode 100644 index 0000000..d5040bf --- /dev/null +++ b/src/Frontend/Spss/Ast/LiteralOperand.php @@ -0,0 +1,17 @@ +span; + } +} diff --git a/src/Frontend/Spss/Ast/MissingInput.php b/src/Frontend/Spss/Ast/MissingInput.php index 91eef12..ba782d7 100644 --- a/src/Frontend/Spss/Ast/MissingInput.php +++ b/src/Frontend/Spss/Ast/MissingInput.php @@ -4,4 +4,14 @@ namespace OpenStatSpec\Frontend\Spss\Ast; -final readonly class MissingInput implements RecodeInput {} +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + +final readonly class MissingInput implements RecodeInput +{ + public function __construct(public SourceSpan $span) {} + + public function span(): SourceSpan + { + return $this->span; + } +} diff --git a/src/Frontend/Spss/Ast/Predicate.php b/src/Frontend/Spss/Ast/Predicate.php new file mode 100644 index 0000000..5af5282 --- /dev/null +++ b/src/Frontend/Spss/Ast/Predicate.php @@ -0,0 +1,12 @@ +span; + } } diff --git a/src/Frontend/Spss/Ast/RecodeInput.php b/src/Frontend/Spss/Ast/RecodeInput.php index c46d949..01db4a0 100644 --- a/src/Frontend/Spss/Ast/RecodeInput.php +++ b/src/Frontend/Spss/Ast/RecodeInput.php @@ -4,4 +4,9 @@ namespace OpenStatSpec\Frontend\Spss\Ast; -interface RecodeInput {} +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + +interface RecodeInput +{ + public function span(): SourceSpan; +} diff --git a/src/Frontend/Spss/Ast/RecodeOutput.php b/src/Frontend/Spss/Ast/RecodeOutput.php index ce88add..ce5c315 100644 --- a/src/Frontend/Spss/Ast/RecodeOutput.php +++ b/src/Frontend/Spss/Ast/RecodeOutput.php @@ -4,10 +4,13 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class RecodeOutput { public function __construct( public RecodeOutputKind $kind, + public SourceSpan $span, public ?ScalarValue $value = null, ) {} } diff --git a/src/Frontend/Spss/Ast/RecodeRule.php b/src/Frontend/Spss/Ast/RecodeRule.php index 520ae97..5c79ef5 100644 --- a/src/Frontend/Spss/Ast/RecodeRule.php +++ b/src/Frontend/Spss/Ast/RecodeRule.php @@ -4,10 +4,13 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class RecodeRule { public function __construct( public RecodeInput $input, public RecodeOutput $output, + public SourceSpan $span, ) {} } diff --git a/src/Frontend/Spss/Ast/RecodeStatement.php b/src/Frontend/Spss/Ast/RecodeStatement.php index b6f8e91..a332991 100644 --- a/src/Frontend/Spss/Ast/RecodeStatement.php +++ b/src/Frontend/Spss/Ast/RecodeStatement.php @@ -4,6 +4,8 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class RecodeStatement implements Statement { /** @@ -14,8 +16,13 @@ public function __construct( public int $sourceLine, public array $sources, + /** @var non-empty-list */ + public array $sourceSpans, public array $rules, public array $targets, + /** @var list */ + public array $targetSpans, + public SourceSpan $span, ) {} public function line(): int diff --git a/src/Frontend/Spss/Ast/ScalarValue.php b/src/Frontend/Spss/Ast/ScalarValue.php index 1612fe2..6f39cf6 100644 --- a/src/Frontend/Spss/Ast/ScalarValue.php +++ b/src/Frontend/Spss/Ast/ScalarValue.php @@ -4,7 +4,18 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ScalarValue { - public function __construct(public int|float|string $value) {} + public function __construct( + public string $value, + public ?string $numericToken, + public SourceSpan $span, + ) {} + + public function isNumeric(): bool + { + return $this->numericToken !== null; + } } diff --git a/src/Frontend/Spss/Ast/StringStatement.php b/src/Frontend/Spss/Ast/StringStatement.php deleted file mode 100644 index 5e50af2..0000000 --- a/src/Frontend/Spss/Ast/StringStatement.php +++ /dev/null @@ -1,20 +0,0 @@ - $variables */ - public function __construct( - public int $lineNumber, - public array $variables, - public int $width, - ) {} - - public function line(): int - { - return $this->lineNumber; - } -} diff --git a/src/Frontend/Spss/Ast/SystemMissingInput.php b/src/Frontend/Spss/Ast/SystemMissingInput.php index ec018f2..a4e3a3a 100644 --- a/src/Frontend/Spss/Ast/SystemMissingInput.php +++ b/src/Frontend/Spss/Ast/SystemMissingInput.php @@ -4,4 +4,14 @@ namespace OpenStatSpec\Frontend\Spss\Ast; -final readonly class SystemMissingInput implements RecodeInput {} +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + +final readonly class SystemMissingInput implements RecodeInput +{ + public function __construct(public SourceSpan $span) {} + + public function span(): SourceSpan + { + return $this->span; + } +} diff --git a/src/Frontend/Spss/Ast/ValueInput.php b/src/Frontend/Spss/Ast/ValueInput.php index 901e177..47d8cf0 100644 --- a/src/Frontend/Spss/Ast/ValueInput.php +++ b/src/Frontend/Spss/Ast/ValueInput.php @@ -4,7 +4,28 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ValueInput implements RecodeInput { - public function __construct(public ScalarValue $value) {} + /** @var non-empty-list */ + public array $values; + public ScalarValue $value; + public SourceSpan $span; + + public function __construct(ScalarValue ...$values) + { + if ($values === []) { + throw new \InvalidArgumentException('A RECODE values selector requires at least one value.'); + } + + $this->values = array_values($values); + $this->value = $values[0]; + $this->span = SourceSpan::cover($values[0]->span, $values[array_key_last($values)]->span); + } + + public function span(): SourceSpan + { + return $this->span; + } } diff --git a/src/Frontend/Spss/Ast/ValueLabel.php b/src/Frontend/Spss/Ast/ValueLabel.php index d39d394..af60443 100644 --- a/src/Frontend/Spss/Ast/ValueLabel.php +++ b/src/Frontend/Spss/Ast/ValueLabel.php @@ -4,7 +4,13 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ValueLabel { - public function __construct(public ScalarValue $value, public string $label) {} + public function __construct( + public ScalarValue $value, + public string $label, + public SourceSpan $span, + ) {} } diff --git a/src/Frontend/Spss/Ast/ValueLabelGroup.php b/src/Frontend/Spss/Ast/ValueLabelGroup.php index 8b7e337..3f87715 100644 --- a/src/Frontend/Spss/Ast/ValueLabelGroup.php +++ b/src/Frontend/Spss/Ast/ValueLabelGroup.php @@ -4,11 +4,19 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ValueLabelGroup { /** * @param non-empty-list $variables * @param non-empty-list $labels */ - public function __construct(public array $variables, public array $labels) {} + public function __construct( + public array $variables, + /** @var non-empty-list */ + public array $variableSpans, + public array $labels, + public SourceSpan $span, + ) {} } diff --git a/src/Frontend/Spss/Ast/ValueLabelsStatement.php b/src/Frontend/Spss/Ast/ValueLabelsStatement.php index 1e4fe20..966e986 100644 --- a/src/Frontend/Spss/Ast/ValueLabelsStatement.php +++ b/src/Frontend/Spss/Ast/ValueLabelsStatement.php @@ -4,10 +4,16 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ValueLabelsStatement implements Statement { /** @param non-empty-list $groups */ - public function __construct(public int $sourceLine, public array $groups) {} + public function __construct( + public int $sourceLine, + public array $groups, + public SourceSpan $span, + ) {} public function line(): int { diff --git a/src/Frontend/Spss/Ast/VariableLabelAssignment.php b/src/Frontend/Spss/Ast/VariableLabelAssignment.php new file mode 100644 index 0000000..b1f4e42 --- /dev/null +++ b/src/Frontend/Spss/Ast/VariableLabelAssignment.php @@ -0,0 +1,17 @@ + $labels */ - public function __construct(public int $sourceLine, public array $labels) {} + public function __construct( + public int $sourceLine, + public array $labels, + /** @var non-empty-list */ + public array $assignments, + public SourceSpan $span, + ) {} public function line(): int { diff --git a/src/Frontend/Spss/Ast/VariableLevelGroup.php b/src/Frontend/Spss/Ast/VariableLevelGroup.php new file mode 100644 index 0000000..28b0625 --- /dev/null +++ b/src/Frontend/Spss/Ast/VariableLevelGroup.php @@ -0,0 +1,19 @@ + $variables */ + public function __construct( + public array $variables, + /** @var non-empty-list */ + public array $variableSpans, + public string $level, + public SourceSpan $span, + ) {} +} diff --git a/src/Frontend/Spss/Ast/VariableLevelStatement.php b/src/Frontend/Spss/Ast/VariableLevelStatement.php new file mode 100644 index 0000000..eff3dcd --- /dev/null +++ b/src/Frontend/Spss/Ast/VariableLevelStatement.php @@ -0,0 +1,18 @@ + $groups */ + public function __construct(public array $groups, public SourceSpan $span) {} + + public function line(): int + { + return $this->span->startLine; + } +} diff --git a/src/Frontend/Spss/Ast/VariableOperand.php b/src/Frontend/Spss/Ast/VariableOperand.php new file mode 100644 index 0000000..b89b094 --- /dev/null +++ b/src/Frontend/Spss/Ast/VariableOperand.php @@ -0,0 +1,17 @@ +span; + } +} diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php index ac27682..ca47603 100644 --- a/src/Frontend/Spss/Binder.php +++ b/src/Frontend/Spss/Binder.php @@ -4,125 +4,557 @@ namespace OpenStatSpec\Frontend\Spss; +use OpenStatSpec\Frontend\Spss\Ast\BooleanPredicate as AstBooleanPredicate; +use OpenStatSpec\Frontend\Spss\Ast\Comparison as AstComparison; +use OpenStatSpec\Frontend\Spss\Ast\ComputeStatement; use OpenStatSpec\Frontend\Spss\Ast\ElseInput; -use OpenStatSpec\Frontend\Spss\Ast\DeleteVariablesStatement; use OpenStatSpec\Frontend\Spss\Ast\ExecuteStatement; +use OpenStatSpec\Frontend\Spss\Ast\ExpressionOperand as AstExpressionOperand; +use OpenStatSpec\Frontend\Spss\Ast\FormatsStatement; +use OpenStatSpec\Frontend\Spss\Ast\IfStatement; +use OpenStatSpec\Frontend\Spss\Ast\LiteralOperand as AstLiteralOperand; use OpenStatSpec\Frontend\Spss\Ast\MissingInput; -use OpenStatSpec\Frontend\Spss\Ast\StringStatement; +use OpenStatSpec\Frontend\Spss\Ast\Predicate as AstPredicate; use OpenStatSpec\Frontend\Spss\Ast\Program; use OpenStatSpec\Frontend\Spss\Ast\RangeInput; +use OpenStatSpec\Frontend\Spss\Ast\RecodeInput; +use OpenStatSpec\Frontend\Spss\Ast\RecodeOutput; +use OpenStatSpec\Frontend\Spss\Ast\RecodeOutputKind; use OpenStatSpec\Frontend\Spss\Ast\RecodeStatement; +use OpenStatSpec\Frontend\Spss\Ast\ScalarValue; +use OpenStatSpec\Frontend\Spss\Ast\SystemMissingInput; +use OpenStatSpec\Frontend\Spss\Ast\ValueInput; use OpenStatSpec\Frontend\Spss\Ast\ValueLabelsStatement; use OpenStatSpec\Frontend\Spss\Ast\VariableLabelsStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableLevelStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableOperand as AstVariableOperand; +use OpenStatSpec\Frontend\Spss\Binding\BoundCompute; +use OpenStatSpec\Frontend\Spss\Binding\BoundConditionalAssign; +use OpenStatSpec\Frontend\Spss\Binding\BoundExecute; +use OpenStatSpec\Frontend\Spss\Binding\BoundFormat; +use OpenStatSpec\Frontend\Spss\Binding\BoundMeasurementLevel; use OpenStatSpec\Frontend\Spss\Binding\BoundProgram; use OpenStatSpec\Frontend\Spss\Binding\BoundRecode; +use OpenStatSpec\Frontend\Spss\Binding\BoundStatement; use OpenStatSpec\Frontend\Spss\Binding\BoundValueLabels; -use OpenStatSpec\Frontend\Spss\Binding\BoundCreateVariable; -use OpenStatSpec\Frontend\Spss\Binding\BoundDeleteVariable; use OpenStatSpec\Frontend\Spss\Binding\BoundVariableLabel; +use OpenStatSpec\Frontend\Spss\Binding\SchemaState; +use OpenStatSpec\Frontend\Spss\Number\DecimalBinary64; +use OpenStatSpec\Frontend\Spss\Request\InputSchema; +use OpenStatSpec\Frontend\Spss\Request\InputVariable; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Plan\Expression\BooleanPredicate; +use OpenStatSpec\Transformation\Plan\Expression\Comparison; +use OpenStatSpec\Transformation\Plan\Expression\LiteralOperand; +use OpenStatSpec\Transformation\Plan\Expression\Operand; +use OpenStatSpec\Transformation\Plan\Expression\Predicate; +use OpenStatSpec\Transformation\Plan\Expression\VariableOperand; +use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; +use OpenStatSpec\Transformation\Plan\Recode\CopyResult; +use OpenStatSpec\Transformation\Plan\Recode\ExactMatch; +use OpenStatSpec\Transformation\Plan\Recode\LiteralResult; +use OpenStatSpec\Transformation\Plan\Recode\RangeMatch; +use OpenStatSpec\Transformation\Plan\Recode\RecodeMatch; +use OpenStatSpec\Transformation\Plan\Recode\RecodeRule; +use OpenStatSpec\Transformation\Plan\Recode\Result; +use OpenStatSpec\Transformation\Plan\Recode\SystemMissingMatch; +use OpenStatSpec\Transformation\Plan\Recode\SystemMissingResult; +use OpenStatSpec\Transformation\Plan\TargetMode; +use OpenStatSpec\Transformation\Plan\Value\Binary64Value; +use OpenStatSpec\Transformation\Plan\Value\StringValue; +use OpenStatSpec\Transformation\Plan\Value\TypedValue; final class Binder { - public function bind(string $datasetId, Program $program): BoundProgram + public function bind(string $inputAlias, InputSchema $inputSchema, Program $program): BoundProgram { - if (trim($datasetId) === '') { - throw new SpssSyntaxException([new Diagnostic(1, 1, 'Dataset id must not be empty.')]); - } - + $schema = new SchemaState($inputSchema); $bound = []; foreach ($program->statements as $statement) { - if ($statement instanceof ExecuteStatement) { + if ($statement instanceof RecodeStatement) { + array_push($bound, ...$this->recode($statement, $schema)); continue; } - if ($statement instanceof StringStatement) { - foreach ($statement->variables as $variable) { - $bound[] = new BoundCreateVariable($variable, 'string', $statement->width); - } + if ($statement instanceof ComputeStatement) { + $bound[] = $this->compute($statement, $schema); continue; } - if ($statement instanceof DeleteVariablesStatement) { - foreach ($statement->variables as $variable) { - $bound[] = new BoundDeleteVariable($variable); - } + if ($statement instanceof IfStatement) { + $bound[] = $this->conditionalAssign($statement, $schema); continue; } - if ($statement instanceof RecodeStatement) { - $targets = $statement->targets === [] ? $statement->sources : $statement->targets; - if (count($statement->sources) !== count($targets)) { - $this->fail($statement->line(), 'RECODE INTO must have exactly one target for each source variable.'); - } - if ($statement->targets !== [] && count(array_unique($targets)) !== count($targets)) { - $this->fail( - $statement->line(), - 'RECODE INTO must not contain duplicate target variables in the same statement.', - ); - } - foreach ($targets as $targetIndex => $target) { - foreach (array_slice($statement->sources, $targetIndex + 1) as $laterSource) { - if ($target === $laterSource) { - $this->fail( - $statement->line(), - 'RECODE INTO targets must not overwrite a source used by a later pair in the same statement.', - ); - } - } - } - $elseSeen = false; - foreach ($statement->rules as $index => $rule) { - if ($rule->input instanceof MissingInput) { + if ($statement instanceof FormatsStatement) { + foreach ($statement->targets as $target) { + $variable = $schema->resolve($target->variable, $target->variableSpan); + if ($variable->storageKind !== 'numeric') { $this->fail( - $statement->line(), - 'MISSING includes user-missing values, which require dataset metadata; use SYSMIS or explicit values.', + 'expression_type_unsupported', + 'Numeric F formats cannot target string variables.', + $target->variableSpan, ); } - if ($rule->input instanceof ElseInput) { - if ($elseSeen || $index !== array_key_last($statement->rules)) { - $this->fail($statement->line(), 'ELSE may occur only once and must be the final RECODE rule.'); - } - $elseSeen = true; - } - if ($rule->input instanceof RangeInput) { - $lower = $rule->input->lower?->value; - $upper = $rule->input->upper?->value; - if ($lower === null && $upper === null) { - $this->fail($statement->line(), 'LOWEST THRU HIGHEST is not a finite canonical range; use ELSE instead.'); - } - if (is_string($lower) || is_string($upper)) { - $this->fail($statement->line(), 'RECODE ranges must use numeric bounds.'); - } - if ($lower !== null && $upper !== null && $lower > $upper) { - $this->fail($statement->line(), 'RECODE range lower bound must not exceed its upper bound.'); - } - } + $this->validateFormat($target->width, $target->decimals, $target->span); + $bound[] = new BoundFormat( + $variable->name, + $target->family, + $target->width, + $target->decimals, + $target->span, + ); } - foreach ($statement->sources as $index => $source) { - $bound[] = new BoundRecode($source, $targets[$index], $statement->rules); + continue; + } + if ($statement instanceof VariableLevelStatement) { + foreach ($statement->groups as $group) { + foreach ($group->variables as $index => $name) { + $span = $group->variableSpans[$index]; + $variable = $schema->resolve($name, $span); + $bound[] = new BoundMeasurementLevel($variable->name, $group->level, $span); + } } continue; } + if ($statement instanceof ExecuteStatement) { + $bound[] = new BoundExecute($statement->span); + continue; + } if ($statement instanceof VariableLabelsStatement) { - foreach ($statement->labels as $variable => $label) { - $bound[] = new BoundVariableLabel($variable, $label); + foreach ($statement->assignments as $assignment) { + $variable = $schema->resolve($assignment->variable, $assignment->variableSpan); + $bound[] = new BoundVariableLabel($variable->name, $assignment->label, $assignment->span); } continue; } if ($statement instanceof ValueLabelsStatement) { - foreach ($statement->groups as $group) { - foreach ($group->variables as $variable) { - $bound[] = new BoundValueLabels($variable, $group->labels); + array_push($bound, ...$this->valueLabels($statement, $schema)); + continue; + } + $this->fail( + 'unsupported_spss_command', + sprintf('Unsupported AST statement %s.', $statement::class), + new SourceSpan(0, 0, $statement->line(), 1, $statement->line(), 1), + ); + } + + if ($bound === []) { + $this->fail( + 'spss_syntax_error', + 'At least one supported SPSS command is required.', + new SourceSpan(0, 0), + ); + } + + return new BoundProgram($inputAlias, $bound); + } + + private function compute(ComputeStatement $statement, SchemaState $schema): BoundCompute + { + [$value, $valueType] = $this->operand($statement->expression, $schema); + if ($valueType !== 'binary64') { + $this->fail( + 'expression_type_unsupported', + 'String assignment expressions are outside Frontend 0.2.', + $statement->expression->span(), + ); + } + + $target = $schema->find($statement->target, $statement->targetSpan); + $this->validateTargetName($statement->target, $statement->targetSpan); + if ($target !== null) { + if ($target->storageKind !== 'numeric') { + $this->fail( + 'expression_type_unsupported', + 'String assignment targets are outside Frontend 0.2.', + $statement->targetSpan, + ); + } + + return new BoundCompute($target->name, TargetMode::Replace, $value, $statement->span); + } + $schema->addNumeric($statement->target); + + return new BoundCompute($statement->target, TargetMode::Create, $value, $statement->span); + } + + private function conditionalAssign(IfStatement $statement, SchemaState $schema): BoundConditionalAssign + { + $condition = $this->predicate($statement->predicate, $schema); + $target = $schema->resolve( + $statement->target, + $statement->targetSpan, + 'conditional_target_missing', + ); + $this->validateTargetName($statement->target, $statement->targetSpan); + if ($target->storageKind !== 'numeric') { + $this->fail( + 'expression_type_unsupported', + 'String assignment targets are outside Frontend 0.2.', + $statement->targetSpan, + ); + } + [$value, $valueType] = $this->operand($statement->expression, $schema); + if ($valueType !== 'binary64') { + $this->fail( + 'expression_type_unsupported', + 'String assignment expressions are outside Frontend 0.2.', + $statement->expression->span(), + ); + } + + return new BoundConditionalAssign($condition, $target->name, $value, $statement->span); + } + + /** @return array{Operand, 'binary64'|'string'} */ + private function operand(AstExpressionOperand $operand, SchemaState $schema): array + { + if ($operand instanceof AstLiteralOperand) { + return [ + new LiteralOperand(Binary64Value::fromBits(DecimalBinary64::bits($operand->token))), + 'binary64', + ]; + } + if ($operand instanceof AstVariableOperand) { + $variable = $schema->resolve($operand->name, $operand->span); + + return [ + new VariableOperand($variable->name), + $this->storageType($variable), + ]; + } + + throw new \LogicException(sprintf('Unsupported operand %s.', $operand::class)); + } + + private function predicate(AstPredicate $predicate, SchemaState $schema): Predicate + { + if ($predicate instanceof AstComparison) { + [$left, $leftType] = $this->operand($predicate->left, $schema); + [$right, $rightType] = $this->operand($predicate->right, $schema); + if ($leftType !== 'binary64' || $rightType !== 'binary64') { + $this->fail( + 'expression_type_unsupported', + 'String predicates are outside Frontend 0.2.', + $predicate->span, + ); + } + + return new Comparison($left, $predicate->operator, $right); + } + if ($predicate instanceof AstBooleanPredicate) { + return new BooleanPredicate( + $predicate->operator, + ...array_map(fn(AstPredicate $operand): Predicate => $this->predicate($operand, $schema), $predicate->operands), + ); + } + + throw new \LogicException(sprintf('Unsupported predicate %s.', $predicate::class)); + } + + /** @return non-empty-list */ + private function recode(RecodeStatement $statement, SchemaState $schema): array + { + $sources = []; + foreach ($statement->sources as $index => $sourceName) { + $sources[] = $schema->resolve($sourceName, $statement->sourceSpans[$index]); + } + + $targetMode = $statement->targets === [] ? TargetMode::Replace : TargetMode::Create; + if ($targetMode === TargetMode::Create && count($statement->targets) !== count($sources)) { + $this->fail( + 'spss_syntax_error', + 'RECODE INTO requires one target for each source.', + $statement->span, + ); + } + $targetNames = $targetMode === TargetMode::Create + ? $statement->targets + : array_map(static fn(InputVariable $source): string => $source->name, $sources); + $seenTargets = []; + foreach ($targetNames as $index => $targetName) { + $targetSpan = $targetMode === TargetMode::Create + ? $statement->targetSpans[$index] + : $statement->sourceSpans[$index]; + $this->validateTargetName($targetName, $targetSpan); + if ($targetMode === TargetMode::Create) { + $key = mb_strtolower($targetName, 'UTF-8'); + if ($schema->contains($targetName) || isset($seenTargets[$key])) { + $this->fail( + 'target_already_exists', + 'RECODE INTO targets must be fresh and unique in the pre-command schema.', + $targetSpan, + ); + } + $seenTargets[$key] = true; + } + } + + $elseRules = array_values(array_filter( + $statement->rules, + static fn(\OpenStatSpec\Frontend\Spss\Ast\RecodeRule $rule): bool => $rule->input instanceof ElseInput, + )); + if (count($elseRules) > 1) { + $this->fail('duplicate_else', 'ELSE may occur only once.', $elseRules[1]->input->span()); + } + if ( + $elseRules !== [] + && !($statement->rules[array_key_last($statement->rules)]->input instanceof ElseInput) + ) { + $this->fail('else_not_last', 'ELSE must be the final RECODE rule.', $elseRules[0]->input->span()); + } + if (count($statement->rules) === count($elseRules)) { + $this->fail( + 'spss_syntax_error', + 'RECODE requires at least one non-ELSE rule.', + $statement->span, + ); + } + + $bound = []; + foreach ($sources as $index => $source) { + $rules = []; + $unmatched = null; + foreach ($statement->rules as $rule) { + $result = $this->recodeResult($rule->output, $source, $targetMode); + if ($rule->input instanceof ElseInput) { + $unmatched = $result; + continue; + } + $rules[] = new RecodeRule($this->recodeMatch($rule->input, $source), $result); + } + $unmatched ??= $targetMode === TargetMode::Create ? new SystemMissingResult() : new CopyResult(); + $resultTypes = array_values(array_unique(array_map( + fn(Result $result): string => $this->resultType($result, $source), + [...array_map(static fn(RecodeRule $rule): Result => $rule->result, $rules), $unmatched], + ))); + $targetSpan = $targetMode === TargetMode::Create + ? $statement->targetSpans[$index] + : $statement->sourceSpans[$index]; + if ($targetMode === TargetMode::Create && in_array('string', $resultTypes, true)) { + $this->fail( + 'string_target_requires_declaration', + 'New string targets require an explicit declaration outside Frontend 0.2.', + $targetSpan, + ); + } + if (count($resultTypes) !== 1) { + $this->fail( + 'mixed_result_types', + 'All RECODE results, including unmatched behavior, must have one type.', + $statement->span, + ); + } + if ($targetMode === TargetMode::Replace && $resultTypes[0] !== $this->storageType($source)) { + $this->fail( + 'type_mismatch', + 'In-place RECODE cannot change the variable storage kind.', + $statement->span, + ); + } + $bound[] = new BoundRecode( + $source->name, + $targetNames[$index], + $targetMode, + $this->nonEmptyRules($rules, $statement->span), + $unmatched, + $statement->span, + ); + } + + if ($targetMode === TargetMode::Create) { + foreach ($targetNames as $targetName) { + $schema->addNumeric($targetName); + } + } + + return $bound; + } + + private function recodeMatch(RecodeInput $input, InputVariable $source): RecodeMatch + { + if ($input instanceof MissingInput) { + $this->fail( + 'spss_syntax_error', + 'MISSING includes user-missing values and is outside the bounded frontend.', + $input->span, + ); + } + if ($input instanceof SystemMissingInput) { + if ($source->storageKind === 'string') { + $this->fail( + 'system_missing_for_string', + 'SYSMIS cannot match a string variable.', + $input->span, + ); + } + + return new SystemMissingMatch(); + } + if ($input instanceof RangeInput) { + if ($input->lower === null || $input->upper === null) { + $this->fail('spss_syntax_error', 'RECODE ranges require two finite endpoints.', $input->span); + } + $lower = $this->typedValue($input->lower); + $upper = $this->typedValue($input->upper); + if (!$lower instanceof Binary64Value || !$upper instanceof Binary64Value || $source->storageKind !== 'numeric') { + $this->fail('type_mismatch', 'RECODE ranges require numeric sources and endpoints.', $input->span); + } + if ($lower->number() > $upper->number()) { + $this->fail('invalid_numeric_range', 'RECODE range lower bound exceeds its upper bound.', $input->span); + } + + return new RangeMatch($lower, $upper); + } + if ($input instanceof ValueInput) { + $values = array_map(fn(ScalarValue $value): TypedValue => $this->typedValue($value), $input->values); + $expectedType = $this->storageType($source); + foreach ($values as $value) { + if ($this->typedValueType($value) !== $expectedType) { + $this->fail('type_mismatch', 'RECODE match values must match the source type.', $input->span); + } + } + + return new ExactMatch(...$values); + } + + throw new \LogicException(sprintf('Unsupported recode input %s.', $input::class)); + } + + private function recodeResult(RecodeOutput $output, InputVariable $source, TargetMode $targetMode): Result + { + if ($output->kind === RecodeOutputKind::Copy) { + return new CopyResult(); + } + if ($output->kind === RecodeOutputKind::SystemMissing) { + if ($source->storageKind === 'string' && $targetMode === TargetMode::Replace) { + $this->fail( + 'system_missing_for_string', + 'SYSMIS cannot be produced for a string variable.', + $output->span, + ); + } + + return new SystemMissingResult(); + } + + return new LiteralResult($this->typedValue( + $output->value ?? throw new \LogicException('A literal RECODE result requires a value.'), + )); + } + + private function resultType(Result $result, InputVariable $source): string + { + if ($result instanceof CopyResult) { + return $this->storageType($source); + } + if ($result instanceof SystemMissingResult) { + return 'binary64'; + } + if ($result instanceof LiteralResult) { + return $this->typedValueType($result->value); + } + + throw new \LogicException(sprintf('Unsupported recode result %s.', $result::class)); + } + + /** @return list */ + private function valueLabels(ValueLabelsStatement $statement, SchemaState $schema): array + { + $bound = []; + foreach ($statement->groups as $group) { + $labels = array_map( + fn(\OpenStatSpec\Frontend\Spss\Ast\ValueLabel $label): ValueLabel => new ValueLabel( + $this->typedValue($label->value), + $label->label, + ), + $group->labels, + ); + foreach ($group->variables as $index => $name) { + $variable = $schema->resolve($name, $group->variableSpans[$index]); + $expectedType = $this->storageType($variable); + $seen = []; + foreach ($labels as $label) { + if ($this->typedValueType($label->value) !== $expectedType) { + $this->fail( + 'type_mismatch', + 'VALUE LABELS codes must match the variable storage kind.', + $group->span, + ); } + $key = $label->value->canonicalKey(); + if (isset($seen[$key])) { + $this->fail( + 'duplicate_value_label', + 'VALUE LABELS contains duplicate canonical codes.', + $group->span, + ); + } + $seen[$key] = true; } - continue; + $bound[] = new BoundValueLabels($variable->name, $labels, $group->span); } + } + + return $bound; + } + + private function typedValue(ScalarValue $value): TypedValue + { + if ($value->isNumeric()) { + return Binary64Value::fromBits(DecimalBinary64::bits( + $value->numericToken ?? throw new \LogicException('Numeric scalar has no source token.'), + )); + } + + return new StringValue((string) $value->value); + } + + private function typedValueType(TypedValue $value): string + { + return $value instanceof Binary64Value ? 'binary64' : 'string'; + } + + /** @return 'binary64'|'string' */ + private function storageType(InputVariable $variable): string + { + return $variable->storageKind === 'numeric' ? 'binary64' : 'string'; + } - $this->fail($statement->line(), sprintf('Unsupported AST statement %s.', $statement::class)); + private function validateTargetName(string $name, SourceSpan $span): void + { + if (str_starts_with($name, '__')) { + $this->fail('reserved_target_name', 'Target names beginning with __ are reserved.', $span); + } + } + + private function validateFormat(int $width, int $decimals, SourceSpan $span): void + { + if ( + $width < 1 + || $width > 40 + || $decimals < 0 + || $decimals > 16 + || ($decimals > 0 && $width < $decimals + 2) + ) { + $this->fail('invalid_format', 'Invalid SPSS F format.', $span); + } + } + + /** + * @param list $rules + * @return non-empty-list + */ + private function nonEmptyRules(array $rules, SourceSpan $span): array + { + if ($rules === []) { + $this->fail('spss_syntax_error', 'RECODE requires at least one non-ELSE rule.', $span); } - return new BoundProgram($datasetId, $bound); + return $rules; } - private function fail(int $line, string $message): never + private function fail(string $code, string $message, SourceSpan $span): never { - throw new SpssSyntaxException([new Diagnostic($line, 1, $message)]); + throw TransformationFailure::at($code, '$.source_text', $message, $span); } } diff --git a/src/Frontend/Spss/Binding/BoundCompute.php b/src/Frontend/Spss/Binding/BoundCompute.php new file mode 100644 index 0000000..265ce2c --- /dev/null +++ b/src/Frontend/Spss/Binding/BoundCompute.php @@ -0,0 +1,19 @@ + $statements */ - public function __construct(public string $datasetId, public array $statements) {} + /** @param non-empty-list $statements */ + public function __construct(public string $inputAlias, public array $statements) {} } diff --git a/src/Frontend/Spss/Binding/BoundRecode.php b/src/Frontend/Spss/Binding/BoundRecode.php index 186885f..a3dfee2 100644 --- a/src/Frontend/Spss/Binding/BoundRecode.php +++ b/src/Frontend/Spss/Binding/BoundRecode.php @@ -4,7 +4,10 @@ namespace OpenStatSpec\Frontend\Spss\Binding; -use OpenStatSpec\Frontend\Spss\Ast\RecodeRule; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Plan\Recode\RecodeRule; +use OpenStatSpec\Transformation\Plan\Recode\Result; +use OpenStatSpec\Transformation\Plan\TargetMode; final readonly class BoundRecode implements BoundStatement { @@ -12,6 +15,9 @@ public function __construct( public string $sourceVariable, public string $targetVariable, + public TargetMode $targetMode, public array $rules, + public Result $unmatched, + public SourceSpan $span, ) {} } diff --git a/src/Frontend/Spss/Binding/BoundValueLabels.php b/src/Frontend/Spss/Binding/BoundValueLabels.php index fb45ec4..bfbf61d 100644 --- a/src/Frontend/Spss/Binding/BoundValueLabels.php +++ b/src/Frontend/Spss/Binding/BoundValueLabels.php @@ -4,10 +4,15 @@ namespace OpenStatSpec\Frontend\Spss\Binding; -use OpenStatSpec\Frontend\Spss\Ast\ValueLabel; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; final readonly class BoundValueLabels implements BoundStatement { /** @param non-empty-list $labels */ - public function __construct(public string $variable, public array $labels) {} + public function __construct( + public string $variable, + public array $labels, + public SourceSpan $span, + ) {} } diff --git a/src/Frontend/Spss/Binding/BoundVariableLabel.php b/src/Frontend/Spss/Binding/BoundVariableLabel.php index 65e76d0..45e5a8d 100644 --- a/src/Frontend/Spss/Binding/BoundVariableLabel.php +++ b/src/Frontend/Spss/Binding/BoundVariableLabel.php @@ -4,7 +4,13 @@ namespace OpenStatSpec\Frontend\Spss\Binding; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class BoundVariableLabel implements BoundStatement { - public function __construct(public string $variable, public string $label) {} + public function __construct( + public string $variable, + public string $label, + public SourceSpan $span, + ) {} } diff --git a/src/Frontend/Spss/Binding/SchemaState.php b/src/Frontend/Spss/Binding/SchemaState.php new file mode 100644 index 0000000..5106ed8 --- /dev/null +++ b/src/Frontend/Spss/Binding/SchemaState.php @@ -0,0 +1,72 @@ + */ + private array $variables; + + /** @var array> */ + private array $index = []; + + public function __construct(InputSchema $schema) + { + $this->variables = $schema->variables; + foreach ($this->variables as $variable) { + $this->index[$this->key($variable->name)][] = $variable; + } + } + + public function resolve( + string $name, + SourceSpan $span, + string $code = 'unknown_variable', + ): InputVariable { + $matches = $this->index[$this->key($name)] ?? []; + if (count($matches) !== 1) { + throw TransformationFailure::at( + $code, + '$.source_text', + sprintf('Variable %s does not resolve uniquely in the current schema.', $name), + $span, + ); + } + + return $matches[0]; + } + + public function find(string $name, SourceSpan $span): ?InputVariable + { + $matches = $this->index[$this->key($name)] ?? []; + if (count($matches) > 1) { + return $this->resolve($name, $span); + } + + return $matches[0] ?? null; + } + + public function contains(string $name): bool + { + return isset($this->index[$this->key($name)]); + } + + public function addNumeric(string $name): void + { + $variable = new InputVariable($name, 'numeric'); + $this->variables[] = $variable; + $this->index[$this->key($name)][] = $variable; + } + + private function key(string $name): string + { + return mb_strtolower($name, 'UTF-8'); + } +} diff --git a/src/Frontend/Spss/Compiler.php b/src/Frontend/Spss/Compiler.php index 35b5dce..a3905a2 100644 --- a/src/Frontend/Spss/Compiler.php +++ b/src/Frontend/Spss/Compiler.php @@ -4,132 +4,83 @@ namespace OpenStatSpec\Frontend\Spss; -use LogicException; -use OpenStatSpec\Frontend\Spss\Ast\ElseInput; -use OpenStatSpec\Frontend\Spss\Ast\RangeInput; -use OpenStatSpec\Frontend\Spss\Ast\RecodeInput; -use OpenStatSpec\Frontend\Spss\Ast\RecodeOutput; -use OpenStatSpec\Frontend\Spss\Ast\RecodeOutputKind; -use OpenStatSpec\Frontend\Spss\Ast\ScalarValue as AstScalarValue; -use OpenStatSpec\Frontend\Spss\Ast\SystemMissingInput; -use OpenStatSpec\Frontend\Spss\Ast\ValueInput; -use OpenStatSpec\Frontend\Spss\Ast\ValueLabel as AstValueLabel; +use OpenStatSpec\Frontend\Spss\Binding\BoundCompute; +use OpenStatSpec\Frontend\Spss\Binding\BoundConditionalAssign; +use OpenStatSpec\Frontend\Spss\Binding\BoundExecute; +use OpenStatSpec\Frontend\Spss\Binding\BoundFormat; +use OpenStatSpec\Frontend\Spss\Binding\BoundMeasurementLevel; use OpenStatSpec\Frontend\Spss\Binding\BoundProgram; use OpenStatSpec\Frontend\Spss\Binding\BoundRecode; use OpenStatSpec\Frontend\Spss\Binding\BoundValueLabels; use OpenStatSpec\Frontend\Spss\Binding\BoundVariableLabel; -use OpenStatSpec\Frontend\Spss\Binding\BoundCreateVariable; -use OpenStatSpec\Frontend\Spss\Binding\BoundDeleteVariable; -use OpenStatSpec\Transformation\Model\Action\AssignValueAction; -use OpenStatSpec\Transformation\Model\CreateVariableOperation; -use OpenStatSpec\Transformation\Model\DeleteVariableOperation; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\Action\SetMissingAction; -use OpenStatSpec\Transformation\Model\RecodeAction; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\RecodeRule; -use OpenStatSpec\Transformation\Model\RecodeSelector; -use OpenStatSpec\Transformation\Model\ScalarValue; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\ExactValueSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\Selector\NumericRangeSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; -use OpenStatSpec\Transformation\Model\ValueLabel; -use OpenStatSpec\Transformation\Validation\PlanValidator; +use OpenStatSpec\Transformation\Plan\Operation; +use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\ConditionalAssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\ExecuteOperation; +use OpenStatSpec\Transformation\Plan\Operation\RecodeOperation; +use OpenStatSpec\Transformation\Plan\Operation\ReplaceValueLabelsOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetFormatOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetMeasurementLevelOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetVariableLabelOperation; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\TransformationPlan; -/** Compiles only bound SPSS semantics into the source-neutral canonical model. */ +/** Compiles only bound SPSS semantics into the source-neutral official plan. */ final class Compiler { - public function __construct(private readonly PlanValidator $validator = new PlanValidator()) {} - public function compile(BoundProgram $program): TransformationPlan { $operations = []; foreach ($program->statements as $statement) { - if ($statement instanceof BoundCreateVariable) { - $operations[] = new CreateVariableOperation( + $operations[] = match (true) { + $statement instanceof BoundRecode => new RecodeOperation( + $statement->sourceVariable, + $statement->targetVariable, + $statement->targetMode, + $statement->rules, + $statement->unmatched, + ), + $statement instanceof BoundCompute => new AssignOperation( + $statement->target, + $statement->targetMode, + $statement->value, + ), + $statement instanceof BoundConditionalAssign => new ConditionalAssignOperation( + $statement->condition, + $statement->target, + $statement->value, + ), + $statement instanceof BoundVariableLabel => new SetVariableLabelOperation( + $statement->variable, + $statement->label, + ), + $statement instanceof BoundValueLabels => new ReplaceValueLabelsOperation( $statement->variable, - $statement->storageKind, - $statement->storageKind === 'string' ? $statement->declaredStringWidth : null, - ); - continue; - } - if ($statement instanceof BoundDeleteVariable) { - $operations[] = new DeleteVariableOperation($statement->variable); - continue; - } - if ($statement instanceof BoundRecode) { - $rules = []; - $hasElse = false; - foreach ($statement->rules as $rule) { - $selector = $this->selector($rule->input); - $hasElse = $hasElse || $selector instanceof ElseSelector; - $rules[] = new RecodeRule($selector, $this->action($rule->output)); - } - if (!$hasElse) { - $defaultAction = $statement->sourceVariable === $statement->targetVariable - ? new CopySourceAction() - : new SetMissingAction(); - $rules[] = new RecodeRule(new ElseSelector(), $defaultAction); - } - $operations[] = new RecodeOperation($statement->sourceVariable, $statement->targetVariable, $rules); - continue; - } - if ($statement instanceof BoundVariableLabel) { - $operations[] = new SetVariableLabelOperation($statement->variable, $statement->label); - continue; - } - if ($statement instanceof BoundValueLabels) { - $labels = array_map( - fn(AstValueLabel $label): ValueLabel => new ValueLabel( - $this->scalar($label->value), - $label->label, - ), $statement->labels, - ); - $operations[] = new SetValueLabelsOperation($statement->variable, $labels); - continue; - } - - throw new LogicException(sprintf('Unsupported bound SPSS statement %s.', $statement::class)); + ), + $statement instanceof BoundFormat => new SetFormatOperation( + $statement->variable, + $statement->family, + $statement->width, + $statement->decimals, + ), + $statement instanceof BoundMeasurementLevel => new SetMeasurementLevelOperation( + $statement->variable, + $statement->level, + ), + $statement instanceof BoundExecute => new ExecuteOperation(), + default => throw new \LogicException(sprintf( + 'Unsupported bound SPSS statement %s.', + $statement::class, + )), + }; } - $plan = new TransformationPlan($program->datasetId, $operations); - $this->validator->assertValid($plan); - - return $plan; - } - - private function selector(RecodeInput $input): RecodeSelector - { - return match (true) { - $input instanceof ValueInput => new ExactValueSelector($this->scalar($input->value)), - $input instanceof RangeInput => new NumericRangeSelector( - $input->lower === null ? null : (float) $input->lower->value, - $input->upper === null ? null : (float) $input->upper->value, - ), - $input instanceof SystemMissingInput => new MissingValueSelector(), - $input instanceof ElseInput => new ElseSelector(), - default => throw new LogicException(sprintf('Unsupported SPSS recode input %s.', $input::class)), - }; - } + $contract = array_any( + $operations, + static fn(Operation $operation): bool => $operation->minimumContract() === PlanContract::V02, + ) ? PlanContract::V02 : PlanContract::V01; - private function action(RecodeOutput $output): RecodeAction - { - return match ($output->kind) { - RecodeOutputKind::Value => new AssignValueAction($this->scalar( - $output->value ?? throw new LogicException('A value output must contain a scalar.'), - )), - RecodeOutputKind::Copy => new CopySourceAction(), - RecodeOutputKind::SystemMissing => new SetMissingAction(), - }; - } - - private function scalar(AstScalarValue $value): ScalarValue - { - return is_string($value->value) ? ScalarValue::string($value->value) : ScalarValue::number($value->value); + return new TransformationPlan($contract, $program->inputAlias, $operations); } } diff --git a/src/Frontend/Spss/Lexer.php b/src/Frontend/Spss/Lexer.php index cb55b51..d464670 100644 --- a/src/Frontend/Spss/Lexer.php +++ b/src/Frontend/Spss/Lexer.php @@ -4,15 +4,16 @@ namespace OpenStatSpec\Frontend\Spss; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Diagnostic\TransformationDiagnostic; + final class Lexer { /** @return list */ public function tokenize(string $source): array { - $source = preg_replace('/^\xEF\xBB\xBF/', '', $source) ?? $source; - $source = str_replace(["\r\n", "\r"], "\n", $source); $tokens = []; - $offset = 0; + $offset = str_starts_with($source, "\xEF\xBB\xBF") ? 3 : 0; $line = 1; $column = 1; $atStatementStart = true; @@ -21,21 +22,23 @@ public function tokenize(string $source): array while ($offset < $length) { $character = $this->characterAt($source, $offset, $line, $column); if (ctype_space($character)) { + if ($character === "\r") { + $this->advanceCarriageReturn($source, $offset, $line, $column); + continue; + } $this->advance($character, $offset, $line, $column); continue; } if ($atStatementStart && $character === '*') { - while ($offset < $length && $this->characterAt($source, $offset, $line, $column) !== '.') { - $this->advance($this->characterAt($source, $offset, $line, $column), $offset, $line, $column); - } - if ($offset === $length) { - $this->fail($line, $column, 'Comment is missing its period terminator.'); - } - $this->advance('.', $offset, $line, $column); - continue; + $this->fail( + new SourceSpan($offset, $offset + 1, $line, $column, $line, $column + 1), + 'Leading-star comments are not supported.', + 'unsupported_spss_command', + ); } + $tokenOffset = $offset; $tokenLine = $line; $tokenColumn = $column; $atStatementStart = false; @@ -45,18 +48,45 @@ public function tokenize(string $source): array continue; } - $punctuation = match ($character) { - '(' => TokenType::LeftParenthesis, - ')' => TokenType::RightParenthesis, - '=' => TokenType::Equals, - ',' => TokenType::Comma, - '/' => TokenType::Slash, - '.' => TokenType::Terminator, + $next = $source[$offset + 1] ?? ''; + if ($character === '/' && $next === '*') { + $this->fail( + new SourceSpan($offset, $offset + 2, $line, $column, $line, $column + 2), + 'Inline block comments are not supported.', + ); + } + $punctuation = match (true) { + $character === '<' && $next === '=' => TokenType::LessThanOrEqual, + $character === '>' && $next === '=' => TokenType::GreaterThanOrEqual, + $character === '<' => TokenType::LessThan, + $character === '>' => TokenType::GreaterThan, + $character === '(' => TokenType::LeftParenthesis, + $character === ')' => TokenType::RightParenthesis, + $character === '=' => TokenType::Equals, + $character === ',' => TokenType::Comma, + $character === '/' => TokenType::Slash, + $character === '.' => TokenType::Terminator, + $character === '+' || $character === '-' || $character === '*' => TokenType::ArithmeticOperator, default => null, }; if ($punctuation !== null) { - $tokens[] = new Token($punctuation, $character, $tokenLine, $tokenColumn); + $lexeme = ($punctuation === TokenType::LessThanOrEqual || $punctuation === TokenType::GreaterThanOrEqual) + ? $character . $next + : $character; $this->advance($character, $offset, $line, $column); + if (strlen($lexeme) === 2) { + $this->advance($next, $offset, $line, $column); + } + $tokens[] = new Token( + $punctuation, + $lexeme, + $tokenLine, + $tokenColumn, + $tokenOffset, + $offset, + $line, + $column, + ); if ($punctuation === TokenType::Terminator) { $atStatementStart = true; } @@ -77,15 +107,41 @@ public function tokenize(string $source): array } $this->advance($identifierCharacter, $offset, $line, $column); } - $tokens[] = new Token(TokenType::Identifier, substr($source, $start, $offset - $start), $tokenLine, $tokenColumn); + $lexeme = substr($source, $start, $offset - $start); + $type = match (strtoupper($lexeme)) { + 'COMPUTE' => TokenType::Compute, + 'IF' => TokenType::If, + 'AND' => TokenType::And, + 'OR' => TokenType::Or, + 'FORMATS' => TokenType::Formats, + 'VARIABLE' => TokenType::Variable, + 'LEVEL' => TokenType::Level, + 'NOMINAL' => TokenType::Nominal, + 'ORDINAL' => TokenType::Ordinal, + 'SCALE' => TokenType::Scale, + default => TokenType::Identifier, + }; + $tokens[] = new Token( + $type, + $lexeme, + $tokenLine, + $tokenColumn, + $tokenOffset, + $offset, + $line, + $column, + ); continue; } $encoded = json_encode($character, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE); - $this->fail($line, $column, sprintf('Unexpected character %s.', $encoded === false ? '?' : $encoded)); + $this->fail( + new SourceSpan($offset, $offset + strlen($character), $line, $column, $line, $column + 1), + sprintf('Unexpected character %s.', $encoded === false ? '?' : $encoded), + ); } - $tokens[] = new Token(TokenType::EndOfFile, '', $line, $column); + $tokens[] = new Token(TokenType::EndOfFile, '', $line, $column, $offset, $offset, $line, $column); return $tokens; } @@ -95,6 +151,7 @@ private function string(string $source, int &$offset, int &$line, int &$column): $quote = $this->characterAt($source, $offset, $line, $column); $tokenLine = $line; $tokenColumn = $column; + $tokenOffset = $offset; $this->advance($quote, $offset, $line, $column); $value = ''; $length = strlen($source); @@ -102,8 +159,13 @@ private function string(string $source, int &$offset, int &$line, int &$column): while ($offset < $length) { $character = $this->characterAt($source, $offset, $line, $column); if ($character !== $quote) { - $value .= $character; - $this->advance($character, $offset, $line, $column); + if ($character === "\r") { + $value .= "\n"; + $this->advanceCarriageReturn($source, $offset, $line, $column); + } else { + $value .= $character; + $this->advance($character, $offset, $line, $column); + } continue; } $this->advance($character, $offset, $line, $column); @@ -113,26 +175,51 @@ private function string(string $source, int &$offset, int &$line, int &$column): continue; } - return new Token(TokenType::String, $value, $tokenLine, $tokenColumn); + return new Token( + TokenType::String, + $value, + $tokenLine, + $tokenColumn, + $tokenOffset, + $offset, + $line, + $column, + ); } - $this->fail($tokenLine, $tokenColumn, 'String literal is not closed.'); + $this->fail( + new SourceSpan($tokenOffset, $offset, $tokenLine, $tokenColumn, $line, $column), + 'String literal is not closed.', + ); } private function number(string $source, int &$offset, int &$line, int &$column): Token { $tokenLine = $line; $tokenColumn = $column; + $tokenOffset = $offset; $remaining = substr($source, $offset); - if (preg_match('/^[+-]?(?:(?:\d+(?:\.\d*)?)|(?:\.\d+))(?:[Ee][+-]?\d+)?/', $remaining, $matches) !== 1) { - $this->fail($line, $column, 'Invalid numeric literal.'); + if (preg_match('/^[+-]?(?:\d+(?:\.\d+)?|\.\d+)(?:[Ee][+-]?\d+)?/', $remaining, $matches) !== 1) { + $this->fail( + new SourceSpan($offset, $offset + 1, $line, $column, $line, $column + 1), + 'Invalid numeric literal.', + ); } $lexeme = $matches[0]; foreach (str_split($lexeme) as $character) { $this->advance($character, $offset, $line, $column); } - return new Token(TokenType::Number, $lexeme, $tokenLine, $tokenColumn); + return new Token( + TokenType::Number, + $lexeme, + $tokenLine, + $tokenColumn, + $tokenOffset, + $offset, + $line, + $column, + ); } private function startsNumber(string $source, int $offset): bool @@ -176,8 +263,7 @@ private function characterAt(string $source, int $offset, int $line, int $column || preg_match('/\A.\z/usD', $character) !== 1 ) { $this->fail( - $line, - $column, + new SourceSpan($offset, $offset + 1, $line, $column, $line, $column + 1), sprintf('Invalid UTF-8 sequence beginning with byte 0x%02X.', $firstByte), ); } @@ -196,8 +282,20 @@ private function advance(string $character, int &$offset, int &$line, int &$colu } } - private function fail(int $line, int $column, string $message): never + private function advanceCarriageReturn(string $source, int &$offset, int &$line, int &$column): void + { + ++$offset; + if (($source[$offset] ?? '') === "\n") { + ++$offset; + } + ++$line; + $column = 1; + } + + private function fail(SourceSpan $span, string $message, string $code = 'spss_syntax_error'): never { - throw new SpssSyntaxException([new Diagnostic($line, $column, $message)]); + throw new SpssSyntaxException([ + new TransformationDiagnostic($code, '$.source_text', $message, $span), + ]); } } diff --git a/src/Frontend/Spss/Number/DecimalBinary64.php b/src/Frontend/Spss/Number/DecimalBinary64.php new file mode 100644 index 0000000..35a5170 --- /dev/null +++ b/src/Frontend/Spss/Number/DecimalBinary64.php @@ -0,0 +1,112 @@ +minus(strlen($fraction)); + + if ($coefficient->isZero()) { + return '0000000000000000'; + } + + $adjustedDecimalExponent = $decimalExponent->plus(strlen($coefficient->toString())); + if ($adjustedDecimalExponent->isGreaterThan(309)) { + self::reject('Numeric literal exceeds the finite IEEE-754 binary64 range.'); + } + if ($adjustedDecimalExponent->isLessThanOrEqualTo(-324)) { + return '0000000000000000'; + } + + $power = $decimalExponent->toInt(); + if ($power >= 0) { + $numerator = $coefficient->multipliedBy(BigInteger::ten()->power($power)); + $denominator = BigInteger::one(); + } else { + $numerator = $coefficient; + $denominator = BigInteger::ten()->power(-$power); + } + + $exponent = self::floorLog2($numerator, $denominator); + if ($exponent > 1023) { + self::reject('Numeric literal exceeds the finite IEEE-754 binary64 range.'); + } + + if ($exponent < -1022) { + $significand = self::roundRatio($numerator->shiftedLeft(1074), $denominator); + if ($significand->isZero()) { + return '0000000000000000'; + } + + $word = $significand; + } else { + $shift = 52 - $exponent; + $significand = $shift >= 0 + ? self::roundRatio($numerator->shiftedLeft($shift), $denominator) + : self::roundRatio($numerator, $denominator->shiftedLeft(-$shift)); + + $hiddenBit = BigInteger::one()->shiftedLeft(52); + if ($significand->isEqualTo($hiddenBit->shiftedLeft(1))) { + $significand = $hiddenBit; + ++$exponent; + if ($exponent > 1023) { + self::reject('Numeric literal exceeds the finite IEEE-754 binary64 range.'); + } + } + + $word = BigInteger::of($exponent + 1023) + ->shiftedLeft(52) + ->plus($significand->minus($hiddenBit)); + } + + if ($negative) { + $word = $word->plus(BigInteger::one()->shiftedLeft(63)); + } + + return str_pad($word->toBase(16), 16, '0', STR_PAD_LEFT); + } + + private static function floorLog2(BigInteger $numerator, BigInteger $denominator): int + { + $exponent = $numerator->getBitLength() - $denominator->getBitLength(); + $comparison = $exponent >= 0 + ? $numerator->compareTo($denominator->shiftedLeft($exponent)) + : $numerator->shiftedLeft(-$exponent)->compareTo($denominator); + + return $comparison < 0 ? $exponent - 1 : $exponent; + } + + private static function roundRatio(BigInteger $numerator, BigInteger $denominator): BigInteger + { + [$quotient, $remainder] = $numerator->quotientAndRemainder($denominator); + $twice = $remainder->multipliedBy(2); + $comparison = $twice->compareTo($denominator); + if ($comparison > 0 || ($comparison === 0 && $quotient->isOdd())) { + return $quotient->plus(1); + } + + return $quotient; + } + + private static function reject(string $message): never + { + throw new SpssSyntaxException([new Diagnostic(1, 1, $message)]); + } +} diff --git a/src/Frontend/Spss/Parser.php b/src/Frontend/Spss/Parser.php index 5d994fd..3d4e812 100644 --- a/src/Frontend/Spss/Parser.php +++ b/src/Frontend/Spss/Parser.php @@ -4,11 +4,19 @@ namespace OpenStatSpec\Frontend\Spss; -use OpenStatSpec\Frontend\Spss\Ast\DeleteVariablesStatement; +use OpenStatSpec\Frontend\Spss\Ast\BooleanPredicate; +use OpenStatSpec\Frontend\Spss\Ast\Comparison; +use OpenStatSpec\Frontend\Spss\Ast\ComputeStatement; use OpenStatSpec\Frontend\Spss\Ast\ElseInput; use OpenStatSpec\Frontend\Spss\Ast\ExecuteStatement; +use OpenStatSpec\Frontend\Spss\Ast\ExpressionOperand; +use OpenStatSpec\Frontend\Spss\Ast\FormatsStatement; +use OpenStatSpec\Frontend\Spss\Ast\FormatTarget; +use OpenStatSpec\Frontend\Spss\Ast\IfStatement; +use OpenStatSpec\Frontend\Spss\Ast\LiteralOperand; use OpenStatSpec\Frontend\Spss\Ast\MissingInput; use OpenStatSpec\Frontend\Spss\Ast\Program; +use OpenStatSpec\Frontend\Spss\Ast\Predicate; use OpenStatSpec\Frontend\Spss\Ast\RangeInput; use OpenStatSpec\Frontend\Spss\Ast\RecodeInput; use OpenStatSpec\Frontend\Spss\Ast\RecodeOutput; @@ -17,12 +25,19 @@ use OpenStatSpec\Frontend\Spss\Ast\RecodeStatement; use OpenStatSpec\Frontend\Spss\Ast\ScalarValue; use OpenStatSpec\Frontend\Spss\Ast\SystemMissingInput; -use OpenStatSpec\Frontend\Spss\Ast\StringStatement; use OpenStatSpec\Frontend\Spss\Ast\ValueInput; use OpenStatSpec\Frontend\Spss\Ast\ValueLabel; use OpenStatSpec\Frontend\Spss\Ast\ValueLabelGroup; use OpenStatSpec\Frontend\Spss\Ast\ValueLabelsStatement; use OpenStatSpec\Frontend\Spss\Ast\VariableLabelsStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableLabelAssignment; +use OpenStatSpec\Frontend\Spss\Ast\VariableLevelGroup; +use OpenStatSpec\Frontend\Spss\Ast\VariableLevelStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableOperand; +use OpenStatSpec\Frontend\Spss\Number\DecimalBinary64; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Diagnostic\TransformationDiagnostic; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; final class Parser { @@ -49,8 +64,13 @@ public function parseTokens(array $tokens): Program if ($this->matchKeyword('RECODE')) { $statements[] = $this->recode($command); } elseif ($this->matchKeyword('VARIABLE')) { - $this->consumeKeyword('LABELS', 'Expected LABELS after VARIABLE.'); - $statements[] = $this->variableLabels($command); + if ($this->matchKeyword('LABELS')) { + $statements[] = $this->variableLabels($command); + } elseif ($this->matchKeyword('LEVEL')) { + $statements[] = $this->variableLevel($command); + } else { + $this->fail($this->current(), 'Expected LABELS or LEVEL after VARIABLE.'); + } } elseif ($this->matchKeyword('VALUE')) { $this->consumeKeyword('LABELS', 'Expected LABELS after VALUE.'); $statements[] = $this->valueLabels($command); @@ -61,14 +81,21 @@ public function parseTokens(array $tokens): Program $this->consumeKeyword('LAB', 'Expected LAB after VAL.'); $statements[] = $this->valueLabels($command); } elseif ($this->matchKeyword('EXECUTE')) { - $statements[] = new ExecuteStatement($command->line); - } elseif ($this->matchKeyword('STRING')) { - $statements[] = $this->string($command); - } elseif ($this->matchKeyword('DELETE')) { - $this->consumeKeyword('VARIABLES', 'Expected VARIABLES after DELETE.'); - $statements[] = $this->deleteVariables($command); + $statements[] = new ExecuteStatement($command->line, $command->span); + } elseif ($this->matchKeyword('COMPUTE')) { + $statements[] = $this->compute($command); + } elseif ($this->matchKeyword('IF')) { + $statements[] = $this->ifStatement($command); + } elseif ($this->matchKeyword('FORMATS')) { + $statements[] = $this->formats($command); } else { - $this->fail($command, sprintf('Unsupported SPSS command %s.', $command->lexeme === '' ? '' : $command->lexeme)); + $code = $command->isWord() + ? 'unsupported_spss_command' + : 'spss_syntax_error'; + $this->fail($command, sprintf( + 'Unsupported SPSS command %s.', + $command->lexeme === '' ? '' : $command->lexeme, + ), $code); } $this->consume(TokenType::Terminator, 'Expected a period after the SPSS command.'); @@ -77,87 +104,374 @@ public function parseTokens(array $tokens): Program return new Program($statements); } + private function compute(Token $command): ComputeStatement + { + $target = $this->consumeIdentifier('Expected a target variable after COMPUTE.'); + $this->consume(TokenType::Equals, 'Expected = after the COMPUTE target.'); + $expression = $this->expressionOperand('Expected a numeric operand after =.'); + $this->rejectUnsupportedExpressionContinuation(); + + return new ComputeStatement( + $target->lexeme, + $target->span, + $expression, + SourceSpan::cover($command->span, $expression->span()), + ); + } + + private function ifStatement(Token $command): IfStatement + { + $this->consume(TokenType::LeftParenthesis, 'IF requires parentheses around the complete predicate.'); + $predicate = $this->parsePredicate(); + $this->consume(TokenType::RightParenthesis, 'Expected ) after the complete IF predicate.'); + $target = $this->consumeIdentifier('Expected an existing target variable after the IF predicate.'); + $this->consume(TokenType::Equals, 'Expected = after the IF target.'); + $expression = $this->expressionOperand('Expected a numeric operand after =.'); + $this->rejectUnsupportedExpressionContinuation(); + + return new IfStatement( + $predicate, + $target->lexeme, + $target->span, + $expression, + SourceSpan::cover($command->span, $expression->span()), + ); + } + + private function parsePredicate(): Predicate + { + return $this->parseOr(); + } + + private function parseOr(): Predicate + { + $predicate = $this->parseAnd(); + while ($this->matchKeyword('OR')) { + $predicate = $this->merge('or', $predicate, $this->parseAnd()); + } + + return $predicate; + } + + private function parseAnd(): Predicate + { + $predicate = $this->comparison(); + while ($this->matchKeyword('AND')) { + $predicate = $this->merge('and', $predicate, $this->comparison()); + } + + return $predicate; + } + + private function comparison(): Predicate + { + if ($this->match(TokenType::LeftParenthesis)) { + $predicate = $this->parsePredicate(); + $this->consume(TokenType::RightParenthesis, 'Expected ) after the parenthesized predicate.'); + + return $predicate; + } + if ($this->current()->isKeyword('NOT')) { + $this->fail($this->current(), 'NOT is not supported in numeric predicates.', 'expression_type_unsupported'); + } + + $left = $this->expressionOperand('Expected a numeric comparison operand.'); + $operator = $this->comparisonOperator(); + $right = $this->expressionOperand('Expected a numeric comparison operand after the operator.'); + $this->rejectUnsupportedExpressionContinuation(); + + return new Comparison($left, $operator->lexeme, $right, SourceSpan::cover($left->span(), $right->span())); + } + + private function comparisonOperator(): Token + { + if ($this->isArithmeticContinuation()) { + $this->fail( + $this->current(), + 'Arithmetic expressions are not supported.', + 'expression_type_unsupported', + ); + } + foreach ([ + TokenType::Equals, + TokenType::LessThan, + TokenType::LessThanOrEqual, + TokenType::GreaterThan, + TokenType::GreaterThanOrEqual, + ] as $type) { + if ($this->match($type)) { + return $this->previous(); + } + } + + $this->fail($this->current(), 'Expected a supported comparison operator.'); + } + + private function expressionOperand(string $message): ExpressionOperand + { + if ($this->match(TokenType::String)) { + $this->fail($this->previous(), 'String expressions are not supported.', 'expression_type_unsupported'); + } + if ($this->current()->isKeyword('NAN') || $this->current()->isKeyword('INFINITY')) { + $this->fail($this->current(), 'NaN and infinity are not finite numeric operands.'); + } + if ($this->match(TokenType::Number)) { + $token = $this->previous(); + try { + DecimalBinary64::bits($token->lexeme); + } catch (TransformationFailure $failure) { + $this->fail($token, $failure->getMessage()); + } + + return new LiteralOperand($token->lexeme, $token->span); + } + if ($this->current()->isWord()) { + $token = $this->advance(); + + return new VariableOperand($token->lexeme, $token->span); + } + + $this->fail($this->current(), $message); + } + + private function rejectUnsupportedExpressionContinuation(): void + { + if ($this->isArithmeticContinuation()) { + $this->fail( + $this->current(), + 'Arithmetic expressions are not supported.', + 'expression_type_unsupported', + ); + } + } + + private function merge(string $operator, Predicate $left, Predicate $right): BooleanPredicate + { + $operands = $left instanceof BooleanPredicate && $left->operator === $operator + ? $left->operands + : [$left]; + array_push( + $operands, + ...($right instanceof BooleanPredicate && $right->operator === $operator ? $right->operands : [$right]), + ); + + return new BooleanPredicate($operator, $operands, SourceSpan::cover($left->span(), $right->span())); + } + + private function isArithmeticContinuation(): bool + { + if ($this->check(TokenType::ArithmeticOperator) || $this->check(TokenType::Slash)) { + return true; + } + + return $this->check(TokenType::Number) + && ($this->current()->lexeme[0] === '+' || $this->current()->lexeme[0] === '-'); + } + + private function formats(Token $command): FormatsStatement + { + $targets = []; + do { + $variable = $this->consumeIdentifier('FORMATS requires a variable and format pair.'); + $this->consume(TokenType::LeftParenthesis, 'Expected ( before the format.'); + $familyAndWidth = $this->consumeIdentifier('Expected an F format such as F8.2.'); + if (preg_match('/\AF([1-9][0-9]*)\z/iD', $familyAndWidth->lexeme, $matches) !== 1) { + $this->fail($familyAndWidth, 'Only the numeric F format family is supported.', 'invalid_format'); + } + $decimalToken = $this->consume(TokenType::Number, 'F format requires . followed by decimals.'); + if ( + $familyAndWidth->span->endOffset !== $decimalToken->span->startOffset + || preg_match('/\A\.([0-9]+)\z/D', $decimalToken->lexeme, $decimalMatches) !== 1 + ) { + $this->fail($decimalToken, 'F format must use the Fwidth.decimals form.', 'invalid_format'); + } + $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after the format.'); + $targets[] = new FormatTarget( + $variable->lexeme, + 'F', + (int) $matches[1], + (int) $decimalMatches[1], + $variable->span, + SourceSpan::cover($variable->span, $rightParenthesis->span), + ); + } while ($this->current()->isWord()); + + return new FormatsStatement( + $targets, + SourceSpan::cover($command->span, $targets[array_key_last($targets)]->span), + ); + } + + private function variableLevel(Token $command): VariableLevelStatement + { + $groups = []; + do { + $variables = []; + $variableSpans = []; + $first = $this->consumeIdentifier('VARIABLE LEVEL requires at least one variable.'); + $variables[] = $first->lexeme; + $variableSpans[] = $first->span; + while ($this->current()->isWord()) { + $variable = $this->advance(); + $variables[] = $variable->lexeme; + $variableSpans[] = $variable->span; + } + $this->consume(TokenType::LeftParenthesis, 'Expected ( before the measurement level.'); + $level = $this->current(); + if ( + !$this->matchKeyword('NOMINAL') + && !$this->matchKeyword('ORDINAL') + && !$this->matchKeyword('SCALE') + ) { + $this->fail($level, 'Expected NOMINAL, ORDINAL, or SCALE.'); + } + $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after the measurement level.'); + $groups[] = new VariableLevelGroup( + $variables, + $variableSpans, + strtolower($level->lexeme), + SourceSpan::cover($first->span, $rightParenthesis->span), + ); + } while ($this->match(TokenType::Slash)); + + return new VariableLevelStatement( + $groups, + SourceSpan::cover($command->span, $groups[array_key_last($groups)]->span), + ); + } + private function recode(Token $command): RecodeStatement { $sources = []; + $sourceSpans = []; do { - $sources[] = $this->consumeIdentifier('Expected a source variable after RECODE.')->lexeme; - } while ($this->check(TokenType::Identifier) && !$this->current()->isKeyword('INTO')); + $source = $this->consumeIdentifier('Expected a source variable after RECODE.'); + $sources[] = $source->lexeme; + $sourceSpans[] = $source->span; + } while ($this->current()->isWord() && !$this->current()->isKeyword('INTO')); $rules = []; while ($this->match(TokenType::LeftParenthesis)) { + $leftParenthesis = $this->previous(); $input = $this->recodeInput(); + if ($this->check(TokenType::Comma)) { + if (!$input instanceof ValueInput) { + $this->fail($this->current(), 'Comma-separated RECODE selectors must all be typed literals.'); + } + $values = [$input->value]; + while ($this->match(TokenType::Comma)) { + $comma = $this->previous(); + $additionalInput = $this->recodeInput(); + if (!$additionalInput instanceof ValueInput) { + $this->fail($comma, 'Comma-separated RECODE selectors must all be typed literals.'); + } + $values[] = $additionalInput->value; + } + $input = new ValueInput(...$values); + } $this->consume(TokenType::Equals, 'Expected = in a RECODE rule.'); $output = $this->recodeOutput(); - $this->consume(TokenType::RightParenthesis, 'Expected ) after a RECODE rule.'); - $rules[] = new RecodeRule($input, $output); + $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after a RECODE rule.'); + $rules[] = new RecodeRule($input, $output, SourceSpan::cover($leftParenthesis->span, $rightParenthesis->span)); } if ($rules === []) { $this->fail($this->current(), 'RECODE requires at least one parenthesized rule.'); } $targets = []; + $targetSpans = []; if ($this->matchKeyword('INTO')) { do { - $targets[] = $this->consumeIdentifier('Expected a target variable after INTO.')->lexeme; - } while ($this->check(TokenType::Identifier)); + $target = $this->consumeIdentifier('Expected a target variable after INTO.'); + $targets[] = $target->lexeme; + $targetSpans[] = $target->span; + } while ($this->current()->isWord()); } - return new RecodeStatement($command->line, $sources, $rules, $targets); + return new RecodeStatement( + $command->line, + $sources, + $sourceSpans, + $rules, + $targets, + $targetSpans, + SourceSpan::cover($command->span, $this->previous()->span), + ); } private function recodeInput(): RecodeInput { if ($this->matchKeyword('SYSMIS')) { - return new SystemMissingInput(); + return new SystemMissingInput($this->previous()->span); } if ($this->matchKeyword('MISSING')) { - return new MissingInput(); + return new MissingInput($this->previous()->span); } if ($this->matchKeyword('ELSE')) { - return new ElseInput(); + return new ElseInput($this->previous()->span); } if ($this->matchKeyword('LOWEST')) { + $lowest = $this->previous(); $this->consumeKeyword('THRU', 'LOWEST must be followed by THRU.'); - $upper = $this->matchKeyword('HIGHEST') ? null : $this->scalar('Expected an upper bound after LOWEST THRU.'); + if ($this->matchKeyword('HIGHEST')) { + return new RangeInput(null, null, SourceSpan::cover($lowest->span, $this->previous()->span)); + } + $upper = $this->scalar('Expected an upper bound after LOWEST THRU.'); - return new RangeInput(null, $upper); + return new RangeInput(null, $upper, SourceSpan::cover($lowest->span, $upper->span)); } $lower = $this->scalar('Expected a value or selector in a RECODE rule.'); if (!$this->matchKeyword('THRU')) { return new ValueInput($lower); } - $upper = $this->matchKeyword('HIGHEST') ? null : $this->scalar('Expected an upper bound after THRU.'); + if ($this->matchKeyword('HIGHEST')) { + return new RangeInput($lower, null, SourceSpan::cover($lower->span, $this->previous()->span)); + } + $upper = $this->scalar('Expected an upper bound after THRU.'); - return new RangeInput($lower, $upper); + return new RangeInput($lower, $upper, SourceSpan::cover($lower->span, $upper->span)); } private function recodeOutput(): RecodeOutput { if ($this->matchKeyword('COPY')) { - return new RecodeOutput(RecodeOutputKind::Copy); + return new RecodeOutput(RecodeOutputKind::Copy, $this->previous()->span); } if ($this->matchKeyword('SYSMIS')) { - return new RecodeOutput(RecodeOutputKind::SystemMissing); + return new RecodeOutput(RecodeOutputKind::SystemMissing, $this->previous()->span); } - return new RecodeOutput(RecodeOutputKind::Value, $this->scalar('Expected a value, COPY, or SYSMIS after =.')); + $value = $this->scalar('Expected a value, COPY, or SYSMIS after =.'); + + return new RecodeOutput(RecodeOutputKind::Value, $value->span, $value); } private function variableLabels(Token $command): VariableLabelsStatement { $labels = []; + $assignments = []; while (!$this->check(TokenType::Terminator) && !$this->check(TokenType::EndOfFile)) { - $variable = $this->consumeIdentifier('Expected a variable name in VARIABLE LABELS.')->lexeme; - $label = $this->consume(TokenType::String, 'Expected a quoted variable label.')->lexeme; - $labels[$variable] = $label; + $variable = $this->consumeIdentifier('Expected a variable name in VARIABLE LABELS.'); + $label = $this->consume(TokenType::String, 'Expected a quoted variable label.'); + $labels[$variable->lexeme] = $label->lexeme; + $assignments[] = new VariableLabelAssignment( + $variable->lexeme, + $label->lexeme, + $variable->span, + SourceSpan::cover($variable->span, $label->span), + ); } if ($labels === []) { $this->fail($this->current(), 'VARIABLE LABELS requires at least one variable and label.'); } - return new VariableLabelsStatement($command->line, $labels); + return new VariableLabelsStatement( + $command->line, + $labels, + $assignments, + SourceSpan::cover($command->span, $assignments[array_key_last($assignments)]->span), + ); } private function valueLabels(Token $command): ValueLabelsStatement @@ -165,8 +479,11 @@ private function valueLabels(Token $command): ValueLabelsStatement $groups = []; do { $variables = []; - while ($this->check(TokenType::Identifier)) { - $variables[] = $this->advance()->lexeme; + $variableSpans = []; + while ($this->current()->isWord()) { + $variable = $this->advance(); + $variables[] = $variable->lexeme; + $variableSpans[] = $variable->span; } if ($variables === []) { $this->fail($this->current(), 'VALUE LABELS requires at least one variable before its value-label pairs.'); @@ -175,75 +492,54 @@ private function valueLabels(Token $command): ValueLabelsStatement $labels = []; while (!$this->check(TokenType::Slash) && !$this->check(TokenType::Terminator) && !$this->check(TokenType::EndOfFile)) { $value = $this->scalar('Expected a value in VALUE LABELS.'); - $label = $this->consume(TokenType::String, 'Expected a quoted label after the value.')->lexeme; - $labels[] = new ValueLabel($value, $label); + $label = $this->consume(TokenType::String, 'Expected a quoted label after the value.'); + $labels[] = new ValueLabel($value, $label->lexeme, SourceSpan::cover($value->span, $label->span)); } if ($labels === []) { $this->fail($this->current(), 'VALUE LABELS requires at least one value-label pair.'); } - $groups[] = new ValueLabelGroup($variables, $labels); + $groups[] = new ValueLabelGroup( + $variables, + $variableSpans, + $labels, + SourceSpan::cover($variableSpans[0], $labels[array_key_last($labels)]->span), + ); } while ($this->match(TokenType::Slash)); - return new ValueLabelsStatement($command->line, $groups); - } - - private function string(Token $command): StringStatement - { - $variables = []; - do { - $variable = $this->consumeIdentifier('Expected a variable name in STRING.')->lexeme; - if (strcasecmp($variable, 'TO') === 0) { - $this->fail($this->previous(), 'STRING variable ranges using TO are not supported.'); - } - $variables[] = $variable; - } while ($this->check(TokenType::Identifier)); - $this->consume(TokenType::LeftParenthesis, 'Expected a width declaration in STRING.'); - $width = $this->consume(TokenType::Identifier, 'Expected a string width such as A20.')->lexeme; - if (preg_match('/\AA([1-9][0-9]*)\z/i', $width, $matches) !== 1) { - $this->fail($this->previous(), 'STRING width must use the SPSS A form.'); - } - $widthValue = (int) $matches[1]; - if ($widthValue > 32767) { - $this->fail($this->previous(), 'STRING width must be at most 32767.'); - } - $this->consume(TokenType::RightParenthesis, 'Expected ) after STRING width.'); - - return new StringStatement($command->line, $variables, $widthValue); + return new ValueLabelsStatement( + $command->line, + $groups, + SourceSpan::cover($command->span, $groups[array_key_last($groups)]->span), + ); } - private function deleteVariables(Token $command): DeleteVariablesStatement - { - $variables = []; - do { - $variable = $this->consumeIdentifier('Expected a variable name in DELETE VARIABLES.')->lexeme; - if (strcasecmp($variable, 'TO') === 0) { - $this->fail($this->previous(), 'DELETE VARIABLES ranges using TO are not supported.'); - } - $variables[] = $variable; - } while ($this->check(TokenType::Identifier)); - - return new DeleteVariablesStatement($command->line, $variables); - } private function scalar(string $message): ScalarValue { if ($this->match(TokenType::String)) { - return new ScalarValue($this->previous()->lexeme); + $token = $this->previous(); + + return new ScalarValue($token->lexeme, null, $token->span); } if (!$this->match(TokenType::Number)) { $this->fail($this->current(), $message); } $lexeme = $this->previous()->lexeme; - $value = (float) $lexeme; - if (!is_finite($value)) { - $this->fail($this->previous(), 'Numeric literals must be finite IEEE-754 binary64 values.'); + try { + DecimalBinary64::bits($lexeme); + } catch (TransformationFailure $failure) { + $this->fail($this->previous(), $failure->getMessage()); } - return new ScalarValue($value); + return new ScalarValue($lexeme, $lexeme, $this->previous()->span); } private function consumeIdentifier(string $message): Token { - return $this->consume(TokenType::Identifier, $message); + if (!$this->current()->isWord()) { + $this->fail($this->current(), $message); + } + + return $this->advance(); } private function consumeKeyword(string $keyword, string $message): Token @@ -304,8 +600,10 @@ private function previous(): Token return $this->tokens[$this->position - 1]; } - private function fail(Token $token, string $message): never + private function fail(Token $token, string $message, string $code = 'spss_syntax_error'): never { - throw new SpssSyntaxException([new Diagnostic($token->line, $token->column, $message)]); + throw new SpssSyntaxException([ + new TransformationDiagnostic($code, '$.source_text', $message, $token->span), + ]); } } diff --git a/src/Frontend/Spss/Request/InputSchema.php b/src/Frontend/Spss/Request/InputSchema.php new file mode 100644 index 0000000..c945af7 --- /dev/null +++ b/src/Frontend/Spss/Request/InputSchema.php @@ -0,0 +1,59 @@ + */ + public array $variables; + + /** @param array $variables */ + public function __construct(array $variables) + { + if ($variables === [] || !array_is_list($variables)) { + self::schema('$.input_schema.variables', 'Variables must be a non-empty ordered array.'); + } + $validated = []; + foreach ($variables as $variable) { + if (!$variable instanceof InputVariable) { + self::schema('$.input_schema.variables', 'Every schema variable must be an InputVariable.'); + } + $validated[] = $variable; + } + $this->variables = $validated; + } + + public static function fromArray(mixed $raw, string $path = '$.input_schema'): self + { + if (!is_array($raw) || array_is_list($raw)) { + self::schema($path, 'Input schema must be an object.'); + } + $keys = array_keys($raw); + sort($keys, SORT_STRING); + if ($keys !== ['variables']) { + self::schema($path, 'Object has missing or extra members.'); + } + $variables = $raw['variables']; + if (!is_array($variables) || !array_is_list($variables) || $variables === []) { + self::schema($path . '.variables', 'Variables must be a non-empty ordered array.'); + } + + return new self(array_map( + static fn(mixed $variable, int $index): InputVariable => InputVariable::fromArray( + $variable, + $path . '.variables[' . $index . ']', + ), + $variables, + array_keys($variables), + )); + } + + private static function schema(string $path, string $message): never + { + throw TransformationFailure::at('plan_schema_invalid', $path, $message); + } +} diff --git a/src/Frontend/Spss/Request/InputVariable.php b/src/Frontend/Spss/Request/InputVariable.php new file mode 100644 index 0000000..e23bbb2 --- /dev/null +++ b/src/Frontend/Spss/Request/InputVariable.php @@ -0,0 +1,213 @@ +, label: string}> $valueLabels */ + public function __construct( + public string $name, + public string $storageKind, + public ?string $variableLabel = null, + public array $valueLabels = [], + public ?string $formatFamily = null, + public ?int $width = null, + public ?int $decimals = null, + public ?string $measurementLevel = null, + ) { + self::nonEmptyString($name, '$.input_schema.variables[].name'); + if (!in_array($storageKind, ['numeric', 'string'], true)) { + self::schema('$.input_schema.variables[].storage_kind', 'Storage kind must be numeric or string.'); + } + self::nullableString($variableLabel, '$.input_schema.variables[].variable_label'); + self::nullableString($formatFamily, '$.input_schema.variables[].format_family'); + if ($width !== null && $width < 1) { + self::schema('$.input_schema.variables[].width', 'Width must be at least one.'); + } + if ($decimals !== null && $decimals < 0) { + self::schema('$.input_schema.variables[].decimals', 'Decimals must not be negative.'); + } + if (!in_array($measurementLevel, [null, 'nominal', 'ordinal', 'scale'], true)) { + self::schema('$.input_schema.variables[].measurement_level', 'Invalid measurement level.'); + } + self::valueLabels($valueLabels, '$.input_schema.variables[].value_labels'); + } + + public static function fromArray(mixed $raw, string $path): self + { + if (!is_array($raw) || array_is_list($raw)) { + self::schema($path, 'Input variable must be an object.'); + } + $allowed = [ + 'name', + 'storage_kind', + 'variable_label', + 'value_labels', + 'format_family', + 'width', + 'decimals', + 'measurement_level', + ]; + if (!array_key_exists('name', $raw) || !array_key_exists('storage_kind', $raw)) { + self::schema($path, 'Input variable is missing a required member.'); + } + foreach (array_keys($raw) as $key) { + if (!in_array($key, $allowed, true)) { + self::schema($path, 'Input variable has an extra member.'); + } + } + + $name = self::nonEmptyString($raw['name'], $path . '.name'); + $storageKind = self::string($raw['storage_kind'], $path . '.storage_kind'); + if (!in_array($storageKind, ['numeric', 'string'], true)) { + self::schema($path . '.storage_kind', 'Storage kind must be numeric or string.'); + } + $variableLabel = self::optionalNullableString($raw, 'variable_label', $path); + $formatFamily = self::optionalNullableString($raw, 'format_family', $path); + $width = self::optionalNullableInteger($raw, 'width', $path, 1); + $decimals = self::optionalNullableInteger($raw, 'decimals', $path, 0); + $measurementLevel = self::optionalNullableString($raw, 'measurement_level', $path); + if (!in_array($measurementLevel, [null, 'nominal', 'ordinal', 'scale'], true)) { + self::schema($path . '.measurement_level', 'Invalid measurement level.'); + } + + return new self( + $name, + $storageKind, + $variableLabel, + self::valueLabels($raw['value_labels'] ?? [], $path . '.value_labels'), + $formatFamily, + $width, + $decimals, + $measurementLevel, + ); + } + + /** @return list, label: string}> */ + private static function valueLabels(mixed $raw, string $path): array + { + if (!is_array($raw) || !array_is_list($raw)) { + self::schema($path, 'Value labels must be an ordered array.'); + } + $labels = []; + foreach ($raw as $index => $label) { + $labelPath = $path . '[' . $index . ']'; + if (!is_array($label) || array_is_list($label)) { + self::schema($labelPath, 'Value label must be an object.'); + } + $keys = array_keys($label); + sort($keys, SORT_STRING); + if ($keys !== ['label', 'value']) { + self::schema($labelPath, 'Value label has missing or extra members.'); + } + $labels[] = [ + 'value' => self::typedValue($label['value'], $labelPath . '.value'), + 'label' => self::string($label['label'], $labelPath . '.label'), + ]; + } + + return $labels; + } + + /** @return array */ + private static function typedValue(mixed $raw, string $path): array + { + if (!is_array($raw) || array_is_list($raw)) { + self::schema($path, 'Typed value must be an object.'); + } + $type = self::string($raw['type'] ?? null, $path . '.type'); + if ($type === 'binary64') { + self::exactKeys($raw, ['type', 'bits'], $path); + $bits = self::string($raw['bits'], $path . '.bits'); + if ( + preg_match('/\A[0-9a-f]{16}\z/D', $bits) !== 1 + || $bits === '8000000000000000' + || substr($bits, 0, 3) === '7ff' + || substr($bits, 0, 3) === 'fff' + ) { + self::schema($path . '.bits', 'Invalid canonical binary64 bits.'); + } + + return ['type' => 'binary64', 'bits' => $bits]; + } + if ($type === 'string') { + self::exactKeys($raw, ['type', 'value'], $path); + + return ['type' => 'string', 'value' => self::string($raw['value'], $path . '.value')]; + } + + self::schema($path . '.type', 'Typed value type must be binary64 or string.'); + } + + /** @param array $raw */ + private static function optionalNullableString(array $raw, string $key, string $path): ?string + { + if (!array_key_exists($key, $raw) || $raw[$key] === null) { + return null; + } + + return self::string($raw[$key], $path . '.' . $key); + } + + /** @param array $raw */ + private static function optionalNullableInteger(array $raw, string $key, string $path, int $minimum): ?int + { + if (!array_key_exists($key, $raw) || $raw[$key] === null) { + return null; + } + if (!is_int($raw[$key]) || $raw[$key] < $minimum) { + self::schema($path . '.' . $key, 'Value is outside the allowed integer range.'); + } + + return $raw[$key]; + } + + /** + * @param array $object + * @param list $expected + */ + private static function exactKeys(array $object, array $expected, string $path): void + { + $keys = array_keys($object); + sort($keys, SORT_STRING); + sort($expected, SORT_STRING); + if ($keys !== $expected) { + self::schema($path, 'Object has missing or extra members.'); + } + } + + private static function nonEmptyString(mixed $value, string $path): string + { + $string = self::string($value, $path); + if ($string === '') { + self::schema($path, 'String must not be empty.'); + } + + return $string; + } + + private static function nullableString(?string $value, string $path): void + { + if ($value !== null && preg_match('//u', $value) !== 1) { + self::schema($path, 'Value must be a valid UTF-8 string.'); + } + } + + private static function string(mixed $value, string $path): string + { + if (!is_string($value) || preg_match('//u', $value) !== 1) { + self::schema($path, 'Value must be a valid UTF-8 string.'); + } + + return $value; + } + + private static function schema(string $path, string $message): never + { + throw TransformationFailure::at('plan_schema_invalid', $path, $message); + } +} diff --git a/src/Frontend/Spss/Request/SpssFrontendRequest.php b/src/Frontend/Spss/Request/SpssFrontendRequest.php new file mode 100644 index 0000000..f2c1fe6 --- /dev/null +++ b/src/Frontend/Spss/Request/SpssFrontendRequest.php @@ -0,0 +1,84 @@ + $request */ + public static function fromArray(array $request): self + { + self::exactKeys($request, ['contract', 'input_alias', 'input_schema', 'source_text'], '$'); + $contract = self::string($request['contract'], '$.contract'); + $inputAlias = self::nonEmptyString($request['input_alias'], '$.input_alias'); + $sourceText = self::nonEmptyString($request['source_text'], '$.source_text'); + + return new self( + $contract, + $inputAlias, + InputSchema::fromArray($request['input_schema'], '$.input_schema'), + $sourceText, + ); + } + + public function sourceHash(): string + { + return hash('sha256', str_replace(["\r\n", "\r"], "\n", $this->sourceText)); + } + + /** + * @param array $object + * @param list $expected + */ + private static function exactKeys(array $object, array $expected, string $path): void + { + $actual = array_keys($object); + sort($actual, SORT_STRING); + sort($expected, SORT_STRING); + if ($actual !== $expected) { + self::schema($path, 'Object has missing or extra members.'); + } + } + + private static function nonEmptyString(mixed $value, string $path): string + { + $string = self::string($value, $path); + if ($string === '') { + self::schema($path, 'String must not be empty.'); + } + + return $string; + } + + private static function string(mixed $value, string $path): string + { + if (!is_string($value) || preg_match('//u', $value) !== 1) { + self::schema($path, 'Value must be a valid UTF-8 string.'); + } + + return $value; + } + + private static function schema(string $path, string $message): never + { + throw TransformationFailure::at('plan_schema_invalid', $path, $message); + } +} diff --git a/src/Frontend/Spss/SpssCompilationResult.php b/src/Frontend/Spss/SpssCompilationResult.php new file mode 100644 index 0000000..561d70b --- /dev/null +++ b/src/Frontend/Spss/SpssCompilationResult.php @@ -0,0 +1,15 @@ +parser->parse($source); } - public function bind(string $datasetId, Program $program): BoundProgram + public function bind(string $inputAlias, InputSchema $inputSchema, Program $program): BoundProgram { - return $this->binder->bind($datasetId, $program); + return $this->binder->bind($inputAlias, $inputSchema, $program); } - public function compile(string $source, string $datasetId): TransformationPlan + public function compile(SpssFrontendRequest $request): SpssCompilationResult { - return $this->compiler->compile($this->binder->bind($datasetId, $this->parser->parse($source))); - } + $sourceHash = $request->sourceHash(); + $program = $this->parser->parse($request->sourceText); + $bound = $this->binder->bind($request->inputAlias, $request->inputSchema, $program); - public function compileForDataset(string $datasetId, string $source): TransformationPlan - { - return $this->compile($source, $datasetId); + return new SpssCompilationResult($this->compiler->compile($bound), $sourceHash); } } diff --git a/src/Frontend/Spss/SpssSyntaxException.php b/src/Frontend/Spss/SpssSyntaxException.php index 58a9c63..0e1de13 100644 --- a/src/Frontend/Spss/SpssSyntaxException.php +++ b/src/Frontend/Spss/SpssSyntaxException.php @@ -4,16 +4,34 @@ namespace OpenStatSpec\Frontend\Spss; -use InvalidArgumentException; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Diagnostic\TransformationDiagnostic; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; -final class SpssSyntaxException extends InvalidArgumentException +final class SpssSyntaxException extends TransformationFailure { /** - * @param non-empty-list $diagnostics + * Plain {@see Diagnostic} entries do not carry a diagnostic code and are + * always wrapped as `spss_syntax_error`; callers that need a more + * specific code must construct a {@see TransformationDiagnostic} + * directly and pass it here. {@see TransformationDiagnostic} entries + * are forwarded verbatim, so the exception never downgrades a code + * supplied by the caller. + * + * @param non-empty-list $diagnostics */ - public function __construct(public readonly array $diagnostics) + public function __construct(array $diagnostics) { - $first = $diagnostics[0]; - parent::__construct(sprintf('SPSS syntax error at %d:%d: %s', $first->line, $first->column, $first->message)); + parent::__construct(array_map( + static fn(Diagnostic|TransformationDiagnostic $diagnostic): TransformationDiagnostic => $diagnostic instanceof TransformationDiagnostic + ? $diagnostic + : new TransformationDiagnostic( + 'spss_syntax_error', + '$.source_text', + $diagnostic->message, + new SourceSpan(0, 0, $diagnostic->line, $diagnostic->column, $diagnostic->line, $diagnostic->column), + ), + $diagnostics, + )); } } diff --git a/src/Frontend/Spss/Token.php b/src/Frontend/Spss/Token.php index 5a68650..cc3711a 100644 --- a/src/Frontend/Spss/Token.php +++ b/src/Frontend/Spss/Token.php @@ -4,17 +4,51 @@ namespace OpenStatSpec\Frontend\Spss; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class Token { + public SourceSpan $span; + public function __construct( public TokenType $type, public string $lexeme, public int $line, public int $column, - ) {} + int $startOffset, + int $endOffset, + int $endLine, + int $endColumn, + ) { + $this->span = new SourceSpan( + $startOffset, + $endOffset, + $line, + $column, + $endLine, + $endColumn, + ); + } public function isKeyword(string $keyword): bool { - return $this->type === TokenType::Identifier && strcasecmp($this->lexeme, $keyword) === 0; + return $this->isWord() && strcasecmp($this->lexeme, $keyword) === 0; + } + + public function isWord(): bool + { + return in_array($this->type, [ + TokenType::Identifier, + TokenType::Compute, + TokenType::If, + TokenType::And, + TokenType::Or, + TokenType::Formats, + TokenType::Variable, + TokenType::Level, + TokenType::Nominal, + TokenType::Ordinal, + TokenType::Scale, + ], true); } } diff --git a/src/Frontend/Spss/TokenType.php b/src/Frontend/Spss/TokenType.php index 46beb41..f369b98 100644 --- a/src/Frontend/Spss/TokenType.php +++ b/src/Frontend/Spss/TokenType.php @@ -7,11 +7,26 @@ enum TokenType { case Identifier; + case Compute; + case If; + case And; + case Or; + case Formats; + case Variable; + case Level; + case Nominal; + case Ordinal; + case Scale; case Number; case String; case LeftParenthesis; case RightParenthesis; case Equals; + case LessThan; + case LessThanOrEqual; + case GreaterThan; + case GreaterThanOrEqual; + case ArithmeticOperator; case Comma; case Slash; case Terminator; diff --git a/src/Spss/SpssAdapter.php b/src/Spss/SpssAdapter.php index aec05a5..c0b71e3 100644 --- a/src/Spss/SpssAdapter.php +++ b/src/Spss/SpssAdapter.php @@ -23,6 +23,7 @@ use OpenStatSpec\Sql\PostgreSqlSchema; use OpenStatSpec\Sql\SqliteWideTableExporter; use OpenStatSpec\Sql\SqliteWideTableImporter; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use PDO; use Throwable; @@ -67,6 +68,7 @@ public function migrateCatalog(): void }; $catalog = new NormativeCatalog($this->connection->pdo); $catalog->createTables(); + (new TransformationAuditMigrator($this->connection->pdo))->migrate(); $this->backfillLegacyDatasets($catalog); CatalogOwnership::markCurrentVersion($this->connection->pdo); } diff --git a/src/Sql/CatalogOwnership.php b/src/Sql/CatalogOwnership.php index 3119e45..8960adf 100644 --- a/src/Sql/CatalogOwnership.php +++ b/src/Sql/CatalogOwnership.php @@ -15,7 +15,8 @@ final class CatalogOwnership private const IDENTITY_TABLE = 'catalog_identity'; private const CONTRACT_ID = 'openstatspec-strict-wide-table-v1'; private const MIGRATION_TABLE = 'openstatspec_schema_migration'; - private const SCHEMA_VERSION = 3; + private const AUDIT_TABLE = 'transformation_apply'; + private const SCHEMA_VERSION = 4; /** @return array */ public static function binding(PDO $pdo): array @@ -93,7 +94,7 @@ public static function ensure(PDO $pdo): void public static function assertReadyForUse(PDO $pdo): void { if (self::tableExists($pdo, self::IDENTITY_TABLE)) { - if (self::validateIdentity($pdo) < self::SCHEMA_VERSION) { + if (self::validateIdentity($pdo) < self::SCHEMA_VERSION || !self::currentMigrationComplete($pdo)) { throw self::migrationRequired(); } self::assertExclusiveNamespace($pdo); @@ -121,6 +122,20 @@ public static function assertReadyForUse(PDO $pdo): void throw self::migrationRequired(); } + /** Transformation preflight must never initialize a fresh namespace. */ + public static function assertReadyForUseReadOnly(PDO $pdo): void + { + if (!self::tableExists($pdo, self::IDENTITY_TABLE) + && !self::tableExists($pdo, self::MIGRATION_TABLE) + && self::catalogCollisions($pdo) === [] + && self::namespaceObjects($pdo) === [] + ) { + throw self::migrationRequired(); + } + + self::assertReadyForUse($pdo); + } + public static function isFreshPending(PDO $pdo): bool { if (!self::tableExists($pdo, self::IDENTITY_TABLE) @@ -148,10 +163,32 @@ public static function isFreshPending(PDO $pdo): bool public static function markCurrentVersion(PDO $pdo): void { self::validateIdentity($pdo); + if (!self::currentMigrationComplete($pdo)) { + throw self::migrationRequired(); + } $statement = $pdo->prepare('UPDATE ' . self::IDENTITY_TABLE . ' SET schema_version = ? WHERE catalog_identity_key = 1'); $statement->execute([self::SCHEMA_VERSION]); } + private static function currentMigrationComplete(PDO $pdo): bool + { + return self::tableExists($pdo, self::MIGRATION_TABLE) + && self::tableExists($pdo, self::AUDIT_TABLE) + && self::validateLegacyMarker($pdo) === self::SCHEMA_VERSION + && self::canSelectColumns($pdo, self::AUDIT_TABLE, self::auditColumns()); + } + + /** @return list */ + private static function auditColumns(): array + { + return [ + 'apply_id', 'contract_id', 'database_profile', 'dataset_id', 'physical_table_schema', + 'physical_table_name', 'source_hash', 'plan_hash', 'canonical_plan_json', 'actor', + 'status', 'dolt_branch', 'dolt_head_before', 'dolt_head_after', 'operation_count', + 'started_at', 'completed_at', + ]; + } + /** @return array */ private static function bindingResult(string $mode, ?string $name, ?string $locator, bool $verified): array { @@ -235,15 +272,19 @@ private static function isRecognizedMarkerCatalog(PDO $pdo, array $collisions, i $canonicalComplete = $canonical !== [] && self::matchesCompleteDefinition($pdo, $canonical, $canonicalDefinition); $legacyComplete = $legacy !== [] && self::matchesCompleteDefinition($pdo, $legacy, $legacyDefinition); $journalsComplete = $journals === [] || self::isRecognizedJournalOnlyLegacyCatalog($pdo, $journals); + $auditPresent = in_array(self::AUDIT_TABLE, $collisions, true); + $auditComplete = !$auditPresent || self::canSelectColumns($pdo, self::AUDIT_TABLE, self::auditColumns()); if (($canonical !== [] && !$canonicalComplete) || ($legacy !== [] && !$legacyComplete) || !$journalsComplete + || !$auditComplete ) { return false; } - return $markerVersion === self::SCHEMA_VERSION - ? $canonicalComplete - : ($canonicalComplete || $legacyComplete); + if ($markerVersion >= 3) { + return $canonicalComplete && ($markerVersion < self::SCHEMA_VERSION || $auditPresent); + } + return $canonicalComplete || $legacyComplete; } /** @param list $collisions */ @@ -707,6 +748,7 @@ private static function catalogTables(): array 'variable_value_label_set', 'missing_rule', 'dataset_attribute', 'variable_attribute', 'document', 'variable_set', 'variable_set_member', 'multiple_response_set', 'multiple_response_member', 'operation', 'fidelity_event', 'datasets', 'variables', + 'transformation_apply', 'dataset_weight_variables', 'dataset_metadata', 'file_technical_metadata', 'documents', 'value_labels', 'missing_rules', 'missing_rule_values', 'variable_display_metadata', 'variable_roles', 'file_attributes', 'variable_attributes', 'variable_sets', diff --git a/src/Sql/MySqlProfile.php b/src/Sql/MySqlProfile.php index 77b81a8..9d10dc6 100644 --- a/src/Sql/MySqlProfile.php +++ b/src/Sql/MySqlProfile.php @@ -6,9 +6,12 @@ use OpenStatSpec\Core\ServerVersionPolicy; use PDO; +use PDOException; class MySqlProfile extends AbstractPdoSqlProfile { + private ?bool $lowerCaseTableNames = null; + public function driverName(): string { return 'mysql'; @@ -105,4 +108,23 @@ private function packetPayloadBytes(PDO $pdo): ?int // and halve the remainder for worst-case emulated-prepare escaping. return intdiv(max(0, (int) $packet - 131_072), 2); } + + /** + * Reports whether the server treats physical table names case-insensitively + * (lower_case_table_names=1 or =2). Cached per profile instance because the + * setting is server-static for the lifetime of the connection. + */ + public function lowerCaseTableNames(PDO $pdo): bool + { + if ($this->lowerCaseTableNames !== null) { + return $this->lowerCaseTableNames; + } + try { + $statement = $pdo->query('SELECT @@lower_case_table_names'); + $value = $statement === false ? false : $statement->fetchColumn(); + } catch (PDOException) { + $value = false; + } + return $this->lowerCaseTableNames = is_numeric($value) && (int) $value !== 0; + } } diff --git a/src/Sql/NormativeCatalog.php b/src/Sql/NormativeCatalog.php index 6fdd8ed..f327598 100644 --- a/src/Sql/NormativeCatalog.php +++ b/src/Sql/NormativeCatalog.php @@ -65,21 +65,34 @@ public function createTables(): void $this->enforceSetOrdinalConstraints($driver); } - $migration = match ($driver) { - 'mysql' => 'INSERT IGNORE INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?)', - 'pgsql', 'sqlite' => 'INSERT INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?) ON CONFLICT (version) DO NOTHING', - default => null, - }; - if ($migration !== null) { + if (in_array($driver, ['mysql', 'pgsql', 'sqlite'], true)) { foreach ([1, 2] as $version) { - $this->pdo->prepare($migration)->execute([$version, self::timestamp()]); + $this->recordMigration($version); } if ($migrationV3Required) { - $this->pdo->prepare($migration)->execute([3, self::timestamp()]); + $this->recordMigration(3); } } } + /** Record one completed normative-catalogue migration idempotently. */ + private function recordMigration(int $version): void + { + if ($version < 1 || $version > 3) { + throw new \InvalidArgumentException('A normative catalogue migration version must be between 1 and 3.'); + } + $driver = (string) $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME); + $migration = match ($driver) { + 'mysql' => 'INSERT IGNORE INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?)', + 'pgsql', 'sqlite' => 'INSERT INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?) ON CONFLICT (version) DO NOTHING', + default => throw new UnsupportedOperation( + DiagnosticCode::UnsupportedSqlDriver, + 'The active SQL profile cannot record catalogue migrations.', + ), + }; + $this->statement($migration)->execute([$version, self::timestamp()]); + } + private function migrationApplied(int $version): bool { $statement = $this->statement('SELECT 1 FROM openstatspec_schema_migration WHERE version = ?'); diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php new file mode 100644 index 0000000..3f113e0 --- /dev/null +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -0,0 +1,348 @@ +pdo->inTransaction()) { + throw new \LogicException('Transformation audit migration cannot run inside an apply transaction.'); + } + + match ((string) $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME)) { + 'sqlite' => $this->migrateSqlite(), + 'pgsql' => $this->migratePostgreSql(), + 'mysql' => $this->migrateMySqlFamily(), + default => throw new UnsupportedOperation( + DiagnosticCode::UnsupportedSqlDriver, + 'The active SQL profile has no transformation audit migration.', + ), + }; + } + + private function migrateSqlite(): void + { + $this->pdo->beginTransaction(); + try { + if (!$this->tableExists('sqlite')) { + $this->pdo->exec($this->createTableSql(self::TABLE, 'sqlite')); + } elseif (!$this->sqliteAcceptsVersion02()) { + $this->assertReadableAuditTable(); + $this->pdo->exec($this->createTableSql(self::SQLITE_REPLACEMENT, 'sqlite')); + $columns = implode(', ', self::columns()); + $this->pdo->exec( + 'INSERT INTO ' . self::SQLITE_REPLACEMENT . ' (' . $columns . ') ' + . 'SELECT ' . $columns . ' FROM ' . self::TABLE, + ); + $this->pdo->exec('DROP TABLE ' . self::TABLE); + $this->pdo->exec('ALTER TABLE ' . self::SQLITE_REPLACEMENT . ' RENAME TO ' . self::TABLE); + } else { + $this->assertReadableAuditTable(); + } + + $violations = $this->pdo->query('PRAGMA foreign_key_check'); + if ($violations === false || $violations->fetchColumn() !== false) { + throw new PDOException('Transformation audit migration produced a foreign-key violation.'); + } + $this->recordMigration(); + $this->pdo->commit(); + } catch (Throwable $exception) { + if ($this->pdo->inTransaction()) { + $this->pdo->rollBack(); + } + throw $exception; + } + } + + private function migratePostgreSql(): void + { + $this->pdo->beginTransaction(); + try { + if (!$this->tableExists('pgsql')) { + $this->pdo->exec($this->createTableSql(self::TABLE, 'pgsql')); + } else { + $this->assertReadableAuditTable(); + $checks = $this->postgreSqlContractChecks(); + if (!$this->checksAreCurrent($checks)) { + if ($checks === []) { + throw new PDOException('The existing transformation audit has no contract check to migrate.'); + } + foreach (array_keys($checks) as $constraint) { + $this->pdo->exec('ALTER TABLE ' . self::TABLE . ' DROP CONSTRAINT ' . $this->quotePostgreSql($constraint)); + } + $this->pdo->exec( + 'ALTER TABLE ' . self::TABLE . ' ADD CONSTRAINT chk_transformation_apply_contract ' + . $this->contractCheck(), + ); + } + } + $this->recordMigration(); + $this->pdo->commit(); + } catch (Throwable $exception) { + if ($this->pdo->inTransaction()) { + $this->pdo->rollBack(); + } + throw $exception; + } + } + + private function migrateMySqlFamily(): void + { + if (!$this->tableExists('mysql')) { + $this->pdo->exec($this->createTableSql(self::TABLE, 'mysql')); + } elseif (!$this->mySqlAcceptsVersion02()) { + $this->assertReadableAuditTable(); + $this->migrateMySqlFamilyByRename(); + } else { + $this->assertReadableAuditTable(); + } + $this->recordMigration(); + } + + /** + * Rebuild the audit table alongside its current name and use a single + * multi-table RENAME to swap the new version into place atomically. + * + * MySQL/MariaDB implicitly commit DDL, so the in-place ALTER TABLE + * DROP CHECK / ADD CONSTRAINT pair cannot be wrapped in a transaction + * and leaves the audit table in a mixed state if the second statement + * fails after the first succeeds. RENAME TABLE with two operands is + * atomic: every concurrent reader sees exactly one of the old or the + * new schema, never an inconsistent mixture. + * + * If any step before the atomic swap fails, the original table keeps + * its old schema and a subsequent migrate() call retries from a clean + * state. + */ + private function migrateMySqlFamilyByRename(): void + { + $staging = self::TABLE . '_v04_pending'; + $archive = self::TABLE . '_pre_v04_archive'; + $columns = implode(', ', self::columns()); + $constraintSuffix = '_v04_' . bin2hex(random_bytes(4)); + + // Drop any orphan archive table from a previous failed migration so + // the RENAME TABLE below has a free target name. This fires only + // when migrateMySqlFamilyByRename() actually runs, i.e. when a + // previous attempt reached the RENAME step but crashed before + // its DROP. A clean v0.4 install takes the short-circuit branch in + // migrateMySqlFamily() and never reaches this line. + $this->pdo->exec('DROP TABLE IF EXISTS ' . $archive); + // A failure after staging CREATE but before the swap can leave the + // pending table behind; remove it before rebuilding from the source. + $this->pdo->exec('DROP TABLE IF EXISTS ' . $staging); + + $this->pdo->exec($this->createTableSql($staging, 'mysql', $constraintSuffix)); + $this->pdo->exec( + 'INSERT INTO ' . $staging . ' (' . $columns . ') ' + . 'SELECT ' . $columns . ' FROM ' . self::TABLE, + ); + $this->pdo->exec( + 'RENAME TABLE ' . self::TABLE . ' TO ' . $archive . ', ' . $staging . ' TO ' . self::TABLE, + ); + $this->pdo->exec('DROP TABLE ' . $archive); + } + + private function createTableSql(string $table, string $driver, string $constraintSuffix = ''): string + { + $applyUuid = $driver === 'pgsql' ? 'UUID' : 'VARCHAR(36)'; + $text = $driver === 'mysql' ? 'LONGTEXT' : 'TEXT'; + $constraint = static fn(string $name): string => $name . $constraintSuffix; + + return 'CREATE TABLE ' . $table . ' (' + . 'apply_id ' . $applyUuid . ' NOT NULL PRIMARY KEY, ' + . 'contract_id ' . $text . ' NOT NULL, ' + . 'database_profile ' . $text . ' NOT NULL, ' + . 'dataset_id VARCHAR(36) NOT NULL, ' + . 'physical_table_schema ' . $text . ' NULL, ' + . 'physical_table_name ' . $text . ' NOT NULL, ' + . 'source_hash CHAR(64) NOT NULL, ' + . 'plan_hash CHAR(64) NOT NULL, ' + . 'canonical_plan_json ' . $text . ' NOT NULL, ' + . 'actor ' . $text . ' NOT NULL, ' + . 'status ' . $text . ' NOT NULL, ' + . 'dolt_branch ' . $text . ' NULL, ' + . 'dolt_head_before ' . $text . ' NULL, ' + . 'dolt_head_after ' . $text . ' NULL, ' + . 'operation_count INTEGER NOT NULL, ' + . 'started_at TIMESTAMP NOT NULL, ' + . 'completed_at TIMESTAMP NOT NULL, ' + . 'CONSTRAINT ' . $constraint('fk_transformation_apply_dataset') . ' FOREIGN KEY (dataset_id) REFERENCES dataset(dataset_id), ' + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_contract') . ' ' . $this->contractCheck() . ', ' + . "CONSTRAINT " . $constraint('chk_transformation_apply_profile') . " CHECK (database_profile IN ('sqlite', 'postgresql', 'mysql', 'mariadb', 'dolt')), " + . "CONSTRAINT " . $constraint('chk_transformation_apply_status') . " CHECK (status IN ('succeeded', 'failed')), " + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_operation_count') . ' CHECK (operation_count > 0), ' + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_dolt') . ' CHECK (' + . "(database_profile <> 'dolt' AND dolt_branch IS NULL AND dolt_head_before IS NULL AND dolt_head_after IS NULL) " + . "OR (database_profile = 'dolt' AND dolt_branch IS NOT NULL AND dolt_head_before IS NOT NULL " + . "AND (status = 'failed' OR dolt_head_after = dolt_head_before))), " + . $this->hashChecks($driver, $constraintSuffix) + . ')'; + } + + private function contractCheck(): string + { + return "CHECK (contract_id IN ('openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'))"; + } + + private function hashChecks(string $driver, string $constraintSuffix = ''): string + { + $constraint = static fn(string $name): string => $name . $constraintSuffix; + if ($driver === 'sqlite') { + return 'CONSTRAINT ' . $constraint('chk_transformation_apply_source_hash') . " CHECK (length(source_hash) = 64 AND source_hash NOT GLOB '*[^0-9a-f]*'), " + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_plan_hash') . " CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*')"; + } + if ($driver === 'pgsql') { + return 'CONSTRAINT ' . $constraint('chk_transformation_apply_source_hash') . " CHECK (source_hash ~ '^[0-9a-f]{64}$'), " + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_plan_hash') . " CHECK (plan_hash ~ '^[0-9a-f]{64}$')"; + } + // MySQL/MariaDB: CAST(... AS BINARY) is the binary collation cast; the BINARY + // keyword as a type modifier is deprecated in MySQL 8.0.17+ in favor of + // COLLATE ... USING BINARY, but the cast form remains supported on every + // release line the adapter claims and reads more obviously to reviewers. + return 'CONSTRAINT ' . $constraint('chk_transformation_apply_source_hash') . " CHECK (CHAR_LENGTH(source_hash) = 64 AND source_hash REGEXP '^[0-9a-f]{64}$' AND CAST(source_hash AS BINARY) = CAST(LOWER(source_hash) AS BINARY)), " + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_plan_hash') . " CHECK (CHAR_LENGTH(plan_hash) = 64 AND plan_hash REGEXP '^[0-9a-f]{64}$' AND CAST(plan_hash AS BINARY) = CAST(LOWER(plan_hash) AS BINARY))"; + } + + private function sqliteAcceptsVersion02(): bool + { + $statement = $this->pdo->query("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = '" . self::TABLE . "'"); + $sql = $statement === false ? false : $statement->fetchColumn(); + return is_string($sql) + && str_contains($sql, 'openstatspec-in-place-transformation-v0.1') + && str_contains($sql, 'openstatspec-in-place-transformation-v0.2'); + } + + /** @return array */ + private function postgreSqlContractChecks(): array + { + $statement = $this->pdo->query(<<<'SQL' +SELECT constraint_metadata.conname, pg_get_constraintdef(constraint_metadata.oid) +FROM pg_constraint constraint_metadata +JOIN pg_class relation ON relation.oid = constraint_metadata.conrelid +JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace +WHERE namespace.nspname = current_schema() + AND relation.relname = 'transformation_apply' + AND constraint_metadata.contype = 'c' + AND pg_get_constraintdef(constraint_metadata.oid) LIKE '%contract_id%' +SQL); + if ($statement === false) { + throw new PDOException('Could not inspect the PostgreSQL transformation audit checks.'); + } + /** @var array $checks */ + $checks = $statement->fetchAll(PDO::FETCH_KEY_PAIR); + return $checks; + } + + /** @return array */ + private function mySqlContractChecks(): array + { + $statement = $this->pdo->query(<<<'SQL' +SELECT table_constraint.constraint_name, check_constraint.check_clause +FROM information_schema.table_constraints table_constraint +JOIN information_schema.check_constraints check_constraint + ON check_constraint.constraint_schema = table_constraint.constraint_schema + AND check_constraint.constraint_name = table_constraint.constraint_name +WHERE table_constraint.constraint_schema = DATABASE() + AND table_constraint.table_name = 'transformation_apply' + AND table_constraint.constraint_type = 'CHECK' + AND check_constraint.check_clause LIKE '%contract_id%' +SQL); + if ($statement === false) { + throw new PDOException('Could not inspect the MySQL-family transformation audit checks.'); + } + /** @var array $checks */ + $checks = $statement->fetchAll(PDO::FETCH_KEY_PAIR); + return $checks; + } + + private function mySqlAcceptsVersion02(): bool + { + foreach ($this->mySqlContractChecks() as $definition) { + if (str_contains($definition, 'openstatspec-in-place-transformation-v0.1') + && str_contains($definition, 'openstatspec-in-place-transformation-v0.2') + ) { + return true; + } + } + return false; + } + + /** @param array $checks */ + private function checksAreCurrent(array $checks): bool + { + if ($checks === []) { + return false; + } + foreach ($checks as $definition) { + if (!str_contains($definition, 'openstatspec-in-place-transformation-v0.1') + || !str_contains($definition, 'openstatspec-in-place-transformation-v0.2') + ) { + return false; + } + } + return true; + } + + private function recordMigration(): void + { + $migration = match ((string) $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME)) { + 'mysql' => 'INSERT IGNORE INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?)', + 'pgsql', 'sqlite' => 'INSERT INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?) ON CONFLICT (version) DO NOTHING', + default => throw new \LogicException('Unsupported audit migration driver.'), + }; + $this->pdo->prepare($migration)->execute([4, gmdate('Y-m-d H:i:s')]); + } + + private function assertReadableAuditTable(): void + { + $statement = $this->pdo->query('SELECT ' . implode(', ', self::columns()) . ' FROM ' . self::TABLE . ' WHERE 1 = 0'); + if ($statement === false) { + throw new PDOException('The existing transformation audit table is not readable.'); + } + } + + private function tableExists(string $driver): bool + { + $sql = match ($driver) { + 'sqlite' => "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", + 'pgsql' => "SELECT 1 FROM information_schema.tables WHERE table_schema = current_schema() AND table_type = 'BASE TABLE' AND table_name = ?", + 'mysql' => "SELECT 1 FROM information_schema.tables WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE' AND table_name = ?", + default => throw new \LogicException('Unsupported audit migration driver.'), + }; + $statement = $this->pdo->prepare($sql); + $statement->execute([self::TABLE]); + return $statement->fetchColumn() !== false; + } + + private function quotePostgreSql(string $identifier): string + { + return '"' . str_replace('"', '""', $identifier) . '"'; + } + + /** @return list */ + private static function columns(): array + { + return [ + 'apply_id', 'contract_id', 'database_profile', 'dataset_id', 'physical_table_schema', + 'physical_table_name', 'source_hash', 'plan_hash', 'canonical_plan_json', 'actor', + 'status', 'dolt_branch', 'dolt_head_before', 'dolt_head_after', 'operation_count', + 'started_at', 'completed_at', + ]; + } +} diff --git a/src/Transformation/Audit/TransformationAuditWriter.php b/src/Transformation/Audit/TransformationAuditWriter.php new file mode 100644 index 0000000..f69fc19 --- /dev/null +++ b/src/Transformation/Audit/TransformationAuditWriter.php @@ -0,0 +1,156 @@ +codec = $codec ?? new PlanCodec(); + } + + public function isFor(PDO $pdo): bool + { + return $this->pdo === $pdo; + } + + public function succeed( + InPlaceApplyRequest $request, + DatasetBinding $dataset, + string $databaseProfile, + ?DoltEvidence $before, + ?DoltEvidence $after, + string $startedAt, + ): string { + if (!$this->pdo->inTransaction()) { + throw new \LogicException('A successful transformation audit must be written inside the open apply transaction.'); + } + if (!in_array($databaseProfile, ['sqlite', 'postgresql', 'mysql', 'mariadb', 'dolt'], true)) { + throw new \InvalidArgumentException('Unsupported transformation audit database profile.'); + } + if ($dataset->datasetId !== $request->datasetId) { + throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply request and target binding identify different datasets.'); + } + if (!self::isTimestamp($startedAt)) { + throw new \InvalidArgumentException('The transformation audit started_at must be a UTC "Y-m-d H:i:s" timestamp.'); + } + + $this->assertDatasetIdentity($dataset); + [$branch, $headBefore, $headAfter] = $this->doltFields($request, $databaseProfile, $before, $after); + $applyId = self::uuidV4(); + $completedAt = gmdate('Y-m-d H:i:s'); + $contract = $request->plan->contract === PlanContract::V01 + ? 'openstatspec-in-place-transformation-v0.1' + : 'openstatspec-in-place-transformation-v0.2'; + + $statement = CheckedPdo::prepare($this->pdo, <<<'SQL' +INSERT INTO transformation_apply ( + apply_id, contract_id, database_profile, dataset_id, physical_table_schema, + physical_table_name, source_hash, plan_hash, canonical_plan_json, actor, + status, dolt_branch, dolt_head_before, dolt_head_after, operation_count, + started_at, completed_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +SQL, 'The transformation audit insert could not be prepared.'); + CheckedPdo::execute($statement, [ + $applyId, + $contract, + $databaseProfile, + $dataset->datasetId, + $dataset->schema, + $dataset->table, + $request->sourceHash, + $this->codec->hash($request->plan), + $this->codec->canonicalJson($request->plan), + $request->actor, + 'succeeded', + $branch, + $headBefore, + $headAfter, + count($request->plan->operations), + $startedAt, + $completedAt, + ], 'The transformation audit insert could not be executed.'); + + return $applyId; + } + + private function assertDatasetIdentity(DatasetBinding $dataset): void + { + $statement = CheckedPdo::prepare( + $this->pdo, + 'SELECT physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', + 'The audit target dataset identity could not be prepared.', + ); + CheckedPdo::execute($statement, [$dataset->datasetId], 'The audit target dataset identity could not be queried.'); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + if (count($rows) !== 1 + || ($rows[0]['physical_table_schema'] ?? null) !== $dataset->schema + || ($rows[0]['physical_table_name'] ?? null) !== $dataset->table + ) { + throw TransformationFailure::at( + 'invalid_dataset_id', + '$.dataset_id', + 'Apply target identity no longer matches the normative dataset row.', + ); + } + } + + /** @return array{string|null, string|null, string|null} */ + private function doltFields( + InPlaceApplyRequest $request, + string $databaseProfile, + ?DoltEvidence $before, + ?DoltEvidence $after, + ): array { + if ($databaseProfile !== 'dolt') { + if ($before !== null || $after !== null) { + throw new \InvalidArgumentException('Non-Dolt audit rows cannot contain Dolt evidence.'); + } + return [null, null, null]; + } + if ($request->expectedBranch === null + || $request->expectedHead === null + || $before === null + || $after === null + ) { + throw TransformationFailure::at('dolt_context_required', '$.expected_context', 'Dolt apply audit requires complete context evidence.'); + } + if ($before->branch() !== $request->expectedBranch + || $before->head() !== $request->expectedHead + || $after->branch() !== $before->branch() + || $after->head() !== $before->head() + ) { + throw TransformationFailure::at('dolt_context_changed', '$.expected_context', 'Dolt context changed before the success audit.'); + } + return [$before->branch(), $before->head(), $after->head()]; + } + + private static function uuidV4(): string + { + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80); + $hex = bin2hex($bytes); + return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-' . substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-' . substr($hex, 20); + } + + private static function isTimestamp(string $value): bool + { + $parsed = \DateTimeImmutable::createFromFormat('!Y-m-d H:i:s', $value, new \DateTimeZone('UTC')); + return $parsed !== false && $parsed->format('Y-m-d H:i:s') === $value; + } +} diff --git a/src/Transformation/Diagnostic/SourceSpan.php b/src/Transformation/Diagnostic/SourceSpan.php new file mode 100644 index 0000000..ca405af --- /dev/null +++ b/src/Transformation/Diagnostic/SourceSpan.php @@ -0,0 +1,39 @@ + $endOffset) { + throw new \InvalidArgumentException('SourceSpan start offset must not exceed its end offset.'); + } + if ($startLine > $endLine || ($startLine === $endLine && $startColumn > $endColumn)) { + throw new \InvalidArgumentException('SourceSpan start position must not exceed its end position.'); + } + } + + public static function cover(self $start, self $end): self + { + if ($start->startOffset > $end->endOffset) { + throw new \InvalidArgumentException('SourceSpan::cover() requires the end span to start at or after the start span.'); + } + return new self( + $start->startOffset, + $end->endOffset, + $start->startLine, + $start->startColumn, + $end->endLine, + $end->endColumn, + ); + } +} diff --git a/src/Transformation/Diagnostic/TransformationDiagnostic.php b/src/Transformation/Diagnostic/TransformationDiagnostic.php new file mode 100644 index 0000000..04d8758 --- /dev/null +++ b/src/Transformation/Diagnostic/TransformationDiagnostic.php @@ -0,0 +1,15 @@ + */ + public readonly array $diagnostics; + + /** @param non-empty-list $diagnostics */ + public function __construct(array $diagnostics) + { + $this->diagnostics = $diagnostics; + + parent::__construct($diagnostics[0]->message); + } + + public static function at(string $code, string $path, string $message, ?SourceSpan $span = null): self + { + return new self([new TransformationDiagnostic($code, $path, $message, $span)]); + } + + public function diagnosticCode(): string + { + return $this->diagnostics[0]->code; + } +} diff --git a/src/Transformation/Execution/BoundOperation.php b/src/Transformation/Execution/BoundOperation.php new file mode 100644 index 0000000..1daa902 --- /dev/null +++ b/src/Transformation/Execution/BoundOperation.php @@ -0,0 +1,23 @@ +target !== null && !$this->target->persisted; + } +} diff --git a/src/Transformation/Execution/BoundPlan.php b/src/Transformation/Execution/BoundPlan.php new file mode 100644 index 0000000..5cc265a --- /dev/null +++ b/src/Transformation/Execution/BoundPlan.php @@ -0,0 +1,19 @@ + $operations */ + public function __construct( + public TransformationPlan $plan, + public DatasetBinding $dataset, + public array $operations, + public BoundSchema $finalSchema, + ) {} +} diff --git a/src/Transformation/Execution/BoundSchema.php b/src/Transformation/Execution/BoundSchema.php new file mode 100644 index 0000000..a6ffa47 --- /dev/null +++ b/src/Transformation/Execution/BoundSchema.php @@ -0,0 +1,24 @@ + $variables */ + public function __construct(public array $variables) {} + + public function variable(string $name, string $path = '$.variable'): VariableBinding + { + $variable = $this->variables[$name] ?? null; + if ($variable === null) { + throw TransformationFailure::at('unknown_variable', $path, 'Plan variable is not registered in the bound dataset.'); + } + + return $variable; + } +} diff --git a/src/Transformation/Execution/CheckedPdo.php b/src/Transformation/Execution/CheckedPdo.php new file mode 100644 index 0000000..8f17963 --- /dev/null +++ b/src/Transformation/Execution/CheckedPdo.php @@ -0,0 +1,82 @@ +prepare($sql); + if ($statement === false) { + throw self::failure($message, $pdo->errorInfo()); + } + + return $statement; + } + + /** @param list $parameters */ + public static function execute(PDOStatement $statement, array $parameters, string $message): void + { + if (!$statement->execute($parameters)) { + throw self::failure($message, $statement->errorInfo()); + } + } + + public static function exec(PDO $pdo, string $sql, string $message): int + { + $affected = $pdo->exec($sql); + if ($affected === false) { + throw self::failure($message, $pdo->errorInfo()); + } + + return $affected; + } + + public static function begin(PDO $pdo): void + { + if (!$pdo->beginTransaction()) { + throw self::failure('The SQL driver did not start the transformation transaction.', $pdo->errorInfo()); + } + } + + public static function commit(PDO $pdo): void + { + if (!$pdo->commit()) { + throw self::failure('The SQL driver did not commit the transformation transaction.', $pdo->errorInfo()); + } + } + + public static function rollback(PDO $pdo, Throwable $applyFailure): void + { + try { + if (!$pdo->rollBack()) { + throw self::failure('The SQL driver did not roll back the failed transformation transaction.', $pdo->errorInfo()); + } + } catch (Throwable $rollbackFailure) { + throw new PDOException( + 'The failed transformation transaction rollback failed: ' . $rollbackFailure->getMessage(), + 0, + $applyFailure, + ); + } + } + + /** @param array $errorInfo */ + private static function failure(string $message, array $errorInfo): PDOException + { + $driverMessage = $errorInfo[2] ?? null; + if (is_string($driverMessage) && $driverMessage !== '') { + $message .= ' ' . $driverMessage; + } + + return new PDOException($message); + } +} diff --git a/src/Transformation/Execution/DatasetBinding.php b/src/Transformation/Execution/DatasetBinding.php index 0da07bb..e930efd 100644 --- a/src/Transformation/Execution/DatasetBinding.php +++ b/src/Transformation/Execution/DatasetBinding.php @@ -4,6 +4,8 @@ namespace OpenStatSpec\Transformation\Execution; +use OpenStatSpec\Sql\PdoSqlProfile; + /** @internal Exact normative-catalog binding for one existing dataset. */ final readonly class DatasetBinding { @@ -13,4 +15,11 @@ public function __construct( public ?string $schema, public string $table, ) {} + + public function qualifiedTable(PdoSqlProfile $profile): string + { + $table = $profile->quoteIdentifier($this->table); + + return $this->schema === null ? $table : $profile->quoteIdentifier($this->schema) . '.' . $table; + } } diff --git a/src/Transformation/Execution/DoltGuard.php b/src/Transformation/Execution/DoltGuard.php index cd52f1c..88afdfb 100644 --- a/src/Transformation/Execution/DoltGuard.php +++ b/src/Transformation/Execution/DoltGuard.php @@ -4,20 +4,42 @@ namespace OpenStatSpec\Transformation\Execution; -use OpenStatSpec\Core\DiagnosticCode; -use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; /** Fail-closed Dolt guard; it observes repository state but never mutates it. */ final readonly class DoltGuard { public function __construct(private DoltEvidenceReader $reader) {} - public function beforeExecution(): DoltEvidence + public function beforeExecution(InPlaceApplyRequest $request): DoltEvidence { + if ($request->expectedBranch === null || $request->expectedHead === null) { + throw TransformationFailure::at( + 'dolt_context_required', + '$.expected_context', + 'Dolt applies require the expected branch and HEAD.', + ); + } + $evidence = $this->reader->read(); + if ($evidence->branch() !== $request->expectedBranch) { + throw TransformationFailure::at( + 'dolt_branch_mismatch', + '$.expected_context.branch', + 'The active Dolt branch does not match the expected branch.', + ); + } + if ($evidence->head() !== $request->expectedHead) { + throw TransformationFailure::at( + 'dolt_head_mismatch', + '$.expected_context.head', + 'Dolt HEAD does not match the expected HEAD.', + ); + } if (!$evidence->isClean()) { - throw new UnsupportedOperation( - DiagnosticCode::SqlProfileOperationUnavailable, + throw TransformationFailure::at( + 'dolt_working_set_dirty', + '$.expected_context.working_set_clean', 'Dolt transformations require a clean working set before execution; dirty tables: ' . implode(', ', $evidence->dirtyTables()) . '.', ); @@ -26,19 +48,22 @@ public function beforeExecution(): DoltEvidence return $evidence; } - public function afterExecution(DoltEvidence $before): DoltEvidence + public function afterExecution(InPlaceApplyRequest $request, DoltEvidence $before): DoltEvidence { $after = $this->reader->read(); - if ($after->branch() !== $before->branch()) { - throw new UnsupportedOperation( - DiagnosticCode::SqlProfileOperationUnavailable, - 'The active Dolt branch changed during transformation execution.', - ); - } - if ($after->head() !== $before->head()) { - throw new UnsupportedOperation( - DiagnosticCode::SqlProfileOperationUnavailable, - 'Dolt HEAD changed during transformation execution; the executor never commits.', + if ($request->expectedBranch === null + || $request->expectedHead === null + || $before->branch() !== $request->expectedBranch + || $before->head() !== $request->expectedHead + || $after->branch() !== $request->expectedBranch + || $after->head() !== $request->expectedHead + || $after->branch() !== $before->branch() + || $after->head() !== $before->head() + ) { + throw TransformationFailure::at( + 'dolt_context_changed', + '$.expected_context', + 'The Dolt branch or HEAD changed during transformation execution.', ); } diff --git a/src/Transformation/Execution/ExecutionResult.php b/src/Transformation/Execution/ExecutionResult.php index c3b5790..5cd4f96 100644 --- a/src/Transformation/Execution/ExecutionResult.php +++ b/src/Transformation/Execution/ExecutionResult.php @@ -10,7 +10,7 @@ public function __construct( private string $datasetId, private string $planHash, private int $operationCount, - private ?string $auditOperationId, + private string $auditOperationId, private ?DoltEvidence $doltBefore, private ?DoltEvidence $doltAfter, ) {} @@ -30,7 +30,7 @@ public function operationCount(): int return $this->operationCount; } - public function auditOperationId(): ?string + public function auditOperationId(): string { return $this->auditOperationId; } diff --git a/src/Transformation/Execution/InPlaceApplyRequest.php b/src/Transformation/Execution/InPlaceApplyRequest.php new file mode 100644 index 0000000..395f990 --- /dev/null +++ b/src/Transformation/Execution/InPlaceApplyRequest.php @@ -0,0 +1,47 @@ +inputAlias) { + throw TransformationFailure::at('unknown_input_alias', '$.input_alias', 'Apply binding does not match the plan alias.'); + } + if (preg_match('/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[89abcd][0-9a-f]{3}-[0-9a-f]{12}\z/D', $datasetId) !== 1) { + throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply dataset identity must be a canonical lowercase UUID with an RFC 4122 (8/9/a/b) or Microsoft (c/d) variant.'); + } + if (preg_match('/\A[0-9a-f]{64}\z/D', $sourceHash) !== 1) { + throw TransformationFailure::at('invalid_source_hash', '$.source_hash', 'Apply source hash must be a lowercase SHA-256 value.'); + } + if (trim($actor) === '') { + throw TransformationFailure::at('actor_required', '$.actor', 'A non-empty actor is required.'); + } + + $branchPresent = $expectedBranch !== null && trim($expectedBranch) !== ''; + $headPresent = $expectedHead !== null && trim($expectedHead) !== ''; + if ($branchPresent !== $headPresent + || ($expectedBranch !== null && !$branchPresent) + || ($expectedHead !== null && !$headPresent) + ) { + throw TransformationFailure::at( + 'dolt_context_required', + '$.expected_context', + 'Expected Dolt branch and HEAD must be supplied together as non-empty values.', + ); + } + } +} diff --git a/src/Transformation/Execution/InPlaceOperationExecutor.php b/src/Transformation/Execution/InPlaceOperationExecutor.php new file mode 100644 index 0000000..968ca91 --- /dev/null +++ b/src/Transformation/Execution/InPlaceOperationExecutor.php @@ -0,0 +1,340 @@ +predicateCompiler = new SqlPredicateCompiler($connection->profile); + } + + public function isFor(PDO $pdo): bool + { + return $this->connection->pdo === $pdo; + } + + public function execute(BoundOperation $bound, DatasetBinding $dataset): void + { + if (!$this->connection->pdo->inTransaction()) { + throw new \LogicException('An in-place operation requires the open apply transaction.'); + } + + $operation = $bound->operation; + if ($operation instanceof AssignOperation) { + $target = $this->target($bound); + if (!$target->persisted) { + $this->createNumericTarget($dataset, $target); + } + $parameters = []; + $value = $this->predicateCompiler->operand($operation->value, $bound->schema, $parameters); + CheckedPdo::execute($this->statement( + 'UPDATE ' . $dataset->qualifiedTable($this->connection->profile) + . ' SET ' . $this->quote($target->physicalName) . ' = ' . $value, + ), $parameters, 'The assignment UPDATE could not be executed.'); + return; + } + + if ($operation instanceof ConditionalAssignOperation) { + $parameters = []; + $value = $this->predicateCompiler->operand($operation->value, $bound->schema, $parameters); + $condition = $this->predicateCompiler->compile($operation->condition, $bound->schema, $parameters); + CheckedPdo::execute($this->statement( + 'UPDATE ' . $dataset->qualifiedTable($this->connection->profile) + . ' SET ' . $this->quote($this->target($bound)->physicalName) . ' = ' . $value + . ' WHERE ' . $condition, + ), $parameters, 'The conditional assignment UPDATE could not be executed.'); + return; + } + + if ($operation instanceof RecodeOperation) { + $target = $this->target($bound); + if (!$target->persisted) { + $this->createNumericTarget($dataset, $target); + } + $this->recode($operation, $bound, $dataset); + return; + } + + if ($operation instanceof SetVariableLabelOperation) { + CheckedPdo::execute($this->statement( + 'UPDATE variable SET variable_label = ? WHERE variable_id = ? AND dataset_id = ?', + ), [$operation->label, $this->target($bound)->variableId, $dataset->datasetId], 'The variable-label UPDATE could not be executed.'); + return; + } + + if ($operation instanceof ReplaceValueLabelsOperation) { + $this->replaceValueLabels($operation, $dataset, $this->target($bound)); + return; + } + + if ($operation instanceof SetFormatOperation) { + CheckedPdo::execute($this->statement( + 'UPDATE variable SET print_format_family = ?, print_format_width = ?, print_format_decimals = ?, ' + . 'write_format_family = ?, write_format_width = ?, write_format_decimals = ? ' + . 'WHERE variable_id = ? AND dataset_id = ?', + ), [ + $operation->family, + $operation->width, + $operation->decimals, + $operation->family, + $operation->width, + $operation->decimals, + $this->target($bound)->variableId, + $dataset->datasetId, + ], 'The variable-format UPDATE could not be executed.'); + return; + } + + if ($operation instanceof SetMeasurementLevelOperation) { + CheckedPdo::execute($this->statement( + 'UPDATE variable SET measurement_level = ? WHERE variable_id = ? AND dataset_id = ?', + ), [$operation->level, $this->target($bound)->variableId, $dataset->datasetId], 'The measurement-level UPDATE could not be executed.'); + return; + } + + if ($operation instanceof ExecuteOperation) { + return; + } + + throw TransformationFailure::at('plan_schema_invalid', '$.operations', 'Unsupported bound operation.'); + } + + private function createNumericTarget(DatasetBinding $dataset, VariableBinding $target): void + { + if (!$this->connection->profile->ddlAtomic()) { + throw TransformationFailure::at( + 'schema_change_not_atomic', + '$.operations', + 'Numeric target creation reached the executor on a non-atomic profile; preflight should have rejected the plan.', + ); + } + + CheckedPdo::exec( + $this->connection->pdo, + 'ALTER TABLE ' . $dataset->qualifiedTable($this->connection->profile) + . ' ADD COLUMN ' . $this->quote($target->physicalName) + . ' ' . $this->connection->profile->numericType() . ' NULL', + 'The numeric target column could not be added.', + ); + CheckedPdo::execute($this->statement( + 'INSERT INTO variable ' + . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?)', + ), [ + $target->variableId, + $dataset->datasetId, + $target->sourceOrdinal, + $target->sourceName, + $target->physicalName, + 'numeric', + null, + ], 'The numeric target catalog row could not be inserted.'); + } + + private function recode(RecodeOperation $operation, BoundOperation $bound, DatasetBinding $dataset): void + { + $source = $bound->source; + if ($source === null) { + throw new \LogicException('A bound recode requires its source.'); + } + $sourceSql = $this->quote($source->physicalName); + $parameters = []; + $when = []; + foreach ($operation->rules as $rule) { + $condition = match (true) { + $rule->match instanceof ExactMatch => $this->exactMatch($sourceSql, $rule->match, $parameters), + $rule->match instanceof RangeMatch => $this->rangeMatch($sourceSql, $rule->match, $parameters), + $rule->match instanceof SystemMissingMatch => '(' . $sourceSql . ' IS NULL)', + default => throw new \LogicException('Unsupported preflighted recode match.'), + }; + $when[] = 'WHEN ' . $condition . ' THEN ' . $this->resultSql($rule->result, $sourceSql, $parameters); + } + $unmatched = $this->resultSql($operation->unmatched, $sourceSql, $parameters); + CheckedPdo::execute($this->statement( + 'UPDATE ' . $dataset->qualifiedTable($this->connection->profile) + . ' SET ' . $this->quote($this->target($bound)->physicalName) + . ' = CASE ' . implode(' ', $when) . ' ELSE ' . $unmatched . ' END', + ), $parameters, 'The recode UPDATE could not be executed.'); + } + + /** @param list $parameters */ + private function exactMatch(string $sourceSql, ExactMatch $match, array &$parameters): string + { + $conditions = []; + foreach ($match->values as $value) { + if ($value instanceof Binary64Value) { + $conditions[] = $sourceSql . ' = ' . $this->predicateCompiler->numericLiteral($value, $parameters); + } else { + $parameters[] = $this->value($value); + $conditions[] = $this->connection->profile->exactValueCondition($sourceSql, true); + } + } + return '(' . implode(' OR ', $conditions) . ')'; + } + + /** @param list $parameters */ + private function rangeMatch(string $sourceSql, RangeMatch $match, array &$parameters): string + { + $lower = $this->predicateCompiler->numericLiteral($match->lower, $parameters); + $upper = $this->predicateCompiler->numericLiteral($match->upper, $parameters); + return '(' . $sourceSql . ' >= ' . $lower . ' AND ' . $sourceSql . ' <= ' . $upper . ')'; + } + + /** @param list $parameters */ + private function resultSql(Result $result, string $sourceSql, array &$parameters): string + { + if ($result instanceof CopyResult) { + return $sourceSql; + } + if ($result instanceof SystemMissingResult) { + return 'NULL'; + } + if ($result instanceof LiteralResult) { + if ($result->value instanceof Binary64Value) { + return $this->predicateCompiler->numericLiteral($result->value, $parameters); + } + $parameters[] = $this->value($result->value); + return '?'; + } + throw new \LogicException('Unsupported preflighted recode result.'); + } + + private function replaceValueLabels( + ReplaceValueLabelsOperation $operation, + DatasetBinding $dataset, + VariableBinding $target, + ): void { + $statement = $this->statement( + 'SELECT link.value_label_set_id, value_set.dataset_id FROM variable_value_label_set link ' + . 'JOIN value_label_set value_set ON value_set.value_label_set_id = link.value_label_set_id ' + . 'WHERE link.variable_id = ?', + ); + CheckedPdo::execute($statement, [$target->variableId], 'The value-label association could not be queried.'); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + if (count($rows) > 1 || (isset($rows[0]['dataset_id']) && $rows[0]['dataset_id'] !== $dataset->datasetId)) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Value-label association is malformed.'); + } + + $setId = isset($rows[0]['value_label_set_id']) && is_string($rows[0]['value_label_set_id']) + ? $rows[0]['value_label_set_id'] + : null; + if ($setId !== null) { + $references = $this->statement('SELECT COUNT(*) FROM variable_value_label_set WHERE value_label_set_id = ?'); + CheckedPdo::execute($references, [$setId], 'The value-label reference count could not be queried.'); + if ((int) $references->fetchColumn() > 1) { + CheckedPdo::execute( + $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?'), + [$target->variableId], + 'The old shared value-label association could not be removed.', + ); + $setId = null; + } else { + CheckedPdo::execute( + $this->statement('DELETE FROM value_label WHERE value_label_set_id = ?'), + [$setId], + 'The old value labels could not be removed.', + ); + } + } + if ($setId === null) { + $setId = NormativeCatalog::uuid(); + CheckedPdo::execute( + $this->statement('INSERT INTO value_label_set (value_label_set_id, dataset_id, name) VALUES (?, ?, ?)'), + [$setId, $dataset->datasetId, null], + 'The value-label set could not be inserted.', + ); + CheckedPdo::execute( + $this->statement('INSERT INTO variable_value_label_set (variable_id, value_label_set_id) VALUES (?, ?)'), + [$target->variableId, $setId], + 'The value-label association could not be inserted.', + ); + } + + $insert = $this->statement( + 'INSERT INTO value_label ' + . '(value_label_id, value_label_set_id, ordinal, code_kind, numeric_code, string_code, label) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?)', + ); + foreach ($operation->labels as $index => $label) { + $numeric = $label->value instanceof Binary64Value; + CheckedPdo::execute($insert, [ + NormativeCatalog::uuid(), + $setId, + $index + 1, + $numeric ? 'numeric' : 'string', + $numeric ? $this->value($label->value) : null, + $numeric ? null : $this->value($label->value), + $label->label, + ], 'A replacement value label could not be inserted.'); + } + } + + /** + * Renders a preflighted typed value as the parameter token fed into + * the prepared UPDATE/INSERT statement. Binary64Value is rendered as + * its locale-independent decimal string so the bound binary64 round + * trips exactly through the engine column; StringValue is rendered + * verbatim. The resulting string is then bound as a PDO parameter, + * which the driver converts to the underlying column type. + */ + private function value(TypedValue $value): string + { + return match (true) { + $value instanceof Binary64Value => $value->decimal(), + $value instanceof StringValue => $value->value, + default => throw new \LogicException('Unsupported preflighted typed value.'), + }; + } + + private function target(BoundOperation $bound): VariableBinding + { + if ($bound->target === null) { + throw new \LogicException('The bound operation requires a target.'); + } + return $bound->target; + } + + private function quote(string $identifier): string + { + return $this->connection->profile->quoteIdentifier($identifier); + } + + private function statement(string $sql): PDOStatement + { + return CheckedPdo::prepare( + $this->connection->pdo, + $sql, + 'Transformation SQL statement could not be prepared.', + ); + } +} diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php index 6db9237..681c849 100644 --- a/src/Transformation/Execution/InPlaceTransformationExecutor.php +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -4,854 +4,91 @@ namespace OpenStatSpec\Transformation\Execution; -use OpenStatSpec\Core\Binary64; use OpenStatSpec\Core\DiagnosticCode; use OpenStatSpec\Core\UnsupportedOperation; -use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; -use OpenStatSpec\Sql\NormativeCatalog; -use OpenStatSpec\Sql\OperationJournal; -use OpenStatSpec\Transformation\Model\CreateVariableOperation; -use OpenStatSpec\Transformation\Model\DeleteVariableOperation; -use OpenStatSpec\Transformation\Model\Action\AssignValueAction; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\Action\SetMissingAction; -use OpenStatSpec\Transformation\Model\RecodeAction; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\ScalarValue; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\ExactValueSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\Selector\NumericRangeSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; -use OpenStatSpec\Transformation\Model\TransformationOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; -use OpenStatSpec\Transformation\Validation\PlanValidator; -use PDO; -use PDOException; -use PDOStatement; -use SPSS\Sav\Variable; +use OpenStatSpec\Transformation\Audit\TransformationAuditWriter; +use OpenStatSpec\Transformation\Plan\PlanCodec; use Throwable; -/** Executes a canonical plan against its registered wide table, in place. */ -final class InPlaceTransformationExecutor +/** Applies one official plan atomically to its existing logical dataset and wide table. */ +final readonly class InPlaceTransformationExecutor { - private readonly PlanValidator $validator; - - /** @var array> */ - private array $operationBindings = []; + private PlanPreflight $preflight; + private InPlaceOperationExecutor $operationExecutor; + private TransformationAuditWriter $auditWriter; + private PlanCodec $codec; public function __construct( - private readonly Connection $connection, - ?PlanValidator $validator = null, - private readonly ?DoltEvidenceReader $doltEvidenceReader = null, + private Connection $connection, + private ?DoltEvidenceReader $doltEvidenceReader = null, + ?TransformationAuditWriter $auditWriter = null, + ?PlanPreflight $preflight = null, + ?InPlaceOperationExecutor $operationExecutor = null, + ?PlanCodec $codec = null, ) { - $this->validator = $validator ?? new PlanValidator(); + $this->codec = $codec ?? new PlanCodec(); + $this->preflight = $preflight ?? new PlanPreflight($connection, $this->codec); + $this->operationExecutor = $operationExecutor ?? new InPlaceOperationExecutor($connection); + $this->auditWriter = $auditWriter ?? new TransformationAuditWriter($connection->pdo, $this->codec); + if (!$this->preflight->isFor($connection->pdo) + || !$this->operationExecutor->isFor($connection->pdo) + || !$this->auditWriter->isFor($connection->pdo) + ) { + throw new \InvalidArgumentException('Every transformation executor collaborator must use the same PDO connection.'); + } } - public function execute(TransformationPlan $plan): ExecutionResult + public function execute(InPlaceApplyRequest $request): ExecutionResult { - $this->validator->assertValid($plan); - $this->connection->assertClaimedSupported(); if ($this->connection->pdo->inTransaction()) { throw new UnsupportedOperation( DiagnosticCode::UnsupportedOperation, 'An in-place transformation cannot start inside a caller-owned active transaction.', ); } - CatalogOwnership::assertReadyForUse($this->connection->pdo); - - $dataset = $this->resolveDataset($plan->datasetId()); - $variables = $this->resolveVariables($dataset); - $plannedVariables = $this->preflight($plan, $dataset, $variables); + $startedAt = gmdate('Y-m-d H:i:s'); + $bound = $this->preflight->bind($request); $guard = $this->doltGuard(); - $doltBefore = $guard?->beforeExecution(); - $journal = $this->existingJournal(); - $operationId = $journal?->start( - 'transform', - $dataset->datasetName, - null, - [], - $this->auditDetails($plan, $dataset, $doltBefore), - ); + $doltBefore = $guard?->beforeExecution($request); $doltAfter = null; + $transactionStarted = false; try { - if (!$this->connection->pdo->beginTransaction()) { - throw new PDOException('The SQL driver did not start the transformation transaction.'); - } - $created = []; - foreach ($plan->operations() as $operationIndex => $operation) { - $this->applyOperation($operation, $dataset, $plannedVariables, $created, $operationIndex); - } - $doltAfter = $doltBefore === null - ? null - : $guard->afterExecution($doltBefore); - if ($journal !== null && $operationId !== null) { - $journal->succeed($operationId, $dataset->datasetName, []); - } - if (!$this->connection->pdo->commit()) { - throw new PDOException('The SQL driver did not commit the transformation transaction.'); - } - } catch (Throwable $exception) { - if ($this->connection->pdo->inTransaction()) { - $this->connection->pdo->rollBack(); - } - if ($journal !== null && $operationId !== null) { - try { - $journal->fail($operationId, $dataset->datasetName, $exception); - } catch (Throwable) { - // Preserve the transformation failure; audit is best-effort after rollback. - } + CheckedPdo::begin($this->connection->pdo); + $transactionStarted = true; + foreach ($bound->operations as $operation) { + $this->operationExecutor->execute($operation, $bound->dataset); + } + $doltAfter = $doltBefore === null ? null : $guard->afterExecution($request, $doltBefore); + $applyId = $this->auditWriter->succeed( + $request, + $bound->dataset, + $this->connection->profileName, + $doltBefore, + $doltAfter, + $startedAt, + ); + CheckedPdo::commit($this->connection->pdo); + $transactionStarted = false; + } catch (Throwable $failure) { + if ($transactionStarted) { + CheckedPdo::rollback($this->connection->pdo, $failure); } - throw $exception; + throw $failure; } return new ExecutionResult( - $dataset->datasetId, - $plan->hash(), - count($plan->operations()), - $operationId, + $bound->dataset->datasetId, + $this->codec->hash($request->plan), + count($request->plan->operations), + $applyId, $doltBefore, $doltAfter, ); } - private function resolveDataset(string $datasetId): DatasetBinding - { - $statement = $this->statement( - 'SELECT dataset_id, dataset_name, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', - ); - $statement->execute([$datasetId]); - $rows = $statement->fetchAll(PDO::FETCH_ASSOC); - if (count($rows) !== 1) { - throw $this->invalidCatalog('The transformation dataset_id resolves to exactly one normative dataset row.'); - } - $row = $rows[0]; - $table = $row['physical_table_name'] ?? null; - $schema = $row['physical_table_schema'] ?? null; - $name = $row['dataset_name'] ?? null; - if (($row['dataset_id'] ?? null) !== $datasetId - || !is_string($table) - || $table === '' - || ($schema !== null && (!is_string($schema) || $schema === '')) - || ($name !== null && !is_string($name)) - ) { - throw $this->invalidCatalog('The normative dataset row has malformed physical table metadata.'); - } - - return new DatasetBinding($datasetId, $name, $schema, $table); - } - - /** @return array */ - private function resolveVariables(DatasetBinding $dataset): array - { - $statement = $this->statement( - 'SELECT variable_id, source_name, physical_name, storage_kind, source_ordinal, declared_string_width ' - . 'FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', - ); - $statement->execute([$dataset->datasetId]); - $variables = []; - $physicalNames = []; - foreach ($statement->fetchAll(PDO::FETCH_ASSOC) as $row) { - $id = $row['variable_id'] ?? null; - $source = $row['source_name'] ?? null; - $physical = $row['physical_name'] ?? null; - $kind = $row['storage_kind'] ?? null; - $ordinal = filter_var($row['source_ordinal'] ?? null, FILTER_VALIDATE_INT); - $declaredStringWidth = $row['declared_string_width'] === null - ? null - : filter_var($row['declared_string_width'], FILTER_VALIDATE_INT); - if (!is_string($id) || $id === '' - || !is_string($source) || $source === '' - || !is_string($physical) || $physical === '' - || !in_array($kind, ['numeric', 'string'], true) - || !is_int($ordinal) || $ordinal < 1 - || ($kind === 'string' && (!is_int($declaredStringWidth) || $declaredStringWidth < 1)) - || isset($variables[$source]) || isset($physicalNames[$physical]) - ) { - throw $this->invalidCatalog('The normative variable mapping is malformed or ambiguous.'); - } - $variables[$source] = new VariableBinding( - $id, - $source, - $physical, - $kind, - $ordinal, - is_int($declaredStringWidth) ? $declaredStringWidth : null, - ); - $physicalNames[$physical] = true; - } - if ($variables === []) { - throw $this->invalidCatalog('The registered dataset has no normative variable mappings.'); - } - - $columns = implode(', ', array_map( - fn(VariableBinding $variable): string => $this->quote($variable->physicalName), - array_values($variables), - )); - try { - $this->connection->pdo->query( - 'SELECT ' . $columns . ' FROM ' . $this->qualifiedTable($dataset) . ' WHERE 1 = 0', - ); - } catch (PDOException $exception) { - throw $this->invalidCatalog('The catalogued physical wide table or variable mapping is not readable: ' . $exception->getMessage()); - } - - return $variables; - } - - /** - * @param array $variables - * @return array - */ - private function preflight(TransformationPlan $plan, DatasetBinding $dataset, array $variables): array - { - $physicalColumnSlots = $this->connection->profileName === 'postgresql' - ? $this->postgresqlPhysicalColumnSlots($dataset) - : null; - $used = ['__case_ordinal' => true]; - $nextOrdinal = 1; - foreach ($variables as $variable) { - $used[$variable->physicalName] = true; - $nextOrdinal = max($nextOrdinal, $variable->sourceOrdinal + 1); - } - - $active = array_fill_keys(array_keys($variables), true); - $this->operationBindings = []; - foreach ($plan->operations() as $operationIndex => $operation) { - if ($operation instanceof CreateVariableOperation) { - if (isset($variables[$operation->targetVariable()]) && isset($active[$operation->targetVariable()])) { - throw $this->invalidCatalog(sprintf( - 'Create variable "%s" collides with an existing catalog variable.', - $operation->targetVariable(), - )); - } - $this->assertCanAlterSchema(count($active), true, $physicalColumnSlots); - if ($physicalColumnSlots !== null) { - ++$physicalColumnSlots; - } - $physical = $this->connection->profile->physicalIdentifier($operation->targetVariable(), $used); - $variables[$operation->targetVariable()] = new VariableBinding( - NormativeCatalog::uuid(), - $operation->targetVariable(), - $physical, - $operation->storageKind(), - $nextOrdinal++, - $operation->declaredStringWidth(), - false, - ); - $used[$physical] = true; - $active[$operation->targetVariable()] = true; - $this->operationBindings[$operationIndex] = ['target' => $variables[$operation->targetVariable()]]; - continue; - } - if ($operation instanceof DeleteVariableOperation) { - $variableName = $operation->targetVariable(); - if (!isset($active[$variableName])) { - throw $this->invalidCatalog(sprintf( - 'Delete variable "%s" is not registered for dataset_id %s.', - $variableName, - $plan->datasetId(), - )); - } - if (count($active) === 1 && !$this->hasFutureCreateOperation($plan->operations(), $operationIndex)) { - throw $this->invalidCatalog('A transformation cannot delete the final dataset variable.'); - } - $this->assertCanAlterSchema(count($active), false); - $this->operationBindings[$operationIndex] = ['target' => $variables[$variableName]]; - unset($active[$variableName]); - continue; - } - if ($operation instanceof RecodeOperation) { - $source = $variables[$operation->sourceVariable()] ?? null; - if ($source === null || !isset($active[$operation->sourceVariable()])) { - throw $this->invalidCatalog(sprintf( - 'Recode source variable "%s" is not registered for dataset_id %s.', - $operation->sourceVariable(), - $plan->datasetId(), - )); - } - $target = $variables[$operation->targetVariable()] ?? null; - if ($target !== null && !isset($active[$operation->targetVariable()])) { - $target = null; - } - if ($target === null) { - $this->assertCanCreateTarget($source, count($active), $physicalColumnSlots); - if ($physicalColumnSlots !== null) { - ++$physicalColumnSlots; - } - $physical = $this->connection->profile->physicalIdentifier($operation->targetVariable(), $used); - $target = new VariableBinding( - NormativeCatalog::uuid(), - $operation->targetVariable(), - $physical, - $source->storageKind, - $nextOrdinal++, - null, - false, - ); - $variables[$operation->targetVariable()] = $target; - $used[$physical] = true; - $active[$operation->targetVariable()] = true; - } - $this->assertRecodeKinds($operation, $source, $target); - $this->operationBindings[$operationIndex] = ['source' => $source, 'target' => $target]; - continue; - } - - $target = $variables[$operation->targetVariable()] ?? null; - if ($target === null || !isset($active[$operation->targetVariable()])) { - throw $this->invalidCatalog(sprintf( - 'Metadata target variable "%s" is not registered for dataset_id %s.', - $operation->targetVariable(), - $plan->datasetId(), - )); - } - $this->operationBindings[$operationIndex] = ['target' => $target]; - if ($operation instanceof SetValueLabelsOperation) { - foreach ($operation->labels() as $label) { - $this->assertScalarKind($label->value(), $target, 'value label'); - } - } - } - - return $variables; - } - - /** @param list $operations */ - private function hasFutureCreateOperation(array $operations, int $operationIndex): bool - { - foreach ($operations as $futureIndex => $operation) { - if ($futureIndex > $operationIndex && $operation instanceof CreateVariableOperation) { - return true; - } - } - return false; - } - private function assertCanAlterSchema(int $registeredVariableCount, bool $creation, ?int $physicalColumnSlots = null): void - { - if (!in_array($this->connection->profileName, ['sqlite', 'postgresql'], true) - || !$this->connection->profile->ddlAtomic() - ) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - sprintf( - '%s cannot atomically alter an existing wide table for an in-place variable transformation.', - $this->connection->profileName, - ), - ); - } - if (!$creation) { - if ($this->connection->profileName === 'sqlite' - && version_compare($this->connection->serverVersion, '3.35.0', '<') - ) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - 'SQLite versions below 3.35.0 do not support DROP COLUMN for DELETE VARIABLES.', - ); - } - - return; - } - - $maximum = $this->connection->profile->effectiveMaximumSourceVariables($this->connection->pdo); - if ($this->connection->profileName === 'postgresql' - && $physicalColumnSlots !== null - && $physicalColumnSlots >= $maximum + 1 - ) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - sprintf( - 'PostgreSQL cannot add a source variable because the wide table already uses %d physical PostgreSQL column slots, including dropped columns; the table limit is %d.', - $physicalColumnSlots, - $maximum + 1, - ), - ); - } - if ($registeredVariableCount >= $maximum) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - sprintf( - '%s supports at most %d source variables in one OpenStatSpec wide table.', - $this->connection->profileName, - $maximum, - ), - ); - } - } - private function assertCanCreateTarget(VariableBinding $source, int $registeredVariableCount, ?int $physicalColumnSlots = null): void - { - if ($source->storageKind === 'string') { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - 'A new string target requires an explicit declared_string_width; register the target variable first.', - ); - } - if (!in_array($this->connection->profileName, ['sqlite', 'postgresql'], true) - || !$this->connection->profile->ddlAtomic() - ) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - sprintf( - '%s cannot atomically add a new INTO target to an existing wide table; register the target variable first.', - $this->connection->profileName, - ), - ); - } - - $this->assertCanAlterSchema($registeredVariableCount, true, $physicalColumnSlots); - } - - private function postgresqlPhysicalColumnSlots(DatasetBinding $dataset): int - { - $statement = $this->statement('SELECT COUNT(*) FROM pg_attribute WHERE attrelid = to_regclass(?) AND attnum > 0'); - $statement->execute([$this->qualifiedTable($dataset)]); - $slots = filter_var($statement->fetchColumn(), FILTER_VALIDATE_INT); - if (!is_int($slots) || $slots < 1) { - throw $this->invalidCatalog('The PostgreSQL wide table has no physical column slots.'); - } - - return $slots; - } - - private function assertRecodeKinds( - RecodeOperation $operation, - VariableBinding $source, - VariableBinding $target, - ): void { - foreach ($operation->rules() as $rule) { - $selector = $rule->selector(); - if ($selector instanceof ExactValueSelector) { - $this->assertScalarKind($selector->value(), $source, 'exact recode selector'); - } elseif ($selector instanceof NumericRangeSelector && $source->storageKind !== 'numeric') { - throw $this->invalidCatalog('A numeric range selector cannot read a string variable.'); - } - - $action = $rule->action(); - if ($action instanceof AssignValueAction) { - $this->assertScalarKind($action->value(), $target, 'recode assignment'); - } elseif ($action instanceof CopySourceAction && $source->storageKind !== $target->storageKind) { - throw $this->invalidCatalog('Copy-source recoding requires source and target variables with the same storage kind.'); - } elseif ($action instanceof CopySourceAction - && $source->storageKind === 'string' - && $this->stringWidth($source) > $this->stringWidth($target) - ) { - throw $this->invalidCatalog(sprintf( - 'Copy-source recoding cannot copy string width %d into variable "%s" width %d.', - $this->stringWidth($source), - $target->sourceName, - $this->stringWidth($target), - )); - } elseif ($action instanceof SetMissingAction && $target->storageKind === 'string') { - throw $this->invalidCatalog(sprintf( - 'System-missing recoding is not representable for string variable "%s".', - $target->sourceName, - )); - } - } - } - - private function assertScalarKind(ScalarValue $value, VariableBinding $variable, string $context): void - { - $kind = $value->type() === 'number' ? 'numeric' : 'string'; - if ($kind !== $variable->storageKind) { - throw $this->invalidCatalog(sprintf( - 'The %s type does not match variable "%s" storage kind %s.', - $context, - $variable->sourceName, - $variable->storageKind, - )); - } - if ($kind === 'string' && strlen($value->stringValue()) > $this->stringWidth($variable)) { - throw $this->invalidCatalog(sprintf( - 'The %s exceeds variable "%s" declared string width %d bytes.', - $context, - $variable->sourceName, - $this->stringWidth($variable), - )); - } - } - - private function stringWidth(VariableBinding $variable): int - { - if ($variable->storageKind !== 'string' - || $variable->declaredStringWidth === null - || $variable->declaredStringWidth < 1 - ) { - throw $this->invalidCatalog(sprintf( - 'String variable "%s" requires a positive declared_string_width.', - $variable->sourceName, - )); - } - - return $variable->declaredStringWidth; - } - - /** - * @param array $variables - * @param array $created - */ - private function applyOperation( - TransformationOperation $operation, - DatasetBinding $dataset, - array $variables, - array &$created, - int $operationIndex, - ): void { - $bindings = $this->operationBindings[$operationIndex] ?? []; - if ($operation instanceof CreateVariableOperation) { - $target = $bindings['target'] ?? $variables[$operation->targetVariable()]; - $this->ensureTargetExists($dataset, $target, $created); - return; - } - if ($operation instanceof DeleteVariableOperation) { - $target = $bindings['target'] ?? $variables[$operation->targetVariable()]; - $this->deleteVariable($dataset, $target); - unset($created[$target->sourceName]); - return; - } - $target = $bindings['target'] ?? $variables[$operation->targetVariable()]; - if ($operation instanceof RecodeOperation) { - $this->ensureTargetExists($dataset, $target, $created); - $this->applyRecode($operation, $dataset, $bindings['source'] ?? $variables[$operation->sourceVariable()], $target); - } elseif ($operation instanceof SetVariableLabelOperation) { - $this->applyVariableLabel($operation, $dataset, $target); - } elseif ($operation instanceof SetValueLabelsOperation) { - $this->applyValueLabels($operation, $dataset, $target); - } - } - - /** @param array $created */ - private function ensureTargetExists(DatasetBinding $dataset, VariableBinding $target, array &$created): void - { - if ($target->persisted || isset($created[$target->sourceName])) { - return; - } - $type = $target->storageKind === 'numeric' - ? $this->connection->profile->numericType() - : $this->connection->profile->textType(); - $columnDefinition = $target->storageKind === 'string' - ? $type . " NOT NULL DEFAULT ''" - : $type . ' NULL'; - $this->connection->pdo->exec(sprintf( - 'ALTER TABLE %s ADD COLUMN %s %s', - $this->qualifiedTable($dataset), - $this->quote($target->physicalName), - $columnDefinition, - )); - $formatFamily = $target->storageKind === 'string' ? Variable::FORMAT_TYPE_A : 5; - $formatWidth = $target->storageKind === 'string' - ? min(255, $this->stringWidth($target)) - : 8; - $this->statement( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width, ' - . 'print_format_family, print_format_width, print_format_decimals, ' - . 'write_format_family, write_format_width, write_format_decimals) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', - )->execute([ - $target->variableId, - $dataset->datasetId, - $target->sourceOrdinal, - $target->sourceName, - $target->physicalName, - $target->storageKind, - $target->declaredStringWidth, - $formatFamily, - $formatWidth, - 0, - $formatFamily, - $formatWidth, - 0, - ]); - $created[$target->sourceName] = true; - } - - private function deleteVariable(DatasetBinding $dataset, VariableBinding $target): void - { - $this->connection->pdo->exec(sprintf( - 'ALTER TABLE %s DROP COLUMN %s', - $this->qualifiedTable($dataset), - $this->quote($target->physicalName), - )); - $association = $this->statement( - 'SELECT vvls.value_label_set_id, vls.dataset_id FROM variable_value_label_set vvls ' - . 'JOIN value_label_set vls ON vls.value_label_set_id = vvls.value_label_set_id ' - . 'WHERE vvls.variable_id = ?', - ); - $association->execute([$target->variableId]); - $rows = $association->fetchAll(PDO::FETCH_ASSOC); - if (count($rows) > 1 || (isset($rows[0]['dataset_id']) && $rows[0]['dataset_id'] !== $dataset->datasetId)) { - throw $this->invalidCatalog('The variable value-label association is malformed or crosses dataset identity.'); - } - $setId = isset($rows[0]['value_label_set_id']) && is_string($rows[0]['value_label_set_id']) - ? $rows[0]['value_label_set_id'] - : null; - - foreach ([ - 'DELETE FROM dataset_weight_variable WHERE variable_id = ?', - 'DELETE FROM variable_set_member WHERE variable_id = ?', - 'DELETE FROM multiple_response_member WHERE variable_id = ?', - 'DELETE FROM variable_attribute WHERE variable_id = ?', - 'DELETE FROM missing_rule WHERE variable_id = ?', - 'DELETE FROM variable_value_label_set WHERE variable_id = ?', - ] as $sql) { - $this->statement($sql)->execute([$target->variableId]); - } - $this->statement('DELETE FROM variable WHERE variable_id = ? AND dataset_id = ?')->execute([ - $target->variableId, - $dataset->datasetId, - ]); - - $orphanedSets = $this->statement( - 'SELECT multiple_response_set_id FROM multiple_response_set WHERE dataset_id = ? ' - . 'AND NOT EXISTS (SELECT 1 FROM multiple_response_member WHERE multiple_response_set_id = multiple_response_set.multiple_response_set_id)', - ); - $orphanedSets->execute([$dataset->datasetId]); - foreach ($orphanedSets->fetchAll(PDO::FETCH_COLUMN) as $orphanedSetId) { - if (!is_string($orphanedSetId)) { - continue; - } - $this->statement('DELETE FROM multiple_response_set WHERE multiple_response_set_id = ? AND dataset_id = ?')->execute([$orphanedSetId, $dataset->datasetId]); - } - - $orphanedVariableSets = $this->statement( - 'SELECT variable_set_id FROM variable_set WHERE dataset_id = ? ' - . 'AND NOT EXISTS (SELECT 1 FROM variable_set_member WHERE variable_set_id = variable_set.variable_set_id)', - ); - $orphanedVariableSets->execute([$dataset->datasetId]); - foreach ($orphanedVariableSets->fetchAll(PDO::FETCH_COLUMN) as $orphanedVariableSetId) { - if (!is_string($orphanedVariableSetId)) { - continue; - } - $this->statement('DELETE FROM variable_set WHERE variable_set_id = ? AND dataset_id = ?')->execute([$orphanedVariableSetId, $dataset->datasetId]); - } - - if ($setId === null) { - return; - } - $references = $this->statement( - 'SELECT COUNT(*) FROM variable_value_label_set WHERE value_label_set_id = ?', - ); - $references->execute([$setId]); - if ((int) $references->fetchColumn() === 0) { - $this->statement('DELETE FROM value_label WHERE value_label_set_id = ?')->execute([$setId]); - $this->statement('DELETE FROM value_label_set WHERE value_label_set_id = ? AND dataset_id = ?')->execute([ - $setId, - $dataset->datasetId, - ]); - } - } - private function applyRecode( - RecodeOperation $operation, - DatasetBinding $dataset, - VariableBinding $source, - VariableBinding $target, - ): void { - $sourceSql = $this->quote($source->physicalName); - $when = []; - $parameters = []; - $else = null; - foreach ($operation->rules() as $rule) { - $selector = $rule->selector(); - if ($selector instanceof ElseSelector) { - $else = $this->actionSql($rule->action(), $sourceSql, $parameters); - continue; - } - $condition = match (true) { - $selector instanceof ExactValueSelector => $this->exactCondition($sourceSql, $selector, $parameters), - $selector instanceof NumericRangeSelector => $this->rangeCondition($sourceSql, $selector, $parameters), - $selector instanceof MissingValueSelector => $sourceSql . ' IS NULL', - default => throw $this->invalidCatalog('The recode selector is not executable.'), - }; - $actionSql = $this->actionSql($rule->action(), $sourceSql, $parameters); - $when[] = 'WHEN ' . $condition . ' THEN ' . $actionSql; - } - if ($else === null) { - throw $this->invalidCatalog('A recode requires one explicit final else action.'); - } - if ($when === []) { - $this->statement(sprintf( - 'UPDATE %s SET %s = %s', - $this->qualifiedTable($dataset), - $this->quote($target->physicalName), - $else, - ))->execute($parameters); - - return; - } - $sql = sprintf( - 'UPDATE %s SET %s = CASE %s ELSE %s END', - $this->qualifiedTable($dataset), - $this->quote($target->physicalName), - implode(' ', $when), - $else, - ); - $this->statement($sql)->execute($parameters); - } - - /** @param list $parameters */ - private function exactCondition(string $sourceSql, ExactValueSelector $selector, array &$parameters): string - { - $parameters[] = $this->boundScalar($selector->value()); - - return $this->connection->profile->exactValueCondition( - $sourceSql, - $selector->value()->type() === 'string', - ); - } - - /** @param list $parameters */ - private function rangeCondition(string $sourceSql, NumericRangeSelector $selector, array &$parameters): string - { - $parts = []; - if ($selector->lower() !== null) { - $parts[] = $sourceSql . ' >= ?'; - $parameters[] = Binary64::encode($selector->lower()); - } - if ($selector->upper() !== null) { - $parts[] = $sourceSql . ' <= ?'; - $parameters[] = Binary64::encode($selector->upper()); - } - - return implode(' AND ', $parts); - } - - /** @param list $parameters */ - private function actionSql(RecodeAction $action, string $sourceSql, array &$parameters): string - { - if ($action instanceof AssignValueAction) { - $parameters[] = $this->boundScalar($action->value()); - - return '?'; - } - if ($action instanceof SetMissingAction) { - return 'NULL'; - } - if ($action instanceof CopySourceAction) { - return $sourceSql; - } - - throw $this->invalidCatalog('The recode action is not executable.'); - } - - private function applyVariableLabel( - SetVariableLabelOperation $operation, - DatasetBinding $dataset, - VariableBinding $target, - ): void { - $this->statement('UPDATE variable SET variable_label = ? WHERE variable_id = ? AND dataset_id = ?')->execute([ - $operation->label(), - $target->variableId, - $dataset->datasetId, - ]); - } - - private function applyValueLabels( - SetValueLabelsOperation $operation, - DatasetBinding $dataset, - VariableBinding $target, - ): void { - $statement = $this->statement( - 'SELECT vvls.value_label_set_id, vls.dataset_id FROM variable_value_label_set vvls ' - . 'JOIN value_label_set vls ON vls.value_label_set_id = vvls.value_label_set_id ' - . 'WHERE vvls.variable_id = ?', - ); - $statement->execute([$target->variableId]); - $rows = $statement->fetchAll(PDO::FETCH_ASSOC); - if (count($rows) > 1 || (isset($rows[0]['dataset_id']) && $rows[0]['dataset_id'] !== $dataset->datasetId)) { - throw $this->invalidCatalog('The variable value-label association is malformed or crosses dataset identity.'); - } - - $setId = isset($rows[0]['value_label_set_id']) && is_string($rows[0]['value_label_set_id']) - ? $rows[0]['value_label_set_id'] - : null; - if ($setId !== null) { - $references = $this->statement( - 'SELECT COUNT(*) FROM variable_value_label_set WHERE value_label_set_id = ?', - ); - $references->execute([$setId]); - $referenceCount = (int) $references->fetchColumn(); - if ($referenceCount > 1) { - $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?')->execute([$target->variableId]); - $setId = null; - } else { - $this->statement('DELETE FROM value_label WHERE value_label_set_id = ?')->execute([$setId]); - } - } - - if ($operation->labels() === []) { - if ($setId !== null) { - $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?')->execute([$target->variableId]); - $this->statement('DELETE FROM value_label_set WHERE value_label_set_id = ?')->execute([$setId]); - } - - return; - } - - if ($setId === null) { - $setId = NormativeCatalog::uuid(); - $this->statement('INSERT INTO value_label_set (value_label_set_id, dataset_id, name) VALUES (?, ?, ?)')->execute([ - $setId, - $dataset->datasetId, - null, - ]); - $this->statement( - 'INSERT INTO variable_value_label_set (variable_id, value_label_set_id) VALUES (?, ?)', - )->execute([$target->variableId, $setId]); - } - - $insert = $this->statement( - 'INSERT INTO value_label ' - . '(value_label_id, value_label_set_id, ordinal, code_kind, numeric_code, string_code, label) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - ); - foreach ($operation->labels() as $index => $label) { - $value = $label->value(); - $insert->execute([ - NormativeCatalog::uuid(), - $setId, - $index + 1, - $value->type() === 'number' ? 'numeric' : 'string', - $value->type() === 'number' ? Binary64::encode($value->numberValue()) : null, - $value->type() === 'string' ? $value->stringValue() : null, - $label->label(), - ]); - } - } - - private function boundScalar(ScalarValue $value): string - { - return $value->type() === 'number' - ? Binary64::encode($value->numberValue()) - : $value->stringValue(); - } - - private function qualifiedTable(DatasetBinding $dataset): string - { - $table = $this->quote($dataset->table); - - return $dataset->schema === null ? $table : $this->quote($dataset->schema) . '.' . $table; - } - - private function quote(string $identifier): string - { - return $this->connection->profile->quoteIdentifier($identifier); - } - - private function statement(string $sql): PDOStatement - { - $statement = $this->connection->pdo->prepare($sql); - if ($statement === false) { - throw new PDOException('The transformation SQL statement could not be prepared.'); - } - - return $statement; - } - private function doltGuard(): ?DoltGuard { if ($this->connection->profileName !== 'dolt') { @@ -860,37 +97,4 @@ private function doltGuard(): ?DoltGuard return new DoltGuard($this->doltEvidenceReader ?? new PdoDoltEvidenceReader($this->connection->pdo)); } - - private function existingJournal(): ?OperationJournal - { - try { - $this->connection->pdo->query('SELECT operation_id FROM operation_catalog WHERE 1 = 0'); - $this->connection->pdo->query('SELECT operation_id FROM fidelity_event_catalog WHERE 1 = 0'); - $this->connection->pdo->query('SELECT operation_id FROM operation WHERE 1 = 0'); - } catch (PDOException) { - return null; - } - - return new OperationJournal($this->connection->pdo); - } - - /** @return array */ - private function auditDetails( - TransformationPlan $plan, - DatasetBinding $dataset, - ?DoltEvidence $doltBefore, - ): array { - return [ - 'plan_hash' => $plan->hash(), - 'dataset_id' => $dataset->datasetId, - 'mode' => 'in_place', - 'dolt_branch_before' => $doltBefore?->branch(), - 'dolt_head_before' => $doltBefore?->head(), - ]; - } - - private function invalidCatalog(string $message): UnsupportedOperation - { - return new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, $message); - } } diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php new file mode 100644 index 0000000..083b0f4 --- /dev/null +++ b/src/Transformation/Execution/PlanPreflight.php @@ -0,0 +1,754 @@ +codec = $codec ?? new PlanCodec(); + } + + public function isFor(PDO $pdo): bool + { + return $this->connection->pdo === $pdo; + } + + public function bind(InPlaceApplyRequest $request): BoundPlan + { + // Re-decode the public model so directly constructed objects receive the + // same complete structural and semantic validation as decoded plans. + $this->codec->fromArray($request->plan->canonicalArray()); + $this->connection->assertClaimedSupported(); + CatalogOwnership::assertReadyForUseReadOnly($this->connection->pdo); + $this->assertAuditReady(); + + $dataset = $this->resolveDataset($request->datasetId); + $this->assertMySqlFamilyTransactionalEngines($dataset); + $this->assertExclusivePhysicalTableBinding($dataset); + $variables = $this->resolveVariables($dataset); + $physicalColumns = $this->physicalColumns($dataset); + $this->assertPhysicalBindings($dataset, $variables, $physicalColumns); + + $usedPhysical = array_fill_keys($physicalColumns, true); + $nextOrdinal = 1; + foreach ($variables as $variable) { + $nextOrdinal = max($nextOrdinal, $variable->sourceOrdinal + 1); + } + $postgresqlSlots = $this->connection->profileName === 'postgresql' + ? $this->postgresqlPhysicalColumnSlots($dataset) + : null; + $bound = []; + + foreach ($request->plan->operations as $index => $operation) { + $path = '$.operations[' . $index . ']'; + if ($this->createsTarget($operation) && !$this->connection->profile->ddlAtomic()) { + throw TransformationFailure::at( + 'schema_change_not_atomic', + $path . '.target_mode', + $this->connection->profileName . ' requires a pre-provisioned target.', + ); + } + $schemaBefore = new BoundSchema($variables); + [$source, $target] = $this->bindOperation( + $operation, + $schemaBefore, + $path, + $variables, + $usedPhysical, + $nextOrdinal, + $postgresqlSlots, + ); + $bound[] = new BoundOperation($operation, $schemaBefore, $source, $target); + if ($target !== null && !$target->persisted) { + $variables[$target->sourceName] = new VariableBinding( + $target->variableId, + $target->sourceName, + $target->physicalName, + $target->storageKind, + $target->sourceOrdinal, + $target->declaredStringWidth, + ); + $usedPhysical[$target->physicalName] = true; + ++$nextOrdinal; + if ($postgresqlSlots !== null) { + ++$postgresqlSlots; + } + } + } + + return new BoundPlan($request->plan, $dataset, $bound, new BoundSchema($variables)); + } + + /** + * @param array $variables + * @param array $usedPhysical + * @return array{VariableBinding|null, VariableBinding|null} + */ + private function bindOperation( + Operation $operation, + BoundSchema $schema, + string $path, + array $variables, + array $usedPhysical, + int $nextOrdinal, + ?int $postgresqlSlots, + ): array { + if ($operation instanceof AssignOperation) { + $this->assertNumericOperand($operation->value, $schema, $path . '.value'); + $target = $this->bindNumericTarget( + $operation->target, + $operation->targetMode, + $path . '.target', + $variables, + $usedPhysical, + $nextOrdinal, + $postgresqlSlots, + ); + return [null, $target]; + } + + if ($operation instanceof ConditionalAssignOperation) { + $target = $variables[$operation->target] ?? null; + if ($target === null) { + throw TransformationFailure::at( + 'conditional_target_missing', + $path . '.target', + 'Conditional assignment requires an existing target.', + ); + } + $this->assertNumericVariable($target, $path . '.target', 'Conditional assignment target must be numeric.'); + $this->assertPredicate($operation->condition, $schema, $path . '.condition'); + $this->assertNumericOperand($operation->value, $schema, $path . '.value'); + return [null, $target]; + } + + if ($operation instanceof RecodeOperation) { + $source = $variables[$operation->source] ?? null; + if ($source === null) { + throw TransformationFailure::at('unknown_variable', $path . '.source', 'Recode source is not registered.'); + } + $target = $this->bindRecodeTarget( + $operation, + $source, + $path, + $variables, + $usedPhysical, + $nextOrdinal, + $postgresqlSlots, + ); + $this->assertRecode($operation, $source, $target, $path); + return [$source, $target]; + } + + if ($operation instanceof SetVariableLabelOperation) { + return [null, $schema->variable($operation->variable, $path . '.variable')]; + } + + if ($operation instanceof ReplaceValueLabelsOperation) { + $target = $schema->variable($operation->variable, $path . '.variable'); + $this->assertValueLabelAssociations($target, $path . '.variable'); + foreach ($operation->labels as $index => $label) { + $this->assertValueKind($label->value, $target, $path . '.labels[' . $index . '].value'); + } + return [null, $target]; + } + + if ($operation instanceof SetFormatOperation) { + $target = $schema->variable($operation->variable, $path . '.variable'); + $this->assertNumericVariable($target, $path . '.variable', 'F format requires a numeric variable.'); + if ($operation->family !== 'F' + || $operation->width < 1 + || $operation->width > 40 + || $operation->decimals < 0 + || $operation->decimals > 16 + || ($operation->decimals > 0 && $operation->width < $operation->decimals + 2) + ) { + throw TransformationFailure::at('invalid_format', $path, 'Invalid SPSS F format.'); + } + return [null, $target]; + } + + if ($operation instanceof SetMeasurementLevelOperation) { + return [null, $schema->variable($operation->variable, $path . '.variable')]; + } + + if ($operation instanceof ExecuteOperation) { + return [null, null]; + } + + throw TransformationFailure::at('plan_schema_invalid', $path, 'Unsupported official operation.'); + } + + /** + * @param array $variables + * @param array $usedPhysical + */ + private function bindNumericTarget( + string $name, + TargetMode $mode, + string $path, + array $variables, + array $usedPhysical, + int $nextOrdinal, + ?int $postgresqlSlots, + ): VariableBinding { + $existing = $variables[$name] ?? null; + if ($mode === TargetMode::Replace) { + if ($existing === null) { + throw TransformationFailure::at('unknown_variable', $path, 'Replace target is not registered.'); + } + $this->assertNumericVariable($existing, $path, 'Assignment target must be numeric.'); + return $existing; + } + if ($existing !== null) { + throw TransformationFailure::at('target_already_exists', $path, 'Create target is already registered.'); + } + + $this->assertCreateCapability(count($variables), $postgresqlSlots); + $physical = $this->connection->profile->physicalIdentifier($name, $usedPhysical); + return new VariableBinding(NormativeCatalog::uuid(), $name, $physical, 'numeric', $nextOrdinal, null, false); + } + + /** + * @param array $variables + * @param array $usedPhysical + */ + private function bindRecodeTarget( + RecodeOperation $operation, + VariableBinding $source, + string $path, + array $variables, + array $usedPhysical, + int $nextOrdinal, + ?int $postgresqlSlots, + ): VariableBinding { + if (str_starts_with($operation->target, '__')) { + throw TransformationFailure::at('reserved_target_name', $path . '.target', 'Target name is reserved.'); + } + $existing = $variables[$operation->target] ?? null; + if ($operation->targetMode === TargetMode::Replace) { + if ($existing === null) { + throw TransformationFailure::at('unknown_variable', $path . '.target', 'Replace target is not registered.'); + } + if ($existing->variableId !== $source->variableId) { + throw TransformationFailure::at( + 'type_mismatch', + $path . '.target', + 'Recode replace requires source and target to identify the same variable.', + ); + } + return $existing; + } + if ($existing !== null) { + throw TransformationFailure::at('target_already_exists', $path . '.target', 'Create target is already registered.'); + } + + $resultKind = $this->recodeResultKind($operation, $source, $path); + if ($resultKind === 'string') { + throw TransformationFailure::at( + 'string_target_requires_declaration', + $path . '.target', + 'A new string target requires separate explicit provisioning.', + ); + } + $this->assertCreateCapability(count($variables), $postgresqlSlots); + $physical = $this->connection->profile->physicalIdentifier($operation->target, $usedPhysical); + return new VariableBinding(NormativeCatalog::uuid(), $operation->target, $physical, 'numeric', $nextOrdinal, null, false); + } + + private function assertRecode( + RecodeOperation $operation, + VariableBinding $source, + VariableBinding $target, + string $path, + ): void { + foreach ($operation->rules as $index => $rule) { + $match = $rule->match; + if ($match instanceof ExactMatch) { + foreach ($match->values as $valueIndex => $value) { + $this->assertValueKind($value, $source, $path . '.rules[' . $index . '].match.values[' . $valueIndex . ']'); + } + } elseif ($match instanceof RangeMatch) { + $this->assertNumericVariable($source, $path . '.rules[' . $index . '].match', 'Numeric range requires a numeric source.'); + } elseif ($match instanceof SystemMissingMatch && !$source->isNumeric()) { + throw TransformationFailure::at('system_missing_for_string', $path . '.rules[' . $index . '].match', 'String variables have no system-missing state.'); + } + if ($rule->result instanceof LiteralResult) { + $this->assertValueKind($rule->result->value, $target, $path . '.rules[' . $index . '].result.value'); + } + } + if ($operation->unmatched instanceof LiteralResult) { + $this->assertValueKind($operation->unmatched->value, $target, $path . '.unmatched.value'); + } + + $kind = $this->recodeResultKind($operation, $source, $path); + if ($kind !== $target->storageKind) { + throw TransformationFailure::at('type_mismatch', $path . '.target', 'Recode result type does not match the target storage kind.'); + } + } + + private function recodeResultKind(RecodeOperation $operation, VariableBinding $source, string $path): string + { + $results = array_map(static fn(RecodeRule $rule): Result => $rule->result, $operation->rules); + $results[] = $operation->unmatched; + $kinds = []; + foreach ($results as $index => $result) { + $kind = match (true) { + $result instanceof CopyResult => $source->storageKind, + $result instanceof SystemMissingResult => 'numeric', + $result instanceof LiteralResult => $this->valueKind($result->value), + default => throw TransformationFailure::at('plan_schema_invalid', $path, 'Unsupported recode result.'), + }; + if ($result instanceof SystemMissingResult && !$source->isNumeric() && $operation->targetMode === TargetMode::Replace) { + throw TransformationFailure::at('system_missing_for_string', $path . '.results[' . $index . ']', 'String variables have no system-missing result.'); + } + $kinds[$kind] = true; + } + if (count($kinds) !== 1) { + throw TransformationFailure::at('mixed_result_types', $path, 'Recode results must have one storage kind.'); + } + + return (string) array_key_first($kinds); + } + + private function assertPredicate(Predicate $predicate, BoundSchema $schema, string $path): void + { + if ($predicate instanceof Comparison) { + $this->assertNumericOperand($predicate->left, $schema, $path . '.left'); + $this->assertNumericOperand($predicate->right, $schema, $path . '.right'); + return; + } + if ($predicate instanceof BooleanPredicate) { + foreach ($predicate->operands as $index => $child) { + $this->assertPredicate($child, $schema, $path . '.operands[' . $index . ']'); + } + return; + } + throw TransformationFailure::at('plan_schema_invalid', $path, 'Unsupported predicate.'); + } + + private function assertNumericOperand(Operand $operand, BoundSchema $schema, string $path): void + { + if ($operand instanceof LiteralOperand) { + if (!$operand->value instanceof Binary64Value) { + throw TransformationFailure::at('expression_type_unsupported', $path, 'Only numeric operands are executable.'); + } + return; + } + if ($operand instanceof VariableOperand) { + $variable = $schema->variable($operand->variable, $path . '.variable'); + if (!$variable->isNumeric()) { + throw TransformationFailure::at('expression_type_unsupported', $path, 'Only numeric operands are executable.'); + } + return; + } + throw TransformationFailure::at('plan_schema_invalid', $path, 'Unsupported operand.'); + } + + private function assertNumericVariable(VariableBinding $variable, string $path, string $message): void + { + if (!$variable->isNumeric()) { + throw TransformationFailure::at('type_mismatch', $path, $message); + } + } + + private function assertValueKind(TypedValue $value, VariableBinding $variable, string $path): void + { + if ($this->valueKind($value) !== $variable->storageKind) { + throw TransformationFailure::at('type_mismatch', $path, 'Typed value does not match variable storage kind.'); + } + if ($value instanceof StringValue + && ($variable->declaredStringWidth === null || strlen($value->value) > $variable->declaredStringWidth) + ) { + throw TransformationFailure::at('type_mismatch', $path, 'String value exceeds the target declared width.'); + } + } + + private function valueKind(TypedValue $value): string + { + return $value instanceof Binary64Value ? 'numeric' : 'string'; + } + + private function assertCreateCapability(int $variableCount, ?int $postgresqlSlots): void + { + $maximum = $this->connection->profile->effectiveMaximumSourceVariables($this->connection->pdo); + if ($variableCount >= $maximum + || ($postgresqlSlots !== null && $postgresqlSlots >= $maximum + 1) + ) { + throw new UnsupportedOperation( + DiagnosticCode::TargetCapabilityExceeded, + $this->connection->profileName . ' cannot add another source variable to this wide table.', + ); + } + } + + private function createsTarget(Operation $operation): bool + { + return ($operation instanceof AssignOperation || $operation instanceof RecodeOperation) + && $operation->targetMode === TargetMode::Create; + } + + private function resolveDataset(string $datasetId): DatasetBinding + { + $statement = $this->statement( + 'SELECT dataset_id, dataset_name, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', + ); + CheckedPdo::execute($statement, [$datasetId], 'The transformation dataset binding could not be queried.'); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + if (count($rows) !== 1) { + throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply dataset must resolve to exactly one catalog row.'); + } + $row = $rows[0]; + $name = $row['dataset_name'] ?? null; + $schema = $row['physical_table_schema'] ?? null; + $table = $row['physical_table_name'] ?? null; + if (($row['dataset_id'] ?? null) !== $datasetId + || ($name !== null && !is_string($name)) + || ($schema !== null && (!is_string($schema) || $schema === '')) + || !is_string($table) + || $table === '' + ) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Dataset physical binding is malformed.'); + } + + return new DatasetBinding($datasetId, $name, $schema, $table); + } + + private function assertExclusivePhysicalTableBinding(DatasetBinding $dataset): void + { + $statement = $this->statement( + 'SELECT dataset_id, physical_table_schema, physical_table_name FROM dataset', + ); + CheckedPdo::execute($statement, [], 'Physical table ownership could not be queried.'); + $canonicalSchema = $this->canonicalPhysicalSchema($dataset->schema); + $canonicalTable = $this->canonicalPhysicalTable($dataset->table); + $owners = array_values(array_filter( + $statement->fetchAll(PDO::FETCH_ASSOC), + function (array $row) use ($canonicalSchema, $canonicalTable): bool { + $schema = $row['physical_table_schema'] ?? null; + $table = $row['physical_table_name'] ?? null; + return is_string($table) + && $this->canonicalPhysicalTable($table) === $canonicalTable + && ($schema !== null && !is_string($schema) + || $this->canonicalPhysicalSchema($schema) === $canonicalSchema); + }, + )); + if (count($owners) !== 1 || ($owners[0]['dataset_id'] ?? null) !== $dataset->datasetId) { + throw TransformationFailure::at( + 'invalid_catalog', + '$.dataset_id', + 'The physical wide table must belong to exactly one logical dataset.', + ); + } + } + + private function canonicalPhysicalSchema(?string $schema): string + { + if ($this->connection->profileName === 'sqlite') { + return $schema === null || strcasecmp($schema, 'main') === 0 + ? 'main' + : strtolower($schema); + } + $profile = $this->connection->profile; + $foldMysqlFamily = $profile instanceof \OpenStatSpec\Sql\MySqlProfile + && $profile->lowerCaseTableNames($this->connection->pdo); + if ($schema !== null) { + return $foldMysqlFamily ? strtolower($schema) : $schema; + } + + $statement = $this->statement( + $this->connection->profileName === 'postgresql' + ? 'SELECT current_schema()' + : 'SELECT DATABASE()', + ); + CheckedPdo::execute($statement, [], 'The active physical schema could not be queried.'); + $active = $statement->fetchColumn(); + if (!is_string($active) || $active === '') { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'The active physical schema is unavailable.'); + } + + return $foldMysqlFamily ? strtolower((string) $active) : (string) $active; + } + + private function canonicalPhysicalTable(string $table): string + { + if ($this->connection->profileName === 'sqlite') { + return strtolower($table); + } + $profile = $this->connection->profile; + if ($profile instanceof \OpenStatSpec\Sql\MySqlProfile + && $profile->lowerCaseTableNames($this->connection->pdo) + ) { + return strtolower($table); + } + return $table; + } + + /** @return array */ + private function resolveVariables(DatasetBinding $dataset): array + { + $statement = $this->statement( + 'SELECT variable_id, source_name, physical_name, storage_kind, source_ordinal, declared_string_width ' + . 'FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', + ); + CheckedPdo::execute($statement, [$dataset->datasetId], 'Variable catalog bindings could not be queried.'); + $variables = []; + $physical = []; + $lastOrdinal = 0; + foreach ($statement->fetchAll(PDO::FETCH_ASSOC) as $row) { + $id = $row['variable_id'] ?? null; + $name = $row['source_name'] ?? null; + $physicalName = $row['physical_name'] ?? null; + $kind = $row['storage_kind'] ?? null; + $ordinal = filter_var($row['source_ordinal'] ?? null, FILTER_VALIDATE_INT); + $width = ($row['declared_string_width'] ?? null) === null + ? null + : filter_var($row['declared_string_width'], FILTER_VALIDATE_INT); + if (!is_string($id) || $id === '' + || !is_string($name) || $name === '' + || !is_string($physicalName) || $physicalName === '' + || str_starts_with($physicalName, '__') + || !in_array($kind, ['numeric', 'string'], true) + || !is_int($ordinal) || $ordinal <= $lastOrdinal + || ($kind === 'string' && (!is_int($width) || $width < 1)) + || isset($variables[$name]) || isset($physical[$physicalName]) + ) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Variable catalog binding is malformed or ambiguous.'); + } + $variables[$name] = new VariableBinding($id, $name, $physicalName, $kind, $ordinal, is_int($width) ? $width : null); + $physical[$physicalName] = true; + $lastOrdinal = $ordinal; + } + if ($variables === []) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Bound dataset has no catalog variables.'); + } + + return $variables; + } + + private function assertValueLabelAssociations(VariableBinding $target, string $path): void + { + $statement = $this->statement( + 'SELECT target_link.value_label_set_id, label_set.dataset_id AS set_dataset_id, ' + . 'linked_variable.dataset_id AS linked_dataset_id ' + . 'FROM variable_value_label_set target_link ' + . 'JOIN value_label_set label_set ON label_set.value_label_set_id = target_link.value_label_set_id ' + . 'JOIN variable_value_label_set linked_link ON linked_link.value_label_set_id = target_link.value_label_set_id ' + . 'JOIN variable linked_variable ON linked_variable.variable_id = linked_link.variable_id ' + . 'WHERE target_link.variable_id = ?', + ); + CheckedPdo::execute($statement, [$target->variableId], 'Value-label associations could not be queried.'); + foreach ($statement->fetchAll(PDO::FETCH_ASSOC) as $row) { + if (!is_string($row['value_label_set_id'] ?? null) + || ($row['set_dataset_id'] ?? null) !== ($row['linked_dataset_id'] ?? null) + ) { + throw TransformationFailure::at( + 'invalid_catalog', + $path, + 'Value-label sets and every linked variable must belong to the same dataset.', + ); + } + } + } + + /** + * @param array $variables + * @param list $physicalColumns + */ + private function assertPhysicalBindings(DatasetBinding $dataset, array $variables, array $physicalColumns): void + { + $columns = array_fill_keys($physicalColumns, true); + if (!isset($columns['__case_ordinal'])) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Wide table lacks the case-order column.'); + } + foreach ($variables as $variable) { + if (!isset($columns[$variable->physicalName])) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Catalog variable lacks its physical column.'); + } + } + + $selection = implode(', ', array_map( + fn(VariableBinding $variable): string => $this->connection->profile->quoteIdentifier($variable->physicalName), + array_values($variables), + )); + try { + $statement = $this->statement( + 'SELECT ' . $selection . ' FROM ' . $dataset->qualifiedTable($this->connection->profile) . ' WHERE 1 = 0', + ); + CheckedPdo::execute($statement, [], 'Cataloged wide-table bindings are not readable.'); + } catch (PDOException) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Cataloged wide-table bindings are not readable.'); + } + } + + /** @return list */ + private function physicalColumns(DatasetBinding $dataset): array + { + if ($this->connection->profileName === 'sqlite') { + $statement = $this->statement( + 'PRAGMA ' . $this->connection->profile->quoteIdentifier($dataset->schema ?? 'main') + . '.table_info(' . $this->connection->profile->quoteIdentifier($dataset->table) . ')', + ); + CheckedPdo::execute($statement, [], 'SQLite physical columns could not be queried.'); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + return array_values(array_map(static fn(array $row): string => (string) ($row['name'] ?? ''), $rows)); + } + if ($this->connection->profileName === 'postgresql') { + $statement = $this->statement( + 'SELECT column_name FROM information_schema.columns ' + . 'WHERE table_schema = COALESCE(?, current_schema()) AND table_name = ? ORDER BY ordinal_position', + ); + CheckedPdo::execute($statement, [$dataset->schema, $dataset->table], 'PostgreSQL physical columns could not be queried.'); + return array_values(array_map('strval', $statement->fetchAll(PDO::FETCH_COLUMN))); + } + if (in_array($this->connection->profileName, ['mysql', 'mariadb', 'dolt'], true)) { + $statement = $this->statement( + 'SELECT column_name FROM information_schema.columns ' + . 'WHERE table_schema = COALESCE(?, DATABASE()) AND table_name = ? ORDER BY ordinal_position', + ); + try { + CheckedPdo::execute($statement, [$dataset->schema, $dataset->table], 'MySQL-family physical columns could not be queried.'); + } catch (PDOException) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'MySQL-family physical columns could not be queried.'); + } + + return array_values(array_map('strval', $statement->fetchAll(PDO::FETCH_COLUMN))); + } + + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Physical columns could not be queried.'); + } + + private function postgresqlPhysicalColumnSlots(DatasetBinding $dataset): int + { + $statement = $this->statement('SELECT COUNT(*) FROM pg_attribute WHERE attrelid = to_regclass(?) AND attnum > 0'); + CheckedPdo::execute( + $statement, + [$dataset->qualifiedTable($this->connection->profile)], + 'PostgreSQL physical column slots could not be queried.', + ); + $slots = filter_var($statement->fetchColumn(), FILTER_VALIDATE_INT); + if (!is_int($slots) || $slots < 1) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'PostgreSQL wide table has no physical column slots.'); + } + return $slots; + } + + private function assertAuditReady(): void + { + try { + $statement = $this->statement( + 'SELECT apply_id, contract_id, database_profile, dataset_id, physical_table_schema, physical_table_name, ' + . 'source_hash, plan_hash, canonical_plan_json, actor, status, dolt_branch, dolt_head_before, dolt_head_after, ' + . 'operation_count, started_at, completed_at FROM transformation_apply WHERE 1 = 0', + ); + CheckedPdo::execute($statement, [], 'The transformation audit schema is not readable.'); + } catch (PDOException) { + throw new UnsupportedOperation( + DiagnosticCode::CatalogMigrationRequired, + 'The transformation audit schema is not ready before apply.', + ); + } + } + + private function assertMySqlFamilyTransactionalEngines(DatasetBinding $dataset): void + { + if (!in_array($this->connection->profileName, ['mysql', 'mariadb'], true)) { + return; + } + + $relations = [ + [$dataset->schema, $dataset->table], + [null, 'dataset'], + [null, 'variable'], + [null, 'value_label_set'], + [null, 'value_label'], + [null, 'variable_value_label_set'], + [null, 'transformation_apply'], + ]; + foreach ($relations as [$schema, $table]) { + try { + $statement = $this->statement( + 'SELECT table_type, engine FROM information_schema.tables ' + . 'WHERE table_schema = COALESCE(?, DATABASE()) AND table_name = ?', + ); + CheckedPdo::execute( + $statement, + [$schema, $table], + 'MySQL-family storage-engine metadata could not be queried.', + ); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + } catch (PDOException $failure) { + throw new UnsupportedOperation( + DiagnosticCode::SqlProfileOperationUnavailable, + 'MySQL-family transactional storage-engine metadata is unavailable: ' . $failure->getMessage(), + ); + } + + $row = isset($rows[0]) ? array_change_key_case($rows[0], CASE_LOWER) : []; + if (count($rows) !== 1 + || !is_string($row['table_type'] ?? null) + || strcasecmp($row['table_type'], 'BASE TABLE') !== 0 + || !is_string($row['engine'] ?? null) + || strcasecmp($row['engine'], 'InnoDB') !== 0 + ) { + throw new UnsupportedOperation( + DiagnosticCode::SqlProfileOperationUnavailable, + 'MySQL-family atomic apply requires every participating data, catalog, and audit table to use InnoDB.', + ); + } + } + } + + private function statement(string $sql): PDOStatement + { + return CheckedPdo::prepare( + $this->connection->pdo, + $sql, + 'Transformation preflight statement could not be prepared.', + ); + } +} diff --git a/src/Transformation/Execution/SqlPredicateCompiler.php b/src/Transformation/Execution/SqlPredicateCompiler.php new file mode 100644 index 0000000..8c56d16 --- /dev/null +++ b/src/Transformation/Execution/SqlPredicateCompiler.php @@ -0,0 +1,83 @@ + $parameters */ + public function compile(Predicate $predicate, BoundSchema $schema, array &$parameters): string + { + return $this->predicate($predicate, $schema, $parameters); + } + + /** @param list $parameters */ + public function operand(Operand $operand, BoundSchema $schema, array &$parameters): string + { + if ($operand instanceof VariableOperand) { + return $this->profile->quoteIdentifier( + $schema->variable($operand->variable)->physicalName, + ); + } + if ($operand instanceof LiteralOperand && $operand->value instanceof Binary64Value) { + return $this->numericLiteral($operand->value, $parameters); + } + + throw TransformationFailure::at('expression_type_unsupported', '$.value', 'Only numeric SQL operands are supported.'); + } + + /** @param list $parameters */ + public function numericLiteral(Binary64Value $value, array &$parameters): string + { + $parameters[] = $value->decimal(); + + return 'CAST(? AS ' . $this->profile->numericType() . ')'; + } + + /** @param list $parameters */ + private function predicate(Predicate $predicate, BoundSchema $schema, array &$parameters): string + { + return match (true) { + $predicate instanceof Comparison => $this->comparison($predicate, $schema, $parameters), + $predicate instanceof BooleanPredicate => $this->boolean($predicate, $schema, $parameters), + default => throw TransformationFailure::at('plan_schema_invalid', '$.condition', 'Unsupported predicate type.'), + }; + } + + /** @param list $parameters */ + private function comparison(Comparison $comparison, BoundSchema $schema, array &$parameters): string + { + $left = $this->operand($comparison->left, $schema, $parameters); + $right = $this->operand($comparison->right, $schema, $parameters); + + return '(' . $left . ' ' . $comparison->operator . ' ' . $right . ')'; + } + + /** @param list $parameters */ + private function boolean(BooleanPredicate $predicate, BoundSchema $schema, array &$parameters): string + { + $children = []; + foreach ($predicate->operands as $child) { + $children[] = $this->predicate($child, $schema, $parameters); + } + + return '(' . implode( + ' ' . strtoupper($predicate->operator) . ' ', + $children, + ) . ')'; + } +} diff --git a/src/Transformation/Execution/VariableBinding.php b/src/Transformation/Execution/VariableBinding.php index 521283f..1e3bd99 100644 --- a/src/Transformation/Execution/VariableBinding.php +++ b/src/Transformation/Execution/VariableBinding.php @@ -16,4 +16,9 @@ public function __construct( public ?int $declaredStringWidth, public bool $persisted = true, ) {} + + public function isNumeric(): bool + { + return $this->storageKind === 'numeric'; + } } diff --git a/src/Transformation/Model/Action/AssignValueAction.php b/src/Transformation/Model/Action/AssignValueAction.php deleted file mode 100644 index 46af1b0..0000000 --- a/src/Transformation/Model/Action/AssignValueAction.php +++ /dev/null @@ -1,29 +0,0 @@ -value; - } - - /** @return array{type: string, value: array} */ - public function canonicalArray(): array - { - return ['type' => $this->type(), 'value' => $this->value->canonicalArray()]; - } -} diff --git a/src/Transformation/Model/Action/CopySourceAction.php b/src/Transformation/Model/Action/CopySourceAction.php deleted file mode 100644 index 64fd47b..0000000 --- a/src/Transformation/Model/Action/CopySourceAction.php +++ /dev/null @@ -1,22 +0,0 @@ - $this->type()]; - } -} diff --git a/src/Transformation/Model/Action/SetMissingAction.php b/src/Transformation/Model/Action/SetMissingAction.php deleted file mode 100644 index 50444e4..0000000 --- a/src/Transformation/Model/Action/SetMissingAction.php +++ /dev/null @@ -1,22 +0,0 @@ - $this->type()]; - } -} diff --git a/src/Transformation/Model/CreateVariableOperation.php b/src/Transformation/Model/CreateVariableOperation.php deleted file mode 100644 index 038e298..0000000 --- a/src/Transformation/Model/CreateVariableOperation.php +++ /dev/null @@ -1,63 +0,0 @@ - self::MAX_STRING_WIDTH) { - throw new \InvalidArgumentException( - 'String variables support at most ' . self::MAX_STRING_WIDTH . ' bytes.', - ); - } - if ($storageKind === 'numeric' && $declaredStringWidth !== null) { - throw new \InvalidArgumentException('Numeric variables cannot declare a string width.'); - } - } - - public function type(): string - { - return 'create_variable'; - } - public function sourceVariable(): string - { - return $this->variable; - } - public function targetVariable(): string - { - return $this->variable; - } - public function storageKind(): string - { - return $this->storageKind; - } - public function declaredStringWidth(): ?int - { - return $this->declaredStringWidth; - } - - public function canonicalArray(): array - { - return array_filter([ - 'type' => $this->type(), - 'source_variable' => $this->variable, - 'target_variable' => $this->variable, - 'storage_kind' => $this->storageKind, - 'declared_string_width' => $this->declaredStringWidth, - ], static fn(mixed $value): bool => $value !== null); - } -} diff --git a/src/Transformation/Model/DeleteVariableOperation.php b/src/Transformation/Model/DeleteVariableOperation.php deleted file mode 100644 index 27a7110..0000000 --- a/src/Transformation/Model/DeleteVariableOperation.php +++ /dev/null @@ -1,31 +0,0 @@ -variable; - } - public function targetVariable(): string - { - return $this->variable; - } - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'source_variable' => $this->variable, - 'target_variable' => $this->variable, - ]; - } -} diff --git a/src/Transformation/Model/RecodeOperation.php b/src/Transformation/Model/RecodeOperation.php deleted file mode 100644 index 27b2b72..0000000 --- a/src/Transformation/Model/RecodeOperation.php +++ /dev/null @@ -1,50 +0,0 @@ - $rules */ - public function __construct( - private string $sourceVariable, - private string $targetVariable, - private array $rules, - ) {} - - public function type(): string - { - return 'recode'; - } - - public function sourceVariable(): string - { - return $this->sourceVariable; - } - - public function targetVariable(): string - { - return $this->targetVariable; - } - - /** @return list */ - public function rules(): array - { - return $this->rules; - } - - /** @return array{type: string, source_variable: string, target_variable: string, rules: list>} */ - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'source_variable' => $this->sourceVariable, - 'target_variable' => $this->targetVariable, - 'rules' => array_map( - static fn(RecodeRule $rule): array => $rule->canonicalArray(), - $this->rules, - ), - ]; - } -} diff --git a/src/Transformation/Model/RecodeRule.php b/src/Transformation/Model/RecodeRule.php deleted file mode 100644 index 905b10b..0000000 --- a/src/Transformation/Model/RecodeRule.php +++ /dev/null @@ -1,32 +0,0 @@ -selector; - } - - public function action(): RecodeAction - { - return $this->action; - } - - /** @return array{selector: array, action: array} */ - public function canonicalArray(): array - { - return [ - 'selector' => $this->selector->canonicalArray(), - 'action' => $this->action->canonicalArray(), - ]; - } -} diff --git a/src/Transformation/Model/ScalarValue.php b/src/Transformation/Model/ScalarValue.php deleted file mode 100644 index a068e19..0000000 --- a/src/Transformation/Model/ScalarValue.php +++ /dev/null @@ -1,80 +0,0 @@ -type; - } - - public function value(): float|string - { - return $this->value; - } - - public function numberValue(): float - { - if (!is_float($this->value)) { - throw new InvalidArgumentException('A string scalar has no numeric value.'); - } - - return $this->value; - } - - public function stringValue(): string - { - if (!is_string($this->value)) { - throw new InvalidArgumentException('A numeric scalar has no string value.'); - } - - return $this->value; - } - - /** @return array{type: string, binary64?: string, value?: string} */ - public function canonicalArray(): array - { - if (is_string($this->value)) { - return ['type' => 'string', 'value' => $this->value]; - } - - return ['type' => 'number', 'binary64' => bin2hex(pack('E', $this->normalisedNumber()))]; - } - - public function identity(): string - { - return $this->type . ':' . ($this->canonicalArray()['binary64'] ?? $this->value); - } - - private function normalisedNumber(): float - { - $value = $this->value; - if (!is_float($value)) { - throw new InvalidArgumentException('A string scalar has no numeric representation.'); - } - - // IEEE-754 considers both zero representations equal for recoding. - return $value === 0.0 ? 0.0 : $value; - } -} diff --git a/src/Transformation/Model/Selector/ElseSelector.php b/src/Transformation/Model/Selector/ElseSelector.php deleted file mode 100644 index 9bfbd57..0000000 --- a/src/Transformation/Model/Selector/ElseSelector.php +++ /dev/null @@ -1,22 +0,0 @@ - $this->type()]; - } -} diff --git a/src/Transformation/Model/Selector/ExactValueSelector.php b/src/Transformation/Model/Selector/ExactValueSelector.php deleted file mode 100644 index 4444f98..0000000 --- a/src/Transformation/Model/Selector/ExactValueSelector.php +++ /dev/null @@ -1,29 +0,0 @@ -value; - } - - /** @return array{type: string, value: array} */ - public function canonicalArray(): array - { - return ['type' => $this->type(), 'value' => $this->value->canonicalArray()]; - } -} diff --git a/src/Transformation/Model/Selector/MissingValueSelector.php b/src/Transformation/Model/Selector/MissingValueSelector.php deleted file mode 100644 index c52bb2a..0000000 --- a/src/Transformation/Model/Selector/MissingValueSelector.php +++ /dev/null @@ -1,22 +0,0 @@ - $this->type()]; - } -} diff --git a/src/Transformation/Model/Selector/NumericRangeSelector.php b/src/Transformation/Model/Selector/NumericRangeSelector.php deleted file mode 100644 index 73af548..0000000 --- a/src/Transformation/Model/Selector/NumericRangeSelector.php +++ /dev/null @@ -1,43 +0,0 @@ -lower; - } - - public function upper(): ?float - { - return $this->upper; - } - - /** @return array{type: string, lower: ?array, upper: ?array, bounds: string} */ - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'lower' => $this->lower === null ? null : ScalarValue::number($this->lower)->canonicalArray(), - 'upper' => $this->upper === null ? null : ScalarValue::number($this->upper)->canonicalArray(), - 'bounds' => 'inclusive', - ]; - } -} diff --git a/src/Transformation/Model/SetValueLabelsOperation.php b/src/Transformation/Model/SetValueLabelsOperation.php deleted file mode 100644 index ab6f4d9..0000000 --- a/src/Transformation/Model/SetValueLabelsOperation.php +++ /dev/null @@ -1,51 +0,0 @@ - $labels */ - public function __construct( - private string $targetVariable, - private array $labels, - ) {} - - public function type(): string - { - return 'set_value_labels'; - } - - public function sourceVariable(): string - { - return $this->targetVariable; - } - - public function targetVariable(): string - { - return $this->targetVariable; - } - - /** @return list */ - public function labels(): array - { - return $this->labels; - } - - /** @return array{type: string, source_variable: string, target_variable: string, replacement: string, labels: list>} */ - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'source_variable' => $this->targetVariable, - 'target_variable' => $this->targetVariable, - 'replacement' => 'complete', - 'labels' => array_map( - static fn(ValueLabel $label): array => $label->canonicalArray(), - $this->labels, - ), - ]; - } -} diff --git a/src/Transformation/Model/SetVariableLabelOperation.php b/src/Transformation/Model/SetVariableLabelOperation.php deleted file mode 100644 index 1293b5e..0000000 --- a/src/Transformation/Model/SetVariableLabelOperation.php +++ /dev/null @@ -1,45 +0,0 @@ -targetVariable; - } - - public function targetVariable(): string - { - return $this->targetVariable; - } - - public function label(): ?string - { - return $this->label; - } - - /** @return array{type: string, source_variable: string, target_variable: string, label: ?string} */ - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'source_variable' => $this->targetVariable, - 'target_variable' => $this->targetVariable, - 'label' => $this->label, - ]; - } -} diff --git a/src/Transformation/Model/TransformationOperation.php b/src/Transformation/Model/TransformationOperation.php deleted file mode 100644 index 8d63194..0000000 --- a/src/Transformation/Model/TransformationOperation.php +++ /dev/null @@ -1,18 +0,0 @@ - */ - public function canonicalArray(): array; -} diff --git a/src/Transformation/Model/TransformationPlan.php b/src/Transformation/Model/TransformationPlan.php deleted file mode 100644 index b063098..0000000 --- a/src/Transformation/Model/TransformationPlan.php +++ /dev/null @@ -1,60 +0,0 @@ - $operations */ - public function __construct( - private string $datasetId, - private array $operations, - ) {} - - public function datasetId(): string - { - return $this->datasetId; - } - - /** @return list */ - public function operations(): array - { - return $this->operations; - } - - /** @return array{contract: string, mode: string, dataset_id: string, operations: list>} */ - public function canonicalArray(): array - { - return [ - 'contract' => self::CONTRACT, - 'mode' => 'in_place', - 'dataset_id' => $this->datasetId, - 'operations' => array_map( - static fn(TransformationOperation $operation): array => $operation->canonicalArray(), - $this->operations, - ), - ]; - } - - public function canonicalJson(): string - { - return CanonicalJson::encode($this->canonicalArray()); - } - - public function hash(): string - { - return hash('sha256', $this->canonicalJson()); - } -} diff --git a/src/Transformation/Model/ValueLabel.php b/src/Transformation/Model/ValueLabel.php deleted file mode 100644 index 9828281..0000000 --- a/src/Transformation/Model/ValueLabel.php +++ /dev/null @@ -1,29 +0,0 @@ -value; - } - - public function label(): string - { - return $this->label; - } - - /** @return array{value: array, label: string} */ - public function canonicalArray(): array - { - return ['value' => $this->value->canonicalArray(), 'label' => $this->label]; - } -} diff --git a/src/Transformation/Plan/Expression/BooleanPredicate.php b/src/Transformation/Plan/Expression/BooleanPredicate.php new file mode 100644 index 0000000..113a2f7 --- /dev/null +++ b/src/Transformation/Plan/Expression/BooleanPredicate.php @@ -0,0 +1,44 @@ + */ + public array $operands; + + public function __construct(public string $operator, Predicate ...$operands) + { + if (!in_array($operator, ['and', 'or'], true)) { + throw TransformationFailure::at('plan_schema_invalid', '$.operator', 'Boolean operator must be and or or.'); + } + if (count($operands) < 2) { + throw TransformationFailure::at('plan_schema_invalid', '$.operands', 'A boolean expression requires at least two operands.'); + } + foreach ($operands as $operand) { + if ($operand instanceof self && $operand->operator === $operator) { + throw TransformationFailure::at( + 'noncanonical_boolean_shape', + '$.operands', + 'A same-operator boolean chain must be flattened in source order.', + ); + } + } + + $this->operands = array_values($operands); + } + + /** @return array{expression: string, operator: string, operands: list>} */ + public function canonicalArray(): array + { + return [ + 'expression' => 'boolean', + 'operator' => $this->operator, + 'operands' => array_map(static fn(Predicate $operand): array => $operand->canonicalArray(), $this->operands), + ]; + } +} diff --git a/src/Transformation/Plan/Expression/Comparison.php b/src/Transformation/Plan/Expression/Comparison.php new file mode 100644 index 0000000..258cbac --- /dev/null +++ b/src/Transformation/Plan/Expression/Comparison.php @@ -0,0 +1,31 @@ +', '>='], true)) { + throw TransformationFailure::at('plan_schema_invalid', '$.operator', 'Unsupported comparison operator.'); + } + } + + /** @return array{expression: string, left: array, operator: string, right: array} */ + public function canonicalArray(): array + { + return [ + 'expression' => 'comparison', + 'left' => $this->left->canonicalArray(), + 'operator' => $this->operator, + 'right' => $this->right->canonicalArray(), + ]; + } +} diff --git a/src/Transformation/Plan/Expression/LiteralOperand.php b/src/Transformation/Plan/Expression/LiteralOperand.php new file mode 100644 index 0000000..e28103b --- /dev/null +++ b/src/Transformation/Plan/Expression/LiteralOperand.php @@ -0,0 +1,18 @@ +} */ + public function canonicalArray(): array + { + return ['kind' => 'literal', 'value' => $this->value->canonicalArray()]; + } +} diff --git a/src/Transformation/Model/RecodeAction.php b/src/Transformation/Plan/Expression/Operand.php similarity index 53% rename from src/Transformation/Model/RecodeAction.php rename to src/Transformation/Plan/Expression/Operand.php index b495cbd..9b631b1 100644 --- a/src/Transformation/Model/RecodeAction.php +++ b/src/Transformation/Plan/Expression/Operand.php @@ -2,12 +2,10 @@ declare(strict_types=1); -namespace OpenStatSpec\Transformation\Model; +namespace OpenStatSpec\Transformation\Plan\Expression; -interface RecodeAction +interface Operand { - public function type(): string; - /** @return array */ public function canonicalArray(): array; } diff --git a/src/Transformation/Model/RecodeSelector.php b/src/Transformation/Plan/Expression/Predicate.php similarity index 53% rename from src/Transformation/Model/RecodeSelector.php rename to src/Transformation/Plan/Expression/Predicate.php index ba45a19..e001c7c 100644 --- a/src/Transformation/Model/RecodeSelector.php +++ b/src/Transformation/Plan/Expression/Predicate.php @@ -2,12 +2,10 @@ declare(strict_types=1); -namespace OpenStatSpec\Transformation\Model; +namespace OpenStatSpec\Transformation\Plan\Expression; -interface RecodeSelector +interface Predicate { - public function type(): string; - /** @return array */ public function canonicalArray(): array; } diff --git a/src/Transformation/Plan/Expression/VariableOperand.php b/src/Transformation/Plan/Expression/VariableOperand.php new file mode 100644 index 0000000..6add27c --- /dev/null +++ b/src/Transformation/Plan/Expression/VariableOperand.php @@ -0,0 +1,16 @@ + 'variable', 'variable' => $this->variable]; + } +} diff --git a/src/Transformation/Plan/Operation.php b/src/Transformation/Plan/Operation.php new file mode 100644 index 0000000..77396ea --- /dev/null +++ b/src/Transformation/Plan/Operation.php @@ -0,0 +1,13 @@ + */ + public function canonicalArray(): array; + + public function minimumContract(): PlanContract; +} diff --git a/src/Transformation/Plan/Operation/AssignOperation.php b/src/Transformation/Plan/Operation/AssignOperation.php new file mode 100644 index 0000000..cfdc3e9 --- /dev/null +++ b/src/Transformation/Plan/Operation/AssignOperation.php @@ -0,0 +1,30 @@ +} */ + public function canonicalArray(): array + { + return ['op' => 'assign', 'target' => $this->target, 'target_mode' => $this->targetMode->value, 'value' => $this->value->canonicalArray()]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/Operation/ConditionalAssignOperation.php b/src/Transformation/Plan/Operation/ConditionalAssignOperation.php new file mode 100644 index 0000000..c15c1a6 --- /dev/null +++ b/src/Transformation/Plan/Operation/ConditionalAssignOperation.php @@ -0,0 +1,30 @@ +, target: string, value: array} */ + public function canonicalArray(): array + { + return ['op' => 'conditional_assign', 'condition' => $this->condition->canonicalArray(), 'target' => $this->target, 'value' => $this->value->canonicalArray()]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/Operation/ExecuteOperation.php b/src/Transformation/Plan/Operation/ExecuteOperation.php new file mode 100644 index 0000000..66bfe6c --- /dev/null +++ b/src/Transformation/Plan/Operation/ExecuteOperation.php @@ -0,0 +1,22 @@ + 'execute']; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/Operation/RecodeOperation.php b/src/Transformation/Plan/Operation/RecodeOperation.php new file mode 100644 index 0000000..6618a74 --- /dev/null +++ b/src/Transformation/Plan/Operation/RecodeOperation.php @@ -0,0 +1,45 @@ + $rules */ + public function __construct( + public string $source, + public string $target, + public TargetMode $targetMode, + public array $rules, + public Result $unmatched, + ) { + if ($rules === []) { + throw new \InvalidArgumentException('A recode operation requires at least one rule.'); + } + } + + /** @return array */ + public function canonicalArray(): array + { + return [ + 'op' => 'recode', + 'source' => $this->source, + 'target' => $this->target, + 'target_mode' => $this->targetMode->value, + 'rules' => array_map(static fn(RecodeRule $rule): array => $rule->canonicalArray(), $this->rules), + 'unmatched' => $this->unmatched->canonicalArray(), + ]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V01; + } +} diff --git a/src/Transformation/Plan/Operation/ReplaceValueLabelsOperation.php b/src/Transformation/Plan/Operation/ReplaceValueLabelsOperation.php new file mode 100644 index 0000000..fc0fa77 --- /dev/null +++ b/src/Transformation/Plan/Operation/ReplaceValueLabelsOperation.php @@ -0,0 +1,32 @@ + $labels */ + public function __construct( + public string $variable, + public array $labels, + ) {} + + /** @return array */ + public function canonicalArray(): array + { + return [ + 'op' => 'replace_value_labels', + 'variable' => $this->variable, + 'labels' => array_map(static fn(ValueLabel $label): array => $label->canonicalArray(), $this->labels), + ]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V01; + } +} diff --git a/src/Transformation/Plan/Operation/SetFormatOperation.php b/src/Transformation/Plan/Operation/SetFormatOperation.php new file mode 100644 index 0000000..ddf0b95 --- /dev/null +++ b/src/Transformation/Plan/Operation/SetFormatOperation.php @@ -0,0 +1,29 @@ + 'set_format', 'variable' => $this->variable, 'family' => $this->family, 'width' => $this->width, 'decimals' => $this->decimals]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/Operation/SetMeasurementLevelOperation.php b/src/Transformation/Plan/Operation/SetMeasurementLevelOperation.php new file mode 100644 index 0000000..ecc2498 --- /dev/null +++ b/src/Transformation/Plan/Operation/SetMeasurementLevelOperation.php @@ -0,0 +1,27 @@ + 'set_measurement_level', 'variable' => $this->variable, 'level' => $this->level]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/Operation/SetVariableLabelOperation.php b/src/Transformation/Plan/Operation/SetVariableLabelOperation.php new file mode 100644 index 0000000..24f2840 --- /dev/null +++ b/src/Transformation/Plan/Operation/SetVariableLabelOperation.php @@ -0,0 +1,27 @@ + 'set_variable_label', 'variable' => $this->variable, 'label' => $this->label]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V01; + } +} diff --git a/src/Transformation/Plan/Operation/ValueLabel.php b/src/Transformation/Plan/Operation/ValueLabel.php new file mode 100644 index 0000000..6025c3a --- /dev/null +++ b/src/Transformation/Plan/Operation/ValueLabel.php @@ -0,0 +1,21 @@ +, label: string} */ + public function canonicalArray(): array + { + return ['value' => $this->value->canonicalArray(), 'label' => $this->label]; + } +} diff --git a/src/Transformation/Plan/PlanCodec.php b/src/Transformation/Plan/PlanCodec.php new file mode 100644 index 0000000..c8f56aa --- /dev/null +++ b/src/Transformation/Plan/PlanCodec.php @@ -0,0 +1,495 @@ +schema('$', 'Plan JSON must be valid JSON.'); + } + + if (!is_array($decoded)) { + $this->schema('$', 'Plan must be a JSON object.'); + } + + return $this->fromArray($decoded); + } + + /** @param array $plan */ + public function fromArray(array $plan): TransformationPlan + { + $this->exactKeys($plan, ['contract', 'input_alias', 'operations'], '$'); + $contract = PlanContract::tryFrom($this->string($plan['contract'], '$.contract')); + if ($contract === null) { + $this->schema('$.contract', 'Plan contract must be openstatspec-transformation-plan-v0.1 or openstatspec-transformation-plan-v0.2.'); + } + + $inputAlias = $this->nonEmptyString($plan['input_alias'], '$.input_alias'); + $rawOperations = $this->nonEmptyList($plan['operations'], '$.operations'); + $operations = []; + foreach ($rawOperations as $index => $rawOperation) { + $path = '$.operations[' . $index . ']'; + $operation = $this->operation($rawOperation, $path); + if ($contract === PlanContract::V01 && $operation->minimumContract() !== PlanContract::V01) { + $this->schema($path, 'Plan contract v0.1 cannot contain v0.2 operations.'); + } + $operations[] = $operation; + } + + return new TransformationPlan($contract, $inputAlias, $operations); + } + + public function canonicalJson(TransformationPlan $plan): string + { + return CanonicalJson::encode($plan->canonicalArray()); + } + + public function hash(TransformationPlan $plan): string + { + return hash('sha256', $this->canonicalJson($plan)); + } + + private function operation(mixed $raw, string $path): Operation + { + $object = $this->object($raw, $path); + $op = $this->string($object['op'] ?? null, $path . '.op'); + + return match ($op) { + 'recode' => $this->recodeOperation($object, $path), + 'assign' => $this->assignOperation($object, $path), + 'conditional_assign' => $this->conditionalAssignOperation($object, $path), + 'set_variable_label' => $this->setVariableLabelOperation($object, $path), + 'replace_value_labels' => $this->replaceValueLabelsOperation($object, $path), + 'set_format' => $this->setFormatOperation($object, $path), + 'set_measurement_level' => $this->setMeasurementLevelOperation($object, $path), + 'execute' => $this->executeOperation($object, $path), + default => $this->schema($path . '.op', 'Unknown operation.'), + }; + } + + /** @param array $object */ + private function assignOperation(array $object, string $path): AssignOperation + { + $this->exactKeys($object, ['op', 'target', 'target_mode', 'value'], $path); + $mode = TargetMode::tryFrom($this->string($object['target_mode'], $path . '.target_mode')); + if ($mode === null) { + $this->schema($path . '.target_mode', 'Assign target mode must be create or replace.'); + } + + return new AssignOperation( + $this->targetName($object['target'], $path . '.target'), + $mode, + $this->numericOperand($object['value'], $path . '.value'), + ); + } + + /** @param array $object */ + private function conditionalAssignOperation(array $object, string $path): ConditionalAssignOperation + { + $this->exactKeys($object, ['op', 'condition', 'target', 'value'], $path); + + return new ConditionalAssignOperation( + $this->predicate($object['condition'], $path . '.condition'), + $this->targetName($object['target'], $path . '.target'), + $this->numericOperand($object['value'], $path . '.value'), + ); + } + + /** @param array $object */ + private function setFormatOperation(array $object, string $path): SetFormatOperation + { + $this->exactKeys($object, ['op', 'variable', 'family', 'width', 'decimals'], $path); + $family = $this->string($object['family'], $path . '.family'); + if ($family !== 'F') { + $this->schema($path . '.family', 'The bounded format profile supports numeric F formats only.'); + } + + $operation = new SetFormatOperation( + $this->nonEmptyString($object['variable'], $path . '.variable'), + $family, + $this->integer($object['width'], $path . '.width'), + $this->integer($object['decimals'], $path . '.decimals'), + ); + if ($operation->width < 1 || $operation->width > 40 + || $operation->decimals < 0 || $operation->decimals > 16 + || ($operation->decimals > 0 && $operation->width < $operation->decimals + 2)) { + throw TransformationFailure::at('invalid_format', $path, 'Invalid SPSS F format.'); + } + + return $operation; + } + + /** @param array $object */ + private function setMeasurementLevelOperation(array $object, string $path): SetMeasurementLevelOperation + { + $this->exactKeys($object, ['op', 'variable', 'level'], $path); + $level = $this->string($object['level'], $path . '.level'); + if (!in_array($level, ['nominal', 'ordinal', 'scale'], true)) { + $this->schema($path . '.level', 'Measurement level must be nominal, ordinal, or scale.'); + } + + return new SetMeasurementLevelOperation( + $this->nonEmptyString($object['variable'], $path . '.variable'), + $level, + ); + } + + /** @param array $object */ + private function executeOperation(array $object, string $path): ExecuteOperation + { + $this->exactKeys($object, ['op'], $path); + return new ExecuteOperation(); + } + + private function predicate(mixed $raw, string $path): Predicate + { + $object = $this->object($raw, $path); + $expression = $this->string($object['expression'] ?? null, $path . '.expression'); + + if ($expression === 'comparison') { + $this->exactKeys($object, ['expression', 'left', 'operator', 'right'], $path); + return new Comparison( + $this->numericOperand($object['left'], $path . '.left'), + $this->string($object['operator'], $path . '.operator'), + $this->numericOperand($object['right'], $path . '.right'), + ); + } + + if ($expression === 'boolean') { + $this->exactKeys($object, ['expression', 'operator', 'operands'], $path); + $operator = $this->string($object['operator'], $path . '.operator'); + $rawOperands = $this->nonEmptyList($object['operands'], $path . '.operands'); + if (count($rawOperands) < 2) { + $this->schema($path . '.operands', 'A boolean expression requires at least two operands.'); + } + $operands = []; + foreach ($rawOperands as $index => $rawOperand) { + $operand = $this->predicate($rawOperand, $path . '.operands[' . $index . ']'); + if ($operand instanceof BooleanPredicate && $operand->operator === $operator) { + throw TransformationFailure::at( + 'noncanonical_boolean_shape', + $path . '.operands[' . $index . ']', + 'A same-operator boolean chain must be flattened in source order.', + ); + } + $operands[] = $operand; + } + + return new BooleanPredicate($operator, ...$operands); + } + + $this->schema($path . '.expression', 'Unknown predicate expression.'); + } + + private function numericOperand(mixed $raw, string $path): Operand + { + $operand = $this->operand($raw, $path); + if ($operand instanceof LiteralOperand && $operand->value instanceof StringValue) { + throw TransformationFailure::at( + 'expression_type_unsupported', + $path, + 'String literal operands are outside the bounded expression profile.', + ); + } + + return $operand; + } + + private function operand(mixed $raw, string $path): Operand + { + $object = $this->object($raw, $path); + $kind = $this->string($object['kind'] ?? null, $path . '.kind'); + if ($kind === 'variable') { + $this->exactKeys($object, ['kind', 'variable'], $path); + return new VariableOperand($this->nonEmptyString($object['variable'], $path . '.variable')); + } + if ($kind === 'literal') { + $this->exactKeys($object, ['kind', 'value'], $path); + return new LiteralOperand($this->typedValue($object['value'], $path . '.value')); + } + + $this->schema($path . '.kind', 'Operand kind must be variable or literal.'); + } + + private function targetName(mixed $value, string $path): string + { + $target = $this->nonEmptyString($value, $path); + if (str_starts_with($target, '__')) { + throw TransformationFailure::at('reserved_target_name', $path, 'Target name is reserved.'); + } + + return $target; + } + + /** @param array $object */ + private function recodeOperation(array $object, string $path): RecodeOperation + { + $this->exactKeys($object, ['op', 'source', 'target', 'target_mode', 'rules', 'unmatched'], $path); + $mode = $this->string($object['target_mode'], $path . '.target_mode'); + if (($targetMode = TargetMode::tryFrom($mode)) === null) { + $this->schema($path . '.target_mode', 'Recode target mode must be create or replace.'); + } + + $rawRules = $this->nonEmptyList($object['rules'], $path . '.rules'); + $rules = []; + foreach ($rawRules as $index => $rawRule) { + $rules[] = $this->rule($rawRule, $path . '.rules[' . $index . ']'); + } + + return new RecodeOperation( + $this->nonEmptyString($object['source'], $path . '.source'), + $this->targetName($object['target'], $path . '.target'), + $targetMode, + $rules, + $this->result($object['unmatched'], $path . '.unmatched'), + ); + } + + /** @param array $object */ + private function setVariableLabelOperation(array $object, string $path): SetVariableLabelOperation + { + $this->exactKeys($object, ['op', 'variable', 'label'], $path); + + return new SetVariableLabelOperation( + $this->nonEmptyString($object['variable'], $path . '.variable'), + $this->string($object['label'], $path . '.label'), + ); + } + + /** @param array $object */ + private function replaceValueLabelsOperation(array $object, string $path): ReplaceValueLabelsOperation + { + $this->exactKeys($object, ['op', 'variable', 'labels'], $path); + $rawLabels = $this->nonEmptyList($object['labels'], $path . '.labels'); + $labels = []; + $seen = []; + foreach ($rawLabels as $index => $rawLabel) { + $label = $this->valueLabel($rawLabel, $path . '.labels[' . $index . ']'); + $key = $label->value->canonicalKey(); + if (isset($seen[$key])) { + throw TransformationFailure::at( + 'duplicate_value_label', + $path . '.labels[' . $index . '].value', + 'Value labels must have unique exact typed values.', + ); + } + $seen[$key] = true; + $labels[] = $label; + } + + return new ReplaceValueLabelsOperation( + $this->nonEmptyString($object['variable'], $path . '.variable'), + $labels, + ); + } + + private function rule(mixed $raw, string $path): RecodeRule + { + $object = $this->object($raw, $path); + $this->exactKeys($object, ['match', 'result'], $path); + + return new RecodeRule( + $this->recodeMatch($object['match'], $path . '.match'), + $this->result($object['result'], $path . '.result'), + ); + } + + private function recodeMatch(mixed $raw, string $path): RecodeMatch + { + $object = $this->object($raw, $path); + $kind = $this->string($object['kind'] ?? null, $path . '.kind'); + + if ($kind === 'values') { + $this->exactKeys($object, ['kind', 'values'], $path); + $rawValues = $this->nonEmptyList($object['values'], $path . '.values'); + $values = []; + foreach ($rawValues as $index => $rawValue) { + $values[] = $this->typedValue($rawValue, $path . '.values[' . $index . ']'); + } + return new ExactMatch(...$values); + } + + if ($kind === 'range') { + $this->exactKeys($object, ['kind', 'lower', 'upper'], $path); + $lower = $this->typedValue($object['lower'], $path . '.lower'); + $upper = $this->typedValue($object['upper'], $path . '.upper'); + if (!$lower instanceof Binary64Value || !$upper instanceof Binary64Value) { + $this->schema($path, 'Range endpoints must be binary64 values.'); + } + if ($lower->number() > $upper->number()) { + throw TransformationFailure::at( + 'invalid_numeric_range', + $path, + 'Range lower endpoint cannot exceed its upper endpoint.', + ); + } + return new RangeMatch($lower, $upper); + } + + if ($kind === 'system_missing') { + $this->exactKeys($object, ['kind'], $path); + return new SystemMissingMatch(); + } + + $this->schema($path . '.kind', 'Unknown match kind.'); + } + + private function result(mixed $raw, string $path): Result + { + $object = $this->object($raw, $path); + $kind = $this->string($object['kind'] ?? null, $path . '.kind'); + + return match ($kind) { + 'literal' => $this->literalResult($object, $path), + 'copy' => $this->emptyResult($object, $path, new CopyResult()), + 'system_missing' => $this->emptyResult($object, $path, new SystemMissingResult()), + default => $this->schema($path . '.kind', 'Unknown result kind.'), + }; + } + + /** @param array $object */ + private function literalResult(array $object, string $path): LiteralResult + { + $this->exactKeys($object, ['kind', 'value'], $path); + return new LiteralResult($this->typedValue($object['value'], $path . '.value')); + } + + /** @param array $object */ + private function emptyResult(array $object, string $path, Result $result): Result + { + $this->exactKeys($object, ['kind'], $path); + return $result; + } + + private function valueLabel(mixed $raw, string $path): ValueLabel + { + $object = $this->object($raw, $path); + $this->exactKeys($object, ['value', 'label'], $path); + + return new ValueLabel( + $this->typedValue($object['value'], $path . '.value'), + $this->string($object['label'], $path . '.label'), + ); + } + + private function typedValue(mixed $raw, string $path): TypedValue + { + $object = $this->object($raw, $path); + $type = $this->string($object['type'] ?? null, $path . '.type'); + if ($type === 'binary64') { + $this->exactKeys($object, ['type', 'bits'], $path); + $bits = $this->string($object['bits'], $path . '.bits'); + try { + return Binary64Value::fromBits($bits); + } catch (TransformationFailure $failure) { + $this->schema($path . '.bits', $failure->getMessage()); + } + } + if ($type === 'string') { + $this->exactKeys($object, ['type', 'value'], $path); + return new StringValue($this->string($object['value'], $path . '.value')); + } + + $this->schema($path . '.type', 'Typed value type must be binary64 or string.'); + } + + /** @return array */ + private function object(mixed $value, string $path): array + { + if (!is_array($value) || array_is_list($value)) { + $this->schema($path, 'Value must be an object.'); + } + return $value; + } + + /** @return non-empty-list */ + private function nonEmptyList(mixed $value, string $path): array + { + if (!is_array($value) || !array_is_list($value) || $value === []) { + $this->schema($path, 'Value must be a non-empty array.'); + } + return $value; + } + + private function integer(mixed $value, string $path): int + { + if (!is_int($value)) { + $this->schema($path, 'Value must be an integer.'); + } + + return $value; + } + + /** + * @param array $object + * @param list $expected + */ + private function exactKeys(array $object, array $expected, string $path): void + { + $actual = array_keys($object); + sort($actual, SORT_STRING); + sort($expected, SORT_STRING); + if ($actual !== $expected) { + $this->schema($path, 'Object has missing or extra members.'); + } + } + + private function nonEmptyString(mixed $value, string $path): string + { + $string = $this->string($value, $path); + if ($string === '') { + $this->schema($path, 'String must not be empty.'); + } + return $string; + } + + private function string(mixed $value, string $path): string + { + if (!is_string($value) || preg_match('//u', $value) !== 1) { + $this->schema($path, 'Value must be a valid UTF-8 string.'); + } + return $value; + } + + private function schema(string $path, string $message): never + { + throw TransformationFailure::at('plan_schema_invalid', $path, $message); + } +} diff --git a/src/Transformation/Plan/PlanContract.php b/src/Transformation/Plan/PlanContract.php new file mode 100644 index 0000000..16936a4 --- /dev/null +++ b/src/Transformation/Plan/PlanContract.php @@ -0,0 +1,11 @@ + 'copy']; + } +} diff --git a/src/Transformation/Plan/Recode/ExactMatch.php b/src/Transformation/Plan/Recode/ExactMatch.php new file mode 100644 index 0000000..c97e819 --- /dev/null +++ b/src/Transformation/Plan/Recode/ExactMatch.php @@ -0,0 +1,31 @@ + */ + public array $values; + + public function __construct(TypedValue ...$values) + { + if ($values === []) { + throw new \InvalidArgumentException('A values match requires at least one value.'); + } + + $this->values = array_values($values); + } + + /** @return array{kind: string, values: list>} */ + public function canonicalArray(): array + { + return [ + 'kind' => 'values', + 'values' => array_map(static fn(TypedValue $value): array => $value->canonicalArray(), $this->values), + ]; + } +} diff --git a/src/Transformation/Plan/Recode/LiteralResult.php b/src/Transformation/Plan/Recode/LiteralResult.php new file mode 100644 index 0000000..ac1d472 --- /dev/null +++ b/src/Transformation/Plan/Recode/LiteralResult.php @@ -0,0 +1,18 @@ +} */ + public function canonicalArray(): array + { + return ['kind' => 'literal', 'value' => $this->value->canonicalArray()]; + } +} diff --git a/src/Transformation/Plan/Recode/Match.php b/src/Transformation/Plan/Recode/Match.php new file mode 100644 index 0000000..5e8a252 --- /dev/null +++ b/src/Transformation/Plan/Recode/Match.php @@ -0,0 +1,5 @@ +, upper: array} */ + public function canonicalArray(): array + { + return [ + 'kind' => 'range', + 'lower' => $this->lower->canonicalArray(), + 'upper' => $this->upper->canonicalArray(), + ]; + } +} diff --git a/src/Transformation/Plan/Recode/RecodeMatch.php b/src/Transformation/Plan/Recode/RecodeMatch.php new file mode 100644 index 0000000..862fac8 --- /dev/null +++ b/src/Transformation/Plan/Recode/RecodeMatch.php @@ -0,0 +1,11 @@ + */ + public function canonicalArray(): array; +} diff --git a/src/Transformation/Plan/Recode/RecodeRule.php b/src/Transformation/Plan/Recode/RecodeRule.php new file mode 100644 index 0000000..3f348bc --- /dev/null +++ b/src/Transformation/Plan/Recode/RecodeRule.php @@ -0,0 +1,19 @@ +, result: array} */ + public function canonicalArray(): array + { + return ['match' => $this->match->canonicalArray(), 'result' => $this->result->canonicalArray()]; + } +} diff --git a/src/Transformation/Plan/Recode/Result.php b/src/Transformation/Plan/Recode/Result.php new file mode 100644 index 0000000..07fc52d --- /dev/null +++ b/src/Transformation/Plan/Recode/Result.php @@ -0,0 +1,11 @@ + */ + public function canonicalArray(): array; +} diff --git a/src/Transformation/Plan/Recode/SystemMissingMatch.php b/src/Transformation/Plan/Recode/SystemMissingMatch.php new file mode 100644 index 0000000..b137f54 --- /dev/null +++ b/src/Transformation/Plan/Recode/SystemMissingMatch.php @@ -0,0 +1,14 @@ + 'system_missing']; + } +} diff --git a/src/Transformation/Plan/Recode/SystemMissingResult.php b/src/Transformation/Plan/Recode/SystemMissingResult.php new file mode 100644 index 0000000..178bd9e --- /dev/null +++ b/src/Transformation/Plan/Recode/SystemMissingResult.php @@ -0,0 +1,14 @@ + 'system_missing']; + } +} diff --git a/src/Transformation/Plan/TargetMode.php b/src/Transformation/Plan/TargetMode.php new file mode 100644 index 0000000..d6943ab --- /dev/null +++ b/src/Transformation/Plan/TargetMode.php @@ -0,0 +1,11 @@ + $operations */ + public function __construct( + public PlanContract $contract, + public string $inputAlias, + public array $operations, + ) {} + + /** @return array{contract: string, input_alias: string, operations: list>} */ + public function canonicalArray(): array + { + return [ + 'contract' => $this->contract->value, + 'input_alias' => $this->inputAlias, + 'operations' => array_map( + static fn(Operation $operation): array => $operation->canonicalArray(), + $this->operations, + ), + ]; + } +} diff --git a/src/Transformation/Plan/Value/Binary64Value.php b/src/Transformation/Plan/Value/Binary64Value.php new file mode 100644 index 0000000..096392e --- /dev/null +++ b/src/Transformation/Plan/Value/Binary64Value.php @@ -0,0 +1,67 @@ + 'binary64', 'bits' => $this->bits]; + } + + public function canonicalKey(): string + { + return 'binary64:' . $this->bits; + } + + public function number(): float + { + $binary = hex2bin($this->bits); + if ($binary === false) { + throw new \LogicException('Validated binary64 bits could not be decoded.'); + } + + $decoded = unpack('Evalue', $binary); + if (!is_array($decoded) || !isset($decoded['value']) || !is_float($decoded['value'])) { + throw new \LogicException('Validated binary64 bits could not be unpacked.'); + } + + return $decoded['value']; + } + + /** Decimal text whose conversion back to binary64 preserves the exact bits. */ + public function decimal(): string + { + $decimal = sprintf('%.17g', $this->number()); + $separator = localeconv()['decimal_point'] ?? '.'; + if (is_string($separator) && $separator !== '' && $separator !== '.') { + $decimal = str_replace($separator, '.', $decimal); + } + + return strtolower($decimal); + } +} diff --git a/src/Transformation/Plan/Value/StringValue.php b/src/Transformation/Plan/Value/StringValue.php new file mode 100644 index 0000000..1d32f80 --- /dev/null +++ b/src/Transformation/Plan/Value/StringValue.php @@ -0,0 +1,21 @@ + 'string', 'value' => $this->value]; + } + + public function canonicalKey(): string + { + return 'string:' . $this->value; + } +} diff --git a/src/Transformation/Plan/Value/TypedValue.php b/src/Transformation/Plan/Value/TypedValue.php new file mode 100644 index 0000000..cdcea17 --- /dev/null +++ b/src/Transformation/Plan/Value/TypedValue.php @@ -0,0 +1,13 @@ + */ + public function canonicalArray(): array; + + public function canonicalKey(): string; +} diff --git a/src/Transformation/Validation/InvalidTransformationPlan.php b/src/Transformation/Validation/InvalidTransformationPlan.php deleted file mode 100644 index 61d9834..0000000 --- a/src/Transformation/Validation/InvalidTransformationPlan.php +++ /dev/null @@ -1,28 +0,0 @@ - $violations */ - public function __construct(private readonly array $violations) - { - $first = $violations[0]; - parent::__construct(sprintf( - 'Invalid transformation plan (%s at %s): %s', - $first->code, - $first->path, - $first->message, - )); - } - - /** @return non-empty-list */ - public function violations(): array - { - return $this->violations; - } -} diff --git a/src/Transformation/Validation/PlanValidator.php b/src/Transformation/Validation/PlanValidator.php deleted file mode 100644 index 2fd866c..0000000 --- a/src/Transformation/Validation/PlanValidator.php +++ /dev/null @@ -1,301 +0,0 @@ - */ - private array $violations = []; - - public function validate(TransformationPlan $plan): ValidationResult - { - $this->violations = []; - if (preg_match('/\A[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/D', $plan->datasetId()) !== 1) { - $this->violation( - 'dataset_id.invalid_uuid', - '$.dataset_id', - 'The dataset identity must be a canonical lowercase RFC 9562 UUID string.', - ); - } - - if ($plan->operations() === []) { - $this->violation('operations.empty', '$.operations', 'A transformation plan must contain at least one operation.'); - } - - foreach ($plan->operations() as $index => $operation) { - $path = '$.operations[' . $index . ']'; - $this->validateOperation($operation, $path); - } - - return new ValidationResult($this->violations); - } - - public function assertValid(TransformationPlan $plan): void - { - $this->validate($plan)->throwIfInvalid(); - } - - private function validateOperation(TransformationOperation $operation, string $path): void - { - if (!in_array($operation::class, [ - CreateVariableOperation::class, - DeleteVariableOperation::class, - RecodeOperation::class, - SetVariableLabelOperation::class, - SetValueLabelsOperation::class, - ], true)) { - $this->violation( - 'operation.unsupported_type', - $path . '.type', - 'Only canonical, source-language-neutral operation types are permitted.', - ); - - return; - } - - $this->validateVariableName($operation->sourceVariable(), $path . '.source_variable'); - $this->validateVariableName($operation->targetVariable(), $path . '.target_variable'); - - if ($operation instanceof CreateVariableOperation) { - if ($operation->storageKind() === 'string') { - $width = $operation->declaredStringWidth(); - if ($width === null || $width < 1) { - $this->violation('create_variable.string_width_required', $path . '.declared_string_width', 'String variables require a positive declared string width.'); - } elseif ($width > CreateVariableOperation::MAX_STRING_WIDTH) { - $this->violation('create_variable.string_width_too_large', $path . '.declared_string_width', 'String variables support at most ' . CreateVariableOperation::MAX_STRING_WIDTH . ' bytes.'); - } - } - } elseif ($operation instanceof DeleteVariableOperation) { - return; - } elseif ($operation instanceof RecodeOperation) { - $this->validateRecode($operation, $path); - } elseif ($operation instanceof SetVariableLabelOperation) { - $this->validateText($operation->label(), $path . '.label'); - } elseif ($operation instanceof SetValueLabelsOperation) { - $this->validateValueLabels($operation, $path); - } - } - - private function validateRecode(RecodeOperation $operation, string $path): void - { - if ($operation->rules() === []) { - $this->violation('recode.rules_empty', $path . '.rules', 'A recode operation must contain ordered rules.'); - - return; - } - - /** @var array $exactValues */ - $exactValues = []; - /** @var list $ranges */ - $ranges = []; - $missingIndex = null; - $elseIndexes = []; - foreach ($operation->rules() as $index => $rule) { - $rulePath = $path . '.rules[' . $index . ']'; - $selector = $rule->selector(); - $this->validateSelector($selector, $rulePath . '.selector'); - $this->validateAction($rule->action(), $rulePath . '.action'); - - if ($selector instanceof ExactValueSelector) { - $identity = $selector->value()->identity(); - if (isset($exactValues[$identity])) { - $this->violation('recode.ambiguous_mapping', $rulePath . '.selector', 'An exact source value is mapped more than once.'); - } else { - $exactValues[$identity] = $index; - } - } elseif ($selector instanceof NumericRangeSelector) { - $ranges[] = ['index' => $index, 'lower' => $selector->lower(), 'upper' => $selector->upper()]; - } elseif ($selector instanceof MissingValueSelector) { - if ($missingIndex !== null) { - $this->violation('recode.ambiguous_mapping', $rulePath . '.selector', 'The system-missing value is mapped more than once.'); - } - $missingIndex = $index; - } elseif ($selector instanceof ElseSelector) { - $elseIndexes[] = $index; - } - } - - if (count($elseIndexes) !== 1) { - $this->violation('recode.else_required_once', $path . '.rules', 'A recode must contain exactly one explicit else rule.'); - } elseif ($elseIndexes[0] !== array_key_last($operation->rules())) { - $this->violation('recode.else_not_last', $path . '.rules[' . $elseIndexes[0] . ']', 'The else rule must be last.'); - } - - $this->validateRangeAmbiguity($ranges, $exactValues, $operation, $path); - } - - private function validateSelector(RecodeSelector $selector, string $path): void - { - if (!in_array($selector::class, [ - ExactValueSelector::class, - NumericRangeSelector::class, - MissingValueSelector::class, - ElseSelector::class, - ], true)) { - $this->violation('selector.unsupported_type', $path . '.type', 'The selector is not part of the canonical source-neutral contract.'); - - return; - } - - if ($selector instanceof ExactValueSelector) { - $this->validateScalar($selector->value(), $path . '.value'); - } elseif ($selector instanceof NumericRangeSelector) { - $lower = $selector->lower(); - $upper = $selector->upper(); - if ($lower === null && $upper === null) { - $this->violation('selector.range_unbounded', $path, 'A numeric range must have at least one finite bound.'); - } - if (($lower !== null && !is_finite($lower)) || ($upper !== null && !is_finite($upper))) { - $this->violation('selector.range_non_finite', $path, 'Numeric range bounds must be finite.'); - } elseif ($lower !== null && $upper !== null && $lower > $upper) { - $this->violation('selector.range_reversed', $path, 'A numeric range lower bound must not exceed its upper bound.'); - } - } - } - - private function validateAction(RecodeAction $action, string $path): void - { - if (!in_array($action::class, [ - AssignValueAction::class, - SetMissingAction::class, - CopySourceAction::class, - ], true)) { - $this->violation('action.unsupported_type', $path . '.type', 'The action is not part of the canonical source-neutral contract.'); - - return; - } - - if ($action instanceof AssignValueAction) { - $this->validateScalar($action->value(), $path . '.value'); - } - } - - private function validateValueLabels(SetValueLabelsOperation $operation, string $path): void - { - /** @var array $values */ - $values = []; - foreach ($operation->labels() as $index => $label) { - $labelPath = $path . '.labels[' . $index . ']'; - $this->validateScalar($label->value(), $labelPath . '.value'); - $this->validateText($label->label(), $labelPath . '.label'); - $identity = $label->value()->identity(); - if (isset($values[$identity])) { - $this->violation('value_labels.duplicate_value', $labelPath . '.value', 'A value may have only one label in a replacement set.'); - } - $values[$identity] = true; - } - } - - private function validateScalar(ScalarValue $value, string $path): void - { - if ($value->type() === 'number') { - if (!is_finite($value->numberValue())) { - $this->violation('value.non_finite', $path, 'Canonical numeric values must be finite binary64 values.'); - } - - return; - } - - $this->validateText($value->stringValue(), $path); - } - - private function validateVariableName(string $name, string $path): void - { - if ($name === '' || strlen($name) > 255 || str_contains($name, "\0") || preg_match('//u', $name) !== 1) { - $this->violation( - 'variable.invalid_name', - $path, - 'Variable names must contain 1-255 bytes of valid UTF-8 scalar text without NUL.', - ); - } - } - - private function validateText(?string $text, string $path): void - { - if ($text !== null && (str_contains($text, "\0") || preg_match('//u', $text) !== 1)) { - $this->violation('text.invalid_unicode', $path, 'Text must contain valid UTF-8 scalar text without NUL.'); - } - } - - /** - * @param list $ranges - * @param array $exactValues - */ - private function validateRangeAmbiguity(array $ranges, array $exactValues, RecodeOperation $operation, string $path): void - { - foreach ($ranges as $leftOffset => $left) { - if (($left['lower'] !== null && !is_finite($left['lower'])) || ($left['upper'] !== null && !is_finite($left['upper']))) { - continue; - } - foreach (array_slice($ranges, $leftOffset + 1) as $right) { - if (($right['lower'] !== null && !is_finite($right['lower'])) || ($right['upper'] !== null && !is_finite($right['upper']))) { - continue; - } - if ($this->rangesOverlap($left, $right)) { - $this->violation( - 'recode.ambiguous_mapping', - $path . '.rules[' . $right['index'] . '].selector', - 'Inclusive numeric ranges must not overlap.', - ); - } - } - - foreach ($exactValues as $identity => $exactIndex) { - if (!str_starts_with($identity, 'number:')) { - continue; - } - $selector = $operation->rules()[$exactIndex]->selector(); - if ($selector instanceof ExactValueSelector && $this->rangeContains($left, $selector->value()->numberValue())) { - $this->violation( - 'recode.ambiguous_mapping', - $path . '.rules[' . $exactIndex . '].selector', - 'An exact numeric selector must not overlap a numeric range.', - ); - } - } - } - } - - /** - * @param array{lower: ?float, upper: ?float} $left - * @param array{lower: ?float, upper: ?float} $right - */ - private function rangesOverlap(array $left, array $right): bool - { - return ($left['upper'] === null || $right['lower'] === null || $left['upper'] >= $right['lower']) - && ($right['upper'] === null || $left['lower'] === null || $right['upper'] >= $left['lower']); - } - - /** @param array{lower: ?float, upper: ?float} $range */ - private function rangeContains(array $range, float $value): bool - { - return ($range['lower'] === null || $value >= $range['lower']) - && ($range['upper'] === null || $value <= $range['upper']); - } - - private function violation(string $code, string $path, string $message): void - { - $this->violations[] = new ValidationViolation($code, $path, $message); - } -} diff --git a/src/Transformation/Validation/ValidationResult.php b/src/Transformation/Validation/ValidationResult.php deleted file mode 100644 index 349ec1d..0000000 --- a/src/Transformation/Validation/ValidationResult.php +++ /dev/null @@ -1,29 +0,0 @@ - $violations */ - public function __construct(private array $violations) {} - - public function isValid(): bool - { - return $this->violations === []; - } - - /** @return list */ - public function violations(): array - { - return $this->violations; - } - - public function throwIfInvalid(): void - { - if ($this->violations !== []) { - throw new InvalidTransformationPlan($this->violations); - } - } -} diff --git a/src/Transformation/Validation/ValidationViolation.php b/src/Transformation/Validation/ValidationViolation.php deleted file mode 100644 index 37ef75c..0000000 --- a/src/Transformation/Validation/ValidationViolation.php +++ /dev/null @@ -1,20 +0,0 @@ - $this->code, 'path' => $this->path, 'message' => $this->message]; - } -} diff --git a/tests/Core/CapabilityDeclarationTest.php b/tests/Core/CapabilityDeclarationTest.php index b198329..ee372ad 100644 --- a/tests/Core/CapabilityDeclarationTest.php +++ b/tests/Core/CapabilityDeclarationTest.php @@ -46,16 +46,22 @@ public function testDeclarationIsMachineReadableAndIncludesEveryProfileLimit(): 'maximum_source_file_bytes' => null, 'limit_basis' => 'runtime_memory_limit', ], $declaration['resource_behavior']); - self::assertSame([ - 'status' => 'legacy_contract_only', - 'legacy_plan_contracts' => ['openstatspec-transformation-plan-v1'], - 'official_plan_contracts' => [], - 'official_frontend_contracts' => [], - 'migration_order' => [ - 'openstatspec-transformation-plan-v0.1', - 'openstatspec-transformation-plan-v0.2', - ], - ], $declaration['transformation_contracts']); + $contracts = $declaration['transformation_contracts']; + self::assertSame('official_conformant', $contracts['status']); + self::assertSame( + ['openstatspec-transformation-plan-v0.1', 'openstatspec-transformation-plan-v0.2'], + $contracts['official_plan_contracts'], + ); + self::assertSame( + ['openstatspec-spss-syntax-frontend-v0.2'], + $contracts['official_frontend_contracts'], + ); + self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $contracts['official_binding_contracts'], + ); + self::assertArrayNotHasKey('legacy_plan_contracts', $contracts); + self::assertFalse(class_exists(\OpenStatSpec\Transformation\Model\TransformationPlan::class)); self::assertSame('3.0.3', $declaration['engine']['active_version']); self::assertSame('>=3.0.0 <4.0.0', $declaration['engine']['claimed_version_range']); self::assertSame(['3.0.3'], $declaration['engine']['ci_tested_versions']); diff --git a/tests/Frontend/Spss/Binding/SchemaStateTest.php b/tests/Frontend/Spss/Binding/SchemaStateTest.php new file mode 100644 index 0000000..9bdbe91 --- /dev/null +++ b/tests/Frontend/Spss/Binding/SchemaStateTest.php @@ -0,0 +1,23 @@ +resolve('ärger', new SourceSpan(0, 5))); + self::assertTrue($schema->contains('ÄRGER')); + } +} diff --git a/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php b/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php new file mode 100644 index 0000000..03ec1a0 --- /dev/null +++ b/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php @@ -0,0 +1,95 @@ + $case */ + #[DataProvider('cases')] + public function testOfficialFrontend02Case(array $case): void + { + /** @var array $requestPayload */ + $requestPayload = $case['request']; + $request = SpssFrontendRequest::fromArray($requestPayload); + self::assertSame($case['expected_source_hash'], $request->sourceHash(), (string) $case['id']); + + try { + $result = (new SpssCompiler())->compile($request); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode(), (string) $case['id']); + self::assertNotNull($failure->diagnostics[0]->span, (string) $case['id']); + + return; + } + + self::assertNull($case['expected_error'], (string) $case['id']); + self::assertSame($case['expected_source_hash'], $result->sourceHash, (string) $case['id']); + $codec = new PlanCodec(); + self::assertSame($case['expected_plan_hash'], $codec->hash($result->plan), (string) $case['id']); + self::assertSame($this->expectedPlan($case), $result->plan->canonicalArray(), (string) $case['id']); + self::assertSame( + $result->plan->canonicalArray(), + $codec->fromJson($codec->canonicalJson($result->plan))->canonicalArray(), + (string) $case['id'], + ); + + if (isset($case['expected_plan_contract'])) { + self::assertSame($case['expected_plan_contract'], $result->plan->contract->value, (string) $case['id']); + } + } + + /** @return iterable}> */ + public static function cases(): iterable + { + $manifest = SpecificationManifest::load('conformance/spss-syntax-frontend-0.2.json'); + /** @var list> $cases */ + $cases = $manifest['cases']; + foreach ($cases as $case) { + yield (string) $case['id'] => [$case]; + } + } + + /** + * @param array $case + * @return array + */ + private function expectedPlan(array $case): array + { + if (isset($case['expected_plan_case'])) { + return $this->namedPlan('conformance/transformation-plan-0.2.json', (string) $case['expected_plan_case'], 'plan'); + } + if (isset($case['expected_plan_case_0_1'])) { + return $this->namedPlan('conformance/spss-syntax-frontend-0.1.json', (string) $case['expected_plan_case_0_1'], 'expected_plan'); + } + if (isset($case['expected_plan_0_1']) && is_array($case['expected_plan_0_1'])) { + return $case['expected_plan_0_1']; + } + + self::fail('Successful frontend case has no expected plan: ' . $case['id']); + } + + /** @return array */ + private function namedPlan(string $manifestPath, string $caseId, string $planKey): array + { + $manifest = SpecificationManifest::load($manifestPath); + /** @var list> $cases */ + $cases = $manifest['cases']; + foreach ($cases as $case) { + if ($case['id'] === $caseId && isset($case[$planKey]) && is_array($case[$planKey])) { + return $case[$planKey]; + } + } + + self::fail('Referenced plan case was not found: ' . $caseId); + } +} diff --git a/tests/Frontend/Spss/DecimalBinary64Test.php b/tests/Frontend/Spss/DecimalBinary64Test.php new file mode 100644 index 0000000..168f3a3 --- /dev/null +++ b/tests/Frontend/Spss/DecimalBinary64Test.php @@ -0,0 +1,86 @@ + */ + public static function tokens(): iterable + { + yield 'negative zero' => ['-0', '0000000000000000']; + yield 'underflowed zero' => ['-1e-9999', '0000000000000000']; + yield 'minimum subnormal' => ['4.9406564584124654417656879286822137236505980e-324', '0000000000000001']; + yield 'one' => ['1', '3ff0000000000000']; + yield 'negative one' => ['-1', 'bff0000000000000']; + yield 'tie rounds even' => ['1.00000000000000011102230246251565404236316680908203125', '3ff0000000000000']; + yield 'tie rounds odd upward' => ['1.00000000000000033306690738754696212708950042724609375', '3ff0000000000002']; + yield 'significand carry' => ['1.99999999999999988897769753748434595763683319091796875', '4000000000000000']; + } + + #[DataProvider('invalidTokens')] + public function testRejectsInvalidOrOverflowingTokens(string $token): void + { + $this->expectException(SpssSyntaxException::class); + + DecimalBinary64::bits($token); + } + + /** @return iterable */ + public static function invalidTokens(): iterable + { + yield 'leading plus' => ['+1']; + yield 'missing integer part' => ['.5']; + yield 'missing fractional part' => ['1.']; + yield 'leading zero' => ['01']; + yield 'NaN' => ['NaN']; + yield 'infinity' => ['Infinity']; + yield 'overflow' => ['1e9999']; + } + + public function testNormalSubnormalMidpointRoundsToEvenMinimumNormal(): void + { + $midpoint = BigInteger::of(2)->power(53) + ->minus(1) + ->multipliedBy(BigInteger::of(5)->power(1075)); + + self::assertSame('0010000000000000', DecimalBinary64::bits(self::fraction($midpoint, 1075))); + } + + public function testValueBelowOverflowMidpointRoundsToMaximumFinite(): void + { + $midpoint = BigInteger::of(2)->power(54) + ->minus(1) + ->multipliedBy(BigInteger::of(2)->power(970)); + + self::assertSame('7fefffffffffffff', DecimalBinary64::bits($midpoint->minus(1)->toString())); + } + + public function testOverflowMidpointIsRejected(): void + { + $midpoint = BigInteger::of(2)->power(54) + ->minus(1) + ->multipliedBy(BigInteger::of(2)->power(970)); + $this->expectException(SpssSyntaxException::class); + + DecimalBinary64::bits($midpoint->toString()); + } + + private static function fraction(BigInteger $numerator, int $decimalPlaces): string + { + return '0.' . str_pad($numerator->toString(), $decimalPlaces, '0', STR_PAD_LEFT); + } +} diff --git a/tests/Frontend/Spss/LexerTest.php b/tests/Frontend/Spss/LexerTest.php index 71b437e..f1b75cf 100644 --- a/tests/Frontend/Spss/LexerTest.php +++ b/tests/Frontend/Spss/LexerTest.php @@ -13,10 +13,10 @@ final class LexerTest extends TestCase { public function testTokenizesMultilineSyntaxAndPreservesQuotedWhitespaceAndEscapes(): void { - $tokens = (new Lexer())->tokenize("* generated comment.\nVARIABLE LABELS score 'A score''s label'.\n"); + $tokens = (new Lexer())->tokenize("\nVARIABLE LABELS score 'A score''s label'.\n"); self::assertSame( - [TokenType::Identifier, TokenType::Identifier, TokenType::Identifier, TokenType::String, TokenType::Terminator, TokenType::EndOfFile], + [TokenType::Variable, TokenType::Identifier, TokenType::Identifier, TokenType::String, TokenType::Terminator, TokenType::EndOfFile], array_column($tokens, 'type'), ); self::assertSame("A score's label", $tokens[3]->lexeme); @@ -57,9 +57,54 @@ public function testRejectsUnknownCharactersWithPositionedDiagnostic(): void (new Lexer())->tokenize('RECODE score (`=1).'); self::fail('Unknown syntax unexpectedly tokenized.'); } catch (SpssSyntaxException $exception) { - self::assertSame(1, $exception->diagnostics[0]->line); - self::assertSame(15, $exception->diagnostics[0]->column); + $span = $exception->diagnostics[0]->span; + self::assertNotNull($span); + self::assertSame(1, $span->startLine); + self::assertSame(15, $span->startColumn); self::assertStringContainsString('Unexpected character', $exception->diagnostics[0]->message); } } + + public function testTokenizesFrontend02KeywordsComparisonsAndExactBoundaries(): void + { + $tokens = (new Lexer())->tokenize("compute target = 0.\nIF (a >= 1 AND b < 2 OR c <= 3) target = 4."); + + self::assertSame(TokenType::Compute, $tokens[0]->type); + self::assertSame(TokenType::If, $tokens[5]->type); + self::assertSame(TokenType::GreaterThanOrEqual, $tokens[8]->type); + self::assertSame(TokenType::And, $tokens[10]->type); + self::assertSame(TokenType::LessThan, $tokens[12]->type); + self::assertSame(TokenType::Or, $tokens[14]->type); + self::assertSame(TokenType::LessThanOrEqual, $tokens[16]->type); + self::assertSame(0, $tokens[0]->span->startOffset); + self::assertSame(7, $tokens[0]->span->endOffset); + self::assertSame(1, $tokens[0]->span->startLine); + self::assertSame(1, $tokens[0]->span->startColumn); + self::assertSame(1, $tokens[0]->span->endLine); + self::assertSame(8, $tokens[0]->span->endColumn); + self::assertSame(20, $tokens[5]->span->startOffset); + self::assertSame(2, $tokens[5]->span->startLine); + self::assertSame(1, $tokens[5]->span->startColumn); + } + + public function testOffsetsRemainExactAcrossCrLfLineEndings(): void + { + $tokens = (new Lexer())->tokenize("COMPUTE x = 0.\r\nEXECUTE."); + + self::assertSame(16, $tokens[5]->span->startOffset); + self::assertSame(2, $tokens[5]->span->startLine); + self::assertSame(1, $tokens[5]->span->startColumn); + } + + public function testNormalizesBareCrInsideStringsWithoutLosingExactOffsets(): void + { + $tokens = (new Lexer())->tokenize("VARIABLE LABELS x 'a\rb'.\rEXECUTE."); + + self::assertSame("a\nb", $tokens[3]->lexeme); + self::assertSame(1, $tokens[3]->span->startLine); + self::assertSame(2, $tokens[3]->span->endLine); + self::assertSame(23, $tokens[3]->span->endOffset); + self::assertSame(3, $tokens[5]->span->startLine); + self::assertSame(25, $tokens[5]->span->startOffset); + } } diff --git a/tests/Frontend/Spss/ParserTest.php b/tests/Frontend/Spss/ParserTest.php index ed46790..7eac5e5 100644 --- a/tests/Frontend/Spss/ParserTest.php +++ b/tests/Frontend/Spss/ParserTest.php @@ -5,6 +5,13 @@ namespace OpenStatSpec\Tests\Frontend\Spss; use OpenStatSpec\Frontend\Spss\Ast\ElseInput; +use OpenStatSpec\Frontend\Spss\Ast\BooleanPredicate; +use OpenStatSpec\Frontend\Spss\Ast\Comparison; +use OpenStatSpec\Frontend\Spss\Ast\ComputeStatement; +use OpenStatSpec\Frontend\Spss\Ast\ExecuteStatement; +use OpenStatSpec\Frontend\Spss\Ast\FormatsStatement; +use OpenStatSpec\Frontend\Spss\Ast\IfStatement; +use OpenStatSpec\Frontend\Spss\Ast\LiteralOperand; use OpenStatSpec\Frontend\Spss\Ast\MissingInput; use OpenStatSpec\Frontend\Spss\Ast\RangeInput; use OpenStatSpec\Frontend\Spss\Ast\RecodeOutputKind; @@ -13,8 +20,15 @@ use OpenStatSpec\Frontend\Spss\Ast\ValueInput; use OpenStatSpec\Frontend\Spss\Ast\ValueLabelsStatement; use OpenStatSpec\Frontend\Spss\Ast\VariableLabelsStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableLevelStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableOperand; use OpenStatSpec\Frontend\Spss\Parser; +use OpenStatSpec\Frontend\Spss\Request\InputSchema; +use OpenStatSpec\Frontend\Spss\Request\InputVariable; +use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; use OpenStatSpec\Frontend\Spss\SpssSyntaxException; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class ParserTest extends TestCase @@ -53,6 +67,22 @@ public function testParsesSupportedRecodeSelectorsActionsAndInto(): void self::assertSame(RecodeOutputKind::Copy, $statement->rules[3]->output->kind); } + public function testParsesCommaSeparatedRecodeSelectorsInSourceOrder(): void + { + $program = (new Parser())->parse('RECODE q1 (1,2,3 = 0).'); + + $statement = $program->statements[0]; + self::assertInstanceOf(RecodeStatement::class, $statement); + self::assertCount(1, $statement->rules); + $rule = $statement->rules[0]; + self::assertInstanceOf(ValueInput::class, $rule->input); + self::assertSame(['1', '2', '3'], array_column($rule->input->values, 'value')); + self::assertSame(['1', '2', '3'], array_column($rule->input->values, 'numericToken')); + self::assertNotNull($rule->output->value); + self::assertSame('0', $rule->output->value->value); + self::assertSame('0', $rule->output->value->numericToken); + } + public function testParsesVariableAndValueLabelGroups(): void { $program = (new Parser())->parse(<<<'SPSS' @@ -68,43 +98,237 @@ public function testParsesVariableAndValueLabelGroups(): void self::assertSame(['band'], $program->statements[1]->groups[1]->variables); } - public function testRejectsStringVariableRangesBeforeCreation(): void + public function testFailsClosedForUnknownOrUnterminatedCommands(): void + { + foreach (['SORT score.', 'RECODE score (1=2)'] as $syntax) { + try { + (new Parser())->parse($syntax); + self::fail('Unsupported or unterminated syntax unexpectedly parsed.'); + } catch (SpssSyntaxException $exception) { + self::assertNotEmpty($exception->diagnostics); + } + } + } + + public function testParsesEveryFrontend02CommandAndPreservesNumericTokenText(): void + { + $program = (new Parser())->parse( + "COMPUTE target = 0.\n" + . "IF (a = 1 AND b >= 2 OR c < 3) target = a.\n" + . "FORMATS target (F8.2).\n" + . "VARIABLE LEVEL target (SCALE).\n" + . 'EXECUTE.', + ); + + self::assertInstanceOf(ComputeStatement::class, $program->statements[0]); + self::assertInstanceOf(LiteralOperand::class, $program->statements[0]->expression); + self::assertSame('0', $program->statements[0]->expression->token); + self::assertInstanceOf(IfStatement::class, $program->statements[1]); + $predicate = $program->statements[1]->predicate; + self::assertInstanceOf(BooleanPredicate::class, $predicate); + self::assertSame('or', $predicate->operator); + self::assertInstanceOf(BooleanPredicate::class, $predicate->operands[0]); + self::assertSame('and', $predicate->operands[0]->operator); + self::assertInstanceOf(FormatsStatement::class, $program->statements[2]); + self::assertSame('F', $program->statements[2]->targets[0]->family); + self::assertSame(8, $program->statements[2]->targets[0]->width); + self::assertSame(2, $program->statements[2]->targets[0]->decimals); + self::assertInstanceOf(VariableLevelStatement::class, $program->statements[3]); + self::assertSame('scale', $program->statements[3]->groups[0]->level); + self::assertInstanceOf(ExecuteStatement::class, $program->statements[4]); + self::assertSame(5, $program->statements[4]->span->startLine); + self::assertSame(0, $program->statements[0]->span->startOffset); + self::assertSame(18, $program->statements[0]->span->endOffset); + } + + public function testBooleanPrecedenceParenthesesAndSameOperatorFlatteningPreserveOrder(): void + { + $program = (new Parser())->parse( + 'IF ((a = 1 OR b = 2) AND c = 3 AND d = 4) target = source.', + ); + + $statement = $program->statements[0]; + self::assertInstanceOf(IfStatement::class, $statement); + $predicate = $statement->predicate; + self::assertInstanceOf(BooleanPredicate::class, $predicate); + self::assertSame('and', $predicate->operator); + self::assertCount(3, $predicate->operands); + $or = $predicate->operands[0]; + self::assertInstanceOf(BooleanPredicate::class, $or); + self::assertSame('or', $or->operator); + $a = $or->operands[0]; + $b = $or->operands[1]; + $c = $predicate->operands[1]; + $d = $predicate->operands[2]; + self::assertInstanceOf(Comparison::class, $a); + self::assertInstanceOf(Comparison::class, $b); + self::assertInstanceOf(Comparison::class, $c); + self::assertInstanceOf(Comparison::class, $d); + self::assertInstanceOf(VariableOperand::class, $a->left); + self::assertInstanceOf(VariableOperand::class, $b->left); + self::assertInstanceOf(VariableOperand::class, $c->left); + self::assertInstanceOf(VariableOperand::class, $d->left); + self::assertSame(['a', 'b', 'c', 'd'], [$a->left->name, $b->left->name, $c->left->name, $d->left->name]); + } + + public function testParsesAllComparisonOperandPermutations(): void + { + $program = (new Parser())->parse('IF (a = b AND 1 < c AND 2 >= 1 AND c > 3) target = -1.25e+2.'); + + $statement = $program->statements[0]; + self::assertInstanceOf(IfStatement::class, $statement); + $predicate = $statement->predicate; + self::assertInstanceOf(BooleanPredicate::class, $predicate); + $first = $predicate->operands[0]; + $second = $predicate->operands[1]; + $third = $predicate->operands[2]; + $fourth = $predicate->operands[3]; + self::assertInstanceOf(Comparison::class, $first); + self::assertInstanceOf(Comparison::class, $second); + self::assertInstanceOf(Comparison::class, $third); + self::assertInstanceOf(Comparison::class, $fourth); + self::assertInstanceOf(VariableOperand::class, $first->left); + self::assertInstanceOf(VariableOperand::class, $first->right); + self::assertInstanceOf(LiteralOperand::class, $second->left); + self::assertInstanceOf(VariableOperand::class, $second->right); + self::assertInstanceOf(LiteralOperand::class, $third->left); + self::assertInstanceOf(LiteralOperand::class, $third->right); + self::assertInstanceOf(VariableOperand::class, $fourth->left); + self::assertInstanceOf(LiteralOperand::class, $fourth->right); + self::assertInstanceOf(LiteralOperand::class, $statement->expression); + self::assertSame('-1.25e+2', $statement->expression->token); + } + + public function testParsesStrictGreaterThanComparison(): void + { + $program = (new Parser())->parse('IF (a > 1) target = 1.'); + $statement = $program->statements[0]; + self::assertInstanceOf(IfStatement::class, $statement); + self::assertInstanceOf(Comparison::class, $statement->predicate); + self::assertSame('>', $statement->predicate->operator); + } + + public function testKeywordTokensRemainValidContextualVariableNames(): void { + $program = (new Parser())->parse( + "RECODE scale (1=2).\n" + . "VARIABLE LABELS compute 'Computed'.\n" + . "COMPUTE if = scale.\n" + . "IF (scale = 1) if = compute.\n" + . "FORMATS scale (F8.2).\n" + . 'VARIABLE LEVEL scale compute (NOMINAL).', + ); + + self::assertInstanceOf(RecodeStatement::class, $program->statements[0]); + self::assertSame(['scale'], $program->statements[0]->sources); + self::assertInstanceOf(ComputeStatement::class, $program->statements[2]); + self::assertSame('if', $program->statements[2]->target); + self::assertInstanceOf(IfStatement::class, $program->statements[3]); + self::assertSame('if', $program->statements[3]->target); + $levels = $program->statements[5]; + self::assertInstanceOf(VariableLevelStatement::class, $levels); + self::assertSame(['scale', 'compute'], $levels->groups[0]->variables); + } + + public function testGroupedMetadataCommandsPreserveGroupAndVariableOrder(): void + { + $program = (new Parser())->parse( + "FORMATS a (F8.2) b (f10.3).\n" + . 'VARIABLE LEVEL a b (ORDINAL) / c target (NOMINAL).', + ); + + $formats = $program->statements[0]; + $levels = $program->statements[1]; + self::assertInstanceOf(FormatsStatement::class, $formats); + self::assertInstanceOf(VariableLevelStatement::class, $levels); + self::assertSame(['a', 'b'], array_column($formats->targets, 'variable')); + self::assertSame([8, 10], array_column($formats->targets, 'width')); + self::assertSame([['a', 'b'], ['c', 'target']], array_column($levels->groups, 'variables')); + self::assertSame(['ordinal', 'nominal'], array_column($levels->groups, 'level')); + } + + #[DataProvider('rejectedFrontend02SyntaxProvider')] + public function testRejectedFrontend02SyntaxHasStableCodeAndExactSpan( + string $source, + string $code, + int $startOffset, + int $endOffset, + ): void { try { - (new Parser())->parse('STRING c1 TO c4 (A7).'); - self::fail('STRING variable ranges unexpectedly parsed.'); + (new Parser())->parse($source); + self::fail('Unsupported frontend syntax unexpectedly parsed.'); } catch (SpssSyntaxException $exception) { - self::assertCount(1, $exception->diagnostics); - self::assertSame( - 'STRING variable ranges using TO are not supported.', - $exception->diagnostics[0]->message, - ); + self::assertInstanceOf(TransformationFailure::class, $exception); + self::assertSame($code, $exception->diagnosticCode()); + $span = $exception->diagnostics[0]->span; + self::assertNotNull($span); + self::assertSame($startOffset, $span->startOffset); + self::assertSame($endOffset, $span->endOffset); + self::assertSame(1, $span->startLine); + self::assertSame($startOffset + 1, $span->startColumn); + self::assertSame(1, $span->endLine); + self::assertSame($endOffset + 1, $span->endColumn); } } - public function testRejectsDeleteVariableRangesBeforeExecution(): void + /** @return iterable */ + public static function rejectedFrontend02SyntaxProvider(): iterable + { + yield 'IF needs outer parentheses' => ['IF a = 1 target = 1.', 'spss_syntax_error', 3, 4]; + yield 'IF ELSE is not in the grammar' => ['IF (a = 1) target = 1 ELSE target = 0.', 'spss_syntax_error', 22, 26]; + yield 'arithmetic is unsupported' => ['COMPUTE target = a + 1.', 'expression_type_unsupported', 19, 20]; + yield 'adjacent addition is unsupported' => ['COMPUTE target = a+1.', 'expression_type_unsupported', 18, 20]; + yield 'adjacent subtraction is unsupported' => ['COMPUTE target = a-1.', 'expression_type_unsupported', 18, 20]; + yield 'predicate arithmetic is unsupported' => ['IF (a + 1 = 2) target = 1.', 'expression_type_unsupported', 6, 7]; + yield 'NOT is unsupported' => ['IF (NOT a = 1) target = 1.', 'expression_type_unsupported', 4, 7]; + yield 'string assignment is unsupported' => ["COMPUTE target = 'x'.", 'expression_type_unsupported', 17, 20]; + yield 'string predicate is unsupported' => ["IF (color = 'R') target = 1.", 'expression_type_unsupported', 12, 15]; + yield 'non-F format is invalid' => ['FORMATS target (A8).', 'invalid_format', 16, 18]; + yield 'unknown level is invalid syntax' => ['VARIABLE LEVEL target (UNKNOWN).', 'spss_syntax_error', 23, 30]; + yield 'star comment is unsupported command' => ['* comment.', 'unsupported_spss_command', 0, 1]; + yield 'COMMENT is unsupported command' => ['COMMENT text.', 'unsupported_spss_command', 0, 7]; + yield 'unknown command is unsupported' => ['SORT CASES BY a.', 'unsupported_spss_command', 0, 4]; + yield 'keyword word is unsupported at command start' => ['SCALE.', 'unsupported_spss_command', 0, 5]; + yield 'inline block comment is syntax error' => ['COMPUTE target = a/* comment */.', 'spss_syntax_error', 18, 20]; + yield 'leading plus is not numeric grammar' => ['COMPUTE target = +1.', 'spss_syntax_error', 17, 19]; + yield 'leading decimal is not numeric grammar' => ['COMPUTE target = .5.', 'spss_syntax_error', 17, 19]; + yield 'leading zero is not numeric grammar' => ['COMPUTE target = 01.', 'spss_syntax_error', 17, 19]; + yield 'NaN is not numeric grammar' => ['COMPUTE target = NaN.', 'spss_syntax_error', 17, 20]; + yield 'Infinity is not numeric grammar' => ['COMPUTE target = Infinity.', 'spss_syntax_error', 17, 25]; + yield 'trailing decimal point is not numeric grammar' => ['COMPUTE target = 1..', 'spss_syntax_error', 19, 20]; + } + + public function testMultilineDiagnosticHasExactStartAndEndCoordinates(): void { try { - (new Parser())->parse('DELETE VARIABLES c1 TO c4.'); - self::fail('DELETE VARIABLES ranges unexpectedly parsed.'); + (new Parser())->parse("COMPUTE target = 0.\nIF x = 1 target = 1."); + self::fail('Unparenthesized IF unexpectedly parsed.'); } catch (SpssSyntaxException $exception) { - self::assertCount(1, $exception->diagnostics); - self::assertSame( - 'DELETE VARIABLES ranges using TO are not supported.', - $exception->diagnostics[0]->message, - ); + $span = $exception->diagnostics[0]->span; + self::assertNotNull($span); + self::assertSame(23, $span->startOffset); + self::assertSame(24, $span->endOffset); + self::assertSame(2, $span->startLine); + self::assertSame(4, $span->startColumn); + self::assertSame(2, $span->endLine); + self::assertSame(5, $span->endColumn); } } - public function testFailsClosedForUnknownOrUnterminatedCommands(): void + public function testRequestDtosRetainExactSourceAndOrderedSchema(): void { - foreach (['COMPUTE score=1.', 'RECODE score (1=2)'] as $syntax) { - try { - (new Parser())->parse($syntax); - self::fail('Unsupported or unterminated syntax unexpectedly parsed.'); - } catch (SpssSyntaxException $exception) { - self::assertNotEmpty($exception->diagnostics); - } - } + $schema = new InputSchema([ + new InputVariable('first', 'numeric'), + new InputVariable('second', 'string'), + ]); + $request = new SpssFrontendRequest( + 'openstatspec-spss-syntax-frontend-v0.2', + 'parent', + $schema, + "COMPUTE target = 0.\r\nEXECUTE.", + ); + + self::assertSame(['first', 'second'], array_column($request->inputSchema->variables, 'name')); + self::assertSame("COMPUTE target = 0.\r\nEXECUTE.", $request->sourceText); } } diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index 246b468..9c80475 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -4,169 +4,327 @@ namespace OpenStatSpec\Tests\Frontend\Spss; +use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; +use OpenStatSpec\Frontend\Spss\Request\InputVariable; use OpenStatSpec\Frontend\Spss\SpssCompiler; -use OpenStatSpec\Frontend\Spss\SpssSyntaxException; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\Action\SetMissingAction; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Plan\PlanContract; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class SpssCompilerTest extends TestCase { - private const DATASET_ID = '018f47a2-4c10-7d34-8f11-93b1c3efc321'; + public function testRequestValidationPreservesOrderedSchemaAndNormalizesOnlySourceLineEndingsForHash(): void + { + $request = SpssFrontendRequest::fromArray([ + 'source_text' => "COMPUTE target = first.\r\nEXECUTE.\r", + 'input_schema' => [ + 'variables' => [ + [ + 'name' => 'first', + 'storage_kind' => 'numeric', + 'variable_label' => 'First', + 'value_labels' => [[ + 'value' => ['type' => 'binary64', 'bits' => '3ff0000000000000'], + 'label' => 'One', + ]], + 'format_family' => 'F', + 'width' => 8, + 'decimals' => 2, + 'measurement_level' => 'scale', + ], + ['name' => 'second', 'storage_kind' => 'string'], + ], + ], + 'input_alias' => 'parent', + 'contract' => 'openstatspec-spss-syntax-frontend-v0.2', + ]); + + self::assertSame(['first', 'second'], array_column($request->inputSchema->variables, 'name')); + self::assertSame("COMPUTE target = first.\r\nEXECUTE.\r", $request->sourceText); + self::assertSame( + hash('sha256', "COMPUTE target = first.\nEXECUTE.\n"), + $request->sourceHash(), + ); + } - public function testCompilesSupportedSubsetIntoValidatedCanonicalPlan(): void + public function testDirectInputVariableConstructionValidatesValueLabelShape(): void { - $plan = (new SpssCompiler())->compile(<<<'SPSS' - RECODE score (1=10) (2 THRU 4=20) (SYSMIS=99) (ELSE=COPY). - VARIABLE LABELS score 'Overall score'. - VALUE LABELS score 10 'Low' 20 'High'. - EXECUTE. - SPSS, self::DATASET_ID); - - self::assertSame(self::DATASET_ID, $plan->datasetId()); - self::assertCount(3, $plan->operations()); - /** @var RecodeOperation $recode */ - $recode = $plan->operations()[0]; - self::assertInstanceOf(RecodeOperation::class, $recode); - self::assertInstanceOf(MissingValueSelector::class, $recode->rules()[2]->selector()); - self::assertInstanceOf(ElseSelector::class, $recode->rules()[3]->selector()); - self::assertInstanceOf(CopySourceAction::class, $recode->rules()[3]->action()); - self::assertInstanceOf(SetVariableLabelOperation::class, $plan->operations()[1]); - self::assertInstanceOf(SetValueLabelsOperation::class, $plan->operations()[2]); - self::assertSame('openstatspec-transformation-plan-v1', $plan->canonicalArray()['contract']); - self::assertSame('in_place', $plan->canonicalArray()['mode']); + $this->expectException(TransformationFailure::class); + + new InputVariable( + 'q1', + 'numeric', + valueLabels: [['value' => ['type' => 'binary64', 'bits' => 'bad'], 'label' => 'Bad']], + ); } - public function testMakesSpssDefaultElseSemanticsExplicit(): void + /** @param array $payload */ + #[DataProvider('invalidRequestProvider')] + public function testRequestRejectsEveryNonSchemaShape(array $payload, string $path): void { - $inPlace = (new SpssCompiler())->compile('RECODE score (1=2).', self::DATASET_ID); - $into = (new SpssCompiler())->compile('RECODE score (1=2) INTO band.', self::DATASET_ID); - /** @var RecodeOperation $inPlaceRecode */ - $inPlaceRecode = $inPlace->operations()[0]; - /** @var RecodeOperation $intoRecode */ - $intoRecode = $into->operations()[0]; - - self::assertInstanceOf(CopySourceAction::class, $inPlaceRecode->rules()[1]->action()); - self::assertInstanceOf(SetMissingAction::class, $intoRecode->rules()[1]->action()); - self::assertInstanceOf(ElseSelector::class, $intoRecode->rules()[1]->selector()); + try { + SpssFrontendRequest::fromArray($payload); + self::fail('An invalid frontend request unexpectedly passed validation.'); + } catch (TransformationFailure $failure) { + self::assertSame('plan_schema_invalid', $failure->diagnosticCode()); + self::assertSame($path, $failure->diagnostics[0]->path); + } } - public function testExpandsParallelSourceAndIntoLists(): void + /** @return iterable, string}> */ + public static function invalidRequestProvider(): iterable { - $plan = (new SpssCompiler())->compile('RECODE first second (1=2) INTO new_first new_second.', self::DATASET_ID); + $valid = self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'numeric']]); - self::assertCount(2, $plan->operations()); - self::assertSame('first', $plan->operations()[0]->sourceVariable()); - self::assertSame('new_first', $plan->operations()[0]->targetVariable()); - self::assertSame('second', $plan->operations()[1]->sourceVariable()); - self::assertSame('new_second', $plan->operations()[1]->targetVariable()); + yield 'extra root member' => [$valid + ['extra' => true], '$']; + yield 'wrong contract' => [[...$valid, 'contract' => 'openstatspec-spss-syntax-frontend-v0.1'], '$.contract']; + yield 'empty alias' => [[...$valid, 'input_alias' => ''], '$.input_alias']; + yield 'empty source' => [[...$valid, 'source_text' => ''], '$.source_text']; + yield 'empty variables' => [[...$valid, 'input_schema' => ['variables' => []]], '$.input_schema.variables']; + yield 'extra schema member' => [[...$valid, 'input_schema' => ['variables' => [['name' => 'q1', 'storage_kind' => 'numeric']], 'extra' => null]], '$.input_schema']; + yield 'empty variable name' => [self::payload('EXECUTE.', [['name' => '', 'storage_kind' => 'numeric']]), '$.input_schema.variables[0].name']; + yield 'invalid storage kind' => [self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'date']]), '$.input_schema.variables[0].storage_kind']; + yield 'extra variable member' => [self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'numeric', 'extra' => true]]), '$.input_schema.variables[0]']; + yield 'invalid value label bits' => [self::payload('EXECUTE.', [[ + 'name' => 'q1', + 'storage_kind' => 'numeric', + 'value_labels' => [['value' => ['type' => 'binary64', 'bits' => '8000000000000000'], 'label' => 'bad']], + ]]), '$.input_schema.variables[0].value_labels[0].value.bits']; + yield 'zero format width' => [self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'numeric', 'width' => 0]]), '$.input_schema.variables[0].width']; + yield 'invalid level' => [self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'numeric', 'measurement_level' => 'interval']]), '$.input_schema.variables[0].measurement_level']; } - public function testPreservesSequentialOperationsOnTheSameTarget(): void + public function testBindingUsesExactCatalogSpellingAndEvolvingCaseInsensitiveSchema(): void { - $plan = (new SpssCompiler())->compile(<<<'SPSS' - RECODE score (1=2). - RECODE score (2=3). - VARIABLE LABELS score 'First label'. - VARIABLE LABELS score 'Replacement label'. - SPSS, self::DATASET_ID); - - self::assertCount(4, $plan->operations()); - self::assertSame( - ['recode', 'recode', 'set_variable_label', 'set_variable_label'], - array_map(static fn($operation): string => $operation->type(), $plan->operations()), - ); + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + "COMPUTE Temp = source.\nIF (TEMP = 1) temp = SOURCE.\nFORMATS tEmP (F8.2).", + [['name' => 'Source', 'storage_kind' => 'numeric']], + ))); + + self::assertSame(PlanContract::V02, $result->plan->contract); + self::assertSame([ + ['op' => 'assign', 'target' => 'Temp', 'target_mode' => 'create', 'value' => ['kind' => 'variable', 'variable' => 'Source']], + [ + 'op' => 'conditional_assign', + 'condition' => [ + 'expression' => 'comparison', + 'left' => ['kind' => 'variable', 'variable' => 'Temp'], + 'operator' => '=', + 'right' => ['kind' => 'literal', 'value' => ['type' => 'binary64', 'bits' => '3ff0000000000000']], + ], + 'target' => 'Temp', + 'value' => ['kind' => 'variable', 'variable' => 'Source'], + ], + ['op' => 'set_format', 'variable' => 'Temp', 'family' => 'F', 'width' => 8, 'decimals' => 2], + ], $result->plan->canonicalArray()['operations']); } - public function testRejectsParallelRecodeThatWouldOverwriteALaterSource(): void + public function testParallelRecodeReadsPrecommandSchemaAndPublishesTargetsAfterTheCommand(): void { - try { - (new SpssCompiler())->compile( - 'RECODE a b (ELSE=COPY) INTO b c.', - self::DATASET_ID, - ); - self::fail('A dependency-unsafe parallel RECODE unexpectedly compiled.'); - } catch (SpssSyntaxException $exception) { - self::assertStringContainsString( - 'overwrite a source used by a later pair', - $exception->diagnostics[0]->message, - ); - } + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + "RECODE FIRST second (1 = 9) INTO First_New Second_New.\nVARIABLE LABELS first_new 'Created'.", + [ + ['name' => 'First', 'storage_kind' => 'numeric'], + ['name' => 'Second', 'storage_kind' => 'numeric'], + ], + ))); + + self::assertSame(PlanContract::V01, $result->plan->contract); + self::assertSame( + [['First', 'First_New'], ['Second', 'Second_New']], + array_map( + static fn(array $operation): array => [$operation['source'], $operation['target']], + array_slice($result->plan->canonicalArray()['operations'], 0, 2), + ), + ); + self::assertSame('First_New', $result->plan->canonicalArray()['operations'][2]['variable']); } - public function testRejectsDuplicateIntoTargetsWithinOneParallelRecode(): void + public function testRecodeAndValueLabelsUseExactDecimalBinary64Bits(): void { - try { - (new SpssCompiler())->compile( - 'RECODE first second (1=2) INTO result result.', - self::DATASET_ID, - ); - self::fail('A parallel RECODE with duplicate INTO targets unexpectedly compiled.'); - } catch (SpssSyntaxException $exception) { - self::assertStringContainsString( - 'duplicate target variables', - $exception->diagnostics[0]->message, - ); - } + $token = '1.00000000000000033306690738754696212708950042724609375'; + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + "RECODE q1 ($token = -0).\nVALUE LABELS q1 $token 'Rounded'.", + [['name' => 'q1', 'storage_kind' => 'numeric']], + ))); + $operations = $result->plan->canonicalArray()['operations']; + + self::assertSame('3ff0000000000002', $operations[0]['rules'][0]['match']['values'][0]['bits']); + self::assertSame('0000000000000000', $operations[0]['rules'][0]['result']['value']['bits']); + self::assertSame('3ff0000000000002', $operations[1]['labels'][0]['value']['bits']); } - public function testBinderRejectsInvalidElsePositionAndIntoArity(): void + public function testStringSourceCanCreateNumericTargetWithExplicitSystemMissingElse(): void { - foreach ([ - 'RECODE score (ELSE=COPY) (1=2).', - 'RECODE first second (1=2) INTO only_one.', - "RECODE score ('a' THRU 'z'=1).", - ] as $syntax) { - $this->expectCompileFailure($syntax); - } + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + "RECODE color ('R' = 1) (ELSE = SYSMIS) INTO code.", + [['name' => 'color', 'storage_kind' => 'string']], + ))); + + self::assertSame('system_missing', $result->plan->canonicalArray()['operations'][0]['unmatched']['kind']); } - public function testFailsClosedForUserMissingSelectorWithoutDatasetMetadata(): void + public function testRecodeWithOnlyElseCannotEmitAnInvalidEmptyRuleList(): void { try { - (new SpssCompiler())->compile('RECODE score (MISSING=0) (ELSE=COPY).', self::DATASET_ID); - self::fail('MISSING unexpectedly compiled as system missing.'); - } catch (SpssSyntaxException $exception) { - self::assertStringContainsString('user-missing', $exception->diagnostics[0]->message); + (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + 'RECODE q1 (ELSE = 0).', + [['name' => 'q1', 'storage_kind' => 'numeric']], + ))); + self::fail('An ELSE-only RECODE unexpectedly emitted an invalid plan.'); + } catch (TransformationFailure $failure) { + self::assertSame('spss_syntax_error', $failure->diagnosticCode()); + self::assertSame([0, 20], [ + $failure->diagnostics[0]->span?->startOffset, + $failure->diagnostics[0]->span?->endOffset, + ]); } } - public function testLargeIntegerLiteralIsParsedDirectlyAsBinary64(): void - { - $plan = (new SpssCompiler())->compile( - 'RECODE score (999999999999999999999=1) (ELSE=COPY).', - self::DATASET_ID, - ); - - /** @var RecodeOperation $recode */ - $recode = $plan->operations()[0]; - /** @var \OpenStatSpec\Transformation\Model\Selector\ExactValueSelector $selector */ - $selector = $recode->rules()[0]->selector(); - self::assertInstanceOf( - \OpenStatSpec\Transformation\Model\Selector\ExactValueSelector::class, - $selector, - ); - self::assertSame((float) '999999999999999999999', $selector->value()->numberValue()); - self::assertNotSame((float) PHP_INT_MAX, $selector->value()->numberValue()); + /** @param list> $variables */ + #[DataProvider('bindingFailureProvider')] + public function testBindingFailuresHaveStableCodesAndExactSourceSpans( + string $source, + array $variables, + string $code, + int $startOffset, + int $endOffset, + ): void { + try { + (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload($source, $variables))); + self::fail('Invalid frontend semantics unexpectedly compiled.'); + } catch (TransformationFailure $failure) { + self::assertSame($code, $failure->diagnosticCode()); + $span = $failure->diagnostics[0]->span; + self::assertNotNull($span); + self::assertSame([$startOffset, $endOffset], [$span->startOffset, $span->endOffset]); + self::assertSame([1, $startOffset + 1, 1, $endOffset + 1], [ + $span->startLine, + $span->startColumn, + $span->endLine, + $span->endColumn, + ]); + } } - public function testRejectsNonFiniteNumericLiteral(): void + /** @return iterable>, string, int, int}> */ + public static function bindingFailureProvider(): iterable { - $this->expectCompileFailure('RECODE score (1e999=1) (ELSE=COPY).'); + yield 'unknown expression variable' => [ + 'COMPUTE target = missing.', + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'unknown_variable', + 17, + 24, + ]; + yield 'missing conditional target' => [ + 'IF (source = 1) missing = 1.', + [['name' => 'source', 'storage_kind' => 'numeric']], + 'conditional_target_missing', + 16, + 23, + ]; + yield 'reserved new compute target' => [ + 'COMPUTE __hidden = 1.', + [['name' => 'source', 'storage_kind' => 'numeric']], + 'reserved_target_name', + 8, + 16, + ]; + yield 'reserved existing compute target' => [ + 'COMPUTE __hidden = 1.', + [['name' => '__Hidden', 'storage_kind' => 'numeric']], + 'reserved_target_name', + 8, + 16, + ]; + yield 'reserved existing conditional target' => [ + 'IF (source = 1) __hidden = 1.', + [ + ['name' => 'source', 'storage_kind' => 'numeric'], + ['name' => '__Hidden', 'storage_kind' => 'numeric'], + ], + 'reserved_target_name', + 16, + 24, + ]; + yield 'reserved existing recode target' => [ + 'RECODE __Hidden (1 = 2).', + [['name' => '__Hidden', 'storage_kind' => 'numeric']], + 'reserved_target_name', + 7, + 15, + ]; + yield 'string compute expression' => [ + 'COMPUTE target = color.', + [['name' => 'color', 'storage_kind' => 'string']], + 'expression_type_unsupported', + 17, + 22, + ]; + yield 'string compute target' => [ + 'COMPUTE color = 1.', + [['name' => 'color', 'storage_kind' => 'string']], + 'expression_type_unsupported', + 8, + 13, + ]; + yield 'value label type mismatch' => [ + "VALUE LABELS q1 'R' 'Red'.", + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'type_mismatch', + 13, + 25, + ]; + yield 'invalid format bounds' => [ + 'FORMATS q1 (F2.2).', + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'invalid_format', + 8, + 17, + ]; + yield 'duplicate value label' => [ + "VALUE LABELS q1 -0 'Minus' 0 'Plus'.", + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'duplicate_value_label', + 13, + 35, + ]; + yield 'system missing string recode' => [ + "RECODE color (SYSMIS = 'x').", + [['name' => 'color', 'storage_kind' => 'string']], + 'system_missing_for_string', + 14, + 20, + ]; + yield 'string target needs declaration' => [ + "RECODE color ('R' = 'red') INTO normalized.", + [['name' => 'color', 'storage_kind' => 'string']], + 'string_target_requires_declaration', + 32, + 42, + ]; + yield 'mixed recode results' => [ + "RECODE q1 (1 = 'one') (ELSE = 0).", + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'mixed_result_types', + 0, + 32, + ]; } - private function expectCompileFailure(string $syntax): void + /** + * @param list> $variables + * @return array + */ + private static function payload(string $source, array $variables): array { - try { - (new SpssCompiler())->compile($syntax, self::DATASET_ID); - self::fail('Semantically invalid syntax unexpectedly compiled.'); - } catch (SpssSyntaxException $exception) { - self::assertNotEmpty($exception->diagnostics); - } + return [ + 'contract' => 'openstatspec-spss-syntax-frontend-v0.2', + 'input_alias' => 'parent', + 'input_schema' => ['variables' => $variables], + 'source_text' => $source, + ]; } } diff --git a/tests/Integration/InPlaceTransformationServiceTest.php b/tests/Integration/InPlaceTransformationServiceTest.php index 0886a29..2c0c961 100644 --- a/tests/Integration/InPlaceTransformationServiceTest.php +++ b/tests/Integration/InPlaceTransformationServiceTest.php @@ -10,20 +10,30 @@ use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Tests\Support\SpecificationManifest; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Execution\DoltEvidence; +use OpenStatSpec\Transformation\Execution\DoltEvidenceReader; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; -use OpenStatSpec\Transformation\Model\Action\AssignValueAction; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\RecodeRule; -use OpenStatSpec\Transformation\Model\ScalarValue; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\ExactValueSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\Selector\NumericRangeSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; -use OpenStatSpec\Transformation\Model\ValueLabel; +use OpenStatSpec\Transformation\Plan\Expression\VariableOperand; +use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\RecodeOperation; +use OpenStatSpec\Transformation\Plan\Operation\ReplaceValueLabelsOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetVariableLabelOperation; +use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; +use OpenStatSpec\Transformation\Plan\PlanCodec; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\Recode\CopyResult; +use OpenStatSpec\Transformation\Plan\Recode\ExactMatch; +use OpenStatSpec\Transformation\Plan\Recode\LiteralResult; +use OpenStatSpec\Transformation\Plan\Recode\RangeMatch; +use OpenStatSpec\Transformation\Plan\Recode\RecodeRule; +use OpenStatSpec\Transformation\Plan\Recode\SystemMissingMatch; +use OpenStatSpec\Transformation\Plan\TargetMode; +use OpenStatSpec\Transformation\Plan\TransformationPlan; +use OpenStatSpec\Transformation\Plan\Value\Binary64Value; use PDO; use PDOStatement; use PHPUnit\Framework\Attributes\DataProvider; @@ -43,7 +53,7 @@ final class InPlaceTransformationServiceTest extends TestCase private ?array $activeFixture = null; /** @var list */ - private array $doltTestDatabases = []; + private array $isolatedTestDatabases = []; private ?string $doltDatabaseRunId = null; @@ -52,13 +62,13 @@ final class InPlaceTransformationServiceTest extends TestCase protected function tearDown(): void { try { - foreach (array_reverse($this->doltTestDatabases) as $testDatabase) { + foreach (array_reverse($this->isolatedTestDatabases) as $testDatabase) { $testDatabase['admin']->exec( 'DROP DATABASE ' . $this->quoteDoltDatabaseName($testDatabase['database']), ); } } finally { - $this->doltTestDatabases = []; + $this->isolatedTestDatabases = []; $this->activeFixture = null; parent::tearDown(); } @@ -94,6 +104,40 @@ public static function createTargetRejectedServices(): iterable } } + /** @return iterable, string|null, string}> */ + public static function createTargetRejectedCases(): iterable + { + foreach (self::createTargetRejectedServices() as $label => $service) { + yield $label . ' / official 0.1' => [...$service, '0.1']; + yield $label . ' / official 0.2' => [...$service, '0.2']; + } + } + + /** @return iterable, string|null}> */ + public static function physicalCaseOrderServices(): iterable + { + foreach (self::createTargetRejectedServices() as $label => $service) { + yield $label => $service; + } + } + + /** @return iterable */ + public static function transactionalMySqlServices(): iterable + { + yield 'MySQL' => ['mysql', 'OPENSTATSPEC_MYSQL', 'mysql']; + yield 'MariaDB' => ['mariadb', 'OPENSTATSPEC_MARIADB', 'mysql']; + } + + /** @return iterable */ + public static function mySqlFamilyCatalogAndAuditEngineCases(): iterable + { + foreach (self::transactionalMySqlServices() as $service => $configuration) { + foreach (['dataset', 'variable', 'value_label_set', 'value_label', 'variable_value_label_set', 'transformation_apply'] as $table) { + yield $service . ' / ' . $table => [...$configuration, $table]; + } + } + } + /** @param list $expectedVersionFamilies */ #[DataProvider('services')] public function testConfiguredServiceMatchesExpectedProfileAndVersionFamily( @@ -162,7 +206,7 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace self::assertSame([], $doltBefore['working_diff']); } $plan = $this->existingTargetPlan(); - $result = (new InPlaceTransformationExecutor($connection))->execute($plan); + $result = (new InPlaceTransformationExecutor($connection))->execute($this->applyRequest($plan, $doltBefore)); $doltAfter = $this->doltRepositoryEvidence($pdo, $connection); if ($expectedProfile === 'dolt') { @@ -174,6 +218,7 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace self::assertSame( [ ['table_name' => 'inplace_existing_target_dolt', 'staged' => false, 'status' => 'modified'], + ['table_name' => 'transformation_apply', 'staged' => false, 'status' => 'modified'], ['table_name' => 'value_label', 'staged' => false, 'status' => 'modified'], ['table_name' => 'value_label_set', 'staged' => false, 'status' => 'modified'], ['table_name' => 'variable', 'staged' => false, 'status' => 'modified'], @@ -185,6 +230,7 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace self::assertSame( [ ['table_name' => 'inplace_existing_target_dolt', 'data_change' => true, 'schema_change' => false], + ['table_name' => 'transformation_apply', 'data_change' => true, 'schema_change' => false], ['table_name' => 'value_label', 'data_change' => true, 'schema_change' => false], ['table_name' => 'value_label_set', 'data_change' => true, 'schema_change' => false], ['table_name' => 'variable', 'data_change' => true, 'schema_change' => false], @@ -196,8 +242,8 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace } self::assertSame($fixture['dataset_id'], $result->datasetId()); - self::assertSame($plan->hash(), $result->planHash()); - self::assertSame(3, $result->operationCount()); + self::assertSame((new PlanCodec())->hash($plan), $result->planHash()); + self::assertSame(4, $result->operationCount()); self::assertSame($fixture['dataset'], $this->datasetRow($pdo)); self::assertSame($fixture['variables'], $this->variableIdentityRows($pdo)); @@ -242,6 +288,376 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace } } + /** @param list $expectedVersionFamilies */ + #[DataProvider('physicalCaseOrderServices')] + public function testMySqlFamilyRejectsMissingPhysicalCaseOrderColumnBeforeMutation( + string $expectedProfile, + ?string $environmentPrefix, + string $driver, + array $expectedVersionFamilies, + ?string $expectedVersionEnvironment, + ): void { + unset($expectedVersionFamilies, $expectedVersionEnvironment); + $pdo = $this->servicePdo($expectedProfile, $environmentPrefix, $driver); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + + try { + $quotedTable = $this->qualifiedTable($connection, $fixture['table_name']); + $quotedOrdinal = $connection->profile->quoteIdentifier('__case_ordinal'); + $pdo->exec('ALTER TABLE ' . $quotedTable . ' DROP PRIMARY KEY, DROP COLUMN ' . $quotedOrdinal); + $auditBefore = (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply', []); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('An apply without the physical case-order column was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + } + + self::assertSame($auditBefore, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply', [])); + self::assertFalse($pdo->inTransaction()); + } finally { + $this->purgeFixture($pdo, $connection, $fixture); + } + } + + /** @param list $expectedVersionFamilies */ + #[DataProvider('physicalCaseOrderServices')] + public function testMySqlFamilyRejectsCaseFoldedTableAliasSharedWithAnotherDataset( + string $expectedProfile, + ?string $environmentPrefix, + string $driver, + array $expectedVersionFamilies, + ?string $expectedVersionEnvironment, + ): void { + unset($expectedVersionFamilies, $expectedVersionEnvironment); + $pdo = $this->servicePdo($expectedProfile, $environmentPrefix, $driver); + $connection = new Connection($pdo); + $lowerCaseTableNames = (int) $this->scalar($pdo, 'SELECT @@lower_case_table_names', []); + if ($lowerCaseTableNames === 0) { + self::markTestSkipped('MySQL-family lower_case_table_names=0 treats table names case-sensitively.'); + } + $fixture = $this->installFixture($pdo, $connection); + + try { + $quotedTable = $this->qualifiedTable($connection, $fixture['table_name']); + $aliasedTable = strtoupper($fixture['table_name']) === $fixture['table_name'] + ? strtolower($fixture['table_name']) + : strtoupper($fixture['table_name']); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([ + '99999999-9999-4999-8999-999999999999', + '1.0', + 'fixture', + null, + $aliasedTable, + 'Case-folded alias', + 0, + '2026-08-17 00:00:00', + ]); + $auditBefore = (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply', []); + self::assertNotSame($fixture['table_name'], $aliasedTable); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('Two case-folded MySQL-family datasets sharing one physical table were accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + } + + self::assertSame($auditBefore, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply', [])); + self::assertFalse($pdo->inTransaction()); + self::assertNotSame( + 0, + (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM information_schema.tables ' + . 'WHERE table_schema = DATABASE() AND table_name = ?', [$fixture['table_name']]), + $quotedTable . ' wide table missing before apply.', + ); + } finally { + $this->purgeFixture($pdo, $connection, $fixture); + } + } + + public function testDoltRequiresExpectedContextBeforeMutation(): void + { + $pdo = $this->servicePdo('dolt', 'OPENSTATSPEC_DOLT', 'mysql'); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A Dolt apply without expected branch and HEAD was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_context_required', $failure->diagnosticCode()); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } + + public function testDoltContextChangeAfterMutationRollsBackWithoutAudit(): void + { + $case = $this->officialInPlaceCase02('dolt-context-changed-after-mutation-rolls-back'); + $pdo = $this->servicePdo('dolt', 'OPENSTATSPEC_DOLT', 'mysql'); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $context = $this->doltRepositoryEvidence($pdo, $connection); + self::assertNotNull($context); + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + $reader = new class ($pdo, $connection, $fixture['table_name'], $context['branch'], $context['head']) implements DoltEvidenceReader { + public bool $mutationObserved = false; + private int $reads = 0; + + public function __construct( + private readonly PDO $pdo, + private readonly Connection $connection, + private readonly string $table, + private readonly string $branch, + private readonly string $head, + ) {} + + public function read(): DoltEvidence + { + if (++$this->reads === 1) { + return new DoltEvidence($this->branch, $this->head, []); + } + + $statement = $this->pdo->query( + 'SELECT ' . $this->connection->profile->quoteIdentifier('destination') + . ' FROM ' . $this->connection->profile->quoteIdentifier($this->table) + . ' WHERE ' . $this->connection->profile->quoteIdentifier('__case_ordinal') . ' = 1', + ); + if ($statement instanceof PDOStatement) { + $this->mutationObserved = (float) $statement->fetchColumn() === 10.0; + } + + return new DoltEvidence($this->branch, 'concurrent-commit', ['variable']); + } + }; + + try { + (new InPlaceTransformationExecutor($connection, $reader))->execute( + $this->applyRequest($this->existingTargetPlan(), $context), + ); + self::fail('A post-mutation Dolt context change was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertTrue($case['mutation_started']); + self::assertTrue($reader->mutationObserved, 'The injected context change must occur after data mutation.'); + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } + + #[DataProvider('transactionalMySqlServices')] + public function testMySqlFamilyDataCatalogAndAuditAreOneTransaction( + string $expectedProfile, + string $environmentPrefix, + string $driver, + ): void { + $pdo = $this->servicePdo($expectedProfile, $environmentPrefix, $driver); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $constraint = 'task9_fail_audit_' . $expectedProfile; + $quotedConstraint = $connection->profile->quoteIdentifier($constraint); + $constraintInstalled = false; + + try { + $pdo->exec( + 'ALTER TABLE transformation_apply ADD CONSTRAINT ' . $quotedConstraint + . " CHECK (actor <> 'integration-test')", + ); + $constraintInstalled = true; + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('The injected MySQL-family audit failure did not abort the apply.'); + } catch (\PDOException $failure) { + self::assertStringContainsString($constraint, $failure->getMessage()); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } finally { + if ($constraintInstalled) { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT ' . $quotedConstraint); + } + $this->purgeFixture($pdo, $connection, $fixture); + } + } + + #[DataProvider('transactionalMySqlServices')] + public function testMySqlFamilyRejectsMyIsamWideTableBeforeForcedAuditFailure( + string $expectedProfile, + string $environmentPrefix, + string $driver, + ): void { + unset($driver); + $pdo = $this->isolatedMySqlFamilyPdo($expectedProfile, $environmentPrefix); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $constraint = 'task9_nontransactional_wide_' . $expectedProfile; + $quotedConstraint = $connection->profile->quoteIdentifier($constraint); + $constraintInstalled = false; + + try { + $pdo->exec( + 'ALTER TABLE ' . $connection->profile->quoteIdentifier($fixture['table_name']) . ' ENGINE=MyISAM', + ); + $pdo->exec( + 'ALTER TABLE transformation_apply ADD CONSTRAINT ' . $quotedConstraint + . " CHECK (actor <> 'integration-test')", + ); + $constraintInstalled = true; + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A MyISAM-bound wide table was accepted for an atomic apply.'); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::SqlProfileOperationUnavailable, $failure->diagnosticCode); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } finally { + if ($constraintInstalled) { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT ' . $quotedConstraint); + } + $this->purgeFixture($pdo, $connection, $fixture); + } + } + + #[DataProvider('mySqlFamilyCatalogAndAuditEngineCases')] + public function testMySqlFamilyRejectsNonTransactionalCatalogAndAuditEngines( + string $expectedProfile, + string $environmentPrefix, + string $driver, + string $table, + ): void { + unset($driver); + $pdo = $this->isolatedMySqlFamilyPdo($expectedProfile, $environmentPrefix); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $this->dropMySqlForeignKeys($pdo, $connection); + $this->dropMySqlSecondaryIndexes($pdo, $connection, $table); + $pdo->exec('ALTER TABLE ' . $connection->profile->quoteIdentifier($table) . ' ENGINE=MyISAM'); + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A non-transactional catalog or audit table was accepted: ' . $table); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::SqlProfileOperationUnavailable, $failure->diagnosticCode); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } + + #[DataProvider('transactionalMySqlServices')] + public function testMySqlFamilyRejectsMissingWideTableEngineMetadata( + string $expectedProfile, + string $environmentPrefix, + string $driver, + ): void { + unset($driver); + $pdo = $this->isolatedMySqlFamilyPdo($expectedProfile, $environmentPrefix); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $quotedTable = $connection->profile->quoteIdentifier($fixture['table_name']); + $rows = $this->rawTableRows( + $pdo, + $connection, + $fixture['table_name'], + ['__case_ordinal', 'source_value', 'destination'], + ); + $pdo->exec('DROP TABLE ' . $quotedTable); + $pdo->exec( + 'CREATE TEMPORARY TABLE ' . $quotedTable . ' (' + . '`__case_ordinal` BIGINT NOT NULL PRIMARY KEY, ' + . '`source_value` DOUBLE NULL, `destination` DOUBLE NULL) ENGINE=InnoDB', + ); + $insert = $pdo->prepare( + 'INSERT INTO ' . $quotedTable . ' (`__case_ordinal`, `source_value`, `destination`) VALUES (?, ?, ?)', + ); + foreach ($rows as $row) { + $insert->execute([$row['__case_ordinal'], $row['source_value'], $row['destination']]); + } + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A table missing from storage-engine metadata was accepted.'); + } catch (UnsupportedOperation $failure) { + self::assertContains($failure->diagnosticCode, [ + DiagnosticCode::SqlProfileOperationUnavailable, + DiagnosticCode::CatalogNamespaceCollision, + ]); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } + + #[DataProvider('transactionalMySqlServices')] + public function testMySqlFamilyRejectsUnknownViewEngineMetadata( + string $expectedProfile, + string $environmentPrefix, + string $driver, + ): void { + unset($driver); + $pdo = $this->isolatedMySqlFamilyPdo($expectedProfile, $environmentPrefix); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $quotedTable = $connection->profile->quoteIdentifier($fixture['table_name']); + $backing = $fixture['table_name'] . '_backing'; + $quotedBacking = $connection->profile->quoteIdentifier($backing); + $pdo->exec('RENAME TABLE ' . $quotedTable . ' TO ' . $quotedBacking); + $pdo->exec('CREATE VIEW ' . $quotedTable . ' AS SELECT * FROM ' . $quotedBacking); + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A view with NULL storage-engine metadata was accepted.'); + } catch (UnsupportedOperation $failure) { + self::assertContains($failure->diagnosticCode, [ + DiagnosticCode::SqlProfileOperationUnavailable, + DiagnosticCode::CatalogNamespaceCollision, + ]); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } finally { + $pdo->exec('DROP VIEW IF EXISTS ' . $quotedTable); + $pdo->exec('RENAME TABLE ' . $quotedBacking . ' TO ' . $quotedTable); + $this->purgeFixture($pdo, $connection, $fixture); + } + } + /** @param list $expectedVersionFamilies */ #[DataProvider('createTargetCapableServices')] public function testCreateTargetTransformationCreatesNumericTargetInPlace( @@ -259,10 +675,10 @@ public function testCreateTargetTransformationCreatesNumericTargetInPlace( try { $datasetCountBefore = (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset', []); $plan = $this->createTargetPlan(); - $result = (new InPlaceTransformationExecutor($connection))->execute($plan); + $result = (new InPlaceTransformationExecutor($connection))->execute($this->applyRequest($plan)); self::assertSame($fixture['dataset_id'], $result->datasetId()); - self::assertSame($plan->hash(), $result->planHash()); + self::assertSame((new PlanCodec())->hash($plan), $result->planHash()); self::assertSame(3, $result->operationCount()); self::assertSame($fixture['dataset'], $this->datasetRow($pdo)); @@ -357,13 +773,14 @@ public function testCreateTargetTransformationCreatesNumericTargetInPlace( } /** @param list $expectedVersionFamilies */ - #[DataProvider('createTargetRejectedServices')] + #[DataProvider('createTargetRejectedCases')] public function testCreateTargetTransformationRejectsNonAtomicProfilesWithoutChangingState( string $expectedProfile, ?string $environmentPrefix, string $driver, array $expectedVersionFamilies, ?string $expectedVersionEnvironment, + string $contractVersion, ): void { unset($expectedVersionFamilies, $expectedVersionEnvironment); $pdo = $this->servicePdo($expectedProfile, $environmentPrefix, $driver); @@ -386,19 +803,21 @@ public function testCreateTargetTransformationRejectsNonAtomicProfilesWithoutCha 'variable_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM variable', []), 'dolt_repository' => $this->doltRepositoryEvidence($pdo, $connection), ]; + $plan = $contractVersion === '0.1' + ? $this->createTargetPlan() + : $this->officialCreateTargetPlan02(); + $officialCase = $contractVersion === '0.2' + ? $this->officialInPlaceCase02($expectedProfile . '-create-target-fails-before-mutation') + : null; try { - (new InPlaceTransformationExecutor($connection))->execute($this->createTargetPlan()); + (new InPlaceTransformationExecutor($connection))->execute($this->applyRequest($plan)); self::fail('Expected non-atomic service profile to reject implicit target creation during preflight.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); - self::assertSame( - sprintf( - '%s cannot atomically add a new INTO target to an existing wide table; register the target variable first.', - $expectedProfile, - ), - $exception->getMessage(), - ); + } catch (TransformationFailure $exception) { + self::assertSame($officialCase['expected_error'] ?? 'schema_change_not_atomic', $exception->diagnosticCode()); + } + if ($officialCase !== null) { + self::assertFalse($officialCase['mutation_started']); } self::assertSame($before['catalog'], $this->fullCatalogSnapshot($pdo)); @@ -440,6 +859,7 @@ public function testFixtureSetupRejectsDirtyDeterministicNamespaceWithoutDeletin $this->assertDoltWorkingSetCleanBeforeFixture($pdo, $connection); (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); $pdo->exec( 'CREATE TABLE ' . $this->qualifiedTable($connection, $tableName) @@ -613,6 +1033,7 @@ private function installFixture(PDO $pdo, Connection $connection, ?array $fixtur { $this->assertDoltWorkingSetCleanBeforeFixture($pdo, $connection); (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); $fixture ??= $this->reserveFixtureIdentity($connection); @@ -721,6 +1142,7 @@ private function purgeFixture(PDO $pdo, Connection $connection, array $fixture): $pdo->prepare( 'DELETE FROM variable_value_label_set WHERE variable_id IN (SELECT variable_id FROM variable WHERE dataset_id = ?)', )->execute([$fixture['dataset_id']]); + $pdo->prepare('DELETE FROM transformation_apply WHERE dataset_id = ?')->execute([$fixture['dataset_id']]); $pdo->prepare( 'DELETE FROM value_label WHERE value_label_set_id IN (SELECT value_label_set_id FROM value_label_set WHERE dataset_id = ?)', )->execute([$fixture['dataset_id']]); @@ -760,58 +1182,103 @@ private function purgeDirtyNamespaceFixture(PDO $pdo, Connection $connection, ar private function existingTargetPlan(): TransformationPlan { - return new TransformationPlan($this->activeFixtureId(), [ - new RecodeOperation('SourceValue', 'Destination', [ + return new TransformationPlan(PlanContract::V02, 'parent', [ + new RecodeOperation('SourceValue', 'SourceValue', TargetMode::Replace, [ new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(10)), + new ExactMatch($this->binary64(1.0)), + new LiteralResult($this->binary64(10.0)), ), new RecodeRule( - new NumericRangeSelector(2.0, 3.0), - new AssignValueAction(ScalarValue::number(20)), + new RangeMatch($this->binary64(2.0), $this->binary64(3.0)), + new LiteralResult($this->binary64(20.0)), ), new RecodeRule( - new MissingValueSelector(), - new AssignValueAction(ScalarValue::number(99)), + new SystemMissingMatch(), + new LiteralResult($this->binary64(99.0)), ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), + ], new CopyResult()), + new AssignOperation('Destination', TargetMode::Replace, new VariableOperand('SourceValue')), new SetVariableLabelOperation('Destination', 'Recoded destination'), - new SetValueLabelsOperation('Destination', [ - new ValueLabel(ScalarValue::number(10), 'Ten'), - new ValueLabel(ScalarValue::number(20), 'Twenty'), - new ValueLabel(ScalarValue::number(99), 'Missing source'), + new ReplaceValueLabelsOperation('Destination', [ + new ValueLabel($this->binary64(10.0), 'Ten'), + new ValueLabel($this->binary64(20.0), 'Twenty'), + new ValueLabel($this->binary64(99.0), 'Missing source'), ]), ]); } private function createTargetPlan(): TransformationPlan { - return new TransformationPlan($this->activeFixtureId(), [ - new RecodeOperation('SourceValue', 'CreatedTarget', [ + return new TransformationPlan(PlanContract::V01, 'parent', [ + new RecodeOperation('SourceValue', 'CreatedTarget', TargetMode::Create, [ new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(10)), + new ExactMatch($this->binary64(1.0)), + new LiteralResult($this->binary64(10.0)), ), new RecodeRule( - new NumericRangeSelector(2.0, 3.0), - new AssignValueAction(ScalarValue::number(20)), + new RangeMatch($this->binary64(2.0), $this->binary64(3.0)), + new LiteralResult($this->binary64(20.0)), ), new RecodeRule( - new MissingValueSelector(), - new AssignValueAction(ScalarValue::number(99)), + new SystemMissingMatch(), + new LiteralResult($this->binary64(99.0)), ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), + ], new CopyResult()), new SetVariableLabelOperation('CreatedTarget', 'Recoded created target'), - new SetValueLabelsOperation('CreatedTarget', [ - new ValueLabel(ScalarValue::number(10), 'Ten'), - new ValueLabel(ScalarValue::number(20), 'Twenty'), - new ValueLabel(ScalarValue::number(99), 'Missing source'), + new ReplaceValueLabelsOperation('CreatedTarget', [ + new ValueLabel($this->binary64(10.0), 'Ten'), + new ValueLabel($this->binary64(20.0), 'Twenty'), + new ValueLabel($this->binary64(99.0), 'Missing source'), ]), ]); } + private function officialCreateTargetPlan02(): TransformationPlan + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.2.json')['cases'] as $case) { + if (is_array($case) + && ($case['id'] ?? null) === 'sqlite-create-target-source-copy' + && is_array($case['plan'] ?? null) + ) { + return (new PlanCodec())->fromArray($case['plan']); + } + } + + throw new RuntimeException('The official 0.2 create-target plan is missing.'); + } + + /** @return array */ + private function officialInPlaceCase02(string $id): array + { + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.2.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id) { + return $case; + } + } + + throw new RuntimeException('The official 0.2 in-place case is missing: ' . $id); + } + + /** @param array{branch: string, head: string}|null $doltBefore */ + private function applyRequest(TransformationPlan $plan, ?array $doltBefore = null): InPlaceApplyRequest + { + return new InPlaceApplyRequest( + $plan, + 'parent', + $this->activeFixtureId(), + str_repeat('a', 64), + 'integration-test', + $doltBefore['branch'] ?? null, + $doltBefore['head'] ?? null, + ); + } + + private function binary64(float $value): Binary64Value + { + $packed = pack('E', $value); + return Binary64Value::fromBits(bin2hex($packed)); + } + private function servicePdo(string $expectedProfile, ?string $environmentPrefix, string $driver): PDO { if (!in_array($driver, PDO::getAvailableDrivers(), true)) { @@ -875,6 +1342,95 @@ private function servicePdo(string $expectedProfile, ?string $environmentPrefix, ); } + private function isolatedMySqlFamilyPdo(string $expectedProfile, string $environmentPrefix): PDO + { + $dsn = getenv($environmentPrefix . '_DSN'); + $adminUser = getenv($environmentPrefix . '_ADMIN_USER'); + $adminPassword = getenv($environmentPrefix . '_ADMIN_PASSWORD'); + if (!is_string($dsn) || $dsn === '') { + self::markTestSkipped($environmentPrefix . '_DSN is not configured.'); + } + if (!is_string($adminUser) || $adminUser === '' || !is_string($adminPassword)) { + self::markTestSkipped( + $environmentPrefix . '_ADMIN_USER and ' . $environmentPrefix + . '_ADMIN_PASSWORD must be explicitly configured for isolated engine tests.', + ); + } + $targetUser = getenv($environmentPrefix . '_USER'); + $targetPassword = getenv($environmentPrefix . '_PASSWORD'); + $options = [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]; + $database = $this->newDoltDatabaseName(); + $admin = new PDO($dsn, $adminUser, $adminPassword, $options); + $admin->exec('CREATE DATABASE ' . $this->quoteDoltDatabaseName($database)); + $this->isolatedTestDatabases[] = ['admin' => $admin, 'database' => $database]; + if (is_string($targetUser) && $targetUser !== '') { + $quotedTargetUser = $admin->quote($targetUser); + if (!is_string($quotedTargetUser)) { + throw new RuntimeException('Unable to quote the MySQL-family target user.'); + } + $admin->exec( + 'GRANT ALL PRIVILEGES ON ' . $this->quoteDoltDatabaseName($database) + . ".* TO {$quotedTargetUser}@'%'", + ); + } + $pdo = new PDO( + $this->dsnForDoltDatabase($dsn, $database), + is_string($targetUser) ? $targetUser : null, + is_string($targetPassword) ? $targetPassword : null, + $options, + ); + self::assertSame($expectedProfile, (new Connection($pdo))->profileName); + self::assertSame($database, $this->scalar($pdo, 'SELECT DATABASE()', [])); + + return $pdo; + } + + private function dropMySqlForeignKeys(PDO $pdo, Connection $connection): void + { + $constraints = $this->rows( + $pdo, + 'SELECT table_name, constraint_name FROM information_schema.referential_constraints ' + . 'WHERE constraint_schema = DATABASE() ORDER BY table_name, constraint_name', + [], + ); + foreach ($constraints as $constraint) { + $constraint = array_change_key_case($constraint, CASE_LOWER); + $table = $constraint['table_name'] ?? null; + $name = $constraint['constraint_name'] ?? null; + if (!is_string($table) || $table === '' || !is_string($name) || $name === '') { + throw new RuntimeException('MySQL-family foreign-key metadata is malformed.'); + } + $pdo->exec( + 'ALTER TABLE ' . $connection->profile->quoteIdentifier($table) + . ' DROP FOREIGN KEY ' . $connection->profile->quoteIdentifier($name), + ); + } + } + + private function dropMySqlSecondaryIndexes(PDO $pdo, Connection $connection, string $table): void + { + $indexRows = $this->rows( + $pdo, + 'SELECT index_name FROM information_schema.statistics ' + . "WHERE table_schema = DATABASE() AND table_name = ? AND index_name <> 'PRIMARY' ORDER BY index_name", + [$table], + ); + $indexes = []; + foreach ($indexRows as $row) { + $row = array_change_key_case($row, CASE_LOWER); + if (!is_string($row['index_name'] ?? null) || $row['index_name'] === '') { + throw new RuntimeException('MySQL-family index metadata is malformed.'); + } + $indexes[$row['index_name']] = true; + } + foreach (array_keys($indexes) as $index) { + $pdo->exec( + 'ALTER TABLE ' . $connection->profile->quoteIdentifier($table) + . ' DROP INDEX ' . $connection->profile->quoteIdentifier($index), + ); + } + } + /** @return array{user: string, password: string}|null */ private function doltAdminCredentials(string|false $user, string|false $password): ?array { @@ -934,7 +1490,7 @@ private function isolatedDoltPdo( $options, ); $admin->exec('CREATE DATABASE ' . $this->quoteDoltDatabaseName($database)); - $this->doltTestDatabases[] = ['admin' => $admin, 'database' => $database]; + $this->isolatedTestDatabases[] = ['admin' => $admin, 'database' => $database]; if ($targetUser !== null && $targetUser !== '') { $quotedTargetUser = $admin->quote($targetUser); if (!is_string($quotedTargetUser)) { @@ -1125,6 +1681,37 @@ private function fullCatalogSnapshot(PDO $pdo): array ]; } + /** + * @param array{dataset_id: string, source_variable_id: string, destination_variable_id: string, table_name: string, dataset_name: string} $fixture + * @return array + */ + private function applyFailureSnapshot(PDO $pdo, Connection $connection, array $fixture): array + { + return [ + 'catalog' => $this->fullCatalogSnapshot($pdo), + 'audit' => $this->rows( + $pdo, + 'SELECT * FROM transformation_apply WHERE dataset_id = ? ORDER BY apply_id', + [$fixture['dataset_id']], + ), + 'rows' => $this->rawTableRows( + $pdo, + $connection, + $fixture['table_name'], + ['__case_ordinal', 'source_value', 'destination'], + ), + 'columns' => $this->tableColumns($pdo, $connection, $fixture['table_name']), + 'tables' => $this->tableNames($pdo), + 'dataset_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset', []), + 'persistent_data_table_count' => (int) $this->scalar( + $pdo, + 'SELECT COUNT(DISTINCT physical_table_name) FROM dataset', + [], + ), + 'dolt_repository' => $this->doltRepositoryEvidence($pdo, $connection), + ]; + } + /** @return list */ private function numericColumn( PDO $pdo, diff --git a/tests/Integration/OfficialInPlaceTransformation01Test.php b/tests/Integration/OfficialInPlaceTransformation01Test.php new file mode 100644 index 0000000..135df21 --- /dev/null +++ b/tests/Integration/OfficialInPlaceTransformation01Test.php @@ -0,0 +1,658 @@ + */ + private array $isolatedTestDatabases = []; + + protected function tearDown(): void + { + $failures = []; + try { + foreach (array_reverse($this->isolatedTestDatabases) as $fixture) { + try { + $fixture['admin']->exec('DROP DATABASE ' . $this->quoteDatabase($fixture['database'])); + } catch (\Throwable $failure) { + $failures[] = $fixture['database'] . ': ' . $failure->getMessage(); + } + } + } finally { + $this->isolatedTestDatabases = []; + parent::tearDown(); + } + if ($failures !== []) { + throw new RuntimeException('Isolated database cleanup failed: ' . implode('; ', $failures)); + } + } + + /** @return iterable}> */ + public static function officialBackendCases(): iterable + { + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.1.json')['cases'] as $case) { + if (is_array($case) && is_string($case['id'] ?? null)) { + yield $case['id'] => [$case]; + } + } + } + + /** @param array $case */ + #[DataProvider('officialBackendCases')] + public function testEveryOfficialBackendManifestCaseRuns(array $case): void + { + $id = $case['id'] ?? null; + self::assertIsString($id); + + match ($id) { + 'mysql-recode-and-labels-preserve-dataset-and-table-identity', + 'dolt-recode-and-labels-preserve-controlled-context' => $this->assertOfficialBackendSuccess($case), + 'reject-dolt-branch-mismatch', + 'reject-dolt-head-mismatch', + 'reject-dolt-dirty-working-set' => $this->assertOfficialDoltContextFailure($case), + 'reject-mysql-nontransactional-create-target' => $this->assertOfficialMySqlCreateRejection($case), + default => throw new RuntimeException('Unhandled official in-place 0.1 case: ' . $id), + }; + } + + public function testSqliteNumericRecodeUsesFirstMatchInclusiveRangeAndSystemMissing(): void + { + $pdo = $this->fixture(); + $connection = new Connection($pdo); + $plan = $this->planCase('numeric-recode-and-declared-labels'); + $request = new InPlaceApplyRequest( + $plan, + 'parent', + self::DATASET_ID, + hash('sha256', 'RECODE q1.'), + 'conformance-runner', + ); + $tablesBefore = $this->tables($pdo); + + $result = (new InPlaceTransformationExecutor($connection))->execute($request); + + self::assertSame('075571b4f84b0f60f289aebb6ec990ad61d34a96b46655802ea8f8bc07a1a405', $result->planHash()); + self::assertSame($tablesBefore, $this->tables($pdo)); + self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset')); + self::assertSame( + [0.0, 0.0, 1.0, 1.0, 1.0, null, null], + array_map( + static fn(mixed $value): ?float => $value === null ? null : (float) $value, + $this->column($pdo, 'SELECT q1_binary FROM data_plan01 ORDER BY __case_ordinal'), + ), + ); + self::assertSame('Positive response', $this->scalar( + $pdo, + "SELECT variable_label FROM variable WHERE dataset_id = '" . self::DATASET_ID . "' AND source_name = 'q1_binary'", + )); + self::assertSame( + [[0.0, 'No'], [1.0, 'Yes']], + array_map( + static fn(array $row): array => [(float) $row['numeric_code'], (string) $row['label']], + $this->rows($pdo, 'SELECT label.numeric_code, label.label FROM value_label label ORDER BY label.ordinal'), + ), + ); + self::assertSame( + ['openstatspec-in-place-transformation-v0.1', $result->planHash(), 3], + $this->normalizedAudit($pdo), + ); + } + + public function testSqliteStringValueLabelsAreReplacedExactlyWithoutChangingOtherMetadata(): void + { + $pdo = $this->fixture(); + $pdo->exec("ALTER TABLE data_plan01 ADD COLUMN color TEXT NOT NULL DEFAULT ''"); + $pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width, variable_label, measurement_level) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', + )->execute(['77777777-7777-4777-8777-777777777777', self::DATASET_ID, 2, 'color', 'color', 'string', 1, 'Colour', 'nominal']); + $pdo->exec("UPDATE data_plan01 SET color = CASE WHEN __case_ordinal % 2 = 0 THEN 'B' ELSE 'R' END"); + $plan = $this->planCase('string-value-label-replacement'); + $request = new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, hash('sha256', 'VALUE LABELS color.'), 'conformance-runner'); + + (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($request); + + self::assertSame( + [['R', 'Punane'], ['B', 'Sinine']], + array_map( + static fn(array $row): array => [(string) $row['string_code'], (string) $row['label']], + $this->rows($pdo, 'SELECT label.string_code, label.label FROM value_label label ORDER BY label.ordinal'), + ), + ); + self::assertSame( + ['variable_label' => 'Colour', 'measurement_level' => 'nominal'], + $this->rows($pdo, "SELECT variable_label, measurement_level FROM variable WHERE source_name = 'color'")[0], + ); + } + + private function fixture(): PDO + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available.'); + } + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]); + $pdo->exec('PRAGMA foreign_keys = ON'); + (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); + CatalogOwnership::markCurrentVersion($pdo); + $pdo->exec('CREATE TABLE data_plan01 (__case_ordinal INTEGER NOT NULL PRIMARY KEY, q1 REAL NULL)'); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([self::DATASET_ID, '1.0', 'fixture', null, 'data_plan01', 'Plan 0.1 fixture', 7, '2026-08-17 00:00:00']); + $pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) VALUES (?, ?, ?, ?, ?, ?)', + )->execute(['88888888-8888-4888-8888-888888888888', self::DATASET_ID, 1, 'q1', 'q1', 'numeric']); + $insert = $pdo->prepare('INSERT INTO data_plan01 (__case_ordinal, q1) VALUES (?, ?)'); + foreach ([[1, 1.0], [2, 2.0], [3, 3.0], [4, 4.0], [5, 5.0], [6, 6.0], [7, null]] as $row) { + $insert->execute($row); + } + return $pdo; + } + + /** @param array $case */ + private function assertOfficialBackendSuccess(array $case): void + { + $profile = $case['database_profile'] ?? null; + $beforeContract = $case['before'] ?? null; + $afterContract = $case['after'] ?? null; + self::assertContains($profile, ['mysql', 'dolt']); + self::assertIsArray($beforeContract); + self::assertIsArray($afterContract); + $database = $profile === 'mysql' ? (string) $beforeContract['physical_table_schema'] : null; + $pdo = $this->isolatedBackendPdo((string) $profile, $database); + $connection = new Connection($pdo); + $fixture = $this->installOfficialBackendFixture($pdo, $connection, $case); + $context = $this->backendContext($pdo, $connection); + $before = $this->officialBackendSnapshot($pdo, $connection, $fixture); + $compiled = $this->compileOfficialSource((string) $case['source_text']); + $request = new InPlaceApplyRequest( + $compiled->plan, + 'parent', + $fixture['dataset_id'], + $compiled->sourceHash, + 'conformance-runner', + $context['branch'] ?? null, + $context['head'] ?? null, + ); + + $result = (new InPlaceTransformationExecutor($connection))->execute($request); + $after = $this->officialBackendSnapshot($pdo, $connection, $fixture); + + self::assertSame($beforeContract['dataset_id'], $result->datasetId()); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($before['case_count'], $after['case_count']); + self::assertSame($before['case_ordinals'], $after['case_ordinals']); + self::assertSame([0.0, 0.0, 1.0, 4.0], $after['score']); + self::assertSame($afterContract['variable_label'], $after['variable_label']); + self::assertSame($afterContract['value_labels'], $after['value_labels']); + self::assertCount(1, $after['audit']); + $audit = $after['audit'][0]; + foreach ($case['required_audit_fields'] as $field) { + self::assertArrayHasKey($field, $audit); + } + self::assertSame('openstatspec-in-place-transformation-v0.1', $audit['contract_id']); + self::assertSame($profile, $audit['database_profile']); + self::assertSame($fixture['dataset_id'], $audit['dataset_id']); + self::assertSame($fixture['schema'], $audit['physical_table_schema']); + self::assertSame($fixture['table'], $audit['physical_table_name']); + self::assertSame($compiled->sourceHash, $audit['source_hash']); + self::assertSame($result->planHash(), $audit['plan_hash']); + self::assertSame('conformance-runner', $audit['actor']); + self::assertSame('succeeded', $audit['status']); + self::assertSame($result->operationCount(), (int) $audit['operation_count']); + $this->assertNoForbiddenArtifactTables($after['tables']); + + if ($profile === 'dolt') { + if ($context === null) { + throw new RuntimeException('Dolt conformance execution did not capture repository context.'); + } + self::assertNotNull($before['repository']); + self::assertNotNull($after['repository']); + self::assertSame($before['repository']['branch'], $after['repository']['branch']); + self::assertSame($before['repository']['head'], $after['repository']['head']); + self::assertSame($before['repository']['history'], $after['repository']['history']); + self::assertNotSame([], $after['repository']['status']); + self::assertContains($fixture['table'], array_column($after['repository']['status'], 'table_name')); + self::assertSame($context['branch'], $audit['dolt_branch']); + self::assertSame($context['head'], $audit['dolt_head_before']); + self::assertSame($context['head'], $audit['dolt_head_after']); + } else { + self::assertNull($audit['dolt_branch']); + self::assertNull($audit['dolt_head_before']); + self::assertNull($audit['dolt_head_after']); + } + } + + /** @param array $case */ + private function assertOfficialDoltContextFailure(array $case): void + { + $pdo = $this->isolatedBackendPdo('dolt'); + $connection = new Connection($pdo); + $fixture = $this->installOfficialBackendFixture($pdo, $connection, $case); + $context = $this->backendContext($pdo, $connection); + self::assertNotNull($context); + $id = (string) $case['id']; + if ($id === 'reject-dolt-dirty-working-set') { + $pdo->exec( + 'UPDATE ' . $this->qualifiedBackendTable($connection, $fixture['schema'], $fixture['table']) + . ' SET `score` = 99 WHERE `__case_ordinal` = 1', + ); + } + $before = $this->officialBackendSnapshot($pdo, $connection, $fixture); + $source = $this->successfulSourceText(); + $compiled = $this->compileOfficialSource($source); + $expectedBranch = $context['branch']; + $expectedHead = $context['head']; + if ($id === 'reject-dolt-branch-mismatch') { + $expectedBranch = (string) $case['expected_context']['branch']; + } elseif ($id === 'reject-dolt-head-mismatch') { + $expectedHead = (string) $case['observed_context']['head']; + } + + try { + (new InPlaceTransformationExecutor($connection))->execute(new InPlaceApplyRequest( + $compiled->plan, + 'parent', + $fixture['dataset_id'], + hash('sha256', $source), + 'conformance-runner', + $expectedBranch, + $expectedHead, + )); + self::fail('An official Dolt context failure was accepted: ' . $id); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->officialBackendSnapshot($pdo, $connection, $fixture)); + } + + /** @param array $case */ + private function assertOfficialMySqlCreateRejection(array $case): void + { + $pdo = $this->isolatedBackendPdo('mysql'); + $connection = new Connection($pdo); + $fixture = $this->installOfficialBackendFixture($pdo, $connection, $case); + $before = $this->officialBackendSnapshot($pdo, $connection, $fixture); + $compiled = $this->compileOfficialSource((string) $case['source_text']); + + try { + (new InPlaceTransformationExecutor($connection))->execute(new InPlaceApplyRequest( + $compiled->plan, + 'parent', + $fixture['dataset_id'], + $compiled->sourceHash, + 'conformance-runner', + )); + self::fail('The official MySQL create-target rejection was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->officialBackendSnapshot($pdo, $connection, $fixture)); + } + + private function compileOfficialSource(string $source): \OpenStatSpec\Frontend\Spss\SpssCompilationResult + { + return (new SpssCompiler())->compile(new SpssFrontendRequest( + SpssFrontendRequest::CONTRACT, + 'parent', + new InputSchema([new InputVariable('score', 'numeric')]), + $source, + )); + } + + private function successfulSourceText(): string + { + return (string) $this->bindingCase('mysql-recode-and-labels-preserve-dataset-and-table-identity')['source_text']; + } + + private function isolatedBackendPdo(string $profile, ?string $preferredDatabase = null): PDO + { + $prefix = match ($profile) { + 'mysql' => 'OPENSTATSPEC_MYSQL', + 'dolt' => 'OPENSTATSPEC_DOLT', + default => throw new RuntimeException('Unsupported official 0.1 backend profile.'), + }; + $dsn = getenv($prefix . '_DSN'); + $adminUser = getenv($prefix . '_ADMIN_USER'); + $adminPassword = getenv($prefix . '_ADMIN_PASSWORD'); + if (!is_string($dsn) || $dsn === '') { + self::markTestSkipped($prefix . '_DSN is not configured.'); + } + if (!is_string($adminUser) || $adminUser === '' || !is_string($adminPassword)) { + self::markTestSkipped($prefix . ' explicit admin credentials are required for isolated official tests.'); + } + $targetUser = getenv($prefix . '_USER'); + $targetPassword = getenv($prefix . '_PASSWORD'); + $options = [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]; + $database = $preferredDatabase ?? sprintf( + 'openstatspec_t9_v01_%d_%s', + getmypid(), + bin2hex(random_bytes(6)), + ); + $quotedDatabase = $this->quoteDatabase($database); + $admin = new PDO($dsn, $adminUser, $adminPassword, $options); + self::assertSame(0, (int) $this->scalar( + $admin, + 'SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = ?', + [$database], + )); + $admin->exec('CREATE DATABASE ' . $quotedDatabase); + $this->isolatedTestDatabases[] = ['admin' => $admin, 'database' => $database]; + if (is_string($targetUser) && $targetUser !== '') { + $quotedTargetUser = $admin->quote($targetUser); + if (!is_string($quotedTargetUser)) { + throw new RuntimeException('Unable to quote the official backend test user.'); + } + $admin->exec('GRANT ALL PRIVILEGES ON ' . $quotedDatabase . ".* TO {$quotedTargetUser}@'%'"); + } + + $pdo = new PDO( + $this->dsnForDatabase($dsn, $database), + is_string($targetUser) ? $targetUser : null, + is_string($targetPassword) ? $targetPassword : null, + $options, + ); + self::assertSame($profile, (new Connection($pdo))->profileName); + self::assertSame($database, $this->scalar($pdo, 'SELECT DATABASE()')); + + return $pdo; + } + + /** + * @param array $case + * @return array{dataset_id: string, schema: string|null, table: string} + */ + private function installOfficialBackendFixture(PDO $pdo, Connection $connection, array $case): array + { + $before = is_array($case['before'] ?? null) ? $case['before'] : []; + $observed = is_array($case['observed_context'] ?? null) ? $case['observed_context'] : []; + if ($connection->profileName === 'dolt') { + $branch = is_string($observed['branch'] ?? null) + ? $observed['branch'] + : (is_string($before['dolt_branch'] ?? null) ? $before['dolt_branch'] : 'main'); + $active = (string) $this->scalar($pdo, 'SELECT active_branch()'); + if ($branch !== $active) { + $checkout = $pdo->prepare('CALL DOLT_CHECKOUT(?, ?)'); + self::assertInstanceOf(PDOStatement::class, $checkout); + $checkout->execute(['-b', $branch]); + } + } + + (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); + CatalogOwnership::markCurrentVersion($pdo); + $datasetId = is_string($before['dataset_id'] ?? null) + ? $before['dataset_id'] + : '11111111-1111-4111-8111-111111111111'; + $schema = is_string($before['physical_table_schema'] ?? null) ? $before['physical_table_schema'] : null; + $table = is_string($before['physical_table_name'] ?? null) ? $before['physical_table_name'] : 'data_survey'; + $qualified = $this->qualifiedBackendTable($connection, $schema, $table); + $pdo->exec( + 'CREATE TABLE ' . $qualified . ' (' + . $connection->profile->quoteIdentifier('__case_ordinal') . ' BIGINT NOT NULL PRIMARY KEY, ' + . $connection->profile->quoteIdentifier('score') . ' DOUBLE NULL)', + ); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, ' + . 'dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([$datasetId, '1.0', 'fixture', $schema, $table, 'Official in-place 0.1', 4, '2026-08-17 00:00:00']); + $pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' + . 'VALUES (?, ?, ?, ?, ?, ?)', + )->execute(['22222222-2222-4222-8222-222222222222', $datasetId, 1, 'score', 'score', 'numeric']); + $insert = $pdo->prepare( + 'INSERT INTO ' . $qualified . ' (' + . $connection->profile->quoteIdentifier('__case_ordinal') . ', ' + . $connection->profile->quoteIdentifier('score') . ') VALUES (?, ?)', + ); + foreach ([[1, 1.0], [2, 2.0], [3, 3.0], [4, 4.0]] as $row) { + $insert->execute($row); + } + + if ($connection->profileName === 'dolt') { + $commit = $pdo->prepare('CALL DOLT_COMMIT(?, ?)'); + self::assertInstanceOf(PDOStatement::class, $commit); + $commit->execute(['-Am', 'Official in-place 0.1 fixture']); + self::assertSame([], $this->rows($pdo, 'SELECT table_name FROM dolt_status')); + } + + return ['dataset_id' => $datasetId, 'schema' => $schema, 'table' => $table]; + } + + /** @return array{branch: string, head: string}|null */ + private function backendContext(PDO $pdo, Connection $connection): ?array + { + if ($connection->profileName !== 'dolt') { + return null; + } + $row = $this->rows( + $pdo, + "SELECT active_branch() AS branch_name, dolt_hashof('HEAD') AS head_hash", + )[0] ?? null; + self::assertIsArray($row); + + return ['branch' => (string) $row['branch_name'], 'head' => (string) $row['head_hash']]; + } + + /** + * @param array{dataset_id: string, schema: string|null, table: string} $fixture + * @return array + */ + private function officialBackendSnapshot(PDO $pdo, Connection $connection, array $fixture): array + { + $qualified = $this->qualifiedBackendTable($connection, $fixture['schema'], $fixture['table']); + $score = array_map( + static fn(mixed $value): ?float => $value === null ? null : (float) $value, + $this->column($pdo, 'SELECT score FROM ' . $qualified . ' ORDER BY __case_ordinal'), + ); + $labels = array_map( + static function (array $row): array { + $numericCode = (float) $row['numeric_code']; + + return [ + floor($numericCode) === $numericCode ? (int) $numericCode : $numericCode, + (string) $row['label'], + ]; + }, + $this->rows( + $pdo, + 'SELECT label.numeric_code, label.label FROM variable ' + . 'JOIN variable_value_label_set link ON link.variable_id = variable.variable_id ' + . 'JOIN value_label label ON label.value_label_set_id = link.value_label_set_id ' + . 'WHERE variable.dataset_id = ? AND variable.source_name = ? ORDER BY label.ordinal', + [$fixture['dataset_id'], 'score'], + ), + ); + + return [ + 'dataset_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset'), + 'persistent_data_table_count' => (int) $this->scalar($pdo, 'SELECT COUNT(DISTINCT physical_table_name) FROM dataset'), + 'dataset_identity' => $this->rows( + $pdo, + 'SELECT dataset_id, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', + [$fixture['dataset_id']], + ), + 'tables' => array_map('strval', $this->column( + $pdo, + 'SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() ORDER BY table_name', + )), + 'case_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM ' . $qualified), + 'case_ordinals' => array_map('intval', $this->column( + $pdo, + 'SELECT __case_ordinal FROM ' . $qualified . ' ORDER BY __case_ordinal', + )), + 'score' => $score, + 'variables' => $this->rows( + $pdo, + 'SELECT * FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', + [$fixture['dataset_id']], + ), + 'variable_label' => $this->scalar( + $pdo, + 'SELECT variable_label FROM variable WHERE dataset_id = ? AND source_name = ?', + [$fixture['dataset_id'], 'score'], + ), + 'value_labels' => $labels, + 'audit' => $this->rows( + $pdo, + 'SELECT * FROM transformation_apply WHERE dataset_id = ? ORDER BY apply_id', + [$fixture['dataset_id']], + ), + 'repository' => $this->backendRepositoryEvidence($pdo, $connection), + ]; + } + + /** @return array{branch: string, head: string, history: list, status: list}|null */ + private function backendRepositoryEvidence(PDO $pdo, Connection $connection): ?array + { + if ($connection->profileName !== 'dolt') { + return null; + } + $identity = $this->rows( + $pdo, + "SELECT active_branch() AS branch_name, dolt_hashof('HEAD') AS head_hash", + )[0] ?? []; + $status = array_map(static function (array $row): array { + $row = array_change_key_case($row, CASE_LOWER); + + return [ + 'table_name' => (string) ($row['table_name'] ?? ''), + 'status' => (string) ($row['status'] ?? ''), + 'staged' => (bool) ($row['staged'] ?? false), + ]; + }, $this->rows($pdo, 'SELECT table_name, status, staged FROM dolt_status ORDER BY table_name')); + + return [ + 'branch' => (string) ($identity['branch_name'] ?? ''), + 'head' => (string) ($identity['head_hash'] ?? ''), + 'history' => array_map('strval', $this->column( + $pdo, + 'SELECT commit_hash FROM dolt_log ORDER BY commit_order DESC, commit_hash', + )), + 'status' => $status, + ]; + } + + /** @param list $tables */ + private function assertNoForbiddenArtifactTables(array $tables): void + { + self::assertSame([], array_values(array_filter( + $tables, + static fn(string $table): bool => preg_match('/derived|output|staging|snapshot|rollback|recovery/i', $table) === 1, + ))); + } + + private function qualifiedBackendTable(Connection $connection, ?string $schema, string $table): string + { + return ($schema === null ? '' : $connection->profile->quoteIdentifier($schema) . '.') + . $connection->profile->quoteIdentifier($table); + } + + private function quoteDatabase(string $database): string + { + if (strlen($database) > 64 || preg_match('/\A[a-z][a-z0-9_]*\z/D', $database) !== 1) { + throw new RuntimeException('Unsafe official backend database name.'); + } + + return '`' . $database . '`'; + } + + private function dsnForDatabase(string $dsn, string $database): string + { + $this->quoteDatabase($database); + if (!str_starts_with(strtolower($dsn), 'mysql:')) { + throw new RuntimeException('Official MySQL-family tests require a mysql PDO DSN.'); + } + $parts = explode(';', substr($dsn, strlen('mysql:'))); + $databaseIndex = null; + foreach ($parts as $index => $part) { + if (strtolower(trim((string) explode('=', $part, 2)[0])) !== 'dbname') { + continue; + } + if ($databaseIndex !== null) { + throw new RuntimeException('Official backend DSN contains duplicate dbname settings.'); + } + $databaseIndex = $index; + } + if ($databaseIndex === null) { + if (end($parts) === '') { + array_pop($parts); + } + $parts[] = 'dbname=' . $database; + } else { + $parts[$databaseIndex] = 'dbname=' . $database; + } + + return 'mysql:' . implode(';', $parts); + } + + private function planCase(string $id): TransformationPlan + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.1.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id && is_array($case['plan'] ?? null)) { + return (new PlanCodec())->fromArray($case['plan']); + } + } + throw new \RuntimeException('Missing plan case: ' . $id); + } + + /** @return array */ + private function bindingCase(string $id): array + { + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.1.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id) { + return $case; + } + } + + throw new \RuntimeException('Missing in-place 0.1 case: ' . $id); + } + + /** @return list */ + private function tables(PDO $pdo): array + { + return array_map('strval', $this->column($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")); + } + + /** @return array{string, string, int} */ + private function normalizedAudit(PDO $pdo): array + { + $row = $this->rows($pdo, 'SELECT contract_id, plan_hash, operation_count FROM transformation_apply')[0]; + return [(string) $row['contract_id'], (string) $row['plan_hash'], (int) $row['operation_count']]; + } + +} diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php new file mode 100644 index 0000000..0c2aeca --- /dev/null +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -0,0 +1,1250 @@ + */ + private array $doltTestDatabases = []; + + protected function tearDown(): void + { + try { + foreach (array_reverse($this->doltTestDatabases) as $fixture) { + $fixture['admin']->exec('DROP DATABASE ' . $this->quoteDoltDatabase($fixture['database'])); + } + } finally { + $this->doltTestDatabases = []; + parent::tearDown(); + } + } + + /** @return iterable}> */ + public static function officialBackendCases(): iterable + { + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.2.json')['cases'] as $case) { + if (is_array($case) && is_string($case['id'] ?? null)) { + yield $case['id'] => [$case]; + } + } + } + + /** @param array $case */ + #[DataProvider('officialBackendCases')] + public function testEveryOfficialBackendManifestCaseRuns(array $case): void + { + $id = $case['id'] ?? null; + self::assertIsString($id); + + match ($id) { + 'dolt-preprovisioned-target-sequential-null-semantics', + 'dolt-preprovisioned-target-or-null-semantics', + 'dolt-preprovisioned-target-variable-missing-propagation', + 'dolt-preprovisioned-target-conditional-variable-missing-propagation' => $this->assertOfficialDoltSuccess($case), + 'dolt-create-target-fails-before-mutation', + 'mysql-create-target-fails-before-mutation', + 'mariadb-create-target-fails-before-mutation' => $this->assertOfficialCreateTargetRejection($case), + 'sqlite-inequality-boundary-semantics' => $this->assertOfficialSqliteInequality($case), + 'dolt-context-changed-after-mutation-rolls-back' => $this->assertOfficialDoltContextChanged($case), + 'dolt-empty-actor-fails-before-mutation' => $this->assertOfficialDoltEmptyActor($case), + 'sqlite-create-target-atomic-success' => $this->assertOfficialSqliteCreate($case), + default => throw new RuntimeException('Unhandled official in-place 0.2 case: ' . $id), + }; + } + + public function testSqliteCreateTargetFailureRollsBackSchemaRowsCatalogAndAudit(): void + { + $case = $this->bindingCase('sqlite-create-target-atomic-success'); + $pdo = $this->sqlite(); + $connection = new Connection($pdo); + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + 'data_atomic_create', + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $pdo->exec("ALTER TABLE transformation_apply ADD COLUMN task8_actor_guard INTEGER NOT NULL DEFAULT 0 CHECK (actor <> 'conformance-runner')"); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + + try { + (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::CREATE_DATASET_ID)); + self::fail('The injected audit failure did not abort the apply.'); + } catch (\PDOException $exception) { + self::assertStringContainsString('constraint', strtolower($exception->getMessage())); + } + + $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + self::assertSame($before, $after); + self::assertNotContains('target', $this->columns($pdo, $connection, 'data_atomic_create')); + self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM variable WHERE dataset_id = ? AND source_name = ?', [self::CREATE_DATASET_ID, 'target'])); + self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM transformation_apply', [])); + self::assertFalse($pdo->inTransaction()); + } + + public function testPostgresqlCreateTargetIsAtomicWhenConfigured(): void + { + $pdo = $this->postgresql(); + $connection = new Connection($pdo); + self::assertSame('postgresql', $connection->profileName); + $table = 'data_atomic_create_task8'; + $case = $this->bindingCase('sqlite-create-target-atomic-success'); + $this->prepareCatalog($pdo); + $this->assertNamespaceClean($pdo, self::CREATE_DATASET_ID, $table); + + try { + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + $table, + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); + (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::CREATE_DATASET_ID)); + $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($case['after']['rows'], $this->numericRows($pdo, $connection, $table, ['source_a', 'source_b', 'target'])); + $this->assertAudit($pdo, [...$case['expected_audit'], 'database_profile' => 'postgresql', 'physical_table_name' => $table]); + } finally { + $this->purgeFixture($pdo, $connection, self::CREATE_DATASET_ID, $table); + } + } + + public function testPostgresqlCreateTargetFailureRollsBackWhenConfigured(): void + { + $pdo = $this->postgresql(); + $connection = new Connection($pdo); + $table = 'data_atomic_rollback_task8'; + $case = $this->bindingCase('sqlite-create-target-atomic-success'); + $this->prepareCatalog($pdo); + $this->assertNamespaceClean($pdo, self::ROLLBACK_DATASET_ID, $table); + $auditGuardInstalled = false; + + try { + $this->installNumericFixture( + $pdo, + $connection, + self::ROLLBACK_DATASET_ID, + $table, + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT IF EXISTS task8_fail_audit'); + $pdo->exec("ALTER TABLE transformation_apply ADD CONSTRAINT task8_fail_audit CHECK (actor <> 'conformance-runner')"); + $auditGuardInstalled = true; + $before = $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + + try { + (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::ROLLBACK_DATASET_ID)); + self::fail('The injected PostgreSQL audit failure did not abort the apply.'); + } catch (\PDOException $exception) { + self::assertStringContainsString('task8_fail_audit', $exception->getMessage()); + } + + self::assertSame($before, $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table)); + self::assertNotContains('target', $this->columns($pdo, $connection, $table)); + self::assertFalse($pdo->inTransaction()); + } finally { + if ($auditGuardInstalled) { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT IF EXISTS task8_fail_audit'); + } + $this->purgeFixture($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + } + } + + /** @param array $case */ + private function assertOfficialCreateTargetRejection(array $case): void + { + $profile = $case['database_profile'] ?? null; + self::assertContains($profile, ['mysql', 'mariadb', 'dolt']); + $pdo = $this->mysqlFamily((string) $profile); + $connection = new Connection($pdo); + self::assertSame($profile, $connection->profileName); + $table = 'data_task9_create_' . $profile; + $this->checkoutDoltFixtureBranch($pdo, $connection, $case); + $this->prepareCatalog($pdo); + $this->assertNamespaceClean($pdo, self::CREATE_DATASET_ID, $table); + + try { + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + $table, + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $context = $this->commitAndReadDoltFixtureContext($pdo, $connection); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); + $request = $this->request( + $this->bindingCase('sqlite-create-target-atomic-success'), + self::CREATE_DATASET_ID, + $context['branch'] ?? null, + $context['head'] ?? null, + ); + + try { + (new InPlaceTransformationExecutor($connection))->execute($request); + self::fail('A non-atomic MySQL-family profile accepted an official create-target plan.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertFalse($pdo->inTransaction()); + self::assertFalse($case['mutation_started']); + self::assertSame($before, $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table)); + } finally { + $this->purgeFixture($pdo, $connection, self::CREATE_DATASET_ID, $table); + } + } + + /** @param array $case */ + private function assertOfficialDoltSuccess(array $case): void + { + $pdo = $this->mysqlFamily('dolt'); + $connection = new Connection($pdo); + self::assertSame('dolt', $connection->profileName); + $this->checkoutDoltFixtureBranch($pdo, $connection, $case); + $beforeContract = is_array($case['before'] ?? null) ? $case['before'] : []; + $afterContract = is_array($case['after'] ?? null) ? $case['after'] : []; + $beforeRows = $beforeContract['rows'] ?? $case['before_rows'] ?? null; + $afterRows = $afterContract['rows'] ?? $case['after_rows'] ?? null; + self::assertIsArray($beforeRows); + self::assertIsArray($afterRows); + $datasetId = $beforeContract['dataset_id'] ?? $case['dataset_id'] ?? null; + $table = $beforeContract['physical_table_name'] ?? $case['physical_table_name'] ?? null; + self::assertIsString($datasetId); + self::assertIsString($table); + [$variables, $fixtureRows] = $this->numericFixtureRows($beforeRows); + $targetIdentity = is_array($beforeContract['target_identity'] ?? null) + ? $beforeContract['target_identity'] + : []; + $variableIds = is_string($targetIdentity['variable_id'] ?? null) + ? ['target' => $targetIdentity['variable_id']] + : []; + + $this->installNumericFixture($pdo, $connection, $datasetId, $table, $variables, $fixtureRows, $variableIds); + $this->installOfficialMissingRules($pdo, $datasetId, $beforeContract); + $context = $this->commitAndReadDoltFixtureContext($pdo, $connection); + self::assertNotNull($context); + $before = $this->snapshot($pdo, $connection, $datasetId, $table); + $targetBefore = $this->targetIdentity($pdo, $datasetId, 'target'); + $metadataBefore = $this->targetMetadata($pdo, $datasetId, 'target'); + $labelsBefore = $this->valueLabels($pdo, $datasetId, 'target'); + $missingBefore = $this->missingRules($pdo, $datasetId, 'target'); + self::assertSame([], $before['repository']['status'] ?? null); + + $result = (new InPlaceTransformationExecutor($connection))->execute( + $this->request($case, $datasetId, $context['branch'], $context['head']), + ); + $after = $this->snapshot($pdo, $connection, $datasetId, $table); + + self::assertSame($datasetId, $result->datasetId()); + self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); + self::assertSame((int) $case['expected_audit']['operation_count'], $result->operationCount()); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($before['columns'], $after['columns']); + self::assertSame($before['variable_count'], $after['variable_count']); + self::assertSame($before['case_ordinals'], $after['case_ordinals']); + self::assertSame($before['case_count'], $after['case_count']); + self::assertSame($afterRows, $this->numericRows($pdo, $connection, $table, $variables)); + self::assertSame($targetBefore, $this->targetIdentity($pdo, $datasetId, 'target')); + $this->assertDoltRepositoryApplyEvidence($before, $after, $table, (string) $case['id']); + $this->assertNoForbiddenArtifactTables($after['tables']); + + if (($case['id'] ?? null) === 'dolt-preprovisioned-target-sequential-null-semantics') { + self::assertSame( + [ + 'variable_id' => $afterContract['target_identity']['variable_id'], + 'source_ordinal' => $afterContract['target_identity']['ordinal'], + 'source_name' => 'target', + 'physical_name' => 'target', + ], + $targetBefore, + ); + self::assertSame( + [ + 'source_ordinal' => 3, + 'storage_kind' => 'numeric', + 'declared_string_width' => null, + 'variable_label' => 'Synthetic conjunction', + 'print_format_family' => 'F', + 'print_format_width' => 1, + 'print_format_decimals' => 0, + 'write_format_family' => 'F', + 'write_format_width' => 1, + 'write_format_decimals' => 0, + 'measurement_level' => 'nominal', + 'variable_role' => null, + 'display_width' => null, + 'display_alignment' => null, + ], + $this->targetMetadata($pdo, $datasetId, 'target'), + ); + self::assertSame([[0.0, 'No'], [1.0, 'Yes']], $this->valueLabels($pdo, $datasetId, 'target')); + self::assertSame($missingBefore, $this->missingRules($pdo, $datasetId, 'target')); + } else { + self::assertSame($metadataBefore, $this->targetMetadata($pdo, $datasetId, 'target')); + self::assertSame($labelsBefore, $this->valueLabels($pdo, $datasetId, 'target')); + self::assertSame($missingBefore, $this->missingRules($pdo, $datasetId, 'target')); + } + + $audit = $this->assertAudit($pdo, [ + ...$case['expected_audit'], + 'dolt_branch' => $context['branch'], + 'dolt_head_before' => $context['head'], + 'dolt_head_after' => $context['head'], + ]); + foreach ($case['required_audit_fields'] ?? [] as $field) { + self::assertArrayHasKey($field, $audit); + } + } + + /** @param array $case */ + private function assertOfficialDoltContextChanged(array $case): void + { + $pdo = $this->mysqlFamily('dolt'); + $connection = new Connection($pdo); + $this->checkoutDoltFixtureBranch($pdo, $connection, $case); + $table = 'data_task9_context_change'; + $this->installNumericFixture( + $pdo, + $connection, + self::ROLLBACK_DATASET_ID, + $table, + ['source_a', 'source_b', 'target'], + [[1, 1.0, 1.0, 0.0]], + ); + $pdo->prepare('UPDATE variable SET variable_label = ? WHERE dataset_id = ? AND source_name = ?') + ->execute(['Before', self::ROLLBACK_DATASET_ID, 'target']); + $context = $this->commitAndReadDoltFixtureContext($pdo, $connection); + self::assertNotNull($context); + $before = $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + self::assertSame($case['before']['rows'], $this->numericRows($pdo, $connection, $table, ['target'])); + self::assertSame($case['before']['target_metadata']['variable_label'], $this->scalar( + $pdo, + 'SELECT variable_label FROM variable WHERE dataset_id = ? AND source_name = ?', + [self::ROLLBACK_DATASET_ID, 'target'], + )); + $concurrentPdo = $this->doltConcurrentSession($pdo, $connection, $context['branch']); + $reader = new class ($pdo, $concurrentPdo, $connection, $table, self::ROLLBACK_DATASET_ID, $context['branch'], $context['head']) implements DoltEvidenceReader { + public int $reads = 0; + public bool $mutationObserved = false; + public ?string $newHead = null; + + public function __construct( + private readonly PDO $executorPdo, + private readonly PDO $concurrentPdo, + private readonly Connection $connection, + private readonly string $table, + private readonly string $datasetId, + private readonly string $branch, + private readonly string $head, + ) {} + + public function read(): DoltEvidence + { + if (++$this->reads === 1) { + return new DoltEvidence($this->branch, $this->head, []); + } + // The second PDO is a real concurrent session: make a DOLT_COMMIT + // that does not touch the executor's target rows but moves HEAD, + // so the executor must fail closed on dolt_context_changed. + $metadata = $this->concurrentPdo->prepare( + 'UPDATE dataset SET dataset_name = ? WHERE dataset_id = ?', + ); + if ($metadata instanceof PDOStatement) { + $metadata->execute(['Concurrent commit probe', $this->datasetId]); + } + $commit = $this->concurrentPdo->prepare('CALL DOLT_COMMIT(?, ?)'); + if ($commit instanceof PDOStatement) { + $commit->execute(['-Am', 'Concurrent commit detected by executor guard']); + } + $head = $this->concurrentPdo->query("SELECT dolt_hashof('HEAD')"); + $this->newHead = $head instanceof PDOStatement ? (string) $head->fetchColumn() : $this->head; + $statement = $this->executorPdo->query( + 'SELECT ' . $this->connection->profile->quoteIdentifier('target') + . ' FROM ' . $this->connection->profile->quoteIdentifier($this->table) + . ' WHERE ' . $this->connection->profile->quoteIdentifier('__case_ordinal') . ' = 1', + ); + $this->mutationObserved = $statement instanceof PDOStatement + && (float) $statement->fetchColumn() === 1.0; + + return new DoltEvidence($this->branch, $this->newHead ?? $this->head, []); + } + }; + + try { + (new InPlaceTransformationExecutor($connection, $reader))->execute(new InPlaceApplyRequest( + $this->planCase('sequential-conditional-binary-existing-target'), + 'parent', + self::ROLLBACK_DATASET_ID, + str_repeat('a', 64), + (string) $case['actor'], + $context['branch'], + $context['head'], + )); + self::fail('The official post-mutation Dolt context change was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertTrue($case['mutation_started']); + self::assertTrue($reader->mutationObserved); + self::assertFalse($pdo->inTransaction()); + $after = $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + $beforeRepository = $before['repository']; + $afterRepository = $after['repository']; + unset($before['repository'], $after['repository']); + self::assertSame($before, $after, 'The failed apply must roll back data, metadata, and audit state.'); + self::assertIsArray($beforeRepository); + self::assertIsArray($afterRepository); + self::assertSame($beforeRepository['branch'], $afterRepository['branch']); + self::assertNotSame($beforeRepository['head'], $afterRepository['head'], 'The independent concurrent commit must remain in Dolt history.'); + self::assertNotNull($reader->newHead); + self::assertSame($reader->newHead, $afterRepository['head']); + self::assertContains($reader->newHead, $afterRepository['history']); + self::assertSame($case['after_failure']['rows'], $this->numericRows($pdo, $connection, $table, ['target'])); + self::assertSame($case['after_failure']['target_metadata']['variable_label'], $this->scalar( + $pdo, + 'SELECT variable_label FROM variable WHERE dataset_id = ? AND source_name = ?', + [self::ROLLBACK_DATASET_ID, 'target'], + )); + self::assertSame($case['after_failure']['audit_row_count'], $this->scalarCount( + $pdo, + 'SELECT COUNT(*) FROM transformation_apply', + [], + )); + } + + /** + * Opens a second PDO against the same Dolt database so the concurrent-commit + * guard test can issue a real DOLT_COMMIT from an independent session + * instead of faking the post-mutation evidence. The PDO is kept alive by + * the anonymous reader closure that captures it; it is released (and + * therefore disconnected) when the reader goes out of scope after the + * executor throws. + */ + private function doltConcurrentSession(PDO $primaryPdo, Connection $primaryConnection, string $branch): PDO + { + $statement = $primaryPdo->query('SELECT DATABASE()'); + $database = $statement instanceof PDOStatement ? (string) $statement->fetchColumn() : ''; + self::assertNotSame('', $database, 'The primary Dolt session must be connected to a database.'); + $prefix = match ($primaryConnection->profileName) { + 'dolt' => 'OPENSTATSPEC_DOLT', + 'mysql' => 'OPENSTATSPEC_MYSQL', + 'mariadb' => 'OPENSTATSPEC_MARIADB', + default => throw new RuntimeException('Concurrent session only supports MySQL-family profiles.'), + }; + $baseDsn = getenv($prefix . '_DSN'); + if (!is_string($baseDsn) || $baseDsn === '') { + self::markTestSkipped($prefix . '_DSN is not configured.'); + } + $user = getenv($prefix . '_USER'); + $password = getenv($prefix . '_PASSWORD'); + $options = [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]; + $concurrent = new PDO( + $this->dsnForDatabase($baseDsn, $database), + is_string($user) ? $user : null, + is_string($password) ? $password : null, + $options, + ); + $active = $concurrent->query('SELECT active_branch()'); + $activeBranch = $active instanceof PDOStatement ? (string) $active->fetchColumn() : ''; + if ($activeBranch !== $branch) { + $checkout = $concurrent->prepare('CALL DOLT_CHECKOUT(?)'); + if ($checkout instanceof PDOStatement) { + $checkout->execute([$branch]); + $result = $checkout->fetch(PDO::FETCH_ASSOC); + if (is_array($result) && array_key_exists('status', $result) && (int) $result['status'] !== 0) { + throw new RuntimeException('DOLT_CHECKOUT failed: ' . (string) ($result['message'] ?? 'unknown error')); + } + } + } + $confirmed = $concurrent->query('SELECT active_branch()'); + $confirmedBranch = $confirmed instanceof PDOStatement ? (string) $confirmed->fetchColumn() : ''; + if ($confirmedBranch !== $branch) { + throw new RuntimeException(sprintf('DOLT_CHECKOUT selected %s instead of %s.', $confirmedBranch, $branch)); + } + return $concurrent; + } + + /** @param array $case */ + private function assertOfficialDoltEmptyActor(array $case): void + { + $pdo = $this->mysqlFamily('dolt'); + $connection = new Connection($pdo); + self::assertSame('dolt', $connection->profileName); + $before = ['tables' => $this->tables($pdo), 'repository' => $this->doltRepositoryEvidence($pdo, $connection)]; + + try { + new InPlaceApplyRequest( + $this->planCase('binding-variable-missing-existing-target'), + 'parent', + '11111111-1111-4111-8111-111111111111', + str_repeat('a', 64), + (string) $case['actor'], + 'feature/recode', + 'provisioning-commit', + ); + self::fail('The official empty actor was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertFalse($case['mutation_started']); + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, [ + 'tables' => $this->tables($pdo), + 'repository' => $this->doltRepositoryEvidence($pdo, $connection), + ]); + } + + /** @param array $case */ + private function assertOfficialSqliteCreate(array $case): void + { + $pdo = $this->sqlite(); + $connection = new Connection($pdo); + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + 'data_atomic_create', + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + $result = (new InPlaceTransformationExecutor($connection))->execute( + $this->request($case, self::CREATE_DATASET_ID), + ); + $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + + self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); + self::assertSame(self::CREATE_DATASET_ID, $result->datasetId()); + self::assertSame(1, $result->operationCount()); + self::assertNotSame('', $result->auditOperationId()); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($before['case_ordinals'], $after['case_ordinals']); + self::assertSame($before['case_count'], $after['case_count']); + self::assertSame(3, $after['variable_count']); + self::assertSame( + $case['after']['rows'], + $this->numericRows($pdo, $connection, 'data_atomic_create', ['source_a', 'source_b', 'target']), + ); + self::assertSame( + [ + 'source_ordinal' => 3, + 'storage_kind' => 'numeric', + 'declared_string_width' => null, + 'variable_label' => null, + 'print_format_family' => null, + 'print_format_width' => null, + 'print_format_decimals' => null, + 'write_format_family' => null, + 'write_format_width' => null, + 'write_format_decimals' => null, + 'measurement_level' => null, + 'variable_role' => null, + 'display_width' => null, + 'display_alignment' => null, + ], + $this->targetMetadata($pdo, self::CREATE_DATASET_ID, 'target'), + ); + self::assertSame([], $this->valueLabels($pdo, self::CREATE_DATASET_ID, 'target')); + self::assertSame([], $this->missingRules($pdo, self::CREATE_DATASET_ID, 'target')); + $this->assertAudit($pdo, $case['expected_audit']); + } + + /** @param array $case */ + private function assertOfficialSqliteInequality(array $case): void + { + $pdo = $this->sqlite(); + $connection = new Connection($pdo); + $this->installNumericFixture( + $pdo, + $connection, + self::INEQUALITY_DATASET_ID, + 'data_inequality', + ['source', 'target_lt', 'target_le', 'target_gt', 'target_ge'], + [[1, 0.0, 0.0, 0.0, 0.0, 0.0], [2, 1.0, 0.0, 0.0, 0.0, 0.0], [3, 2.0, 0.0, 0.0, 0.0, 0.0]], + ); + $before = $this->snapshot($pdo, $connection, self::INEQUALITY_DATASET_ID, 'data_inequality'); + $result = (new InPlaceTransformationExecutor($connection))->execute( + $this->request($case, self::INEQUALITY_DATASET_ID), + ); + $after = $this->snapshot($pdo, $connection, self::INEQUALITY_DATASET_ID, 'data_inequality'); + + self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); + self::assertSame($case['after_rows'], $this->numericRows( + $pdo, + $connection, + 'data_inequality', + ['source', 'target_lt', 'target_le', 'target_gt', 'target_ge'], + )); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($before['variables'], $after['variables']); + self::assertSame($before['case_ordinals'], $after['case_ordinals']); + self::assertSame($before['case_count'], $after['case_count']); + $this->assertAudit($pdo, $case['expected_audit']); + } + + private function sqlite(): PDO + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available.'); + } + return new PDO('sqlite::memory:', options: [ + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_STRINGIFY_FETCHES => false, + ]); + } + + private function postgresql(): PDO + { + $dsn = getenv('OPENSTATSPEC_PG_DSN'); + if (!is_string($dsn) || $dsn === '') { + self::markTestSkipped('OPENSTATSPEC_PG_DSN is not configured.'); + } + if (!in_array('pgsql', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO PostgreSQL is not available.'); + } + $user = getenv('OPENSTATSPEC_PG_USER'); + $password = getenv('OPENSTATSPEC_PG_PASSWORD'); + return new PDO( + $dsn, + is_string($user) ? $user : null, + is_string($password) ? $password : null, + [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false], + ); + } + + private function mysqlFamily(string $profile): PDO + { + $prefix = match ($profile) { + 'mysql' => 'OPENSTATSPEC_MYSQL', + 'mariadb' => 'OPENSTATSPEC_MARIADB', + 'dolt' => 'OPENSTATSPEC_DOLT', + default => throw new RuntimeException('Unsupported MySQL-family test profile.'), + }; + $dsn = getenv($prefix . '_DSN'); + if (!is_string($dsn) || $dsn === '') { + self::markTestSkipped($prefix . '_DSN is not configured.'); + } + if (!in_array('mysql', PDO::getAvailableDrivers(), true)) { + throw new RuntimeException('Configured MySQL-family service requires the PDO MySQL driver.'); + } + $user = getenv($prefix . '_USER'); + $password = getenv($prefix . '_PASSWORD'); + $options = [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]; + + if ($profile !== 'dolt') { + return new PDO( + $dsn, + is_string($user) ? $user : null, + is_string($password) ? $password : null, + $options, + ); + } + + $adminUser = getenv($prefix . '_ADMIN_USER'); + $adminPassword = getenv($prefix . '_ADMIN_PASSWORD'); + if (!is_string($adminUser) || $adminUser === '' || !is_string($adminPassword)) { + throw new RuntimeException('Configured Dolt service requires explicit admin credentials for isolation.'); + } + $database = sprintf('openstatspec_t9_%d_%s', getmypid(), bin2hex(random_bytes(6))); + $admin = new PDO($dsn, $adminUser, $adminPassword, $options); + $admin->exec('CREATE DATABASE ' . $this->quoteDoltDatabase($database)); + $this->doltTestDatabases[] = ['admin' => $admin, 'database' => $database]; + if (is_string($user) && $user !== '') { + $quotedUser = $admin->quote($user); + if (!is_string($quotedUser)) { + throw new RuntimeException('Unable to quote the Dolt test user.'); + } + $admin->exec( + 'GRANT ALL PRIVILEGES ON ' . $this->quoteDoltDatabase($database) . ".* TO {$quotedUser}@'%'", + ); + } + + return new PDO( + $this->dsnForDatabase($dsn, $database), + is_string($user) ? $user : null, + is_string($password) ? $password : null, + $options, + ); + } + + /** + * @param list $variables + * @param list> $rows + * @param array $variableIds + */ + private function installNumericFixture( + PDO $pdo, + Connection $connection, + string $datasetId, + string $table, + array $variables, + array $rows, + array $variableIds = [], + ): void { + if ($connection->profileName === 'sqlite') { + $pdo->exec('PRAGMA foreign_keys = ON'); + } + $this->prepareCatalog($pdo); + $quotedColumns = array_map($connection->profile->quoteIdentifier(...), $variables); + $pdo->exec( + 'CREATE TABLE ' . $connection->profile->quoteIdentifier($table) . ' (' + . $connection->profile->quoteIdentifier('__case_ordinal') . ' BIGINT NOT NULL PRIMARY KEY, ' + . implode(' ' . $connection->profile->numericType() . ' NULL, ', $quotedColumns) + . ' ' . $connection->profile->numericType() . ' NULL)', + ); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([$datasetId, '1.0', 'fixture', null, $table, 'Task 8 fixture', count($rows), '2026-08-17 00:00:00']); + $insertVariable = $pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) VALUES (?, ?, ?, ?, ?, ?)', + ); + foreach ($variables as $index => $variable) { + $insertVariable->execute([ + $variableIds[$variable] ?? $this->fixtureUuid($datasetId . ':' . $variable), + $datasetId, + $index + 1, + $variable, + $variable, + 'numeric', + ]); + } + $columns = [$connection->profile->quoteIdentifier('__case_ordinal'), ...$quotedColumns]; + $insertCase = $pdo->prepare( + 'INSERT INTO ' . $connection->profile->quoteIdentifier($table) + . ' (' . implode(', ', $columns) . ') VALUES (' . implode(', ', array_fill(0, count($columns), '?')) . ')', + ); + foreach ($rows as $row) { + $insertCase->execute($row); + } + } + + private function prepareCatalog(PDO $pdo): void + { + (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); + CatalogOwnership::markCurrentVersion($pdo); + } + + /** @param array $case */ + private function request( + array $case, + string $datasetId, + ?string $expectedBranch = null, + ?string $expectedHead = null, + ): InPlaceApplyRequest { + return new InPlaceApplyRequest( + plan: $this->planCase((string) $case['applied_plan_case']), + inputAlias: 'parent', + datasetId: $datasetId, + sourceHash: (string) $case['expected_audit']['source_hash'], + actor: (string) $case['actor'], + expectedBranch: $expectedBranch, + expectedHead: $expectedHead, + ); + } + + /** @return array */ + private function bindingCase(string $id): array + { + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.2.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id) { + return $case; + } + } + throw new \RuntimeException('Missing in-place fixture: ' . $id); + } + + private function planCase(string $id): \OpenStatSpec\Transformation\Plan\TransformationPlan + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.2.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id && is_array($case['plan'] ?? null)) { + return (new PlanCodec())->fromArray($case['plan']); + } + } + throw new \RuntimeException('Missing plan fixture: ' . $id); + } + + /** @return array */ + private function snapshot(PDO $pdo, Connection $connection, string $datasetId, string $table): array + { + return [ + 'dataset_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM dataset', []), + 'persistent_data_table_count' => $this->scalarCount($pdo, 'SELECT COUNT(DISTINCT physical_table_name) FROM dataset', []), + 'dataset_identity' => $this->rows($pdo, 'SELECT dataset_id, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', [$datasetId]), + 'tables' => $this->tables($pdo), + 'columns' => $this->columns($pdo, $connection, $table), + 'variables' => $this->rows($pdo, 'SELECT * FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', [$datasetId]), + 'variable_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM variable WHERE dataset_id = ?', [$datasetId]), + 'case_ordinals' => array_map('intval', $this->column($pdo, 'SELECT __case_ordinal FROM ' . $connection->profile->quoteIdentifier($table) . ' ORDER BY __case_ordinal')), + 'case_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM ' . $connection->profile->quoteIdentifier($table), []), + 'rows' => $this->rows($pdo, 'SELECT * FROM ' . $connection->profile->quoteIdentifier($table) . ' ORDER BY __case_ordinal', []), + 'audit_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM transformation_apply', []), + 'repository' => $this->doltRepositoryEvidence($pdo, $connection), + ]; + } + + /** + * @param list $variables + * @return list> + */ + private function numericRows(PDO $pdo, Connection $connection, string $table, array $variables): array + { + $columns = ['__case_ordinal', ...$variables]; + $rows = $this->rows( + $pdo, + 'SELECT ' . implode(', ', array_map($connection->profile->quoteIdentifier(...), $columns)) + . ' FROM ' . $connection->profile->quoteIdentifier($table) + . ' ORDER BY ' . $connection->profile->quoteIdentifier('__case_ordinal'), + [], + ); + return array_map(static function (array $row) use ($columns): array { + $normalized = []; + foreach ($columns as $column) { + $value = $row[$column]; + if ($column === '__case_ordinal') { + $normalized[$column] = (int) $value; + continue; + } + if ($value === null) { + $normalized[$column] = null; + continue; + } + $number = (float) $value; + $normalized[$column] = floor($number) === $number ? (int) $number : $number; + } + return $normalized; + }, $rows); + } + + /** @return array */ + private function targetMetadata(PDO $pdo, string $datasetId, string $target): array + { + $rows = $this->rows( + $pdo, + 'SELECT source_ordinal, storage_kind, declared_string_width, variable_label, print_format_family, print_format_width, ' + . 'print_format_decimals, write_format_family, write_format_width, write_format_decimals, measurement_level, ' + . 'variable_role, display_width, display_alignment FROM variable WHERE dataset_id = ? AND source_name = ?', + [$datasetId, $target], + ); + self::assertCount(1, $rows); + foreach (['source_ordinal', 'declared_string_width', 'print_format_width', 'print_format_decimals', 'write_format_width', 'write_format_decimals', 'display_width'] as $integer) { + if ($rows[0][$integer] !== null) { + $rows[0][$integer] = (int) $rows[0][$integer]; + } + } + return $rows[0]; + } + + /** @return list */ + private function valueLabels(PDO $pdo, string $datasetId, string $target): array + { + $rows = $this->rows( + $pdo, + 'SELECT label.code_kind, label.numeric_code, label.string_code, label.label FROM variable ' + . 'JOIN variable_value_label_set link ON link.variable_id = variable.variable_id ' + . 'JOIN value_label label ON label.value_label_set_id = link.value_label_set_id ' + . 'WHERE variable.dataset_id = ? AND variable.source_name = ? ORDER BY label.ordinal', + [$datasetId, $target], + ); + return array_map(static fn(array $row): array => [ + $row['code_kind'] === 'numeric' ? (float) $row['numeric_code'] : (string) $row['string_code'], + (string) $row['label'], + ], $rows); + } + + /** @return list */ + private function missingRules(PDO $pdo, string $datasetId, string $target): array + { + return array_map(static fn(array $row): array => [ + 'ordinal' => (int) $row['ordinal'], + 'rule_kind' => (string) $row['rule_kind'], + 'code_kind' => (string) $row['code_kind'], + 'numeric_value' => $row['numeric_value'] === null ? null : (float) $row['numeric_value'], + ], $this->rows( + $pdo, + 'SELECT missing_rule.ordinal, missing_rule.rule_kind, missing_rule.code_kind, missing_rule.numeric_value ' + . 'FROM variable JOIN missing_rule ON missing_rule.variable_id = variable.variable_id ' + . 'WHERE variable.dataset_id = ? AND variable.source_name = ? ORDER BY missing_rule.ordinal', + [$datasetId, $target], + )); + } + + /** + * @param array $expected + * @return array + */ + private function assertAudit(PDO $pdo, array $expected): array + { + $rows = $this->rows($pdo, 'SELECT * FROM transformation_apply ORDER BY started_at, apply_id', []); + self::assertCount(1, $rows); + foreach ($expected as $field => $value) { + self::assertArrayHasKey($field, $rows[0]); + self::assertSame($value, $field === 'operation_count' ? (int) $rows[0][$field] : $rows[0][$field], $field); + } + self::assertMatchesRegularExpression('/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/', (string) $rows[0]['apply_id']); + self::assertNotSame('', (string) $rows[0]['started_at']); + self::assertNotSame('', (string) $rows[0]['completed_at']); + + return $rows[0]; + } + + private function assertNamespaceClean(PDO $pdo, string $datasetId, string $table): void + { + self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM dataset WHERE dataset_id = ? OR physical_table_name = ?', [$datasetId, $table])); + self::assertNotContains($table, $this->tables($pdo)); + } + + private function purgeFixture(PDO $pdo, Connection $connection, string $datasetId, string $table): void + { + if ($pdo->inTransaction()) { + $pdo->rollBack(); + } + $pdo->prepare('DELETE FROM transformation_apply WHERE dataset_id = ?')->execute([$datasetId]); + $pdo->prepare('DELETE FROM variable_value_label_set WHERE variable_id IN (SELECT variable_id FROM variable WHERE dataset_id = ?)')->execute([$datasetId]); + $pdo->prepare('DELETE FROM value_label WHERE value_label_set_id IN (SELECT value_label_set_id FROM value_label_set WHERE dataset_id = ?)')->execute([$datasetId]); + $pdo->prepare('DELETE FROM value_label_set WHERE dataset_id = ?')->execute([$datasetId]); + $pdo->prepare('DELETE FROM missing_rule WHERE variable_id IN (SELECT variable_id FROM variable WHERE dataset_id = ?)')->execute([$datasetId]); + $pdo->prepare('DELETE FROM variable WHERE dataset_id = ?')->execute([$datasetId]); + $pdo->prepare('DELETE FROM dataset WHERE dataset_id = ?')->execute([$datasetId]); + $pdo->exec('DROP TABLE IF EXISTS ' . $connection->profile->quoteIdentifier($table)); + } + + /** @return array{branch: string, head: string}|null */ + private function commitAndReadDoltFixtureContext(PDO $pdo, Connection $connection): ?array + { + if ($connection->profileName !== 'dolt') { + return null; + } + $statement = $pdo->prepare('CALL DOLT_COMMIT(?, ?)'); + self::assertInstanceOf(PDOStatement::class, $statement); + $statement->execute(['-Am', 'Task 9 official create-target fixture']); + $identity = $pdo->query("SELECT active_branch() AS branch_name, dolt_hashof('HEAD') AS head_hash"); + self::assertInstanceOf(PDOStatement::class, $identity); + $row = $identity->fetch(PDO::FETCH_ASSOC); + self::assertIsArray($row); + $status = $pdo->query('SELECT table_name FROM dolt_status'); + self::assertInstanceOf(PDOStatement::class, $status); + self::assertSame([], $status->fetchAll(PDO::FETCH_COLUMN)); + + return ['branch' => (string) $row['branch_name'], 'head' => (string) $row['head_hash']]; + } + + /** @param array $case */ + private function checkoutDoltFixtureBranch(PDO $pdo, Connection $connection, array $case): void + { + if ($connection->profileName !== 'dolt') { + return; + } + $expectedContext = is_array($case['expected_context'] ?? null) ? $case['expected_context'] : []; + $branch = $case['expected_branch'] ?? $expectedContext['branch'] ?? null; + if (!is_string($branch) || $branch === '') { + throw new RuntimeException('Official Dolt case is missing its expected branch.'); + } + $active = (string) $this->scalar($pdo, 'SELECT active_branch()', []); + if ($active === $branch) { + return; + } + $statement = $pdo->prepare('CALL DOLT_CHECKOUT(?, ?)'); + self::assertInstanceOf(PDOStatement::class, $statement); + $statement->execute(['-b', $branch]); + self::assertSame($branch, $this->scalar($pdo, 'SELECT active_branch()', [])); + } + + /** + * @param array $rows + * @return array{list, list>} + */ + private function numericFixtureRows(array $rows): array + { + $first = $rows[0] ?? null; + if (!is_array($first) || !is_int($first['__case_ordinal'] ?? null)) { + throw new RuntimeException('Official numeric fixture rows are malformed.'); + } + $variables = []; + foreach (array_keys($first) as $column) { + if (is_string($column) && $column !== '__case_ordinal') { + $variables[] = $column; + } + } + if ($variables === []) { + throw new RuntimeException('Official numeric fixture has no variables.'); + } + $fixtureRows = []; + foreach ($rows as $row) { + if (!is_array($row) || !is_int($row['__case_ordinal'] ?? null)) { + throw new RuntimeException('Official numeric fixture row is malformed.'); + } + $fixtureRow = [$row['__case_ordinal']]; + foreach ($variables as $variable) { + if (!array_key_exists($variable, $row)) { + throw new RuntimeException('Official numeric fixture row is missing a variable.'); + } + $value = $row[$variable]; + if ($value !== null && !is_int($value) && !is_float($value)) { + throw new RuntimeException('Official numeric fixture value is not numeric or null.'); + } + $fixtureRow[] = $value; + } + $fixtureRows[] = $fixtureRow; + } + + return [$variables, $fixtureRows]; + } + + /** @param array $beforeContract */ + private function installOfficialMissingRules(PDO $pdo, string $datasetId, array $beforeContract): void + { + $targetMetadata = is_array($beforeContract['target_metadata'] ?? null) + ? $beforeContract['target_metadata'] + : []; + $missingValues = $targetMetadata['missing_values'] ?? []; + if (!is_array($missingValues) || $missingValues === []) { + return; + } + $targetId = $this->scalar( + $pdo, + 'SELECT variable_id FROM variable WHERE dataset_id = ? AND source_name = ?', + [$datasetId, 'target'], + ); + if (!is_string($targetId) || $targetId === '') { + throw new RuntimeException('Official target variable identity is unavailable.'); + } + $insert = $pdo->prepare( + 'INSERT INTO missing_rule (missing_rule_id, variable_id, ordinal, rule_kind, code_kind, numeric_value) ' + . 'VALUES (?, ?, ?, ?, ?, ?)', + ); + self::assertInstanceOf(PDOStatement::class, $insert); + foreach ($missingValues as $missing) { + if (!is_array($missing) || !is_int($missing['ordinal'] ?? null)) { + throw new RuntimeException('Official missing-value fixture is malformed.'); + } + $insert->execute([ + $this->fixtureUuid($datasetId . ':missing:' . $missing['ordinal']), + $targetId, + $missing['ordinal'], + $missing['rule_kind'] ?? null, + $missing['code_kind'] ?? null, + $missing['numeric_value'] ?? null, + ]); + } + } + + /** @return array{variable_id: string, source_ordinal: int, source_name: string, physical_name: string} */ + private function targetIdentity(PDO $pdo, string $datasetId, string $target): array + { + $rows = $this->rows( + $pdo, + 'SELECT variable_id, source_ordinal, source_name, physical_name FROM variable ' + . 'WHERE dataset_id = ? AND source_name = ?', + [$datasetId, $target], + ); + self::assertCount(1, $rows); + + return [ + 'variable_id' => (string) $rows[0]['variable_id'], + 'source_ordinal' => (int) $rows[0]['source_ordinal'], + 'source_name' => (string) $rows[0]['source_name'], + 'physical_name' => (string) $rows[0]['physical_name'], + ]; + } + + /** @return array{branch: string, head: string, history: list, status: list}|null */ + private function doltRepositoryEvidence(PDO $pdo, Connection $connection): ?array + { + if ($connection->profileName !== 'dolt') { + return null; + } + $identity = $this->rows( + $pdo, + "SELECT active_branch() AS branch_name, dolt_hashof('HEAD') AS head_hash", + [], + )[0] ?? []; + $status = array_map(static function (array $row): array { + $row = array_change_key_case($row, CASE_LOWER); + + return [ + 'table_name' => (string) ($row['table_name'] ?? ''), + 'status' => (string) ($row['status'] ?? ''), + 'staged' => (bool) ($row['staged'] ?? false), + ]; + }, $this->rows($pdo, 'SELECT table_name, status, staged FROM dolt_status ORDER BY table_name', [])); + + return [ + 'branch' => (string) ($identity['branch_name'] ?? ''), + 'head' => (string) ($identity['head_hash'] ?? ''), + 'history' => array_map('strval', $this->column( + $pdo, + 'SELECT commit_hash FROM dolt_log ORDER BY commit_order DESC, commit_hash', + )), + 'status' => $status, + ]; + } + + /** + * @param array $before + * @param array $after + */ + private function assertDoltRepositoryApplyEvidence(array $before, array $after, string $table, string $caseId): void + { + $beforeRepository = $before['repository'] ?? null; + $afterRepository = $after['repository'] ?? null; + self::assertIsArray($beforeRepository); + self::assertIsArray($afterRepository); + self::assertSame($beforeRepository['branch'], $afterRepository['branch']); + self::assertSame($beforeRepository['head'], $afterRepository['head']); + self::assertSame($beforeRepository['history'], $afterRepository['history']); + $changedTables = [$table, 'transformation_apply']; + if ($caseId === 'dolt-preprovisioned-target-sequential-null-semantics') { + $changedTables = [ + $table, + 'transformation_apply', + 'value_label', + 'value_label_set', + 'variable', + 'variable_value_label_set', + ]; + } + sort($changedTables); + self::assertSame(array_map(static fn(string $changed): array => [ + 'table_name' => $changed, + 'status' => 'modified', + 'staged' => false, + ], $changedTables), $afterRepository['status']); + } + + /** @param list $tables */ + private function assertNoForbiddenArtifactTables(array $tables): void + { + self::assertSame([], array_values(array_filter( + $tables, + static fn(string $table): bool => preg_match('/derived|output|staging|snapshot|rollback|recovery/i', $table) === 1, + ))); + } + + private function quoteDoltDatabase(string $database): string + { + if (strlen($database) > 64 || preg_match('/\A[a-z][a-z0-9_]*\z/D', $database) !== 1) { + throw new RuntimeException('Unsafe Dolt test database name.'); + } + + return '`' . $database . '`'; + } + + private function dsnForDatabase(string $dsn, string $database): string + { + $this->quoteDoltDatabase($database); + if (!str_starts_with(strtolower($dsn), 'mysql:')) { + throw new RuntimeException('Dolt tests require a MySQL PDO DSN.'); + } + $parts = explode(';', substr($dsn, strlen('mysql:'))); + $found = null; + foreach ($parts as $index => $part) { + if (strtolower(trim((string) explode('=', $part, 2)[0])) !== 'dbname') { + continue; + } + if ($found !== null) { + throw new RuntimeException('Dolt DSN contains duplicate dbname settings.'); + } + $found = $index; + } + if ($found === null) { + if (end($parts) === '') { + array_pop($parts); + } + $parts[] = 'dbname=' . $database; + } else { + $parts[$found] = 'dbname=' . $database; + } + + return 'mysql:' . implode(';', $parts); + } + + /** @return list */ + private function tables(PDO $pdo): array + { + $sql = match ((string) $pdo->getAttribute(PDO::ATTR_DRIVER_NAME)) { + 'sqlite' => "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + 'pgsql' => "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = current_schema() ORDER BY tablename", + 'mysql' => 'SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() ORDER BY table_name', + default => throw new RuntimeException('Unsupported integration driver.'), + }; + return array_map('strval', $this->column($pdo, $sql)); + } + + /** @return list */ + private function columns(PDO $pdo, Connection $connection, string $table): array + { + if ($connection->profileName === 'sqlite') { + return array_map( + static fn(array $row): string => (string) $row['name'], + $this->rows($pdo, 'PRAGMA table_info(' . $connection->profile->quoteIdentifier($table) . ')', []), + ); + } + $schema = $connection->profileName === 'postgresql' ? 'current_schema()' : 'DATABASE()'; + return array_map('strval', $this->column( + $pdo, + 'SELECT column_name FROM information_schema.columns WHERE table_schema = ' . $schema . ' AND table_name = ? ORDER BY ordinal_position', + [$table], + )); + } + + private function fixtureUuid(string $scope): string + { + $hash = hash('sha256', $scope); + return substr($hash, 0, 8) . '-' . substr($hash, 8, 4) . '-4' . substr($hash, 12, 3) . '-8' . substr($hash, 15, 3) . '-' . substr($hash, 18, 12); + } + + /** @param list $parameters */ + private function scalarCount(PDO $pdo, string $sql, array $parameters): int + { + return (int) $this->scalar($pdo, $sql, $parameters); + } +} diff --git a/tests/Integration/PostgreSqlSpssRoundTripTest.php b/tests/Integration/PostgreSqlSpssRoundTripTest.php index b6beec5..3902a01 100644 --- a/tests/Integration/PostgreSqlSpssRoundTripTest.php +++ b/tests/Integration/PostgreSqlSpssRoundTripTest.php @@ -10,10 +10,15 @@ use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Spss\PhpSpssEngine; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; -use OpenStatSpec\Transformation\Model\CreateVariableOperation; -use OpenStatSpec\Transformation\Model\DeleteVariableOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; +use OpenStatSpec\Transformation\Plan\Expression\LiteralOperand; +use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\TargetMode; +use OpenStatSpec\Transformation\Plan\TransformationPlan; +use OpenStatSpec\Transformation\Plan\Value\Binary64Value; use OpenStatSpec\Spss\SpssAdapter; use PDO; use PDOException; @@ -134,6 +139,7 @@ public function testDeleteThenCreateAtThePostgreSqlPhysicalColumnLimitFailsBefor try { (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); $columns = ['__case_ordinal BIGINT NOT NULL PRIMARY KEY']; for ($ordinal = 1; $ordinal <= 1599; ++$ordinal) { @@ -156,17 +162,27 @@ public function testDeleteThenCreateAtThePostgreSqlPhysicalColumnLimitFailsBefor } $pdo->commit(); - $plan = new TransformationPlan($datasetId, [ - new DeleteVariableOperation('V1599'), - new CreateVariableOperation('Replacement', 'string', 1), + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation( + 'Replacement', + TargetMode::Create, + new LiteralOperand(Binary64Value::fromBits('0000000000000000')), + ), ]); + $request = new InPlaceApplyRequest( + $plan, + 'parent', + $datasetId, + hash('sha256', 'PostgreSQL physical-column slot fixture'), + 'integration-test', + ); try { - (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($plan); + (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($request); self::fail('PostgreSQL accepted a replacement after the physical column limit was reached.'); } catch (UnsupportedOperation $exception) { self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); - self::assertStringContainsString('physical PostgreSQL column slots', $exception->getMessage()); + self::assertStringContainsString('cannot add another source variable to this wide table', $exception->getMessage()); } self::assertSame(1599, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM variable WHERE dataset_id = ?', [$datasetId])); diff --git a/tests/Integration/ServerCatalogMigrationTest.php b/tests/Integration/ServerCatalogMigrationTest.php index 11834d5..df2f47d 100644 --- a/tests/Integration/ServerCatalogMigrationTest.php +++ b/tests/Integration/ServerCatalogMigrationTest.php @@ -32,10 +32,21 @@ public function testLegacyV3ConstraintsAndSourceOrderMigrateOnConfiguredServers( $pdo->prepare('INSERT INTO multiple_response_sets (dataset_name, set_ordinal, name, set_type, category_labels, label_source) VALUES (?, ?, ?, ?, ?, ?)')->execute([$datasetName, 1, '$Legacy MR', 'dichotomy', 'counted_values', 'set_label']); $pdo->prepare('INSERT INTO variable_set (variable_set_id, dataset_id, source_ordinal, set_name) VALUES (?, ?, NULL, ?)')->execute(['vs-' . $token, $datasetId, 'Legacy variable set']); $pdo->prepare('INSERT INTO multiple_response_set (multiple_response_set_id, dataset_id, source_ordinal, set_name, set_kind, category_label_behavior) VALUES (?, ?, NULL, ?, ?, ?)')->execute(['mr-' . $token, $datasetId, '$Legacy MR', 'MD', 'counted_values']); + $legacyApplyId = $this->makeVersion01Audit($pdo, $name, $datasetId, 'unused_' . $token); $adapter->migrateCatalog(); self::assertSame(2, (int) $this->scalar($pdo, 'SELECT source_ordinal FROM variable_set WHERE dataset_id = ?', [$datasetId]), $name . ': variable-set source order'); self::assertSame(1, (int) $this->scalar($pdo, 'SELECT source_ordinal FROM multiple_response_set WHERE dataset_id = ?', [$datasetId]), $name . ': MR-set source order'); + self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply WHERE apply_id = ?', [$legacyApplyId]), $name . ': existing 0.1 audit row'); + $this->assertAuditContracts($pdo, $name, $datasetId, 'unused_' . $token); + $this->assertUppercaseHashesRejected($pdo, $name, $datasetId, 'unused_' . $token); + if ($name !== 'postgresql') { + $this->dropAuditContractChecks($pdo, $name); + $pdo->exec('DELETE FROM openstatspec_schema_migration WHERE version = 4'); + + $adapter->migrateCatalog(); + $this->assertUnknownAuditContractRejected($pdo, $name, $datasetId, 'unused_' . $token); + } $this->assertNotNullAndUnique($pdo, $name); foreach ([ ['INSERT INTO variable_set (variable_set_id, dataset_id, source_ordinal, set_name) VALUES (?, ?, ?, ?)', ['vs-duplicate-' . $token, $datasetId, 2, 'Duplicate variable set']], @@ -50,7 +61,9 @@ public function testLegacyV3ConstraintsAndSourceOrderMigrateOnConfiguredServers( } $adapter->migrateCatalog(); self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM openstatspec_schema_migration WHERE version = 3', []), $name . ': migration marker must be idempotent'); + self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM openstatspec_schema_migration WHERE version = 4', []), $name . ': audit migration marker must be idempotent'); } finally { + $pdo->prepare('DELETE FROM transformation_apply WHERE dataset_id = ?')->execute([$datasetId]); $pdo->prepare('DELETE FROM multiple_response_set WHERE dataset_id = ?')->execute([$datasetId]); $pdo->prepare('DELETE FROM variable_set WHERE dataset_id = ?')->execute([$datasetId]); $pdo->prepare('DELETE FROM multiple_response_sets WHERE dataset_name = ?')->execute([$datasetName]); @@ -60,6 +73,163 @@ public function testLegacyV3ConstraintsAndSourceOrderMigrateOnConfiguredServers( } } + private function makeVersion01Audit(PDO $pdo, string $profile, string $datasetId, string $table): string + { + $driver = (string) $pdo->getAttribute(PDO::ATTR_DRIVER_NAME); + if ($driver === 'pgsql') { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT chk_transformation_apply_contract'); + } else { + $this->dropAuditContractChecks($pdo, $profile); + } + $pdo->exec("ALTER TABLE transformation_apply ADD CONSTRAINT chk_transformation_apply_contract CHECK (contract_id = 'openstatspec-in-place-transformation-v0.1')"); + $pdo->exec('DELETE FROM openstatspec_schema_migration WHERE version = 4'); + + $applyId = $this->uuidV4(); + $dolt = $profile === 'dolt'; + $pdo->prepare('INSERT INTO transformation_apply VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')->execute([ + $applyId, + 'openstatspec-in-place-transformation-v0.1', + $profile, + $datasetId, + null, + $table, + str_repeat('c', 64), + str_repeat('d', 64), + '{}', + 'legacy-migration-test', + 'succeeded', + $dolt ? 'migration-test' : null, + $dolt ? 'migration-head' : null, + $dolt ? 'migration-head' : null, + 1, + '2026-08-16 00:00:00', + '2026-08-16 00:00:01', + ]); + return $applyId; + } + + private function assertAuditContracts(PDO $pdo, string $profile, string $datasetId, string $table): void + { + $insert = $pdo->prepare(<<<'SQL' +INSERT INTO transformation_apply ( + apply_id, contract_id, database_profile, dataset_id, physical_table_schema, + physical_table_name, source_hash, plan_hash, canonical_plan_json, actor, + status, dolt_branch, dolt_head_before, dolt_head_after, operation_count, + started_at, completed_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +SQL); + foreach (['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'] as $contract) { + $dolt = $profile === 'dolt'; + $insert->execute([ + $this->uuidV4(), + $contract, + $profile, + $datasetId, + null, + $table, + str_repeat('a', 64), + str_repeat('b', 64), + '{}', + 'migration-test', + 'succeeded', + $dolt ? 'migration-test' : null, + $dolt ? 'migration-head' : null, + $dolt ? 'migration-head' : null, + 1, + '2026-08-17 00:00:00', + '2026-08-17 00:00:01', + ]); + } + self::assertSame( + 2, + (int) $this->scalar($pdo, "SELECT COUNT(*) FROM transformation_apply WHERE dataset_id = ? AND actor = 'migration-test'", [$datasetId]), + $profile . ': both official audit contracts', + ); + } + + private function assertUppercaseHashesRejected(PDO $pdo, string $profile, string $datasetId, string $table): void + { + foreach (['source' => [str_repeat('A', 64), str_repeat('b', 64)], 'plan' => [str_repeat('a', 64), str_repeat('B', 64)]] as $hash => [$sourceHash, $planHash]) { + try { + $this->insertAudit($pdo, $profile, $datasetId, $table, 'openstatspec-in-place-transformation-v0.2', $sourceHash, $planHash); + self::fail($profile . ': uppercase ' . $hash . ' hash was accepted'); + } catch (PDOException) { + // The exact lowercase-hex database check is the assertion. + } + } + } + + private function assertUnknownAuditContractRejected(PDO $pdo, string $profile, string $datasetId, string $table): void + { + try { + $this->insertAudit($pdo, $profile, $datasetId, $table, 'foreign-contract', str_repeat('a', 64), str_repeat('b', 64)); + self::fail($profile . ': missing contract check was not recovered'); + } catch (PDOException) { + // A retry after the interrupted upgrade restored the current check. + } + } + + private function insertAudit( + PDO $pdo, + string $profile, + string $datasetId, + string $table, + string $contract, + string $sourceHash, + string $planHash, + ): void { + $dolt = $profile === 'dolt'; + $pdo->prepare('INSERT INTO transformation_apply VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')->execute([ + $this->uuidV4(), + $contract, + $profile, + $datasetId, + null, + $table, + $sourceHash, + $planHash, + '{}', + 'migration-rejection-test', + 'succeeded', + $dolt ? 'migration-test' : null, + $dolt ? 'migration-head' : null, + $dolt ? 'migration-head' : null, + 1, + '2026-08-17 00:00:00', + '2026-08-17 00:00:01', + ]); + } + + private function dropAuditContractChecks(PDO $pdo, string $profile): void + { + $statement = $pdo->query(<<<'SQL' +SELECT table_constraint.constraint_name +FROM information_schema.table_constraints table_constraint +JOIN information_schema.check_constraints check_constraint + ON check_constraint.constraint_schema = table_constraint.constraint_schema + AND check_constraint.constraint_name = table_constraint.constraint_name +WHERE table_constraint.constraint_schema = DATABASE() + AND table_constraint.table_name = 'transformation_apply' + AND table_constraint.constraint_type = 'CHECK' + AND check_constraint.check_clause LIKE '%contract_id%' +SQL); + $constraints = $statement === false ? [] : $statement->fetchAll(PDO::FETCH_COLUMN); + $drop = $profile === 'mariadb' ? 'DROP CONSTRAINT' : 'DROP CHECK'; + foreach ($constraints as $constraint) { + self::assertIsString($constraint); + $pdo->exec('ALTER TABLE transformation_apply ' . $drop . ' `' . str_replace('`', '``', $constraint) . '`'); + } + } + + private function uuidV4(): string + { + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80); + $hex = bin2hex($bytes); + return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-' . substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-' . substr($hex, 20); + } + private function makeV3Legacy(PDO $pdo): void { $driver = (string) $pdo->getAttribute(PDO::ATTR_DRIVER_NAME); diff --git a/tests/Release/CiTransformationGateTest.php b/tests/Release/CiTransformationGateTest.php new file mode 100644 index 0000000..256ac72 --- /dev/null +++ b/tests/Release/CiTransformationGateTest.php @@ -0,0 +1,65 @@ +job($workflow, 'mysql-integration'); + $jobEnvironment = $this->environment($job, 6); + $serviceEnvironment = $this->environment($job, 10); + + self::assertArrayHasKey('OPENSTATSPEC_MYSQL_ADMIN_USER', $jobEnvironment); + self::assertArrayHasKey('OPENSTATSPEC_MYSQL_ADMIN_PASSWORD', $jobEnvironment); + self::assertSame('"%"', $serviceEnvironment['MYSQL_ROOT_HOST']); + self::assertSame('root', $jobEnvironment['OPENSTATSPEC_MYSQL_ADMIN_USER']); + self::assertSame( + $serviceEnvironment['MYSQL_ROOT_PASSWORD'], + $jobEnvironment['OPENSTATSPEC_MYSQL_ADMIN_PASSWORD'], + ); + self::assertStringContainsString('OfficialInPlaceTransformation01Test', $job); + self::assertStringContainsString( + 'vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php ' + . '--filter \'/mysql/\' --fail-on-skipped', + $job, + ); + } + + private function job(string $workflow, string $name): string + { + $matched = preg_match( + '/^ ' . preg_quote($name, '/') . ':\R(?(?:^ {4,}.*(?:\R|\z))*)/m', + $workflow, + $matches, + ); + self::assertSame(1, $matched, 'The required CI job is missing.'); + + return $matches['job']; + } + + /** @return array */ + private function environment(string $job, int $indent): array + { + $matched = preg_match_all( + '/^ {' . $indent . '}([A-Z][A-Z0-9_]+):\s*(\S.*?)\s*$/m', + $job, + $matches, + PREG_SET_ORDER, + ); + self::assertNotFalse($matched); + + $environment = []; + foreach ($matches as $match) { + $environment[$match[1]] = $match[2]; + } + + return $environment; + } +} diff --git a/tests/Release/SpecificationPinTest.php b/tests/Release/SpecificationPinTest.php index 548b5fe..8e459a2 100644 --- a/tests/Release/SpecificationPinTest.php +++ b/tests/Release/SpecificationPinTest.php @@ -5,12 +5,14 @@ namespace OpenStatSpec\Tests\Release; use OpenStatSpec\Core\CapabilityDeclaration; +use OpenStatSpec\Tests\Support\SpecificationManifest; use PHPUnit\Framework\TestCase; final class SpecificationPinTest extends TestCase { public function testEverySpecificationCheckoutUsesTheStableReleaseCommit(): void { + self::assertSame('v0.3.0', CapabilityDeclaration::SPECIFICATION_RELEASE); self::assertSame( 'cd8f198c68b849eb8ed018a894670a0904c2181d', CapabilityDeclaration::SPECIFICATION_COMMIT, @@ -56,5 +58,18 @@ public function testEverySpecificationCheckoutUsesTheStableReleaseCommit(): void array_fill(0, count($refs), CapabilityDeclaration::SPECIFICATION_COMMIT), $refs, ); + + foreach ([ + 'transformation/plan-0.1.schema.json', + 'transformation/plan-0.2.schema.json', + 'transformation/spss-syntax-frontend-0.2.schema.json', + 'conformance/transformation-plan-0.1.json', + 'conformance/transformation-plan-0.2.json', + 'conformance/spss-syntax-frontend-0.2.json', + 'conformance/in-place-transformation-0.1.json', + 'conformance/in-place-transformation-0.2.json', + ] as $relative) { + self::assertFileExists(SpecificationManifest::path($relative)); + } } } diff --git a/tests/Spss/SpssAdapterTest.php b/tests/Spss/SpssAdapterTest.php index 0237625..fed5365 100644 --- a/tests/Spss/SpssAdapterTest.php +++ b/tests/Spss/SpssAdapterTest.php @@ -62,7 +62,7 @@ public function testSuccessfulMigrationAdvancesOlderIdentityVersion(): void (new SpssAdapter($pdo, new FakeSpssEngine($this->fixture())))->migrateCatalog(); - self::assertSame([['schema_version' => 3]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); + self::assertSame([['schema_version' => 4]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); } public function testFailedMigrationDoesNotAdvanceOlderIdentityVersion(): void @@ -102,7 +102,7 @@ public function testImportRequiresExplicitMigrationForOlderIdentityWithoutJourna $result = $adapter->import('fixture.sav', 'Old catalog import'); self::assertSame(2, $result->caseCount); - self::assertSame([['schema_version' => 3]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); + self::assertSame([['schema_version' => 4]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); } public function testExportRequiresExplicitMigrationForOlderIdentityWithoutJournalMutation(): void @@ -127,7 +127,7 @@ public function testExportRequiresExplicitMigrationForOlderIdentityWithoutJourna $result = $adapter->export('Old catalog export', 'old-catalog.sav'); self::assertSame(2, $result->caseCount); - self::assertSame([['schema_version' => 3]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); + self::assertSame([['schema_version' => 4]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); } public function testReadOnlyFreshInitializationFailureDoesNotCreateCurrentIdentity(): void @@ -175,7 +175,7 @@ public function testTypedDatasetImportCreatesWideTableAndExportPreservesSupporte $import = $adapter->import('fixture.sav', 'Customer survey'); self::assertSame('Customer survey', $import->datasetName); - self::assertSame([['schema_version' => 3]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); + self::assertSame([['schema_version' => 4]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); self::assertSame(2, $import->caseCount); self::assertSame([], $import->diagnostics); self::assertMatchesRegularExpression('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/', $import->operationId); @@ -940,7 +940,7 @@ public function testMigrateCatalogBackfillsLegacyDatasetsAndIsIdempotent(): void self::rows($pdo, 'SELECT spec_version, source_format, physical_table_name, dataset_name, source_case_count FROM dataset'), ); self::assertSame( - [['version' => 1], ['version' => 2], ['version' => 3]], + [['version' => 1], ['version' => 2], ['version' => 3], ['version' => 4]], self::rows($pdo, 'SELECT version FROM openstatspec_schema_migration ORDER BY version'), ); self::assertSame( diff --git a/tests/Sql/CatalogOwnershipTest.php b/tests/Sql/CatalogOwnershipTest.php index 1a68d98..a7ca3b9 100644 --- a/tests/Sql/CatalogOwnershipTest.php +++ b/tests/Sql/CatalogOwnershipTest.php @@ -8,6 +8,7 @@ use OpenStatSpec\Core\UnsupportedOperation; use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use PDO; use PDOStatement; use PHPUnit\Framework\Attributes\DataProvider; @@ -191,6 +192,7 @@ public function testCurrentIdentityRejectsLaterUnrelatedApplicationTable(): void $pdo = $this->sqlite(); CatalogOwnership::ensure($pdo); (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); $pdo->exec('CREATE TABLE application_data (id INTEGER PRIMARY KEY)'); @@ -209,11 +211,63 @@ public function testRegisteredPhysicalDataTableIsAllowed(): void (new NormativeCatalog($pdo))->createTables(); $pdo->exec('CREATE TABLE dataset_registered (case_ordinal INTEGER PRIMARY KEY)'); $pdo->exec("INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_name, dataset_name, source_case_count, imported_at) VALUES ('registered', '1.0', 'sav', 'dataset_registered', 'registered', 0, '2026-07-28 00:00:00')"); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); CatalogOwnership::ensure($pdo); - self::assertSame(3, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); + self::assertSame(4, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); + } + + public function testCurrentVersionCannotBeMarkedBeforeAuditMigrationCompletes(): void + { + $pdo = $this->sqlite(); + CatalogOwnership::ensure($pdo); + (new NormativeCatalog($pdo))->createTables(); + + try { + CatalogOwnership::markCurrentVersion($pdo); + self::fail('Schema version 4 was marked without audit migration 4.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $exception->diagnosticCode); + } + + self::assertSame(1, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); + self::assertSame([1, 2, 3], array_map( + 'intval', + $this->query($pdo, 'SELECT version FROM openstatspec_schema_migration ORDER BY version')->fetchAll(PDO::FETCH_COLUMN), + )); + } + + public function testVersionFourIdentityWithoutAuditMigrationIsNotReadyForUse(): void + { + $pdo = $this->sqlite(); + CatalogOwnership::ensure($pdo); + (new NormativeCatalog($pdo))->createTables(); + $pdo->exec('UPDATE catalog_identity SET schema_version = 4'); + + try { + CatalogOwnership::assertReadyForUse($pdo); + self::fail('Schema version 4 without audit migration 4 was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $exception->diagnosticCode); + } + } + + public function testVersionFourMarkerWithoutAuditTableIsNotReadyForUse(): void + { + $pdo = $this->sqlite(); + CatalogOwnership::ensure($pdo); + (new NormativeCatalog($pdo))->createTables(); + $pdo->exec("INSERT INTO openstatspec_schema_migration VALUES (4, '2026-08-17 00:00:00')"); + $pdo->exec('UPDATE catalog_identity SET schema_version = 4'); + + try { + CatalogOwnership::assertReadyForUse($pdo); + self::fail('Schema version and marker 4 without the audit table were accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $exception->diagnosticCode); + } } public function testOlderIdentityVersionIsAcceptedWithoutPrematureUpgrade(): void @@ -231,7 +285,7 @@ public function testFutureIdentityVersionFailsClosed(): void { $pdo = $this->sqlite(); CatalogOwnership::ensure($pdo); - $pdo->exec('UPDATE catalog_identity SET schema_version = 4'); + $pdo->exec('UPDATE catalog_identity SET schema_version = 5'); try { CatalogOwnership::ensure($pdo); @@ -240,7 +294,7 @@ public function testFutureIdentityVersionFailsClosed(): void self::assertSame(DiagnosticCode::CatalogNamespaceCollision, $exception->diagnosticCode); } - self::assertSame(4, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); + self::assertSame(5, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); } public function testForeignCatalogNameFailsBeforeOwnershipObjectsAreCreated(): void diff --git a/tests/Support/OfficialFixturePdoAccessors.php b/tests/Support/OfficialFixturePdoAccessors.php new file mode 100644 index 0000000..cac411f --- /dev/null +++ b/tests/Support/OfficialFixturePdoAccessors.php @@ -0,0 +1,47 @@ + $parameters */ + private function scalar(PDO $pdo, string $sql, array $parameters = []): mixed + { + $statement = $pdo->prepare($sql); + $statement->execute($parameters); + return $statement->fetchColumn(); + } + + /** + * @param list $parameters + * @return list + */ + private function column(PDO $pdo, string $sql, array $parameters = []): array + { + $statement = $pdo->prepare($sql); + $statement->execute($parameters); + return array_values($statement->fetchAll(PDO::FETCH_COLUMN)); + } + + /** + * @param list $parameters + * @return list> + */ + private function rows(PDO $pdo, string $sql, array $parameters = []): array + { + $statement = $pdo->prepare($sql); + $statement->execute($parameters); + return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); + } +} diff --git a/tests/Support/SpecificationManifest.php b/tests/Support/SpecificationManifest.php new file mode 100644 index 0000000..0d77f25 --- /dev/null +++ b/tests/Support/SpecificationManifest.php @@ -0,0 +1,37 @@ + */ + public static function load(string $relative): array + { + $decoded = json_decode( + (string) file_get_contents(self::path($relative)), + true, + flags: JSON_THROW_ON_ERROR, + ); + if (!is_array($decoded)) { + throw new RuntimeException('Specification manifest must decode to an object.'); + } + return $decoded; + } +} diff --git a/tests/Transformation/Audit/TransformationAuditMigratorTest.php b/tests/Transformation/Audit/TransformationAuditMigratorTest.php new file mode 100644 index 0000000..0312b39 --- /dev/null +++ b/tests/Transformation/Audit/TransformationAuditMigratorTest.php @@ -0,0 +1,396 @@ +plan(), + inputAlias: 'parent', + datasetId: self::DATASET_ID, + sourceHash: str_repeat('a', 64), + actor: 'conformance-runner', + ); + + self::assertSame('parent', $request->inputAlias); + self::assertSame(self::DATASET_ID, $request->datasetId); + self::assertSame(str_repeat('a', 64), $request->sourceHash); + self::assertSame('conformance-runner', $request->actor); + self::assertNull($request->expectedBranch); + self::assertNull($request->expectedHead); + } + + public function testApplyRequestRejectsInvalidBindingsBeforeExecution(): void + { + $plan = $this->plan(); + $hash = str_repeat('a', 64); + foreach ([ + 'alias mismatch' => [ + 'unknown_input_alias', + static fn() => new InPlaceApplyRequest($plan, 'other', self::DATASET_ID, $hash, 'conformance-runner'), + ], + 'malformed dataset UUID' => [ + 'invalid_dataset_id', + static fn() => new InPlaceApplyRequest($plan, 'parent', 'not-a-uuid', $hash, 'conformance-runner'), + ], + 'malformed source hash' => [ + 'invalid_source_hash', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, str_repeat('A', 64), 'conformance-runner'), + ], + 'empty actor' => [ + 'actor_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, " \t"), + ], + 'branch without head' => [ + 'dolt_context_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, 'conformance-runner', 'feature/recode'), + ], + 'head without branch' => [ + 'dolt_context_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, 'conformance-runner', expectedHead: 'expected-head'), + ], + 'empty branch' => [ + 'dolt_context_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, 'conformance-runner', '', 'expected-head'), + ], + 'empty head' => [ + 'dolt_context_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, 'conformance-runner', 'feature/recode', ''), + ], + ] as $description => [$expectedCode, $factory]) { + try { + $factory(); + self::fail($description . ' was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($expectedCode, $failure->diagnosticCode(), $description); + } + } + } + + public function testFreshSqliteMigrationCreatesTheExactCompactAuditTable(): void + { + $pdo = $this->sqlite(); + + (new TransformationAuditMigrator($pdo))->migrate(); + (new TransformationAuditMigrator($pdo))->migrate(); + + self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $this->acceptedContracts($pdo), + ); + self::assertSame([ + 'apply_id', + 'contract_id', + 'database_profile', + 'dataset_id', + 'physical_table_schema', + 'physical_table_name', + 'source_hash', + 'plan_hash', + 'canonical_plan_json', + 'actor', + 'status', + 'dolt_branch', + 'dolt_head_before', + 'dolt_head_after', + 'operation_count', + 'started_at', + 'completed_at', + ], $this->columnNames($pdo)); + self::assertSame([1, 2, 3, 4], $this->migrationVersions($pdo)); + self::assertSame( + ['transformation_apply'], + $this->query($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'transformation_apply%' ORDER BY name")->fetchAll(PDO::FETCH_COLUMN), + ); + } + + public function testAdapterMigrationRecordsFreshVersionsOneThroughFourExactlyOnce(): void + { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $adapter = new SpssAdapter($pdo); + + $adapter->migrateCatalog(); + $adapter->migrateCatalog(); + + self::assertSame([1, 2, 3, 4], $this->migrationVersions($pdo)); + self::assertSame(4, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); + self::assertSame( + ['transformation_apply'], + $this->query($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'transformation_apply'")->fetchAll(PDO::FETCH_COLUMN), + ); + } + + public function testSqliteMigrationPreservesAnExistingVersion01AuditRowWithoutResidue(): void + { + $pdo = $this->sqlite(); + $this->createVersion01AuditTable($pdo); + $this->insertVersion01AuditRow($pdo); + $before = $this->query($pdo, 'SELECT * FROM transformation_apply')->fetch(PDO::FETCH_ASSOC); + + (new TransformationAuditMigrator($pdo))->migrate(); + + self::assertSame($before, $this->query($pdo, 'SELECT * FROM transformation_apply')->fetch(PDO::FETCH_ASSOC)); + self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $this->acceptedContracts($pdo), + ); + self::assertSame( + ['transformation_apply'], + $this->query($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'transformation_apply%' ORDER BY name")->fetchAll(PDO::FETCH_COLUMN), + ); + self::assertSame([1, 2, 3, 4], $this->migrationVersions($pdo)); + } + + public function testFailedSqliteRebuildRollsBackToTheVersion01Table(): void + { + $pdo = $this->sqlite(false); + $this->createVersion01AuditTable($pdo); + $this->insertVersion01AuditRow($pdo, '22222222-2222-4222-8222-222222222222'); + $pdo->exec('PRAGMA foreign_keys = ON'); + + try { + (new TransformationAuditMigrator($pdo))->migrate(); + self::fail('Migration accepted an audit row whose dataset foreign key is missing.'); + } catch (\Throwable) { + // The unchanged v0.1 table below proves the native transaction rolled back. + } + + self::assertSame(['openstatspec-in-place-transformation-v0.1'], $this->acceptedContracts($pdo)); + self::assertSame(1, (int) $this->query($pdo, 'SELECT COUNT(*) FROM transformation_apply')->fetchColumn()); + self::assertSame( + ['transformation_apply'], + $this->query($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'transformation_apply%' ORDER BY name")->fetchAll(PDO::FETCH_COLUMN), + ); + self::assertSame([1, 2, 3], $this->migrationVersions($pdo)); + } + + public function testSuccessWriterAddsOneCompactRowInsideTheCallerTransaction(): void + { + $pdo = $this->sqlite(); + (new TransformationAuditMigrator($pdo))->migrate(); + $request = new InPlaceApplyRequest( + $this->plan(), + 'parent', + self::DATASET_ID, + str_repeat('a', 64), + 'conformance-runner', + ); + $dataset = new DatasetBinding(self::DATASET_ID, 'Survey', null, 'data_survey'); + + $pdo->beginTransaction(); + $applyId = (new TransformationAuditWriter($pdo))->succeed( + $request, + $dataset, + 'sqlite', + null, + null, + '2026-08-17 12:00:00', + ); + $pdo->commit(); + + $row = $this->query($pdo, 'SELECT * FROM transformation_apply')->fetch(PDO::FETCH_ASSOC); + self::assertIsArray($row); + self::assertSame($applyId, $row['apply_id']); + self::assertMatchesRegularExpression('/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/D', $applyId); + self::assertSame('openstatspec-in-place-transformation-v0.1', $row['contract_id']); + self::assertSame('sqlite', $row['database_profile']); + self::assertSame(self::DATASET_ID, $row['dataset_id']); + self::assertNull($row['physical_table_schema']); + self::assertSame('data_survey', $row['physical_table_name']); + self::assertSame(str_repeat('a', 64), $row['source_hash']); + self::assertSame('73a4893b0bf20cd731786b6998526ba84affcdb39c034c561d1ad29548ac38fe', $row['plan_hash']); + self::assertSame('{"contract":"openstatspec-transformation-plan-v0.1","input_alias":"parent","operations":[{"label":"Score","op":"set_variable_label","variable":"score"}]}', $row['canonical_plan_json']); + self::assertSame('conformance-runner', $row['actor']); + self::assertSame('succeeded', $row['status']); + self::assertNull($row['dolt_branch']); + self::assertNull($row['dolt_head_before']); + self::assertNull($row['dolt_head_after']); + self::assertSame(1, (int) $row['operation_count']); + self::assertSame('2026-08-17 12:00:00', $row['started_at']); + self::assertIsString($row['completed_at']); + self::assertGreaterThanOrEqual($row['started_at'], $row['completed_at']); + self::assertSame(1, (int) $this->query($pdo, 'SELECT COUNT(*) FROM transformation_apply')->fetchColumn()); + } + + public function testSuccessWriterRequiresTheOpenApplyTransaction(): void + { + $pdo = $this->sqlite(); + (new TransformationAuditMigrator($pdo))->migrate(); + $request = new InPlaceApplyRequest( + $this->plan(), + 'parent', + self::DATASET_ID, + str_repeat('a', 64), + 'conformance-runner', + ); + + $this->expectException(\LogicException::class); + (new TransformationAuditWriter($pdo))->succeed( + $request, + new DatasetBinding(self::DATASET_ID, 'Survey', null, 'data_survey'), + 'sqlite', + null, + null, + '2026-08-17 12:00:00', + ); + } + + public function testSuccessWriterUsesTheVersion02BindingContractForAVersion02Plan(): void + { + $pdo = $this->sqlite(); + (new TransformationAuditMigrator($pdo))->migrate(); + $request = new InPlaceApplyRequest( + new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]), + 'parent', + self::DATASET_ID, + str_repeat('a', 64), + 'conformance-runner', + ); + + $pdo->beginTransaction(); + (new TransformationAuditWriter($pdo))->succeed( + $request, + new DatasetBinding(self::DATASET_ID, 'Survey', null, 'data_survey'), + 'sqlite', + null, + null, + '2026-08-17 12:00:00', + ); + $pdo->commit(); + + self::assertSame( + 'openstatspec-in-place-transformation-v0.2', + $this->query($pdo, 'SELECT contract_id FROM transformation_apply')->fetchColumn(), + ); + } + + private function plan(): TransformationPlan + { + return new TransformationPlan( + PlanContract::V01, + 'parent', + [new SetVariableLabelOperation('score', 'Score')], + ); + } + + private function sqlite(bool $foreignKeys = true): PDO + { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $pdo->exec('PRAGMA foreign_keys = ' . ($foreignKeys ? 'ON' : 'OFF')); + $pdo->exec('CREATE TABLE dataset (dataset_id VARCHAR(36) NOT NULL PRIMARY KEY, physical_table_schema TEXT NULL, physical_table_name TEXT NOT NULL)'); + $pdo->exec("INSERT INTO dataset VALUES ('" . self::DATASET_ID . "', NULL, 'data_survey')"); + $pdo->exec('CREATE TABLE openstatspec_schema_migration (version INTEGER NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL)'); + $insert = $pdo->prepare('INSERT INTO openstatspec_schema_migration VALUES (?, ?)'); + foreach ([1, 2, 3] as $version) { + $insert->execute([$version, '2026-08-17 00:00:00']); + } + return $pdo; + } + + private function createVersion01AuditTable(PDO $pdo): void + { + $pdo->exec(<<<'SQL' +CREATE TABLE transformation_apply ( + apply_id VARCHAR(36) NOT NULL PRIMARY KEY, + contract_id TEXT NOT NULL, + database_profile TEXT NOT NULL, + dataset_id VARCHAR(36) NOT NULL REFERENCES dataset(dataset_id), + physical_table_schema TEXT NULL, + physical_table_name TEXT NOT NULL, + source_hash CHAR(64) NOT NULL, + plan_hash CHAR(64) NOT NULL, + canonical_plan_json TEXT NOT NULL, + actor TEXT NOT NULL, + status TEXT NOT NULL, + dolt_branch TEXT NULL, + dolt_head_before TEXT NULL, + dolt_head_after TEXT NULL, + operation_count INTEGER NOT NULL, + started_at TIMESTAMP NOT NULL, + completed_at TIMESTAMP NOT NULL, + CHECK (contract_id = 'openstatspec-in-place-transformation-v0.1'), + CHECK (database_profile IN ('sqlite', 'postgresql', 'mysql', 'mariadb', 'dolt')), + CHECK (status IN ('succeeded', 'failed')), + CHECK (operation_count > 0), + CHECK ((database_profile <> 'dolt' AND dolt_branch IS NULL AND dolt_head_before IS NULL AND dolt_head_after IS NULL) OR (database_profile = 'dolt' AND dolt_branch IS NOT NULL AND dolt_head_before IS NOT NULL AND (status = 'failed' OR dolt_head_after = dolt_head_before))) +) +SQL); + } + + private function insertVersion01AuditRow(PDO $pdo, string $datasetId = self::DATASET_ID): void + { + $pdo->prepare('INSERT INTO transformation_apply VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')->execute([ + 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + 'openstatspec-in-place-transformation-v0.1', + 'sqlite', + $datasetId, + null, + 'data_survey', + str_repeat('b', 64), + str_repeat('c', 64), + '{"contract":"openstatspec-transformation-plan-v0.1"}', + 'legacy-runner', + 'succeeded', + null, + null, + null, + 1, + '2026-08-17 00:00:00', + '2026-08-17 00:00:01', + ]); + } + + /** @return list */ + private function acceptedContracts(PDO $pdo): array + { + $sql = $this->query($pdo, "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'transformation_apply'")->fetchColumn(); + self::assertIsString($sql); + return array_values(array_filter( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + static fn(string $contract): bool => str_contains($sql, $contract), + )); + } + + /** @return list */ + private function columnNames(PDO $pdo): array + { + $columns = $this->query($pdo, 'PRAGMA table_info(transformation_apply)')->fetchAll(PDO::FETCH_ASSOC); + return array_values(array_map(static fn(array $column): string => (string) $column['name'], $columns)); + } + + /** @return list */ + private function migrationVersions(PDO $pdo): array + { + $versions = $this->query($pdo, 'SELECT version FROM openstatspec_schema_migration ORDER BY version')->fetchAll(PDO::FETCH_COLUMN); + return array_values(array_map(static fn(mixed $version): int => (int) $version, $versions)); + } + + private function query(PDO $pdo, string $sql): PDOStatement + { + $statement = $pdo->query($sql); + if ($statement === false) { + self::fail('Could not execute audit test query.'); + } + return $statement; + } +} diff --git a/tests/Transformation/Canonical/TransformationPlanTest.php b/tests/Transformation/Canonical/TransformationPlanTest.php deleted file mode 100644 index e58e531..0000000 --- a/tests/Transformation/Canonical/TransformationPlanTest.php +++ /dev/null @@ -1,227 +0,0 @@ -validPlan(); - (new PlanValidator())->assertValid($plan); - - self::assertSame(self::DATASET_ID, $plan->datasetId()); - self::assertCount(3, $plan->operations()); - self::assertSame('age', $plan->operations()[0]->sourceVariable()); - self::assertSame('age_group', $plan->operations()[0]->targetVariable()); - self::assertSame(TransformationPlan::CONTRACT, $plan->canonicalArray()['contract']); - self::assertSame('in_place', $plan->canonicalArray()['mode']); - self::assertArrayNotHasKey('output_dataset_id', $plan->canonicalArray()); - self::assertArrayNotHasKey('output_table', $plan->canonicalArray()); - self::assertSame($plan->canonicalJson(), $plan->canonicalJson()); - self::assertMatchesRegularExpression('/\A[0-9a-f]{64}\z/D', $plan->hash()); - self::assertSame(hash('sha256', $plan->canonicalJson()), $plan->hash()); - self::assertStringNotContainsString('spss', strtolower($plan->canonicalJson())); - self::assertStringNotContainsString('stata', strtolower($plan->canonicalJson())); - self::assertStringNotContainsString('sas', strtolower($plan->canonicalJson())); - } - - public function testCanonicalJsonHasStableObjectKeyOrderingAndTypedNumbers(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('score', 'score', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(-0.0)), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - ]); - - self::assertSame( - '{"contract":"openstatspec-transformation-plan-v1","dataset_id":"123e4567-e89b-42d3-a456-426614174000","mode":"in_place","operations":[{"rules":[{"action":{"type":"assign","value":{"binary64":"0000000000000000","type":"number"}},"selector":{"type":"exact","value":{"binary64":"3ff0000000000000","type":"number"}}},{"action":{"type":"copy_source"},"selector":{"type":"else"}}],"source_variable":"score","target_variable":"score","type":"recode"}]}', - $plan->canonicalJson(), - ); - self::assertSame('d562adfb994ddad015bd0fee06dc56026c6fa405476ca980e92190c00162ba52', $plan->hash()); - } - - public function testValidatorCollectsIdentityNameAndLabelViolations(): void - { - $duplicateLabels = [ - new ValueLabel(ScalarValue::number(1), 'One'), - new ValueLabel(ScalarValue::number(1.0), 'Still one'), - ]; - $plan = new TransformationPlan('NOT-A-UUID', [ - new SetVariableLabelOperation("invalid\0name", "bad\0label"), - new SetVariableLabelOperation("invalid\0name", 'again'), - new SetValueLabelsOperation('status', $duplicateLabels), - ]); - - $result = (new PlanValidator())->validate($plan); - - self::assertFalse($result->isValid()); - self::assertSame([ - 'dataset_id.invalid_uuid', - 'variable.invalid_name', - 'variable.invalid_name', - 'text.invalid_unicode', - 'variable.invalid_name', - 'variable.invalid_name', - 'value_labels.duplicate_value', - ], $this->codes($result->violations())); - - $this->expectException(InvalidTransformationPlan::class); - $result->throwIfInvalid(); - } - - public function testCreateVariableRejectsStringWidthAboveSpssMaximum(): void - { - $this->expectException('InvalidArgumentException'); - $this->expectExceptionMessage('at most 32767'); - - new CreateVariableOperation('TooWide', 'string', 32768); - } - - public function testValidatorAllowsMaximumCanonicalStringWidth(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('MaxText', 'string', 32767), - ]); - - self::assertTrue((new PlanValidator())->validate($plan)->isValid()); - } - - public function testValidatorAllowsSequentialOperationsOnTheSameTarget(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new SetVariableLabelOperation('status', 'First label'), - new SetVariableLabelOperation('status', 'Replacement label'), - new RecodeOperation('status', 'status', [ - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - new RecodeOperation('status', 'status', [ - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(1))), - ]), - ]); - - self::assertTrue((new PlanValidator())->validate($plan)->isValid()); - } - - public function testValidatorAllowsSourceNeutralCatalogVariableNames(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new SetVariableLabelOperation('@score', 'At-prefixed variable'), - new SetVariableLabelOperation('#temp', 'Scratch variable'), - new SetVariableLabelOperation('$weight', 'System variable'), - new SetVariableLabelOperation('wave-1 score', 'Catalog name outside SPSS syntax'), - ]); - - self::assertTrue((new PlanValidator())->validate($plan)->isValid()); - } - - public function testValidatorRejectsAmbiguousAndIncompleteRecodeMappings(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('score', 'score', [ - new RecodeRule(new ExactValueSelector(ScalarValue::number(2)), new SetMissingAction()), - new RecodeRule(new ExactValueSelector(ScalarValue::number(2.0)), new CopySourceAction()), - new RecodeRule(new NumericRangeSelector(0.0, 10.0), new CopySourceAction()), - new RecodeRule(new NumericRangeSelector(10.0, null), new CopySourceAction()), - new RecodeRule(new MissingValueSelector(), new SetMissingAction()), - new RecodeRule(new MissingValueSelector(), new CopySourceAction()), - ]), - ]); - - $codes = $this->codes((new PlanValidator())->validate($plan)->violations()); - - self::assertContains('recode.else_required_once', $codes); - self::assertContains('recode.ambiguous_mapping', $codes); - self::assertGreaterThanOrEqual(4, count(array_filter( - $codes, - static fn(string $code): bool => $code === 'recode.ambiguous_mapping', - ))); - } - - public function testValidatorRejectsMalformedRangesElseOrderingAndNonFiniteValues(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('score', 'score', [ - new RecodeRule(new NumericRangeSelector(null, null), new CopySourceAction()), - new RecodeRule(new NumericRangeSelector(5.0, 4.0), new CopySourceAction()), - new RecodeRule(new NumericRangeSelector(INF, null), new CopySourceAction()), - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(NAN))), - new RecodeRule(new ExactValueSelector(ScalarValue::string('late')), new CopySourceAction()), - ]), - ]); - - $codes = $this->codes((new PlanValidator())->validate($plan)->violations()); - - self::assertContains('selector.range_unbounded', $codes); - self::assertContains('selector.range_reversed', $codes); - self::assertContains('selector.range_non_finite', $codes); - self::assertContains('value.non_finite', $codes); - self::assertContains('recode.else_not_last', $codes); - } - - public function testEmptyPlanAndEmptyRecodeAreRejected(): void - { - $empty = (new PlanValidator())->validate(new TransformationPlan(self::DATASET_ID, [])); - self::assertSame(['operations.empty'], $this->codes($empty->violations())); - - $emptyRecode = (new PlanValidator())->validate(new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('score', 'score', []), - ])); - self::assertSame(['recode.rules_empty'], $this->codes($emptyRecode->violations())); - } - - private function validPlan(): TransformationPlan - { - return new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('age', 'age_group', [ - new RecodeRule(new NumericRangeSelector(null, 17.0), new AssignValueAction(ScalarValue::number(1))), - new RecodeRule(new NumericRangeSelector(18.0, null), new AssignValueAction(ScalarValue::number(2))), - new RecodeRule(new MissingValueSelector(), new SetMissingAction()), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - new SetVariableLabelOperation('age_group', 'Age group'), - new SetValueLabelsOperation('age_group', [ - new ValueLabel(ScalarValue::number(1), 'Child'), - new ValueLabel(ScalarValue::number(2), 'Adult'), - ]), - ]); - } - - /** - * @param list<\OpenStatSpec\Transformation\Validation\ValidationViolation> $violations - * @return list - */ - private function codes(array $violations): array - { - return array_map( - static fn(\OpenStatSpec\Transformation\Validation\ValidationViolation $violation): string => $violation->code, - $violations, - ); - } -} diff --git a/tests/Transformation/Conformance/ManifestInventoryTest.php b/tests/Transformation/Conformance/ManifestInventoryTest.php new file mode 100644 index 0000000..d6b8b5a --- /dev/null +++ b/tests/Transformation/Conformance/ManifestInventoryTest.php @@ -0,0 +1,20 @@ +, string}> */ + public static function invalidPlans(): iterable + { + yield 'missing root member' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'operations' => [['op' => 'set_variable_label', 'variable' => 'q1', 'label' => 'Question']], + ], 'plan_schema_invalid']; + yield 'extra operation member' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [['op' => 'set_variable_label', 'variable' => 'q1', 'label' => 'Question', 'extra' => true]], + ], 'plan_schema_invalid']; + yield 'empty input alias' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => '', + 'operations' => [['op' => 'set_variable_label', 'variable' => 'q1', 'label' => 'Question']], + ], 'plan_schema_invalid']; + yield 'empty operations' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [], + ], 'plan_schema_invalid']; + yield 'unknown operation' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [['op' => 'delete_variable']], + ], 'plan_schema_invalid']; + yield 'malformed binary64 bits' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'replace_value_labels', + 'variable' => 'q1', + 'labels' => [['value' => ['type' => 'binary64', 'bits' => 'not-binary64'], 'label' => 'One']], + ]], + ], 'plan_schema_invalid']; + yield 'non-finite binary64 bits' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'replace_value_labels', + 'variable' => 'q1', + 'labels' => [['value' => ['type' => 'binary64', 'bits' => '7ff0000000000000'], 'label' => 'Infinity']], + ]], + ], 'plan_schema_invalid']; + yield 'negative zero binary64 bits' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'replace_value_labels', + 'variable' => 'q1', + 'labels' => [['value' => ['type' => 'binary64', 'bits' => '8000000000000000'], 'label' => 'Zero']], + ]], + ], 'plan_schema_invalid']; + yield 'descending range' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'recode', + 'source' => 'q1', + 'target' => 'q1', + 'target_mode' => 'replace', + 'rules' => [[ + 'match' => [ + 'kind' => 'range', + 'lower' => ['type' => 'binary64', 'bits' => '4000000000000000'], + 'upper' => ['type' => 'binary64', 'bits' => '3ff0000000000000'], + ], + 'result' => ['kind' => 'copy'], + ]], + 'unmatched' => ['kind' => 'copy'], + ]], + ], 'invalid_numeric_range']; + yield 'reserved recode target' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'recode', + 'source' => 'source', + 'target' => '__hidden', + 'target_mode' => 'replace', + 'rules' => [[ + 'match' => ['kind' => 'values', 'values' => [['type' => 'binary64', 'bits' => '3ff0000000000000']]], + 'result' => ['kind' => 'copy'], + ]], + 'unmatched' => ['kind' => 'copy'], + ]], + ], 'reserved_target_name']; + yield 'duplicate exact typed label values' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'replace_value_labels', + 'variable' => 'q1', + 'labels' => [ + ['value' => ['type' => 'string', 'value' => 'A'], 'label' => 'Alpha'], + ['value' => ['type' => 'string', 'value' => 'A'], 'label' => 'Again'], + ], + ]], + ], 'duplicate_value_label']; + } + + /** @param array $plan */ + #[DataProvider('invalidPlans')] + public function testRejectsInvalidPlanWithStableDiagnostic(array $plan, string $expectedCode): void + { + try { + (new PlanCodec())->fromArray($plan); + self::fail('Expected ' . $expectedCode . '.'); + } catch (TransformationFailure $failure) { + self::assertSame($expectedCode, $failure->diagnosticCode()); + } + } + + public function testRejectsInvalidUtf8AsPlanSchemaInvalid(): void + { + $plan = [ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => "invalid-\xB1", + 'operations' => [[ + 'op' => 'set_variable_label', + 'variable' => 'q1', + 'label' => 'Question', + ]], + ]; + + try { + (new PlanCodec())->fromArray($plan); + self::fail('Expected plan_schema_invalid for malformed UTF-8.'); + } catch (TransformationFailure $failure) { + self::assertSame('plan_schema_invalid', $failure->diagnosticCode()); + } + } + + /** @return iterable}> */ + public static function cases(): iterable + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.1.json')['cases'] as $case) { + yield $case['id'] => [$case]; + } + } + + /** @param array $case */ + #[DataProvider('cases')] + public function testOfficialPlan01(array $case): void + { + if (is_string($case['expected_error'] ?? null)) { + try { + (new PlanCodec())->fromArray($case['plan']); + self::fail('Expected ' . $case['expected_error']); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + return; + } + + $plan = (new PlanCodec())->fromArray($case['plan']); + self::assertSame($case['plan'], $plan->canonicalArray()); + self::assertSame($case['expected_plan_hash'], (new PlanCodec())->hash($plan)); + } +} diff --git a/tests/Transformation/Conformance/TransformationPlan02Test.php b/tests/Transformation/Conformance/TransformationPlan02Test.php new file mode 100644 index 0000000..077ddf2 --- /dev/null +++ b/tests/Transformation/Conformance/TransformationPlan02Test.php @@ -0,0 +1,63 @@ + 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'assign', + 'target' => 'target', + 'target_mode' => 'replace', + 'value' => ['kind' => 'variable', 'variable' => 'source'], + ]], + ]; + + try { + (new PlanCodec())->fromArray($plan); + self::fail('Expected plan_schema_invalid.'); + } catch (TransformationFailure $failure) { + self::assertSame('plan_schema_invalid', $failure->diagnosticCode()); + } + } + + /** @return iterable}> */ + public static function cases(): iterable + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.2.json')['cases'] as $case) { + yield $case['id'] => [$case]; + } + } + + /** @param array $case */ + #[DataProvider('cases')] + public function testOfficialPlan02(array $case): void + { + if (is_string($case['expected_error'] ?? null)) { + try { + (new PlanCodec())->fromArray($case['plan']); + self::fail('Expected ' . $case['expected_error']); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + return; + } + + $codec = new PlanCodec(); + $plan = $codec->fromArray($case['plan']); + self::assertSame($case['plan'], $plan->canonicalArray()); + self::assertSame($case['expected_plan_hash'], $codec->hash($plan)); + } +} diff --git a/tests/Transformation/Diagnostic/SourceSpanTest.php b/tests/Transformation/Diagnostic/SourceSpanTest.php new file mode 100644 index 0000000..c524fa9 --- /dev/null +++ b/tests/Transformation/Diagnostic/SourceSpanTest.php @@ -0,0 +1,60 @@ +startOffset); + self::assertSame(20, $span->endOffset); + } + + public function testConstructorRejectsInvertedOffsets(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('start offset must not exceed its end offset'); + new SourceSpan(20, 10); + } + + public function testConstructorRejectsInvertedLineColumn(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('start position must not exceed its end position'); + new SourceSpan(0, 10, 2, 1, 1, 1); + } + + public function testConstructorRejectsInvertedColumnOnSameLine(): void + { + $this->expectException(\InvalidArgumentException::class); + new SourceSpan(0, 10, 1, 10, 1, 1); + } + + public function testCoverAcceptsProperlyOrderedSpans(): void + { + $start = new SourceSpan(10, 15, 1, 1, 1, 6); + $end = new SourceSpan(20, 25, 1, 11, 1, 16); + $covered = SourceSpan::cover($start, $end); + self::assertSame(10, $covered->startOffset); + self::assertSame(25, $covered->endOffset); + self::assertSame(1, $covered->startLine); + self::assertSame(1, $covered->startColumn); + self::assertSame(1, $covered->endLine); + self::assertSame(16, $covered->endColumn); + } + + public function testCoverRejectsEndSpanBeforeStartSpan(): void + { + $start = new SourceSpan(20, 25, 2, 1, 2, 6); + $end = new SourceSpan(10, 15, 1, 1, 1, 6); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('cover() requires the end span'); + SourceSpan::cover($start, $end); + } +} diff --git a/tests/Transformation/Execution/DoltGuardTest.php b/tests/Transformation/Execution/DoltGuardTest.php index 52631eb..a17f731 100644 --- a/tests/Transformation/Execution/DoltGuardTest.php +++ b/tests/Transformation/Execution/DoltGuardTest.php @@ -4,15 +4,68 @@ namespace OpenStatSpec\Tests\Transformation\Execution; -use OpenStatSpec\Core\DiagnosticCode; -use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; use OpenStatSpec\Transformation\Execution\DoltEvidence; use OpenStatSpec\Transformation\Execution\DoltEvidenceReader; use OpenStatSpec\Transformation\Execution\DoltGuard; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; +use OpenStatSpec\Transformation\Plan\Operation\ExecuteOperation; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\TransformationPlan; use PHPUnit\Framework\TestCase; final class DoltGuardTest extends TestCase { + public function testItRequiresCallerSuppliedDoltContextBeforeReadingRepositoryState(): void + { + $reader = new class implements DoltEvidenceReader { + public int $reads = 0; + + public function read(): DoltEvidence + { + ++$this->reads; + return new DoltEvidence('feature/recode', 'expected-head', []); + } + }; + $guard = new DoltGuard($reader); + + try { + $guard->beforeExecution($this->request()); + self::fail('A Dolt apply without expected branch and HEAD was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_context_required', $failure->diagnosticCode()); + self::assertSame(0, $reader->reads); + } + } + + public function testItRejectsAnUnexpectedBranchBeforeExecution(): void + { + $guard = new DoltGuard($this->reader( + new DoltEvidence('main', 'expected-head', []), + )); + + try { + $guard->beforeExecution($this->request('feature/recode', 'expected-head')); + self::fail('An unexpected Dolt branch was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_branch_mismatch', $failure->diagnosticCode()); + } + } + + public function testItRejectsAnUnexpectedHeadBeforeExecution(): void + { + $guard = new DoltGuard($this->reader( + new DoltEvidence('feature/recode', 'other-head', []), + )); + + try { + $guard->beforeExecution($this->request('feature/recode', 'expected-head')); + self::fail('An unexpected Dolt HEAD was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_head_mismatch', $failure->diagnosticCode()); + } + } + public function testItRejectsADirtyWorkingSetBeforeExecution(): void { $guard = new DoltGuard($this->reader( @@ -20,12 +73,11 @@ public function testItRejectsADirtyWorkingSetBeforeExecution(): void )); try { - $guard->beforeExecution(); + $guard->beforeExecution($this->request('main', 'abc123')); self::fail('A dirty Dolt working set was accepted.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::SqlProfileOperationUnavailable, $exception->diagnosticCode); - self::assertStringContainsString('clean working set', $exception->getMessage()); - self::assertStringContainsString('respondents', $exception->getMessage()); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_working_set_dirty', $failure->diagnosticCode()); + self::assertStringContainsString('respondents', $failure->getMessage()); } } @@ -36,27 +88,15 @@ public function testItCapturesPostEvidenceWithoutRequiringTheExpectedEditToBeCle new DoltEvidence('main', 'abc123', ['respondents', 'variable']), )); - $before = $guard->beforeExecution(); - $after = $guard->afterExecution($before); + $request = $this->request('main', 'abc123'); + $before = $guard->beforeExecution($request); + $after = $guard->afterExecution($request, $before); self::assertTrue($before->isClean()); self::assertFalse($after->isClean()); self::assertSame(['respondents', 'variable'], $after->dirtyTables()); } - public function testItRejectsBranchOrHeadMutation(): void - { - $guard = new DoltGuard($this->reader( - new DoltEvidence('main', 'abc123', []), - new DoltEvidence('other', 'abc123', ['respondents']), - )); - $before = $guard->beforeExecution(); - - $this->expectException(UnsupportedOperation::class); - $this->expectExceptionMessage('branch changed'); - $guard->afterExecution($before); - } - private function reader(DoltEvidence ...$evidence): DoltEvidenceReader { return new class (array_values($evidence)) implements DoltEvidenceReader { @@ -74,4 +114,17 @@ public function read(): DoltEvidence } }; } + + private function request(?string $branch = null, ?string $head = null): InPlaceApplyRequest + { + return new InPlaceApplyRequest( + new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]), + 'parent', + '11111111-1111-4111-8111-111111111111', + str_repeat('a', 64), + 'conformance-runner', + $branch, + $head, + ); + } } diff --git a/tests/Transformation/Execution/DoltHeadGuardTest.php b/tests/Transformation/Execution/DoltHeadGuardTest.php index 1def689..96bc9ff 100644 --- a/tests/Transformation/Execution/DoltHeadGuardTest.php +++ b/tests/Transformation/Execution/DoltHeadGuardTest.php @@ -4,15 +4,19 @@ namespace OpenStatSpec\Tests\Transformation\Execution; -use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; use OpenStatSpec\Transformation\Execution\DoltEvidence; use OpenStatSpec\Transformation\Execution\DoltEvidenceReader; use OpenStatSpec\Transformation\Execution\DoltGuard; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; +use OpenStatSpec\Transformation\Plan\Operation\ExecuteOperation; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\TransformationPlan; use PHPUnit\Framework\TestCase; final class DoltHeadGuardTest extends TestCase { - public function testItRejectsAHeadChangeWithoutMutatingRepositoryState(): void + public function testItRejectsAHeadChangeWithTheOfficialPostMutationDiagnostic(): void { $reader = new class implements DoltEvidenceReader { private int $reads = 0; @@ -20,15 +24,55 @@ public function testItRejectsAHeadChangeWithoutMutatingRepositoryState(): void public function read(): DoltEvidence { return ++$this->reads === 1 - ? new DoltEvidence('main', 'before123', []) - : new DoltEvidence('main', 'after456', ['respondents']); + ? new DoltEvidence('feature/recode', 'expected-head', []) + : new DoltEvidence('feature/recode', 'after456', ['respondents']); } }; $guard = new DoltGuard($reader); - $before = $guard->beforeExecution(); + $request = $this->request(); + $before = $guard->beforeExecution($request); - $this->expectException(UnsupportedOperation::class); - $this->expectExceptionMessage('HEAD changed'); - $guard->afterExecution($before); + try { + $guard->afterExecution($request, $before); + self::fail('A post-mutation Dolt HEAD change was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_context_changed', $failure->diagnosticCode()); + } + } + + public function testItRejectsABranchChangeWithTheOfficialPostMutationDiagnostic(): void + { + $guard = new DoltGuard(new class implements DoltEvidenceReader { + private int $reads = 0; + + public function read(): DoltEvidence + { + return ++$this->reads === 1 + ? new DoltEvidence('feature/recode', 'expected-head', []) + : new DoltEvidence('main', 'expected-head', ['respondents']); + } + }); + $request = $this->request(); + $before = $guard->beforeExecution($request); + + try { + $guard->afterExecution($request, $before); + self::fail('A post-mutation Dolt branch change was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_context_changed', $failure->diagnosticCode()); + } + } + + private function request(): InPlaceApplyRequest + { + return new InPlaceApplyRequest( + new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]), + 'parent', + '11111111-1111-4111-8111-111111111111', + str_repeat('a', 64), + 'conformance-runner', + 'feature/recode', + 'expected-head', + ); } } diff --git a/tests/Transformation/Execution/InPlaceApplyRequestTest.php b/tests/Transformation/Execution/InPlaceApplyRequestTest.php new file mode 100644 index 0000000..088a649 --- /dev/null +++ b/tests/Transformation/Execution/InPlaceApplyRequestTest.php @@ -0,0 +1,99 @@ +build('018f47f2-8b6a-7c3d-8e1f-123456789abc'); + self::assertSame('018f47f2-8b6a-7c3d-8e1f-123456789abc', $request->datasetId); + } + + public function testMicrosoftVariantDatasetIdIsAccepted(): void + { + // RFC 4122 reserves the 10xx variant bits (8/9/a/b) and the 110x bits (c/d) for + // Microsoft backward compatibility; deployment tooling that emits Microsoft-style + // GUIDs must not be rejected as invalid. + $request = $this->build('018f47f2-8b6a-7c3d-8e1f-123456789cda'); + self::assertSame('018f47f2-8b6a-7c3d-8e1f-123456789cda', $request->datasetId); + } + + public function testNcsVariantDatasetIdIsRejected(): void + { + // NCS (variant bits 0xxx) is the legacy Apollo NCS UUID; OpenStatSpec does not + // accept it because every catalog row is RFC 4122 or Microsoft variant. + $this->expectException(TransformationFailure::class); + $this->expectExceptionMessage('Apply dataset identity must be a canonical lowercase UUID'); + $this->build('018f47f2-8b6a-7c3d-0e1f-123456789abc'); + } + + public function testReservedVariantDatasetIdIsRejected(): void + { + // 1110xxxx (e/f) is reserved for future use; OpenStatSpec rejects it to keep + // variant semantics tight. + $this->expectException(TransformationFailure::class); + $this->build('018f47f2-8b6a-7c3d-ee1f-123456789abc'); + } + + /** @return iterable */ + public static function malformedDatasetIds(): iterable + { + yield 'empty string' => ['']; + yield 'wrong length' => ['018f47f2-8b6a-7c3d-8e1f-123456789']; + yield 'wrong separator' => ['018f47f28b6a7c3d8e1f123456789abc']; + yield 'uppercase hex' => ['018F47F2-8B6A-7C3D-8E1F-123456789ABC']; + yield 'non-hex character' => ['018f47f2-8b6a-7c3d-8e1f-123456789abz']; + } + + #[DataProvider('malformedDatasetIds')] + public function testMalformedDatasetIdIsRejected(string $datasetId): void + { + $this->expectException(TransformationFailure::class); + $this->build($datasetId); + } + + public function testEmptyActorIsRejected(): void + { + $this->expectException(TransformationFailure::class); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + new InPlaceApplyRequest($plan, 'parent', '018f47f2-8b6a-7c3d-8e1f-123456789abc', str_repeat('a', 64), ''); + } + + public function testDoltBranchAndHeadMustBeSuppliedTogether(): void + { + $this->expectException(TransformationFailure::class); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + new InPlaceApplyRequest($plan, 'parent', '018f47f2-8b6a-7c3d-8e1f-123456789abc', str_repeat('a', 64), 'actor', 'main', null); + } + + public function testInvalidSourceHashIsRejected(): void + { + $this->expectException(TransformationFailure::class); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + new InPlaceApplyRequest($plan, 'parent', '018f47f2-8b6a-7c3d-8e1f-123456789abc', 'not-a-hash', 'actor'); + } + + public function testAliasMismatchIsRejected(): void + { + $this->expectException(TransformationFailure::class); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + new InPlaceApplyRequest($plan, 'other', '018f47f2-8b6a-7c3d-8e1f-123456789abc', str_repeat('a', 64), 'actor'); + } + + private function build(string $datasetId): InPlaceApplyRequest + { + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + return new InPlaceApplyRequest($plan, 'parent', $datasetId, str_repeat('a', 64), 'conformance-runner'); + } +} diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index 4b7a5ab..50517ce 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -4,724 +4,710 @@ namespace OpenStatSpec\Tests\Transformation\Execution; -use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Core\DiagnosticCode; use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Sql\PostgreSqlProfile; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; +use OpenStatSpec\Transformation\Audit\TransformationAuditWriter; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Execution\BoundSchema; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; -use OpenStatSpec\Transformation\Model\Action\AssignValueAction; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\Action\SetMissingAction; -use OpenStatSpec\Transformation\Model\CreateVariableOperation; -use OpenStatSpec\Transformation\Model\DeleteVariableOperation; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\RecodeRule; -use OpenStatSpec\Transformation\Model\ScalarValue; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\ExactValueSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\Selector\NumericRangeSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; -use OpenStatSpec\Transformation\Model\ValueLabel; +use OpenStatSpec\Transformation\Execution\SqlPredicateCompiler; +use OpenStatSpec\Transformation\Plan\Expression\Comparison; +use OpenStatSpec\Transformation\Plan\Expression\LiteralOperand; +use OpenStatSpec\Transformation\Plan\Expression\VariableOperand; +use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\ConditionalAssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\RecodeOperation; +use OpenStatSpec\Transformation\Plan\Operation\ReplaceValueLabelsOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetFormatOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetVariableLabelOperation; +use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\Recode\CopyResult; +use OpenStatSpec\Transformation\Plan\Recode\ExactMatch; +use OpenStatSpec\Transformation\Plan\Recode\LiteralResult; +use OpenStatSpec\Transformation\Plan\Recode\RecodeRule; +use OpenStatSpec\Transformation\Plan\TargetMode; +use OpenStatSpec\Transformation\Plan\TransformationPlan; +use OpenStatSpec\Transformation\Plan\Value\Binary64Value; +use OpenStatSpec\Transformation\Plan\Value\StringValue; use PDO; +use PDOException; use PDOStatement; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +final class FaultInjectingSqlitePdo extends PDO +{ + public bool $failBegin = false; + public bool $failCommit = false; + public bool $failRollback = false; + + public function beginTransaction(): bool + { + return $this->failBegin ? false : parent::beginTransaction(); + } + + public function commit(): bool + { + return $this->failCommit ? false : parent::commit(); + } + + public function rollBack(): bool + { + return $this->failRollback ? false : parent::rollBack(); + } +} + final class InPlaceTransformationExecutorTest extends TestCase { - private const DATASET_ID = '018f47f2-8b6a-7c3d-9e1f-123456789abc'; + private const DATASET_ID = '018f47f2-8b6a-4c3d-8e1f-123456789abc'; - private PDO $pdo; + private FaultInjectingSqlitePdo $pdo; protected function setUp(): void { if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { - self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + self::markTestSkipped('PDO SQLite is not available.'); } - $this->pdo = new PDO('sqlite::memory:', options: [ + $this->pdo = new FaultInjectingSqlitePdo('sqlite::memory:', options: [ PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false, ]); $this->pdo->exec('PRAGMA foreign_keys = ON'); (new NormativeCatalog($this->pdo))->createTables(); - $this->pdo->exec( - 'CREATE TABLE respondents (__case_ordinal INTEGER NOT NULL PRIMARY KEY, source_value REAL NULL, destination REAL NULL)', - ); + (new TransformationAuditMigrator($this->pdo))->migrate(); + CatalogOwnership::markCurrentVersion($this->pdo); + $this->pdo->exec('CREATE TABLE "odd""table" (__case_ordinal INTEGER NOT NULL PRIMARY KEY, "odd""source" REAL NULL, "target"";DROP TABLE dataset;--" REAL NULL CHECK ("target"";DROP TABLE dataset;--" <> 12345.0), text_value TEXT COLLATE NOCASE NOT NULL)'); $this->pdo->prepare( - 'INSERT INTO dataset ' - . '(dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)', - )->execute([self::DATASET_ID, '1.0', 'fixture', null, 'respondents', 'survey', 5, '2026-07-31 00:00:00']); - $insertVariable = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' - . 'VALUES (?, ?, ?, ?, ?, ?)', - ); - $insertVariable->execute(['018f47f2-8b6a-7c3d-9e1f-123456789abd', self::DATASET_ID, 1, 'SourceValue', 'source_value', 'numeric']); - $insertVariable->execute(['018f47f2-8b6a-7c3d-9e1f-123456789abe', self::DATASET_ID, 2, 'Destination', 'destination', 'numeric']); - $insertCase = $this->pdo->prepare( - 'INSERT INTO respondents (__case_ordinal, source_value, destination) VALUES (?, ?, ?)', + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([self::DATASET_ID, '1.0', 'fixture', null, 'odd"table', 'Executor fixture', 2, '2026-08-17 00:00:00']); + $insert = $this->pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) VALUES (?, ?, ?, ?, ?, ?, ?)', ); - foreach ([[1, 1.0, -1.0], [2, 2.0, -1.0], [3, 3.0, -1.0], [4, 9.0, -1.0], [5, null, -1.0]] as $case) { - $insertCase->execute($case); - } - CatalogOwnership::markCurrentVersion($this->pdo); + $insert->execute(['11111111-1111-4111-8111-111111111111', self::DATASET_ID, 1, 'source', 'odd"source', 'numeric', null]); + $insert->execute(['22222222-2222-4222-8222-222222222222', self::DATASET_ID, 2, 'target', 'target";DROP TABLE dataset;--', 'numeric', null]); + $insert->execute(['33333333-3333-4333-8333-333333333333', self::DATASET_ID, 3, 'text', 'text_value', 'string', 8]); + $insertCase = $this->pdo->prepare('INSERT INTO "odd""table" (__case_ordinal, "odd""source", "target"";DROP TABLE dataset;--", text_value) VALUES (?, ?, ?, ?)'); + $insertCase->execute([1, 2.0, 7.0, 'A']); + $insertCase->execute([2, null, 7.0, 'B']); } - public function testCanonicalOperationsMutateDataAndMetadataInPlace(): void + public function testIdentifiersAndValuesAreBoundWithoutSqlInjection(): void { - $tablesBefore = $this->tableNames(); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'Destination', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(10)), - ), - new RecodeRule( - new NumericRangeSelector(2.0, 3.0), - new AssignValueAction(ScalarValue::number(20)), - ), - new RecodeRule( - new MissingValueSelector(), - new AssignValueAction(ScalarValue::number(99)), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - new SetVariableLabelOperation('Destination', 'Recoded destination'), - new SetValueLabelsOperation('Destination', [ - new ValueLabel(ScalarValue::number(10), 'Ten'), - new ValueLabel(ScalarValue::number(20), 'Twenty'), - new ValueLabel(ScalarValue::number(99), 'Missing source'), - ]), + $label = "x'); DROP TABLE dataset; --"; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), + new SetVariableLabelOperation('target', $label), ]); - $result = (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame(self::DATASET_ID, $result->datasetId()); - self::assertSame($plan->hash(), $result->planHash()); - self::assertSame(3, $result->operationCount()); - self::assertNull($result->auditOperationId(), 'Execution must not create a missing journal schema.'); - self::assertSame([10.0, 20.0, 20.0, 9.0, 99.0], $this->query( - 'SELECT destination FROM respondents ORDER BY __case_ordinal', - )->fetchAll(PDO::FETCH_COLUMN)); - self::assertSame('Recoded destination', $this->query( - "SELECT variable_label FROM variable WHERE source_name = 'Destination'", - )->fetchColumn()); - self::assertSame( - [['10.0', 'Ten'], ['20.0', 'Twenty'], ['99.0', 'Missing source']], - $this->query( - 'SELECT CAST(vl.numeric_code AS TEXT), vl.label FROM value_label vl ORDER BY vl.ordinal', - )->fetchAll(PDO::FETCH_NUM), - ); - self::assertSame($tablesBefore, $this->tableNames()); - self::assertSame(1, (int) $this->query('SELECT COUNT(*) FROM dataset')->fetchColumn()); - self::assertFalse(in_array('operation_catalog', $this->tableNames(), true)); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + + self::assertSame([2.0, null], $this->numericColumn('target";DROP TABLE dataset;--')); + self::assertSame($label, $this->scalar("SELECT variable_label FROM variable WHERE source_name = 'target'")); + self::assertSame(1, (int) $this->scalar('SELECT COUNT(*) FROM dataset')); + self::assertSame(1, (int) $this->scalar('SELECT COUNT(*) FROM transformation_apply')); } - public function testSqliteAddsANewTargetInsideTheSameWideTableAndTransaction(): void + public function testSuccessAuditCapturesStartedAtBeforeOperationsAndCompletedAtAfter(): void { - $tablesBefore = $this->tableNames(); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'CreatedTarget', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(100)), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), ]); + $startedBefore = gmdate('Y-m-d H:i:s'); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame($tablesBefore, $this->tableNames(), 'A transformation must not create a table or snapshot.'); - self::assertSame(1, (int) $this->query('SELECT COUNT(*) FROM dataset')->fetchColumn()); - self::assertSame(3, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertSame('createdtarget', $this->query( - "SELECT physical_name FROM variable WHERE source_name = 'CreatedTarget'", - )->fetchColumn()); - self::assertSame( - ['5', 8, 0, '5', 8, 0], - $this->query( - "SELECT print_format_family, print_format_width, print_format_decimals, " - . "write_format_family, write_format_width, write_format_decimals " - . "FROM variable WHERE source_name = 'CreatedTarget'", - )->fetch(PDO::FETCH_NUM), - ); - self::assertSame([100.0, 2.0, 3.0, 9.0, null], $this->query( - 'SELECT createdtarget FROM respondents ORDER BY __case_ordinal', - )->fetchAll(PDO::FETCH_COLUMN)); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + $row = $this->rows('SELECT started_at, completed_at FROM transformation_apply ORDER BY apply_id DESC LIMIT 1'); + self::assertCount(1, $row); + + $completedAfter = gmdate('Y-m-d H:i:s'); + self::assertGreaterThanOrEqual($startedBefore, $row[0]['started_at']); + self::assertLessThanOrEqual($completedAfter, $row[0]['completed_at']); + self::assertLessThanOrEqual($row[0]['started_at'], $row[0]['completed_at']); } - public function testImplicitRecodeTargetRemainsActiveForLaterMetadataOperation(): void + public function testStringRecodeUsesExactBoundValuesDespiteColumnCollation(): void { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'CreatedTarget', [ - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - new SetVariableLabelOperation('CreatedTarget', 'Created target'), + $payload = "x');--"; + $this->pdo->exec("UPDATE \"odd\"\"table\" SET text_value = CASE __case_ordinal WHEN 1 THEN 'Match' ELSE 'match' END"); + $plan = new TransformationPlan(PlanContract::V01, 'parent', [ + new RecodeOperation('text', 'text', TargetMode::Replace, [ + new RecodeRule(new ExactMatch(new StringValue('Match')), new LiteralResult(new StringValue($payload))), + ], new CopyResult()), ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); - self::assertSame('Created target', $this->query( - "SELECT variable_label FROM variable WHERE source_name = 'CreatedTarget'", - )->fetchColumn()); + self::assertSame([$payload, 'match'], $this->column('SELECT text_value FROM "odd""table" ORDER BY __case_ordinal')); + self::assertSame(1, (int) $this->scalar('SELECT COUNT(*) FROM dataset')); } - public function testDeleteThenRecodeCanReuseVariableName(): void + public function testBinary64LiteralAssignmentPreservesTheExactAdjacentValue(): void { - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('Destination'), - new RecodeOperation('SourceValue', 'Destination', [ - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(7))), - ]), + $bits = '3ff0000000000001'; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits($bits))), ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); - $physicalName = (string) $this->query( - "SELECT physical_name FROM variable WHERE source_name = 'Destination'", - )->fetchColumn(); - self::assertNotSame('destination', $physicalName); - self::assertSame( - [7.0, 7.0, 7.0, 7.0, 7.0], - $this->query('SELECT ' . $physicalName . ' FROM respondents ORDER BY __case_ordinal')->fetchAll(PDO::FETCH_COLUMN), - ); + self::assertSame([$bits, $bits], array_map( + static fn(mixed $value): string => bin2hex(pack('E', (float) $value)), + $this->column('SELECT "target"";DROP TABLE dataset;--" FROM "odd""table" ORDER BY __case_ordinal'), + )); } - public function testExplicitStringCreateAndDeleteRemovesDataAndMetadata(): void + public function testLiteralOnlyNumericPredicateUsesNumericOrderingAndExactResultBits(): void { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('TempText', 'string', 20), - new SetVariableLabelOperation('TempText', 'Temporary text'), - new SetValueLabelsOperation('TempText', [ - new ValueLabel(ScalarValue::string('yes'), 'Yes'), - ]), - new DeleteVariableOperation('TempText'), + $bits = '3ff0000000000001'; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new ConditionalAssignOperation( + new Comparison( + new LiteralOperand(Binary64Value::fromBits('4000000000000000')), + '<', + new LiteralOperand(Binary64Value::fromBits('4024000000000000')), + ), + 'target', + new LiteralOperand(Binary64Value::fromBits($bits)), + ), ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); - self::assertSame(2, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertFalse(in_array('temptext', $this->tableColumns(), true)); - self::assertSame(0, (int) $this->query( - 'SELECT COUNT(*) FROM variable_value_label_set', - )->fetchColumn()); - self::assertSame(0, (int) $this->query( - 'SELECT COUNT(*) FROM value_label', - )->fetchColumn()); + self::assertSame([$bits, $bits], array_map( + static fn(mixed $value): string => bin2hex(pack('E', (float) $value)), + $this->column('SELECT "target"";DROP TABLE dataset;--" FROM "odd""table" ORDER BY __case_ordinal'), + )); } - - public function testDeletingOnlyVariableThenRecreatingWithinPlan(): void + public function testPostgresqlNumericLiteralsUseExactTextAndNumericCasts(): void { - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('Destination'), - new DeleteVariableOperation('SourceValue'), - new CreateVariableOperation('SourceValue', 'string', 20), - ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::assertSame(1, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - $physicalName = (string) $this->query("SELECT physical_name FROM variable WHERE source_name = 'SourceValue'")->fetchColumn(); - self::assertNotSame('source_value', $physicalName); - self::assertSame('string', $this->query("SELECT storage_kind FROM variable WHERE source_name = 'SourceValue'")->fetchColumn()); - self::assertTrue(in_array($physicalName, $this->tableColumns(), true)); - self::assertFalse(in_array('source_value', $this->tableColumns(), true)); - } - public function testCreatedVariableCannotUseCaseOrdinalColumn(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('__case_ordinal', 'string', 1), - ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - $physicalName = (string) $this->query("SELECT physical_name FROM variable WHERE source_name = '__case_ordinal'")->fetchColumn(); - self::assertNotSame('__case_ordinal', $physicalName); - self::assertTrue(in_array($physicalName, $this->tableColumns(), true)); + $parameters = []; + $sql = (new SqlPredicateCompiler(new PostgreSqlProfile()))->compile( + new Comparison( + new LiteralOperand(Binary64Value::fromBits('3ff0000000000001')), + '<', + new LiteralOperand(Binary64Value::fromBits('4024000000000000')), + ), + new BoundSchema([]), + $parameters, + ); + + self::assertSame('(CAST(? AS DOUBLE PRECISION) < CAST(? AS DOUBLE PRECISION))', $sql); + self::assertSame(['1.0000000000000002', '10'], $parameters); } - public function testDeleteThenCreateCanReuseVariableName(): void + + public function testRecodeMatchAndResultPreserveExactBinary64Values(): void { - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('SourceValue'), - new CreateVariableOperation('SourceValue', 'string', 20), + $this->pdo->exec('UPDATE "odd""table" SET "odd""source" = CAST(\'1.0000000000000002\' AS REAL) WHERE __case_ordinal = 1'); + $resultBits = '4000000000000001'; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new RecodeOperation('source', 'source', TargetMode::Replace, [ + new RecodeRule( + new ExactMatch(Binary64Value::fromBits('3ff0000000000001')), + new LiteralResult(Binary64Value::fromBits($resultBits)), + ), + ], new CopyResult()), ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); - $physicalName = (string) $this->query( - "SELECT physical_name FROM variable WHERE source_name = 'SourceValue'", - )->fetchColumn(); - self::assertNotSame('source_value', $physicalName); - self::assertSame('string', $this->query( - "SELECT storage_kind FROM variable WHERE source_name = 'SourceValue'", - )->fetchColumn()); - self::assertTrue(in_array($physicalName, $this->tableColumns(), true)); - self::assertFalse(in_array('source_value', $this->tableColumns(), true)); + $values = $this->column('SELECT "odd""source" FROM "odd""table" ORDER BY __case_ordinal'); + self::assertSame($resultBits, bin2hex(pack('E', (float) $values[0]))); + self::assertNull($values[1]); } - public function testDeletingLastMultipleResponseMemberRemovesSet(): void + public function testNumericValueLabelPreservesExactBinary64Code(): void { - $this->pdo->prepare('INSERT INTO multiple_response_set (multiple_response_set_id, dataset_id, source_ordinal, set_name, set_kind) VALUES (?, ?, ?, ?, ?)')->execute(['mr-source', self::DATASET_ID, 1, '$MR', 'MC']); - $this->pdo->prepare('INSERT INTO multiple_response_member (multiple_response_set_id, variable_id, source_ordinal) VALUES (?, ?, ?)')->execute(['mr-source', '018f47f2-8b6a-7c3d-9e1f-123456789abd', 1]); - - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('SourceValue'), + $bits = '3ff0000000000001'; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new ReplaceValueLabelsOperation('target', [ + new ValueLabel(Binary64Value::fromBits($bits), 'Adjacent'), + ]), ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); - self::assertSame(0, (int) $this->query('SELECT COUNT(*) FROM multiple_response_set')->fetchColumn()); - self::assertSame(0, (int) $this->query('SELECT COUNT(*) FROM multiple_response_member')->fetchColumn()); + self::assertSame($bits, bin2hex(pack('E', (float) $this->scalar('SELECT numeric_code FROM value_label')))); } - public function testDeletingLastVariableSetMemberRemovesSet(): void + public function testStringRecodeLiteralWiderThanTargetFailsBeforeAnyMutation(): void { - $this->pdo->prepare('INSERT INTO variable_set (variable_set_id, dataset_id, source_ordinal, set_name) VALUES (?, ?, ?, ?)')->execute(['vs-source', self::DATASET_ID, 1, 'Core']); - $this->pdo->prepare('INSERT INTO variable_set_member (variable_set_id, variable_id, source_ordinal) VALUES (?, ?, ?)')->execute(['vs-source', '018f47f2-8b6a-7c3d-9e1f-123456789abd', 1]); - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('SourceValue'), + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + new RecodeOperation('text', 'text', TargetMode::Replace, [ + new RecodeRule(new ExactMatch(new StringValue('A')), new LiteralResult(new StringValue('123456789'))), + ], new CopyResult()), ]); + $before = $this->state(); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A recode literal wider than the target was applied.'); + } catch (TransformationFailure $failure) { + self::assertSame('type_mismatch', $failure->diagnosticCode()); + } - self::assertSame(0, (int) $this->query('SELECT COUNT(*) FROM variable_set')->fetchColumn()); - self::assertSame(0, (int) $this->query('SELECT COUNT(*) FROM variable_set_member')->fetchColumn()); + self::assertSame($before, $this->state()); } - public function testCreateDeleteCreateRecreatesVariable(): void + public function testReservedPhysicalVariableBindingFailsBeforeMutation(): void { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('TempText', 'string', 20), - new DeleteVariableOperation('TempText'), - new CreateVariableOperation('TempText', 'string', 30), + $this->pdo->exec("UPDATE variable SET physical_name = '__case_ordinal' WHERE source_name = 'source'"); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), ]); + $before = $this->state(); + $failure = null; - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + } catch (TransformationFailure $caught) { + $failure = $caught; + } - self::assertTrue(in_array('temptext_2', $this->tableColumns(), true)); - self::assertSame(30, (int) $this->query( - "SELECT declared_string_width FROM variable WHERE source_name = 'TempText'", - )->fetchColumn()); + self::assertInstanceOf(TransformationFailure::class, $failure, 'A variable bound to the reserved case-order column was accepted.'); + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + self::assertSame($before, $this->state()); } - public function testExplicitStringCreateInitializesBlankValuesAndAFormats(): void + public function testSharedPhysicalTableBindingFailsBeforeMutation(): void { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('TempText', 'string', 20), + $this->pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute(['99999999-9999-4999-8999-999999999999', '1.0', 'fixture', null, 'odd"table', 'Aliased table', 2, '2026-08-17 00:00:00']); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), ]); + $before = $this->state(); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame(['', '', '', '', ''], $this->query( - 'SELECT temptext FROM respondents ORDER BY __case_ordinal', - )->fetchAll(PDO::FETCH_COLUMN)); - self::assertSame(0, (int) $this->query( - 'SELECT COUNT(*) FROM respondents WHERE temptext IS NULL', - )->fetchColumn()); - self::assertSame( - ['1', 20, 0, '1', 20, 0], - $this->query( - "SELECT print_format_family, print_format_width, print_format_decimals, " - . "write_format_family, write_format_width, write_format_decimals " - . "FROM variable WHERE source_name = 'TempText'", - )->fetch(PDO::FETCH_NUM), - ); + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('Two datasets sharing one physical table were accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + } + + self::assertSame($before, $this->state()); } - public function testRecodeTargetCanReuseSlotFreedByEarlierDelete(): void + public function testSqliteNullAndMainSchemasCannotShareOnePhysicalTable(): void { - $connection = new Connection($this->pdo); - $maximum = $connection->profile->effectiveMaximumSourceVariables($this->pdo); - $columns = ['__case_ordinal INTEGER NOT NULL PRIMARY KEY']; - for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { - $columns[] = 'v' . $ordinal . ' REAL NULL'; - } + $this->pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute(['99999999-9999-4999-8999-999999999999', '1.0', 'fixture', 'main', 'odd"table', 'Main-schema alias', 2, '2026-08-17 00:00:00']); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), + ]); + $before = $this->state(); + $beforeDatasetCount = (int) $this->scalar('SELECT COUNT(*) FROM dataset'); + $beforePersistentTableCount = (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"); + $failure = null; - $this->pdo->exec('DROP TABLE respondents'); - $this->pdo->exec('CREATE TABLE respondents (' . implode(', ', $columns) . ')'); - $this->pdo->exec('DELETE FROM variable'); - $insert = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' - . 'VALUES (?, ?, ?, ?, ?, ?)', - ); - $this->pdo->beginTransaction(); - for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { - $name = 'V' . $ordinal; - $insert->execute([ - sprintf('00000000-0000-4000-8000-%012d', $ordinal), - self::DATASET_ID, - $ordinal, - $name, - strtolower($name), - 'numeric', - ]); + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + } catch (TransformationFailure $caught) { + $failure = $caught; } - $this->pdo->commit(); - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('V' . $maximum), - new RecodeOperation('V1', 'Replacement', [ - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), + self::assertInstanceOf(TransformationFailure::class, $failure, 'SQLite NULL and main schemas were treated as different physical namespaces.'); + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + self::assertSame($before, $this->state()); + self::assertSame($beforeDatasetCount, (int) $this->scalar('SELECT COUNT(*) FROM dataset')); + self::assertSame($beforePersistentTableCount, (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")); + } + + public function testSqliteQuotedTableCaseAliasesCannotShareOnePhysicalTable(): void + { + $this->pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute(['99999999-9999-4999-8999-999999999999', '1.0', 'fixture', 'main', 'ODD"TABLE', 'Case-folded alias', 2, '2026-08-17 00:00:00']); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), ]); + $before = $this->state(); + $beforeDatasetCount = (int) $this->scalar('SELECT COUNT(*) FROM dataset'); + $beforePersistentTableCount = (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"); + $failure = null; - (new InPlaceTransformationExecutor($connection))->execute($plan); + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + } catch (TransformationFailure $caught) { + $failure = $caught; + } - self::assertSame($maximum, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertSame($maximum, count($this->query('PRAGMA table_info(respondents)')->fetchAll()) - 1); - self::assertFalse(in_array('v' . $maximum, $this->tableColumns(), true)); - self::assertTrue(in_array('replacement', $this->tableColumns(), true)); + self::assertInstanceOf(TransformationFailure::class, $failure, 'SQLite quoted table case aliases were treated as different physical tables.'); + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + self::assertSame($before, $this->state()); + self::assertSame($beforeDatasetCount, (int) $this->scalar('SELECT COUNT(*) FROM dataset')); + self::assertSame($beforePersistentTableCount, (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")); } - public function testExplicitLongStringCreateCapsAFormatWidthsButKeepsStorageWidth(): void + public function testSqliteAttachedSchemaPhysicalColumnsAreIntrospectedInBoundNamespace(): void { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('LongText', 'string', 400), + $this->pdo->exec("ATTACH DATABASE ':memory:' AS other"); + $this->pdo->exec('CREATE TABLE other."odd""table" ("odd""source" REAL NULL, "target"";DROP TABLE dataset;--" REAL NULL, text_value TEXT)'); + $insert = $this->pdo->prepare('INSERT INTO other."odd""table" ("odd""source", "target"";DROP TABLE dataset;--", text_value) VALUES (?, ?, ?)'); + $insert->execute([2.0, 7.0, 'A']); + $insert->execute([null, 7.0, 'B']); + $this->pdo->exec("UPDATE dataset SET physical_table_schema = 'other'"); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame( - [400, '1', 255, '1', 255], - $this->query( - "SELECT declared_string_width, print_format_family, print_format_width, " - . "write_format_family, write_format_width " - . "FROM variable WHERE source_name = 'LongText'", - )->fetch(PDO::FETCH_NUM), - ); + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('An attached SQLite table without the case-order column was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + } finally { + $this->pdo->exec('DETACH DATABASE other'); + } } - public function testNewTargetIsRejectedBeforeAlterAtTheEffectiveColumnLimit(): void + public function testCrossDatasetValueLabelAssociationFailsBeforeEarlierUpdate(): void { - $connection = new Connection($this->pdo); - $maximum = $connection->profile->effectiveMaximumSourceVariables($this->pdo); - $columns = ['__case_ordinal INTEGER NOT NULL PRIMARY KEY']; - for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { - $columns[] = 'v' . $ordinal . ' REAL NULL'; - } + $otherDataset = '99999999-9999-4999-8999-999999999999'; + $this->pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([$otherDataset, '1.0', 'fixture', null, 'other_table', 'Other dataset', 0, '2026-08-17 00:00:00']); + $this->pdo->prepare('INSERT INTO value_label_set (value_label_set_id, dataset_id, name) VALUES (?, ?, ?)') + ->execute(['aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', $otherDataset, null]); + $this->pdo->prepare('INSERT INTO variable_value_label_set (variable_id, value_label_set_id) VALUES (?, ?)') + ->execute(['22222222-2222-4222-8222-222222222222', 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa']); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand($this->binary64(12345.0))), + new ReplaceValueLabelsOperation('target', [ + new ValueLabel(Binary64Value::fromBits('3ff0000000000000'), 'One'), + ]), + ]); + $before = $this->state(); + $failure = null; - $this->pdo->exec('DROP TABLE respondents'); - $this->pdo->exec('CREATE TABLE respondents (' . implode(', ', $columns) . ')'); - $this->pdo->exec('DELETE FROM variable'); - $insert = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' - . 'VALUES (?, ?, ?, ?, ?, ?)', - ); - $this->pdo->beginTransaction(); - for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { - $name = 'V' . $ordinal; - $insert->execute([ - sprintf('00000000-0000-4000-8000-%012d', $ordinal), - self::DATASET_ID, - $ordinal, - $name, - strtolower($name), - 'numeric', - ]); + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + } catch (TransformationFailure $caught) { + $failure = $caught; } - $this->pdo->commit(); - $columnsBefore = count($this->query('PRAGMA table_info(respondents)')->fetchAll()); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('V1', 'OverflowTarget', [ - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), + self::assertInstanceOf(TransformationFailure::class, $failure, 'A cross-dataset value-label association reached row mutation.'); + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + self::assertSame($before, $this->state()); + } + + public function testFreshDatabasePreflightFailureIsReadOnly(): void + { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]); try { - (new InPlaceTransformationExecutor($connection))->execute($plan); - self::fail('A target beyond the effective source-variable limit was created.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); - self::assertStringContainsString('at most ' . $maximum . ' source variables', $exception->getMessage()); + (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($this->request($plan)); + self::fail('Fresh-database preflight unexpectedly executed.'); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $failure->diagnosticCode); } - self::assertSame($maximum, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertSame($columnsBefore, count($this->query('PRAGMA table_info(respondents)')->fetchAll())); + $statement = $pdo->query("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') ORDER BY name"); + self::assertInstanceOf(PDOStatement::class, $statement); + self::assertSame([], $statement->fetchAll(PDO::FETCH_COLUMN)); } - public function testRecodeWithOnlyElseAssignsTheExpressionDirectly(): void + /** @return iterable */ + public static function invalidBindings(): iterable { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'Destination', [ - new RecodeRule( - new ElseSelector(), - new AssignValueAction(ScalarValue::number(7)), + yield 'unknown value variable' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('missing')), + ]), + 'unknown_variable', + ]; + yield 'create target already exists' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Create, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + ]), + 'target_already_exists', + ]; + yield 'replace target does not exist' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('missing', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + ]), + 'unknown_variable', + ]; + yield 'conditional target does not exist' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new ConditionalAssignOperation( + new Comparison( + new VariableOperand('source'), + '=', + new LiteralOperand(Binary64Value::fromBits('3ff0000000000000')), + ), + 'missing', + new LiteralOperand(Binary64Value::fromBits('3ff0000000000000')), ), ]), + 'conditional_target_missing', + ]; + yield 'string expression variable' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('text')), + ]), + 'expression_type_unsupported', + ]; + yield 'format target is string' => [ + new TransformationPlan(PlanContract::V02, 'parent', [new SetFormatOperation('text', 'F', 8, 2)]), + 'type_mismatch', + ]; + yield 'invalid direct format object' => [ + new TransformationPlan(PlanContract::V02, 'parent', [new SetFormatOperation('target', 'F', 2, 2)]), + 'invalid_format', + ]; + yield 'later invalid operation prevents earlier mutation' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + new SetVariableLabelOperation('missing', 'Never applied'), + ]), + 'unknown_variable', + ]; + } + + #[DataProvider('invalidBindings')] + public function testCompleteBindingValidationRunsBeforeMutation(TransformationPlan $plan, string $code): void + { + $before = $this->state(); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('Invalid plan unexpectedly mutated the dataset.'); + } catch (TransformationFailure $failure) { + self::assertSame($code, $failure->diagnosticCode()); + } + + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); + } + + public function testAuditSchemaMustBeReadyBeforeMutation(): void + { + $this->pdo->exec('DROP TABLE transformation_apply'); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]); + $before = $this->numericColumn('target";DROP TABLE dataset;--'); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('Apply executed without the explicit audit schema.'); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $failure->diagnosticCode); + } - self::assertSame( - [7.0, 7.0, 7.0, 7.0, 7.0], - $this->query('SELECT destination FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), - ); + self::assertSame($before, $this->numericColumn('target";DROP TABLE dataset;--')); + self::assertFalse($this->pdo->inTransaction()); } - public function testCallerOwnedTransactionIsRejectedWithoutRollbackOrJournalWork(): void + public function testCallerOwnedTransactionIsRejectedWithoutTouchingIt(): void { - $tablesBefore = $this->tableNames(); $this->pdo->beginTransaction(); - $this->pdo->exec('UPDATE respondents SET source_value = 42 WHERE __case_ordinal = 1'); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'Destination', [ - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(7))), - ]), + $this->pdo->exec('UPDATE "odd""table" SET "target"";DROP TABLE dataset;--" = 42 WHERE __case_ordinal = 1'); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]); try { - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::fail('Execution unexpectedly joined a caller-owned transaction.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::UnsupportedOperation, $exception->diagnosticCode); - self::assertStringContainsString('caller-owned active transaction', $exception->getMessage()); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('Apply unexpectedly joined the caller transaction.'); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::UnsupportedOperation, $failure->diagnosticCode); } self::assertTrue($this->pdo->inTransaction()); - self::assertSame(42.0, $this->query( - 'SELECT source_value FROM respondents WHERE __case_ordinal = 1', - )->fetchColumn()); - self::assertSame($tablesBefore, $this->tableNames()); - self::assertFalse(in_array('operation_catalog', $this->tableNames(), true)); - self::assertSame( - [-1.0, -1.0, -1.0, -1.0, -1.0], - $this->query('SELECT destination FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), - ); + self::assertSame([42.0, 7.0], $this->numericColumn('target";DROP TABLE dataset;--')); + $this->pdo->rollBack(); + } - self::assertTrue($this->pdo->commit()); - self::assertFalse($this->pdo->inTransaction()); - self::assertSame(42.0, $this->query( - 'SELECT source_value FROM respondents WHERE __case_ordinal = 1', - )->fetchColumn()); + public function testCrossPdoAuditWriterIsRejectedBeforeApply(): void + { + $other = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $before = $this->state(); + + try { + new InPlaceTransformationExecutor( + new Connection($this->pdo), + auditWriter: new TransformationAuditWriter($other), + ); + self::fail('An audit writer from another PDO connection was accepted.'); + } catch (\InvalidArgumentException $failure) { + self::assertStringContainsString('same PDO', $failure->getMessage()); + } + + self::assertSame($before, $this->state()); } - public function testSystemMissingActionForStringTargetFailsBeforeMutation(): void + public function testSilentStatementExecuteFailureRollsBackWithoutAudit(): void { - $this->pdo->exec("ALTER TABLE respondents ADD COLUMN source_text TEXT NOT NULL DEFAULT ''"); - $this->pdo->exec("ALTER TABLE respondents ADD COLUMN destination_text TEXT NOT NULL DEFAULT 'original'"); - $insertVariable = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - ); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789abf', - self::DATASET_ID, - 3, - 'SourceText', - 'source_text', - 'string', - 8, - ]); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789ac0', - self::DATASET_ID, - 4, - 'DestinationText', - 'destination_text', - 'string', - 8, - ]); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'DestinationText', [ - new RecodeRule(new ElseSelector(), new SetMissingAction()), - ]), + $this->pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_SILENT); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand($this->binary64(12345.0))), ]); + $before = $this->state(); try { - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::fail('System-missing recoding into a string target unexpectedly executed.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); - self::assertStringContainsString('not representable', $exception->getMessage()); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A failed UPDATE returned a successful apply result.'); + } catch (PDOException $failure) { + self::assertStringContainsString('execute', strtolower($failure->getMessage())); } - self::assertSame( - ['original', 'original', 'original', 'original', 'original'], - $this->query('SELECT destination_text FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), - ); + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); } - public function testExactStringSelectorIgnoresCaseInsensitiveColumnCollation(): void + public function testSilentAuditExecuteFailureRollsBackDataAndPublishesNoSuccess(): void { - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT COLLATE NOCASE NULL'); - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN destination_text TEXT NULL'); - $insertVariable = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - ); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789abf', self::DATASET_ID, 3, - 'SourceText', 'source_text', 'string', 8, - ]); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789ac0', self::DATASET_ID, 4, - 'DestinationText', 'destination_text', 'string', 8, - ]); - $this->pdo->exec("UPDATE respondents SET source_text = CASE __case_ordinal WHEN 1 THEN 'Match' WHEN 2 THEN 'match' ELSE 'other' END"); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'DestinationText', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::string('Match')), - new AssignValueAction(ScalarValue::string('exact')), - ), - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::string('else'))), - ]), + $this->pdo->exec("ALTER TABLE transformation_apply ADD COLUMN task8_actor_guard INTEGER NOT NULL DEFAULT 0 CHECK (actor <> 'force-audit-failure')"); + $this->pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_SILENT); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]); + $before = $this->state(); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute(new InPlaceApplyRequest( + $plan, + 'parent', + self::DATASET_ID, + str_repeat('a', 64), + 'force-audit-failure', + )); + self::fail('A failed audit INSERT returned a successful apply result.'); + } catch (PDOException $failure) { + self::assertStringContainsString('audit', strtolower($failure->getMessage())); + } - self::assertSame( - ['exact', 'else', 'else', 'else', 'else'], - $this->query('SELECT destination_text FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), - ); + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); } - public function testNewStringTargetRequiresExplicitCatalogWidth(): void + public function testFalseBeginOutcomeFailsBeforeMutation(): void { - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT NULL'); - $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - )->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789abf', - self::DATASET_ID, - 3, - 'SourceText', - 'source_text', - 'string', - 8, - ]); - $tablesBefore = $this->tableNames(); - $variablesBefore = (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn(); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'CreatedText', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::string('a')), - new AssignValueAction(ScalarValue::string('b')), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), + $this->pdo->failBegin = true; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]); + $before = $this->state(); try { - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::fail('A string target without declared_string_width unexpectedly executed.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A false beginTransaction outcome was accepted.'); + } catch (PDOException $failure) { + self::assertStringContainsString('start', strtolower($failure->getMessage())); } - self::assertSame($tablesBefore, $this->tableNames()); - self::assertSame($variablesBefore, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertFalse(in_array('createdtext', $this->tableColumns(), true)); + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); } - public function testExistingStringTargetEnforcesNormativeByteWidthBeforeMutation(): void + public function testFalseCommitOutcomeRollsBackDataAndAudit(): void { - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT NULL'); - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN destination_text TEXT NULL'); - $insertVariable = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - ); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789abf', - self::DATASET_ID, - 3, - 'SourceText', - 'source_text', - 'string', - 4, - ]); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789ac0', - self::DATASET_ID, - 4, - 'DestinationText', - 'destination_text', - 'string', - 1, + $this->pdo->failCommit = true; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]); + $before = $this->state(); - $plans = [ - new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'DestinationText', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::string('a')), - new AssignValueAction(ScalarValue::string('long')), - ), - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::string('x'))), - ]), - ]), - new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'DestinationText', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::string('a')), - new AssignValueAction(ScalarValue::string('x')), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - ]), - new TransformationPlan(self::DATASET_ID, [ - new SetValueLabelsOperation('DestinationText', [ - new ValueLabel(ScalarValue::string('long'), 'Too wide'), - ]), - ]), - ]; + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A false commit outcome returned a successful apply result.'); + } catch (PDOException $failure) { + self::assertStringContainsString('commit', strtolower($failure->getMessage())); + } - foreach ($plans as $plan) { - try { - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::fail('A string operation wider than declared_string_width unexpectedly executed.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); + } + + public function testFalseRollbackOutcomeIsSurfaced(): void + { + $this->pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_SILENT); + $this->pdo->failRollback = true; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand($this->binary64(12345.0))), + ]); + $before = $this->state(); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A false rollback outcome was ignored.'); + } catch (PDOException $failure) { + self::assertStringContainsString('rollback', strtolower($failure->getMessage())); + } finally { + $this->pdo->failRollback = false; + if ($this->pdo->inTransaction()) { + $this->pdo->rollBack(); } } - self::assertSame( - [null, null, null, null, null], - $this->query('SELECT destination_text FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), - ); + + self::assertSame($before, $this->state()); } - /** @return list */ - private function tableNames(): array + private function request(TransformationPlan $plan): InPlaceApplyRequest { - $names = $this->query( - "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", - )->fetchAll(PDO::FETCH_COLUMN); + return new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, str_repeat('a', 64), 'unit-test'); + } - return array_values(array_map(static fn(mixed $name): string => (string) $name, $names)); + private function binary64(float $value): Binary64Value + { + return Binary64Value::fromBits(bin2hex(pack('E', $value))); } - /** @return list */ - private function tableColumns(): array + /** @return array */ + private function state(): array { - $names = $this->query('PRAGMA table_info(respondents)')->fetchAll(PDO::FETCH_ASSOC); + return [ + 'rows' => $this->rows('SELECT * FROM "odd""table" ORDER BY __case_ordinal'), + 'variables' => $this->rows('SELECT * FROM variable ORDER BY source_ordinal'), + 'audit' => $this->rows('SELECT * FROM transformation_apply ORDER BY apply_id'), + 'tables' => $this->column("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name"), + ]; + } - return array_values(array_map( - static fn(array $column): string => (string) $column['name'], - $names, - )); + /** @return list */ + private function numericColumn(string $column): array + { + $quoted = (new Connection($this->pdo))->profile->quoteIdentifier($column); + return array_map( + static fn(mixed $value): ?float => $value === null ? null : (float) $value, + $this->column('SELECT ' . $quoted . ' FROM "odd""table" ORDER BY __case_ordinal'), + ); } - private function query(string $sql): PDOStatement + private function scalar(string $sql): mixed { $statement = $this->pdo->query($sql); self::assertInstanceOf(PDOStatement::class, $statement); + return $statement->fetchColumn(); + } - return $statement; + /** @return list */ + private function column(string $sql): array + { + $statement = $this->pdo->query($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + return array_values($statement->fetchAll(PDO::FETCH_COLUMN)); + } + + /** @return list> */ + private function rows(string $sql): array + { + $statement = $this->pdo->query($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); } } diff --git a/tests/Transformation/Plan/BooleanPredicateTest.php b/tests/Transformation/Plan/BooleanPredicateTest.php new file mode 100644 index 0000000..935e92d --- /dev/null +++ b/tests/Transformation/Plan/BooleanPredicateTest.php @@ -0,0 +1,39 @@ +comparison('source_a'), $this->comparison('source_b')), + $this->comparison('source_c'), + ); + self::fail('Expected noncanonical_boolean_shape.'); + } catch (TransformationFailure $failure) { + self::assertSame('noncanonical_boolean_shape', $failure->diagnosticCode()); + } + } + + private function comparison(string $variable): Comparison + { + return new Comparison( + new VariableOperand($variable), + '=', + new LiteralOperand(Binary64Value::fromBits('3ff0000000000000')), + ); + } +} diff --git a/tests/Transformation/Plan/RecodeOperationTest.php b/tests/Transformation/Plan/RecodeOperationTest.php new file mode 100644 index 0000000..f78ad3b --- /dev/null +++ b/tests/Transformation/Plan/RecodeOperationTest.php @@ -0,0 +1,40 @@ +expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('requires at least one rule'); + $operation = new RecodeOperation('source', 'target', TargetMode::Replace, [], new CopyResult()); + self::assertInstanceOf(RecodeOperation::class, $operation); + } + + public function testValidRuleListRemainsCanonicalizable(): void + { + $operation = new RecodeOperation( + 'source', + 'target', + TargetMode::Create, + [new RecodeRule( + new ExactMatch(Binary64Value::fromBits('3ff0000000000000')), + new LiteralResult(Binary64Value::fromBits('4000000000000000')), + )], + new CopyResult(), + ); + self::assertCount(1, $operation->canonicalArray()['rules']); + } +} diff --git a/tests/Transformation/Plan/Value/Binary64ValueTest.php b/tests/Transformation/Plan/Value/Binary64ValueTest.php new file mode 100644 index 0000000..f46e877 --- /dev/null +++ b/tests/Transformation/Plan/Value/Binary64ValueTest.php @@ -0,0 +1,102 @@ +expectException(\OpenStatSpec\Transformation\Diagnostic\TransformationFailure::class); + Binary64Value::fromBits('7ff8000000000000'); + } + + public function testFromBitsRejectsNegativeZero(): void + { + $this->expectException(\OpenStatSpec\Transformation\Diagnostic\TransformationFailure::class); + Binary64Value::fromBits('8000000000000000'); + } + + public function testFromBitsRejectsInvalidLength(): void + { + $this->expectException(\OpenStatSpec\Transformation\Diagnostic\TransformationFailure::class); + Binary64Value::fromBits('3ff000000000000'); + } + + public function testFromBitsRejectsUppercaseHex(): void + { + $this->expectException(\OpenStatSpec\Transformation\Diagnostic\TransformationFailure::class); + Binary64Value::fromBits('3FF0000000000000'); + } + + public function testNumberReturnsCanonicalFloatForOne(): void + { + self::assertSame(1.0, Binary64Value::fromBits('3ff0000000000000')->number()); + } + + public function testCanonicalArrayReturnsBinary64Shape(): void + { + self::assertSame( + ['type' => 'binary64', 'bits' => '3ff0000000000000'], + Binary64Value::fromBits('3ff0000000000000')->canonicalArray(), + ); + } + + public function testCanonicalKeyIsStableAndUnambiguous(): void + { + self::assertSame( + 'binary64:3ff0000000000000', + Binary64Value::fromBits('3ff0000000000000')->canonicalKey(), + ); + } + + public function testDecimalAlwaysUsesDotSeparator(): void + { + // The decimal() output must always round-trip back through PHP's float + // parser; if localeconv() were to leak a comma separator, the string + // would parse as zero and silently lose precision. Use 1.1 (binary64 + // approximation that always expands with a fractional part). + $onePointOne = Binary64Value::fromBits('3ff199999999999a')->decimal(); + self::assertStringContainsString('.', $onePointOne); + self::assertStringNotContainsString(',', $onePointOne); + $negativeOnePointOne = Binary64Value::fromBits('bff199999999999a')->decimal(); + self::assertStringContainsString('.', $negativeOnePointOne); + self::assertStringNotContainsString(',', $negativeOnePointOne); + } + + public function testDecimalNormalizesLocaleWithCommaSeparator(): void + { + $original = setlocale(LC_NUMERIC, '0'); + $candidateLocales = ['de_DE.utf8', 'fr_FR.utf8', 'nl_NL.utf8', 'de_DE.UTF-8', 'fr_FR.UTF-8', 'nl_NL.UTF-8']; + $restored = false; + $commaLocale = null; + foreach ($candidateLocales as $locale) { + if (setlocale(LC_NUMERIC, $locale) !== false) { + $commaLocale = $locale; + break; + } + } + try { + if ($commaLocale === null) { + self::markTestSkipped('No locale with a comma decimal separator is available on this system.'); + } + $separator = localeconv()['decimal_point'] ?? '.'; + self::assertSame(',', $separator, 'The selected locale must actually use a comma decimal separator.'); + $decimal = Binary64Value::fromBits('4024000000000000')->decimal(); + self::assertStringContainsString('.', $decimal); + self::assertStringNotContainsString(',', $decimal); + } finally { + if ($original !== false) { + setlocale(LC_NUMERIC, $original); + $restored = true; + } elseif ($commaLocale !== null) { + setlocale(LC_NUMERIC, 'C'); + } + self::assertTrue($restored || $commaLocale !== null); + } + } +}