From a2c2dbcefa8770d476e04480d20e11725590959d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 14:28:24 +0300 Subject: [PATCH 01/53] docs: design transformation and SPSS 0.2 support --- ...formation-plan-spss-frontend-0.2-design.md | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md diff --git a/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md b/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md new file mode 100644 index 0000000..8a114cd --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md @@ -0,0 +1,189 @@ +# Transformation Plan and SPSS Frontend 0.2 Design + +**Date:** 2026-08-17 +**Status:** Approved for implementation planning + +## Goal + +Replace the package-local `openstatspec-transformation-plan-v1` API with the +official OpenStatSpec Transformation Plan and SPSS-like Syntax Frontend +contracts from the package's pinned specification checkout. The release must +pass every case in the official Transformation Plan 0.2, SPSS Frontend 0.2, +and In-Place Transformation 0.2 manifests, including the referenced 0.1 golden +plans required by Frontend 0.2. + +This is an intentional breaking change for `v0.6.0`. There is no compatibility +adapter for the package-local plan, and its non-standard `create_variable` and +`delete_variable` operations are removed from the transformation API. + +## Contract boundary + +The canonical model supports both official plan identifiers: + +- `openstatspec-transformation-plan-v0.1` for programs containing only the + unchanged 0.1 command subset; and +- `openstatspec-transformation-plan-v0.2` when a 0.2 command is present. + +Supporting official 0.1 output is part of Frontend 0.2 conformance, not legacy +package compatibility. A plan contains exactly `contract`, `input_alias`, and +a non-empty ordered `operations` array. Dataset UUIDs, SQL details, actor +identity, and Dolt state do not belong in canonical plan JSON. + +The package exposes typed plan objects plus a strict plan codec. Decoding first +validates the structure selected by `contract`, rejects unknown members and +non-canonical values, and then constructs typed operations. Encoding produces +the exact restricted-RFC-8785 bytes used for SHA-256 plan identity. It never +silently upgrades 0.1 to 0.2 or changes operation order, Unicode, expression +shape, or binary64 bits. + +## Public API + +The old constructors and method signatures are removed rather than deprecated. +The replacement boundary consists of: + +- a `TransformationPlan` carrying contract, input alias, and typed operations; +- a `PlanCodec` for strict array/JSON decoding, canonical JSON, and plan hash; +- an `SpssFrontendRequest` carrying the 0.2 request contract, input alias, + ordered input schema, and exact source text; +- an `SpssCompilationResult` carrying the emitted 0.1-or-0.2 plan and the + LF-normalized source hash; +- an `SpssCompiler::compile(SpssFrontendRequest)` facade; and +- an `InPlaceApplyRequest` carrying the plan, input-alias-to-dataset binding, + source hash, non-empty actor, and optional expected Dolt branch and HEAD. + +`InPlaceTransformationExecutor::execute(InPlaceApplyRequest)` is the only +public mutation entry point. This makes the official alias-based plan portable +while keeping the concrete dataset identity and controlled execution context +explicit at apply time. + +## Components + +### Canonical model and validation + +The current transformation model is replaced with the official 0.1 and 0.2 +operation vocabulary. Version 0.1 includes `recode`, `set_variable_label`, and +`replace_value_labels`. Version 0.2 additionally includes `assign`, +`conditional_assign`, `set_format`, `set_measurement_level`, and `execute`. + +Numeric literals retain exact 16-character lowercase binary64 bits. Operands, +comparisons, and n-ary boolean expressions are separate immutable types. +Validation rejects non-finite or negative-zero encodings, invalid target +states, string expressions, invalid formats, duplicate labels, reserved names, +and nested same-operator boolean nodes before SQL mutation. + +### SPSS frontend + +The lexer, parser, binder, and compiler remain separate stages, but they consume +the official request's ordered schema rather than a dataset UUID. Binding is +ASCII case-insensitive and stores the exact catalog spelling in the plan. It +tracks variables created by earlier commands and preserves the simultaneous +input semantics of multi-variable `RECODE`. + +The existing 0.1 commands are narrowed to the official forms. Version 0.2 adds +`COMPUTE`, parenthesized `IF`, `FORMATS`, `VARIABLE LEVEL`, and `EXECUTE`. +Unsupported commands and expression forms fail closed. Decimal numeric tokens +are converted directly to correctly rounded binary64 bits with ties-to-even; +the conversion does not pass through a host floating-point intermediate, and +signed or underflowed zero becomes positive zero. + +The compiler emits an exact 0.1 plan when the program contains only 0.1 +commands. Any 0.2 command promotes the complete emitted plan to 0.2 without +dropping or reordering earlier operations. + +### In-place SQL executor + +Before mutation, the executor validates the complete plan, resolves the input +alias to exactly one dataset, binds every ordered operation against the evolving +schema, checks storage types and target state, verifies backend capabilities, +and validates the apply context. No journal or audit row is started until this +preflight succeeds. + +Existing-target `recode` and `assign` use direct updates. +`conditional_assign` uses a direct update whose predicate relies on SQL +three-valued truth, so only TRUE rows change. Metadata operations update only +their normative catalog fields. `execute` emits no SQL and never commits. + +SQLite and PostgreSQL may create a numeric target only where the full schema, +data, catalog, and audit change is proven to share one native transaction. +MySQL, MariaDB, and Dolt reject create-target plans before mutation with +`schema_change_not_atomic`; they never simulate atomicity with a copied table +or recovery artifact. + +### Dolt context and audit + +Every apply requires a non-empty actor. Dolt additionally requires the expected +branch and expected HEAD and a clean working set. Branch, HEAD, and cleanliness +are checked before mutation; branch and HEAD are checked again before success. +The executor never calls `DOLT_COMMIT`, switches branches, resets, merges, tags, +or creates a persistent recovery object. + +The explicit catalog migration creates or upgrades the versioned +`transformation_apply` table. It stores one compact audit row with the +official binding contract, target identity, canonical plan and source hashes, +actor, status, operation count, timestamps, and optional Dolt evidence. The +table accepts both official 0.1 and 0.2 binding contracts and preserves any +existing official audit rows. Migration runs before an apply and is not hidden +inside the apply transaction. + +## Data flow and atomicity + +1. Validate the frontend request and normalize only source line endings for + `source_hash`. +2. Parse, bind against the supplied schema, and compile the complete source. +3. Validate and canonicalize the emitted plan, then calculate `plan_hash`. +4. Build an apply request that maps the plan alias to one dataset and supplies + actor and any Dolt expectations. +5. Preflight the entire apply without writing data, metadata, schema, or audit. +6. Execute operations in order inside one engine-native transaction and write + the compact audit in that same boundary. +7. On any failure, roll back the native transaction and publish no successful + audit or partial state. + +A successful apply preserves dataset identity, physical table identity, case +order, case count, dataset count, and persistent data-table count. It creates +no derived dataset, full-table copy, staging relation, snapshot, rollback table, +dataset-version row, or hidden recovery layer. Dolt alone owns version history +and rollback. + +## Diagnostics + +Frontend and plan failures expose the stable specification diagnostic codes and +source spans where applicable. Apply failures expose the stable binding codes, +including actor, Dolt context, and backend capability failures. Exceptions may +add a concise human explanation but must not leak credentials, unrelated row +values, or driver connection strings. + +Every validation, binding, capability, actor, branch, HEAD, and clean-working-set +failure occurs before mutation. A detected post-mutation Dolt context change +causes the open transaction to roll back. + +## Verification + +Automated tests load the manifests from the pinned OpenStatSpec `v0.3.0` +specification checkout and treat their expected canonical JSON, hashes, errors, +row results, metadata, audit fields, and forbidden artifacts as independent +goldens. Required coverage is: + +- all 26 Transformation Plan 0.2 cases and every referenced Plan 0.1 golden; +- all 44 SPSS Frontend 0.2 cases and their exact source and plan hashes; +- all 11 In-Place Transformation 0.2 cases; +- focused unit tests for exact decimal-to-binary64 conversion, boolean + precedence/flattening, version selection, and diagnostic spans; +- SQLite execution and rollback tests for successful target creation; +- PostgreSQL execution evidence for the supported atomic create path; +- MySQL, MariaDB, and Dolt rejection evidence for create-target plans; +- live Dolt success, mismatch, dirty-state, context-change, and no-commit + evidence; and +- the existing full lint, style, PHPStan, PHPUnit, and GitHub service matrix. + +The capability declaration and documentation claim Transformation Plan 0.1 and +0.2, SPSS Frontend 0.2, and In-Place Transformation 0.1 and 0.2 only after these +gates pass. Migration notes show the removed package-local API and the new +request/compile/apply flow, and the completed change is released as `v0.6.0`. + +## Out of scope + +Full IBM SPSS syntax, arbitrary arithmetic or functions, string predicates, +automatic target provisioning on non-atomic engines, additional frontends, +automatic Dolt commits, and OpenStatSpec-managed dataset versioning or rollback +are not part of this feature. From a234bb645b49ef91075f1a1f614e38f06101226e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 14:40:03 +0300 Subject: [PATCH 02/53] docs: plan transformation and SPSS 0.2 support --- ...7-transformation-plan-spss-frontend-0.2.md | 1071 +++++++++++++++++ 1 file changed, 1071 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-17-transformation-plan-spss-frontend-0.2.md diff --git a/docs/superpowers/plans/2026-08-17-transformation-plan-spss-frontend-0.2.md b/docs/superpowers/plans/2026-08-17-transformation-plan-spss-frontend-0.2.md new file mode 100644 index 0000000..b1d03fa --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-transformation-plan-spss-frontend-0.2.md @@ -0,0 +1,1071 @@ +# Transformation Plan and SPSS Frontend 0.2 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (- [ ]) syntax for tracking. + +**Goal:** Replace the package-local transformation API with official OpenStatSpec Transformation Plan 0.1/0.2, SPSS Frontend 0.2, and In-Place Transformation 0.1/0.2 conformance. + +**Architecture:** Add a strict official plan model and codec beside the legacy model, migrate the SPSS compiler and in-place executor to it, then delete the legacy model before completion. The frontend compiles an alias-based request to a versioned canonical plan; a separate apply request binds that alias to one dataset and carries source hash, actor, and Dolt expectations. + +**Tech Stack:** PHP 8.4.1+, PDO, PHPUnit 11, PHPStan 2, PHP-CS-Fixer 3, brick/math 0.19, OpenStatSpec specification v0.3.0 fixtures, SQLite/PostgreSQL/MySQL/MariaDB/Dolt. + +## Global Constraints + +- The breaking release target is v0.6.0; do not retain a compatibility adapter for openstatspec-transformation-plan-v1. +- Frontend 0.2 must still emit exact official Plan 0.1 for programs containing only the 0.1 command subset. +- Pass all 4 Plan 0.1, 26 Plan 0.2, 44 Frontend 0.2, 6 In-Place 0.1, and 11 In-Place 0.2 cases. +- Preserve dataset identity, physical table identity, case order, case count, dataset count, and persistent data-table count. +- Never create a derived dataset, persistent copy, snapshot, staging relation, rollback table, dataset-version row, or recovery layer. +- Dolt is the sole version/history/rollback layer; never commit, switch, merge, reset, or tag Dolt from the adapter. +- Validate the complete plan, target, actor, backend capabilities, and Dolt context before mutation. +- SQLite and PostgreSQL may create numeric targets atomically; MySQL, MariaDB, and Dolt must reject them with schema_change_not_atomic. +- Canonical hashes use exact restricted RFC 8785 bytes; decimal source numbers must round directly to binary64 ties-to-even without a host-float intermediate. +- Run every task through a red test, minimal implementation, focused green test, full relevant regression, and a focused commit. + +--- + +## File Structure + +New production boundaries: + +- src/Transformation/Plan: official immutable plan, operation, expression, value, codec, and validation types. +- src/Transformation/Diagnostic: stable diagnostic value and exception types shared by plan, frontend, and executor. +- src/Frontend/Spss/Request: official frontend request and ordered input-schema types. +- src/Frontend/Spss/Number: exact finite-decimal to binary64 conversion. +- src/Transformation/Audit: explicit schema migration and compact transformation audit writer. +- src/Transformation/Execution: apply request, target binding, preflight, SQL predicate compiler, and executor. + +New test boundaries: + +- tests/Support/SpecificationManifest.php: one strict loader for the pinned sibling/CI specification checkout. +- tests/Transformation/Conformance: official Plan 0.1 and 0.2 golden tests. +- tests/Frontend/Spss/Conformance: official Frontend 0.2 golden tests, including exact referenced Plan 0.1 output. +- tests/Integration/OfficialInPlaceTransformation01Test.php: the 6 official 0.1 binding cases. +- tests/Integration/OfficialInPlaceTransformation02Test.php: the 11 official 0.2 binding cases. + +--- + +### Task 1: Pin and load the official transformation manifests + +**Files:** +- Create: tests/Support/SpecificationManifest.php +- Create: tests/Transformation/Conformance/ManifestInventoryTest.php +- Modify: tests/Release/SpecificationPinTest.php + +**Interfaces:** +- Consumes: OPENSTATSPEC_SPECIFICATION_DIR or the workspace sibling specification directory. +- Produces: SpecificationManifest::load(string): array and SpecificationManifest::path(string): string. + +- [ ] **Step 1: Write the failing manifest inventory test** + +~~~php +public function testPinnedTransformationManifestInventory(): void +{ + self::assertCount(4, SpecificationManifest::load('conformance/transformation-plan-0.1.json')['cases']); + self::assertCount(26, SpecificationManifest::load('conformance/transformation-plan-0.2.json')['cases']); + self::assertCount(44, SpecificationManifest::load('conformance/spss-syntax-frontend-0.2.json')['cases']); + self::assertCount(6, SpecificationManifest::load('conformance/in-place-transformation-0.1.json')['cases']); + self::assertCount(11, SpecificationManifest::load('conformance/in-place-transformation-0.2.json')['cases']); +} +~~~ + +- [ ] **Step 2: Run the test and verify the missing loader failure** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/ManifestInventoryTest.php + +Expected: FAIL because OpenStatSpec\Tests\Support\SpecificationManifest does not exist. + +- [ ] **Step 3: Implement the strict loader** + +~~~php +final class SpecificationManifest +{ + public static function path(string $relative): string + { + $configured = getenv('OPENSTATSPEC_SPECIFICATION_DIR'); + $root = is_string($configured) && $configured !== '' + ? $configured + : dirname(__DIR__, 3) . '/specification'; + $path = $root . '/' . ltrim($relative, '/'); + if (!is_file($path)) { + throw new RuntimeException('Pinned specification fixture is missing: ' . $relative); + } + return $path; + } + + /** @return array */ + public static function load(string $relative): array + { + $decoded = json_decode( + (string) file_get_contents(self::path($relative)), + true, + flags: JSON_THROW_ON_ERROR, + ); + if (!is_array($decoded)) { + throw new RuntimeException('Specification manifest must decode to an object.'); + } + return $decoded; + } +} +~~~ + +- [ ] **Step 4: Extend the pin test to cover transformation schemas and manifests** + +Assert that CapabilityDeclaration::SPECIFICATION_RELEASE remains v0.3.0, the commit remains cd8f198c68b849eb8ed018a894670a0904c2181d, and all five CI specification checkouts use that commit. Then assert the three used schema files (Plan 0.1, Plan 0.2, and Frontend 0.2) and five manifests used by this plan exist below the resolved root. + +- [ ] **Step 5: Run focused and release tests** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/ManifestInventoryTest.php tests/Release/SpecificationPinTest.php + +Expected: PASS with exact inventory counts and no skipped tests. + +- [ ] **Step 6: Commit** + +~~~bash +git add tests/Support/SpecificationManifest.php tests/Transformation/Conformance/ManifestInventoryTest.php tests/Release/SpecificationPinTest.php +git commit -m "test: load official transformation conformance manifests" +~~~ + +--- + +### Task 2: Implement the official Plan 0.1 model and canonical codec + +**Files:** +- Create: src/Transformation/Diagnostic/TransformationDiagnostic.php +- Create: src/Transformation/Diagnostic/TransformationFailure.php +- Create: src/Transformation/Diagnostic/SourceSpan.php +- Create: src/Transformation/Plan/PlanContract.php +- Create: src/Transformation/Plan/TransformationPlan.php +- Create: src/Transformation/Plan/PlanCodec.php +- Create: src/Transformation/Plan/Operation.php +- Create: src/Transformation/Plan/TargetMode.php +- Create: src/Transformation/Plan/Value/TypedValue.php +- Create: src/Transformation/Plan/Value/Binary64Value.php +- Create: src/Transformation/Plan/Value/StringValue.php +- Create: src/Transformation/Plan/Recode/Match.php +- Create: src/Transformation/Plan/Recode/ExactMatch.php +- Create: src/Transformation/Plan/Recode/RangeMatch.php +- Create: src/Transformation/Plan/Recode/SystemMissingMatch.php +- Create: src/Transformation/Plan/Recode/Result.php +- Create: src/Transformation/Plan/Recode/LiteralResult.php +- Create: src/Transformation/Plan/Recode/CopyResult.php +- Create: src/Transformation/Plan/Recode/SystemMissingResult.php +- Create: src/Transformation/Plan/Recode/RecodeRule.php +- Create: src/Transformation/Plan/Operation/RecodeOperation.php +- Create: src/Transformation/Plan/Operation/SetVariableLabelOperation.php +- Create: src/Transformation/Plan/Operation/ReplaceValueLabelsOperation.php +- Create: src/Transformation/Plan/Operation/ValueLabel.php +- Create: tests/Transformation/Conformance/TransformationPlan01Test.php + +**Interfaces:** +- Consumes: CanonicalJson::encode(array): string and SpecificationManifest::load(string): array. +- Produces: PlanCodec::fromArray(array): TransformationPlan, PlanCodec::fromJson(string): TransformationPlan, PlanCodec::canonicalJson(TransformationPlan): string, and PlanCodec::hash(TransformationPlan): string. + +- [ ] **Step 1: Write the failing Plan 0.1 golden test** + +~~~php +#[DataProvider('cases')] +public function testOfficialPlan01(array $case): void +{ + if (is_string($case['expected_error'])) { + try { + (new PlanCodec())->fromArray($case['plan']); + self::fail('Expected ' . $case['expected_error']); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + return; + } + + $plan = (new PlanCodec())->fromArray($case['plan']); + self::assertSame($case['plan'], $plan->canonicalArray()); + self::assertSame($case['expected_plan_hash'], (new PlanCodec())->hash($plan)); +} +~~~ + +- [ ] **Step 2: Run the test and verify it is red** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/TransformationPlan01Test.php + +Expected: FAIL because PlanCodec and the official plan model do not exist. + +- [ ] **Step 3: Add stable diagnostics and the contract root** + +~~~php +enum PlanContract: string +{ + case V01 = 'openstatspec-transformation-plan-v0.1'; + case V02 = 'openstatspec-transformation-plan-v0.2'; +} + +interface Operation +{ + /** @return array */ + public function canonicalArray(): array; + public function minimumContract(): PlanContract; +} + +final readonly class TransformationPlan +{ + /** @param non-empty-list $operations */ + public function __construct( + public PlanContract $contract, + public string $inputAlias, + public array $operations, + ) {} + + /** @return array{contract:string,input_alias:string,operations:list>} */ + public function canonicalArray(): array + { + return [ + 'contract' => $this->contract->value, + 'input_alias' => $this->inputAlias, + 'operations' => array_map( + static fn(Operation $operation): array => $operation->canonicalArray(), + $this->operations, + ), + ]; + } +} +~~~ + +The non-final TransformationFailure stores a non-empty list of TransformationDiagnostic and exposes diagnosticCode() as the first stable code. TransformationFailure::at(string $code, string $path, string $message, ?SourceSpan $span = null) creates a one-diagnostic failure. Diagnostic values carry code, JSON path, message, and the optional shared SourceSpan without credentials or row values; SpssSyntaxException extends this shared failure type in Task 5. + +- [ ] **Step 4: Implement exact typed values and 0.1 operation shapes** + +Use immutable classes whose canonicalArray methods emit exactly these keys: + +~~~php +new RecodeOperation( + source: 'q1', + target: 'q1', + targetMode: TargetMode::Replace, + rules: [new RecodeRule(new ExactMatch(Binary64Value::fromBits('3ff0000000000000')), new LiteralResult(Binary64Value::fromBits('0000000000000000')))], + unmatched: new CopyResult(), +); + +// canonicalArray() +[ + 'op' => 'recode', + 'source' => 'q1', + 'target' => 'q1', + 'target_mode' => 'replace', + 'rules' => [['match' => ['kind' => 'exact', 'value' => ['type' => 'binary64', 'bits' => '3ff0000000000000']], 'result' => ['kind' => 'literal', 'value' => ['type' => 'binary64', 'bits' => '0000000000000000']]]], + 'unmatched' => ['kind' => 'copy'], +]; +~~~ + +SetVariableLabelOperation emits op, variable, label. ReplaceValueLabelsOperation emits op, variable, and ordered labels; each label emits value then label. Binary64Value accepts exactly 16 lowercase hexadecimal bits, rejects exponent 0x7ff and negative zero with plan_schema_invalid, and StringValue preserves exact Unicode. + +- [ ] **Step 5: Implement strict recursive PlanCodec decoding** + +PlanCodec must reject missing/extra members, empty aliases/operations, unknown operations, malformed typed values, descending ranges, and duplicate exact typed label values. Map the two semantic 0.1 failures to invalid_numeric_range and duplicate_value_label; all structural failures map to plan_schema_invalid. + +~~~php +public function canonicalJson(TransformationPlan $plan): string +{ + return CanonicalJson::encode($plan->canonicalArray()); +} + +public function hash(TransformationPlan $plan): string +{ + return hash('sha256', $this->canonicalJson($plan)); +} +~~~ + +- [ ] **Step 6: Run Plan 0.1 goldens and canonical regressions** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/TransformationPlan01Test.php tests/Transformation/Canonical/TransformationPlanTest.php + +Expected: all four official 0.1 cases pass; legacy tests remain green because their namespace has not been removed yet. + +- [ ] **Step 7: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Diagnostic src/Transformation/Plan tests/Transformation/Conformance/TransformationPlan01Test.php +git commit -m "feat: add official transformation plan 0.1 model" +~~~ + +--- + +### Task 3: Extend the model and codec to Plan 0.2 + +**Files:** +- Create: src/Transformation/Plan/Expression/Operand.php +- Create: src/Transformation/Plan/Expression/LiteralOperand.php +- Create: src/Transformation/Plan/Expression/VariableOperand.php +- Create: src/Transformation/Plan/Expression/Predicate.php +- Create: src/Transformation/Plan/Expression/Comparison.php +- Create: src/Transformation/Plan/Expression/BooleanPredicate.php +- Create: src/Transformation/Plan/Operation/AssignOperation.php +- Create: src/Transformation/Plan/Operation/ConditionalAssignOperation.php +- Create: src/Transformation/Plan/Operation/SetFormatOperation.php +- Create: src/Transformation/Plan/Operation/SetMeasurementLevelOperation.php +- Create: src/Transformation/Plan/Operation/ExecuteOperation.php +- Modify: src/Transformation/Plan/PlanCodec.php +- Create: tests/Transformation/Conformance/TransformationPlan02Test.php +- Create: tests/Transformation/Plan/BooleanPredicateTest.php + +**Interfaces:** +- Consumes: official Plan 0.1 model and codec. +- Produces: typed 0.2 operations and predicates accepted by the same PlanCodec API. + +- [ ] **Step 1: Write the failing 26-case Plan 0.2 golden test** + +Reuse the Task 2 data-provider structure with conformance/transformation-plan-0.2.json. Add a focused assertion that a direct BooleanPredicate child with the same operator throws noncanonical_boolean_shape. + +- [ ] **Step 2: Run the tests and verify unsupported 0.2 operations fail** + +Run: vendor/bin/phpunit tests/Transformation/Conformance/TransformationPlan02Test.php tests/Transformation/Plan/BooleanPredicateTest.php + +Expected: FAIL on the first assign or conditional_assign case. + +- [ ] **Step 3: Implement operands and predicates** + +~~~php +final readonly class Comparison implements Predicate +{ + public function __construct( + public Operand $left, + public string $operator, + public Operand $right, + ) { + if (!in_array($operator, ['=', '<', '<=', '>', '>='], true)) { + throw TransformationFailure::at('plan_schema_invalid', '$.operator', 'Unsupported comparison operator.'); + } + } + + public function canonicalArray(): array + { + return [ + 'expression' => 'comparison', + 'left' => $this->left->canonicalArray(), + 'operator' => $this->operator, + 'right' => $this->right->canonicalArray(), + ]; + } +} +~~~ + +BooleanPredicate accepts and/or plus at least two ordered operands. Reject a direct BooleanPredicate child with the same operator as noncanonical_boolean_shape; never reorder, deduplicate, simplify, distribute, or constant-fold. + +- [ ] **Step 4: Implement the five 0.2 operations** + +AssignOperation emits op, target, target_mode, value. ConditionalAssignOperation emits op, condition, target, value. SetFormatOperation emits op, variable, family F, width, decimals. SetMeasurementLevelOperation emits op, variable, level. ExecuteOperation emits only op. + +Validation rules: + +~~~php +if ($operation instanceof SetFormatOperation + && ($operation->width < 1 || $operation->width > 40 + || $operation->decimals < 0 || $operation->decimals > 16 + || ($operation->decimals > 0 && $operation->width < $operation->decimals + 2))) { + throw TransformationFailure::at('invalid_format', $path, 'Invalid SPSS F format.'); +} +~~~ + +Reject string literal operands in assign, conditional assignment, or comparison with expression_type_unsupported; reject names beginning with double underscore using reserved_target_name where the name is a target. + +- [ ] **Step 5: Enforce contract selection** + +PlanContract::V01 accepts only operations whose minimumContract is V01. PlanContract::V02 accepts both 0.1 and 0.2 operations. Decoding never promotes or rewrites a declared 0.1 plan. + +- [ ] **Step 6: Run all official plan tests** + +Run: vendor/bin/phpunit tests/Transformation/Conformance tests/Transformation/Plan + +Expected: 4 Plan 0.1 and 26 Plan 0.2 cases pass with exact hashes and stable expected errors. + +- [ ] **Step 7: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Plan tests/Transformation/Conformance/TransformationPlan02Test.php tests/Transformation/Plan +git commit -m "feat: implement transformation plan 0.2 operations" +~~~ + +--- + +### Task 4: Convert SPSS finite decimals to binary64 exactly + +**Files:** +- Modify: composer.json +- Modify: composer.lock +- Create: src/Frontend/Spss/Number/DecimalBinary64.php +- Create: tests/Frontend/Spss/DecimalBinary64Test.php + +**Interfaces:** +- Consumes: brick/math ^0.19 BigInteger arithmetic. +- Produces: DecimalBinary64::bits(string): string returning exactly 16 lowercase hexadecimal digits. + +- [ ] **Step 1: Add the dependency and failing boundary tests** + +Run: composer require brick/math:^0.19 --no-interaction + +~~~php +#[DataProvider('tokens')] +public function testExactBits(string $token, string $bits): void +{ + self::assertSame($bits, DecimalBinary64::bits($token)); +} + +public static function tokens(): iterable +{ + yield 'negative zero' => ['-0', '0000000000000000']; + yield 'underflowed zero' => ['-1e-9999', '0000000000000000']; + yield 'minimum subnormal' => ['4.9406564584124654417656879286822137236505980e-324', '0000000000000001']; + yield 'one' => ['1', '3ff0000000000000']; + yield 'tie rounds even' => ['1.00000000000000011102230246251565404236316680908203125', '3ff0000000000000']; +} +~~~ + +Add rejection tests for +1, .5, 1., 01, NaN, Infinity, and overflow 1e9999. + +- [ ] **Step 2: Run the test and verify it is red** + +Run: vendor/bin/phpunit tests/Frontend/Spss/DecimalBinary64Test.php + +Expected: FAIL because DecimalBinary64 does not exist. + +- [ ] **Step 3: Implement grammar and exact rational construction** + +Parse the token with this exact pattern: + +~~~php +private const FINITE = '/\A(-?)(0|[1-9][0-9]*)(?:\.([0-9]+))?(?:[eE]([+-]?[0-9]+))?\z/D'; +~~~ + +Build coefficient and decimal exponent with BigInteger only. Represent the absolute value as numerator/denominator where positive decimal exponents multiply the numerator by 10^n and negative exponents multiply the denominator by 10^n. + +- [ ] **Step 4: Implement ties-to-even rounding** + +~~~php +private static function roundRatio(BigInteger $numerator, BigInteger $denominator): BigInteger +{ + [$quotient, $remainder] = $numerator->quotientAndRemainder($denominator); + $twice = $remainder->multipliedBy(2); + $comparison = $twice->compareTo($denominator); + if ($comparison > 0 || ($comparison === 0 && $quotient->isOdd())) { + return $quotient->plus(1); + } + return $quotient; +} +~~~ + +Find floor(log2(numerator/denominator)) by integer comparison. For normal values round numerator multiplied by 2^(52-e); for subnormals round numerator multiplied by 2^1074. Handle significand carry, overflow, sign bit, minimum normal, and zero before rendering a 64-bit hexadecimal word. + +- [ ] **Step 5: Run focused and frontend-number tests** + +Run: vendor/bin/phpunit tests/Frontend/Spss/DecimalBinary64Test.php + +Expected: all accepted tokens produce exact bits and every rejected token throws spss_syntax_error. + +- [ ] **Step 6: Run quality checks and commit** + +Run: composer validate --strict && composer analyse && composer style + +~~~bash +git add composer.json composer.lock src/Frontend/Spss/Number/DecimalBinary64.php tests/Frontend/Spss/DecimalBinary64Test.php +git commit -m "feat: add exact SPSS decimal binary64 conversion" +~~~ + +--- + +### Task 5: Parse the official SPSS Frontend 0.2 grammar + +**Files:** +- Create: src/Frontend/Spss/Request/SpssFrontendRequest.php +- Create: src/Frontend/Spss/Request/InputSchema.php +- Create: src/Frontend/Spss/Request/InputVariable.php +- Create: src/Frontend/Spss/Ast/ComputeStatement.php +- Create: src/Frontend/Spss/Ast/IfStatement.php +- Create: src/Frontend/Spss/Ast/FormatsStatement.php +- Create: src/Frontend/Spss/Ast/FormatTarget.php +- Create: src/Frontend/Spss/Ast/VariableLevelStatement.php +- Create: src/Frontend/Spss/Ast/VariableLevelGroup.php +- Create: src/Frontend/Spss/Ast/ExpressionOperand.php +- Create: src/Frontend/Spss/Ast/LiteralOperand.php +- Create: src/Frontend/Spss/Ast/VariableOperand.php +- Create: src/Frontend/Spss/Ast/Predicate.php +- Create: src/Frontend/Spss/Ast/Comparison.php +- Create: src/Frontend/Spss/Ast/BooleanPredicate.php +- Modify: src/Frontend/Spss/TokenType.php +- Modify: src/Frontend/Spss/Lexer.php +- Modify: src/Frontend/Spss/Parser.php +- Modify: src/Frontend/Spss/SpssSyntaxException.php +- Modify: tests/Frontend/Spss/LexerTest.php +- Modify: tests/Frontend/Spss/ParserTest.php + +**Interfaces:** +- Consumes: exact numeric-token grammar and DecimalBinary64 rejection behavior. +- Produces: Parser::parse(string): Program containing ordered 0.1/0.2 statement AST nodes with source spans. + +- [ ] **Step 1: Write failing parser tests for every new command** + +~~~php +$program = (new Parser())->parse( + "COMPUTE target = 0.\n" + . "IF (a = 1 AND b >= 2 OR c < 3) target = a.\n" + . "FORMATS target (F8.2).\n" + . "VARIABLE LEVEL target (SCALE).\n" + . "EXECUTE." +); + +self::assertInstanceOf(ComputeStatement::class, $program->statements[0]); +self::assertInstanceOf(IfStatement::class, $program->statements[1]); +self::assertInstanceOf(FormatsStatement::class, $program->statements[2]); +self::assertInstanceOf(VariableLevelStatement::class, $program->statements[3]); +self::assertInstanceOf(ExecuteStatement::class, $program->statements[4]); +~~~ + +Add rejection tests for IF without outer parentheses, IF ELSE, arithmetic, NOT, string expressions, non-F formats, invalid levels, comments, and unsupported commands. Assert each exception exposes the stable code and start/end source span. + +- [ ] **Step 2: Run parser tests and verify they are red** + +Run: vendor/bin/phpunit tests/Frontend/Spss/LexerTest.php tests/Frontend/Spss/ParserTest.php + +Expected: FAIL on COMPUTE tokenization. + +- [ ] **Step 3: Extend tokens and lexer** + +Add ASCII-case-insensitive keyword tokens for COMPUTE, IF, AND, OR, FORMATS, VARIABLE LEVEL, NOMINAL, ORDINAL, and SCALE; add equals and ordered comparison tokens. Preserve the exact source offset plus 1-based line/column at both token boundaries. + +- [ ] **Step 4: Implement precedence and flattening in the parser** + +~~~php +private function parsePredicate(): Predicate +{ + return $this->parseOr(); +} + +private function merge(string $operator, Predicate $left, Predicate $right): BooleanPredicate +{ + $operands = $left instanceof BooleanPredicate && $left->operator === $operator + ? $left->operands + : [$left]; + array_push($operands, ...($right instanceof BooleanPredicate && $right->operator === $operator + ? $right->operands + : [$right])); + return new BooleanPredicate($operator, $operands, SourceSpan::cover($left->span, $right->span)); +} +~~~ + +parseOr calls parseAnd; parseAnd calls comparison; comparison parses two operands. Parentheses recurse but same-operator nodes are flattened in source order. IF requires one outer parenthesized predicate. + +- [ ] **Step 5: Implement grouped metadata commands and stable failures** + +FORMATS emits ordered FormatTarget pairs with uppercase F and integer width/decimals. VARIABLE LEVEL emits ordered groups and lowercase levels. Invalid productions throw spss_syntax_error; recognized but unsupported commands throw unsupported_spss_command. + +- [ ] **Step 6: Run lexer/parser regression tests** + +Run: vendor/bin/phpunit tests/Frontend/Spss/LexerTest.php tests/Frontend/Spss/ParserTest.php + +Expected: all old official-subset parsing and new 0.2 parsing tests pass. + +- [ ] **Step 7: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Frontend/Spss tests/Frontend/Spss/LexerTest.php tests/Frontend/Spss/ParserTest.php +git commit -m "feat: parse SPSS frontend 0.2 commands" +~~~ + +--- + +### Task 6: Bind and compile all official Frontend 0.2 cases + +**Files:** +- Create: src/Frontend/Spss/SpssCompilationResult.php +- Create: src/Frontend/Spss/Binding/SchemaState.php +- Create: src/Frontend/Spss/Binding/BoundCompute.php +- Create: src/Frontend/Spss/Binding/BoundConditionalAssign.php +- Create: src/Frontend/Spss/Binding/BoundFormat.php +- Create: src/Frontend/Spss/Binding/BoundMeasurementLevel.php +- Create: src/Frontend/Spss/Binding/BoundExecute.php +- Modify: src/Frontend/Spss/Binder.php +- Modify: src/Frontend/Spss/Compiler.php +- Modify: src/Frontend/Spss/SpssCompiler.php +- Create: tests/Frontend/Spss/Conformance/SpssFrontend02Test.php +- Modify: tests/Frontend/Spss/SpssCompilerTest.php + +**Interfaces:** +- Consumes: SpssFrontendRequest, Parser AST, DecimalBinary64, and official plan types. +- Produces: SpssFrontendRequest::sourceHash(): string and SpssCompiler::compile(SpssFrontendRequest): SpssCompilationResult with public readonly plan and sourceHash. + +- [ ] **Step 1: Write failing manifest-driven frontend tests** + +~~~php +$request = SpssFrontendRequest::fromArray($case['request']); +self::assertSame($case['expected_source_hash'], $request->sourceHash()); +try { + $result = (new SpssCompiler())->compile($request); +} catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + return; +} + +self::assertNull($case['expected_error']); +self::assertSame($case['expected_source_hash'], $result->sourceHash); +self::assertSame($case['expected_plan_hash'], (new PlanCodec())->hash($result->plan)); +if (isset($case['expected_plan_contract'])) { + self::assertSame($case['expected_plan_contract'], $result->plan->contract->value); +} +~~~ + +Load the official 0.2 frontend manifest; references to exact 0.1 plans are checked against conformance/transformation-plan-0.1.json rather than copied expectations. + +- [ ] **Step 2: Run the frontend conformance tests and verify red** + +Run: vendor/bin/phpunit tests/Frontend/Spss/Conformance + +Expected: FAIL because SpssFrontendRequest and the new compiler signature do not exist. + +- [ ] **Step 3: Validate the request and source hash** + +SpssFrontendRequest::fromArray accepts exactly contract, input_alias, input_schema, and source_text; contract must equal openstatspec-spss-syntax-frontend-v0.2. InputSchema requires a non-empty ordered variables array; InputVariable validates name, storage_kind, and optional label/value-label/format/measurement fields. + +~~~php +$normalized = str_replace(["\r\n", "\r"], "\n", $request->sourceText); +$sourceHash = hash('sha256', $normalized); +~~~ + +sourceHash() performs this normalization independently of parsing so failed compilations still have the exact conformance source identity. + +- [ ] **Step 4: Implement case-insensitive evolving schema binding** + +SchemaState indexes ASCII-lowercase names while retaining exact catalog spelling. RECODE pairs resolve against pre-command state; targets created by an earlier command become available to later commands. COMPUTE creates or replaces a numeric target. IF requires an existing numeric target. FORMATS requires numeric targets; VARIABLE LEVEL accepts numeric or string targets. + +Map binding failures exactly to unknown_variable, conditional_target_missing, type_mismatch, expression_type_unsupported, invalid_format, reserved_target_name, duplicate_value_label, mixed_result_types, system_missing_for_string, and string_target_requires_declaration. + +- [ ] **Step 5: Compile ordered operations and select the plan contract** + +~~~php +$contract = array_any( + $operations, + static fn(Operation $operation): bool => $operation->minimumContract() === PlanContract::V02, +) ? PlanContract::V02 : PlanContract::V01; + +$plan = new TransformationPlan($contract, $request->inputAlias, $operations); +return new SpssCompilationResult($plan, $sourceHash); +~~~ + +Compile DecimalBinary64 bits directly into Binary64Value. Preserve command, group, variable, rule, and boolean operand order. EXECUTE always remains an ExecuteOperation. + +- [ ] **Step 6: Run all 44 official Frontend 0.2 cases** + +Run: vendor/bin/phpunit tests/Frontend/Spss/Conformance tests/Frontend/Spss/SpssCompilerTest.php + +Expected: all 44 Frontend 0.2 cases pass with exact plan/source hashes and stable errors, including the required exact Plan 0.1 outputs. + +- [ ] **Step 7: Run plan plus frontend regressions and commit** + +Run: vendor/bin/phpunit tests/Transformation/Conformance tests/Frontend/Spss +Run: composer analyse && composer style + +~~~bash +git add src/Frontend/Spss tests/Frontend/Spss +git commit -m "feat: compile official SPSS frontend 0.2 plans" +~~~ + +--- + +### Task 7: Add the explicit transformation audit migration and apply request + +**Files:** +- Create: src/Transformation/Execution/InPlaceApplyRequest.php +- Create: src/Transformation/Audit/TransformationAuditMigrator.php +- Create: src/Transformation/Audit/TransformationAuditWriter.php +- Modify: src/Sql/CatalogOwnership.php +- Modify: src/Sql/NormativeCatalog.php +- Modify: src/Spss/SpssAdapter.php +- Create: tests/Transformation/Audit/TransformationAuditMigratorTest.php +- Modify: tests/Integration/ServerCatalogMigrationTest.php + +**Interfaces:** +- Consumes: TransformationPlan and PlanCodec. +- Produces: InPlaceApplyRequest(plan, inputAlias, datasetId, sourceHash, actor, expectedBranch, expectedHead) and TransformationAuditWriter::succeed(...): string. + +- [ ] **Step 1: Write failing SQLite migration and request-validation tests** + +~~~php +$request = new InPlaceApplyRequest( + plan: $plan, + inputAlias: 'parent', + datasetId: '11111111-1111-4111-8111-111111111111', + sourceHash: str_repeat('a', 64), + actor: 'conformance-runner', +); +self::assertSame('parent', $request->inputAlias); + +(new TransformationAuditMigrator($pdo))->migrate(); +self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $this->acceptedContracts($pdo), +); +~~~ + +Also create an old 0.1-only transformation_apply table with one row, migrate it, and assert the row and logical table name survive with no backup table residue. + +- [ ] **Step 2: Run tests and verify red** + +Run: vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php + +Expected: FAIL because the request and migrator do not exist. + +- [ ] **Step 3: Implement fail-fast apply request validation** + +The constructor rejects alias mismatch, malformed dataset UUID, malformed source hash, and empty actor before any database call. expectedBranch and expectedHead must be both null or both non-empty. + +~~~php +if ($inputAlias !== $plan->inputAlias) { + throw TransformationFailure::at('unknown_input_alias', '$.input_alias', 'Apply binding does not match the plan alias.'); +} +if (trim($actor) === '') { + throw TransformationFailure::at('actor_required', '$.actor', 'A non-empty actor is required.'); +} +~~~ + +- [ ] **Step 4: Implement schema version 4 migration** + +Bump CatalogOwnership::SCHEMA_VERSION from 3 to 4. TransformationAuditMigrator creates the exact normative columns and checks from sql/transformation-plan-profile-schema.sql using profile-specific UUID/text types. + +For SQLite 0.1-only tables: begin a native transaction, create transformation_apply_v04, copy compact audit rows, drop the old table, rename the new table to transformation_apply, verify foreign keys, and commit. On failure roll back so the old table remains. PostgreSQL alters the contract check transactionally. MySQL/MariaDB/Dolt replace the check in the explicit migration workflow before applies; never perform this DDL inside an apply. + +- [ ] **Step 5: Implement the success-only audit writer** + +~~~php +public function succeed( + InPlaceApplyRequest $request, + DatasetBinding $dataset, + string $databaseProfile, + ?DoltEvidence $before, + ?DoltEvidence $after, +): string +~~~ + +Insert exactly one succeeded row inside the open apply transaction after operations and post-Dolt checks but before commit. Store canonical plan JSON, plan/source hashes, actor, operation count, target identity, timestamps, and nullable Dolt fields. Do not write a failed row after rollback because the 0.2 binding requires failed applies to leave audit unchanged. + +- [ ] **Step 6: Wire explicit catalog migration** + +SpssAdapter::migrateCatalog calls TransformationAuditMigrator after NormativeCatalog::createTables and before CatalogOwnership::markCurrentVersion. Fresh catalogs receive migration markers 1 through 4 exactly once. + +- [ ] **Step 7: Run SQLite and configured server migration tests** + +Run: vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php tests/Integration/ServerCatalogMigrationTest.php + +Expected: SQLite always passes; configured servers pass; unconfigured services are the only skips. + +- [ ] **Step 8: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Audit src/Transformation/Execution/InPlaceApplyRequest.php src/Sql/CatalogOwnership.php src/Sql/NormativeCatalog.php src/Spss/SpssAdapter.php tests/Transformation/Audit tests/Integration/ServerCatalogMigrationTest.php +git commit -m "feat: add official transformation apply audit" +~~~ + +--- + +### Task 8: Execute official plans in place on SQLite and PostgreSQL + +**Files:** +- Create: src/Transformation/Execution/PlanPreflight.php +- Create: src/Transformation/Execution/BoundPlan.php +- Create: src/Transformation/Execution/BoundOperation.php +- Create: src/Transformation/Execution/SqlPredicateCompiler.php +- Create: src/Transformation/Execution/InPlaceOperationExecutor.php +- Modify: src/Transformation/Execution/InPlaceTransformationExecutor.php +- Modify: src/Transformation/Execution/ExecutionResult.php +- Modify: src/Transformation/Execution/DatasetBinding.php +- Modify: src/Transformation/Execution/VariableBinding.php +- Modify: tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +- Modify: tests/Integration/InPlaceTransformationServiceTest.php +- Create: tests/Integration/OfficialInPlaceTransformation01Test.php +- Create: tests/Integration/OfficialInPlaceTransformation02Test.php + +**Interfaces:** +- Consumes: InPlaceApplyRequest, official plan operations, Connection profile, and TransformationAuditWriter. +- Produces: InPlaceTransformationExecutor::execute(InPlaceApplyRequest): ExecutionResult. + +- [ ] **Step 1: Add failing SQLite official binding cases** + +In OfficialInPlaceTransformation02Test, materialize fixtures for sqlite-create-target-atomic-success and sqlite-inequality-boundary-semantics from the manifest. Snapshot dataset count, persistent data-table count, dataset/table identity, case order/count, variables, metadata, and audit count before apply. + +~~~php +$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($request); +self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); +self::assertSame($before['dataset_count'], $this->datasetCount($pdo)); +self::assertSame($before['persistent_data_table_count'], $this->persistentDataTableCount($pdo)); +self::assertSame($case['after']['rows'], $this->rows($pdo, $table)); +~~~ + +Add the rollback probe that injects failure after catalog target creation but before audit and asserts no physical column, catalog variable, row change, or audit row remains. + +- [ ] **Step 2: Run focused tests and verify red** + +Run: vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation02Test.php --filter sqlite + +Expected: FAIL because the executor still accepts the legacy plan signature. + +- [ ] **Step 3: Implement complete preflight** + +PlanPreflight resolves the alias-bound dataset and ordered catalog variables, then simulates every operation against an evolving schema. It validates unknown variables, create/replace target state, numeric expression types, recode result type, format validity, backend create capability, and audit schema readiness before transaction start. + +Return an immutable BoundPlan containing the unchanged plan plus per-operation source/target physical bindings. Do not store row values or create database objects during preflight. + +- [ ] **Step 4: Implement deterministic SQL predicate compilation** + +~~~php +private function comparison(Comparison $comparison, BoundSchema $schema, array &$parameters): string +{ + $left = $this->operand($comparison->left, $schema, $parameters); + $right = $this->operand($comparison->right, $schema, $parameters); + return '(' . $left . ' ' . $comparison->operator . ' ' . $right . ')'; +} + +private function boolean(BooleanPredicate $predicate, BoundSchema $schema, array &$parameters): string +{ + return '(' . implode( + ' ' . strtoupper($predicate->operator) . ' ', + array_map(fn(Predicate $child): string => $this->predicate($child, $schema, $parameters), $predicate->operands), + ) . ')'; +} +~~~ + +Literal operands append a PDO parameter; variable operands use the profile-quoted physical column. SQL NULL comparisons naturally become UNKNOWN; conditional UPDATE changes only rows whose WHERE predicate is TRUE. + +- [ ] **Step 5: Implement ordered operation execution** + +InPlaceOperationExecutor receives one BoundOperation at a time so the public InPlaceTransformationExecutor remains a small transaction orchestrator. Use direct UPDATE for existing recode/assign, ALTER TABLE plus catalog insert plus UPDATE for atomic create, conditional UPDATE WHERE predicate, exact catalog updates for labels/format/measurement, and no SQL for execute. Each operation observes prior operations in the same transaction. + +SetFormatOperation updates print and write family/width/decimals only. SetMeasurementLevelOperation updates measurement_level only. New variables receive a unique UUID, next ordinal, nullable numeric column, and no label/value-label/missing/display/measurement metadata. + +- [ ] **Step 6: Put audit in the same transaction** + +Begin one native transaction after preflight, apply every operation, run post-context checks, insert the success audit, then commit. Any Throwable rolls back; do not start OperationJournal for transformations. + +- [ ] **Step 7: Add PostgreSQL atomic-create evidence** + +Extend OfficialInPlaceTransformation02Test to run the SQLite create-target case against OPENSTATSPEC_PG_DSN with PostgreSQL identifiers and catalog fixture setup. Assert schema, rows, catalog, audit, and rollback probe share the transaction and leave no extra persistent relation. + +- [ ] **Step 8: Run executor regressions** + +Run: vendor/bin/phpunit tests/Transformation/Execution/InPlaceTransformationExecutorTest.php tests/Integration/InPlaceTransformationServiceTest.php tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php + +Expected: SQLite cases pass, PostgreSQL passes when configured, and only unconfigured services skip. + +- [ ] **Step 9: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Execution tests/Transformation/Execution tests/Integration/InPlaceTransformationServiceTest.php tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php +git commit -m "feat: execute official plans atomically in place" +~~~ + +--- + +### Task 9: Enforce MySQL-family and controlled Dolt bindings + +**Files:** +- Modify: src/Transformation/Execution/DoltGuard.php +- Modify: src/Transformation/Execution/DoltEvidence.php +- Modify: src/Transformation/Execution/PdoDoltEvidenceReader.php +- Modify: src/Transformation/Execution/PlanPreflight.php +- Modify: src/Transformation/Execution/InPlaceTransformationExecutor.php +- Modify: tests/Transformation/Execution/DoltGuardTest.php +- Modify: tests/Transformation/Execution/DoltHeadGuardTest.php +- Modify: tests/Integration/OfficialInPlaceTransformation01Test.php +- Modify: tests/Integration/OfficialInPlaceTransformation02Test.php +- Modify: tests/Integration/InPlaceTransformationServiceTest.php + +**Interfaces:** +- Consumes: InPlaceApplyRequest expected branch/HEAD and Connection::profileName. +- Produces: DoltGuard::beforeExecution(InPlaceApplyRequest): DoltEvidence and afterExecution(InPlaceApplyRequest, DoltEvidence): DoltEvidence. + +- [ ] **Step 1: Add failing backend rejection tests** + +For MySQL, MariaDB, and Dolt create-target manifest cases, snapshot data, catalog, audit, dataset count, and persistent table count. Execute the create plan and assert schema_change_not_atomic plus mutation_started=false and exact before/after equality. Include the official 0.1 MySQL create-target rejection as well as the 0.2 cases. + +- [ ] **Step 2: Add failing Dolt context tests** + +Cover empty actor, missing expected context, branch mismatch, HEAD mismatch, dirty working set, and injected post-mutation context change. Assert exact codes: actor_required, dolt_context_required, dolt_branch_mismatch, dolt_head_mismatch, dolt_working_set_dirty, and dolt_context_changed. + +- [ ] **Step 3: Run focused tests and verify red** + +Run: vendor/bin/phpunit tests/Transformation/Execution/DoltGuardTest.php tests/Transformation/Execution/DoltHeadGuardTest.php tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php + +Expected: FAIL because current DoltGuard compares only its own before/after observations. + +- [ ] **Step 4: Enforce profile capabilities during preflight** + +~~~php +if ($operation->createsTarget() && !$connection->profile->ddlAtomic()) { + throw TransformationFailure::at( + 'schema_change_not_atomic', + '$.operations[' . $index . '].target_mode', + $connection->profileName . ' requires a pre-provisioned target.', + ); +} +~~~ + +This runs before beginning a transaction or writing audit. MySQL, MariaDB, and Dolt never attempt compensating transformation cleanup. + +- [ ] **Step 5: Enforce caller-supplied Dolt context** + +Before mutation, require expectedBranch and expectedHead, compare them to active_branch() and dolt_hashof('HEAD'), and require dolt_status to be empty. After operations but before audit/commit, re-read branch and HEAD and compare them to both the request and before evidence. + +Never call a Dolt mutation procedure. A successful apply leaves one inspectable dirty working set and equal before/after HEAD in the audit. + +- [ ] **Step 6: Run all official 0.1 and 0.2 in-place cases** + +Run: vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php + +Expected: all 6 official 0.1 and 11 official 0.2 cases for configured services pass; only absent service configurations skip. + +- [ ] **Step 7: Run the existing service evidence test** + +Run: vendor/bin/phpunit tests/Integration/InPlaceTransformationServiceTest.php + +Expected: every existing-target service case preserves the global no-new-dataset/table invariant; live Dolt evidence proves no commit and stable branch/HEAD. + +- [ ] **Step 8: Run static checks and commit** + +Run: composer analyse && composer style + +~~~bash +git add src/Transformation/Execution tests/Transformation/Execution tests/Integration/OfficialInPlaceTransformation01Test.php tests/Integration/OfficialInPlaceTransformation02Test.php tests/Integration/InPlaceTransformationServiceTest.php +git commit -m "feat: enforce official SQL and Dolt apply context" +~~~ + +--- + +### Task 10: Remove the legacy API and publish conformance claims + +**Files:** +- Delete: src/Transformation/Model +- Delete: src/Transformation/Validation +- Delete: src/Frontend/Spss/Binding/BoundCreateVariable.php +- Delete: src/Frontend/Spss/Binding/BoundDeleteVariable.php +- Delete: src/Frontend/Spss/Ast/StringStatement.php +- Delete: src/Frontend/Spss/Ast/DeleteVariablesStatement.php +- Delete: tests/Transformation/Canonical/TransformationPlanTest.php +- Modify: tests/Frontend/Spss/SpssCompilerTest.php +- Modify: tests/Core/CapabilityDeclarationTest.php +- Modify: src/Core/CapabilityDeclaration.php +- Modify: README.md +- Modify: docs/architecture.md +- Modify: docs/transformations.md +- Modify: docs/release-readiness.md +- Modify: CHANGELOG.md +- Modify: .github/workflows/ci.yml + +**Interfaces:** +- Consumes: completed official model, frontend, audit, and executor. +- Produces: only the official public API and truthful machine-readable conformance claims. + +- [ ] **Step 1: Write failing capability and legacy-absence tests** + +~~~php +$contracts = $declaration['transformation_contracts']; +self::assertSame('official_conformant', $contracts['status']); +self::assertSame( + ['openstatspec-transformation-plan-v0.1', 'openstatspec-transformation-plan-v0.2'], + $contracts['official_plan_contracts'], +); +self::assertSame( + ['openstatspec-spss-syntax-frontend-v0.2'], + $contracts['official_frontend_contracts'], +); +self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $contracts['official_binding_contracts'], +); +self::assertArrayNotHasKey('legacy_plan_contracts', $contracts); +self::assertFalse(class_exists(OpenStatSpec\Transformation\Model\TransformationPlan::class)); +~~~ + +- [ ] **Step 2: Run capability tests and verify red** + +Run: vendor/bin/phpunit tests/Core/CapabilityDeclarationTest.php + +Expected: FAIL because the declaration still reports legacy_contract_only. + +- [ ] **Step 3: Migrate every remaining caller and delete the legacy tree** + +Search: + +~~~bash +rg -n "Transformation\\Model|Transformation\\Validation|openstatspec-transformation-plan-v1|CreateVariableOperation|DeleteVariableOperation|compileForDataset|compile\(" src tests README.md docs +~~~ + +Update remaining fixtures to SpssFrontendRequest, SpssCompilationResult, PlanCodec, and InPlaceApplyRequest. Delete legacy model/validator classes and the non-standard STRING/DELETE frontend path. The final search must return no production or documentation reference to the package-local contract or removed operations. + +- [ ] **Step 4: Update capabilities and migration documentation** + +Document the exact new flow: + +~~~php +$compiled = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray($request)); +$apply = new InPlaceApplyRequest( + plan: $compiled->plan, + inputAlias: 'parent', + datasetId: $datasetId, + sourceHash: $compiled->sourceHash, + actor: 'analyst@example.org', + expectedBranch: $branch, + expectedHead: $head, +); +$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($apply); +~~~ + +README and migration notes state that v0.6.0 removes openstatspec-transformation-plan-v1 plus STRING/DELETE transformation commands. Explain pre-provisioning on MySQL/MariaDB/Dolt and that Dolt commits remain caller-owned. + +- [ ] **Step 5: Put the new integration gate in every service job** + +Add OfficialInPlaceTransformation01Test and OfficialInPlaceTransformation02Test to PostgreSQL, MySQL, MariaDB, and Dolt PHPUnit filters without removing existing filters. The normal PHP 8.4/8.5 job continues running full composer check and therefore all plan/frontend fixture tests. + +- [ ] **Step 6: Run every official conformance suite** + +Run: + +~~~bash +vendor/bin/phpunit tests/Transformation/Conformance +vendor/bin/phpunit tests/Frontend/Spss/Conformance +vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php +vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation02Test.php +~~~ + +Expected: exact counts 4 Plan 0.1, 26 Plan 0.2, 44 Frontend 0.2, 6 In-Place 0.1, and 11 In-Place 0.2 cases for configured profiles. + +- [ ] **Step 7: Run the complete local quality gate** + +Run: composer check + +Expected: composer validation, PHP syntax, style, PHPStan, and the full PHPUnit suite pass; skips are limited to unconfigured external database services. + +- [ ] **Step 8: Review forbidden artifacts and API removal** + +Run: + +~~~bash +rg -n "derived_dataset|snapshot|rollback|staging|DOLT_COMMIT|openstatspec-transformation-plan-v1" src tests README.md docs +git diff --check +git status --short +~~~ + +Inspect every match and confirm there is no implementation path that creates a forbidden artifact or commits Dolt. Confirm the legacy contract has no remaining class, capability, example, or test. + +- [ ] **Step 9: Commit** + +~~~bash +git add -A +git commit -m "feat: claim official transformation and SPSS 0.2 conformance" +~~~ + +- [ ] **Step 10: Push and open the review PR** + +Push codex/transformation-plan-spss-0.2, open a draft PR targeting main, and monitor every GitHub Actions job plus Codex review. Address only verified actionable feedback, rerun the affected local gates, and do not mark ready until all service matrices are green. From 9fc8667875732251e8e6a268a7eca2a421c1524b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 14:43:13 +0300 Subject: [PATCH 03/53] test: load official transformation conformance manifests --- tests/Release/SpecificationPinTest.php | 15 ++++++++ tests/Support/SpecificationManifest.php | 37 +++++++++++++++++++ .../Conformance/ManifestInventoryTest.php | 20 ++++++++++ 3 files changed, 72 insertions(+) create mode 100644 tests/Support/SpecificationManifest.php create mode 100644 tests/Transformation/Conformance/ManifestInventoryTest.php diff --git a/tests/Release/SpecificationPinTest.php b/tests/Release/SpecificationPinTest.php index 548b5fe..8e459a2 100644 --- a/tests/Release/SpecificationPinTest.php +++ b/tests/Release/SpecificationPinTest.php @@ -5,12 +5,14 @@ namespace OpenStatSpec\Tests\Release; use OpenStatSpec\Core\CapabilityDeclaration; +use OpenStatSpec\Tests\Support\SpecificationManifest; use PHPUnit\Framework\TestCase; final class SpecificationPinTest extends TestCase { public function testEverySpecificationCheckoutUsesTheStableReleaseCommit(): void { + self::assertSame('v0.3.0', CapabilityDeclaration::SPECIFICATION_RELEASE); self::assertSame( 'cd8f198c68b849eb8ed018a894670a0904c2181d', CapabilityDeclaration::SPECIFICATION_COMMIT, @@ -56,5 +58,18 @@ public function testEverySpecificationCheckoutUsesTheStableReleaseCommit(): void array_fill(0, count($refs), CapabilityDeclaration::SPECIFICATION_COMMIT), $refs, ); + + foreach ([ + 'transformation/plan-0.1.schema.json', + 'transformation/plan-0.2.schema.json', + 'transformation/spss-syntax-frontend-0.2.schema.json', + 'conformance/transformation-plan-0.1.json', + 'conformance/transformation-plan-0.2.json', + 'conformance/spss-syntax-frontend-0.2.json', + 'conformance/in-place-transformation-0.1.json', + 'conformance/in-place-transformation-0.2.json', + ] as $relative) { + self::assertFileExists(SpecificationManifest::path($relative)); + } } } diff --git a/tests/Support/SpecificationManifest.php b/tests/Support/SpecificationManifest.php new file mode 100644 index 0000000..0d77f25 --- /dev/null +++ b/tests/Support/SpecificationManifest.php @@ -0,0 +1,37 @@ + */ + public static function load(string $relative): array + { + $decoded = json_decode( + (string) file_get_contents(self::path($relative)), + true, + flags: JSON_THROW_ON_ERROR, + ); + if (!is_array($decoded)) { + throw new RuntimeException('Specification manifest must decode to an object.'); + } + return $decoded; + } +} diff --git a/tests/Transformation/Conformance/ManifestInventoryTest.php b/tests/Transformation/Conformance/ManifestInventoryTest.php new file mode 100644 index 0000000..d6b8b5a --- /dev/null +++ b/tests/Transformation/Conformance/ManifestInventoryTest.php @@ -0,0 +1,20 @@ + Date: Mon, 17 Aug 2026 14:53:26 +0300 Subject: [PATCH 04/53] feat: add official transformation plan 0.1 model --- src/Transformation/Diagnostic/SourceSpan.php | 29 ++ .../Diagnostic/TransformationDiagnostic.php | 15 + .../Diagnostic/TransformationFailure.php | 31 ++ src/Transformation/Plan/Operation.php | 13 + .../Plan/Operation/RecodeOperation.php | 41 +++ .../Operation/ReplaceValueLabelsOperation.php | 32 ++ .../Operation/SetVariableLabelOperation.php | 27 ++ .../Plan/Operation/ValueLabel.php | 21 ++ src/Transformation/Plan/PlanCodec.php | 320 ++++++++++++++++++ src/Transformation/Plan/PlanContract.php | 11 + src/Transformation/Plan/Recode/CopyResult.php | 14 + src/Transformation/Plan/Recode/ExactMatch.php | 31 ++ .../Plan/Recode/LiteralResult.php | 18 + src/Transformation/Plan/Recode/Match.php | 5 + src/Transformation/Plan/Recode/RangeMatch.php | 25 ++ .../Plan/Recode/RecodeMatch.php | 11 + src/Transformation/Plan/Recode/RecodeRule.php | 19 ++ src/Transformation/Plan/Recode/Result.php | 11 + .../Plan/Recode/SystemMissingMatch.php | 14 + .../Plan/Recode/SystemMissingResult.php | 14 + src/Transformation/Plan/TargetMode.php | 11 + .../Plan/TransformationPlan.php | 28 ++ .../Plan/Value/Binary64Value.php | 55 +++ src/Transformation/Plan/Value/StringValue.php | 21 ++ src/Transformation/Plan/Value/TypedValue.php | 13 + .../Conformance/TransformationPlan01Test.php | 41 +++ 26 files changed, 871 insertions(+) create mode 100644 src/Transformation/Diagnostic/SourceSpan.php create mode 100644 src/Transformation/Diagnostic/TransformationDiagnostic.php create mode 100644 src/Transformation/Diagnostic/TransformationFailure.php create mode 100644 src/Transformation/Plan/Operation.php create mode 100644 src/Transformation/Plan/Operation/RecodeOperation.php create mode 100644 src/Transformation/Plan/Operation/ReplaceValueLabelsOperation.php create mode 100644 src/Transformation/Plan/Operation/SetVariableLabelOperation.php create mode 100644 src/Transformation/Plan/Operation/ValueLabel.php create mode 100644 src/Transformation/Plan/PlanCodec.php create mode 100644 src/Transformation/Plan/PlanContract.php create mode 100644 src/Transformation/Plan/Recode/CopyResult.php create mode 100644 src/Transformation/Plan/Recode/ExactMatch.php create mode 100644 src/Transformation/Plan/Recode/LiteralResult.php create mode 100644 src/Transformation/Plan/Recode/Match.php create mode 100644 src/Transformation/Plan/Recode/RangeMatch.php create mode 100644 src/Transformation/Plan/Recode/RecodeMatch.php create mode 100644 src/Transformation/Plan/Recode/RecodeRule.php create mode 100644 src/Transformation/Plan/Recode/Result.php create mode 100644 src/Transformation/Plan/Recode/SystemMissingMatch.php create mode 100644 src/Transformation/Plan/Recode/SystemMissingResult.php create mode 100644 src/Transformation/Plan/TargetMode.php create mode 100644 src/Transformation/Plan/TransformationPlan.php create mode 100644 src/Transformation/Plan/Value/Binary64Value.php create mode 100644 src/Transformation/Plan/Value/StringValue.php create mode 100644 src/Transformation/Plan/Value/TypedValue.php create mode 100644 tests/Transformation/Conformance/TransformationPlan01Test.php diff --git a/src/Transformation/Diagnostic/SourceSpan.php b/src/Transformation/Diagnostic/SourceSpan.php new file mode 100644 index 0000000..27c84a8 --- /dev/null +++ b/src/Transformation/Diagnostic/SourceSpan.php @@ -0,0 +1,29 @@ +startOffset, + $end->endOffset, + $start->startLine, + $start->startColumn, + $end->endLine, + $end->endColumn, + ); + } +} diff --git a/src/Transformation/Diagnostic/TransformationDiagnostic.php b/src/Transformation/Diagnostic/TransformationDiagnostic.php new file mode 100644 index 0000000..04d8758 --- /dev/null +++ b/src/Transformation/Diagnostic/TransformationDiagnostic.php @@ -0,0 +1,15 @@ + */ + public readonly array $diagnostics; + + /** @param non-empty-list $diagnostics */ + public function __construct(array $diagnostics) + { + $this->diagnostics = $diagnostics; + + parent::__construct($diagnostics[0]->message); + } + + public static function at(string $code, string $path, string $message, ?SourceSpan $span = null): self + { + return new self([new TransformationDiagnostic($code, $path, $message, $span)]); + } + + public function diagnosticCode(): string + { + return $this->diagnostics[0]->code; + } +} diff --git a/src/Transformation/Plan/Operation.php b/src/Transformation/Plan/Operation.php new file mode 100644 index 0000000..77396ea --- /dev/null +++ b/src/Transformation/Plan/Operation.php @@ -0,0 +1,13 @@ + */ + public function canonicalArray(): array; + + public function minimumContract(): PlanContract; +} diff --git a/src/Transformation/Plan/Operation/RecodeOperation.php b/src/Transformation/Plan/Operation/RecodeOperation.php new file mode 100644 index 0000000..9636596 --- /dev/null +++ b/src/Transformation/Plan/Operation/RecodeOperation.php @@ -0,0 +1,41 @@ + $rules */ + public function __construct( + public string $source, + public string $target, + public TargetMode $targetMode, + public array $rules, + public Result $unmatched, + ) {} + + /** @return array */ + public function canonicalArray(): array + { + return [ + 'op' => 'recode', + 'source' => $this->source, + 'target' => $this->target, + 'target_mode' => $this->targetMode->value, + 'rules' => array_map(static fn(RecodeRule $rule): array => $rule->canonicalArray(), $this->rules), + 'unmatched' => $this->unmatched->canonicalArray(), + ]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V01; + } +} diff --git a/src/Transformation/Plan/Operation/ReplaceValueLabelsOperation.php b/src/Transformation/Plan/Operation/ReplaceValueLabelsOperation.php new file mode 100644 index 0000000..fc0fa77 --- /dev/null +++ b/src/Transformation/Plan/Operation/ReplaceValueLabelsOperation.php @@ -0,0 +1,32 @@ + $labels */ + public function __construct( + public string $variable, + public array $labels, + ) {} + + /** @return array */ + public function canonicalArray(): array + { + return [ + 'op' => 'replace_value_labels', + 'variable' => $this->variable, + 'labels' => array_map(static fn(ValueLabel $label): array => $label->canonicalArray(), $this->labels), + ]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V01; + } +} diff --git a/src/Transformation/Plan/Operation/SetVariableLabelOperation.php b/src/Transformation/Plan/Operation/SetVariableLabelOperation.php new file mode 100644 index 0000000..24f2840 --- /dev/null +++ b/src/Transformation/Plan/Operation/SetVariableLabelOperation.php @@ -0,0 +1,27 @@ + 'set_variable_label', 'variable' => $this->variable, 'label' => $this->label]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V01; + } +} diff --git a/src/Transformation/Plan/Operation/ValueLabel.php b/src/Transformation/Plan/Operation/ValueLabel.php new file mode 100644 index 0000000..6025c3a --- /dev/null +++ b/src/Transformation/Plan/Operation/ValueLabel.php @@ -0,0 +1,21 @@ +, label: string} */ + public function canonicalArray(): array + { + return ['value' => $this->value->canonicalArray(), 'label' => $this->label]; + } +} diff --git a/src/Transformation/Plan/PlanCodec.php b/src/Transformation/Plan/PlanCodec.php new file mode 100644 index 0000000..5eb47a9 --- /dev/null +++ b/src/Transformation/Plan/PlanCodec.php @@ -0,0 +1,320 @@ +schema('$', 'Plan JSON must be valid JSON.'); + } + + if (!is_array($decoded)) { + $this->schema('$', 'Plan must be a JSON object.'); + } + + return $this->fromArray($decoded); + } + + /** @param array $plan */ + public function fromArray(array $plan): TransformationPlan + { + $this->exactKeys($plan, ['contract', 'input_alias', 'operations'], '$'); + $contract = $plan['contract']; + if (!is_string($contract) || $contract !== PlanContract::V01->value) { + $this->schema('$.contract', 'Plan contract must be openstatspec-transformation-plan-v0.1.'); + } + + $inputAlias = $this->nonEmptyString($plan['input_alias'], '$.input_alias'); + $rawOperations = $this->nonEmptyList($plan['operations'], '$.operations'); + $operations = []; + foreach ($rawOperations as $index => $rawOperation) { + $operations[] = $this->operation($rawOperation, '$.operations[' . $index . ']'); + } + + return new TransformationPlan(PlanContract::V01, $inputAlias, $operations); + } + + public function canonicalJson(TransformationPlan $plan): string + { + return CanonicalJson::encode($plan->canonicalArray()); + } + + public function hash(TransformationPlan $plan): string + { + return hash('sha256', $this->canonicalJson($plan)); + } + + private function operation(mixed $raw, string $path): Operation + { + $object = $this->object($raw, $path); + $op = $object['op'] ?? null; + if (!is_string($op)) { + $this->schema($path . '.op', 'Operation discriminator must be a string.'); + } + + return match ($op) { + 'recode' => $this->recodeOperation($object, $path), + 'set_variable_label' => $this->setVariableLabelOperation($object, $path), + 'replace_value_labels' => $this->replaceValueLabelsOperation($object, $path), + default => $this->schema($path . '.op', 'Unknown operation.'), + }; + } + + /** @param array $object */ + private function recodeOperation(array $object, string $path): RecodeOperation + { + $this->exactKeys($object, ['op', 'source', 'target', 'target_mode', 'rules', 'unmatched'], $path); + $mode = $object['target_mode']; + if (!is_string($mode) || ($targetMode = TargetMode::tryFrom($mode)) === null) { + $this->schema($path . '.target_mode', 'Recode target mode must be create or replace.'); + } + + $rawRules = $this->nonEmptyList($object['rules'], $path . '.rules'); + $rules = []; + foreach ($rawRules as $index => $rawRule) { + $rules[] = $this->rule($rawRule, $path . '.rules[' . $index . ']'); + } + + return new RecodeOperation( + $this->nonEmptyString($object['source'], $path . '.source'), + $this->nonEmptyString($object['target'], $path . '.target'), + $targetMode, + $rules, + $this->result($object['unmatched'], $path . '.unmatched'), + ); + } + + /** @param array $object */ + private function setVariableLabelOperation(array $object, string $path): SetVariableLabelOperation + { + $this->exactKeys($object, ['op', 'variable', 'label'], $path); + + return new SetVariableLabelOperation( + $this->nonEmptyString($object['variable'], $path . '.variable'), + $this->string($object['label'], $path . '.label'), + ); + } + + /** @param array $object */ + private function replaceValueLabelsOperation(array $object, string $path): ReplaceValueLabelsOperation + { + $this->exactKeys($object, ['op', 'variable', 'labels'], $path); + $rawLabels = $this->nonEmptyList($object['labels'], $path . '.labels'); + $labels = []; + $seen = []; + foreach ($rawLabels as $index => $rawLabel) { + $label = $this->valueLabel($rawLabel, $path . '.labels[' . $index . ']'); + $key = $label->value->canonicalKey(); + if (isset($seen[$key])) { + throw TransformationFailure::at( + 'duplicate_value_label', + $path . '.labels[' . $index . '].value', + 'Value labels must have unique exact typed values.', + ); + } + $seen[$key] = true; + $labels[] = $label; + } + + return new ReplaceValueLabelsOperation( + $this->nonEmptyString($object['variable'], $path . '.variable'), + $labels, + ); + } + + private function rule(mixed $raw, string $path): RecodeRule + { + $object = $this->object($raw, $path); + $this->exactKeys($object, ['match', 'result'], $path); + + return new RecodeRule( + $this->recodeMatch($object['match'], $path . '.match'), + $this->result($object['result'], $path . '.result'), + ); + } + + private function recodeMatch(mixed $raw, string $path): RecodeMatch + { + $object = $this->object($raw, $path); + $kind = $object['kind'] ?? null; + if (!is_string($kind)) { + $this->schema($path . '.kind', 'Match kind must be a string.'); + } + + if ($kind === 'values') { + $this->exactKeys($object, ['kind', 'values'], $path); + $rawValues = $this->nonEmptyList($object['values'], $path . '.values'); + $values = []; + foreach ($rawValues as $index => $rawValue) { + $values[] = $this->typedValue($rawValue, $path . '.values[' . $index . ']'); + } + return new ExactMatch(...$values); + } + + if ($kind === 'range') { + $this->exactKeys($object, ['kind', 'lower', 'upper'], $path); + $lower = $this->typedValue($object['lower'], $path . '.lower'); + $upper = $this->typedValue($object['upper'], $path . '.upper'); + if (!$lower instanceof Binary64Value || !$upper instanceof Binary64Value) { + $this->schema($path, 'Range endpoints must be binary64 values.'); + } + if ($lower->number() > $upper->number()) { + throw TransformationFailure::at( + 'invalid_numeric_range', + $path, + 'Range lower endpoint cannot exceed its upper endpoint.', + ); + } + return new RangeMatch($lower, $upper); + } + + if ($kind === 'system_missing') { + $this->exactKeys($object, ['kind'], $path); + return new SystemMissingMatch(); + } + + $this->schema($path . '.kind', 'Unknown match kind.'); + } + + private function result(mixed $raw, string $path): Result + { + $object = $this->object($raw, $path); + $kind = $object['kind'] ?? null; + if (!is_string($kind)) { + $this->schema($path . '.kind', 'Result kind must be a string.'); + } + + return match ($kind) { + 'literal' => $this->literalResult($object, $path), + 'copy' => $this->emptyResult($object, $path, new CopyResult()), + 'system_missing' => $this->emptyResult($object, $path, new SystemMissingResult()), + default => $this->schema($path . '.kind', 'Unknown result kind.'), + }; + } + + /** @param array $object */ + private function literalResult(array $object, string $path): LiteralResult + { + $this->exactKeys($object, ['kind', 'value'], $path); + return new LiteralResult($this->typedValue($object['value'], $path . '.value')); + } + + /** @param array $object */ + private function emptyResult(array $object, string $path, Result $result): Result + { + $this->exactKeys($object, ['kind'], $path); + return $result; + } + + private function valueLabel(mixed $raw, string $path): ValueLabel + { + $object = $this->object($raw, $path); + $this->exactKeys($object, ['value', 'label'], $path); + + return new ValueLabel( + $this->typedValue($object['value'], $path . '.value'), + $this->string($object['label'], $path . '.label'), + ); + } + + private function typedValue(mixed $raw, string $path): TypedValue + { + $object = $this->object($raw, $path); + $type = $object['type'] ?? null; + if ($type === 'binary64') { + $this->exactKeys($object, ['type', 'bits'], $path); + $bits = $this->string($object['bits'], $path . '.bits'); + try { + return Binary64Value::fromBits($bits); + } catch (TransformationFailure $failure) { + $this->schema($path . '.bits', $failure->getMessage()); + } + } + if ($type === 'string') { + $this->exactKeys($object, ['type', 'value'], $path); + return new StringValue($this->string($object['value'], $path . '.value')); + } + + $this->schema($path . '.type', 'Typed value type must be binary64 or string.'); + } + + /** @return array */ + private function object(mixed $value, string $path): array + { + if (!is_array($value) || array_is_list($value)) { + $this->schema($path, 'Value must be an object.'); + } + return $value; + } + + /** @return non-empty-list */ + private function nonEmptyList(mixed $value, string $path): array + { + if (!is_array($value) || !array_is_list($value) || $value === []) { + $this->schema($path, 'Value must be a non-empty array.'); + } + return $value; + } + + /** + * @param array $object + * @param list $expected + */ + private function exactKeys(array $object, array $expected, string $path): void + { + $actual = array_keys($object); + sort($actual, SORT_STRING); + sort($expected, SORT_STRING); + if ($actual !== $expected) { + $this->schema($path, 'Object has missing or extra members.'); + } + } + + private function nonEmptyString(mixed $value, string $path): string + { + $string = $this->string($value, $path); + if ($string === '') { + $this->schema($path, 'String must not be empty.'); + } + return $string; + } + + private function string(mixed $value, string $path): string + { + if (!is_string($value)) { + $this->schema($path, 'Value must be a string.'); + } + return $value; + } + + private function schema(string $path, string $message): never + { + throw TransformationFailure::at('plan_schema_invalid', $path, $message); + } +} diff --git a/src/Transformation/Plan/PlanContract.php b/src/Transformation/Plan/PlanContract.php new file mode 100644 index 0000000..16936a4 --- /dev/null +++ b/src/Transformation/Plan/PlanContract.php @@ -0,0 +1,11 @@ + 'copy']; + } +} diff --git a/src/Transformation/Plan/Recode/ExactMatch.php b/src/Transformation/Plan/Recode/ExactMatch.php new file mode 100644 index 0000000..c97e819 --- /dev/null +++ b/src/Transformation/Plan/Recode/ExactMatch.php @@ -0,0 +1,31 @@ + */ + public array $values; + + public function __construct(TypedValue ...$values) + { + if ($values === []) { + throw new \InvalidArgumentException('A values match requires at least one value.'); + } + + $this->values = array_values($values); + } + + /** @return array{kind: string, values: list>} */ + public function canonicalArray(): array + { + return [ + 'kind' => 'values', + 'values' => array_map(static fn(TypedValue $value): array => $value->canonicalArray(), $this->values), + ]; + } +} diff --git a/src/Transformation/Plan/Recode/LiteralResult.php b/src/Transformation/Plan/Recode/LiteralResult.php new file mode 100644 index 0000000..ac1d472 --- /dev/null +++ b/src/Transformation/Plan/Recode/LiteralResult.php @@ -0,0 +1,18 @@ +} */ + public function canonicalArray(): array + { + return ['kind' => 'literal', 'value' => $this->value->canonicalArray()]; + } +} diff --git a/src/Transformation/Plan/Recode/Match.php b/src/Transformation/Plan/Recode/Match.php new file mode 100644 index 0000000..5e8a252 --- /dev/null +++ b/src/Transformation/Plan/Recode/Match.php @@ -0,0 +1,5 @@ +, upper: array} */ + public function canonicalArray(): array + { + return [ + 'kind' => 'range', + 'lower' => $this->lower->canonicalArray(), + 'upper' => $this->upper->canonicalArray(), + ]; + } +} diff --git a/src/Transformation/Plan/Recode/RecodeMatch.php b/src/Transformation/Plan/Recode/RecodeMatch.php new file mode 100644 index 0000000..862fac8 --- /dev/null +++ b/src/Transformation/Plan/Recode/RecodeMatch.php @@ -0,0 +1,11 @@ + */ + public function canonicalArray(): array; +} diff --git a/src/Transformation/Plan/Recode/RecodeRule.php b/src/Transformation/Plan/Recode/RecodeRule.php new file mode 100644 index 0000000..3f348bc --- /dev/null +++ b/src/Transformation/Plan/Recode/RecodeRule.php @@ -0,0 +1,19 @@ +, result: array} */ + public function canonicalArray(): array + { + return ['match' => $this->match->canonicalArray(), 'result' => $this->result->canonicalArray()]; + } +} diff --git a/src/Transformation/Plan/Recode/Result.php b/src/Transformation/Plan/Recode/Result.php new file mode 100644 index 0000000..07fc52d --- /dev/null +++ b/src/Transformation/Plan/Recode/Result.php @@ -0,0 +1,11 @@ + */ + public function canonicalArray(): array; +} diff --git a/src/Transformation/Plan/Recode/SystemMissingMatch.php b/src/Transformation/Plan/Recode/SystemMissingMatch.php new file mode 100644 index 0000000..b137f54 --- /dev/null +++ b/src/Transformation/Plan/Recode/SystemMissingMatch.php @@ -0,0 +1,14 @@ + 'system_missing']; + } +} diff --git a/src/Transformation/Plan/Recode/SystemMissingResult.php b/src/Transformation/Plan/Recode/SystemMissingResult.php new file mode 100644 index 0000000..178bd9e --- /dev/null +++ b/src/Transformation/Plan/Recode/SystemMissingResult.php @@ -0,0 +1,14 @@ + 'system_missing']; + } +} diff --git a/src/Transformation/Plan/TargetMode.php b/src/Transformation/Plan/TargetMode.php new file mode 100644 index 0000000..d6943ab --- /dev/null +++ b/src/Transformation/Plan/TargetMode.php @@ -0,0 +1,11 @@ + $operations */ + public function __construct( + public PlanContract $contract, + public string $inputAlias, + public array $operations, + ) {} + + /** @return array{contract: string, input_alias: string, operations: list>} */ + public function canonicalArray(): array + { + return [ + 'contract' => $this->contract->value, + 'input_alias' => $this->inputAlias, + 'operations' => array_map( + static fn(Operation $operation): array => $operation->canonicalArray(), + $this->operations, + ), + ]; + } +} diff --git a/src/Transformation/Plan/Value/Binary64Value.php b/src/Transformation/Plan/Value/Binary64Value.php new file mode 100644 index 0000000..07fd425 --- /dev/null +++ b/src/Transformation/Plan/Value/Binary64Value.php @@ -0,0 +1,55 @@ + 'binary64', 'bits' => $this->bits]; + } + + public function canonicalKey(): string + { + return 'binary64:' . $this->bits; + } + + public function number(): float + { + $binary = hex2bin($this->bits); + if ($binary === false) { + throw new \LogicException('Validated binary64 bits could not be decoded.'); + } + + $decoded = unpack('Evalue', $binary); + if (!is_array($decoded) || !isset($decoded['value']) || !is_float($decoded['value'])) { + throw new \LogicException('Validated binary64 bits could not be unpacked.'); + } + + return $decoded['value']; + } +} diff --git a/src/Transformation/Plan/Value/StringValue.php b/src/Transformation/Plan/Value/StringValue.php new file mode 100644 index 0000000..1d32f80 --- /dev/null +++ b/src/Transformation/Plan/Value/StringValue.php @@ -0,0 +1,21 @@ + 'string', 'value' => $this->value]; + } + + public function canonicalKey(): string + { + return 'string:' . $this->value; + } +} diff --git a/src/Transformation/Plan/Value/TypedValue.php b/src/Transformation/Plan/Value/TypedValue.php new file mode 100644 index 0000000..cdcea17 --- /dev/null +++ b/src/Transformation/Plan/Value/TypedValue.php @@ -0,0 +1,13 @@ + */ + public function canonicalArray(): array; + + public function canonicalKey(): string; +} diff --git a/tests/Transformation/Conformance/TransformationPlan01Test.php b/tests/Transformation/Conformance/TransformationPlan01Test.php new file mode 100644 index 0000000..5f4ae41 --- /dev/null +++ b/tests/Transformation/Conformance/TransformationPlan01Test.php @@ -0,0 +1,41 @@ +}> */ + public static function cases(): iterable + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.1.json')['cases'] as $case) { + yield $case['id'] => [$case]; + } + } + + /** @param array $case */ + #[DataProvider('cases')] + public function testOfficialPlan01(array $case): void + { + if (is_string($case['expected_error'])) { + try { + (new PlanCodec())->fromArray($case['plan']); + self::fail('Expected ' . $case['expected_error']); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + return; + } + + $plan = (new PlanCodec())->fromArray($case['plan']); + self::assertSame($case['plan'], $plan->canonicalArray()); + self::assertSame($case['expected_plan_hash'], (new PlanCodec())->hash($plan)); + } +} From a8c006c729493aafb5a5a6d91ebfb5349ab80b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 14:59:11 +0300 Subject: [PATCH 05/53] fix: validate transformation plan schema strings --- src/Transformation/Plan/PlanCodec.php | 29 ++--- .../Conformance/TransformationPlan01Test.php | 119 ++++++++++++++++++ 2 files changed, 129 insertions(+), 19 deletions(-) diff --git a/src/Transformation/Plan/PlanCodec.php b/src/Transformation/Plan/PlanCodec.php index 5eb47a9..c37ef78 100644 --- a/src/Transformation/Plan/PlanCodec.php +++ b/src/Transformation/Plan/PlanCodec.php @@ -45,8 +45,8 @@ public function fromJson(string $json): TransformationPlan public function fromArray(array $plan): TransformationPlan { $this->exactKeys($plan, ['contract', 'input_alias', 'operations'], '$'); - $contract = $plan['contract']; - if (!is_string($contract) || $contract !== PlanContract::V01->value) { + $contract = $this->string($plan['contract'], '$.contract'); + if ($contract !== PlanContract::V01->value) { $this->schema('$.contract', 'Plan contract must be openstatspec-transformation-plan-v0.1.'); } @@ -73,10 +73,7 @@ public function hash(TransformationPlan $plan): string private function operation(mixed $raw, string $path): Operation { $object = $this->object($raw, $path); - $op = $object['op'] ?? null; - if (!is_string($op)) { - $this->schema($path . '.op', 'Operation discriminator must be a string.'); - } + $op = $this->string($object['op'] ?? null, $path . '.op'); return match ($op) { 'recode' => $this->recodeOperation($object, $path), @@ -90,8 +87,8 @@ private function operation(mixed $raw, string $path): Operation private function recodeOperation(array $object, string $path): RecodeOperation { $this->exactKeys($object, ['op', 'source', 'target', 'target_mode', 'rules', 'unmatched'], $path); - $mode = $object['target_mode']; - if (!is_string($mode) || ($targetMode = TargetMode::tryFrom($mode)) === null) { + $mode = $this->string($object['target_mode'], $path . '.target_mode'); + if (($targetMode = TargetMode::tryFrom($mode)) === null) { $this->schema($path . '.target_mode', 'Recode target mode must be create or replace.'); } @@ -162,10 +159,7 @@ private function rule(mixed $raw, string $path): RecodeRule private function recodeMatch(mixed $raw, string $path): RecodeMatch { $object = $this->object($raw, $path); - $kind = $object['kind'] ?? null; - if (!is_string($kind)) { - $this->schema($path . '.kind', 'Match kind must be a string.'); - } + $kind = $this->string($object['kind'] ?? null, $path . '.kind'); if ($kind === 'values') { $this->exactKeys($object, ['kind', 'values'], $path); @@ -205,10 +199,7 @@ private function recodeMatch(mixed $raw, string $path): RecodeMatch private function result(mixed $raw, string $path): Result { $object = $this->object($raw, $path); - $kind = $object['kind'] ?? null; - if (!is_string($kind)) { - $this->schema($path . '.kind', 'Result kind must be a string.'); - } + $kind = $this->string($object['kind'] ?? null, $path . '.kind'); return match ($kind) { 'literal' => $this->literalResult($object, $path), @@ -246,7 +237,7 @@ private function valueLabel(mixed $raw, string $path): ValueLabel private function typedValue(mixed $raw, string $path): TypedValue { $object = $this->object($raw, $path); - $type = $object['type'] ?? null; + $type = $this->string($object['type'] ?? null, $path . '.type'); if ($type === 'binary64') { $this->exactKeys($object, ['type', 'bits'], $path); $bits = $this->string($object['bits'], $path . '.bits'); @@ -307,8 +298,8 @@ private function nonEmptyString(mixed $value, string $path): string private function string(mixed $value, string $path): string { - if (!is_string($value)) { - $this->schema($path, 'Value must be a string.'); + if (!is_string($value) || preg_match('//u', $value) !== 1) { + $this->schema($path, 'Value must be a valid UTF-8 string.'); } return $value; } diff --git a/tests/Transformation/Conformance/TransformationPlan01Test.php b/tests/Transformation/Conformance/TransformationPlan01Test.php index 5f4ae41..0adb5c8 100644 --- a/tests/Transformation/Conformance/TransformationPlan01Test.php +++ b/tests/Transformation/Conformance/TransformationPlan01Test.php @@ -12,6 +12,125 @@ final class TransformationPlan01Test extends TestCase { + /** @return iterable, string}> */ + public static function invalidPlans(): iterable + { + yield 'missing root member' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'operations' => [['op' => 'set_variable_label', 'variable' => 'q1', 'label' => 'Question']], + ], 'plan_schema_invalid']; + yield 'extra operation member' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [['op' => 'set_variable_label', 'variable' => 'q1', 'label' => 'Question', 'extra' => true]], + ], 'plan_schema_invalid']; + yield 'empty input alias' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => '', + 'operations' => [['op' => 'set_variable_label', 'variable' => 'q1', 'label' => 'Question']], + ], 'plan_schema_invalid']; + yield 'empty operations' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [], + ], 'plan_schema_invalid']; + yield 'unknown operation' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [['op' => 'delete_variable']], + ], 'plan_schema_invalid']; + yield 'malformed binary64 bits' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'replace_value_labels', + 'variable' => 'q1', + 'labels' => [['value' => ['type' => 'binary64', 'bits' => 'not-binary64'], 'label' => 'One']], + ]], + ], 'plan_schema_invalid']; + yield 'non-finite binary64 bits' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'replace_value_labels', + 'variable' => 'q1', + 'labels' => [['value' => ['type' => 'binary64', 'bits' => '7ff0000000000000'], 'label' => 'Infinity']], + ]], + ], 'plan_schema_invalid']; + yield 'negative zero binary64 bits' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'replace_value_labels', + 'variable' => 'q1', + 'labels' => [['value' => ['type' => 'binary64', 'bits' => '8000000000000000'], 'label' => 'Zero']], + ]], + ], 'plan_schema_invalid']; + yield 'descending range' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'recode', + 'source' => 'q1', + 'target' => 'q1', + 'target_mode' => 'replace', + 'rules' => [[ + 'match' => [ + 'kind' => 'range', + 'lower' => ['type' => 'binary64', 'bits' => '4000000000000000'], + 'upper' => ['type' => 'binary64', 'bits' => '3ff0000000000000'], + ], + 'result' => ['kind' => 'copy'], + ]], + 'unmatched' => ['kind' => 'copy'], + ]], + ], 'invalid_numeric_range']; + yield 'duplicate exact typed label values' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'replace_value_labels', + 'variable' => 'q1', + 'labels' => [ + ['value' => ['type' => 'string', 'value' => 'A'], 'label' => 'Alpha'], + ['value' => ['type' => 'string', 'value' => 'A'], 'label' => 'Again'], + ], + ]], + ], 'duplicate_value_label']; + } + + /** @param array $plan */ + #[DataProvider('invalidPlans')] + public function testRejectsInvalidPlanWithStableDiagnostic(array $plan, string $expectedCode): void + { + try { + (new PlanCodec())->fromArray($plan); + self::fail('Expected ' . $expectedCode . '.'); + } catch (TransformationFailure $failure) { + self::assertSame($expectedCode, $failure->diagnosticCode()); + } + } + + public function testRejectsInvalidUtf8AsPlanSchemaInvalid(): void + { + $plan = [ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => "invalid-\xB1", + 'operations' => [[ + 'op' => 'set_variable_label', + 'variable' => 'q1', + 'label' => 'Question', + ]], + ]; + + try { + (new PlanCodec())->fromArray($plan); + self::fail('Expected plan_schema_invalid for malformed UTF-8.'); + } catch (TransformationFailure $failure) { + self::assertSame('plan_schema_invalid', $failure->diagnosticCode()); + } + } + /** @return iterable}> */ public static function cases(): iterable { From 1a90c93f969a26dc847c2a345dd1834908eb7278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 15:10:08 +0300 Subject: [PATCH 06/53] feat: implement transformation plan 0.2 operations --- .../Plan/Expression/BooleanPredicate.php | 44 ++++ .../Plan/Expression/Comparison.php | 31 +++ .../Plan/Expression/LiteralOperand.php | 18 ++ .../Plan/Expression/Operand.php | 11 + .../Plan/Expression/Predicate.php | 11 + .../Plan/Expression/VariableOperand.php | 16 ++ .../Plan/Operation/AssignOperation.php | 30 +++ .../Operation/ConditionalAssignOperation.php | 30 +++ .../Plan/Operation/ExecuteOperation.php | 22 ++ .../Plan/Operation/SetFormatOperation.php | 29 +++ .../SetMeasurementLevelOperation.php | 27 +++ src/Transformation/Plan/PlanCodec.php | 194 +++++++++++++++++- .../Conformance/TransformationPlan02Test.php | 63 ++++++ .../Plan/BooleanPredicateTest.php | 39 ++++ 14 files changed, 560 insertions(+), 5 deletions(-) create mode 100644 src/Transformation/Plan/Expression/BooleanPredicate.php create mode 100644 src/Transformation/Plan/Expression/Comparison.php create mode 100644 src/Transformation/Plan/Expression/LiteralOperand.php create mode 100644 src/Transformation/Plan/Expression/Operand.php create mode 100644 src/Transformation/Plan/Expression/Predicate.php create mode 100644 src/Transformation/Plan/Expression/VariableOperand.php create mode 100644 src/Transformation/Plan/Operation/AssignOperation.php create mode 100644 src/Transformation/Plan/Operation/ConditionalAssignOperation.php create mode 100644 src/Transformation/Plan/Operation/ExecuteOperation.php create mode 100644 src/Transformation/Plan/Operation/SetFormatOperation.php create mode 100644 src/Transformation/Plan/Operation/SetMeasurementLevelOperation.php create mode 100644 tests/Transformation/Conformance/TransformationPlan02Test.php create mode 100644 tests/Transformation/Plan/BooleanPredicateTest.php diff --git a/src/Transformation/Plan/Expression/BooleanPredicate.php b/src/Transformation/Plan/Expression/BooleanPredicate.php new file mode 100644 index 0000000..113a2f7 --- /dev/null +++ b/src/Transformation/Plan/Expression/BooleanPredicate.php @@ -0,0 +1,44 @@ + */ + public array $operands; + + public function __construct(public string $operator, Predicate ...$operands) + { + if (!in_array($operator, ['and', 'or'], true)) { + throw TransformationFailure::at('plan_schema_invalid', '$.operator', 'Boolean operator must be and or or.'); + } + if (count($operands) < 2) { + throw TransformationFailure::at('plan_schema_invalid', '$.operands', 'A boolean expression requires at least two operands.'); + } + foreach ($operands as $operand) { + if ($operand instanceof self && $operand->operator === $operator) { + throw TransformationFailure::at( + 'noncanonical_boolean_shape', + '$.operands', + 'A same-operator boolean chain must be flattened in source order.', + ); + } + } + + $this->operands = array_values($operands); + } + + /** @return array{expression: string, operator: string, operands: list>} */ + public function canonicalArray(): array + { + return [ + 'expression' => 'boolean', + 'operator' => $this->operator, + 'operands' => array_map(static fn(Predicate $operand): array => $operand->canonicalArray(), $this->operands), + ]; + } +} diff --git a/src/Transformation/Plan/Expression/Comparison.php b/src/Transformation/Plan/Expression/Comparison.php new file mode 100644 index 0000000..258cbac --- /dev/null +++ b/src/Transformation/Plan/Expression/Comparison.php @@ -0,0 +1,31 @@ +', '>='], true)) { + throw TransformationFailure::at('plan_schema_invalid', '$.operator', 'Unsupported comparison operator.'); + } + } + + /** @return array{expression: string, left: array, operator: string, right: array} */ + public function canonicalArray(): array + { + return [ + 'expression' => 'comparison', + 'left' => $this->left->canonicalArray(), + 'operator' => $this->operator, + 'right' => $this->right->canonicalArray(), + ]; + } +} diff --git a/src/Transformation/Plan/Expression/LiteralOperand.php b/src/Transformation/Plan/Expression/LiteralOperand.php new file mode 100644 index 0000000..e28103b --- /dev/null +++ b/src/Transformation/Plan/Expression/LiteralOperand.php @@ -0,0 +1,18 @@ +} */ + public function canonicalArray(): array + { + return ['kind' => 'literal', 'value' => $this->value->canonicalArray()]; + } +} diff --git a/src/Transformation/Plan/Expression/Operand.php b/src/Transformation/Plan/Expression/Operand.php new file mode 100644 index 0000000..9b631b1 --- /dev/null +++ b/src/Transformation/Plan/Expression/Operand.php @@ -0,0 +1,11 @@ + */ + public function canonicalArray(): array; +} diff --git a/src/Transformation/Plan/Expression/Predicate.php b/src/Transformation/Plan/Expression/Predicate.php new file mode 100644 index 0000000..e001c7c --- /dev/null +++ b/src/Transformation/Plan/Expression/Predicate.php @@ -0,0 +1,11 @@ + */ + public function canonicalArray(): array; +} diff --git a/src/Transformation/Plan/Expression/VariableOperand.php b/src/Transformation/Plan/Expression/VariableOperand.php new file mode 100644 index 0000000..6add27c --- /dev/null +++ b/src/Transformation/Plan/Expression/VariableOperand.php @@ -0,0 +1,16 @@ + 'variable', 'variable' => $this->variable]; + } +} diff --git a/src/Transformation/Plan/Operation/AssignOperation.php b/src/Transformation/Plan/Operation/AssignOperation.php new file mode 100644 index 0000000..cfdc3e9 --- /dev/null +++ b/src/Transformation/Plan/Operation/AssignOperation.php @@ -0,0 +1,30 @@ +} */ + public function canonicalArray(): array + { + return ['op' => 'assign', 'target' => $this->target, 'target_mode' => $this->targetMode->value, 'value' => $this->value->canonicalArray()]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/Operation/ConditionalAssignOperation.php b/src/Transformation/Plan/Operation/ConditionalAssignOperation.php new file mode 100644 index 0000000..c15c1a6 --- /dev/null +++ b/src/Transformation/Plan/Operation/ConditionalAssignOperation.php @@ -0,0 +1,30 @@ +, target: string, value: array} */ + public function canonicalArray(): array + { + return ['op' => 'conditional_assign', 'condition' => $this->condition->canonicalArray(), 'target' => $this->target, 'value' => $this->value->canonicalArray()]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/Operation/ExecuteOperation.php b/src/Transformation/Plan/Operation/ExecuteOperation.php new file mode 100644 index 0000000..66bfe6c --- /dev/null +++ b/src/Transformation/Plan/Operation/ExecuteOperation.php @@ -0,0 +1,22 @@ + 'execute']; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/Operation/SetFormatOperation.php b/src/Transformation/Plan/Operation/SetFormatOperation.php new file mode 100644 index 0000000..ddf0b95 --- /dev/null +++ b/src/Transformation/Plan/Operation/SetFormatOperation.php @@ -0,0 +1,29 @@ + 'set_format', 'variable' => $this->variable, 'family' => $this->family, 'width' => $this->width, 'decimals' => $this->decimals]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/Operation/SetMeasurementLevelOperation.php b/src/Transformation/Plan/Operation/SetMeasurementLevelOperation.php new file mode 100644 index 0000000..ecc2498 --- /dev/null +++ b/src/Transformation/Plan/Operation/SetMeasurementLevelOperation.php @@ -0,0 +1,27 @@ + 'set_measurement_level', 'variable' => $this->variable, 'level' => $this->level]; + } + + public function minimumContract(): PlanContract + { + return PlanContract::V02; + } +} diff --git a/src/Transformation/Plan/PlanCodec.php b/src/Transformation/Plan/PlanCodec.php index c37ef78..549704e 100644 --- a/src/Transformation/Plan/PlanCodec.php +++ b/src/Transformation/Plan/PlanCodec.php @@ -7,8 +7,19 @@ use JsonException; use OpenStatSpec\Transformation\Canonical\CanonicalJson; use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Plan\Expression\BooleanPredicate; +use OpenStatSpec\Transformation\Plan\Expression\Comparison; +use OpenStatSpec\Transformation\Plan\Expression\LiteralOperand; +use OpenStatSpec\Transformation\Plan\Expression\Operand; +use OpenStatSpec\Transformation\Plan\Expression\Predicate; +use OpenStatSpec\Transformation\Plan\Expression\VariableOperand; +use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\ConditionalAssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\ExecuteOperation; use OpenStatSpec\Transformation\Plan\Operation\RecodeOperation; use OpenStatSpec\Transformation\Plan\Operation\ReplaceValueLabelsOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetFormatOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetMeasurementLevelOperation; use OpenStatSpec\Transformation\Plan\Operation\SetVariableLabelOperation; use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; use OpenStatSpec\Transformation\Plan\Recode\CopyResult; @@ -45,19 +56,24 @@ public function fromJson(string $json): TransformationPlan public function fromArray(array $plan): TransformationPlan { $this->exactKeys($plan, ['contract', 'input_alias', 'operations'], '$'); - $contract = $this->string($plan['contract'], '$.contract'); - if ($contract !== PlanContract::V01->value) { - $this->schema('$.contract', 'Plan contract must be openstatspec-transformation-plan-v0.1.'); + $contract = PlanContract::tryFrom($this->string($plan['contract'], '$.contract')); + if ($contract === null) { + $this->schema('$.contract', 'Plan contract must be openstatspec-transformation-plan-v0.1 or openstatspec-transformation-plan-v0.2.'); } $inputAlias = $this->nonEmptyString($plan['input_alias'], '$.input_alias'); $rawOperations = $this->nonEmptyList($plan['operations'], '$.operations'); $operations = []; foreach ($rawOperations as $index => $rawOperation) { - $operations[] = $this->operation($rawOperation, '$.operations[' . $index . ']'); + $path = '$.operations[' . $index . ']'; + $operation = $this->operation($rawOperation, $path); + if ($contract === PlanContract::V01 && $operation->minimumContract() !== PlanContract::V01) { + $this->schema($path, 'Plan contract v0.1 cannot contain v0.2 operations.'); + } + $operations[] = $operation; } - return new TransformationPlan(PlanContract::V01, $inputAlias, $operations); + return new TransformationPlan($contract, $inputAlias, $operations); } public function canonicalJson(TransformationPlan $plan): string @@ -77,12 +93,171 @@ private function operation(mixed $raw, string $path): Operation return match ($op) { 'recode' => $this->recodeOperation($object, $path), + 'assign' => $this->assignOperation($object, $path), + 'conditional_assign' => $this->conditionalAssignOperation($object, $path), 'set_variable_label' => $this->setVariableLabelOperation($object, $path), 'replace_value_labels' => $this->replaceValueLabelsOperation($object, $path), + 'set_format' => $this->setFormatOperation($object, $path), + 'set_measurement_level' => $this->setMeasurementLevelOperation($object, $path), + 'execute' => $this->executeOperation($object, $path), default => $this->schema($path . '.op', 'Unknown operation.'), }; } + /** @param array $object */ + private function assignOperation(array $object, string $path): AssignOperation + { + $this->exactKeys($object, ['op', 'target', 'target_mode', 'value'], $path); + $mode = TargetMode::tryFrom($this->string($object['target_mode'], $path . '.target_mode')); + if ($mode === null) { + $this->schema($path . '.target_mode', 'Assign target mode must be create or replace.'); + } + + return new AssignOperation( + $this->targetName($object['target'], $path . '.target'), + $mode, + $this->numericOperand($object['value'], $path . '.value'), + ); + } + + /** @param array $object */ + private function conditionalAssignOperation(array $object, string $path): ConditionalAssignOperation + { + $this->exactKeys($object, ['op', 'condition', 'target', 'value'], $path); + + return new ConditionalAssignOperation( + $this->predicate($object['condition'], $path . '.condition'), + $this->targetName($object['target'], $path . '.target'), + $this->numericOperand($object['value'], $path . '.value'), + ); + } + + /** @param array $object */ + private function setFormatOperation(array $object, string $path): SetFormatOperation + { + $this->exactKeys($object, ['op', 'variable', 'family', 'width', 'decimals'], $path); + $family = $this->string($object['family'], $path . '.family'); + if ($family !== 'F') { + $this->schema($path . '.family', 'The bounded format profile supports numeric F formats only.'); + } + + $operation = new SetFormatOperation( + $this->nonEmptyString($object['variable'], $path . '.variable'), + $family, + $this->integer($object['width'], $path . '.width'), + $this->integer($object['decimals'], $path . '.decimals'), + ); + if ($operation->width < 1 || $operation->width > 40 + || $operation->decimals < 0 || $operation->decimals > 16 + || ($operation->decimals > 0 && $operation->width < $operation->decimals + 2)) { + throw TransformationFailure::at('invalid_format', $path, 'Invalid SPSS F format.'); + } + + return $operation; + } + + /** @param array $object */ + private function setMeasurementLevelOperation(array $object, string $path): SetMeasurementLevelOperation + { + $this->exactKeys($object, ['op', 'variable', 'level'], $path); + $level = $this->string($object['level'], $path . '.level'); + if (!in_array($level, ['nominal', 'ordinal', 'scale'], true)) { + $this->schema($path . '.level', 'Measurement level must be nominal, ordinal, or scale.'); + } + + return new SetMeasurementLevelOperation( + $this->nonEmptyString($object['variable'], $path . '.variable'), + $level, + ); + } + + /** @param array $object */ + private function executeOperation(array $object, string $path): ExecuteOperation + { + $this->exactKeys($object, ['op'], $path); + return new ExecuteOperation(); + } + + private function predicate(mixed $raw, string $path): Predicate + { + $object = $this->object($raw, $path); + $expression = $this->string($object['expression'] ?? null, $path . '.expression'); + + if ($expression === 'comparison') { + $this->exactKeys($object, ['expression', 'left', 'operator', 'right'], $path); + return new Comparison( + $this->numericOperand($object['left'], $path . '.left'), + $this->string($object['operator'], $path . '.operator'), + $this->numericOperand($object['right'], $path . '.right'), + ); + } + + if ($expression === 'boolean') { + $this->exactKeys($object, ['expression', 'operator', 'operands'], $path); + $operator = $this->string($object['operator'], $path . '.operator'); + $rawOperands = $this->nonEmptyList($object['operands'], $path . '.operands'); + if (count($rawOperands) < 2) { + $this->schema($path . '.operands', 'A boolean expression requires at least two operands.'); + } + $operands = []; + foreach ($rawOperands as $index => $rawOperand) { + $operand = $this->predicate($rawOperand, $path . '.operands[' . $index . ']'); + if ($operand instanceof BooleanPredicate && $operand->operator === $operator) { + throw TransformationFailure::at( + 'noncanonical_boolean_shape', + $path . '.operands[' . $index . ']', + 'A same-operator boolean chain must be flattened in source order.', + ); + } + $operands[] = $operand; + } + + return new BooleanPredicate($operator, ...$operands); + } + + $this->schema($path . '.expression', 'Unknown predicate expression.'); + } + + private function numericOperand(mixed $raw, string $path): Operand + { + $operand = $this->operand($raw, $path); + if ($operand instanceof LiteralOperand && $operand->value instanceof StringValue) { + throw TransformationFailure::at( + 'expression_type_unsupported', + $path, + 'String literal operands are outside the bounded expression profile.', + ); + } + + return $operand; + } + + private function operand(mixed $raw, string $path): Operand + { + $object = $this->object($raw, $path); + $kind = $this->string($object['kind'] ?? null, $path . '.kind'); + if ($kind === 'variable') { + $this->exactKeys($object, ['kind', 'variable'], $path); + return new VariableOperand($this->nonEmptyString($object['variable'], $path . '.variable')); + } + if ($kind === 'literal') { + $this->exactKeys($object, ['kind', 'value'], $path); + return new LiteralOperand($this->typedValue($object['value'], $path . '.value')); + } + + $this->schema($path . '.kind', 'Operand kind must be variable or literal.'); + } + + private function targetName(mixed $value, string $path): string + { + $target = $this->nonEmptyString($value, $path); + if (str_starts_with($target, '__')) { + throw TransformationFailure::at('reserved_target_name', $path, 'Target name is reserved.'); + } + + return $target; + } + /** @param array $object */ private function recodeOperation(array $object, string $path): RecodeOperation { @@ -273,6 +448,15 @@ private function nonEmptyList(mixed $value, string $path): array return $value; } + private function integer(mixed $value, string $path): int + { + if (!is_int($value)) { + $this->schema($path, 'Value must be an integer.'); + } + + return $value; + } + /** * @param array $object * @param list $expected diff --git a/tests/Transformation/Conformance/TransformationPlan02Test.php b/tests/Transformation/Conformance/TransformationPlan02Test.php new file mode 100644 index 0000000..0e979a0 --- /dev/null +++ b/tests/Transformation/Conformance/TransformationPlan02Test.php @@ -0,0 +1,63 @@ + 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'assign', + 'target' => 'target', + 'target_mode' => 'replace', + 'value' => ['kind' => 'variable', 'variable' => 'source'], + ]], + ]; + + try { + (new PlanCodec())->fromArray($plan); + self::fail('Expected plan_schema_invalid.'); + } catch (TransformationFailure $failure) { + self::assertSame('plan_schema_invalid', $failure->diagnosticCode()); + } + } + + /** @return iterable}> */ + public static function cases(): iterable + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.2.json')['cases'] as $case) { + yield $case['id'] => [$case]; + } + } + + /** @param array $case */ + #[DataProvider('cases')] + public function testOfficialPlan02(array $case): void + { + if (is_string($case['expected_error'])) { + try { + (new PlanCodec())->fromArray($case['plan']); + self::fail('Expected ' . $case['expected_error']); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + return; + } + + $codec = new PlanCodec(); + $plan = $codec->fromArray($case['plan']); + self::assertSame($case['plan'], $plan->canonicalArray()); + self::assertSame($case['expected_plan_hash'], $codec->hash($plan)); + } +} diff --git a/tests/Transformation/Plan/BooleanPredicateTest.php b/tests/Transformation/Plan/BooleanPredicateTest.php new file mode 100644 index 0000000..935e92d --- /dev/null +++ b/tests/Transformation/Plan/BooleanPredicateTest.php @@ -0,0 +1,39 @@ +comparison('source_a'), $this->comparison('source_b')), + $this->comparison('source_c'), + ); + self::fail('Expected noncanonical_boolean_shape.'); + } catch (TransformationFailure $failure) { + self::assertSame('noncanonical_boolean_shape', $failure->diagnosticCode()); + } + } + + private function comparison(string $variable): Comparison + { + return new Comparison( + new VariableOperand($variable), + '=', + new LiteralOperand(Binary64Value::fromBits('3ff0000000000000')), + ); + } +} From d7e37b6d5945cd79e61faf5ee7b85462ce3d5599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 15:20:21 +0300 Subject: [PATCH 07/53] feat: add exact SPSS decimal binary64 conversion --- composer.json | 3 +- composer.lock | 65 ++++++++++- src/Frontend/Spss/Number/DecimalBinary64.php | 112 +++++++++++++++++++ tests/Frontend/Spss/DecimalBinary64Test.php | 86 ++++++++++++++ 4 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 src/Frontend/Spss/Number/DecimalBinary64.php create mode 100644 tests/Frontend/Spss/DecimalBinary64Test.php diff --git a/composer.json b/composer.json index 5a4d2c7..d24bc22 100644 --- a/composer.json +++ b/composer.json @@ -12,8 +12,9 @@ ] }, "require": { - "ext-pdo": "*", "php": "^8.4.1", + "ext-pdo": "*", + "brick/math": "^0.19", "openstatspec/spss-sav": "^3.0" }, "repositories": [ diff --git a/composer.lock b/composer.lock index 6eea277..de1f593 100644 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,67 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "70656320a6598594d04aefc21dfb1b74", + "content-hash": "fc67717f8a9244dbd2043737f27ac270", "packages": [ + { + "name": "brick/math", + "version": "0.19.1", + "source": { + "type": "git", + "url": "https://github.com/brick/math.git", + "reference": "a89bc96a7cf3d7b59e725afe57ccb95eb03cf6ce" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/brick/math/zipball/a89bc96a7cf3d7b59e725afe57ccb95eb03cf6ce", + "reference": "a89bc96a7cf3d7b59e725afe57ccb95eb03cf6ce", + "shasum": "" + }, + "require": { + "php": "^8.2" + }, + "require-dev": { + "phpstan/phpstan": "2.1.22", + "phpunit/phpunit": "^11.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Brick\\Math\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "Arbitrary-precision arithmetic library", + "keywords": [ + "Arbitrary-precision", + "BigInteger", + "BigRational", + "arithmetic", + "bigdecimal", + "bignum", + "bignumber", + "brick", + "decimal", + "integer", + "math", + "mathematics", + "rational" + ], + "support": { + "issues": "https://github.com/brick/math/issues", + "source": "https://github.com/brick/math/tree/0.19.1" + }, + "funding": [ + { + "url": "https://github.com/BenMorel", + "type": "github" + } + ], + "time": "2026-08-08T23:03:16+00:00" + }, { "name": "openstatspec/spss-sav", "version": "3.0.3", @@ -4668,8 +4727,8 @@ "prefer-stable": true, "prefer-lowest": false, "platform": { - "ext-pdo": "*", - "php": "^8.4.1" + "php": "^8.4.1", + "ext-pdo": "*" }, "platform-dev": {}, "platform-overrides": { diff --git a/src/Frontend/Spss/Number/DecimalBinary64.php b/src/Frontend/Spss/Number/DecimalBinary64.php new file mode 100644 index 0000000..35a5170 --- /dev/null +++ b/src/Frontend/Spss/Number/DecimalBinary64.php @@ -0,0 +1,112 @@ +minus(strlen($fraction)); + + if ($coefficient->isZero()) { + return '0000000000000000'; + } + + $adjustedDecimalExponent = $decimalExponent->plus(strlen($coefficient->toString())); + if ($adjustedDecimalExponent->isGreaterThan(309)) { + self::reject('Numeric literal exceeds the finite IEEE-754 binary64 range.'); + } + if ($adjustedDecimalExponent->isLessThanOrEqualTo(-324)) { + return '0000000000000000'; + } + + $power = $decimalExponent->toInt(); + if ($power >= 0) { + $numerator = $coefficient->multipliedBy(BigInteger::ten()->power($power)); + $denominator = BigInteger::one(); + } else { + $numerator = $coefficient; + $denominator = BigInteger::ten()->power(-$power); + } + + $exponent = self::floorLog2($numerator, $denominator); + if ($exponent > 1023) { + self::reject('Numeric literal exceeds the finite IEEE-754 binary64 range.'); + } + + if ($exponent < -1022) { + $significand = self::roundRatio($numerator->shiftedLeft(1074), $denominator); + if ($significand->isZero()) { + return '0000000000000000'; + } + + $word = $significand; + } else { + $shift = 52 - $exponent; + $significand = $shift >= 0 + ? self::roundRatio($numerator->shiftedLeft($shift), $denominator) + : self::roundRatio($numerator, $denominator->shiftedLeft(-$shift)); + + $hiddenBit = BigInteger::one()->shiftedLeft(52); + if ($significand->isEqualTo($hiddenBit->shiftedLeft(1))) { + $significand = $hiddenBit; + ++$exponent; + if ($exponent > 1023) { + self::reject('Numeric literal exceeds the finite IEEE-754 binary64 range.'); + } + } + + $word = BigInteger::of($exponent + 1023) + ->shiftedLeft(52) + ->plus($significand->minus($hiddenBit)); + } + + if ($negative) { + $word = $word->plus(BigInteger::one()->shiftedLeft(63)); + } + + return str_pad($word->toBase(16), 16, '0', STR_PAD_LEFT); + } + + private static function floorLog2(BigInteger $numerator, BigInteger $denominator): int + { + $exponent = $numerator->getBitLength() - $denominator->getBitLength(); + $comparison = $exponent >= 0 + ? $numerator->compareTo($denominator->shiftedLeft($exponent)) + : $numerator->shiftedLeft(-$exponent)->compareTo($denominator); + + return $comparison < 0 ? $exponent - 1 : $exponent; + } + + private static function roundRatio(BigInteger $numerator, BigInteger $denominator): BigInteger + { + [$quotient, $remainder] = $numerator->quotientAndRemainder($denominator); + $twice = $remainder->multipliedBy(2); + $comparison = $twice->compareTo($denominator); + if ($comparison > 0 || ($comparison === 0 && $quotient->isOdd())) { + return $quotient->plus(1); + } + + return $quotient; + } + + private static function reject(string $message): never + { + throw new SpssSyntaxException([new Diagnostic(1, 1, $message)]); + } +} diff --git a/tests/Frontend/Spss/DecimalBinary64Test.php b/tests/Frontend/Spss/DecimalBinary64Test.php new file mode 100644 index 0000000..168f3a3 --- /dev/null +++ b/tests/Frontend/Spss/DecimalBinary64Test.php @@ -0,0 +1,86 @@ + */ + public static function tokens(): iterable + { + yield 'negative zero' => ['-0', '0000000000000000']; + yield 'underflowed zero' => ['-1e-9999', '0000000000000000']; + yield 'minimum subnormal' => ['4.9406564584124654417656879286822137236505980e-324', '0000000000000001']; + yield 'one' => ['1', '3ff0000000000000']; + yield 'negative one' => ['-1', 'bff0000000000000']; + yield 'tie rounds even' => ['1.00000000000000011102230246251565404236316680908203125', '3ff0000000000000']; + yield 'tie rounds odd upward' => ['1.00000000000000033306690738754696212708950042724609375', '3ff0000000000002']; + yield 'significand carry' => ['1.99999999999999988897769753748434595763683319091796875', '4000000000000000']; + } + + #[DataProvider('invalidTokens')] + public function testRejectsInvalidOrOverflowingTokens(string $token): void + { + $this->expectException(SpssSyntaxException::class); + + DecimalBinary64::bits($token); + } + + /** @return iterable */ + public static function invalidTokens(): iterable + { + yield 'leading plus' => ['+1']; + yield 'missing integer part' => ['.5']; + yield 'missing fractional part' => ['1.']; + yield 'leading zero' => ['01']; + yield 'NaN' => ['NaN']; + yield 'infinity' => ['Infinity']; + yield 'overflow' => ['1e9999']; + } + + public function testNormalSubnormalMidpointRoundsToEvenMinimumNormal(): void + { + $midpoint = BigInteger::of(2)->power(53) + ->minus(1) + ->multipliedBy(BigInteger::of(5)->power(1075)); + + self::assertSame('0010000000000000', DecimalBinary64::bits(self::fraction($midpoint, 1075))); + } + + public function testValueBelowOverflowMidpointRoundsToMaximumFinite(): void + { + $midpoint = BigInteger::of(2)->power(54) + ->minus(1) + ->multipliedBy(BigInteger::of(2)->power(970)); + + self::assertSame('7fefffffffffffff', DecimalBinary64::bits($midpoint->minus(1)->toString())); + } + + public function testOverflowMidpointIsRejected(): void + { + $midpoint = BigInteger::of(2)->power(54) + ->minus(1) + ->multipliedBy(BigInteger::of(2)->power(970)); + $this->expectException(SpssSyntaxException::class); + + DecimalBinary64::bits($midpoint->toString()); + } + + private static function fraction(BigInteger $numerator, int $decimalPlaces): string + { + return '0.' . str_pad($numerator->toString(), $decimalPlaces, '0', STR_PAD_LEFT); + } +} From d4af7affa0bcb3d430d4ab3df9d8239d6f4c32ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 15:50:24 +0300 Subject: [PATCH 08/53] feat: parse SPSS frontend 0.2 commands --- src/Frontend/Spss/Ast/BooleanPredicate.php | 22 ++ src/Frontend/Spss/Ast/Comparison.php | 22 ++ src/Frontend/Spss/Ast/ComputeStatement.php | 21 ++ src/Frontend/Spss/Ast/ExecuteStatement.php | 4 +- src/Frontend/Spss/Ast/ExpressionOperand.php | 12 + src/Frontend/Spss/Ast/FormatTarget.php | 18 ++ src/Frontend/Spss/Ast/FormatsStatement.php | 18 ++ src/Frontend/Spss/Ast/IfStatement.php | 22 ++ src/Frontend/Spss/Ast/LiteralOperand.php | 17 ++ src/Frontend/Spss/Ast/Predicate.php | 12 + src/Frontend/Spss/Ast/VariableLevelGroup.php | 17 ++ .../Spss/Ast/VariableLevelStatement.php | 18 ++ src/Frontend/Spss/Ast/VariableOperand.php | 17 ++ src/Frontend/Spss/Lexer.php | 158 ++++++++-- src/Frontend/Spss/Parser.php | 281 +++++++++++++++++- src/Frontend/Spss/Request/InputSchema.php | 11 + src/Frontend/Spss/Request/InputVariable.php | 20 ++ .../Spss/Request/SpssFrontendRequest.php | 15 + src/Frontend/Spss/SpssSyntaxException.php | 23 +- src/Frontend/Spss/Token.php | 38 ++- src/Frontend/Spss/TokenType.php | 15 + tests/Frontend/Spss/LexerTest.php | 53 +++- tests/Frontend/Spss/ParserTest.php | 208 ++++++++++++- 23 files changed, 983 insertions(+), 59 deletions(-) create mode 100644 src/Frontend/Spss/Ast/BooleanPredicate.php create mode 100644 src/Frontend/Spss/Ast/Comparison.php create mode 100644 src/Frontend/Spss/Ast/ComputeStatement.php create mode 100644 src/Frontend/Spss/Ast/ExpressionOperand.php create mode 100644 src/Frontend/Spss/Ast/FormatTarget.php create mode 100644 src/Frontend/Spss/Ast/FormatsStatement.php create mode 100644 src/Frontend/Spss/Ast/IfStatement.php create mode 100644 src/Frontend/Spss/Ast/LiteralOperand.php create mode 100644 src/Frontend/Spss/Ast/Predicate.php create mode 100644 src/Frontend/Spss/Ast/VariableLevelGroup.php create mode 100644 src/Frontend/Spss/Ast/VariableLevelStatement.php create mode 100644 src/Frontend/Spss/Ast/VariableOperand.php create mode 100644 src/Frontend/Spss/Request/InputSchema.php create mode 100644 src/Frontend/Spss/Request/InputVariable.php create mode 100644 src/Frontend/Spss/Request/SpssFrontendRequest.php diff --git a/src/Frontend/Spss/Ast/BooleanPredicate.php b/src/Frontend/Spss/Ast/BooleanPredicate.php new file mode 100644 index 0000000..1e58826 --- /dev/null +++ b/src/Frontend/Spss/Ast/BooleanPredicate.php @@ -0,0 +1,22 @@ + $operands */ + public function __construct( + public string $operator, + public array $operands, + public SourceSpan $span, + ) {} + + public function span(): SourceSpan + { + return $this->span; + } +} diff --git a/src/Frontend/Spss/Ast/Comparison.php b/src/Frontend/Spss/Ast/Comparison.php new file mode 100644 index 0000000..122d43e --- /dev/null +++ b/src/Frontend/Spss/Ast/Comparison.php @@ -0,0 +1,22 @@ +span; + } +} diff --git a/src/Frontend/Spss/Ast/ComputeStatement.php b/src/Frontend/Spss/Ast/ComputeStatement.php new file mode 100644 index 0000000..5627ccc --- /dev/null +++ b/src/Frontend/Spss/Ast/ComputeStatement.php @@ -0,0 +1,21 @@ +span->startLine; + } +} diff --git a/src/Frontend/Spss/Ast/ExecuteStatement.php b/src/Frontend/Spss/Ast/ExecuteStatement.php index 16483e3..49057f7 100644 --- a/src/Frontend/Spss/Ast/ExecuteStatement.php +++ b/src/Frontend/Spss/Ast/ExecuteStatement.php @@ -4,9 +4,11 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ExecuteStatement implements Statement { - public function __construct(public int $sourceLine) {} + public function __construct(public int $sourceLine, public SourceSpan $span) {} public function line(): int { diff --git a/src/Frontend/Spss/Ast/ExpressionOperand.php b/src/Frontend/Spss/Ast/ExpressionOperand.php new file mode 100644 index 0000000..1f85d96 --- /dev/null +++ b/src/Frontend/Spss/Ast/ExpressionOperand.php @@ -0,0 +1,12 @@ + $targets */ + public function __construct(public array $targets, public SourceSpan $span) {} + + public function line(): int + { + return $this->span->startLine; + } +} diff --git a/src/Frontend/Spss/Ast/IfStatement.php b/src/Frontend/Spss/Ast/IfStatement.php new file mode 100644 index 0000000..45c6698 --- /dev/null +++ b/src/Frontend/Spss/Ast/IfStatement.php @@ -0,0 +1,22 @@ +span->startLine; + } +} diff --git a/src/Frontend/Spss/Ast/LiteralOperand.php b/src/Frontend/Spss/Ast/LiteralOperand.php new file mode 100644 index 0000000..d5040bf --- /dev/null +++ b/src/Frontend/Spss/Ast/LiteralOperand.php @@ -0,0 +1,17 @@ +span; + } +} diff --git a/src/Frontend/Spss/Ast/Predicate.php b/src/Frontend/Spss/Ast/Predicate.php new file mode 100644 index 0000000..5af5282 --- /dev/null +++ b/src/Frontend/Spss/Ast/Predicate.php @@ -0,0 +1,12 @@ + $variables */ + public function __construct( + public array $variables, + public string $level, + public SourceSpan $span, + ) {} +} diff --git a/src/Frontend/Spss/Ast/VariableLevelStatement.php b/src/Frontend/Spss/Ast/VariableLevelStatement.php new file mode 100644 index 0000000..eff3dcd --- /dev/null +++ b/src/Frontend/Spss/Ast/VariableLevelStatement.php @@ -0,0 +1,18 @@ + $groups */ + public function __construct(public array $groups, public SourceSpan $span) {} + + public function line(): int + { + return $this->span->startLine; + } +} diff --git a/src/Frontend/Spss/Ast/VariableOperand.php b/src/Frontend/Spss/Ast/VariableOperand.php new file mode 100644 index 0000000..b89b094 --- /dev/null +++ b/src/Frontend/Spss/Ast/VariableOperand.php @@ -0,0 +1,17 @@ +span; + } +} diff --git a/src/Frontend/Spss/Lexer.php b/src/Frontend/Spss/Lexer.php index cb55b51..2efd681 100644 --- a/src/Frontend/Spss/Lexer.php +++ b/src/Frontend/Spss/Lexer.php @@ -4,15 +4,16 @@ namespace OpenStatSpec\Frontend\Spss; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Diagnostic\TransformationDiagnostic; + final class Lexer { /** @return list */ public function tokenize(string $source): array { - $source = preg_replace('/^\xEF\xBB\xBF/', '', $source) ?? $source; - $source = str_replace(["\r\n", "\r"], "\n", $source); $tokens = []; - $offset = 0; + $offset = str_starts_with($source, "\xEF\xBB\xBF") ? 3 : 0; $line = 1; $column = 1; $atStatementStart = true; @@ -21,21 +22,23 @@ public function tokenize(string $source): array while ($offset < $length) { $character = $this->characterAt($source, $offset, $line, $column); if (ctype_space($character)) { + if ($character === "\r") { + $this->advanceCarriageReturn($source, $offset, $line, $column); + continue; + } $this->advance($character, $offset, $line, $column); continue; } if ($atStatementStart && $character === '*') { - while ($offset < $length && $this->characterAt($source, $offset, $line, $column) !== '.') { - $this->advance($this->characterAt($source, $offset, $line, $column), $offset, $line, $column); - } - if ($offset === $length) { - $this->fail($line, $column, 'Comment is missing its period terminator.'); - } - $this->advance('.', $offset, $line, $column); - continue; + $this->fail( + new SourceSpan($offset, $offset + 1, $line, $column, $line, $column + 1), + 'Leading-star comments are not supported.', + 'unsupported_spss_command', + ); } + $tokenOffset = $offset; $tokenLine = $line; $tokenColumn = $column; $atStatementStart = false; @@ -45,18 +48,39 @@ public function tokenize(string $source): array continue; } - $punctuation = match ($character) { - '(' => TokenType::LeftParenthesis, - ')' => TokenType::RightParenthesis, - '=' => TokenType::Equals, - ',' => TokenType::Comma, - '/' => TokenType::Slash, - '.' => TokenType::Terminator, + $next = $source[$offset + 1] ?? ''; + $punctuation = match (true) { + $character === '<' && $next === '=' => TokenType::LessThanOrEqual, + $character === '>' && $next === '=' => TokenType::GreaterThanOrEqual, + $character === '<' => TokenType::LessThan, + $character === '>' => TokenType::GreaterThan, + $character === '(' => TokenType::LeftParenthesis, + $character === ')' => TokenType::RightParenthesis, + $character === '=' => TokenType::Equals, + $character === ',' => TokenType::Comma, + $character === '/' => TokenType::Slash, + $character === '.' => TokenType::Terminator, + $character === '+' || $character === '-' || $character === '*' => TokenType::ArithmeticOperator, default => null, }; if ($punctuation !== null) { - $tokens[] = new Token($punctuation, $character, $tokenLine, $tokenColumn); + $lexeme = ($punctuation === TokenType::LessThanOrEqual || $punctuation === TokenType::GreaterThanOrEqual) + ? $character . $next + : $character; $this->advance($character, $offset, $line, $column); + if (strlen($lexeme) === 2) { + $this->advance($next, $offset, $line, $column); + } + $tokens[] = new Token( + $punctuation, + $lexeme, + $tokenLine, + $tokenColumn, + $tokenOffset, + $offset, + $line, + $column, + ); if ($punctuation === TokenType::Terminator) { $atStatementStart = true; } @@ -77,15 +101,41 @@ public function tokenize(string $source): array } $this->advance($identifierCharacter, $offset, $line, $column); } - $tokens[] = new Token(TokenType::Identifier, substr($source, $start, $offset - $start), $tokenLine, $tokenColumn); + $lexeme = substr($source, $start, $offset - $start); + $type = match (strtoupper($lexeme)) { + 'COMPUTE' => TokenType::Compute, + 'IF' => TokenType::If, + 'AND' => TokenType::And, + 'OR' => TokenType::Or, + 'FORMATS' => TokenType::Formats, + 'VARIABLE' => TokenType::Variable, + 'LEVEL' => TokenType::Level, + 'NOMINAL' => TokenType::Nominal, + 'ORDINAL' => TokenType::Ordinal, + 'SCALE' => TokenType::Scale, + default => TokenType::Identifier, + }; + $tokens[] = new Token( + $type, + $lexeme, + $tokenLine, + $tokenColumn, + $tokenOffset, + $offset, + $line, + $column, + ); continue; } $encoded = json_encode($character, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE); - $this->fail($line, $column, sprintf('Unexpected character %s.', $encoded === false ? '?' : $encoded)); + $this->fail( + new SourceSpan($offset, $offset + strlen($character), $line, $column, $line, $column + 1), + sprintf('Unexpected character %s.', $encoded === false ? '?' : $encoded), + ); } - $tokens[] = new Token(TokenType::EndOfFile, '', $line, $column); + $tokens[] = new Token(TokenType::EndOfFile, '', $line, $column, $offset, $offset, $line, $column); return $tokens; } @@ -95,6 +145,7 @@ private function string(string $source, int &$offset, int &$line, int &$column): $quote = $this->characterAt($source, $offset, $line, $column); $tokenLine = $line; $tokenColumn = $column; + $tokenOffset = $offset; $this->advance($quote, $offset, $line, $column); $value = ''; $length = strlen($source); @@ -102,8 +153,13 @@ private function string(string $source, int &$offset, int &$line, int &$column): while ($offset < $length) { $character = $this->characterAt($source, $offset, $line, $column); if ($character !== $quote) { - $value .= $character; - $this->advance($character, $offset, $line, $column); + if ($character === "\r") { + $value .= "\n"; + $this->advanceCarriageReturn($source, $offset, $line, $column); + } else { + $value .= $character; + $this->advance($character, $offset, $line, $column); + } continue; } $this->advance($character, $offset, $line, $column); @@ -113,26 +169,51 @@ private function string(string $source, int &$offset, int &$line, int &$column): continue; } - return new Token(TokenType::String, $value, $tokenLine, $tokenColumn); + return new Token( + TokenType::String, + $value, + $tokenLine, + $tokenColumn, + $tokenOffset, + $offset, + $line, + $column, + ); } - $this->fail($tokenLine, $tokenColumn, 'String literal is not closed.'); + $this->fail( + new SourceSpan($tokenOffset, $offset, $tokenLine, $tokenColumn, $line, $column), + 'String literal is not closed.', + ); } private function number(string $source, int &$offset, int &$line, int &$column): Token { $tokenLine = $line; $tokenColumn = $column; + $tokenOffset = $offset; $remaining = substr($source, $offset); - if (preg_match('/^[+-]?(?:(?:\d+(?:\.\d*)?)|(?:\.\d+))(?:[Ee][+-]?\d+)?/', $remaining, $matches) !== 1) { - $this->fail($line, $column, 'Invalid numeric literal.'); + if (preg_match('/^[+-]?(?:\d+(?:\.\d+)?|\.\d+)(?:[Ee][+-]?\d+)?/', $remaining, $matches) !== 1) { + $this->fail( + new SourceSpan($offset, $offset + 1, $line, $column, $line, $column + 1), + 'Invalid numeric literal.', + ); } $lexeme = $matches[0]; foreach (str_split($lexeme) as $character) { $this->advance($character, $offset, $line, $column); } - return new Token(TokenType::Number, $lexeme, $tokenLine, $tokenColumn); + return new Token( + TokenType::Number, + $lexeme, + $tokenLine, + $tokenColumn, + $tokenOffset, + $offset, + $line, + $column, + ); } private function startsNumber(string $source, int $offset): bool @@ -176,8 +257,7 @@ private function characterAt(string $source, int $offset, int $line, int $column || preg_match('/\A.\z/usD', $character) !== 1 ) { $this->fail( - $line, - $column, + new SourceSpan($offset, $offset + 1, $line, $column, $line, $column + 1), sprintf('Invalid UTF-8 sequence beginning with byte 0x%02X.', $firstByte), ); } @@ -196,8 +276,20 @@ private function advance(string $character, int &$offset, int &$line, int &$colu } } - private function fail(int $line, int $column, string $message): never + private function advanceCarriageReturn(string $source, int &$offset, int &$line, int &$column): void + { + ++$offset; + if (($source[$offset] ?? '') === "\n") { + ++$offset; + } + ++$line; + $column = 1; + } + + private function fail(SourceSpan $span, string $message, string $code = 'spss_syntax_error'): never { - throw new SpssSyntaxException([new Diagnostic($line, $column, $message)]); + throw new SpssSyntaxException([ + new TransformationDiagnostic($code, '$.source_text', $message, $span), + ]); } } diff --git a/src/Frontend/Spss/Parser.php b/src/Frontend/Spss/Parser.php index 5d994fd..ef2294f 100644 --- a/src/Frontend/Spss/Parser.php +++ b/src/Frontend/Spss/Parser.php @@ -4,11 +4,20 @@ namespace OpenStatSpec\Frontend\Spss; +use OpenStatSpec\Frontend\Spss\Ast\BooleanPredicate; +use OpenStatSpec\Frontend\Spss\Ast\Comparison; +use OpenStatSpec\Frontend\Spss\Ast\ComputeStatement; use OpenStatSpec\Frontend\Spss\Ast\DeleteVariablesStatement; use OpenStatSpec\Frontend\Spss\Ast\ElseInput; use OpenStatSpec\Frontend\Spss\Ast\ExecuteStatement; +use OpenStatSpec\Frontend\Spss\Ast\ExpressionOperand; +use OpenStatSpec\Frontend\Spss\Ast\FormatsStatement; +use OpenStatSpec\Frontend\Spss\Ast\FormatTarget; +use OpenStatSpec\Frontend\Spss\Ast\IfStatement; +use OpenStatSpec\Frontend\Spss\Ast\LiteralOperand; use OpenStatSpec\Frontend\Spss\Ast\MissingInput; use OpenStatSpec\Frontend\Spss\Ast\Program; +use OpenStatSpec\Frontend\Spss\Ast\Predicate; use OpenStatSpec\Frontend\Spss\Ast\RangeInput; use OpenStatSpec\Frontend\Spss\Ast\RecodeInput; use OpenStatSpec\Frontend\Spss\Ast\RecodeOutput; @@ -23,6 +32,13 @@ use OpenStatSpec\Frontend\Spss\Ast\ValueLabelGroup; use OpenStatSpec\Frontend\Spss\Ast\ValueLabelsStatement; use OpenStatSpec\Frontend\Spss\Ast\VariableLabelsStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableLevelGroup; +use OpenStatSpec\Frontend\Spss\Ast\VariableLevelStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableOperand; +use OpenStatSpec\Frontend\Spss\Number\DecimalBinary64; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Diagnostic\TransformationDiagnostic; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; final class Parser { @@ -49,8 +65,13 @@ public function parseTokens(array $tokens): Program if ($this->matchKeyword('RECODE')) { $statements[] = $this->recode($command); } elseif ($this->matchKeyword('VARIABLE')) { - $this->consumeKeyword('LABELS', 'Expected LABELS after VARIABLE.'); - $statements[] = $this->variableLabels($command); + if ($this->matchKeyword('LABELS')) { + $statements[] = $this->variableLabels($command); + } elseif ($this->matchKeyword('LEVEL')) { + $statements[] = $this->variableLevel($command); + } else { + $this->fail($this->current(), 'Expected LABELS or LEVEL after VARIABLE.'); + } } elseif ($this->matchKeyword('VALUE')) { $this->consumeKeyword('LABELS', 'Expected LABELS after VALUE.'); $statements[] = $this->valueLabels($command); @@ -61,14 +82,26 @@ public function parseTokens(array $tokens): Program $this->consumeKeyword('LAB', 'Expected LAB after VAL.'); $statements[] = $this->valueLabels($command); } elseif ($this->matchKeyword('EXECUTE')) { - $statements[] = new ExecuteStatement($command->line); + $statements[] = new ExecuteStatement($command->line, $command->span); + } elseif ($this->matchKeyword('COMPUTE')) { + $statements[] = $this->compute($command); + } elseif ($this->matchKeyword('IF')) { + $statements[] = $this->ifStatement($command); + } elseif ($this->matchKeyword('FORMATS')) { + $statements[] = $this->formats($command); } elseif ($this->matchKeyword('STRING')) { $statements[] = $this->string($command); } elseif ($this->matchKeyword('DELETE')) { $this->consumeKeyword('VARIABLES', 'Expected VARIABLES after DELETE.'); $statements[] = $this->deleteVariables($command); } else { - $this->fail($command, sprintf('Unsupported SPSS command %s.', $command->lexeme === '' ? '' : $command->lexeme)); + $code = $command->isWord() + ? 'unsupported_spss_command' + : 'spss_syntax_error'; + $this->fail($command, sprintf( + 'Unsupported SPSS command %s.', + $command->lexeme === '' ? '' : $command->lexeme, + ), $code); } $this->consume(TokenType::Terminator, 'Expected a period after the SPSS command.'); @@ -77,12 +110,230 @@ public function parseTokens(array $tokens): Program return new Program($statements); } + private function compute(Token $command): ComputeStatement + { + $target = $this->consumeIdentifier('Expected a target variable after COMPUTE.'); + $this->consume(TokenType::Equals, 'Expected = after the COMPUTE target.'); + $expression = $this->expressionOperand('Expected a numeric operand after =.'); + $this->rejectUnsupportedExpressionContinuation(); + + return new ComputeStatement( + $target->lexeme, + $expression, + SourceSpan::cover($command->span, $expression->span()), + ); + } + + private function ifStatement(Token $command): IfStatement + { + $this->consume(TokenType::LeftParenthesis, 'IF requires parentheses around the complete predicate.'); + $predicate = $this->parsePredicate(); + $this->consume(TokenType::RightParenthesis, 'Expected ) after the complete IF predicate.'); + $target = $this->consumeIdentifier('Expected an existing target variable after the IF predicate.'); + $this->consume(TokenType::Equals, 'Expected = after the IF target.'); + $expression = $this->expressionOperand('Expected a numeric operand after =.'); + $this->rejectUnsupportedExpressionContinuation(); + + return new IfStatement( + $predicate, + $target->lexeme, + $expression, + SourceSpan::cover($command->span, $expression->span()), + ); + } + + private function parsePredicate(): Predicate + { + return $this->parseOr(); + } + + private function parseOr(): Predicate + { + $predicate = $this->parseAnd(); + while ($this->matchKeyword('OR')) { + $predicate = $this->merge('or', $predicate, $this->parseAnd()); + } + + return $predicate; + } + + private function parseAnd(): Predicate + { + $predicate = $this->comparison(); + while ($this->matchKeyword('AND')) { + $predicate = $this->merge('and', $predicate, $this->comparison()); + } + + return $predicate; + } + + private function comparison(): Predicate + { + if ($this->match(TokenType::LeftParenthesis)) { + $predicate = $this->parsePredicate(); + $this->consume(TokenType::RightParenthesis, 'Expected ) after the parenthesized predicate.'); + + return $predicate; + } + if ($this->current()->isKeyword('NOT')) { + $this->fail($this->current(), 'NOT is not supported in numeric predicates.', 'expression_type_unsupported'); + } + + $left = $this->expressionOperand('Expected a numeric comparison operand.'); + $operator = $this->comparisonOperator(); + $right = $this->expressionOperand('Expected a numeric comparison operand after the operator.'); + $this->rejectUnsupportedExpressionContinuation(); + + return new Comparison($left, $operator->lexeme, $right, SourceSpan::cover($left->span(), $right->span())); + } + + private function comparisonOperator(): Token + { + if ($this->check(TokenType::ArithmeticOperator) || $this->check(TokenType::Slash)) { + $this->fail( + $this->current(), + 'Arithmetic expressions are not supported.', + 'expression_type_unsupported', + ); + } + foreach ([ + TokenType::Equals, + TokenType::LessThan, + TokenType::LessThanOrEqual, + TokenType::GreaterThan, + TokenType::GreaterThanOrEqual, + ] as $type) { + if ($this->match($type)) { + return $this->previous(); + } + } + + $this->fail($this->current(), 'Expected a supported comparison operator.'); + } + + private function expressionOperand(string $message): ExpressionOperand + { + if ($this->match(TokenType::String)) { + $this->fail($this->previous(), 'String expressions are not supported.', 'expression_type_unsupported'); + } + if ($this->current()->isKeyword('NAN') || $this->current()->isKeyword('INFINITY')) { + $this->fail($this->current(), 'NaN and infinity are not finite numeric operands.'); + } + if ($this->match(TokenType::Number)) { + $token = $this->previous(); + try { + DecimalBinary64::bits($token->lexeme); + } catch (TransformationFailure $failure) { + $this->fail($token, $failure->getMessage()); + } + + return new LiteralOperand($token->lexeme, $token->span); + } + if ($this->current()->isWord()) { + $token = $this->advance(); + + return new VariableOperand($token->lexeme, $token->span); + } + + $this->fail($this->current(), $message); + } + + private function rejectUnsupportedExpressionContinuation(): void + { + if ($this->check(TokenType::ArithmeticOperator) || $this->check(TokenType::Slash)) { + $this->fail( + $this->current(), + 'Arithmetic expressions are not supported.', + 'expression_type_unsupported', + ); + } + } + + private function merge(string $operator, Predicate $left, Predicate $right): BooleanPredicate + { + $operands = $left instanceof BooleanPredicate && $left->operator === $operator + ? $left->operands + : [$left]; + array_push( + $operands, + ...($right instanceof BooleanPredicate && $right->operator === $operator ? $right->operands : [$right]), + ); + + return new BooleanPredicate($operator, $operands, SourceSpan::cover($left->span(), $right->span())); + } + + private function formats(Token $command): FormatsStatement + { + $targets = []; + do { + $variable = $this->consumeIdentifier('FORMATS requires a variable and format pair.'); + $this->consume(TokenType::LeftParenthesis, 'Expected ( before the format.'); + $familyAndWidth = $this->consumeIdentifier('Expected an F format such as F8.2.'); + if (preg_match('/\AF([1-9][0-9]*)\z/iD', $familyAndWidth->lexeme, $matches) !== 1) { + $this->fail($familyAndWidth, 'Only the numeric F format family is supported.', 'invalid_format'); + } + $decimalToken = $this->consume(TokenType::Number, 'F format requires . followed by decimals.'); + if ( + $familyAndWidth->span->endOffset !== $decimalToken->span->startOffset + || preg_match('/\A\.([0-9]+)\z/D', $decimalToken->lexeme, $decimalMatches) !== 1 + ) { + $this->fail($decimalToken, 'F format must use the Fwidth.decimals form.', 'invalid_format'); + } + $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after the format.'); + $targets[] = new FormatTarget( + $variable->lexeme, + 'F', + (int) $matches[1], + (int) $decimalMatches[1], + SourceSpan::cover($variable->span, $rightParenthesis->span), + ); + } while ($this->current()->isWord()); + + return new FormatsStatement( + $targets, + SourceSpan::cover($command->span, $targets[array_key_last($targets)]->span), + ); + } + + private function variableLevel(Token $command): VariableLevelStatement + { + $groups = []; + do { + $variables = []; + $first = $this->consumeIdentifier('VARIABLE LEVEL requires at least one variable.'); + $variables[] = $first->lexeme; + while ($this->current()->isWord()) { + $variables[] = $this->advance()->lexeme; + } + $this->consume(TokenType::LeftParenthesis, 'Expected ( before the measurement level.'); + $level = $this->current(); + if ( + !$this->matchKeyword('NOMINAL') + && !$this->matchKeyword('ORDINAL') + && !$this->matchKeyword('SCALE') + ) { + $this->fail($level, 'Expected NOMINAL, ORDINAL, or SCALE.'); + } + $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after the measurement level.'); + $groups[] = new VariableLevelGroup( + $variables, + strtolower($level->lexeme), + SourceSpan::cover($first->span, $rightParenthesis->span), + ); + } while ($this->match(TokenType::Slash)); + + return new VariableLevelStatement( + $groups, + SourceSpan::cover($command->span, $groups[array_key_last($groups)]->span), + ); + } + private function recode(Token $command): RecodeStatement { $sources = []; do { $sources[] = $this->consumeIdentifier('Expected a source variable after RECODE.')->lexeme; - } while ($this->check(TokenType::Identifier) && !$this->current()->isKeyword('INTO')); + } while ($this->current()->isWord() && !$this->current()->isKeyword('INTO')); $rules = []; while ($this->match(TokenType::LeftParenthesis)) { @@ -100,7 +351,7 @@ private function recode(Token $command): RecodeStatement if ($this->matchKeyword('INTO')) { do { $targets[] = $this->consumeIdentifier('Expected a target variable after INTO.')->lexeme; - } while ($this->check(TokenType::Identifier)); + } while ($this->current()->isWord()); } return new RecodeStatement($command->line, $sources, $rules, $targets); @@ -165,7 +416,7 @@ private function valueLabels(Token $command): ValueLabelsStatement $groups = []; do { $variables = []; - while ($this->check(TokenType::Identifier)) { + while ($this->current()->isWord()) { $variables[] = $this->advance()->lexeme; } if ($variables === []) { @@ -196,7 +447,7 @@ private function string(Token $command): StringStatement $this->fail($this->previous(), 'STRING variable ranges using TO are not supported.'); } $variables[] = $variable; - } while ($this->check(TokenType::Identifier)); + } while ($this->current()->isWord()); $this->consume(TokenType::LeftParenthesis, 'Expected a width declaration in STRING.'); $width = $this->consume(TokenType::Identifier, 'Expected a string width such as A20.')->lexeme; if (preg_match('/\AA([1-9][0-9]*)\z/i', $width, $matches) !== 1) { @@ -220,7 +471,7 @@ private function deleteVariables(Token $command): DeleteVariablesStatement $this->fail($this->previous(), 'DELETE VARIABLES ranges using TO are not supported.'); } $variables[] = $variable; - } while ($this->check(TokenType::Identifier)); + } while ($this->current()->isWord()); return new DeleteVariablesStatement($command->line, $variables); } @@ -243,7 +494,11 @@ private function scalar(string $message): ScalarValue private function consumeIdentifier(string $message): Token { - return $this->consume(TokenType::Identifier, $message); + if (!$this->current()->isWord()) { + $this->fail($this->current(), $message); + } + + return $this->advance(); } private function consumeKeyword(string $keyword, string $message): Token @@ -304,8 +559,10 @@ private function previous(): Token return $this->tokens[$this->position - 1]; } - private function fail(Token $token, string $message): never + private function fail(Token $token, string $message, string $code = 'spss_syntax_error'): never { - throw new SpssSyntaxException([new Diagnostic($token->line, $token->column, $message)]); + throw new SpssSyntaxException([ + new TransformationDiagnostic($code, '$.source_text', $message, $token->span), + ]); } } diff --git a/src/Frontend/Spss/Request/InputSchema.php b/src/Frontend/Spss/Request/InputSchema.php new file mode 100644 index 0000000..0b99407 --- /dev/null +++ b/src/Frontend/Spss/Request/InputSchema.php @@ -0,0 +1,11 @@ + $variables */ + public function __construct(public array $variables) {} +} diff --git a/src/Frontend/Spss/Request/InputVariable.php b/src/Frontend/Spss/Request/InputVariable.php new file mode 100644 index 0000000..ff80040 --- /dev/null +++ b/src/Frontend/Spss/Request/InputVariable.php @@ -0,0 +1,20 @@ +, label: string}> $valueLabels */ + public function __construct( + public string $name, + public string $storageKind, + public ?string $variableLabel = null, + public array $valueLabels = [], + public ?string $formatFamily = null, + public ?int $width = null, + public ?int $decimals = null, + public ?string $measurementLevel = null, + ) {} +} diff --git a/src/Frontend/Spss/Request/SpssFrontendRequest.php b/src/Frontend/Spss/Request/SpssFrontendRequest.php new file mode 100644 index 0000000..494ad43 --- /dev/null +++ b/src/Frontend/Spss/Request/SpssFrontendRequest.php @@ -0,0 +1,15 @@ + $diagnostics + * @param non-empty-list $diagnostics */ - public function __construct(public readonly array $diagnostics) + public function __construct(array $diagnostics) { - $first = $diagnostics[0]; - parent::__construct(sprintf('SPSS syntax error at %d:%d: %s', $first->line, $first->column, $first->message)); + parent::__construct(array_map( + static fn(Diagnostic|TransformationDiagnostic $diagnostic): TransformationDiagnostic => $diagnostic instanceof TransformationDiagnostic + ? $diagnostic + : new TransformationDiagnostic( + 'spss_syntax_error', + '$.source_text', + $diagnostic->message, + new SourceSpan(0, 0, $diagnostic->line, $diagnostic->column, $diagnostic->line, $diagnostic->column), + ), + $diagnostics, + )); } } diff --git a/src/Frontend/Spss/Token.php b/src/Frontend/Spss/Token.php index 5a68650..cc3711a 100644 --- a/src/Frontend/Spss/Token.php +++ b/src/Frontend/Spss/Token.php @@ -4,17 +4,51 @@ namespace OpenStatSpec\Frontend\Spss; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class Token { + public SourceSpan $span; + public function __construct( public TokenType $type, public string $lexeme, public int $line, public int $column, - ) {} + int $startOffset, + int $endOffset, + int $endLine, + int $endColumn, + ) { + $this->span = new SourceSpan( + $startOffset, + $endOffset, + $line, + $column, + $endLine, + $endColumn, + ); + } public function isKeyword(string $keyword): bool { - return $this->type === TokenType::Identifier && strcasecmp($this->lexeme, $keyword) === 0; + return $this->isWord() && strcasecmp($this->lexeme, $keyword) === 0; + } + + public function isWord(): bool + { + return in_array($this->type, [ + TokenType::Identifier, + TokenType::Compute, + TokenType::If, + TokenType::And, + TokenType::Or, + TokenType::Formats, + TokenType::Variable, + TokenType::Level, + TokenType::Nominal, + TokenType::Ordinal, + TokenType::Scale, + ], true); } } diff --git a/src/Frontend/Spss/TokenType.php b/src/Frontend/Spss/TokenType.php index 46beb41..f369b98 100644 --- a/src/Frontend/Spss/TokenType.php +++ b/src/Frontend/Spss/TokenType.php @@ -7,11 +7,26 @@ enum TokenType { case Identifier; + case Compute; + case If; + case And; + case Or; + case Formats; + case Variable; + case Level; + case Nominal; + case Ordinal; + case Scale; case Number; case String; case LeftParenthesis; case RightParenthesis; case Equals; + case LessThan; + case LessThanOrEqual; + case GreaterThan; + case GreaterThanOrEqual; + case ArithmeticOperator; case Comma; case Slash; case Terminator; diff --git a/tests/Frontend/Spss/LexerTest.php b/tests/Frontend/Spss/LexerTest.php index 71b437e..f1b75cf 100644 --- a/tests/Frontend/Spss/LexerTest.php +++ b/tests/Frontend/Spss/LexerTest.php @@ -13,10 +13,10 @@ final class LexerTest extends TestCase { public function testTokenizesMultilineSyntaxAndPreservesQuotedWhitespaceAndEscapes(): void { - $tokens = (new Lexer())->tokenize("* generated comment.\nVARIABLE LABELS score 'A score''s label'.\n"); + $tokens = (new Lexer())->tokenize("\nVARIABLE LABELS score 'A score''s label'.\n"); self::assertSame( - [TokenType::Identifier, TokenType::Identifier, TokenType::Identifier, TokenType::String, TokenType::Terminator, TokenType::EndOfFile], + [TokenType::Variable, TokenType::Identifier, TokenType::Identifier, TokenType::String, TokenType::Terminator, TokenType::EndOfFile], array_column($tokens, 'type'), ); self::assertSame("A score's label", $tokens[3]->lexeme); @@ -57,9 +57,54 @@ public function testRejectsUnknownCharactersWithPositionedDiagnostic(): void (new Lexer())->tokenize('RECODE score (`=1).'); self::fail('Unknown syntax unexpectedly tokenized.'); } catch (SpssSyntaxException $exception) { - self::assertSame(1, $exception->diagnostics[0]->line); - self::assertSame(15, $exception->diagnostics[0]->column); + $span = $exception->diagnostics[0]->span; + self::assertNotNull($span); + self::assertSame(1, $span->startLine); + self::assertSame(15, $span->startColumn); self::assertStringContainsString('Unexpected character', $exception->diagnostics[0]->message); } } + + public function testTokenizesFrontend02KeywordsComparisonsAndExactBoundaries(): void + { + $tokens = (new Lexer())->tokenize("compute target = 0.\nIF (a >= 1 AND b < 2 OR c <= 3) target = 4."); + + self::assertSame(TokenType::Compute, $tokens[0]->type); + self::assertSame(TokenType::If, $tokens[5]->type); + self::assertSame(TokenType::GreaterThanOrEqual, $tokens[8]->type); + self::assertSame(TokenType::And, $tokens[10]->type); + self::assertSame(TokenType::LessThan, $tokens[12]->type); + self::assertSame(TokenType::Or, $tokens[14]->type); + self::assertSame(TokenType::LessThanOrEqual, $tokens[16]->type); + self::assertSame(0, $tokens[0]->span->startOffset); + self::assertSame(7, $tokens[0]->span->endOffset); + self::assertSame(1, $tokens[0]->span->startLine); + self::assertSame(1, $tokens[0]->span->startColumn); + self::assertSame(1, $tokens[0]->span->endLine); + self::assertSame(8, $tokens[0]->span->endColumn); + self::assertSame(20, $tokens[5]->span->startOffset); + self::assertSame(2, $tokens[5]->span->startLine); + self::assertSame(1, $tokens[5]->span->startColumn); + } + + public function testOffsetsRemainExactAcrossCrLfLineEndings(): void + { + $tokens = (new Lexer())->tokenize("COMPUTE x = 0.\r\nEXECUTE."); + + self::assertSame(16, $tokens[5]->span->startOffset); + self::assertSame(2, $tokens[5]->span->startLine); + self::assertSame(1, $tokens[5]->span->startColumn); + } + + public function testNormalizesBareCrInsideStringsWithoutLosingExactOffsets(): void + { + $tokens = (new Lexer())->tokenize("VARIABLE LABELS x 'a\rb'.\rEXECUTE."); + + self::assertSame("a\nb", $tokens[3]->lexeme); + self::assertSame(1, $tokens[3]->span->startLine); + self::assertSame(2, $tokens[3]->span->endLine); + self::assertSame(23, $tokens[3]->span->endOffset); + self::assertSame(3, $tokens[5]->span->startLine); + self::assertSame(25, $tokens[5]->span->startOffset); + } } diff --git a/tests/Frontend/Spss/ParserTest.php b/tests/Frontend/Spss/ParserTest.php index ed46790..937bdab 100644 --- a/tests/Frontend/Spss/ParserTest.php +++ b/tests/Frontend/Spss/ParserTest.php @@ -5,6 +5,13 @@ namespace OpenStatSpec\Tests\Frontend\Spss; use OpenStatSpec\Frontend\Spss\Ast\ElseInput; +use OpenStatSpec\Frontend\Spss\Ast\BooleanPredicate; +use OpenStatSpec\Frontend\Spss\Ast\Comparison; +use OpenStatSpec\Frontend\Spss\Ast\ComputeStatement; +use OpenStatSpec\Frontend\Spss\Ast\ExecuteStatement; +use OpenStatSpec\Frontend\Spss\Ast\FormatsStatement; +use OpenStatSpec\Frontend\Spss\Ast\IfStatement; +use OpenStatSpec\Frontend\Spss\Ast\LiteralOperand; use OpenStatSpec\Frontend\Spss\Ast\MissingInput; use OpenStatSpec\Frontend\Spss\Ast\RangeInput; use OpenStatSpec\Frontend\Spss\Ast\RecodeOutputKind; @@ -13,8 +20,15 @@ use OpenStatSpec\Frontend\Spss\Ast\ValueInput; use OpenStatSpec\Frontend\Spss\Ast\ValueLabelsStatement; use OpenStatSpec\Frontend\Spss\Ast\VariableLabelsStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableLevelStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableOperand; use OpenStatSpec\Frontend\Spss\Parser; +use OpenStatSpec\Frontend\Spss\Request\InputSchema; +use OpenStatSpec\Frontend\Spss\Request\InputVariable; +use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; use OpenStatSpec\Frontend\Spss\SpssSyntaxException; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class ParserTest extends TestCase @@ -98,7 +112,7 @@ public function testRejectsDeleteVariableRangesBeforeExecution(): void public function testFailsClosedForUnknownOrUnterminatedCommands(): void { - foreach (['COMPUTE score=1.', 'RECODE score (1=2)'] as $syntax) { + foreach (['SORT score.', 'RECODE score (1=2)'] as $syntax) { try { (new Parser())->parse($syntax); self::fail('Unsupported or unterminated syntax unexpectedly parsed.'); @@ -107,4 +121,196 @@ public function testFailsClosedForUnknownOrUnterminatedCommands(): void } } } + + public function testParsesEveryFrontend02CommandAndPreservesNumericTokenText(): void + { + $program = (new Parser())->parse( + "COMPUTE target = 0.\n" + . "IF (a = 1 AND b >= 2 OR c < 3) target = a.\n" + . "FORMATS target (F8.2).\n" + . "VARIABLE LEVEL target (SCALE).\n" + . 'EXECUTE.', + ); + + self::assertInstanceOf(ComputeStatement::class, $program->statements[0]); + self::assertInstanceOf(LiteralOperand::class, $program->statements[0]->expression); + self::assertSame('0', $program->statements[0]->expression->token); + self::assertInstanceOf(IfStatement::class, $program->statements[1]); + $predicate = $program->statements[1]->predicate; + self::assertInstanceOf(BooleanPredicate::class, $predicate); + self::assertSame('or', $predicate->operator); + self::assertInstanceOf(BooleanPredicate::class, $predicate->operands[0]); + self::assertSame('and', $predicate->operands[0]->operator); + self::assertInstanceOf(FormatsStatement::class, $program->statements[2]); + self::assertSame('F', $program->statements[2]->targets[0]->family); + self::assertSame(8, $program->statements[2]->targets[0]->width); + self::assertSame(2, $program->statements[2]->targets[0]->decimals); + self::assertInstanceOf(VariableLevelStatement::class, $program->statements[3]); + self::assertSame('scale', $program->statements[3]->groups[0]->level); + self::assertInstanceOf(ExecuteStatement::class, $program->statements[4]); + self::assertSame(5, $program->statements[4]->span->startLine); + self::assertSame(0, $program->statements[0]->span->startOffset); + self::assertSame(18, $program->statements[0]->span->endOffset); + } + + public function testBooleanPrecedenceParenthesesAndSameOperatorFlatteningPreserveOrder(): void + { + $program = (new Parser())->parse( + 'IF ((a = 1 OR b = 2) AND c = 3 AND d = 4) target = source.', + ); + + $statement = $program->statements[0]; + self::assertInstanceOf(IfStatement::class, $statement); + $predicate = $statement->predicate; + self::assertInstanceOf(BooleanPredicate::class, $predicate); + self::assertSame('and', $predicate->operator); + self::assertCount(3, $predicate->operands); + $or = $predicate->operands[0]; + self::assertInstanceOf(BooleanPredicate::class, $or); + self::assertSame('or', $or->operator); + $a = $or->operands[0]; + $b = $or->operands[1]; + $c = $predicate->operands[1]; + $d = $predicate->operands[2]; + self::assertInstanceOf(Comparison::class, $a); + self::assertInstanceOf(Comparison::class, $b); + self::assertInstanceOf(Comparison::class, $c); + self::assertInstanceOf(Comparison::class, $d); + self::assertInstanceOf(VariableOperand::class, $a->left); + self::assertInstanceOf(VariableOperand::class, $b->left); + self::assertInstanceOf(VariableOperand::class, $c->left); + self::assertInstanceOf(VariableOperand::class, $d->left); + self::assertSame(['a', 'b', 'c', 'd'], [$a->left->name, $b->left->name, $c->left->name, $d->left->name]); + } + + public function testParsesAllComparisonOperandPermutations(): void + { + $program = (new Parser())->parse('IF (a = b AND 1 < c AND 2 >= 1) target = -1.25e+2.'); + + $statement = $program->statements[0]; + self::assertInstanceOf(IfStatement::class, $statement); + $predicate = $statement->predicate; + self::assertInstanceOf(BooleanPredicate::class, $predicate); + $first = $predicate->operands[0]; + $second = $predicate->operands[1]; + self::assertInstanceOf(Comparison::class, $first); + self::assertInstanceOf(Comparison::class, $second); + self::assertInstanceOf(VariableOperand::class, $first->left); + self::assertInstanceOf(VariableOperand::class, $first->right); + self::assertInstanceOf(LiteralOperand::class, $second->left); + self::assertInstanceOf(VariableOperand::class, $second->right); + self::assertInstanceOf(LiteralOperand::class, $statement->expression); + self::assertSame('-1.25e+2', $statement->expression->token); + } + + public function testParsesStrictGreaterThanComparison(): void + { + $program = (new Parser())->parse('IF (a > 1) target = 1.'); + $statement = $program->statements[0]; + self::assertInstanceOf(IfStatement::class, $statement); + self::assertInstanceOf(Comparison::class, $statement->predicate); + self::assertSame('>', $statement->predicate->operator); + } + + public function testKeywordTokensRemainValidContextualVariableNames(): void + { + $program = (new Parser())->parse( + "RECODE scale (1=2).\n" + . "VARIABLE LABELS compute 'Computed'.\n" + . "COMPUTE if = scale.\n" + . "IF (scale = 1) if = compute.\n" + . "FORMATS scale (F8.2).\n" + . 'VARIABLE LEVEL scale compute (NOMINAL).', + ); + + self::assertInstanceOf(RecodeStatement::class, $program->statements[0]); + self::assertSame(['scale'], $program->statements[0]->sources); + self::assertInstanceOf(ComputeStatement::class, $program->statements[2]); + self::assertSame('if', $program->statements[2]->target); + self::assertInstanceOf(IfStatement::class, $program->statements[3]); + self::assertSame('if', $program->statements[3]->target); + $levels = $program->statements[5]; + self::assertInstanceOf(VariableLevelStatement::class, $levels); + self::assertSame(['scale', 'compute'], $levels->groups[0]->variables); + } + + public function testGroupedMetadataCommandsPreserveGroupAndVariableOrder(): void + { + $program = (new Parser())->parse( + "FORMATS a (F8.2) b (f10.3).\n" + . 'VARIABLE LEVEL a b (ORDINAL) / c target (NOMINAL).', + ); + + $formats = $program->statements[0]; + $levels = $program->statements[1]; + self::assertInstanceOf(FormatsStatement::class, $formats); + self::assertInstanceOf(VariableLevelStatement::class, $levels); + self::assertSame(['a', 'b'], array_column($formats->targets, 'variable')); + self::assertSame([8, 10], array_column($formats->targets, 'width')); + self::assertSame([['a', 'b'], ['c', 'target']], array_column($levels->groups, 'variables')); + self::assertSame(['ordinal', 'nominal'], array_column($levels->groups, 'level')); + } + + #[DataProvider('rejectedFrontend02SyntaxProvider')] + public function testRejectedFrontend02SyntaxHasStableCodeAndExactSpan( + string $source, + string $code, + int $startOffset, + int $endOffset, + ): void { + try { + (new Parser())->parse($source); + self::fail('Unsupported frontend syntax unexpectedly parsed.'); + } catch (SpssSyntaxException $exception) { + self::assertInstanceOf(TransformationFailure::class, $exception); + self::assertSame($code, $exception->diagnosticCode()); + $span = $exception->diagnostics[0]->span; + self::assertNotNull($span); + self::assertSame($startOffset, $span->startOffset); + self::assertSame($endOffset, $span->endOffset); + self::assertSame(1, $span->startLine); + self::assertGreaterThan(0, $span->endColumn); + } + } + + /** @return iterable */ + public static function rejectedFrontend02SyntaxProvider(): iterable + { + yield 'IF needs outer parentheses' => ['IF a = 1 target = 1.', 'spss_syntax_error', 3, 4]; + yield 'IF ELSE is not in the grammar' => ['IF (a = 1) target = 1 ELSE target = 0.', 'spss_syntax_error', 22, 26]; + yield 'arithmetic is unsupported' => ['COMPUTE target = a + 1.', 'expression_type_unsupported', 19, 20]; + yield 'predicate arithmetic is unsupported' => ['IF (a + 1 = 2) target = 1.', 'expression_type_unsupported', 6, 7]; + yield 'NOT is unsupported' => ['IF (NOT a = 1) target = 1.', 'expression_type_unsupported', 4, 7]; + yield 'string assignment is unsupported' => ["COMPUTE target = 'x'.", 'expression_type_unsupported', 17, 20]; + yield 'string predicate is unsupported' => ["IF (color = 'R') target = 1.", 'expression_type_unsupported', 12, 15]; + yield 'non-F format is invalid' => ['FORMATS target (A8).', 'invalid_format', 16, 18]; + yield 'unknown level is invalid syntax' => ['VARIABLE LEVEL target (UNKNOWN).', 'spss_syntax_error', 23, 30]; + yield 'star comment is unsupported command' => ['* comment.', 'unsupported_spss_command', 0, 1]; + yield 'COMMENT is unsupported command' => ['COMMENT text.', 'unsupported_spss_command', 0, 7]; + yield 'unknown command is unsupported' => ['SORT CASES BY a.', 'unsupported_spss_command', 0, 4]; + yield 'keyword word is unsupported at command start' => ['SCALE.', 'unsupported_spss_command', 0, 5]; + yield 'leading plus is not numeric grammar' => ['COMPUTE target = +1.', 'spss_syntax_error', 17, 19]; + yield 'leading decimal is not numeric grammar' => ['COMPUTE target = .5.', 'spss_syntax_error', 17, 19]; + yield 'leading zero is not numeric grammar' => ['COMPUTE target = 01.', 'spss_syntax_error', 17, 19]; + yield 'NaN is not numeric grammar' => ['COMPUTE target = NaN.', 'spss_syntax_error', 17, 20]; + yield 'Infinity is not numeric grammar' => ['COMPUTE target = Infinity.', 'spss_syntax_error', 17, 25]; + yield 'trailing decimal point is not numeric grammar' => ['COMPUTE target = 1..', 'spss_syntax_error', 19, 20]; + } + + public function testRequestDtosRetainExactSourceAndOrderedSchema(): void + { + $schema = new InputSchema([ + new InputVariable('first', 'numeric'), + new InputVariable('second', 'string'), + ]); + $request = new SpssFrontendRequest( + 'openstatspec-spss-syntax-frontend-v0.2', + 'parent', + $schema, + "COMPUTE target = 0.\r\nEXECUTE.", + ); + + self::assertSame(['first', 'second'], array_column($request->inputSchema->variables, 'name')); + self::assertSame("COMPUTE target = 0.\r\nEXECUTE.", $request->sourceText); + } } From ddbec077054a0479a930eb609ef4395c92717c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 16:11:37 +0300 Subject: [PATCH 09/53] fix: preserve SPSS frontend grammar edge cases --- src/Frontend/Spss/Ast/ValueInput.php | 14 ++++++++- src/Frontend/Spss/Compiler.php | 12 +++++++- src/Frontend/Spss/Lexer.php | 6 ++++ src/Frontend/Spss/Parser.php | 29 ++++++++++++++++-- tests/Frontend/Spss/ParserTest.php | 38 +++++++++++++++++++++++- tests/Frontend/Spss/SpssCompilerTest.php | 16 ++++++++++ 6 files changed, 110 insertions(+), 5 deletions(-) diff --git a/src/Frontend/Spss/Ast/ValueInput.php b/src/Frontend/Spss/Ast/ValueInput.php index 901e177..59c5da3 100644 --- a/src/Frontend/Spss/Ast/ValueInput.php +++ b/src/Frontend/Spss/Ast/ValueInput.php @@ -6,5 +6,17 @@ final readonly class ValueInput implements RecodeInput { - public function __construct(public ScalarValue $value) {} + /** @var non-empty-list */ + public array $values; + public ScalarValue $value; + + public function __construct(ScalarValue ...$values) + { + if ($values === []) { + throw new \InvalidArgumentException('A RECODE values selector requires at least one value.'); + } + + $this->values = array_values($values); + $this->value = $values[0]; + } } diff --git a/src/Frontend/Spss/Compiler.php b/src/Frontend/Spss/Compiler.php index 35b5dce..d2076c7 100644 --- a/src/Frontend/Spss/Compiler.php +++ b/src/Frontend/Spss/Compiler.php @@ -65,9 +65,19 @@ public function compile(BoundProgram $program): TransformationPlan $rules = []; $hasElse = false; foreach ($statement->rules as $rule) { + $action = $this->action($rule->output); + if ($rule->input instanceof ValueInput) { + foreach ($rule->input->values as $value) { + $rules[] = new RecodeRule( + new ExactValueSelector($this->scalar($value)), + $action, + ); + } + continue; + } $selector = $this->selector($rule->input); $hasElse = $hasElse || $selector instanceof ElseSelector; - $rules[] = new RecodeRule($selector, $this->action($rule->output)); + $rules[] = new RecodeRule($selector, $action); } if (!$hasElse) { $defaultAction = $statement->sourceVariable === $statement->targetVariable diff --git a/src/Frontend/Spss/Lexer.php b/src/Frontend/Spss/Lexer.php index 2efd681..d464670 100644 --- a/src/Frontend/Spss/Lexer.php +++ b/src/Frontend/Spss/Lexer.php @@ -49,6 +49,12 @@ public function tokenize(string $source): array } $next = $source[$offset + 1] ?? ''; + if ($character === '/' && $next === '*') { + $this->fail( + new SourceSpan($offset, $offset + 2, $line, $column, $line, $column + 2), + 'Inline block comments are not supported.', + ); + } $punctuation = match (true) { $character === '<' && $next === '=' => TokenType::LessThanOrEqual, $character === '>' && $next === '=' => TokenType::GreaterThanOrEqual, diff --git a/src/Frontend/Spss/Parser.php b/src/Frontend/Spss/Parser.php index ef2294f..ab1945c 100644 --- a/src/Frontend/Spss/Parser.php +++ b/src/Frontend/Spss/Parser.php @@ -189,7 +189,7 @@ private function comparison(): Predicate private function comparisonOperator(): Token { - if ($this->check(TokenType::ArithmeticOperator) || $this->check(TokenType::Slash)) { + if ($this->isArithmeticContinuation()) { $this->fail( $this->current(), 'Arithmetic expressions are not supported.', @@ -240,7 +240,7 @@ private function expressionOperand(string $message): ExpressionOperand private function rejectUnsupportedExpressionContinuation(): void { - if ($this->check(TokenType::ArithmeticOperator) || $this->check(TokenType::Slash)) { + if ($this->isArithmeticContinuation()) { $this->fail( $this->current(), 'Arithmetic expressions are not supported.', @@ -262,6 +262,16 @@ private function merge(string $operator, Predicate $left, Predicate $right): Boo return new BooleanPredicate($operator, $operands, SourceSpan::cover($left->span(), $right->span())); } + private function isArithmeticContinuation(): bool + { + if ($this->check(TokenType::ArithmeticOperator) || $this->check(TokenType::Slash)) { + return true; + } + + return $this->check(TokenType::Number) + && ($this->current()->lexeme[0] === '+' || $this->current()->lexeme[0] === '-'); + } + private function formats(Token $command): FormatsStatement { $targets = []; @@ -338,6 +348,21 @@ private function recode(Token $command): RecodeStatement $rules = []; while ($this->match(TokenType::LeftParenthesis)) { $input = $this->recodeInput(); + if ($this->check(TokenType::Comma)) { + if (!$input instanceof ValueInput) { + $this->fail($this->current(), 'Comma-separated RECODE selectors must all be typed literals.'); + } + $values = [$input->value]; + while ($this->match(TokenType::Comma)) { + $comma = $this->previous(); + $additionalInput = $this->recodeInput(); + if (!$additionalInput instanceof ValueInput) { + $this->fail($comma, 'Comma-separated RECODE selectors must all be typed literals.'); + } + $values[] = $additionalInput->value; + } + $input = new ValueInput(...$values); + } $this->consume(TokenType::Equals, 'Expected = in a RECODE rule.'); $output = $this->recodeOutput(); $this->consume(TokenType::RightParenthesis, 'Expected ) after a RECODE rule.'); diff --git a/tests/Frontend/Spss/ParserTest.php b/tests/Frontend/Spss/ParserTest.php index 937bdab..f84bf81 100644 --- a/tests/Frontend/Spss/ParserTest.php +++ b/tests/Frontend/Spss/ParserTest.php @@ -67,6 +67,20 @@ public function testParsesSupportedRecodeSelectorsActionsAndInto(): void self::assertSame(RecodeOutputKind::Copy, $statement->rules[3]->output->kind); } + public function testParsesCommaSeparatedRecodeSelectorsInSourceOrder(): void + { + $program = (new Parser())->parse('RECODE q1 (1,2,3 = 0).'); + + $statement = $program->statements[0]; + self::assertInstanceOf(RecodeStatement::class, $statement); + self::assertCount(1, $statement->rules); + $rule = $statement->rules[0]; + self::assertInstanceOf(ValueInput::class, $rule->input); + self::assertSame([1.0, 2.0, 3.0], array_column($rule->input->values, 'value')); + self::assertNotNull($rule->output->value); + self::assertSame(0.0, $rule->output->value->value); + } + public function testParsesVariableAndValueLabelGroups(): void { $program = (new Parser())->parse(<<<'SPSS' @@ -269,7 +283,9 @@ public function testRejectedFrontend02SyntaxHasStableCodeAndExactSpan( self::assertSame($startOffset, $span->startOffset); self::assertSame($endOffset, $span->endOffset); self::assertSame(1, $span->startLine); - self::assertGreaterThan(0, $span->endColumn); + self::assertSame($startOffset + 1, $span->startColumn); + self::assertSame(1, $span->endLine); + self::assertSame($endOffset + 1, $span->endColumn); } } @@ -279,6 +295,8 @@ public static function rejectedFrontend02SyntaxProvider(): iterable yield 'IF needs outer parentheses' => ['IF a = 1 target = 1.', 'spss_syntax_error', 3, 4]; yield 'IF ELSE is not in the grammar' => ['IF (a = 1) target = 1 ELSE target = 0.', 'spss_syntax_error', 22, 26]; yield 'arithmetic is unsupported' => ['COMPUTE target = a + 1.', 'expression_type_unsupported', 19, 20]; + yield 'adjacent addition is unsupported' => ['COMPUTE target = a+1.', 'expression_type_unsupported', 18, 20]; + yield 'adjacent subtraction is unsupported' => ['COMPUTE target = a-1.', 'expression_type_unsupported', 18, 20]; yield 'predicate arithmetic is unsupported' => ['IF (a + 1 = 2) target = 1.', 'expression_type_unsupported', 6, 7]; yield 'NOT is unsupported' => ['IF (NOT a = 1) target = 1.', 'expression_type_unsupported', 4, 7]; yield 'string assignment is unsupported' => ["COMPUTE target = 'x'.", 'expression_type_unsupported', 17, 20]; @@ -289,6 +307,7 @@ public static function rejectedFrontend02SyntaxProvider(): iterable yield 'COMMENT is unsupported command' => ['COMMENT text.', 'unsupported_spss_command', 0, 7]; yield 'unknown command is unsupported' => ['SORT CASES BY a.', 'unsupported_spss_command', 0, 4]; yield 'keyword word is unsupported at command start' => ['SCALE.', 'unsupported_spss_command', 0, 5]; + yield 'inline block comment is syntax error' => ['COMPUTE target = a/* comment */.', 'spss_syntax_error', 18, 20]; yield 'leading plus is not numeric grammar' => ['COMPUTE target = +1.', 'spss_syntax_error', 17, 19]; yield 'leading decimal is not numeric grammar' => ['COMPUTE target = .5.', 'spss_syntax_error', 17, 19]; yield 'leading zero is not numeric grammar' => ['COMPUTE target = 01.', 'spss_syntax_error', 17, 19]; @@ -297,6 +316,23 @@ public static function rejectedFrontend02SyntaxProvider(): iterable yield 'trailing decimal point is not numeric grammar' => ['COMPUTE target = 1..', 'spss_syntax_error', 19, 20]; } + public function testMultilineDiagnosticHasExactStartAndEndCoordinates(): void + { + try { + (new Parser())->parse("COMPUTE target = 0.\nIF x = 1 target = 1."); + self::fail('Unparenthesized IF unexpectedly parsed.'); + } catch (SpssSyntaxException $exception) { + $span = $exception->diagnostics[0]->span; + self::assertNotNull($span); + self::assertSame(23, $span->startOffset); + self::assertSame(24, $span->endOffset); + self::assertSame(2, $span->startLine); + self::assertSame(4, $span->startColumn); + self::assertSame(2, $span->endLine); + self::assertSame(5, $span->endColumn); + } + } + public function testRequestDtosRetainExactSourceAndOrderedSchema(): void { $schema = new InputSchema([ diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index 246b468..458a603 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -56,6 +56,22 @@ public function testMakesSpssDefaultElseSemanticsExplicit(): void self::assertInstanceOf(ElseSelector::class, $intoRecode->rules()[1]->selector()); } + public function testExpandsGroupedRecodeValuesWithoutDroppingSelectors(): void + { + $plan = (new SpssCompiler())->compile('RECODE score (1,2=0).', self::DATASET_ID); + $operation = $plan->operations()[0]; + self::assertInstanceOf(RecodeOperation::class, $operation); + self::assertCount(3, $operation->rules()); + + $first = $operation->rules()[0]->selector(); + $second = $operation->rules()[1]->selector(); + self::assertInstanceOf(\OpenStatSpec\Transformation\Model\Selector\ExactValueSelector::class, $first); + self::assertInstanceOf(\OpenStatSpec\Transformation\Model\Selector\ExactValueSelector::class, $second); + self::assertSame(1.0, $first->value()->numberValue()); + self::assertSame(2.0, $second->value()->numberValue()); + self::assertInstanceOf(ElseSelector::class, $operation->rules()[2]->selector()); + } + public function testExpandsParallelSourceAndIntoLists(): void { $plan = (new SpssCompiler())->compile('RECODE first second (1=2) INTO new_first new_second.', self::DATASET_ID); From 24267477812fefe02bf766ee52211ef3dd43c799 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 16:44:00 +0300 Subject: [PATCH 10/53] feat: compile official SPSS frontend 0.2 plans --- .../task-6-report.md | 88 +++ src/Frontend/Spss/Ast/ComputeStatement.php | 1 + .../Spss/Ast/DeleteVariablesStatement.php | 4 + src/Frontend/Spss/Ast/ElseInput.php | 12 +- src/Frontend/Spss/Ast/FormatTarget.php | 1 + src/Frontend/Spss/Ast/IfStatement.php | 1 + src/Frontend/Spss/Ast/MissingInput.php | 12 +- src/Frontend/Spss/Ast/RangeInput.php | 8 + src/Frontend/Spss/Ast/RecodeInput.php | 7 +- src/Frontend/Spss/Ast/RecodeOutput.php | 3 + src/Frontend/Spss/Ast/RecodeRule.php | 3 + src/Frontend/Spss/Ast/RecodeStatement.php | 7 + src/Frontend/Spss/Ast/ScalarValue.php | 13 +- src/Frontend/Spss/Ast/StringStatement.php | 4 + src/Frontend/Spss/Ast/SystemMissingInput.php | 12 +- src/Frontend/Spss/Ast/ValueInput.php | 9 + src/Frontend/Spss/Ast/ValueLabel.php | 8 +- src/Frontend/Spss/Ast/ValueLabelGroup.php | 10 +- .../Spss/Ast/ValueLabelsStatement.php | 8 +- .../Spss/Ast/VariableLabelAssignment.php | 17 + .../Spss/Ast/VariableLabelsStatement.php | 10 +- src/Frontend/Spss/Ast/VariableLevelGroup.php | 2 + src/Frontend/Spss/Binder.php | 589 +++++++++++++++--- src/Frontend/Spss/Binding/BoundCompute.php | 19 + .../Spss/Binding/BoundConditionalAssign.php | 19 + src/Frontend/Spss/Binding/BoundExecute.php | 12 + src/Frontend/Spss/Binding/BoundFormat.php | 18 + .../Spss/Binding/BoundMeasurementLevel.php | 16 + src/Frontend/Spss/Binding/BoundProgram.php | 4 +- src/Frontend/Spss/Binding/BoundRecode.php | 8 +- .../Spss/Binding/BoundValueLabels.php | 9 +- .../Spss/Binding/BoundVariableLabel.php | 8 +- src/Frontend/Spss/Binding/SchemaState.php | 72 +++ src/Frontend/Spss/Compiler.php | 183 ++---- src/Frontend/Spss/Parser.php | 137 +++- src/Frontend/Spss/Request/InputSchema.php | 52 +- src/Frontend/Spss/Request/InputVariable.php | 195 +++++- .../Spss/Request/SpssFrontendRequest.php | 71 ++- src/Frontend/Spss/SpssCompilationResult.php | 15 + src/Frontend/Spss/SpssCompiler.php | 20 +- .../Spss/Conformance/SpssFrontend02Test.php | 89 +++ tests/Frontend/Spss/ParserTest.php | 6 +- tests/Frontend/Spss/SpssCompilerTest.php | 399 ++++++++---- 43 files changed, 1785 insertions(+), 396 deletions(-) create mode 100644 .superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md create mode 100644 src/Frontend/Spss/Ast/VariableLabelAssignment.php create mode 100644 src/Frontend/Spss/Binding/BoundCompute.php create mode 100644 src/Frontend/Spss/Binding/BoundConditionalAssign.php create mode 100644 src/Frontend/Spss/Binding/BoundExecute.php create mode 100644 src/Frontend/Spss/Binding/BoundFormat.php create mode 100644 src/Frontend/Spss/Binding/BoundMeasurementLevel.php create mode 100644 src/Frontend/Spss/Binding/SchemaState.php create mode 100644 src/Frontend/Spss/SpssCompilationResult.php create mode 100644 tests/Frontend/Spss/Conformance/SpssFrontend02Test.php diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md new file mode 100644 index 0000000..782e4c2 --- /dev/null +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md @@ -0,0 +1,88 @@ +# Task 6 report: bind and compile SPSS Frontend 0.2 + +## Status + +Complete. The change is committed as one Task 6 commit and was not pushed. + +## Implementation + +- Added strict `SpssFrontendRequest::fromArray()` validation for the exact Frontend 0.2 request shape, its ordered non-empty input schema, and every optional variable metadata field. +- Added source identity hashing that normalizes only CRLF and bare CR to LF before SHA-256 and remains available independently of parsing or binding success. +- Added an ASCII-case-insensitive evolving `SchemaState` that retains exact catalog spelling, resolves earlier created variables in later commands, and binds multi-variable `RECODE` sources and targets against pre-command state. +- Replaced the legacy binder/compiler boundary with typed bound nodes for recode, compute, conditional assignment, labels, formats, measurement levels, and execute. +- Compiled ordered bound statements one-for-one into the official Plan operation types and selected Plan 0.1 or 0.2 from each operation's minimum contract. +- Converted every decimal literal directly through `DecimalBinary64` into canonical `Binary64Value` bits, including exact rounding and positive-zero canonicalization, without a host-float parser intermediate. +- Added command, group, variable, expression, rule, value, result, and unsupported-command spans needed for stable binding diagnostics while preserving all source ordering. +- Added manifest-driven coverage for all 44 official Frontend 0.2 cases, exact source/plan hashes, exact canonical plans, referenced official Frontend 0.1 golden plans, plan contracts, stable failure codes, and non-null source spans. +- Added focused request, evolving-schema, simultaneous-recode, exact-binary64, diagnostic-code/span, invalid-empty-recode, and unsupported-legacy-command regressions. + +## TDD evidence + +### RED + +The official manifest test was added before the request/compiler implementation: + +```text +vendor/bin/phpunit tests/Frontend/Spss/Conformance +44 errors: SpssFrontendRequest::fromArray() did not exist. +``` + +The focused compiler tests were then added before the new facade and binder: + +```text +vendor/bin/phpunit tests/Frontend/Spss/SpssCompilerTest.php +25 errors: the strict request constructor and compile(request) boundary did not exist. +``` + +Further tests independently reproduced missing direct-constructor value-label validation, invalid ELSE-only RECODE output, loss of exact numeric source tokens, and zero-width unsupported-command spans before each corresponding fix. + +### GREEN + +```text +vendor/bin/phpunit tests/Frontend/Spss/Conformance tests/Frontend/Spss/SpssCompilerTest.php +OK (74 tests, 269 assertions) +``` + +The 44 manifest cases are included in that count. + +## Self-review + +- **Request boundary:** exact root/schema/value object members are enforced; strings are valid UTF-8; ordered variables are non-empty; storage, typed values, formats, and measurement metadata are validated before parsing. +- **Source identity:** hashing is isolated from parsing and changes only line-ending bytes. +- **Resolution and ordering:** matching is ASCII-case-insensitive, emitted names preserve schema spelling, ambiguous names fail closed, new names retain source spelling, commands/groups/variables/rules/boolean operands remain ordered, and parallel RECODE targets are published only after the whole command binds. +- **Type and target semantics:** COMPUTE creates or replaces numeric targets; IF requires an existing numeric target; numeric-only expressions/formats, value-label types, ranges, result homogeneity, reserved names, fresh targets, and string limitations fail at the narrowest available source span. +- **Plan boundary:** emitted operations are official immutable Plan types; `PlanCodec` produces each official hash; unchanged subsets stay Plan 0.1 and any 0.2 operation promotes the whole ordered plan to 0.2. +- **Failure behavior:** request, syntax, binding, and unsupported-command failures occur before mutation and carry the stable code/path/span contract. +- **Dolt/global constraints:** Task 6 changes only the pure frontend and tests. It adds no SQL mutation, dataset/table copy, derived dataset, staging/snapshot/rollback artifact, audit/version row, or Dolt operation; Dolt remains the sole version-history layer. +- **Scope:** the staged diff contains only Task 6 frontend source/tests and this report; unrelated changes were not included. + +## Verification + +```text +vendor/bin/phpunit tests/Transformation/Conformance tests/Frontend/Spss +OK (179 tests, 697 assertions) +``` + +```text +composer analyse +[OK] No errors +``` + +The CRLF-normalized staged-tree pre-commit hook ran the complete `composer check` gate successfully: + +```text +composer validate --strict: valid +PHP lint: all source/test/tool files valid +php-cs-fixer: Found 0 of 213 files that can be fixed +PHPStan: [OK] No errors +PHPUnit: 406 tests, 2361 assertions, 30 skipped; exit 0 +``` + +`git diff --cached --check` also passed. + +The repository-wide working-tree `composer style` command was also run. It exits 8 because 133 of 213 repository files have pre-existing CRLF line endings; the normalized staged-tree hook runs the same style check against LF-normalized Git blobs and is the authoritative clean gate for the committed tree. + +## Notes + +- The Frontend 0.2 manifest's two `expected_plan_case_0_1` identifiers refer to cases in `spss-syntax-frontend-0.1.json`; they do not exist in `transformation-plan-0.1.json`. The conformance test therefore resolves those official references from their actual owning manifest rather than copying the golden plans. +- Verification used installed PHP 8.5.9 with Composer's declared PHP 8.4.1 platform floor. diff --git a/src/Frontend/Spss/Ast/ComputeStatement.php b/src/Frontend/Spss/Ast/ComputeStatement.php index 5627ccc..36eb30f 100644 --- a/src/Frontend/Spss/Ast/ComputeStatement.php +++ b/src/Frontend/Spss/Ast/ComputeStatement.php @@ -10,6 +10,7 @@ { public function __construct( public string $target, + public SourceSpan $targetSpan, public ExpressionOperand $expression, public SourceSpan $span, ) {} diff --git a/src/Frontend/Spss/Ast/DeleteVariablesStatement.php b/src/Frontend/Spss/Ast/DeleteVariablesStatement.php index cb36b74..ccedcff 100644 --- a/src/Frontend/Spss/Ast/DeleteVariablesStatement.php +++ b/src/Frontend/Spss/Ast/DeleteVariablesStatement.php @@ -4,12 +4,16 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class DeleteVariablesStatement implements Statement { /** @param list $variables */ public function __construct( public int $lineNumber, public array $variables, + public SourceSpan $commandSpan, + public SourceSpan $span, ) {} public function line(): int diff --git a/src/Frontend/Spss/Ast/ElseInput.php b/src/Frontend/Spss/Ast/ElseInput.php index 344da43..ad8ad67 100644 --- a/src/Frontend/Spss/Ast/ElseInput.php +++ b/src/Frontend/Spss/Ast/ElseInput.php @@ -4,4 +4,14 @@ namespace OpenStatSpec\Frontend\Spss\Ast; -final readonly class ElseInput implements RecodeInput {} +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + +final readonly class ElseInput implements RecodeInput +{ + public function __construct(public SourceSpan $span) {} + + public function span(): SourceSpan + { + return $this->span; + } +} diff --git a/src/Frontend/Spss/Ast/FormatTarget.php b/src/Frontend/Spss/Ast/FormatTarget.php index 58f461e..2706753 100644 --- a/src/Frontend/Spss/Ast/FormatTarget.php +++ b/src/Frontend/Spss/Ast/FormatTarget.php @@ -13,6 +13,7 @@ public function __construct( public string $family, public int $width, public int $decimals, + public SourceSpan $variableSpan, public SourceSpan $span, ) {} } diff --git a/src/Frontend/Spss/Ast/IfStatement.php b/src/Frontend/Spss/Ast/IfStatement.php index 45c6698..ed699eb 100644 --- a/src/Frontend/Spss/Ast/IfStatement.php +++ b/src/Frontend/Spss/Ast/IfStatement.php @@ -11,6 +11,7 @@ public function __construct( public Predicate $predicate, public string $target, + public SourceSpan $targetSpan, public ExpressionOperand $expression, public SourceSpan $span, ) {} diff --git a/src/Frontend/Spss/Ast/MissingInput.php b/src/Frontend/Spss/Ast/MissingInput.php index 91eef12..ba782d7 100644 --- a/src/Frontend/Spss/Ast/MissingInput.php +++ b/src/Frontend/Spss/Ast/MissingInput.php @@ -4,4 +4,14 @@ namespace OpenStatSpec\Frontend\Spss\Ast; -final readonly class MissingInput implements RecodeInput {} +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + +final readonly class MissingInput implements RecodeInput +{ + public function __construct(public SourceSpan $span) {} + + public function span(): SourceSpan + { + return $this->span; + } +} diff --git a/src/Frontend/Spss/Ast/RangeInput.php b/src/Frontend/Spss/Ast/RangeInput.php index ab44e1e..835924a 100644 --- a/src/Frontend/Spss/Ast/RangeInput.php +++ b/src/Frontend/Spss/Ast/RangeInput.php @@ -4,10 +4,18 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class RangeInput implements RecodeInput { public function __construct( public ?ScalarValue $lower, public ?ScalarValue $upper, + public SourceSpan $span, ) {} + + public function span(): SourceSpan + { + return $this->span; + } } diff --git a/src/Frontend/Spss/Ast/RecodeInput.php b/src/Frontend/Spss/Ast/RecodeInput.php index c46d949..01db4a0 100644 --- a/src/Frontend/Spss/Ast/RecodeInput.php +++ b/src/Frontend/Spss/Ast/RecodeInput.php @@ -4,4 +4,9 @@ namespace OpenStatSpec\Frontend\Spss\Ast; -interface RecodeInput {} +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + +interface RecodeInput +{ + public function span(): SourceSpan; +} diff --git a/src/Frontend/Spss/Ast/RecodeOutput.php b/src/Frontend/Spss/Ast/RecodeOutput.php index ce88add..ce5c315 100644 --- a/src/Frontend/Spss/Ast/RecodeOutput.php +++ b/src/Frontend/Spss/Ast/RecodeOutput.php @@ -4,10 +4,13 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class RecodeOutput { public function __construct( public RecodeOutputKind $kind, + public SourceSpan $span, public ?ScalarValue $value = null, ) {} } diff --git a/src/Frontend/Spss/Ast/RecodeRule.php b/src/Frontend/Spss/Ast/RecodeRule.php index 520ae97..5c79ef5 100644 --- a/src/Frontend/Spss/Ast/RecodeRule.php +++ b/src/Frontend/Spss/Ast/RecodeRule.php @@ -4,10 +4,13 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class RecodeRule { public function __construct( public RecodeInput $input, public RecodeOutput $output, + public SourceSpan $span, ) {} } diff --git a/src/Frontend/Spss/Ast/RecodeStatement.php b/src/Frontend/Spss/Ast/RecodeStatement.php index b6f8e91..a332991 100644 --- a/src/Frontend/Spss/Ast/RecodeStatement.php +++ b/src/Frontend/Spss/Ast/RecodeStatement.php @@ -4,6 +4,8 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class RecodeStatement implements Statement { /** @@ -14,8 +16,13 @@ public function __construct( public int $sourceLine, public array $sources, + /** @var non-empty-list */ + public array $sourceSpans, public array $rules, public array $targets, + /** @var list */ + public array $targetSpans, + public SourceSpan $span, ) {} public function line(): int diff --git a/src/Frontend/Spss/Ast/ScalarValue.php b/src/Frontend/Spss/Ast/ScalarValue.php index 1612fe2..6f39cf6 100644 --- a/src/Frontend/Spss/Ast/ScalarValue.php +++ b/src/Frontend/Spss/Ast/ScalarValue.php @@ -4,7 +4,18 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ScalarValue { - public function __construct(public int|float|string $value) {} + public function __construct( + public string $value, + public ?string $numericToken, + public SourceSpan $span, + ) {} + + public function isNumeric(): bool + { + return $this->numericToken !== null; + } } diff --git a/src/Frontend/Spss/Ast/StringStatement.php b/src/Frontend/Spss/Ast/StringStatement.php index 5e50af2..a9eb3fb 100644 --- a/src/Frontend/Spss/Ast/StringStatement.php +++ b/src/Frontend/Spss/Ast/StringStatement.php @@ -4,6 +4,8 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class StringStatement implements Statement { /** @param list $variables */ @@ -11,6 +13,8 @@ public function __construct( public int $lineNumber, public array $variables, public int $width, + public SourceSpan $commandSpan, + public SourceSpan $span, ) {} public function line(): int diff --git a/src/Frontend/Spss/Ast/SystemMissingInput.php b/src/Frontend/Spss/Ast/SystemMissingInput.php index ec018f2..a4e3a3a 100644 --- a/src/Frontend/Spss/Ast/SystemMissingInput.php +++ b/src/Frontend/Spss/Ast/SystemMissingInput.php @@ -4,4 +4,14 @@ namespace OpenStatSpec\Frontend\Spss\Ast; -final readonly class SystemMissingInput implements RecodeInput {} +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + +final readonly class SystemMissingInput implements RecodeInput +{ + public function __construct(public SourceSpan $span) {} + + public function span(): SourceSpan + { + return $this->span; + } +} diff --git a/src/Frontend/Spss/Ast/ValueInput.php b/src/Frontend/Spss/Ast/ValueInput.php index 59c5da3..47d8cf0 100644 --- a/src/Frontend/Spss/Ast/ValueInput.php +++ b/src/Frontend/Spss/Ast/ValueInput.php @@ -4,11 +4,14 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ValueInput implements RecodeInput { /** @var non-empty-list */ public array $values; public ScalarValue $value; + public SourceSpan $span; public function __construct(ScalarValue ...$values) { @@ -18,5 +21,11 @@ public function __construct(ScalarValue ...$values) $this->values = array_values($values); $this->value = $values[0]; + $this->span = SourceSpan::cover($values[0]->span, $values[array_key_last($values)]->span); + } + + public function span(): SourceSpan + { + return $this->span; } } diff --git a/src/Frontend/Spss/Ast/ValueLabel.php b/src/Frontend/Spss/Ast/ValueLabel.php index d39d394..af60443 100644 --- a/src/Frontend/Spss/Ast/ValueLabel.php +++ b/src/Frontend/Spss/Ast/ValueLabel.php @@ -4,7 +4,13 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ValueLabel { - public function __construct(public ScalarValue $value, public string $label) {} + public function __construct( + public ScalarValue $value, + public string $label, + public SourceSpan $span, + ) {} } diff --git a/src/Frontend/Spss/Ast/ValueLabelGroup.php b/src/Frontend/Spss/Ast/ValueLabelGroup.php index 8b7e337..3f87715 100644 --- a/src/Frontend/Spss/Ast/ValueLabelGroup.php +++ b/src/Frontend/Spss/Ast/ValueLabelGroup.php @@ -4,11 +4,19 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ValueLabelGroup { /** * @param non-empty-list $variables * @param non-empty-list $labels */ - public function __construct(public array $variables, public array $labels) {} + public function __construct( + public array $variables, + /** @var non-empty-list */ + public array $variableSpans, + public array $labels, + public SourceSpan $span, + ) {} } diff --git a/src/Frontend/Spss/Ast/ValueLabelsStatement.php b/src/Frontend/Spss/Ast/ValueLabelsStatement.php index 1e4fe20..966e986 100644 --- a/src/Frontend/Spss/Ast/ValueLabelsStatement.php +++ b/src/Frontend/Spss/Ast/ValueLabelsStatement.php @@ -4,10 +4,16 @@ namespace OpenStatSpec\Frontend\Spss\Ast; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class ValueLabelsStatement implements Statement { /** @param non-empty-list $groups */ - public function __construct(public int $sourceLine, public array $groups) {} + public function __construct( + public int $sourceLine, + public array $groups, + public SourceSpan $span, + ) {} public function line(): int { diff --git a/src/Frontend/Spss/Ast/VariableLabelAssignment.php b/src/Frontend/Spss/Ast/VariableLabelAssignment.php new file mode 100644 index 0000000..b1f4e42 --- /dev/null +++ b/src/Frontend/Spss/Ast/VariableLabelAssignment.php @@ -0,0 +1,17 @@ + $labels */ - public function __construct(public int $sourceLine, public array $labels) {} + public function __construct( + public int $sourceLine, + public array $labels, + /** @var non-empty-list */ + public array $assignments, + public SourceSpan $span, + ) {} public function line(): int { diff --git a/src/Frontend/Spss/Ast/VariableLevelGroup.php b/src/Frontend/Spss/Ast/VariableLevelGroup.php index 0406ae3..28b0625 100644 --- a/src/Frontend/Spss/Ast/VariableLevelGroup.php +++ b/src/Frontend/Spss/Ast/VariableLevelGroup.php @@ -11,6 +11,8 @@ /** @param non-empty-list $variables */ public function __construct( public array $variables, + /** @var non-empty-list */ + public array $variableSpans, public string $level, public SourceSpan $span, ) {} diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php index ac27682..9904169 100644 --- a/src/Frontend/Spss/Binder.php +++ b/src/Frontend/Spss/Binder.php @@ -4,125 +4,564 @@ namespace OpenStatSpec\Frontend\Spss; -use OpenStatSpec\Frontend\Spss\Ast\ElseInput; +use OpenStatSpec\Frontend\Spss\Ast\BooleanPredicate as AstBooleanPredicate; +use OpenStatSpec\Frontend\Spss\Ast\Comparison as AstComparison; +use OpenStatSpec\Frontend\Spss\Ast\ComputeStatement; use OpenStatSpec\Frontend\Spss\Ast\DeleteVariablesStatement; +use OpenStatSpec\Frontend\Spss\Ast\ElseInput; use OpenStatSpec\Frontend\Spss\Ast\ExecuteStatement; +use OpenStatSpec\Frontend\Spss\Ast\ExpressionOperand as AstExpressionOperand; +use OpenStatSpec\Frontend\Spss\Ast\FormatsStatement; +use OpenStatSpec\Frontend\Spss\Ast\IfStatement; +use OpenStatSpec\Frontend\Spss\Ast\LiteralOperand as AstLiteralOperand; use OpenStatSpec\Frontend\Spss\Ast\MissingInput; -use OpenStatSpec\Frontend\Spss\Ast\StringStatement; +use OpenStatSpec\Frontend\Spss\Ast\Predicate as AstPredicate; use OpenStatSpec\Frontend\Spss\Ast\Program; use OpenStatSpec\Frontend\Spss\Ast\RangeInput; +use OpenStatSpec\Frontend\Spss\Ast\RecodeInput; +use OpenStatSpec\Frontend\Spss\Ast\RecodeOutput; +use OpenStatSpec\Frontend\Spss\Ast\RecodeOutputKind; use OpenStatSpec\Frontend\Spss\Ast\RecodeStatement; +use OpenStatSpec\Frontend\Spss\Ast\ScalarValue; +use OpenStatSpec\Frontend\Spss\Ast\StringStatement; +use OpenStatSpec\Frontend\Spss\Ast\SystemMissingInput; +use OpenStatSpec\Frontend\Spss\Ast\ValueInput; use OpenStatSpec\Frontend\Spss\Ast\ValueLabelsStatement; use OpenStatSpec\Frontend\Spss\Ast\VariableLabelsStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableLevelStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableOperand as AstVariableOperand; +use OpenStatSpec\Frontend\Spss\Binding\BoundCompute; +use OpenStatSpec\Frontend\Spss\Binding\BoundConditionalAssign; +use OpenStatSpec\Frontend\Spss\Binding\BoundExecute; +use OpenStatSpec\Frontend\Spss\Binding\BoundFormat; +use OpenStatSpec\Frontend\Spss\Binding\BoundMeasurementLevel; use OpenStatSpec\Frontend\Spss\Binding\BoundProgram; use OpenStatSpec\Frontend\Spss\Binding\BoundRecode; +use OpenStatSpec\Frontend\Spss\Binding\BoundStatement; use OpenStatSpec\Frontend\Spss\Binding\BoundValueLabels; -use OpenStatSpec\Frontend\Spss\Binding\BoundCreateVariable; -use OpenStatSpec\Frontend\Spss\Binding\BoundDeleteVariable; use OpenStatSpec\Frontend\Spss\Binding\BoundVariableLabel; +use OpenStatSpec\Frontend\Spss\Binding\SchemaState; +use OpenStatSpec\Frontend\Spss\Number\DecimalBinary64; +use OpenStatSpec\Frontend\Spss\Request\InputSchema; +use OpenStatSpec\Frontend\Spss\Request\InputVariable; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Plan\Expression\BooleanPredicate; +use OpenStatSpec\Transformation\Plan\Expression\Comparison; +use OpenStatSpec\Transformation\Plan\Expression\LiteralOperand; +use OpenStatSpec\Transformation\Plan\Expression\Operand; +use OpenStatSpec\Transformation\Plan\Expression\Predicate; +use OpenStatSpec\Transformation\Plan\Expression\VariableOperand; +use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; +use OpenStatSpec\Transformation\Plan\Recode\CopyResult; +use OpenStatSpec\Transformation\Plan\Recode\ExactMatch; +use OpenStatSpec\Transformation\Plan\Recode\LiteralResult; +use OpenStatSpec\Transformation\Plan\Recode\RangeMatch; +use OpenStatSpec\Transformation\Plan\Recode\RecodeMatch; +use OpenStatSpec\Transformation\Plan\Recode\RecodeRule; +use OpenStatSpec\Transformation\Plan\Recode\Result; +use OpenStatSpec\Transformation\Plan\Recode\SystemMissingMatch; +use OpenStatSpec\Transformation\Plan\Recode\SystemMissingResult; +use OpenStatSpec\Transformation\Plan\TargetMode; +use OpenStatSpec\Transformation\Plan\Value\Binary64Value; +use OpenStatSpec\Transformation\Plan\Value\StringValue; +use OpenStatSpec\Transformation\Plan\Value\TypedValue; final class Binder { - public function bind(string $datasetId, Program $program): BoundProgram + public function bind(string $inputAlias, InputSchema $inputSchema, Program $program): BoundProgram { - if (trim($datasetId) === '') { - throw new SpssSyntaxException([new Diagnostic(1, 1, 'Dataset id must not be empty.')]); - } - + $schema = new SchemaState($inputSchema); $bound = []; foreach ($program->statements as $statement) { - if ($statement instanceof ExecuteStatement) { + if ($statement instanceof RecodeStatement) { + array_push($bound, ...$this->recode($statement, $schema)); continue; } - if ($statement instanceof StringStatement) { - foreach ($statement->variables as $variable) { - $bound[] = new BoundCreateVariable($variable, 'string', $statement->width); - } + if ($statement instanceof ComputeStatement) { + $bound[] = $this->compute($statement, $schema); continue; } - if ($statement instanceof DeleteVariablesStatement) { - foreach ($statement->variables as $variable) { - $bound[] = new BoundDeleteVariable($variable); - } + if ($statement instanceof IfStatement) { + $bound[] = $this->conditionalAssign($statement, $schema); continue; } - if ($statement instanceof RecodeStatement) { - $targets = $statement->targets === [] ? $statement->sources : $statement->targets; - if (count($statement->sources) !== count($targets)) { - $this->fail($statement->line(), 'RECODE INTO must have exactly one target for each source variable.'); - } - if ($statement->targets !== [] && count(array_unique($targets)) !== count($targets)) { - $this->fail( - $statement->line(), - 'RECODE INTO must not contain duplicate target variables in the same statement.', - ); - } - foreach ($targets as $targetIndex => $target) { - foreach (array_slice($statement->sources, $targetIndex + 1) as $laterSource) { - if ($target === $laterSource) { - $this->fail( - $statement->line(), - 'RECODE INTO targets must not overwrite a source used by a later pair in the same statement.', - ); - } - } - } - $elseSeen = false; - foreach ($statement->rules as $index => $rule) { - if ($rule->input instanceof MissingInput) { + if ($statement instanceof FormatsStatement) { + foreach ($statement->targets as $target) { + $variable = $schema->resolve($target->variable, $target->variableSpan); + if ($variable->storageKind !== 'numeric') { $this->fail( - $statement->line(), - 'MISSING includes user-missing values, which require dataset metadata; use SYSMIS or explicit values.', + 'expression_type_unsupported', + 'Numeric F formats cannot target string variables.', + $target->variableSpan, ); } - if ($rule->input instanceof ElseInput) { - if ($elseSeen || $index !== array_key_last($statement->rules)) { - $this->fail($statement->line(), 'ELSE may occur only once and must be the final RECODE rule.'); - } - $elseSeen = true; - } - if ($rule->input instanceof RangeInput) { - $lower = $rule->input->lower?->value; - $upper = $rule->input->upper?->value; - if ($lower === null && $upper === null) { - $this->fail($statement->line(), 'LOWEST THRU HIGHEST is not a finite canonical range; use ELSE instead.'); - } - if (is_string($lower) || is_string($upper)) { - $this->fail($statement->line(), 'RECODE ranges must use numeric bounds.'); - } - if ($lower !== null && $upper !== null && $lower > $upper) { - $this->fail($statement->line(), 'RECODE range lower bound must not exceed its upper bound.'); - } - } + $this->validateFormat($target->width, $target->decimals, $target->span); + $bound[] = new BoundFormat( + $variable->name, + $target->family, + $target->width, + $target->decimals, + $target->span, + ); } - foreach ($statement->sources as $index => $source) { - $bound[] = new BoundRecode($source, $targets[$index], $statement->rules); + continue; + } + if ($statement instanceof VariableLevelStatement) { + foreach ($statement->groups as $group) { + foreach ($group->variables as $index => $name) { + $span = $group->variableSpans[$index]; + $variable = $schema->resolve($name, $span); + $bound[] = new BoundMeasurementLevel($variable->name, $group->level, $span); + } } continue; } + if ($statement instanceof ExecuteStatement) { + $bound[] = new BoundExecute($statement->span); + continue; + } if ($statement instanceof VariableLabelsStatement) { - foreach ($statement->labels as $variable => $label) { - $bound[] = new BoundVariableLabel($variable, $label); + foreach ($statement->assignments as $assignment) { + $variable = $schema->resolve($assignment->variable, $assignment->variableSpan); + $bound[] = new BoundVariableLabel($variable->name, $assignment->label, $assignment->span); } continue; } if ($statement instanceof ValueLabelsStatement) { - foreach ($statement->groups as $group) { - foreach ($group->variables as $variable) { - $bound[] = new BoundValueLabels($variable, $group->labels); + array_push($bound, ...$this->valueLabels($statement, $schema)); + continue; + } + if ($statement instanceof StringStatement || $statement instanceof DeleteVariablesStatement) { + $this->fail( + 'unsupported_spss_command', + 'STRING and DELETE VARIABLES are outside Frontend 0.2.', + $statement->commandSpan, + ); + } + + $this->fail( + 'unsupported_spss_command', + sprintf('Unsupported AST statement %s.', $statement::class), + new SourceSpan(0, 0, $statement->line(), 1, $statement->line(), 1), + ); + } + + if ($bound === []) { + $this->fail( + 'spss_syntax_error', + 'At least one supported SPSS command is required.', + new SourceSpan(0, 0), + ); + } + + return new BoundProgram($inputAlias, $bound); + } + + private function compute(ComputeStatement $statement, SchemaState $schema): BoundCompute + { + [$value, $valueType] = $this->operand($statement->expression, $schema); + if ($valueType !== 'binary64') { + $this->fail( + 'expression_type_unsupported', + 'String assignment expressions are outside Frontend 0.2.', + $statement->expression->span(), + ); + } + + $target = $schema->find($statement->target, $statement->targetSpan); + if ($target !== null) { + if ($target->storageKind !== 'numeric') { + $this->fail( + 'expression_type_unsupported', + 'String assignment targets are outside Frontend 0.2.', + $statement->targetSpan, + ); + } + + return new BoundCompute($target->name, TargetMode::Replace, $value, $statement->span); + } + $this->validateTargetName($statement->target, $statement->targetSpan); + $schema->addNumeric($statement->target); + + return new BoundCompute($statement->target, TargetMode::Create, $value, $statement->span); + } + + private function conditionalAssign(IfStatement $statement, SchemaState $schema): BoundConditionalAssign + { + $condition = $this->predicate($statement->predicate, $schema); + $target = $schema->resolve( + $statement->target, + $statement->targetSpan, + 'conditional_target_missing', + ); + if ($target->storageKind !== 'numeric') { + $this->fail( + 'expression_type_unsupported', + 'String assignment targets are outside Frontend 0.2.', + $statement->targetSpan, + ); + } + [$value, $valueType] = $this->operand($statement->expression, $schema); + if ($valueType !== 'binary64') { + $this->fail( + 'expression_type_unsupported', + 'String assignment expressions are outside Frontend 0.2.', + $statement->expression->span(), + ); + } + + return new BoundConditionalAssign($condition, $target->name, $value, $statement->span); + } + + /** @return array{Operand, 'binary64'|'string'} */ + private function operand(AstExpressionOperand $operand, SchemaState $schema): array + { + if ($operand instanceof AstLiteralOperand) { + return [ + new LiteralOperand(Binary64Value::fromBits(DecimalBinary64::bits($operand->token))), + 'binary64', + ]; + } + if ($operand instanceof AstVariableOperand) { + $variable = $schema->resolve($operand->name, $operand->span); + + return [ + new VariableOperand($variable->name), + $this->storageType($variable), + ]; + } + + throw new \LogicException(sprintf('Unsupported operand %s.', $operand::class)); + } + + private function predicate(AstPredicate $predicate, SchemaState $schema): Predicate + { + if ($predicate instanceof AstComparison) { + [$left, $leftType] = $this->operand($predicate->left, $schema); + [$right, $rightType] = $this->operand($predicate->right, $schema); + if ($leftType !== 'binary64' || $rightType !== 'binary64') { + $this->fail( + 'expression_type_unsupported', + 'String predicates are outside Frontend 0.2.', + $predicate->span, + ); + } + + return new Comparison($left, $predicate->operator, $right); + } + if ($predicate instanceof AstBooleanPredicate) { + return new BooleanPredicate( + $predicate->operator, + ...array_map(fn(AstPredicate $operand): Predicate => $this->predicate($operand, $schema), $predicate->operands), + ); + } + + throw new \LogicException(sprintf('Unsupported predicate %s.', $predicate::class)); + } + + /** @return non-empty-list */ + private function recode(RecodeStatement $statement, SchemaState $schema): array + { + $sources = []; + foreach ($statement->sources as $index => $sourceName) { + $sources[] = $schema->resolve($sourceName, $statement->sourceSpans[$index]); + } + + $targetMode = $statement->targets === [] ? TargetMode::Replace : TargetMode::Create; + if ($targetMode === TargetMode::Create && count($statement->targets) !== count($sources)) { + $this->fail( + 'spss_syntax_error', + 'RECODE INTO requires one target for each source.', + $statement->span, + ); + } + $targetNames = $targetMode === TargetMode::Create + ? $statement->targets + : array_map(static fn(InputVariable $source): string => $source->name, $sources); + if ($targetMode === TargetMode::Create) { + $seenTargets = []; + foreach ($targetNames as $index => $targetName) { + $targetSpan = $statement->targetSpans[$index]; + $this->validateTargetName($targetName, $targetSpan); + $key = strtolower($targetName); + if ($schema->contains($targetName) || isset($seenTargets[$key])) { + $this->fail( + 'target_already_exists', + 'RECODE INTO targets must be fresh and unique in the pre-command schema.', + $targetSpan, + ); + } + $seenTargets[$key] = true; + } + } + + $elseRules = array_values(array_filter( + $statement->rules, + static fn(\OpenStatSpec\Frontend\Spss\Ast\RecodeRule $rule): bool => $rule->input instanceof ElseInput, + )); + if (count($elseRules) > 1) { + $this->fail('duplicate_else', 'ELSE may occur only once.', $elseRules[1]->input->span()); + } + if ( + $elseRules !== [] + && !($statement->rules[array_key_last($statement->rules)]->input instanceof ElseInput) + ) { + $this->fail('else_not_last', 'ELSE must be the final RECODE rule.', $elseRules[0]->input->span()); + } + if (count($statement->rules) === count($elseRules)) { + $this->fail( + 'spss_syntax_error', + 'RECODE requires at least one non-ELSE rule.', + $statement->span, + ); + } + + $bound = []; + foreach ($sources as $index => $source) { + $rules = []; + $unmatched = null; + foreach ($statement->rules as $rule) { + $result = $this->recodeResult($rule->output, $source); + if ($rule->input instanceof ElseInput) { + $unmatched = $result; + continue; + } + $rules[] = new RecodeRule($this->recodeMatch($rule->input, $source), $result); + } + $unmatched ??= $targetMode === TargetMode::Create ? new SystemMissingResult() : new CopyResult(); + $resultTypes = array_values(array_unique(array_map( + fn(Result $result): string => $this->resultType($result, $source), + [...array_map(static fn(RecodeRule $rule): Result => $rule->result, $rules), $unmatched], + ))); + $targetSpan = $targetMode === TargetMode::Create + ? $statement->targetSpans[$index] + : $statement->sourceSpans[$index]; + if ($targetMode === TargetMode::Create && in_array('string', $resultTypes, true)) { + $this->fail( + 'string_target_requires_declaration', + 'New string targets require an explicit declaration outside Frontend 0.2.', + $targetSpan, + ); + } + if (count($resultTypes) !== 1) { + $this->fail( + 'mixed_result_types', + 'All RECODE results, including unmatched behavior, must have one type.', + $statement->span, + ); + } + if ($targetMode === TargetMode::Replace && $resultTypes[0] !== $this->storageType($source)) { + $this->fail( + 'type_mismatch', + 'In-place RECODE cannot change the variable storage kind.', + $statement->span, + ); + } + $bound[] = new BoundRecode( + $source->name, + $targetNames[$index], + $targetMode, + $this->nonEmptyRules($rules, $statement->span), + $unmatched, + $statement->span, + ); + } + + if ($targetMode === TargetMode::Create) { + foreach ($targetNames as $targetName) { + $schema->addNumeric($targetName); + } + } + + return $bound; + } + + private function recodeMatch(RecodeInput $input, InputVariable $source): RecodeMatch + { + if ($input instanceof MissingInput) { + $this->fail( + 'spss_syntax_error', + 'MISSING includes user-missing values and is outside the bounded frontend.', + $input->span, + ); + } + if ($input instanceof SystemMissingInput) { + if ($source->storageKind === 'string') { + $this->fail( + 'system_missing_for_string', + 'SYSMIS cannot match a string variable.', + $input->span, + ); + } + + return new SystemMissingMatch(); + } + if ($input instanceof RangeInput) { + if ($input->lower === null || $input->upper === null) { + $this->fail('spss_syntax_error', 'RECODE ranges require two finite endpoints.', $input->span); + } + $lower = $this->typedValue($input->lower); + $upper = $this->typedValue($input->upper); + if (!$lower instanceof Binary64Value || !$upper instanceof Binary64Value || $source->storageKind !== 'numeric') { + $this->fail('type_mismatch', 'RECODE ranges require numeric sources and endpoints.', $input->span); + } + if ($lower->number() > $upper->number()) { + $this->fail('invalid_numeric_range', 'RECODE range lower bound exceeds its upper bound.', $input->span); + } + + return new RangeMatch($lower, $upper); + } + if ($input instanceof ValueInput) { + $values = array_map(fn(ScalarValue $value): TypedValue => $this->typedValue($value), $input->values); + $expectedType = $this->storageType($source); + foreach ($values as $value) { + if ($this->typedValueType($value) !== $expectedType) { + $this->fail('type_mismatch', 'RECODE match values must match the source type.', $input->span); + } + } + + return new ExactMatch(...$values); + } + + throw new \LogicException(sprintf('Unsupported recode input %s.', $input::class)); + } + + private function recodeResult(RecodeOutput $output, InputVariable $source): Result + { + if ($output->kind === RecodeOutputKind::Copy) { + return new CopyResult(); + } + if ($output->kind === RecodeOutputKind::SystemMissing) { + if ($source->storageKind === 'string') { + $this->fail( + 'system_missing_for_string', + 'SYSMIS cannot be produced for a string variable.', + $output->span, + ); + } + + return new SystemMissingResult(); + } + + return new LiteralResult($this->typedValue( + $output->value ?? throw new \LogicException('A literal RECODE result requires a value.'), + )); + } + + private function resultType(Result $result, InputVariable $source): string + { + if ($result instanceof CopyResult) { + return $this->storageType($source); + } + if ($result instanceof SystemMissingResult) { + return 'binary64'; + } + if ($result instanceof LiteralResult) { + return $this->typedValueType($result->value); + } + + throw new \LogicException(sprintf('Unsupported recode result %s.', $result::class)); + } + + /** @return list */ + private function valueLabels(ValueLabelsStatement $statement, SchemaState $schema): array + { + $bound = []; + foreach ($statement->groups as $group) { + $labels = array_map( + fn(\OpenStatSpec\Frontend\Spss\Ast\ValueLabel $label): ValueLabel => new ValueLabel( + $this->typedValue($label->value), + $label->label, + ), + $group->labels, + ); + foreach ($group->variables as $index => $name) { + $variable = $schema->resolve($name, $group->variableSpans[$index]); + $expectedType = $this->storageType($variable); + $seen = []; + foreach ($labels as $label) { + if ($this->typedValueType($label->value) !== $expectedType) { + $this->fail( + 'type_mismatch', + 'VALUE LABELS codes must match the variable storage kind.', + $group->span, + ); + } + $key = $label->value->canonicalKey(); + if (isset($seen[$key])) { + $this->fail( + 'duplicate_value_label', + 'VALUE LABELS contains duplicate canonical codes.', + $group->span, + ); } + $seen[$key] = true; } - continue; + $bound[] = new BoundValueLabels($variable->name, $labels, $group->span); } + } - $this->fail($statement->line(), sprintf('Unsupported AST statement %s.', $statement::class)); + return $bound; + } + + private function typedValue(ScalarValue $value): TypedValue + { + if ($value->isNumeric()) { + return Binary64Value::fromBits(DecimalBinary64::bits( + $value->numericToken ?? throw new \LogicException('Numeric scalar has no source token.'), + )); + } + + return new StringValue((string) $value->value); + } + + private function typedValueType(TypedValue $value): string + { + return $value instanceof Binary64Value ? 'binary64' : 'string'; + } + + /** @return 'binary64'|'string' */ + private function storageType(InputVariable $variable): string + { + return $variable->storageKind === 'numeric' ? 'binary64' : 'string'; + } + + private function validateTargetName(string $name, SourceSpan $span): void + { + if (str_starts_with($name, '__')) { + $this->fail('reserved_target_name', 'Target names beginning with __ are reserved.', $span); + } + } + + private function validateFormat(int $width, int $decimals, SourceSpan $span): void + { + if ( + $width < 1 + || $width > 40 + || $decimals < 0 + || $decimals > 16 + || ($decimals > 0 && $width < $decimals + 2) + ) { + $this->fail('invalid_format', 'Invalid SPSS F format.', $span); + } + } + + /** + * @param list $rules + * @return non-empty-list + */ + private function nonEmptyRules(array $rules, SourceSpan $span): array + { + if ($rules === []) { + $this->fail('spss_syntax_error', 'RECODE requires at least one non-ELSE rule.', $span); } - return new BoundProgram($datasetId, $bound); + return $rules; } - private function fail(int $line, string $message): never + private function fail(string $code, string $message, SourceSpan $span): never { - throw new SpssSyntaxException([new Diagnostic($line, 1, $message)]); + throw TransformationFailure::at($code, '$.source_text', $message, $span); } } diff --git a/src/Frontend/Spss/Binding/BoundCompute.php b/src/Frontend/Spss/Binding/BoundCompute.php new file mode 100644 index 0000000..265ce2c --- /dev/null +++ b/src/Frontend/Spss/Binding/BoundCompute.php @@ -0,0 +1,19 @@ + $statements */ - public function __construct(public string $datasetId, public array $statements) {} + /** @param non-empty-list $statements */ + public function __construct(public string $inputAlias, public array $statements) {} } diff --git a/src/Frontend/Spss/Binding/BoundRecode.php b/src/Frontend/Spss/Binding/BoundRecode.php index 186885f..a3dfee2 100644 --- a/src/Frontend/Spss/Binding/BoundRecode.php +++ b/src/Frontend/Spss/Binding/BoundRecode.php @@ -4,7 +4,10 @@ namespace OpenStatSpec\Frontend\Spss\Binding; -use OpenStatSpec\Frontend\Spss\Ast\RecodeRule; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Plan\Recode\RecodeRule; +use OpenStatSpec\Transformation\Plan\Recode\Result; +use OpenStatSpec\Transformation\Plan\TargetMode; final readonly class BoundRecode implements BoundStatement { @@ -12,6 +15,9 @@ public function __construct( public string $sourceVariable, public string $targetVariable, + public TargetMode $targetMode, public array $rules, + public Result $unmatched, + public SourceSpan $span, ) {} } diff --git a/src/Frontend/Spss/Binding/BoundValueLabels.php b/src/Frontend/Spss/Binding/BoundValueLabels.php index fb45ec4..bfbf61d 100644 --- a/src/Frontend/Spss/Binding/BoundValueLabels.php +++ b/src/Frontend/Spss/Binding/BoundValueLabels.php @@ -4,10 +4,15 @@ namespace OpenStatSpec\Frontend\Spss\Binding; -use OpenStatSpec\Frontend\Spss\Ast\ValueLabel; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; +use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; final readonly class BoundValueLabels implements BoundStatement { /** @param non-empty-list $labels */ - public function __construct(public string $variable, public array $labels) {} + public function __construct( + public string $variable, + public array $labels, + public SourceSpan $span, + ) {} } diff --git a/src/Frontend/Spss/Binding/BoundVariableLabel.php b/src/Frontend/Spss/Binding/BoundVariableLabel.php index 65e76d0..45e5a8d 100644 --- a/src/Frontend/Spss/Binding/BoundVariableLabel.php +++ b/src/Frontend/Spss/Binding/BoundVariableLabel.php @@ -4,7 +4,13 @@ namespace OpenStatSpec\Frontend\Spss\Binding; +use OpenStatSpec\Transformation\Diagnostic\SourceSpan; + final readonly class BoundVariableLabel implements BoundStatement { - public function __construct(public string $variable, public string $label) {} + public function __construct( + public string $variable, + public string $label, + public SourceSpan $span, + ) {} } diff --git a/src/Frontend/Spss/Binding/SchemaState.php b/src/Frontend/Spss/Binding/SchemaState.php new file mode 100644 index 0000000..b26c8d8 --- /dev/null +++ b/src/Frontend/Spss/Binding/SchemaState.php @@ -0,0 +1,72 @@ + */ + private array $variables; + + /** @var array> */ + private array $index = []; + + public function __construct(InputSchema $schema) + { + $this->variables = $schema->variables; + foreach ($this->variables as $variable) { + $this->index[$this->key($variable->name)][] = $variable; + } + } + + public function resolve( + string $name, + SourceSpan $span, + string $code = 'unknown_variable', + ): InputVariable { + $matches = $this->index[$this->key($name)] ?? []; + if (count($matches) !== 1) { + throw TransformationFailure::at( + $code, + '$.source_text', + sprintf('Variable %s does not resolve uniquely in the current schema.', $name), + $span, + ); + } + + return $matches[0]; + } + + public function find(string $name, SourceSpan $span): ?InputVariable + { + $matches = $this->index[$this->key($name)] ?? []; + if (count($matches) > 1) { + return $this->resolve($name, $span); + } + + return $matches[0] ?? null; + } + + public function contains(string $name): bool + { + return isset($this->index[$this->key($name)]); + } + + public function addNumeric(string $name): void + { + $variable = new InputVariable($name, 'numeric'); + $this->variables[] = $variable; + $this->index[$this->key($name)][] = $variable; + } + + private function key(string $name): string + { + return strtolower($name); + } +} diff --git a/src/Frontend/Spss/Compiler.php b/src/Frontend/Spss/Compiler.php index d2076c7..a3905a2 100644 --- a/src/Frontend/Spss/Compiler.php +++ b/src/Frontend/Spss/Compiler.php @@ -4,142 +4,83 @@ namespace OpenStatSpec\Frontend\Spss; -use LogicException; -use OpenStatSpec\Frontend\Spss\Ast\ElseInput; -use OpenStatSpec\Frontend\Spss\Ast\RangeInput; -use OpenStatSpec\Frontend\Spss\Ast\RecodeInput; -use OpenStatSpec\Frontend\Spss\Ast\RecodeOutput; -use OpenStatSpec\Frontend\Spss\Ast\RecodeOutputKind; -use OpenStatSpec\Frontend\Spss\Ast\ScalarValue as AstScalarValue; -use OpenStatSpec\Frontend\Spss\Ast\SystemMissingInput; -use OpenStatSpec\Frontend\Spss\Ast\ValueInput; -use OpenStatSpec\Frontend\Spss\Ast\ValueLabel as AstValueLabel; +use OpenStatSpec\Frontend\Spss\Binding\BoundCompute; +use OpenStatSpec\Frontend\Spss\Binding\BoundConditionalAssign; +use OpenStatSpec\Frontend\Spss\Binding\BoundExecute; +use OpenStatSpec\Frontend\Spss\Binding\BoundFormat; +use OpenStatSpec\Frontend\Spss\Binding\BoundMeasurementLevel; use OpenStatSpec\Frontend\Spss\Binding\BoundProgram; use OpenStatSpec\Frontend\Spss\Binding\BoundRecode; use OpenStatSpec\Frontend\Spss\Binding\BoundValueLabels; use OpenStatSpec\Frontend\Spss\Binding\BoundVariableLabel; -use OpenStatSpec\Frontend\Spss\Binding\BoundCreateVariable; -use OpenStatSpec\Frontend\Spss\Binding\BoundDeleteVariable; -use OpenStatSpec\Transformation\Model\Action\AssignValueAction; -use OpenStatSpec\Transformation\Model\CreateVariableOperation; -use OpenStatSpec\Transformation\Model\DeleteVariableOperation; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\Action\SetMissingAction; -use OpenStatSpec\Transformation\Model\RecodeAction; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\RecodeRule; -use OpenStatSpec\Transformation\Model\RecodeSelector; -use OpenStatSpec\Transformation\Model\ScalarValue; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\ExactValueSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\Selector\NumericRangeSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; -use OpenStatSpec\Transformation\Model\ValueLabel; -use OpenStatSpec\Transformation\Validation\PlanValidator; +use OpenStatSpec\Transformation\Plan\Operation; +use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\ConditionalAssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\ExecuteOperation; +use OpenStatSpec\Transformation\Plan\Operation\RecodeOperation; +use OpenStatSpec\Transformation\Plan\Operation\ReplaceValueLabelsOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetFormatOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetMeasurementLevelOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetVariableLabelOperation; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\TransformationPlan; -/** Compiles only bound SPSS semantics into the source-neutral canonical model. */ +/** Compiles only bound SPSS semantics into the source-neutral official plan. */ final class Compiler { - public function __construct(private readonly PlanValidator $validator = new PlanValidator()) {} - public function compile(BoundProgram $program): TransformationPlan { $operations = []; foreach ($program->statements as $statement) { - if ($statement instanceof BoundCreateVariable) { - $operations[] = new CreateVariableOperation( + $operations[] = match (true) { + $statement instanceof BoundRecode => new RecodeOperation( + $statement->sourceVariable, + $statement->targetVariable, + $statement->targetMode, + $statement->rules, + $statement->unmatched, + ), + $statement instanceof BoundCompute => new AssignOperation( + $statement->target, + $statement->targetMode, + $statement->value, + ), + $statement instanceof BoundConditionalAssign => new ConditionalAssignOperation( + $statement->condition, + $statement->target, + $statement->value, + ), + $statement instanceof BoundVariableLabel => new SetVariableLabelOperation( + $statement->variable, + $statement->label, + ), + $statement instanceof BoundValueLabels => new ReplaceValueLabelsOperation( $statement->variable, - $statement->storageKind, - $statement->storageKind === 'string' ? $statement->declaredStringWidth : null, - ); - continue; - } - if ($statement instanceof BoundDeleteVariable) { - $operations[] = new DeleteVariableOperation($statement->variable); - continue; - } - if ($statement instanceof BoundRecode) { - $rules = []; - $hasElse = false; - foreach ($statement->rules as $rule) { - $action = $this->action($rule->output); - if ($rule->input instanceof ValueInput) { - foreach ($rule->input->values as $value) { - $rules[] = new RecodeRule( - new ExactValueSelector($this->scalar($value)), - $action, - ); - } - continue; - } - $selector = $this->selector($rule->input); - $hasElse = $hasElse || $selector instanceof ElseSelector; - $rules[] = new RecodeRule($selector, $action); - } - if (!$hasElse) { - $defaultAction = $statement->sourceVariable === $statement->targetVariable - ? new CopySourceAction() - : new SetMissingAction(); - $rules[] = new RecodeRule(new ElseSelector(), $defaultAction); - } - $operations[] = new RecodeOperation($statement->sourceVariable, $statement->targetVariable, $rules); - continue; - } - if ($statement instanceof BoundVariableLabel) { - $operations[] = new SetVariableLabelOperation($statement->variable, $statement->label); - continue; - } - if ($statement instanceof BoundValueLabels) { - $labels = array_map( - fn(AstValueLabel $label): ValueLabel => new ValueLabel( - $this->scalar($label->value), - $label->label, - ), $statement->labels, - ); - $operations[] = new SetValueLabelsOperation($statement->variable, $labels); - continue; - } - - throw new LogicException(sprintf('Unsupported bound SPSS statement %s.', $statement::class)); + ), + $statement instanceof BoundFormat => new SetFormatOperation( + $statement->variable, + $statement->family, + $statement->width, + $statement->decimals, + ), + $statement instanceof BoundMeasurementLevel => new SetMeasurementLevelOperation( + $statement->variable, + $statement->level, + ), + $statement instanceof BoundExecute => new ExecuteOperation(), + default => throw new \LogicException(sprintf( + 'Unsupported bound SPSS statement %s.', + $statement::class, + )), + }; } - $plan = new TransformationPlan($program->datasetId, $operations); - $this->validator->assertValid($plan); - - return $plan; - } - - private function selector(RecodeInput $input): RecodeSelector - { - return match (true) { - $input instanceof ValueInput => new ExactValueSelector($this->scalar($input->value)), - $input instanceof RangeInput => new NumericRangeSelector( - $input->lower === null ? null : (float) $input->lower->value, - $input->upper === null ? null : (float) $input->upper->value, - ), - $input instanceof SystemMissingInput => new MissingValueSelector(), - $input instanceof ElseInput => new ElseSelector(), - default => throw new LogicException(sprintf('Unsupported SPSS recode input %s.', $input::class)), - }; - } + $contract = array_any( + $operations, + static fn(Operation $operation): bool => $operation->minimumContract() === PlanContract::V02, + ) ? PlanContract::V02 : PlanContract::V01; - private function action(RecodeOutput $output): RecodeAction - { - return match ($output->kind) { - RecodeOutputKind::Value => new AssignValueAction($this->scalar( - $output->value ?? throw new LogicException('A value output must contain a scalar.'), - )), - RecodeOutputKind::Copy => new CopySourceAction(), - RecodeOutputKind::SystemMissing => new SetMissingAction(), - }; - } - - private function scalar(AstScalarValue $value): ScalarValue - { - return is_string($value->value) ? ScalarValue::string($value->value) : ScalarValue::number($value->value); + return new TransformationPlan($contract, $program->inputAlias, $operations); } } diff --git a/src/Frontend/Spss/Parser.php b/src/Frontend/Spss/Parser.php index ab1945c..2745216 100644 --- a/src/Frontend/Spss/Parser.php +++ b/src/Frontend/Spss/Parser.php @@ -32,6 +32,7 @@ use OpenStatSpec\Frontend\Spss\Ast\ValueLabelGroup; use OpenStatSpec\Frontend\Spss\Ast\ValueLabelsStatement; use OpenStatSpec\Frontend\Spss\Ast\VariableLabelsStatement; +use OpenStatSpec\Frontend\Spss\Ast\VariableLabelAssignment; use OpenStatSpec\Frontend\Spss\Ast\VariableLevelGroup; use OpenStatSpec\Frontend\Spss\Ast\VariableLevelStatement; use OpenStatSpec\Frontend\Spss\Ast\VariableOperand; @@ -119,6 +120,7 @@ private function compute(Token $command): ComputeStatement return new ComputeStatement( $target->lexeme, + $target->span, $expression, SourceSpan::cover($command->span, $expression->span()), ); @@ -137,6 +139,7 @@ private function ifStatement(Token $command): IfStatement return new IfStatement( $predicate, $target->lexeme, + $target->span, $expression, SourceSpan::cover($command->span, $expression->span()), ); @@ -295,6 +298,7 @@ private function formats(Token $command): FormatsStatement 'F', (int) $matches[1], (int) $decimalMatches[1], + $variable->span, SourceSpan::cover($variable->span, $rightParenthesis->span), ); } while ($this->current()->isWord()); @@ -310,10 +314,14 @@ private function variableLevel(Token $command): VariableLevelStatement $groups = []; do { $variables = []; + $variableSpans = []; $first = $this->consumeIdentifier('VARIABLE LEVEL requires at least one variable.'); $variables[] = $first->lexeme; + $variableSpans[] = $first->span; while ($this->current()->isWord()) { - $variables[] = $this->advance()->lexeme; + $variable = $this->advance(); + $variables[] = $variable->lexeme; + $variableSpans[] = $variable->span; } $this->consume(TokenType::LeftParenthesis, 'Expected ( before the measurement level.'); $level = $this->current(); @@ -327,6 +335,7 @@ private function variableLevel(Token $command): VariableLevelStatement $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after the measurement level.'); $groups[] = new VariableLevelGroup( $variables, + $variableSpans, strtolower($level->lexeme), SourceSpan::cover($first->span, $rightParenthesis->span), ); @@ -341,12 +350,16 @@ private function variableLevel(Token $command): VariableLevelStatement private function recode(Token $command): RecodeStatement { $sources = []; + $sourceSpans = []; do { - $sources[] = $this->consumeIdentifier('Expected a source variable after RECODE.')->lexeme; + $source = $this->consumeIdentifier('Expected a source variable after RECODE.'); + $sources[] = $source->lexeme; + $sourceSpans[] = $source->span; } while ($this->current()->isWord() && !$this->current()->isKeyword('INTO')); $rules = []; while ($this->match(TokenType::LeftParenthesis)) { + $leftParenthesis = $this->previous(); $input = $this->recodeInput(); if ($this->check(TokenType::Comma)) { if (!$input instanceof ValueInput) { @@ -365,75 +378,107 @@ private function recode(Token $command): RecodeStatement } $this->consume(TokenType::Equals, 'Expected = in a RECODE rule.'); $output = $this->recodeOutput(); - $this->consume(TokenType::RightParenthesis, 'Expected ) after a RECODE rule.'); - $rules[] = new RecodeRule($input, $output); + $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after a RECODE rule.'); + $rules[] = new RecodeRule($input, $output, SourceSpan::cover($leftParenthesis->span, $rightParenthesis->span)); } if ($rules === []) { $this->fail($this->current(), 'RECODE requires at least one parenthesized rule.'); } $targets = []; + $targetSpans = []; if ($this->matchKeyword('INTO')) { do { - $targets[] = $this->consumeIdentifier('Expected a target variable after INTO.')->lexeme; + $target = $this->consumeIdentifier('Expected a target variable after INTO.'); + $targets[] = $target->lexeme; + $targetSpans[] = $target->span; } while ($this->current()->isWord()); } - return new RecodeStatement($command->line, $sources, $rules, $targets); + return new RecodeStatement( + $command->line, + $sources, + $sourceSpans, + $rules, + $targets, + $targetSpans, + SourceSpan::cover($command->span, $this->previous()->span), + ); } private function recodeInput(): RecodeInput { if ($this->matchKeyword('SYSMIS')) { - return new SystemMissingInput(); + return new SystemMissingInput($this->previous()->span); } if ($this->matchKeyword('MISSING')) { - return new MissingInput(); + return new MissingInput($this->previous()->span); } if ($this->matchKeyword('ELSE')) { - return new ElseInput(); + return new ElseInput($this->previous()->span); } if ($this->matchKeyword('LOWEST')) { + $lowest = $this->previous(); $this->consumeKeyword('THRU', 'LOWEST must be followed by THRU.'); - $upper = $this->matchKeyword('HIGHEST') ? null : $this->scalar('Expected an upper bound after LOWEST THRU.'); + if ($this->matchKeyword('HIGHEST')) { + return new RangeInput(null, null, SourceSpan::cover($lowest->span, $this->previous()->span)); + } + $upper = $this->scalar('Expected an upper bound after LOWEST THRU.'); - return new RangeInput(null, $upper); + return new RangeInput(null, $upper, SourceSpan::cover($lowest->span, $upper->span)); } $lower = $this->scalar('Expected a value or selector in a RECODE rule.'); if (!$this->matchKeyword('THRU')) { return new ValueInput($lower); } - $upper = $this->matchKeyword('HIGHEST') ? null : $this->scalar('Expected an upper bound after THRU.'); + if ($this->matchKeyword('HIGHEST')) { + return new RangeInput($lower, null, SourceSpan::cover($lower->span, $this->previous()->span)); + } + $upper = $this->scalar('Expected an upper bound after THRU.'); - return new RangeInput($lower, $upper); + return new RangeInput($lower, $upper, SourceSpan::cover($lower->span, $upper->span)); } private function recodeOutput(): RecodeOutput { if ($this->matchKeyword('COPY')) { - return new RecodeOutput(RecodeOutputKind::Copy); + return new RecodeOutput(RecodeOutputKind::Copy, $this->previous()->span); } if ($this->matchKeyword('SYSMIS')) { - return new RecodeOutput(RecodeOutputKind::SystemMissing); + return new RecodeOutput(RecodeOutputKind::SystemMissing, $this->previous()->span); } - return new RecodeOutput(RecodeOutputKind::Value, $this->scalar('Expected a value, COPY, or SYSMIS after =.')); + $value = $this->scalar('Expected a value, COPY, or SYSMIS after =.'); + + return new RecodeOutput(RecodeOutputKind::Value, $value->span, $value); } private function variableLabels(Token $command): VariableLabelsStatement { $labels = []; + $assignments = []; while (!$this->check(TokenType::Terminator) && !$this->check(TokenType::EndOfFile)) { - $variable = $this->consumeIdentifier('Expected a variable name in VARIABLE LABELS.')->lexeme; - $label = $this->consume(TokenType::String, 'Expected a quoted variable label.')->lexeme; - $labels[$variable] = $label; + $variable = $this->consumeIdentifier('Expected a variable name in VARIABLE LABELS.'); + $label = $this->consume(TokenType::String, 'Expected a quoted variable label.'); + $labels[$variable->lexeme] = $label->lexeme; + $assignments[] = new VariableLabelAssignment( + $variable->lexeme, + $label->lexeme, + $variable->span, + SourceSpan::cover($variable->span, $label->span), + ); } if ($labels === []) { $this->fail($this->current(), 'VARIABLE LABELS requires at least one variable and label.'); } - return new VariableLabelsStatement($command->line, $labels); + return new VariableLabelsStatement( + $command->line, + $labels, + $assignments, + SourceSpan::cover($command->span, $assignments[array_key_last($assignments)]->span), + ); } private function valueLabels(Token $command): ValueLabelsStatement @@ -441,8 +486,11 @@ private function valueLabels(Token $command): ValueLabelsStatement $groups = []; do { $variables = []; + $variableSpans = []; while ($this->current()->isWord()) { - $variables[] = $this->advance()->lexeme; + $variable = $this->advance(); + $variables[] = $variable->lexeme; + $variableSpans[] = $variable->span; } if ($variables === []) { $this->fail($this->current(), 'VALUE LABELS requires at least one variable before its value-label pairs.'); @@ -451,16 +499,25 @@ private function valueLabels(Token $command): ValueLabelsStatement $labels = []; while (!$this->check(TokenType::Slash) && !$this->check(TokenType::Terminator) && !$this->check(TokenType::EndOfFile)) { $value = $this->scalar('Expected a value in VALUE LABELS.'); - $label = $this->consume(TokenType::String, 'Expected a quoted label after the value.')->lexeme; - $labels[] = new ValueLabel($value, $label); + $label = $this->consume(TokenType::String, 'Expected a quoted label after the value.'); + $labels[] = new ValueLabel($value, $label->lexeme, SourceSpan::cover($value->span, $label->span)); } if ($labels === []) { $this->fail($this->current(), 'VALUE LABELS requires at least one value-label pair.'); } - $groups[] = new ValueLabelGroup($variables, $labels); + $groups[] = new ValueLabelGroup( + $variables, + $variableSpans, + $labels, + SourceSpan::cover($variableSpans[0], $labels[array_key_last($labels)]->span), + ); } while ($this->match(TokenType::Slash)); - return new ValueLabelsStatement($command->line, $groups); + return new ValueLabelsStatement( + $command->line, + $groups, + SourceSpan::cover($command->span, $groups[array_key_last($groups)]->span), + ); } private function string(Token $command): StringStatement @@ -482,9 +539,15 @@ private function string(Token $command): StringStatement if ($widthValue > 32767) { $this->fail($this->previous(), 'STRING width must be at most 32767.'); } - $this->consume(TokenType::RightParenthesis, 'Expected ) after STRING width.'); + $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after STRING width.'); - return new StringStatement($command->line, $variables, $widthValue); + return new StringStatement( + $command->line, + $variables, + $widthValue, + $command->span, + SourceSpan::cover($command->span, $rightParenthesis->span), + ); } private function deleteVariables(Token $command): DeleteVariablesStatement @@ -498,23 +561,31 @@ private function deleteVariables(Token $command): DeleteVariablesStatement $variables[] = $variable; } while ($this->current()->isWord()); - return new DeleteVariablesStatement($command->line, $variables); + return new DeleteVariablesStatement( + $command->line, + $variables, + $command->span, + SourceSpan::cover($command->span, $this->previous()->span), + ); } private function scalar(string $message): ScalarValue { if ($this->match(TokenType::String)) { - return new ScalarValue($this->previous()->lexeme); + $token = $this->previous(); + + return new ScalarValue($token->lexeme, null, $token->span); } if (!$this->match(TokenType::Number)) { $this->fail($this->current(), $message); } $lexeme = $this->previous()->lexeme; - $value = (float) $lexeme; - if (!is_finite($value)) { - $this->fail($this->previous(), 'Numeric literals must be finite IEEE-754 binary64 values.'); + try { + DecimalBinary64::bits($lexeme); + } catch (TransformationFailure $failure) { + $this->fail($this->previous(), $failure->getMessage()); } - return new ScalarValue($value); + return new ScalarValue($lexeme, $lexeme, $this->previous()->span); } private function consumeIdentifier(string $message): Token diff --git a/src/Frontend/Spss/Request/InputSchema.php b/src/Frontend/Spss/Request/InputSchema.php index 0b99407..c945af7 100644 --- a/src/Frontend/Spss/Request/InputSchema.php +++ b/src/Frontend/Spss/Request/InputSchema.php @@ -4,8 +4,56 @@ namespace OpenStatSpec\Frontend\Spss\Request; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; + final readonly class InputSchema { - /** @param non-empty-list $variables */ - public function __construct(public array $variables) {} + /** @var non-empty-list */ + public array $variables; + + /** @param array $variables */ + public function __construct(array $variables) + { + if ($variables === [] || !array_is_list($variables)) { + self::schema('$.input_schema.variables', 'Variables must be a non-empty ordered array.'); + } + $validated = []; + foreach ($variables as $variable) { + if (!$variable instanceof InputVariable) { + self::schema('$.input_schema.variables', 'Every schema variable must be an InputVariable.'); + } + $validated[] = $variable; + } + $this->variables = $validated; + } + + public static function fromArray(mixed $raw, string $path = '$.input_schema'): self + { + if (!is_array($raw) || array_is_list($raw)) { + self::schema($path, 'Input schema must be an object.'); + } + $keys = array_keys($raw); + sort($keys, SORT_STRING); + if ($keys !== ['variables']) { + self::schema($path, 'Object has missing or extra members.'); + } + $variables = $raw['variables']; + if (!is_array($variables) || !array_is_list($variables) || $variables === []) { + self::schema($path . '.variables', 'Variables must be a non-empty ordered array.'); + } + + return new self(array_map( + static fn(mixed $variable, int $index): InputVariable => InputVariable::fromArray( + $variable, + $path . '.variables[' . $index . ']', + ), + $variables, + array_keys($variables), + )); + } + + private static function schema(string $path, string $message): never + { + throw TransformationFailure::at('plan_schema_invalid', $path, $message); + } } diff --git a/src/Frontend/Spss/Request/InputVariable.php b/src/Frontend/Spss/Request/InputVariable.php index ff80040..e23bbb2 100644 --- a/src/Frontend/Spss/Request/InputVariable.php +++ b/src/Frontend/Spss/Request/InputVariable.php @@ -4,6 +4,8 @@ namespace OpenStatSpec\Frontend\Spss\Request; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; + final readonly class InputVariable { /** @param list, label: string}> $valueLabels */ @@ -16,5 +18,196 @@ public function __construct( public ?int $width = null, public ?int $decimals = null, public ?string $measurementLevel = null, - ) {} + ) { + self::nonEmptyString($name, '$.input_schema.variables[].name'); + if (!in_array($storageKind, ['numeric', 'string'], true)) { + self::schema('$.input_schema.variables[].storage_kind', 'Storage kind must be numeric or string.'); + } + self::nullableString($variableLabel, '$.input_schema.variables[].variable_label'); + self::nullableString($formatFamily, '$.input_schema.variables[].format_family'); + if ($width !== null && $width < 1) { + self::schema('$.input_schema.variables[].width', 'Width must be at least one.'); + } + if ($decimals !== null && $decimals < 0) { + self::schema('$.input_schema.variables[].decimals', 'Decimals must not be negative.'); + } + if (!in_array($measurementLevel, [null, 'nominal', 'ordinal', 'scale'], true)) { + self::schema('$.input_schema.variables[].measurement_level', 'Invalid measurement level.'); + } + self::valueLabels($valueLabels, '$.input_schema.variables[].value_labels'); + } + + public static function fromArray(mixed $raw, string $path): self + { + if (!is_array($raw) || array_is_list($raw)) { + self::schema($path, 'Input variable must be an object.'); + } + $allowed = [ + 'name', + 'storage_kind', + 'variable_label', + 'value_labels', + 'format_family', + 'width', + 'decimals', + 'measurement_level', + ]; + if (!array_key_exists('name', $raw) || !array_key_exists('storage_kind', $raw)) { + self::schema($path, 'Input variable is missing a required member.'); + } + foreach (array_keys($raw) as $key) { + if (!in_array($key, $allowed, true)) { + self::schema($path, 'Input variable has an extra member.'); + } + } + + $name = self::nonEmptyString($raw['name'], $path . '.name'); + $storageKind = self::string($raw['storage_kind'], $path . '.storage_kind'); + if (!in_array($storageKind, ['numeric', 'string'], true)) { + self::schema($path . '.storage_kind', 'Storage kind must be numeric or string.'); + } + $variableLabel = self::optionalNullableString($raw, 'variable_label', $path); + $formatFamily = self::optionalNullableString($raw, 'format_family', $path); + $width = self::optionalNullableInteger($raw, 'width', $path, 1); + $decimals = self::optionalNullableInteger($raw, 'decimals', $path, 0); + $measurementLevel = self::optionalNullableString($raw, 'measurement_level', $path); + if (!in_array($measurementLevel, [null, 'nominal', 'ordinal', 'scale'], true)) { + self::schema($path . '.measurement_level', 'Invalid measurement level.'); + } + + return new self( + $name, + $storageKind, + $variableLabel, + self::valueLabels($raw['value_labels'] ?? [], $path . '.value_labels'), + $formatFamily, + $width, + $decimals, + $measurementLevel, + ); + } + + /** @return list, label: string}> */ + private static function valueLabels(mixed $raw, string $path): array + { + if (!is_array($raw) || !array_is_list($raw)) { + self::schema($path, 'Value labels must be an ordered array.'); + } + $labels = []; + foreach ($raw as $index => $label) { + $labelPath = $path . '[' . $index . ']'; + if (!is_array($label) || array_is_list($label)) { + self::schema($labelPath, 'Value label must be an object.'); + } + $keys = array_keys($label); + sort($keys, SORT_STRING); + if ($keys !== ['label', 'value']) { + self::schema($labelPath, 'Value label has missing or extra members.'); + } + $labels[] = [ + 'value' => self::typedValue($label['value'], $labelPath . '.value'), + 'label' => self::string($label['label'], $labelPath . '.label'), + ]; + } + + return $labels; + } + + /** @return array */ + private static function typedValue(mixed $raw, string $path): array + { + if (!is_array($raw) || array_is_list($raw)) { + self::schema($path, 'Typed value must be an object.'); + } + $type = self::string($raw['type'] ?? null, $path . '.type'); + if ($type === 'binary64') { + self::exactKeys($raw, ['type', 'bits'], $path); + $bits = self::string($raw['bits'], $path . '.bits'); + if ( + preg_match('/\A[0-9a-f]{16}\z/D', $bits) !== 1 + || $bits === '8000000000000000' + || substr($bits, 0, 3) === '7ff' + || substr($bits, 0, 3) === 'fff' + ) { + self::schema($path . '.bits', 'Invalid canonical binary64 bits.'); + } + + return ['type' => 'binary64', 'bits' => $bits]; + } + if ($type === 'string') { + self::exactKeys($raw, ['type', 'value'], $path); + + return ['type' => 'string', 'value' => self::string($raw['value'], $path . '.value')]; + } + + self::schema($path . '.type', 'Typed value type must be binary64 or string.'); + } + + /** @param array $raw */ + private static function optionalNullableString(array $raw, string $key, string $path): ?string + { + if (!array_key_exists($key, $raw) || $raw[$key] === null) { + return null; + } + + return self::string($raw[$key], $path . '.' . $key); + } + + /** @param array $raw */ + private static function optionalNullableInteger(array $raw, string $key, string $path, int $minimum): ?int + { + if (!array_key_exists($key, $raw) || $raw[$key] === null) { + return null; + } + if (!is_int($raw[$key]) || $raw[$key] < $minimum) { + self::schema($path . '.' . $key, 'Value is outside the allowed integer range.'); + } + + return $raw[$key]; + } + + /** + * @param array $object + * @param list $expected + */ + private static function exactKeys(array $object, array $expected, string $path): void + { + $keys = array_keys($object); + sort($keys, SORT_STRING); + sort($expected, SORT_STRING); + if ($keys !== $expected) { + self::schema($path, 'Object has missing or extra members.'); + } + } + + private static function nonEmptyString(mixed $value, string $path): string + { + $string = self::string($value, $path); + if ($string === '') { + self::schema($path, 'String must not be empty.'); + } + + return $string; + } + + private static function nullableString(?string $value, string $path): void + { + if ($value !== null && preg_match('//u', $value) !== 1) { + self::schema($path, 'Value must be a valid UTF-8 string.'); + } + } + + private static function string(mixed $value, string $path): string + { + if (!is_string($value) || preg_match('//u', $value) !== 1) { + self::schema($path, 'Value must be a valid UTF-8 string.'); + } + + return $value; + } + + private static function schema(string $path, string $message): never + { + throw TransformationFailure::at('plan_schema_invalid', $path, $message); + } } diff --git a/src/Frontend/Spss/Request/SpssFrontendRequest.php b/src/Frontend/Spss/Request/SpssFrontendRequest.php index 494ad43..f2c1fe6 100644 --- a/src/Frontend/Spss/Request/SpssFrontendRequest.php +++ b/src/Frontend/Spss/Request/SpssFrontendRequest.php @@ -4,12 +4,81 @@ namespace OpenStatSpec\Frontend\Spss\Request; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; + final readonly class SpssFrontendRequest { + public const CONTRACT = 'openstatspec-spss-syntax-frontend-v0.2'; + public function __construct( public string $contract, public string $inputAlias, public InputSchema $inputSchema, public string $sourceText, - ) {} + ) { + if ($contract !== self::CONTRACT) { + self::schema('$.contract', 'Unsupported SPSS frontend contract.'); + } + self::nonEmptyString($inputAlias, '$.input_alias'); + self::nonEmptyString($sourceText, '$.source_text'); + } + + /** @param array $request */ + public static function fromArray(array $request): self + { + self::exactKeys($request, ['contract', 'input_alias', 'input_schema', 'source_text'], '$'); + $contract = self::string($request['contract'], '$.contract'); + $inputAlias = self::nonEmptyString($request['input_alias'], '$.input_alias'); + $sourceText = self::nonEmptyString($request['source_text'], '$.source_text'); + + return new self( + $contract, + $inputAlias, + InputSchema::fromArray($request['input_schema'], '$.input_schema'), + $sourceText, + ); + } + + public function sourceHash(): string + { + return hash('sha256', str_replace(["\r\n", "\r"], "\n", $this->sourceText)); + } + + /** + * @param array $object + * @param list $expected + */ + private static function exactKeys(array $object, array $expected, string $path): void + { + $actual = array_keys($object); + sort($actual, SORT_STRING); + sort($expected, SORT_STRING); + if ($actual !== $expected) { + self::schema($path, 'Object has missing or extra members.'); + } + } + + private static function nonEmptyString(mixed $value, string $path): string + { + $string = self::string($value, $path); + if ($string === '') { + self::schema($path, 'String must not be empty.'); + } + + return $string; + } + + private static function string(mixed $value, string $path): string + { + if (!is_string($value) || preg_match('//u', $value) !== 1) { + self::schema($path, 'Value must be a valid UTF-8 string.'); + } + + return $value; + } + + private static function schema(string $path, string $message): never + { + throw TransformationFailure::at('plan_schema_invalid', $path, $message); + } } diff --git a/src/Frontend/Spss/SpssCompilationResult.php b/src/Frontend/Spss/SpssCompilationResult.php new file mode 100644 index 0000000..561d70b --- /dev/null +++ b/src/Frontend/Spss/SpssCompilationResult.php @@ -0,0 +1,15 @@ +parser->parse($source); } - public function bind(string $datasetId, Program $program): BoundProgram + public function bind(string $inputAlias, InputSchema $inputSchema, Program $program): BoundProgram { - return $this->binder->bind($datasetId, $program); + return $this->binder->bind($inputAlias, $inputSchema, $program); } - public function compile(string $source, string $datasetId): TransformationPlan + public function compile(SpssFrontendRequest $request): SpssCompilationResult { - return $this->compiler->compile($this->binder->bind($datasetId, $this->parser->parse($source))); - } + $sourceHash = $request->sourceHash(); + $program = $this->parser->parse($request->sourceText); + $bound = $this->binder->bind($request->inputAlias, $request->inputSchema, $program); - public function compileForDataset(string $datasetId, string $source): TransformationPlan - { - return $this->compile($source, $datasetId); + return new SpssCompilationResult($this->compiler->compile($bound), $sourceHash); } } diff --git a/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php b/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php new file mode 100644 index 0000000..d31d8ea --- /dev/null +++ b/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php @@ -0,0 +1,89 @@ + $case */ + #[DataProvider('cases')] + public function testOfficialFrontend02Case(array $case): void + { + /** @var array $requestPayload */ + $requestPayload = $case['request']; + $request = SpssFrontendRequest::fromArray($requestPayload); + self::assertSame($case['expected_source_hash'], $request->sourceHash(), (string) $case['id']); + + try { + $result = (new SpssCompiler())->compile($request); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode(), (string) $case['id']); + self::assertNotNull($failure->diagnostics[0]->span, (string) $case['id']); + + return; + } + + self::assertNull($case['expected_error'], (string) $case['id']); + self::assertSame($case['expected_source_hash'], $result->sourceHash, (string) $case['id']); + self::assertSame($case['expected_plan_hash'], (new PlanCodec())->hash($result->plan), (string) $case['id']); + self::assertSame($this->expectedPlan($case), $result->plan->canonicalArray(), (string) $case['id']); + + if (isset($case['expected_plan_contract'])) { + self::assertSame($case['expected_plan_contract'], $result->plan->contract->value, (string) $case['id']); + } + } + + /** @return iterable}> */ + public static function cases(): iterable + { + $manifest = SpecificationManifest::load('conformance/spss-syntax-frontend-0.2.json'); + /** @var list> $cases */ + $cases = $manifest['cases']; + foreach ($cases as $case) { + yield (string) $case['id'] => [$case]; + } + } + + /** + * @param array $case + * @return array + */ + private function expectedPlan(array $case): array + { + if (isset($case['expected_plan_case'])) { + return $this->namedPlan('conformance/transformation-plan-0.2.json', (string) $case['expected_plan_case'], 'plan'); + } + if (isset($case['expected_plan_case_0_1'])) { + return $this->namedPlan('conformance/spss-syntax-frontend-0.1.json', (string) $case['expected_plan_case_0_1'], 'expected_plan'); + } + if (isset($case['expected_plan_0_1']) && is_array($case['expected_plan_0_1'])) { + return $case['expected_plan_0_1']; + } + + self::fail('Successful frontend case has no expected plan: ' . $case['id']); + } + + /** @return array */ + private function namedPlan(string $manifestPath, string $caseId, string $planKey): array + { + $manifest = SpecificationManifest::load($manifestPath); + /** @var list> $cases */ + $cases = $manifest['cases']; + foreach ($cases as $case) { + if ($case['id'] === $caseId && isset($case[$planKey]) && is_array($case[$planKey])) { + return $case[$planKey]; + } + } + + self::fail('Referenced plan case was not found: ' . $caseId); + } +} diff --git a/tests/Frontend/Spss/ParserTest.php b/tests/Frontend/Spss/ParserTest.php index f84bf81..ede9cd8 100644 --- a/tests/Frontend/Spss/ParserTest.php +++ b/tests/Frontend/Spss/ParserTest.php @@ -76,9 +76,11 @@ public function testParsesCommaSeparatedRecodeSelectorsInSourceOrder(): void self::assertCount(1, $statement->rules); $rule = $statement->rules[0]; self::assertInstanceOf(ValueInput::class, $rule->input); - self::assertSame([1.0, 2.0, 3.0], array_column($rule->input->values, 'value')); + self::assertSame(['1', '2', '3'], array_column($rule->input->values, 'value')); + self::assertSame(['1', '2', '3'], array_column($rule->input->values, 'numericToken')); self::assertNotNull($rule->output->value); - self::assertSame(0.0, $rule->output->value->value); + self::assertSame('0', $rule->output->value->value); + self::assertSame('0', $rule->output->value->numericToken); } public function testParsesVariableAndValueLabelGroups(): void diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index 458a603..5189d51 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -4,185 +4,312 @@ namespace OpenStatSpec\Tests\Frontend\Spss; +use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; +use OpenStatSpec\Frontend\Spss\Request\InputVariable; use OpenStatSpec\Frontend\Spss\SpssCompiler; -use OpenStatSpec\Frontend\Spss\SpssSyntaxException; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\Action\SetMissingAction; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Plan\PlanContract; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class SpssCompilerTest extends TestCase { - private const DATASET_ID = '018f47a2-4c10-7d34-8f11-93b1c3efc321'; + public function testRequestValidationPreservesOrderedSchemaAndNormalizesOnlySourceLineEndingsForHash(): void + { + $request = SpssFrontendRequest::fromArray([ + 'source_text' => "COMPUTE target = first.\r\nEXECUTE.\r", + 'input_schema' => [ + 'variables' => [ + [ + 'name' => 'first', + 'storage_kind' => 'numeric', + 'variable_label' => 'First', + 'value_labels' => [[ + 'value' => ['type' => 'binary64', 'bits' => '3ff0000000000000'], + 'label' => 'One', + ]], + 'format_family' => 'F', + 'width' => 8, + 'decimals' => 2, + 'measurement_level' => 'scale', + ], + ['name' => 'second', 'storage_kind' => 'string'], + ], + ], + 'input_alias' => 'parent', + 'contract' => 'openstatspec-spss-syntax-frontend-v0.2', + ]); + + self::assertSame(['first', 'second'], array_column($request->inputSchema->variables, 'name')); + self::assertSame("COMPUTE target = first.\r\nEXECUTE.\r", $request->sourceText); + self::assertSame( + hash('sha256', "COMPUTE target = first.\nEXECUTE.\n"), + $request->sourceHash(), + ); + } - public function testCompilesSupportedSubsetIntoValidatedCanonicalPlan(): void + public function testDirectInputVariableConstructionValidatesValueLabelShape(): void { - $plan = (new SpssCompiler())->compile(<<<'SPSS' - RECODE score (1=10) (2 THRU 4=20) (SYSMIS=99) (ELSE=COPY). - VARIABLE LABELS score 'Overall score'. - VALUE LABELS score 10 'Low' 20 'High'. - EXECUTE. - SPSS, self::DATASET_ID); - - self::assertSame(self::DATASET_ID, $plan->datasetId()); - self::assertCount(3, $plan->operations()); - /** @var RecodeOperation $recode */ - $recode = $plan->operations()[0]; - self::assertInstanceOf(RecodeOperation::class, $recode); - self::assertInstanceOf(MissingValueSelector::class, $recode->rules()[2]->selector()); - self::assertInstanceOf(ElseSelector::class, $recode->rules()[3]->selector()); - self::assertInstanceOf(CopySourceAction::class, $recode->rules()[3]->action()); - self::assertInstanceOf(SetVariableLabelOperation::class, $plan->operations()[1]); - self::assertInstanceOf(SetValueLabelsOperation::class, $plan->operations()[2]); - self::assertSame('openstatspec-transformation-plan-v1', $plan->canonicalArray()['contract']); - self::assertSame('in_place', $plan->canonicalArray()['mode']); + $this->expectException(TransformationFailure::class); + + new InputVariable( + 'q1', + 'numeric', + valueLabels: [['value' => ['type' => 'binary64', 'bits' => 'bad'], 'label' => 'Bad']], + ); } - public function testMakesSpssDefaultElseSemanticsExplicit(): void + /** @param array $payload */ + #[DataProvider('invalidRequestProvider')] + public function testRequestRejectsEveryNonSchemaShape(array $payload, string $path): void { - $inPlace = (new SpssCompiler())->compile('RECODE score (1=2).', self::DATASET_ID); - $into = (new SpssCompiler())->compile('RECODE score (1=2) INTO band.', self::DATASET_ID); - /** @var RecodeOperation $inPlaceRecode */ - $inPlaceRecode = $inPlace->operations()[0]; - /** @var RecodeOperation $intoRecode */ - $intoRecode = $into->operations()[0]; - - self::assertInstanceOf(CopySourceAction::class, $inPlaceRecode->rules()[1]->action()); - self::assertInstanceOf(SetMissingAction::class, $intoRecode->rules()[1]->action()); - self::assertInstanceOf(ElseSelector::class, $intoRecode->rules()[1]->selector()); + try { + SpssFrontendRequest::fromArray($payload); + self::fail('An invalid frontend request unexpectedly passed validation.'); + } catch (TransformationFailure $failure) { + self::assertSame('plan_schema_invalid', $failure->diagnosticCode()); + self::assertSame($path, $failure->diagnostics[0]->path); + } } - public function testExpandsGroupedRecodeValuesWithoutDroppingSelectors(): void + /** @return iterable, string}> */ + public static function invalidRequestProvider(): iterable { - $plan = (new SpssCompiler())->compile('RECODE score (1,2=0).', self::DATASET_ID); - $operation = $plan->operations()[0]; - self::assertInstanceOf(RecodeOperation::class, $operation); - self::assertCount(3, $operation->rules()); - - $first = $operation->rules()[0]->selector(); - $second = $operation->rules()[1]->selector(); - self::assertInstanceOf(\OpenStatSpec\Transformation\Model\Selector\ExactValueSelector::class, $first); - self::assertInstanceOf(\OpenStatSpec\Transformation\Model\Selector\ExactValueSelector::class, $second); - self::assertSame(1.0, $first->value()->numberValue()); - self::assertSame(2.0, $second->value()->numberValue()); - self::assertInstanceOf(ElseSelector::class, $operation->rules()[2]->selector()); + $valid = self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'numeric']]); + + yield 'extra root member' => [$valid + ['extra' => true], '$']; + yield 'wrong contract' => [[...$valid, 'contract' => 'openstatspec-spss-syntax-frontend-v0.1'], '$.contract']; + yield 'empty alias' => [[...$valid, 'input_alias' => ''], '$.input_alias']; + yield 'empty source' => [[...$valid, 'source_text' => ''], '$.source_text']; + yield 'empty variables' => [[...$valid, 'input_schema' => ['variables' => []]], '$.input_schema.variables']; + yield 'extra schema member' => [[...$valid, 'input_schema' => ['variables' => [['name' => 'q1', 'storage_kind' => 'numeric']], 'extra' => null]], '$.input_schema']; + yield 'empty variable name' => [self::payload('EXECUTE.', [['name' => '', 'storage_kind' => 'numeric']]), '$.input_schema.variables[0].name']; + yield 'invalid storage kind' => [self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'date']]), '$.input_schema.variables[0].storage_kind']; + yield 'extra variable member' => [self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'numeric', 'extra' => true]]), '$.input_schema.variables[0]']; + yield 'invalid value label bits' => [self::payload('EXECUTE.', [[ + 'name' => 'q1', + 'storage_kind' => 'numeric', + 'value_labels' => [['value' => ['type' => 'binary64', 'bits' => '8000000000000000'], 'label' => 'bad']], + ]]), '$.input_schema.variables[0].value_labels[0].value.bits']; + yield 'zero format width' => [self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'numeric', 'width' => 0]]), '$.input_schema.variables[0].width']; + yield 'invalid level' => [self::payload('EXECUTE.', [['name' => 'q1', 'storage_kind' => 'numeric', 'measurement_level' => 'interval']]), '$.input_schema.variables[0].measurement_level']; } - public function testExpandsParallelSourceAndIntoLists(): void + public function testBindingUsesExactCatalogSpellingAndEvolvingCaseInsensitiveSchema(): void { - $plan = (new SpssCompiler())->compile('RECODE first second (1=2) INTO new_first new_second.', self::DATASET_ID); + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + "COMPUTE Temp = source.\nIF (TEMP = 1) temp = SOURCE.\nFORMATS tEmP (F8.2).", + [['name' => 'Source', 'storage_kind' => 'numeric']], + ))); - self::assertCount(2, $plan->operations()); - self::assertSame('first', $plan->operations()[0]->sourceVariable()); - self::assertSame('new_first', $plan->operations()[0]->targetVariable()); - self::assertSame('second', $plan->operations()[1]->sourceVariable()); - self::assertSame('new_second', $plan->operations()[1]->targetVariable()); + self::assertSame(PlanContract::V02, $result->plan->contract); + self::assertSame([ + ['op' => 'assign', 'target' => 'Temp', 'target_mode' => 'create', 'value' => ['kind' => 'variable', 'variable' => 'Source']], + [ + 'op' => 'conditional_assign', + 'condition' => [ + 'expression' => 'comparison', + 'left' => ['kind' => 'variable', 'variable' => 'Temp'], + 'operator' => '=', + 'right' => ['kind' => 'literal', 'value' => ['type' => 'binary64', 'bits' => '3ff0000000000000']], + ], + 'target' => 'Temp', + 'value' => ['kind' => 'variable', 'variable' => 'Source'], + ], + ['op' => 'set_format', 'variable' => 'Temp', 'family' => 'F', 'width' => 8, 'decimals' => 2], + ], $result->plan->canonicalArray()['operations']); } - public function testPreservesSequentialOperationsOnTheSameTarget(): void + public function testParallelRecodeReadsPrecommandSchemaAndPublishesTargetsAfterTheCommand(): void { - $plan = (new SpssCompiler())->compile(<<<'SPSS' - RECODE score (1=2). - RECODE score (2=3). - VARIABLE LABELS score 'First label'. - VARIABLE LABELS score 'Replacement label'. - SPSS, self::DATASET_ID); - - self::assertCount(4, $plan->operations()); + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + "RECODE FIRST second (1 = 9) INTO First_New Second_New.\nVARIABLE LABELS first_new 'Created'.", + [ + ['name' => 'First', 'storage_kind' => 'numeric'], + ['name' => 'Second', 'storage_kind' => 'numeric'], + ], + ))); + + self::assertSame(PlanContract::V01, $result->plan->contract); self::assertSame( - ['recode', 'recode', 'set_variable_label', 'set_variable_label'], - array_map(static fn($operation): string => $operation->type(), $plan->operations()), + [['First', 'First_New'], ['Second', 'Second_New']], + array_map( + static fn(array $operation): array => [$operation['source'], $operation['target']], + array_slice($result->plan->canonicalArray()['operations'], 0, 2), + ), ); + self::assertSame('First_New', $result->plan->canonicalArray()['operations'][2]['variable']); } - public function testRejectsParallelRecodeThatWouldOverwriteALaterSource(): void + public function testRecodeAndValueLabelsUseExactDecimalBinary64Bits(): void { - try { - (new SpssCompiler())->compile( - 'RECODE a b (ELSE=COPY) INTO b c.', - self::DATASET_ID, - ); - self::fail('A dependency-unsafe parallel RECODE unexpectedly compiled.'); - } catch (SpssSyntaxException $exception) { - self::assertStringContainsString( - 'overwrite a source used by a later pair', - $exception->diagnostics[0]->message, - ); - } + $token = '1.00000000000000033306690738754696212708950042724609375'; + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + "RECODE q1 ($token = -0).\nVALUE LABELS q1 $token 'Rounded'.", + [['name' => 'q1', 'storage_kind' => 'numeric']], + ))); + $operations = $result->plan->canonicalArray()['operations']; + + self::assertSame('3ff0000000000002', $operations[0]['rules'][0]['match']['values'][0]['bits']); + self::assertSame('0000000000000000', $operations[0]['rules'][0]['result']['value']['bits']); + self::assertSame('3ff0000000000002', $operations[1]['labels'][0]['value']['bits']); } - public function testRejectsDuplicateIntoTargetsWithinOneParallelRecode(): void + public function testRecodeWithOnlyElseCannotEmitAnInvalidEmptyRuleList(): void { try { - (new SpssCompiler())->compile( - 'RECODE first second (1=2) INTO result result.', - self::DATASET_ID, - ); - self::fail('A parallel RECODE with duplicate INTO targets unexpectedly compiled.'); - } catch (SpssSyntaxException $exception) { - self::assertStringContainsString( - 'duplicate target variables', - $exception->diagnostics[0]->message, - ); + (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + 'RECODE q1 (ELSE = 0).', + [['name' => 'q1', 'storage_kind' => 'numeric']], + ))); + self::fail('An ELSE-only RECODE unexpectedly emitted an invalid plan.'); + } catch (TransformationFailure $failure) { + self::assertSame('spss_syntax_error', $failure->diagnosticCode()); + self::assertSame([0, 20], [ + $failure->diagnostics[0]->span?->startOffset, + $failure->diagnostics[0]->span?->endOffset, + ]); } } - public function testBinderRejectsInvalidElsePositionAndIntoArity(): void + public function testLegacySchemaCommandsFailAsUnsupportedAtTheExactCommandToken(): void { - foreach ([ - 'RECODE score (ELSE=COPY) (1=2).', - 'RECODE first second (1=2) INTO only_one.', - "RECODE score ('a' THRU 'z'=1).", - ] as $syntax) { - $this->expectCompileFailure($syntax); + foreach (['STRING note (A4).', 'DELETE VARIABLES q1.'] as $source) { + try { + (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + $source, + [['name' => 'q1', 'storage_kind' => 'numeric']], + ))); + self::fail('A non-Frontend-0.2 schema command unexpectedly compiled.'); + } catch (TransformationFailure $failure) { + self::assertSame('unsupported_spss_command', $failure->diagnosticCode()); + self::assertSame([0, 6], [ + $failure->diagnostics[0]->span?->startOffset, + $failure->diagnostics[0]->span?->endOffset, + ]); + } } } - public function testFailsClosedForUserMissingSelectorWithoutDatasetMetadata(): void - { + /** @param list> $variables */ + #[DataProvider('bindingFailureProvider')] + public function testBindingFailuresHaveStableCodesAndExactSourceSpans( + string $source, + array $variables, + string $code, + int $startOffset, + int $endOffset, + ): void { try { - (new SpssCompiler())->compile('RECODE score (MISSING=0) (ELSE=COPY).', self::DATASET_ID); - self::fail('MISSING unexpectedly compiled as system missing.'); - } catch (SpssSyntaxException $exception) { - self::assertStringContainsString('user-missing', $exception->diagnostics[0]->message); + (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload($source, $variables))); + self::fail('Invalid frontend semantics unexpectedly compiled.'); + } catch (TransformationFailure $failure) { + self::assertSame($code, $failure->diagnosticCode()); + $span = $failure->diagnostics[0]->span; + self::assertNotNull($span); + self::assertSame([$startOffset, $endOffset], [$span->startOffset, $span->endOffset]); + self::assertSame([1, $startOffset + 1, 1, $endOffset + 1], [ + $span->startLine, + $span->startColumn, + $span->endLine, + $span->endColumn, + ]); } } - public function testLargeIntegerLiteralIsParsedDirectlyAsBinary64(): void + /** @return iterable>, string, int, int}> */ + public static function bindingFailureProvider(): iterable { - $plan = (new SpssCompiler())->compile( - 'RECODE score (999999999999999999999=1) (ELSE=COPY).', - self::DATASET_ID, - ); - - /** @var RecodeOperation $recode */ - $recode = $plan->operations()[0]; - /** @var \OpenStatSpec\Transformation\Model\Selector\ExactValueSelector $selector */ - $selector = $recode->rules()[0]->selector(); - self::assertInstanceOf( - \OpenStatSpec\Transformation\Model\Selector\ExactValueSelector::class, - $selector, - ); - self::assertSame((float) '999999999999999999999', $selector->value()->numberValue()); - self::assertNotSame((float) PHP_INT_MAX, $selector->value()->numberValue()); + yield 'unknown expression variable' => [ + 'COMPUTE target = missing.', + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'unknown_variable', + 17, + 24, + ]; + yield 'missing conditional target' => [ + 'IF (source = 1) missing = 1.', + [['name' => 'source', 'storage_kind' => 'numeric']], + 'conditional_target_missing', + 16, + 23, + ]; + yield 'reserved compute target' => [ + 'COMPUTE __hidden = 1.', + [['name' => 'source', 'storage_kind' => 'numeric']], + 'reserved_target_name', + 8, + 16, + ]; + yield 'string compute expression' => [ + 'COMPUTE target = color.', + [['name' => 'color', 'storage_kind' => 'string']], + 'expression_type_unsupported', + 17, + 22, + ]; + yield 'string compute target' => [ + 'COMPUTE color = 1.', + [['name' => 'color', 'storage_kind' => 'string']], + 'expression_type_unsupported', + 8, + 13, + ]; + yield 'value label type mismatch' => [ + "VALUE LABELS q1 'R' 'Red'.", + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'type_mismatch', + 13, + 25, + ]; + yield 'invalid format bounds' => [ + 'FORMATS q1 (F2.2).', + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'invalid_format', + 8, + 17, + ]; + yield 'duplicate value label' => [ + "VALUE LABELS q1 -0 'Minus' 0 'Plus'.", + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'duplicate_value_label', + 13, + 35, + ]; + yield 'system missing string recode' => [ + "RECODE color (SYSMIS = 'x').", + [['name' => 'color', 'storage_kind' => 'string']], + 'system_missing_for_string', + 14, + 20, + ]; + yield 'string target needs declaration' => [ + "RECODE color ('R' = 'red') INTO normalized.", + [['name' => 'color', 'storage_kind' => 'string']], + 'string_target_requires_declaration', + 32, + 42, + ]; + yield 'mixed recode results' => [ + "RECODE q1 (1 = 'one') (ELSE = 0).", + [['name' => 'q1', 'storage_kind' => 'numeric']], + 'mixed_result_types', + 0, + 32, + ]; } - public function testRejectsNonFiniteNumericLiteral(): void + /** + * @param list> $variables + * @return array + */ + private static function payload(string $source, array $variables): array { - $this->expectCompileFailure('RECODE score (1e999=1) (ELSE=COPY).'); - } - - private function expectCompileFailure(string $syntax): void - { - try { - (new SpssCompiler())->compile($syntax, self::DATASET_ID); - self::fail('Semantically invalid syntax unexpectedly compiled.'); - } catch (SpssSyntaxException $exception) { - self::assertNotEmpty($exception->diagnostics); - } + return [ + 'contract' => 'openstatspec-spss-syntax-frontend-v0.2', + 'input_alias' => 'parent', + 'input_schema' => ['variables' => $variables], + 'source_text' => $source, + ]; } } From 26fad1bb7219c7409972d30ddc998fcd5d205a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 17:00:37 +0300 Subject: [PATCH 11/53] fix: reject reserved existing SPSS targets --- .../task-6-report.md | 21 ++++++++++++++----- src/Frontend/Spss/Binder.php | 3 ++- .../Spss/Conformance/SpssFrontend02Test.php | 8 ++++++- tests/Frontend/Spss/SpssCompilerTest.php | 19 ++++++++++++++++- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md index 782e4c2..8565d6d 100644 --- a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-6-report.md @@ -2,7 +2,7 @@ ## Status -Complete. The change is committed as one Task 6 commit and was not pushed. +Complete in the initial Task 6 commit plus one review-fix commit. Neither was pushed. ## Implementation @@ -15,6 +15,7 @@ Complete. The change is committed as one Task 6 commit and was not pushed. - Added command, group, variable, expression, rule, value, result, and unsupported-command spans needed for stable binding diagnostics while preserving all source ordering. - Added manifest-driven coverage for all 44 official Frontend 0.2 cases, exact source/plan hashes, exact canonical plans, referenced official Frontend 0.1 golden plans, plan contracts, stable failure codes, and non-null source spans. - Added focused request, evolving-schema, simultaneous-recode, exact-binary64, diagnostic-code/span, invalid-empty-recode, and unsupported-legacy-command regressions. +- Review fix: existing reserved `__` COMPUTE and IF targets now fail in the binder with `reserved_target_name` at the exact target span, matching strict Plan validation; every successful official frontend plan is also decoded again through strict `PlanCodec` and compared canonically. ## TDD evidence @@ -36,11 +37,21 @@ vendor/bin/phpunit tests/Frontend/Spss/SpssCompilerTest.php Further tests independently reproduced missing direct-constructor value-label validation, invalid ELSE-only RECODE output, loss of exact numeric source tokens, and zero-width unsupported-command spans before each corresponding fix. +The review regressions were added before the binder fix: + +```text +vendor/bin/phpunit tests/Frontend/Spss/SpssCompilerTest.php --filter 'reserved existing' +FAILURES! Tests: 2, Assertions: 2, Failures: 2. +Both existing reserved targets unexpectedly compiled. +``` + +The root cause was that `validateTargetName()` ran only on the missing-target COMPUTE branch and never on IF; both existing-target paths could therefore emit a plan rejected by strict `PlanCodec`. + ### GREEN ```text vendor/bin/phpunit tests/Frontend/Spss/Conformance tests/Frontend/Spss/SpssCompilerTest.php -OK (74 tests, 269 assertions) +OK (76 tests, 301 assertions) ``` The 44 manifest cases are included in that count. @@ -51,7 +62,7 @@ The 44 manifest cases are included in that count. - **Source identity:** hashing is isolated from parsing and changes only line-ending bytes. - **Resolution and ordering:** matching is ASCII-case-insensitive, emitted names preserve schema spelling, ambiguous names fail closed, new names retain source spelling, commands/groups/variables/rules/boolean operands remain ordered, and parallel RECODE targets are published only after the whole command binds. - **Type and target semantics:** COMPUTE creates or replaces numeric targets; IF requires an existing numeric target; numeric-only expressions/formats, value-label types, ranges, result homogeneity, reserved names, fresh targets, and string limitations fail at the narrowest available source span. -- **Plan boundary:** emitted operations are official immutable Plan types; `PlanCodec` produces each official hash; unchanged subsets stay Plan 0.1 and any 0.2 operation promotes the whole ordered plan to 0.2. +- **Plan boundary:** emitted operations are official immutable Plan types; every successful official plan round-trips through strict `PlanCodec`; unchanged subsets stay Plan 0.1 and any 0.2 operation promotes the whole ordered plan to 0.2. - **Failure behavior:** request, syntax, binding, and unsupported-command failures occur before mutation and carry the stable code/path/span contract. - **Dolt/global constraints:** Task 6 changes only the pure frontend and tests. It adds no SQL mutation, dataset/table copy, derived dataset, staging/snapshot/rollback artifact, audit/version row, or Dolt operation; Dolt remains the sole version-history layer. - **Scope:** the staged diff contains only Task 6 frontend source/tests and this report; unrelated changes were not included. @@ -60,7 +71,7 @@ The 44 manifest cases are included in that count. ```text vendor/bin/phpunit tests/Transformation/Conformance tests/Frontend/Spss -OK (179 tests, 697 assertions) +OK (181 tests, 729 assertions) ``` ```text @@ -75,7 +86,7 @@ composer validate --strict: valid PHP lint: all source/test/tool files valid php-cs-fixer: Found 0 of 213 files that can be fixed PHPStan: [OK] No errors -PHPUnit: 406 tests, 2361 assertions, 30 skipped; exit 0 +PHPUnit: 408 tests, 2393 assertions, 30 skipped; exit 0 ``` `git diff --cached --check` also passed. diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php index 9904169..170a90a 100644 --- a/src/Frontend/Spss/Binder.php +++ b/src/Frontend/Spss/Binder.php @@ -170,6 +170,7 @@ private function compute(ComputeStatement $statement, SchemaState $schema): Boun } $target = $schema->find($statement->target, $statement->targetSpan); + $this->validateTargetName($statement->target, $statement->targetSpan); if ($target !== null) { if ($target->storageKind !== 'numeric') { $this->fail( @@ -181,7 +182,6 @@ private function compute(ComputeStatement $statement, SchemaState $schema): Boun return new BoundCompute($target->name, TargetMode::Replace, $value, $statement->span); } - $this->validateTargetName($statement->target, $statement->targetSpan); $schema->addNumeric($statement->target); return new BoundCompute($statement->target, TargetMode::Create, $value, $statement->span); @@ -195,6 +195,7 @@ private function conditionalAssign(IfStatement $statement, SchemaState $schema): $statement->targetSpan, 'conditional_target_missing', ); + $this->validateTargetName($statement->target, $statement->targetSpan); if ($target->storageKind !== 'numeric') { $this->fail( 'expression_type_unsupported', diff --git a/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php b/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php index d31d8ea..03ec1a0 100644 --- a/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php +++ b/tests/Frontend/Spss/Conformance/SpssFrontend02Test.php @@ -34,8 +34,14 @@ public function testOfficialFrontend02Case(array $case): void self::assertNull($case['expected_error'], (string) $case['id']); self::assertSame($case['expected_source_hash'], $result->sourceHash, (string) $case['id']); - self::assertSame($case['expected_plan_hash'], (new PlanCodec())->hash($result->plan), (string) $case['id']); + $codec = new PlanCodec(); + self::assertSame($case['expected_plan_hash'], $codec->hash($result->plan), (string) $case['id']); self::assertSame($this->expectedPlan($case), $result->plan->canonicalArray(), (string) $case['id']); + self::assertSame( + $result->plan->canonicalArray(), + $codec->fromJson($codec->canonicalJson($result->plan))->canonicalArray(), + (string) $case['id'], + ); if (isset($case['expected_plan_contract'])) { self::assertSame($case['expected_plan_contract'], $result->plan->contract->value, (string) $case['id']); diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index 5189d51..73f0737 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -234,13 +234,30 @@ public static function bindingFailureProvider(): iterable 16, 23, ]; - yield 'reserved compute target' => [ + yield 'reserved new compute target' => [ 'COMPUTE __hidden = 1.', [['name' => 'source', 'storage_kind' => 'numeric']], 'reserved_target_name', 8, 16, ]; + yield 'reserved existing compute target' => [ + 'COMPUTE __hidden = 1.', + [['name' => '__Hidden', 'storage_kind' => 'numeric']], + 'reserved_target_name', + 8, + 16, + ]; + yield 'reserved existing conditional target' => [ + 'IF (source = 1) __hidden = 1.', + [ + ['name' => 'source', 'storage_kind' => 'numeric'], + ['name' => '__Hidden', 'storage_kind' => 'numeric'], + ], + 'reserved_target_name', + 16, + 24, + ]; yield 'string compute expression' => [ 'COMPUTE target = color.', [['name' => 'color', 'storage_kind' => 'string']], From da006f6cf9c431bbfb550942027196f986afae97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 17:24:34 +0300 Subject: [PATCH 12/53] feat: add official transformation apply audit --- .../task-7-report.md | 33 ++ src/Spss/SpssAdapter.php | 2 + src/Sql/CatalogOwnership.php | 18 +- src/Sql/NormativeCatalog.php | 29 +- .../Audit/TransformationAuditMigrator.php | 300 ++++++++++++++ .../Audit/TransformationAuditWriter.php | 145 +++++++ .../Execution/InPlaceApplyRequest.php | 47 +++ .../ServerCatalogMigrationTest.php | 89 ++++ tests/Spss/SpssAdapterTest.php | 10 +- tests/Sql/CatalogOwnershipTest.php | 6 +- .../Audit/TransformationAuditMigratorTest.php | 386 ++++++++++++++++++ 11 files changed, 1045 insertions(+), 20 deletions(-) create mode 100644 .superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md create mode 100644 src/Transformation/Audit/TransformationAuditMigrator.php create mode 100644 src/Transformation/Audit/TransformationAuditWriter.php create mode 100644 src/Transformation/Execution/InPlaceApplyRequest.php create mode 100644 tests/Transformation/Audit/TransformationAuditMigratorTest.php diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md new file mode 100644 index 0000000..6215bb5 --- /dev/null +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md @@ -0,0 +1,33 @@ +# Task 7 report: explicit transformation audit migration and apply request + +## Implementation + +- Added `InPlaceApplyRequest` with fail-fast alias, canonical UUID, lowercase SHA-256, non-empty actor, and paired Dolt-context validation. +- Added explicit schema-v4 `TransformationAuditMigrator` for SQLite, PostgreSQL, MySQL, MariaDB, and Dolt. SQLite rebuilds the one logical audit table transactionally; server profiles replace only the contract check before apply. +- Added success-only `TransformationAuditWriter`, which requires the caller's open apply transaction and stores canonical plan identity, source identity, actor, target identity, operation count, timestamps, and nullable Dolt evidence. +- Wired `SpssAdapter::migrateCatalog()` to run the audit migration after normative table creation and before marking schema version 4. +- Preserved official 0.1 audit rows while allowing both 0.1 and 0.2 binding contracts. No dataset/data table, snapshot, staging, rollback, or recovery relation was added. + +## TDD evidence + +- RED: `vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php` reported 6 missing-class errors across 7 tests. +- GREEN: the final focused command `vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php tests/Integration/ServerCatalogMigrationTest.php` passed 10 tests and 54 assertions; the only skip was the unconfigured server integration profile. +- Wiring mutation check: removing the adapter's explicit migrator call made `testAdapterMigrationRecordsFreshVersionsOneThroughFourExactlyOnce` fail because marker 4 was absent; restoring the call returned the suite to green. + +## Verification + +- `composer analyse`: `[OK] No errors`. +- Working-tree `composer style`: exits 8 because 133 pre-existing CRLF files are reported as fixable. +- CRLF-normalized staged-tree `composer check`: Composer validation, PHP lint, style (`0 of 217`), PHPStan, and PHPUnit all passed; PHPUnit ran 417 tests with 2,447 assertions and 30 environment skips. +- `git diff --cached --check`: passed. + +## Self-review + +- SQLite v0.1 migration preserves the row and logical table name, verifies foreign keys before commit, leaves no `_v04` residue, and rolls back to the old table on copy failure. +- PostgreSQL changes the old contract check in one native transaction; MySQL-family/Dolt DDL is explicit, idempotent, and never invoked by the audit writer or inside an apply transaction. +- The writer has no failure-row API and performs no migration, dataset copy, table copy, Dolt commit, branch operation, or recovery/version write. +- Fresh catalogs record versions 1 through 4 exactly once; v3 ownership recognition remains backward compatible, while v4 marker recognition requires the compact audit relation. + +## Environment note + +No server DSNs were configured locally, so PostgreSQL/MySQL/MariaDB/Dolt integration cases were the only focused skip; their migration test now exercises preservation of a pre-existing 0.1 row when those services are configured. diff --git a/src/Spss/SpssAdapter.php b/src/Spss/SpssAdapter.php index aec05a5..c0b71e3 100644 --- a/src/Spss/SpssAdapter.php +++ b/src/Spss/SpssAdapter.php @@ -23,6 +23,7 @@ use OpenStatSpec\Sql\PostgreSqlSchema; use OpenStatSpec\Sql\SqliteWideTableExporter; use OpenStatSpec\Sql\SqliteWideTableImporter; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use PDO; use Throwable; @@ -67,6 +68,7 @@ public function migrateCatalog(): void }; $catalog = new NormativeCatalog($this->connection->pdo); $catalog->createTables(); + (new TransformationAuditMigrator($this->connection->pdo))->migrate(); $this->backfillLegacyDatasets($catalog); CatalogOwnership::markCurrentVersion($this->connection->pdo); } diff --git a/src/Sql/CatalogOwnership.php b/src/Sql/CatalogOwnership.php index 3119e45..17599d4 100644 --- a/src/Sql/CatalogOwnership.php +++ b/src/Sql/CatalogOwnership.php @@ -15,7 +15,7 @@ final class CatalogOwnership private const IDENTITY_TABLE = 'catalog_identity'; private const CONTRACT_ID = 'openstatspec-strict-wide-table-v1'; private const MIGRATION_TABLE = 'openstatspec_schema_migration'; - private const SCHEMA_VERSION = 3; + private const SCHEMA_VERSION = 4; /** @return array */ public static function binding(PDO $pdo): array @@ -235,15 +235,24 @@ private static function isRecognizedMarkerCatalog(PDO $pdo, array $collisions, i $canonicalComplete = $canonical !== [] && self::matchesCompleteDefinition($pdo, $canonical, $canonicalDefinition); $legacyComplete = $legacy !== [] && self::matchesCompleteDefinition($pdo, $legacy, $legacyDefinition); $journalsComplete = $journals === [] || self::isRecognizedJournalOnlyLegacyCatalog($pdo, $journals); + $auditPresent = in_array('transformation_apply', $collisions, true); + $auditComplete = !$auditPresent || self::canSelectColumns($pdo, 'transformation_apply', [ + 'apply_id', 'contract_id', 'database_profile', 'dataset_id', 'physical_table_schema', + 'physical_table_name', 'source_hash', 'plan_hash', 'canonical_plan_json', 'actor', + 'status', 'dolt_branch', 'dolt_head_before', 'dolt_head_after', 'operation_count', + 'started_at', 'completed_at', + ]); if (($canonical !== [] && !$canonicalComplete) || ($legacy !== [] && !$legacyComplete) || !$journalsComplete + || !$auditComplete ) { return false; } - return $markerVersion === self::SCHEMA_VERSION - ? $canonicalComplete - : ($canonicalComplete || $legacyComplete); + if ($markerVersion >= 3) { + return $canonicalComplete && ($markerVersion < self::SCHEMA_VERSION || $auditPresent); + } + return $canonicalComplete || $legacyComplete; } /** @param list $collisions */ @@ -707,6 +716,7 @@ private static function catalogTables(): array 'variable_value_label_set', 'missing_rule', 'dataset_attribute', 'variable_attribute', 'document', 'variable_set', 'variable_set_member', 'multiple_response_set', 'multiple_response_member', 'operation', 'fidelity_event', 'datasets', 'variables', + 'transformation_apply', 'dataset_weight_variables', 'dataset_metadata', 'file_technical_metadata', 'documents', 'value_labels', 'missing_rules', 'missing_rule_values', 'variable_display_metadata', 'variable_roles', 'file_attributes', 'variable_attributes', 'variable_sets', diff --git a/src/Sql/NormativeCatalog.php b/src/Sql/NormativeCatalog.php index 6fdd8ed..4345013 100644 --- a/src/Sql/NormativeCatalog.php +++ b/src/Sql/NormativeCatalog.php @@ -65,21 +65,34 @@ public function createTables(): void $this->enforceSetOrdinalConstraints($driver); } - $migration = match ($driver) { - 'mysql' => 'INSERT IGNORE INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?)', - 'pgsql', 'sqlite' => 'INSERT INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?) ON CONFLICT (version) DO NOTHING', - default => null, - }; - if ($migration !== null) { + if (in_array($driver, ['mysql', 'pgsql', 'sqlite'], true)) { foreach ([1, 2] as $version) { - $this->pdo->prepare($migration)->execute([$version, self::timestamp()]); + $this->recordMigration($version); } if ($migrationV3Required) { - $this->pdo->prepare($migration)->execute([3, self::timestamp()]); + $this->recordMigration(3); } } } + /** Record one completed explicit catalogue migration idempotently. */ + public function recordMigration(int $version): void + { + if ($version < 1) { + throw new \InvalidArgumentException('A catalogue migration version must be positive.'); + } + $driver = (string) $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME); + $migration = match ($driver) { + 'mysql' => 'INSERT IGNORE INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?)', + 'pgsql', 'sqlite' => 'INSERT INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?) ON CONFLICT (version) DO NOTHING', + default => throw new UnsupportedOperation( + DiagnosticCode::UnsupportedSqlDriver, + 'The active SQL profile cannot record catalogue migrations.', + ), + }; + $this->statement($migration)->execute([$version, self::timestamp()]); + } + private function migrationApplied(int $version): bool { $statement = $this->statement('SELECT 1 FROM openstatspec_schema_migration WHERE version = ?'); diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php new file mode 100644 index 0000000..d4b4dca --- /dev/null +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -0,0 +1,300 @@ +pdo->inTransaction()) { + throw new \LogicException('Transformation audit migration cannot run inside an apply transaction.'); + } + + match ((string) $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME)) { + 'sqlite' => $this->migrateSqlite(), + 'pgsql' => $this->migratePostgreSql(), + 'mysql' => $this->migrateMySqlFamily(), + default => throw new UnsupportedOperation( + DiagnosticCode::UnsupportedSqlDriver, + 'The active SQL profile has no transformation audit migration.', + ), + }; + } + + private function migrateSqlite(): void + { + $this->pdo->beginTransaction(); + try { + if (!$this->tableExists('sqlite')) { + $this->pdo->exec($this->createTableSql(self::TABLE, 'sqlite')); + } elseif (!$this->sqliteAcceptsVersion02()) { + $this->assertReadableAuditTable(); + $this->pdo->exec($this->createTableSql(self::SQLITE_REPLACEMENT, 'sqlite')); + $columns = implode(', ', self::columns()); + $this->pdo->exec( + 'INSERT INTO ' . self::SQLITE_REPLACEMENT . ' (' . $columns . ') ' + . 'SELECT ' . $columns . ' FROM ' . self::TABLE, + ); + $this->pdo->exec('DROP TABLE ' . self::TABLE); + $this->pdo->exec('ALTER TABLE ' . self::SQLITE_REPLACEMENT . ' RENAME TO ' . self::TABLE); + } else { + $this->assertReadableAuditTable(); + } + + $violations = $this->pdo->query('PRAGMA foreign_key_check'); + if ($violations === false || $violations->fetchColumn() !== false) { + throw new PDOException('Transformation audit migration produced a foreign-key violation.'); + } + (new NormativeCatalog($this->pdo))->recordMigration(4); + $this->pdo->commit(); + } catch (Throwable $exception) { + if ($this->pdo->inTransaction()) { + $this->pdo->rollBack(); + } + throw $exception; + } + } + + private function migratePostgreSql(): void + { + $this->pdo->beginTransaction(); + try { + if (!$this->tableExists('pgsql')) { + $this->pdo->exec($this->createTableSql(self::TABLE, 'pgsql')); + } else { + $this->assertReadableAuditTable(); + $checks = $this->postgreSqlContractChecks(); + if (!$this->checksAreCurrent($checks)) { + if ($checks === []) { + throw new PDOException('The existing transformation audit has no contract check to migrate.'); + } + foreach (array_keys($checks) as $constraint) { + $this->pdo->exec('ALTER TABLE ' . self::TABLE . ' DROP CONSTRAINT ' . $this->quotePostgreSql($constraint)); + } + $this->pdo->exec( + 'ALTER TABLE ' . self::TABLE . ' ADD CONSTRAINT chk_transformation_apply_contract ' + . $this->contractCheck(), + ); + } + } + (new NormativeCatalog($this->pdo))->recordMigration(4); + $this->pdo->commit(); + } catch (Throwable $exception) { + if ($this->pdo->inTransaction()) { + $this->pdo->rollBack(); + } + throw $exception; + } + } + + private function migrateMySqlFamily(): void + { + if (!$this->tableExists('mysql')) { + $this->pdo->exec($this->createTableSql(self::TABLE, 'mysql')); + } else { + $this->assertReadableAuditTable(); + $checks = $this->mySqlContractChecks(); + if (!$this->checksAreCurrent($checks)) { + if ($checks === []) { + throw new PDOException('The existing transformation audit has no contract check to migrate.'); + } + $dropSyntax = $this->isMariaDb() ? ' DROP CONSTRAINT ' : ' DROP CHECK '; + foreach (array_keys($checks) as $constraint) { + $this->pdo->exec('ALTER TABLE ' . self::TABLE . $dropSyntax . $this->quoteMySql($constraint)); + } + $this->pdo->exec( + 'ALTER TABLE ' . self::TABLE . ' ADD CONSTRAINT chk_transformation_apply_contract ' + . $this->contractCheck(), + ); + } + } + (new NormativeCatalog($this->pdo))->recordMigration(4); + } + + private function createTableSql(string $table, string $driver): string + { + $applyUuid = $driver === 'pgsql' ? 'UUID' : 'VARCHAR(36)'; + $text = $driver === 'mysql' ? 'LONGTEXT' : 'TEXT'; + + return 'CREATE TABLE ' . $table . ' (' + . 'apply_id ' . $applyUuid . ' NOT NULL PRIMARY KEY, ' + . 'contract_id ' . $text . ' NOT NULL, ' + . 'database_profile ' . $text . ' NOT NULL, ' + . 'dataset_id VARCHAR(36) NOT NULL, ' + . 'physical_table_schema ' . $text . ' NULL, ' + . 'physical_table_name ' . $text . ' NOT NULL, ' + . 'source_hash CHAR(64) NOT NULL, ' + . 'plan_hash CHAR(64) NOT NULL, ' + . 'canonical_plan_json ' . $text . ' NOT NULL, ' + . 'actor ' . $text . ' NOT NULL, ' + . 'status ' . $text . ' NOT NULL, ' + . 'dolt_branch ' . $text . ' NULL, ' + . 'dolt_head_before ' . $text . ' NULL, ' + . 'dolt_head_after ' . $text . ' NULL, ' + . 'operation_count INTEGER NOT NULL, ' + . 'started_at TIMESTAMP NOT NULL, ' + . 'completed_at TIMESTAMP NOT NULL, ' + . 'CONSTRAINT fk_transformation_apply_dataset FOREIGN KEY (dataset_id) REFERENCES dataset(dataset_id), ' + . 'CONSTRAINT chk_transformation_apply_contract ' . $this->contractCheck() . ', ' + . "CONSTRAINT chk_transformation_apply_profile CHECK (database_profile IN ('sqlite', 'postgresql', 'mysql', 'mariadb', 'dolt')), " + . "CONSTRAINT chk_transformation_apply_status CHECK (status IN ('succeeded', 'failed')), " + . 'CONSTRAINT chk_transformation_apply_operation_count CHECK (operation_count > 0), ' + . 'CONSTRAINT chk_transformation_apply_dolt CHECK (' + . "(database_profile <> 'dolt' AND dolt_branch IS NULL AND dolt_head_before IS NULL AND dolt_head_after IS NULL) " + . "OR (database_profile = 'dolt' AND dolt_branch IS NOT NULL AND dolt_head_before IS NOT NULL " + . "AND (status = 'failed' OR dolt_head_after = dolt_head_before))), " + . $this->hashChecks($driver) + . ')'; + } + + private function contractCheck(): string + { + return "CHECK (contract_id IN ('openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'))"; + } + + private function hashChecks(string $driver): string + { + if ($driver === 'sqlite') { + return "CONSTRAINT chk_transformation_apply_source_hash CHECK (length(source_hash) = 64 AND source_hash NOT GLOB '*[^0-9a-f]*'), " + . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*')"; + } + if ($driver === 'pgsql') { + return "CONSTRAINT chk_transformation_apply_source_hash CHECK (source_hash ~ '^[0-9a-f]{64}$'), " + . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (plan_hash ~ '^[0-9a-f]{64}$')"; + } + return "CONSTRAINT chk_transformation_apply_source_hash CHECK (source_hash REGEXP BINARY '^[0-9a-f]{64}$'), " + . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (plan_hash REGEXP BINARY '^[0-9a-f]{64}$')"; + } + + private function sqliteAcceptsVersion02(): bool + { + $statement = $this->pdo->query("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = '" . self::TABLE . "'"); + $sql = $statement === false ? false : $statement->fetchColumn(); + return is_string($sql) + && str_contains($sql, 'openstatspec-in-place-transformation-v0.1') + && str_contains($sql, 'openstatspec-in-place-transformation-v0.2'); + } + + /** @return array */ + private function postgreSqlContractChecks(): array + { + $statement = $this->pdo->query(<<<'SQL' +SELECT constraint_metadata.conname, pg_get_constraintdef(constraint_metadata.oid) +FROM pg_constraint constraint_metadata +JOIN pg_class relation ON relation.oid = constraint_metadata.conrelid +JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace +WHERE namespace.nspname = current_schema() + AND relation.relname = 'transformation_apply' + AND constraint_metadata.contype = 'c' + AND pg_get_constraintdef(constraint_metadata.oid) LIKE '%contract_id%' +SQL); + if ($statement === false) { + throw new PDOException('Could not inspect the PostgreSQL transformation audit checks.'); + } + /** @var array $checks */ + $checks = $statement->fetchAll(PDO::FETCH_KEY_PAIR); + return $checks; + } + + /** @return array */ + private function mySqlContractChecks(): array + { + $statement = $this->pdo->query(<<<'SQL' +SELECT table_constraint.constraint_name, check_constraint.check_clause +FROM information_schema.table_constraints table_constraint +JOIN information_schema.check_constraints check_constraint + ON check_constraint.constraint_schema = table_constraint.constraint_schema + AND check_constraint.constraint_name = table_constraint.constraint_name +WHERE table_constraint.constraint_schema = DATABASE() + AND table_constraint.table_name = 'transformation_apply' + AND table_constraint.constraint_type = 'CHECK' + AND check_constraint.check_clause LIKE '%contract_id%' +SQL); + if ($statement === false) { + throw new PDOException('Could not inspect the MySQL-family transformation audit checks.'); + } + /** @var array $checks */ + $checks = $statement->fetchAll(PDO::FETCH_KEY_PAIR); + return $checks; + } + + /** @param array $checks */ + private function checksAreCurrent(array $checks): bool + { + if ($checks === []) { + return false; + } + foreach ($checks as $definition) { + if (!str_contains($definition, 'openstatspec-in-place-transformation-v0.1') + || !str_contains($definition, 'openstatspec-in-place-transformation-v0.2') + ) { + return false; + } + } + return true; + } + + private function assertReadableAuditTable(): void + { + $statement = $this->pdo->query('SELECT ' . implode(', ', self::columns()) . ' FROM ' . self::TABLE . ' WHERE 1 = 0'); + if ($statement === false) { + throw new PDOException('The existing transformation audit table is not readable.'); + } + } + + private function tableExists(string $driver): bool + { + $sql = match ($driver) { + 'sqlite' => "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", + 'pgsql' => "SELECT 1 FROM information_schema.tables WHERE table_schema = current_schema() AND table_type = 'BASE TABLE' AND table_name = ?", + 'mysql' => "SELECT 1 FROM information_schema.tables WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE' AND table_name = ?", + default => throw new \LogicException('Unsupported audit migration driver.'), + }; + $statement = $this->pdo->prepare($sql); + $statement->execute([self::TABLE]); + return $statement->fetchColumn() !== false; + } + + private function isMariaDb(): bool + { + $statement = $this->pdo->query('SELECT VERSION()'); + $version = $statement === false ? false : $statement->fetchColumn(); + return is_string($version) && str_contains(strtolower($version), 'mariadb'); + } + + private function quotePostgreSql(string $identifier): string + { + return '"' . str_replace('"', '""', $identifier) . '"'; + } + + private function quoteMySql(string $identifier): string + { + return '`' . str_replace('`', '``', $identifier) . '`'; + } + + /** @return list */ + private static function columns(): array + { + return [ + 'apply_id', 'contract_id', 'database_profile', 'dataset_id', 'physical_table_schema', + 'physical_table_name', 'source_hash', 'plan_hash', 'canonical_plan_json', 'actor', + 'status', 'dolt_branch', 'dolt_head_before', 'dolt_head_after', 'operation_count', + 'started_at', 'completed_at', + ]; + } +} diff --git a/src/Transformation/Audit/TransformationAuditWriter.php b/src/Transformation/Audit/TransformationAuditWriter.php new file mode 100644 index 0000000..65b1e5c --- /dev/null +++ b/src/Transformation/Audit/TransformationAuditWriter.php @@ -0,0 +1,145 @@ +codec = $codec ?? new PlanCodec(); + } + + public function succeed( + InPlaceApplyRequest $request, + DatasetBinding $dataset, + string $databaseProfile, + ?DoltEvidence $before, + ?DoltEvidence $after, + ): string { + if (!$this->pdo->inTransaction()) { + throw new \LogicException('A successful transformation audit must be written inside the open apply transaction.'); + } + if (!in_array($databaseProfile, ['sqlite', 'postgresql', 'mysql', 'mariadb', 'dolt'], true)) { + throw new \InvalidArgumentException('Unsupported transformation audit database profile.'); + } + if ($dataset->datasetId !== $request->datasetId) { + throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply request and target binding identify different datasets.'); + } + + $this->assertDatasetIdentity($dataset); + [$branch, $headBefore, $headAfter] = $this->doltFields($request, $databaseProfile, $before, $after); + $applyId = self::uuidV4(); + $timestamp = gmdate('Y-m-d H:i:s'); + $contract = $request->plan->contract === PlanContract::V01 + ? 'openstatspec-in-place-transformation-v0.1' + : 'openstatspec-in-place-transformation-v0.2'; + + $statement = $this->pdo->prepare(<<<'SQL' +INSERT INTO transformation_apply ( + apply_id, contract_id, database_profile, dataset_id, physical_table_schema, + physical_table_name, source_hash, plan_hash, canonical_plan_json, actor, + status, dolt_branch, dolt_head_before, dolt_head_after, operation_count, + started_at, completed_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +SQL); + if ($statement === false) { + throw new PDOException('The transformation audit insert could not be prepared.'); + } + $statement->execute([ + $applyId, + $contract, + $databaseProfile, + $dataset->datasetId, + $dataset->schema, + $dataset->table, + $request->sourceHash, + $this->codec->hash($request->plan), + $this->codec->canonicalJson($request->plan), + $request->actor, + 'succeeded', + $branch, + $headBefore, + $headAfter, + count($request->plan->operations), + $timestamp, + $timestamp, + ]); + + return $applyId; + } + + private function assertDatasetIdentity(DatasetBinding $dataset): void + { + $statement = $this->pdo->prepare( + 'SELECT physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', + ); + if ($statement === false) { + throw new PDOException('The audit target dataset identity could not be prepared.'); + } + $statement->execute([$dataset->datasetId]); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + if (count($rows) !== 1 + || ($rows[0]['physical_table_schema'] ?? null) !== $dataset->schema + || ($rows[0]['physical_table_name'] ?? null) !== $dataset->table + ) { + throw TransformationFailure::at( + 'invalid_dataset_id', + '$.dataset_id', + 'Apply target identity no longer matches the normative dataset row.', + ); + } + } + + /** @return array{string|null, string|null, string|null} */ + private function doltFields( + InPlaceApplyRequest $request, + string $databaseProfile, + ?DoltEvidence $before, + ?DoltEvidence $after, + ): array { + if ($databaseProfile !== 'dolt') { + if ($before !== null || $after !== null) { + throw new \InvalidArgumentException('Non-Dolt audit rows cannot contain Dolt evidence.'); + } + return [null, null, null]; + } + if ($request->expectedBranch === null + || $request->expectedHead === null + || $before === null + || $after === null + ) { + throw TransformationFailure::at('dolt_context_required', '$.expected_context', 'Dolt apply audit requires complete context evidence.'); + } + if ($before->branch() !== $request->expectedBranch + || $before->head() !== $request->expectedHead + || $after->branch() !== $before->branch() + || $after->head() !== $before->head() + ) { + throw TransformationFailure::at('dolt_context_changed', '$.expected_context', 'Dolt context changed before the success audit.'); + } + return [$before->branch(), $before->head(), $after->head()]; + } + + private static function uuidV4(): string + { + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80); + $hex = bin2hex($bytes); + return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-' . substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-' . substr($hex, 20); + } +} diff --git a/src/Transformation/Execution/InPlaceApplyRequest.php b/src/Transformation/Execution/InPlaceApplyRequest.php new file mode 100644 index 0000000..8c19ee0 --- /dev/null +++ b/src/Transformation/Execution/InPlaceApplyRequest.php @@ -0,0 +1,47 @@ +inputAlias) { + throw TransformationFailure::at('unknown_input_alias', '$.input_alias', 'Apply binding does not match the plan alias.'); + } + if (preg_match('/\A[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/D', $datasetId) !== 1) { + throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply dataset identity must be a canonical lowercase UUID.'); + } + if (preg_match('/\A[0-9a-f]{64}\z/D', $sourceHash) !== 1) { + throw TransformationFailure::at('invalid_source_hash', '$.source_hash', 'Apply source hash must be a lowercase SHA-256 value.'); + } + if (trim($actor) === '') { + throw TransformationFailure::at('actor_required', '$.actor', 'A non-empty actor is required.'); + } + + $branchPresent = $expectedBranch !== null && trim($expectedBranch) !== ''; + $headPresent = $expectedHead !== null && trim($expectedHead) !== ''; + if ($branchPresent !== $headPresent + || ($expectedBranch !== null && !$branchPresent) + || ($expectedHead !== null && !$headPresent) + ) { + throw TransformationFailure::at( + 'dolt_context_required', + '$.expected_context', + 'Expected Dolt branch and HEAD must be supplied together as non-empty values.', + ); + } + } +} diff --git a/tests/Integration/ServerCatalogMigrationTest.php b/tests/Integration/ServerCatalogMigrationTest.php index 11834d5..953f974 100644 --- a/tests/Integration/ServerCatalogMigrationTest.php +++ b/tests/Integration/ServerCatalogMigrationTest.php @@ -32,10 +32,13 @@ public function testLegacyV3ConstraintsAndSourceOrderMigrateOnConfiguredServers( $pdo->prepare('INSERT INTO multiple_response_sets (dataset_name, set_ordinal, name, set_type, category_labels, label_source) VALUES (?, ?, ?, ?, ?, ?)')->execute([$datasetName, 1, '$Legacy MR', 'dichotomy', 'counted_values', 'set_label']); $pdo->prepare('INSERT INTO variable_set (variable_set_id, dataset_id, source_ordinal, set_name) VALUES (?, ?, NULL, ?)')->execute(['vs-' . $token, $datasetId, 'Legacy variable set']); $pdo->prepare('INSERT INTO multiple_response_set (multiple_response_set_id, dataset_id, source_ordinal, set_name, set_kind, category_label_behavior) VALUES (?, ?, NULL, ?, ?, ?)')->execute(['mr-' . $token, $datasetId, '$Legacy MR', 'MD', 'counted_values']); + $legacyApplyId = $this->makeVersion01Audit($pdo, $name, $datasetId, 'unused_' . $token); $adapter->migrateCatalog(); self::assertSame(2, (int) $this->scalar($pdo, 'SELECT source_ordinal FROM variable_set WHERE dataset_id = ?', [$datasetId]), $name . ': variable-set source order'); self::assertSame(1, (int) $this->scalar($pdo, 'SELECT source_ordinal FROM multiple_response_set WHERE dataset_id = ?', [$datasetId]), $name . ': MR-set source order'); + self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply WHERE apply_id = ?', [$legacyApplyId]), $name . ': existing 0.1 audit row'); + $this->assertAuditContracts($pdo, $name, $datasetId, 'unused_' . $token); $this->assertNotNullAndUnique($pdo, $name); foreach ([ ['INSERT INTO variable_set (variable_set_id, dataset_id, source_ordinal, set_name) VALUES (?, ?, ?, ?)', ['vs-duplicate-' . $token, $datasetId, 2, 'Duplicate variable set']], @@ -50,7 +53,9 @@ public function testLegacyV3ConstraintsAndSourceOrderMigrateOnConfiguredServers( } $adapter->migrateCatalog(); self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM openstatspec_schema_migration WHERE version = 3', []), $name . ': migration marker must be idempotent'); + self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM openstatspec_schema_migration WHERE version = 4', []), $name . ': audit migration marker must be idempotent'); } finally { + $pdo->prepare('DELETE FROM transformation_apply WHERE dataset_id = ?')->execute([$datasetId]); $pdo->prepare('DELETE FROM multiple_response_set WHERE dataset_id = ?')->execute([$datasetId]); $pdo->prepare('DELETE FROM variable_set WHERE dataset_id = ?')->execute([$datasetId]); $pdo->prepare('DELETE FROM multiple_response_sets WHERE dataset_name = ?')->execute([$datasetName]); @@ -60,6 +65,90 @@ public function testLegacyV3ConstraintsAndSourceOrderMigrateOnConfiguredServers( } } + private function makeVersion01Audit(PDO $pdo, string $profile, string $datasetId, string $table): string + { + $driver = (string) $pdo->getAttribute(PDO::ATTR_DRIVER_NAME); + if ($driver === 'pgsql') { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT chk_transformation_apply_contract'); + } else { + $drop = $profile === 'mariadb' ? 'DROP CONSTRAINT' : 'DROP CHECK'; + $pdo->exec('ALTER TABLE transformation_apply ' . $drop . ' chk_transformation_apply_contract'); + } + $pdo->exec("ALTER TABLE transformation_apply ADD CONSTRAINT chk_transformation_apply_contract CHECK (contract_id = 'openstatspec-in-place-transformation-v0.1')"); + $pdo->exec('DELETE FROM openstatspec_schema_migration WHERE version = 4'); + + $applyId = $this->uuidV4(); + $dolt = $profile === 'dolt'; + $pdo->prepare('INSERT INTO transformation_apply VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')->execute([ + $applyId, + 'openstatspec-in-place-transformation-v0.1', + $profile, + $datasetId, + null, + $table, + str_repeat('c', 64), + str_repeat('d', 64), + '{}', + 'legacy-migration-test', + 'succeeded', + $dolt ? 'migration-test' : null, + $dolt ? 'migration-head' : null, + $dolt ? 'migration-head' : null, + 1, + '2026-08-16 00:00:00', + '2026-08-16 00:00:01', + ]); + return $applyId; + } + + private function assertAuditContracts(PDO $pdo, string $profile, string $datasetId, string $table): void + { + $insert = $pdo->prepare(<<<'SQL' +INSERT INTO transformation_apply ( + apply_id, contract_id, database_profile, dataset_id, physical_table_schema, + physical_table_name, source_hash, plan_hash, canonical_plan_json, actor, + status, dolt_branch, dolt_head_before, dolt_head_after, operation_count, + started_at, completed_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +SQL); + foreach (['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'] as $contract) { + $dolt = $profile === 'dolt'; + $insert->execute([ + $this->uuidV4(), + $contract, + $profile, + $datasetId, + null, + $table, + str_repeat('a', 64), + str_repeat('b', 64), + '{}', + 'migration-test', + 'succeeded', + $dolt ? 'migration-test' : null, + $dolt ? 'migration-head' : null, + $dolt ? 'migration-head' : null, + 1, + '2026-08-17 00:00:00', + '2026-08-17 00:00:01', + ]); + } + self::assertSame( + 2, + (int) $this->scalar($pdo, "SELECT COUNT(*) FROM transformation_apply WHERE dataset_id = ? AND actor = 'migration-test'", [$datasetId]), + $profile . ': both official audit contracts', + ); + } + + private function uuidV4(): string + { + $bytes = random_bytes(16); + $bytes[6] = chr((ord($bytes[6]) & 0x0f) | 0x40); + $bytes[8] = chr((ord($bytes[8]) & 0x3f) | 0x80); + $hex = bin2hex($bytes); + return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-' . substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-' . substr($hex, 20); + } + private function makeV3Legacy(PDO $pdo): void { $driver = (string) $pdo->getAttribute(PDO::ATTR_DRIVER_NAME); diff --git a/tests/Spss/SpssAdapterTest.php b/tests/Spss/SpssAdapterTest.php index 0237625..fed5365 100644 --- a/tests/Spss/SpssAdapterTest.php +++ b/tests/Spss/SpssAdapterTest.php @@ -62,7 +62,7 @@ public function testSuccessfulMigrationAdvancesOlderIdentityVersion(): void (new SpssAdapter($pdo, new FakeSpssEngine($this->fixture())))->migrateCatalog(); - self::assertSame([['schema_version' => 3]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); + self::assertSame([['schema_version' => 4]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); } public function testFailedMigrationDoesNotAdvanceOlderIdentityVersion(): void @@ -102,7 +102,7 @@ public function testImportRequiresExplicitMigrationForOlderIdentityWithoutJourna $result = $adapter->import('fixture.sav', 'Old catalog import'); self::assertSame(2, $result->caseCount); - self::assertSame([['schema_version' => 3]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); + self::assertSame([['schema_version' => 4]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); } public function testExportRequiresExplicitMigrationForOlderIdentityWithoutJournalMutation(): void @@ -127,7 +127,7 @@ public function testExportRequiresExplicitMigrationForOlderIdentityWithoutJourna $result = $adapter->export('Old catalog export', 'old-catalog.sav'); self::assertSame(2, $result->caseCount); - self::assertSame([['schema_version' => 3]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); + self::assertSame([['schema_version' => 4]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); } public function testReadOnlyFreshInitializationFailureDoesNotCreateCurrentIdentity(): void @@ -175,7 +175,7 @@ public function testTypedDatasetImportCreatesWideTableAndExportPreservesSupporte $import = $adapter->import('fixture.sav', 'Customer survey'); self::assertSame('Customer survey', $import->datasetName); - self::assertSame([['schema_version' => 3]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); + self::assertSame([['schema_version' => 4]], self::rows($pdo, 'SELECT schema_version FROM catalog_identity')); self::assertSame(2, $import->caseCount); self::assertSame([], $import->diagnostics); self::assertMatchesRegularExpression('/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/', $import->operationId); @@ -940,7 +940,7 @@ public function testMigrateCatalogBackfillsLegacyDatasetsAndIsIdempotent(): void self::rows($pdo, 'SELECT spec_version, source_format, physical_table_name, dataset_name, source_case_count FROM dataset'), ); self::assertSame( - [['version' => 1], ['version' => 2], ['version' => 3]], + [['version' => 1], ['version' => 2], ['version' => 3], ['version' => 4]], self::rows($pdo, 'SELECT version FROM openstatspec_schema_migration ORDER BY version'), ); self::assertSame( diff --git a/tests/Sql/CatalogOwnershipTest.php b/tests/Sql/CatalogOwnershipTest.php index 1a68d98..515a560 100644 --- a/tests/Sql/CatalogOwnershipTest.php +++ b/tests/Sql/CatalogOwnershipTest.php @@ -213,7 +213,7 @@ public function testRegisteredPhysicalDataTableIsAllowed(): void CatalogOwnership::ensure($pdo); - self::assertSame(3, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); + self::assertSame(4, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); } public function testOlderIdentityVersionIsAcceptedWithoutPrematureUpgrade(): void @@ -231,7 +231,7 @@ public function testFutureIdentityVersionFailsClosed(): void { $pdo = $this->sqlite(); CatalogOwnership::ensure($pdo); - $pdo->exec('UPDATE catalog_identity SET schema_version = 4'); + $pdo->exec('UPDATE catalog_identity SET schema_version = 5'); try { CatalogOwnership::ensure($pdo); @@ -240,7 +240,7 @@ public function testFutureIdentityVersionFailsClosed(): void self::assertSame(DiagnosticCode::CatalogNamespaceCollision, $exception->diagnosticCode); } - self::assertSame(4, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); + self::assertSame(5, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); } public function testForeignCatalogNameFailsBeforeOwnershipObjectsAreCreated(): void diff --git a/tests/Transformation/Audit/TransformationAuditMigratorTest.php b/tests/Transformation/Audit/TransformationAuditMigratorTest.php new file mode 100644 index 0000000..9f8d9e5 --- /dev/null +++ b/tests/Transformation/Audit/TransformationAuditMigratorTest.php @@ -0,0 +1,386 @@ +plan(), + inputAlias: 'parent', + datasetId: self::DATASET_ID, + sourceHash: str_repeat('a', 64), + actor: 'conformance-runner', + ); + + self::assertSame('parent', $request->inputAlias); + self::assertSame(self::DATASET_ID, $request->datasetId); + self::assertSame(str_repeat('a', 64), $request->sourceHash); + self::assertSame('conformance-runner', $request->actor); + self::assertNull($request->expectedBranch); + self::assertNull($request->expectedHead); + } + + public function testApplyRequestRejectsInvalidBindingsBeforeExecution(): void + { + $plan = $this->plan(); + $hash = str_repeat('a', 64); + foreach ([ + 'alias mismatch' => [ + 'unknown_input_alias', + static fn() => new InPlaceApplyRequest($plan, 'other', self::DATASET_ID, $hash, 'conformance-runner'), + ], + 'malformed dataset UUID' => [ + 'invalid_dataset_id', + static fn() => new InPlaceApplyRequest($plan, 'parent', 'not-a-uuid', $hash, 'conformance-runner'), + ], + 'malformed source hash' => [ + 'invalid_source_hash', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, str_repeat('A', 64), 'conformance-runner'), + ], + 'empty actor' => [ + 'actor_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, " \t"), + ], + 'branch without head' => [ + 'dolt_context_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, 'conformance-runner', 'feature/recode'), + ], + 'head without branch' => [ + 'dolt_context_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, 'conformance-runner', expectedHead: 'expected-head'), + ], + 'empty branch' => [ + 'dolt_context_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, 'conformance-runner', '', 'expected-head'), + ], + 'empty head' => [ + 'dolt_context_required', + static fn() => new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, $hash, 'conformance-runner', 'feature/recode', ''), + ], + ] as $description => [$expectedCode, $factory]) { + try { + $factory(); + self::fail($description . ' was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($expectedCode, $failure->diagnosticCode(), $description); + } + } + } + + public function testFreshSqliteMigrationCreatesTheExactCompactAuditTable(): void + { + $pdo = $this->sqlite(); + + (new TransformationAuditMigrator($pdo))->migrate(); + (new TransformationAuditMigrator($pdo))->migrate(); + + self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $this->acceptedContracts($pdo), + ); + self::assertSame([ + 'apply_id', + 'contract_id', + 'database_profile', + 'dataset_id', + 'physical_table_schema', + 'physical_table_name', + 'source_hash', + 'plan_hash', + 'canonical_plan_json', + 'actor', + 'status', + 'dolt_branch', + 'dolt_head_before', + 'dolt_head_after', + 'operation_count', + 'started_at', + 'completed_at', + ], $this->columnNames($pdo)); + self::assertSame([1, 2, 3, 4], $this->migrationVersions($pdo)); + self::assertSame( + ['transformation_apply'], + $this->query($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'transformation_apply%' ORDER BY name")->fetchAll(PDO::FETCH_COLUMN), + ); + } + + public function testAdapterMigrationRecordsFreshVersionsOneThroughFourExactlyOnce(): void + { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $adapter = new SpssAdapter($pdo); + + $adapter->migrateCatalog(); + $adapter->migrateCatalog(); + + self::assertSame([1, 2, 3, 4], $this->migrationVersions($pdo)); + self::assertSame(4, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); + self::assertSame( + ['transformation_apply'], + $this->query($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'transformation_apply'")->fetchAll(PDO::FETCH_COLUMN), + ); + } + + public function testSqliteMigrationPreservesAnExistingVersion01AuditRowWithoutResidue(): void + { + $pdo = $this->sqlite(); + $this->createVersion01AuditTable($pdo); + $this->insertVersion01AuditRow($pdo); + $before = $this->query($pdo, 'SELECT * FROM transformation_apply')->fetch(PDO::FETCH_ASSOC); + + (new TransformationAuditMigrator($pdo))->migrate(); + + self::assertSame($before, $this->query($pdo, 'SELECT * FROM transformation_apply')->fetch(PDO::FETCH_ASSOC)); + self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $this->acceptedContracts($pdo), + ); + self::assertSame( + ['transformation_apply'], + $this->query($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'transformation_apply%' ORDER BY name")->fetchAll(PDO::FETCH_COLUMN), + ); + self::assertSame([1, 2, 3, 4], $this->migrationVersions($pdo)); + } + + public function testFailedSqliteRebuildRollsBackToTheVersion01Table(): void + { + $pdo = $this->sqlite(false); + $this->createVersion01AuditTable($pdo); + $this->insertVersion01AuditRow($pdo, '22222222-2222-4222-8222-222222222222'); + $pdo->exec('PRAGMA foreign_keys = ON'); + + try { + (new TransformationAuditMigrator($pdo))->migrate(); + self::fail('Migration accepted an audit row whose dataset foreign key is missing.'); + } catch (\Throwable) { + // The unchanged v0.1 table below proves the native transaction rolled back. + } + + self::assertSame(['openstatspec-in-place-transformation-v0.1'], $this->acceptedContracts($pdo)); + self::assertSame(1, (int) $this->query($pdo, 'SELECT COUNT(*) FROM transformation_apply')->fetchColumn()); + self::assertSame( + ['transformation_apply'], + $this->query($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'transformation_apply%' ORDER BY name")->fetchAll(PDO::FETCH_COLUMN), + ); + self::assertSame([1, 2, 3], $this->migrationVersions($pdo)); + } + + public function testSuccessWriterAddsOneCompactRowInsideTheCallerTransaction(): void + { + $pdo = $this->sqlite(); + (new TransformationAuditMigrator($pdo))->migrate(); + $request = new InPlaceApplyRequest( + $this->plan(), + 'parent', + self::DATASET_ID, + str_repeat('a', 64), + 'conformance-runner', + ); + $dataset = new DatasetBinding(self::DATASET_ID, 'Survey', null, 'data_survey'); + + $pdo->beginTransaction(); + $applyId = (new TransformationAuditWriter($pdo))->succeed($request, $dataset, 'sqlite', null, null); + $pdo->commit(); + + $row = $this->query($pdo, 'SELECT * FROM transformation_apply')->fetch(PDO::FETCH_ASSOC); + self::assertIsArray($row); + self::assertSame($applyId, $row['apply_id']); + self::assertMatchesRegularExpression('/\A[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/D', $applyId); + self::assertSame('openstatspec-in-place-transformation-v0.1', $row['contract_id']); + self::assertSame('sqlite', $row['database_profile']); + self::assertSame(self::DATASET_ID, $row['dataset_id']); + self::assertNull($row['physical_table_schema']); + self::assertSame('data_survey', $row['physical_table_name']); + self::assertSame(str_repeat('a', 64), $row['source_hash']); + self::assertSame('73a4893b0bf20cd731786b6998526ba84affcdb39c034c561d1ad29548ac38fe', $row['plan_hash']); + self::assertSame('{"contract":"openstatspec-transformation-plan-v0.1","input_alias":"parent","operations":[{"label":"Score","op":"set_variable_label","variable":"score"}]}', $row['canonical_plan_json']); + self::assertSame('conformance-runner', $row['actor']); + self::assertSame('succeeded', $row['status']); + self::assertNull($row['dolt_branch']); + self::assertNull($row['dolt_head_before']); + self::assertNull($row['dolt_head_after']); + self::assertSame(1, (int) $row['operation_count']); + self::assertIsString($row['started_at']); + self::assertIsString($row['completed_at']); + self::assertSame(1, (int) $this->query($pdo, 'SELECT COUNT(*) FROM transformation_apply')->fetchColumn()); + } + + public function testSuccessWriterRequiresTheOpenApplyTransaction(): void + { + $pdo = $this->sqlite(); + (new TransformationAuditMigrator($pdo))->migrate(); + $request = new InPlaceApplyRequest( + $this->plan(), + 'parent', + self::DATASET_ID, + str_repeat('a', 64), + 'conformance-runner', + ); + + $this->expectException(\LogicException::class); + (new TransformationAuditWriter($pdo))->succeed( + $request, + new DatasetBinding(self::DATASET_ID, 'Survey', null, 'data_survey'), + 'sqlite', + null, + null, + ); + } + + public function testSuccessWriterUsesTheVersion02BindingContractForAVersion02Plan(): void + { + $pdo = $this->sqlite(); + (new TransformationAuditMigrator($pdo))->migrate(); + $request = new InPlaceApplyRequest( + new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]), + 'parent', + self::DATASET_ID, + str_repeat('a', 64), + 'conformance-runner', + ); + + $pdo->beginTransaction(); + (new TransformationAuditWriter($pdo))->succeed( + $request, + new DatasetBinding(self::DATASET_ID, 'Survey', null, 'data_survey'), + 'sqlite', + null, + null, + ); + $pdo->commit(); + + self::assertSame( + 'openstatspec-in-place-transformation-v0.2', + $this->query($pdo, 'SELECT contract_id FROM transformation_apply')->fetchColumn(), + ); + } + + private function plan(): TransformationPlan + { + return new TransformationPlan( + PlanContract::V01, + 'parent', + [new SetVariableLabelOperation('score', 'Score')], + ); + } + + private function sqlite(bool $foreignKeys = true): PDO + { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $pdo->exec('PRAGMA foreign_keys = ' . ($foreignKeys ? 'ON' : 'OFF')); + $pdo->exec('CREATE TABLE dataset (dataset_id VARCHAR(36) NOT NULL PRIMARY KEY, physical_table_schema TEXT NULL, physical_table_name TEXT NOT NULL)'); + $pdo->exec("INSERT INTO dataset VALUES ('" . self::DATASET_ID . "', NULL, 'data_survey')"); + $pdo->exec('CREATE TABLE openstatspec_schema_migration (version INTEGER NOT NULL PRIMARY KEY, applied_at TIMESTAMP NOT NULL)'); + $insert = $pdo->prepare('INSERT INTO openstatspec_schema_migration VALUES (?, ?)'); + foreach ([1, 2, 3] as $version) { + $insert->execute([$version, '2026-08-17 00:00:00']); + } + return $pdo; + } + + private function createVersion01AuditTable(PDO $pdo): void + { + $pdo->exec(<<<'SQL' +CREATE TABLE transformation_apply ( + apply_id VARCHAR(36) NOT NULL PRIMARY KEY, + contract_id TEXT NOT NULL, + database_profile TEXT NOT NULL, + dataset_id VARCHAR(36) NOT NULL REFERENCES dataset(dataset_id), + physical_table_schema TEXT NULL, + physical_table_name TEXT NOT NULL, + source_hash CHAR(64) NOT NULL, + plan_hash CHAR(64) NOT NULL, + canonical_plan_json TEXT NOT NULL, + actor TEXT NOT NULL, + status TEXT NOT NULL, + dolt_branch TEXT NULL, + dolt_head_before TEXT NULL, + dolt_head_after TEXT NULL, + operation_count INTEGER NOT NULL, + started_at TIMESTAMP NOT NULL, + completed_at TIMESTAMP NOT NULL, + CHECK (contract_id = 'openstatspec-in-place-transformation-v0.1'), + CHECK (database_profile IN ('sqlite', 'postgresql', 'mysql', 'mariadb', 'dolt')), + CHECK (status IN ('succeeded', 'failed')), + CHECK (operation_count > 0), + CHECK ((database_profile <> 'dolt' AND dolt_branch IS NULL AND dolt_head_before IS NULL AND dolt_head_after IS NULL) OR (database_profile = 'dolt' AND dolt_branch IS NOT NULL AND dolt_head_before IS NOT NULL AND (status = 'failed' OR dolt_head_after = dolt_head_before))) +) +SQL); + } + + private function insertVersion01AuditRow(PDO $pdo, string $datasetId = self::DATASET_ID): void + { + $pdo->prepare('INSERT INTO transformation_apply VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')->execute([ + 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + 'openstatspec-in-place-transformation-v0.1', + 'sqlite', + $datasetId, + null, + 'data_survey', + str_repeat('b', 64), + str_repeat('c', 64), + '{"contract":"openstatspec-transformation-plan-v0.1"}', + 'legacy-runner', + 'succeeded', + null, + null, + null, + 1, + '2026-08-17 00:00:00', + '2026-08-17 00:00:01', + ]); + } + + /** @return list */ + private function acceptedContracts(PDO $pdo): array + { + $sql = $this->query($pdo, "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'transformation_apply'")->fetchColumn(); + self::assertIsString($sql); + return array_values(array_filter( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + static fn(string $contract): bool => str_contains($sql, $contract), + )); + } + + /** @return list */ + private function columnNames(PDO $pdo): array + { + $columns = $this->query($pdo, 'PRAGMA table_info(transformation_apply)')->fetchAll(PDO::FETCH_ASSOC); + return array_values(array_map(static fn(array $column): string => (string) $column['name'], $columns)); + } + + /** @return list */ + private function migrationVersions(PDO $pdo): array + { + $versions = $this->query($pdo, 'SELECT version FROM openstatspec_schema_migration ORDER BY version')->fetchAll(PDO::FETCH_COLUMN); + return array_values(array_map(static fn(mixed $version): int => (int) $version, $versions)); + } + + private function query(PDO $pdo, string $sql): PDOStatement + { + $statement = $pdo->query($sql); + if ($statement === false) { + self::fail('Could not execute audit test query.'); + } + return $statement; + } +} From d605d876d98cb00fc21db215b6a96a990c9c7014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 18:00:04 +0300 Subject: [PATCH 13/53] fix: harden transformation audit migration --- .../task-7-report.md | 18 +++- src/Sql/CatalogOwnership.php | 34 ++++++-- src/Sql/NormativeCatalog.php | 8 +- .../Audit/TransformationAuditMigrator.php | 45 ++++++---- .../InPlaceTransformationServiceTest.php | 3 + .../PostgreSqlSpssRoundTripTest.php | 2 + .../ServerCatalogMigrationTest.php | 85 ++++++++++++++++++- tests/Sql/CatalogOwnershipTest.php | 54 ++++++++++++ .../InPlaceTransformationExecutorTest.php | 2 + 9 files changed, 218 insertions(+), 33 deletions(-) diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md index 6215bb5..3ba172b 100644 --- a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md @@ -7,27 +7,37 @@ - Added success-only `TransformationAuditWriter`, which requires the caller's open apply transaction and stores canonical plan identity, source identity, actor, target identity, operation count, timestamps, and nullable Dolt evidence. - Wired `SpssAdapter::migrateCatalog()` to run the audit migration after normative table creation and before marking schema version 4. - Preserved official 0.1 audit rows while allowing both 0.1 and 0.2 binding contracts. No dataset/data table, snapshot, staging, rollback, or recovery relation was added. +- MySQL-family hashes now use a MySQL 8.4/MariaDB/Dolt-compatible exact lowercase-hex check without `REGEXP BINARY`. +- MySQL-family contract upgrades add the v0.1+v0.2 check before dropping older checks. A retry can recover an already check-less table, and an interrupted add-before-drop attempt retains the old check. +- Schema version 4 can be marked or accepted as ready only when the contiguous 1..4 migration history and readable compact audit relation both exist. Normative migration recording is private and limited to versions 1..3; the audit migrator alone records version 4. ## TDD evidence - RED: `vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php` reported 6 missing-class errors across 7 tests. - GREEN: the final focused command `vendor/bin/phpunit tests/Transformation/Audit/TransformationAuditMigratorTest.php tests/Integration/ServerCatalogMigrationTest.php` passed 10 tests and 54 assertions; the only skip was the unconfigured server integration profile. - Wiring mutation check: removing the adapter's explicit migrator call made `testAdapterMigrationRecordsFreshVersionsOneThroughFourExactlyOnce` fail because marker 4 was absent; restoring the call returned the suite to green. +- Review RED: the two new ownership regressions both failed because `markCurrentVersion()` and readiness accepted an audit-less v4 claim. +- Review RED: the configured MySQL 8.4 integration test failed during fresh migration with SQLSTATE 3995 from `REGEXP BINARY`. +- Review GREEN: ownership regressions cover both the reported `identity=4 / migrations=1..3 / audit=absent` state and a forged marker 4 with no audit table. The server test also rejects uppercase source/plan hashes and simulates recovery from a missing MySQL-family contract check. +- Review follow-up RED/GREEN: a second MySQL run against the same migrated schema first failed because the legacy fixture hard-coded the pre-upgrade check name; introspecting the current check made two consecutive runs pass. ## Verification - `composer analyse`: `[OK] No errors`. -- Working-tree `composer style`: exits 8 because 133 pre-existing CRLF files are reported as fixable. -- CRLF-normalized staged-tree `composer check`: Composer validation, PHP lint, style (`0 of 217`), PHPStan, and PHPUnit all passed; PHPUnit ran 417 tests with 2,447 assertions and 30 environment skips. +- Working-tree `composer style`: exits 8 because 126 pre-existing CRLF files are reported as fixable. +- CRLF-normalized staged-tree `composer check`: Composer validation, PHP lint, style, PHPStan, and PHPUnit all passed; PHPUnit ran 420 tests with 2,454 assertions and 30 environment skips. +- Focused SQLite/unconfigured-server command passed 35 tests and 144 assertions with the one expected unconfigured-server skip. +- Live MySQL 8.4.9 integration passed twice consecutively against one schema, each run with 1 test and 13 assertions, including fresh/idempotent DDL, legacy upgrade, exact hash rejection, and missing-check retry recovery. +- Live MariaDB 11.6.2 DDL probe passed fresh migration, lowercase acceptance, uppercase source/plan rejection, missing-check retry recovery, and exactly one v4 marker. This locally available version is outside the adapter's claimed MariaDB families, so the probe invoked the migrator directly rather than misrepresenting it as a supported full adapter run. - `git diff --cached --check`: passed. ## Self-review - SQLite v0.1 migration preserves the row and logical table name, verifies foreign keys before commit, leaves no `_v04` residue, and rolls back to the old table on copy failure. -- PostgreSQL changes the old contract check in one native transaction; MySQL-family/Dolt DDL is explicit, idempotent, and never invoked by the audit writer or inside an apply transaction. +- PostgreSQL changes the old contract check in one native transaction; MySQL-family/Dolt DDL is explicit, add-before-drop, retryable, idempotent, and never invoked by the audit writer or inside an apply transaction. - The writer has no failure-row API and performs no migration, dataset copy, table copy, Dolt commit, branch operation, or recovery/version write. - Fresh catalogs record versions 1 through 4 exactly once; v3 ownership recognition remains backward compatible, while v4 marker recognition requires the compact audit relation. ## Environment note -No server DSNs were configured locally, so PostgreSQL/MySQL/MariaDB/Dolt integration cases were the only focused skip; their migration test now exercises preservation of a pre-existing 0.1 row when those services are configured. +No PostgreSQL or Dolt service was available locally. Their configured-server integration branches remain covered by the same portable migration test; PostgreSQL's check replacement remains transactionally atomic, while Dolt follows the live-probed MySQL-family DDL path. diff --git a/src/Sql/CatalogOwnership.php b/src/Sql/CatalogOwnership.php index 17599d4..a0a10a3 100644 --- a/src/Sql/CatalogOwnership.php +++ b/src/Sql/CatalogOwnership.php @@ -15,6 +15,7 @@ final class CatalogOwnership private const IDENTITY_TABLE = 'catalog_identity'; private const CONTRACT_ID = 'openstatspec-strict-wide-table-v1'; private const MIGRATION_TABLE = 'openstatspec_schema_migration'; + private const AUDIT_TABLE = 'transformation_apply'; private const SCHEMA_VERSION = 4; /** @return array */ @@ -93,7 +94,7 @@ public static function ensure(PDO $pdo): void public static function assertReadyForUse(PDO $pdo): void { if (self::tableExists($pdo, self::IDENTITY_TABLE)) { - if (self::validateIdentity($pdo) < self::SCHEMA_VERSION) { + if (self::validateIdentity($pdo) < self::SCHEMA_VERSION || !self::currentMigrationComplete($pdo)) { throw self::migrationRequired(); } self::assertExclusiveNamespace($pdo); @@ -148,10 +149,32 @@ public static function isFreshPending(PDO $pdo): bool public static function markCurrentVersion(PDO $pdo): void { self::validateIdentity($pdo); + if (!self::currentMigrationComplete($pdo)) { + throw self::migrationRequired(); + } $statement = $pdo->prepare('UPDATE ' . self::IDENTITY_TABLE . ' SET schema_version = ? WHERE catalog_identity_key = 1'); $statement->execute([self::SCHEMA_VERSION]); } + private static function currentMigrationComplete(PDO $pdo): bool + { + return self::tableExists($pdo, self::MIGRATION_TABLE) + && self::tableExists($pdo, self::AUDIT_TABLE) + && self::validateLegacyMarker($pdo) === self::SCHEMA_VERSION + && self::canSelectColumns($pdo, self::AUDIT_TABLE, self::auditColumns()); + } + + /** @return list */ + private static function auditColumns(): array + { + return [ + 'apply_id', 'contract_id', 'database_profile', 'dataset_id', 'physical_table_schema', + 'physical_table_name', 'source_hash', 'plan_hash', 'canonical_plan_json', 'actor', + 'status', 'dolt_branch', 'dolt_head_before', 'dolt_head_after', 'operation_count', + 'started_at', 'completed_at', + ]; + } + /** @return array */ private static function bindingResult(string $mode, ?string $name, ?string $locator, bool $verified): array { @@ -235,13 +258,8 @@ private static function isRecognizedMarkerCatalog(PDO $pdo, array $collisions, i $canonicalComplete = $canonical !== [] && self::matchesCompleteDefinition($pdo, $canonical, $canonicalDefinition); $legacyComplete = $legacy !== [] && self::matchesCompleteDefinition($pdo, $legacy, $legacyDefinition); $journalsComplete = $journals === [] || self::isRecognizedJournalOnlyLegacyCatalog($pdo, $journals); - $auditPresent = in_array('transformation_apply', $collisions, true); - $auditComplete = !$auditPresent || self::canSelectColumns($pdo, 'transformation_apply', [ - 'apply_id', 'contract_id', 'database_profile', 'dataset_id', 'physical_table_schema', - 'physical_table_name', 'source_hash', 'plan_hash', 'canonical_plan_json', 'actor', - 'status', 'dolt_branch', 'dolt_head_before', 'dolt_head_after', 'operation_count', - 'started_at', 'completed_at', - ]); + $auditPresent = in_array(self::AUDIT_TABLE, $collisions, true); + $auditComplete = !$auditPresent || self::canSelectColumns($pdo, self::AUDIT_TABLE, self::auditColumns()); if (($canonical !== [] && !$canonicalComplete) || ($legacy !== [] && !$legacyComplete) || !$journalsComplete diff --git a/src/Sql/NormativeCatalog.php b/src/Sql/NormativeCatalog.php index 4345013..f327598 100644 --- a/src/Sql/NormativeCatalog.php +++ b/src/Sql/NormativeCatalog.php @@ -75,11 +75,11 @@ public function createTables(): void } } - /** Record one completed explicit catalogue migration idempotently. */ - public function recordMigration(int $version): void + /** Record one completed normative-catalogue migration idempotently. */ + private function recordMigration(int $version): void { - if ($version < 1) { - throw new \InvalidArgumentException('A catalogue migration version must be positive.'); + if ($version < 1 || $version > 3) { + throw new \InvalidArgumentException('A normative catalogue migration version must be between 1 and 3.'); } $driver = (string) $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME); $migration = match ($driver) { diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php index d4b4dca..f25be49 100644 --- a/src/Transformation/Audit/TransformationAuditMigrator.php +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -6,7 +6,6 @@ use OpenStatSpec\Core\DiagnosticCode; use OpenStatSpec\Core\UnsupportedOperation; -use OpenStatSpec\Sql\NormativeCatalog; use PDO; use PDOException; use Throwable; @@ -60,7 +59,7 @@ private function migrateSqlite(): void if ($violations === false || $violations->fetchColumn() !== false) { throw new PDOException('Transformation audit migration produced a foreign-key violation.'); } - (new NormativeCatalog($this->pdo))->recordMigration(4); + $this->recordMigration(); $this->pdo->commit(); } catch (Throwable $exception) { if ($this->pdo->inTransaction()) { @@ -92,7 +91,7 @@ private function migratePostgreSql(): void ); } } - (new NormativeCatalog($this->pdo))->recordMigration(4); + $this->recordMigration(); $this->pdo->commit(); } catch (Throwable $exception) { if ($this->pdo->inTransaction()) { @@ -109,21 +108,21 @@ private function migrateMySqlFamily(): void } else { $this->assertReadableAuditTable(); $checks = $this->mySqlContractChecks(); - if (!$this->checksAreCurrent($checks)) { - if ($checks === []) { - throw new PDOException('The existing transformation audit has no contract check to migrate.'); - } - $dropSyntax = $this->isMariaDb() ? ' DROP CONSTRAINT ' : ' DROP CHECK '; - foreach (array_keys($checks) as $constraint) { - $this->pdo->exec('ALTER TABLE ' . self::TABLE . $dropSyntax . $this->quoteMySql($constraint)); - } + $current = array_filter($checks, $this->checkIsCurrent(...)); + if ($current === []) { $this->pdo->exec( - 'ALTER TABLE ' . self::TABLE . ' ADD CONSTRAINT chk_transformation_apply_contract ' + 'ALTER TABLE ' . self::TABLE . ' ADD CONSTRAINT chk_transformation_apply_contract_v04 ' . $this->contractCheck(), ); } + if (count($current) !== count($checks)) { + $dropSyntax = $this->isMariaDb() ? ' DROP CONSTRAINT ' : ' DROP CHECK '; + foreach (array_diff_key($checks, $current) as $constraint => $_definition) { + $this->pdo->exec('ALTER TABLE ' . self::TABLE . $dropSyntax . $this->quoteMySql($constraint)); + } + } } - (new NormativeCatalog($this->pdo))->recordMigration(4); + $this->recordMigration(); } private function createTableSql(string $table, string $driver): string @@ -177,8 +176,8 @@ private function hashChecks(string $driver): string return "CONSTRAINT chk_transformation_apply_source_hash CHECK (source_hash ~ '^[0-9a-f]{64}$'), " . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (plan_hash ~ '^[0-9a-f]{64}$')"; } - return "CONSTRAINT chk_transformation_apply_source_hash CHECK (source_hash REGEXP BINARY '^[0-9a-f]{64}$'), " - . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (plan_hash REGEXP BINARY '^[0-9a-f]{64}$')"; + return "CONSTRAINT chk_transformation_apply_source_hash CHECK (CHAR_LENGTH(source_hash) = 64 AND source_hash REGEXP '^[0-9a-f]{64}$' AND CAST(source_hash AS BINARY) = CAST(LOWER(source_hash) AS BINARY)), " + . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (CHAR_LENGTH(plan_hash) = 64 AND plan_hash REGEXP '^[0-9a-f]{64}$' AND CAST(plan_hash AS BINARY) = CAST(LOWER(plan_hash) AS BINARY))"; } private function sqliteAcceptsVersion02(): bool @@ -249,6 +248,22 @@ private function checksAreCurrent(array $checks): bool return true; } + private function checkIsCurrent(string $definition): bool + { + return str_contains($definition, 'openstatspec-in-place-transformation-v0.1') + && str_contains($definition, 'openstatspec-in-place-transformation-v0.2'); + } + + private function recordMigration(): void + { + $migration = match ((string) $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME)) { + 'mysql' => 'INSERT IGNORE INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?)', + 'pgsql', 'sqlite' => 'INSERT INTO openstatspec_schema_migration (version, applied_at) VALUES (?, ?) ON CONFLICT (version) DO NOTHING', + default => throw new \LogicException('Unsupported audit migration driver.'), + }; + $this->pdo->prepare($migration)->execute([4, gmdate('Y-m-d H:i:s')]); + } + private function assertReadableAuditTable(): void { $statement = $this->pdo->query('SELECT ' . implode(', ', self::columns()) . ' FROM ' . self::TABLE . ' WHERE 1 = 0'); diff --git a/tests/Integration/InPlaceTransformationServiceTest.php b/tests/Integration/InPlaceTransformationServiceTest.php index 0886a29..52d1be9 100644 --- a/tests/Integration/InPlaceTransformationServiceTest.php +++ b/tests/Integration/InPlaceTransformationServiceTest.php @@ -10,6 +10,7 @@ use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Model\Action\AssignValueAction; use OpenStatSpec\Transformation\Model\Action\CopySourceAction; @@ -440,6 +441,7 @@ public function testFixtureSetupRejectsDirtyDeterministicNamespaceWithoutDeletin $this->assertDoltWorkingSetCleanBeforeFixture($pdo, $connection); (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); $pdo->exec( 'CREATE TABLE ' . $this->qualifiedTable($connection, $tableName) @@ -613,6 +615,7 @@ private function installFixture(PDO $pdo, Connection $connection, ?array $fixtur { $this->assertDoltWorkingSetCleanBeforeFixture($pdo, $connection); (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); $fixture ??= $this->reserveFixtureIdentity($connection); diff --git a/tests/Integration/PostgreSqlSpssRoundTripTest.php b/tests/Integration/PostgreSqlSpssRoundTripTest.php index b6beec5..10a4b2a 100644 --- a/tests/Integration/PostgreSqlSpssRoundTripTest.php +++ b/tests/Integration/PostgreSqlSpssRoundTripTest.php @@ -10,6 +10,7 @@ use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Spss\PhpSpssEngine; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Model\CreateVariableOperation; use OpenStatSpec\Transformation\Model\DeleteVariableOperation; @@ -134,6 +135,7 @@ public function testDeleteThenCreateAtThePostgreSqlPhysicalColumnLimitFailsBefor try { (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); $columns = ['__case_ordinal BIGINT NOT NULL PRIMARY KEY']; for ($ordinal = 1; $ordinal <= 1599; ++$ordinal) { diff --git a/tests/Integration/ServerCatalogMigrationTest.php b/tests/Integration/ServerCatalogMigrationTest.php index 953f974..df2f47d 100644 --- a/tests/Integration/ServerCatalogMigrationTest.php +++ b/tests/Integration/ServerCatalogMigrationTest.php @@ -39,6 +39,14 @@ public function testLegacyV3ConstraintsAndSourceOrderMigrateOnConfiguredServers( self::assertSame(1, (int) $this->scalar($pdo, 'SELECT source_ordinal FROM multiple_response_set WHERE dataset_id = ?', [$datasetId]), $name . ': MR-set source order'); self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply WHERE apply_id = ?', [$legacyApplyId]), $name . ': existing 0.1 audit row'); $this->assertAuditContracts($pdo, $name, $datasetId, 'unused_' . $token); + $this->assertUppercaseHashesRejected($pdo, $name, $datasetId, 'unused_' . $token); + if ($name !== 'postgresql') { + $this->dropAuditContractChecks($pdo, $name); + $pdo->exec('DELETE FROM openstatspec_schema_migration WHERE version = 4'); + + $adapter->migrateCatalog(); + $this->assertUnknownAuditContractRejected($pdo, $name, $datasetId, 'unused_' . $token); + } $this->assertNotNullAndUnique($pdo, $name); foreach ([ ['INSERT INTO variable_set (variable_set_id, dataset_id, source_ordinal, set_name) VALUES (?, ?, ?, ?)', ['vs-duplicate-' . $token, $datasetId, 2, 'Duplicate variable set']], @@ -71,8 +79,7 @@ private function makeVersion01Audit(PDO $pdo, string $profile, string $datasetId if ($driver === 'pgsql') { $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT chk_transformation_apply_contract'); } else { - $drop = $profile === 'mariadb' ? 'DROP CONSTRAINT' : 'DROP CHECK'; - $pdo->exec('ALTER TABLE transformation_apply ' . $drop . ' chk_transformation_apply_contract'); + $this->dropAuditContractChecks($pdo, $profile); } $pdo->exec("ALTER TABLE transformation_apply ADD CONSTRAINT chk_transformation_apply_contract CHECK (contract_id = 'openstatspec-in-place-transformation-v0.1')"); $pdo->exec('DELETE FROM openstatspec_schema_migration WHERE version = 4'); @@ -140,6 +147,80 @@ private function assertAuditContracts(PDO $pdo, string $profile, string $dataset ); } + private function assertUppercaseHashesRejected(PDO $pdo, string $profile, string $datasetId, string $table): void + { + foreach (['source' => [str_repeat('A', 64), str_repeat('b', 64)], 'plan' => [str_repeat('a', 64), str_repeat('B', 64)]] as $hash => [$sourceHash, $planHash]) { + try { + $this->insertAudit($pdo, $profile, $datasetId, $table, 'openstatspec-in-place-transformation-v0.2', $sourceHash, $planHash); + self::fail($profile . ': uppercase ' . $hash . ' hash was accepted'); + } catch (PDOException) { + // The exact lowercase-hex database check is the assertion. + } + } + } + + private function assertUnknownAuditContractRejected(PDO $pdo, string $profile, string $datasetId, string $table): void + { + try { + $this->insertAudit($pdo, $profile, $datasetId, $table, 'foreign-contract', str_repeat('a', 64), str_repeat('b', 64)); + self::fail($profile . ': missing contract check was not recovered'); + } catch (PDOException) { + // A retry after the interrupted upgrade restored the current check. + } + } + + private function insertAudit( + PDO $pdo, + string $profile, + string $datasetId, + string $table, + string $contract, + string $sourceHash, + string $planHash, + ): void { + $dolt = $profile === 'dolt'; + $pdo->prepare('INSERT INTO transformation_apply VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')->execute([ + $this->uuidV4(), + $contract, + $profile, + $datasetId, + null, + $table, + $sourceHash, + $planHash, + '{}', + 'migration-rejection-test', + 'succeeded', + $dolt ? 'migration-test' : null, + $dolt ? 'migration-head' : null, + $dolt ? 'migration-head' : null, + 1, + '2026-08-17 00:00:00', + '2026-08-17 00:00:01', + ]); + } + + private function dropAuditContractChecks(PDO $pdo, string $profile): void + { + $statement = $pdo->query(<<<'SQL' +SELECT table_constraint.constraint_name +FROM information_schema.table_constraints table_constraint +JOIN information_schema.check_constraints check_constraint + ON check_constraint.constraint_schema = table_constraint.constraint_schema + AND check_constraint.constraint_name = table_constraint.constraint_name +WHERE table_constraint.constraint_schema = DATABASE() + AND table_constraint.table_name = 'transformation_apply' + AND table_constraint.constraint_type = 'CHECK' + AND check_constraint.check_clause LIKE '%contract_id%' +SQL); + $constraints = $statement === false ? [] : $statement->fetchAll(PDO::FETCH_COLUMN); + $drop = $profile === 'mariadb' ? 'DROP CONSTRAINT' : 'DROP CHECK'; + foreach ($constraints as $constraint) { + self::assertIsString($constraint); + $pdo->exec('ALTER TABLE transformation_apply ' . $drop . ' `' . str_replace('`', '``', $constraint) . '`'); + } + } + private function uuidV4(): string { $bytes = random_bytes(16); diff --git a/tests/Sql/CatalogOwnershipTest.php b/tests/Sql/CatalogOwnershipTest.php index 515a560..a7ca3b9 100644 --- a/tests/Sql/CatalogOwnershipTest.php +++ b/tests/Sql/CatalogOwnershipTest.php @@ -8,6 +8,7 @@ use OpenStatSpec\Core\UnsupportedOperation; use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use PDO; use PDOStatement; use PHPUnit\Framework\Attributes\DataProvider; @@ -191,6 +192,7 @@ public function testCurrentIdentityRejectsLaterUnrelatedApplicationTable(): void $pdo = $this->sqlite(); CatalogOwnership::ensure($pdo); (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); $pdo->exec('CREATE TABLE application_data (id INTEGER PRIMARY KEY)'); @@ -209,6 +211,7 @@ public function testRegisteredPhysicalDataTableIsAllowed(): void (new NormativeCatalog($pdo))->createTables(); $pdo->exec('CREATE TABLE dataset_registered (case_ordinal INTEGER PRIMARY KEY)'); $pdo->exec("INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_name, dataset_name, source_case_count, imported_at) VALUES ('registered', '1.0', 'sav', 'dataset_registered', 'registered', 0, '2026-07-28 00:00:00')"); + (new TransformationAuditMigrator($pdo))->migrate(); CatalogOwnership::markCurrentVersion($pdo); CatalogOwnership::ensure($pdo); @@ -216,6 +219,57 @@ public function testRegisteredPhysicalDataTableIsAllowed(): void self::assertSame(4, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); } + public function testCurrentVersionCannotBeMarkedBeforeAuditMigrationCompletes(): void + { + $pdo = $this->sqlite(); + CatalogOwnership::ensure($pdo); + (new NormativeCatalog($pdo))->createTables(); + + try { + CatalogOwnership::markCurrentVersion($pdo); + self::fail('Schema version 4 was marked without audit migration 4.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $exception->diagnosticCode); + } + + self::assertSame(1, (int) $this->query($pdo, 'SELECT schema_version FROM catalog_identity')->fetchColumn()); + self::assertSame([1, 2, 3], array_map( + 'intval', + $this->query($pdo, 'SELECT version FROM openstatspec_schema_migration ORDER BY version')->fetchAll(PDO::FETCH_COLUMN), + )); + } + + public function testVersionFourIdentityWithoutAuditMigrationIsNotReadyForUse(): void + { + $pdo = $this->sqlite(); + CatalogOwnership::ensure($pdo); + (new NormativeCatalog($pdo))->createTables(); + $pdo->exec('UPDATE catalog_identity SET schema_version = 4'); + + try { + CatalogOwnership::assertReadyForUse($pdo); + self::fail('Schema version 4 without audit migration 4 was accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $exception->diagnosticCode); + } + } + + public function testVersionFourMarkerWithoutAuditTableIsNotReadyForUse(): void + { + $pdo = $this->sqlite(); + CatalogOwnership::ensure($pdo); + (new NormativeCatalog($pdo))->createTables(); + $pdo->exec("INSERT INTO openstatspec_schema_migration VALUES (4, '2026-08-17 00:00:00')"); + $pdo->exec('UPDATE catalog_identity SET schema_version = 4'); + + try { + CatalogOwnership::assertReadyForUse($pdo); + self::fail('Schema version and marker 4 without the audit table were accepted.'); + } catch (UnsupportedOperation $exception) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $exception->diagnosticCode); + } + } + public function testOlderIdentityVersionIsAcceptedWithoutPrematureUpgrade(): void { $pdo = $this->sqlite(); diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index 4b7a5ab..aadedc8 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -9,6 +9,7 @@ use OpenStatSpec\Core\UnsupportedOperation; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Model\Action\AssignValueAction; use OpenStatSpec\Transformation\Model\Action\CopySourceAction; @@ -68,6 +69,7 @@ protected function setUp(): void foreach ([[1, 1.0, -1.0], [2, 2.0, -1.0], [3, 3.0, -1.0], [4, 9.0, -1.0], [5, null, -1.0]] as $case) { $insertCase->execute($case); } + (new TransformationAuditMigrator($this->pdo))->migrate(); CatalogOwnership::markCurrentVersion($this->pdo); } From 25fba2808ec9b75b6b83b157c011493954a76fbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 18:37:06 +0300 Subject: [PATCH 14/53] feat: execute official plans atomically in place --- .../task-8-report.md | 43 + .../Execution/BoundOperation.php | 23 + src/Transformation/Execution/BoundPlan.php | 19 + src/Transformation/Execution/BoundSchema.php | 24 + .../Execution/DatasetBinding.php | 9 + .../Execution/ExecutionResult.php | 4 +- .../Execution/InPlaceOperationExecutor.php | 305 ++++++ .../InPlaceTransformationExecutor.php | 890 +----------------- .../Execution/PlanPreflight.php | 573 +++++++++++ .../Execution/SqlPredicateCompiler.php | 76 ++ .../Execution/VariableBinding.php | 5 + .../InPlaceTransformationServiceTest.php | 133 +-- .../OfficialInPlaceTransformation01Test.php | 163 ++++ .../OfficialInPlaceTransformation02Test.php | 613 ++++++++++++ .../PostgreSqlSpssRoundTripTest.php | 28 +- .../InPlaceTransformationExecutorTest.php | 796 ++++------------ 16 files changed, 2162 insertions(+), 1542 deletions(-) create mode 100644 .superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md create mode 100644 src/Transformation/Execution/BoundOperation.php create mode 100644 src/Transformation/Execution/BoundPlan.php create mode 100644 src/Transformation/Execution/BoundSchema.php create mode 100644 src/Transformation/Execution/InPlaceOperationExecutor.php create mode 100644 src/Transformation/Execution/PlanPreflight.php create mode 100644 src/Transformation/Execution/SqlPredicateCompiler.php create mode 100644 tests/Integration/OfficialInPlaceTransformation01Test.php create mode 100644 tests/Integration/OfficialInPlaceTransformation02Test.php diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md new file mode 100644 index 0000000..e33b7e3 --- /dev/null +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md @@ -0,0 +1,43 @@ +# Task 8 report: atomic official-plan execution on SQLite and PostgreSQL + +## Implementation + +- Replaced the legacy executor entry point with `execute(InPlaceApplyRequest)` and split mutation-free binding from ordered SQL execution. +- Added immutable bound plan/operation/schema objects. Preflight revalidates the official canonical plan, catalog ownership and audit readiness, resolves the exact dataset/table and variable bindings, verifies physical columns, and simulates the complete evolving schema before starting a transaction. +- Added deterministic, parameterized predicate and operand SQL. All physical identifiers are profile-quoted; literals, labels, and values are bound parameters. Boolean operands retain source order and SQL three-valued truth. +- Implemented ordered in-place `recode`, `assign`, `conditional_assign`, variable-label, value-label, format, measurement-level, and `execute` behavior. First-match recodes, inclusive ranges, exact typed values, numeric system missing, unmatched results, and sequential dependencies follow the official 0.1/0.2 plans. +- SQLite and PostgreSQL create nullable numeric targets with one same-table `ALTER TABLE`, one minimal catalog variable row, and the data update inside the public transaction. New variables receive a fresh UUID and next ordinal with no implicit label, value-label, missing, format, display, role, or measurement metadata. +- Success audit insertion shares the data/catalog/schema transaction and returns a non-null audit ID. Any apply failure rolls back; the executor no longer starts or writes `OperationJournal` records. +- Migrated executor integration callers to official plans/apply requests, including the PostgreSQL physical-column-limit regression. + +## TDD evidence + +- RED: the initial Task 8 focused suite produced 16 errors because the legacy executor rejected `InPlaceApplyRequest` at its old plan-only signature. +- GREEN: the final focused command passed 44 tests and 387 assertions; 18 configured-service cases skipped. +- SQLite success tests prove unchanged dataset count, persistent data-table count, dataset/table identity, table set, case order/count, and existing variable identity while applying exact row and metadata changes. +- SQLite injected-audit failure proves transactional rollback of the added physical column, catalog variable, row updates, and audit row. +- Preflight tests cover unknown/type/target/format/audit errors, a later invalid operation preventing an earlier valid mutation, and rejection of caller-owned transactions. +- SQL tests cover malicious catalog identifiers and bound label/string payloads, including exact string recoding over a case-insensitive SQLite column. +- Official 0.1/0.2 tests cover ordered first-match recode, inclusive ranges, numeric system missing, copy/unmatched behavior, sequential assign and conditional assign, SQL NULL truth, complete value-label replacement, exact format/measurement updates, and preservation of unrelated missing/display metadata. +- PostgreSQL success and injected-audit rollback probes are environment-gated and use the same official atomic-create case. Both initialize a fresh catalog before namespace checks and verify no additional persistent relation. + +## Verification + +- Focused executor regressions: 44 tests, 387 assertions, 18 skips; passed. +- `composer analyse`: passed with no errors. +- `composer lint`: passed. +- Full CRLF-normalized staged-tree `composer check`: Composer validation, PHP lint, style, PHPStan, and PHPUnit passed; PHPUnit ran 419 tests with 2,611 assertions and 32 environment skips. +- `git diff --cached --check`: passed. +- Working-tree `composer style`: exits 8 because 125 repository files retain pre-existing CRLF line endings. The LF-normalized staged-tree style gate found zero fixable files across all 225 checked files. + +## Self-review + +- Successful applies retain the one dataset row and one physical wide table; create-target changes only that table's schema and its normative variable catalog. +- Production executor code creates no derived/output/staging/snapshot/rollback/history/recovery relation, stores no copied dataset state, performs no compensating cleanup, and invokes no Dolt mutation procedure. +- Preflight is read-only and completes before `beginTransaction`; ordered operation SQL, Dolt post-context observation where applicable, and exactly one success audit share the native transaction. +- Metadata writes target only the named normative fields. Value-label replacement is copy-on-write for a shared label set and leaves missing values and unrelated metadata untouched. +- PostgreSQL physical-slot accounting includes dropped column slots through `pg_attribute`, so target creation fails before mutation at the server's wide-table limit. + +## Environment note + +`OPENSTATSPEC_PG_DSN`, `OPENSTATSPEC_PG_USER`, and `OPENSTATSPEC_PG_PASSWORD` were not configured locally, so the two new live PostgreSQL success/rollback probes and the existing PostgreSQL integrations skipped. SQLite supplied the local native-DDL transaction evidence; the PostgreSQL paths remain in the normalized full suite and run automatically when the documented environment is configured. diff --git a/src/Transformation/Execution/BoundOperation.php b/src/Transformation/Execution/BoundOperation.php new file mode 100644 index 0000000..1daa902 --- /dev/null +++ b/src/Transformation/Execution/BoundOperation.php @@ -0,0 +1,23 @@ +target !== null && !$this->target->persisted; + } +} diff --git a/src/Transformation/Execution/BoundPlan.php b/src/Transformation/Execution/BoundPlan.php new file mode 100644 index 0000000..5cc265a --- /dev/null +++ b/src/Transformation/Execution/BoundPlan.php @@ -0,0 +1,19 @@ + $operations */ + public function __construct( + public TransformationPlan $plan, + public DatasetBinding $dataset, + public array $operations, + public BoundSchema $finalSchema, + ) {} +} diff --git a/src/Transformation/Execution/BoundSchema.php b/src/Transformation/Execution/BoundSchema.php new file mode 100644 index 0000000..a6ffa47 --- /dev/null +++ b/src/Transformation/Execution/BoundSchema.php @@ -0,0 +1,24 @@ + $variables */ + public function __construct(public array $variables) {} + + public function variable(string $name, string $path = '$.variable'): VariableBinding + { + $variable = $this->variables[$name] ?? null; + if ($variable === null) { + throw TransformationFailure::at('unknown_variable', $path, 'Plan variable is not registered in the bound dataset.'); + } + + return $variable; + } +} diff --git a/src/Transformation/Execution/DatasetBinding.php b/src/Transformation/Execution/DatasetBinding.php index 0da07bb..e930efd 100644 --- a/src/Transformation/Execution/DatasetBinding.php +++ b/src/Transformation/Execution/DatasetBinding.php @@ -4,6 +4,8 @@ namespace OpenStatSpec\Transformation\Execution; +use OpenStatSpec\Sql\PdoSqlProfile; + /** @internal Exact normative-catalog binding for one existing dataset. */ final readonly class DatasetBinding { @@ -13,4 +15,11 @@ public function __construct( public ?string $schema, public string $table, ) {} + + public function qualifiedTable(PdoSqlProfile $profile): string + { + $table = $profile->quoteIdentifier($this->table); + + return $this->schema === null ? $table : $profile->quoteIdentifier($this->schema) . '.' . $table; + } } diff --git a/src/Transformation/Execution/ExecutionResult.php b/src/Transformation/Execution/ExecutionResult.php index c3b5790..5cd4f96 100644 --- a/src/Transformation/Execution/ExecutionResult.php +++ b/src/Transformation/Execution/ExecutionResult.php @@ -10,7 +10,7 @@ public function __construct( private string $datasetId, private string $planHash, private int $operationCount, - private ?string $auditOperationId, + private string $auditOperationId, private ?DoltEvidence $doltBefore, private ?DoltEvidence $doltAfter, ) {} @@ -30,7 +30,7 @@ public function operationCount(): int return $this->operationCount; } - public function auditOperationId(): ?string + public function auditOperationId(): string { return $this->auditOperationId; } diff --git a/src/Transformation/Execution/InPlaceOperationExecutor.php b/src/Transformation/Execution/InPlaceOperationExecutor.php new file mode 100644 index 0000000..b5d306b --- /dev/null +++ b/src/Transformation/Execution/InPlaceOperationExecutor.php @@ -0,0 +1,305 @@ +predicateCompiler = new SqlPredicateCompiler($connection->profile); + } + + public function execute(BoundOperation $bound, DatasetBinding $dataset): void + { + if (!$this->connection->pdo->inTransaction()) { + throw new \LogicException('An in-place operation requires the open apply transaction.'); + } + + $operation = $bound->operation; + if ($operation instanceof AssignOperation) { + $target = $this->target($bound); + if (!$target->persisted) { + $this->createNumericTarget($dataset, $target); + } + $parameters = []; + $value = $this->predicateCompiler->operand($operation->value, $bound->schema, $parameters); + $this->statement( + 'UPDATE ' . $dataset->qualifiedTable($this->connection->profile) + . ' SET ' . $this->quote($target->physicalName) . ' = ' . $value, + )->execute($parameters); + return; + } + + if ($operation instanceof ConditionalAssignOperation) { + $parameters = []; + $value = $this->predicateCompiler->operand($operation->value, $bound->schema, $parameters); + $condition = $this->predicateCompiler->compile($operation->condition, $bound->schema, $parameters); + $this->statement( + 'UPDATE ' . $dataset->qualifiedTable($this->connection->profile) + . ' SET ' . $this->quote($this->target($bound)->physicalName) . ' = ' . $value + . ' WHERE ' . $condition, + )->execute($parameters); + return; + } + + if ($operation instanceof RecodeOperation) { + $target = $this->target($bound); + if (!$target->persisted) { + $this->createNumericTarget($dataset, $target); + } + $this->recode($operation, $bound, $dataset); + return; + } + + if ($operation instanceof SetVariableLabelOperation) { + $this->statement( + 'UPDATE variable SET variable_label = ? WHERE variable_id = ? AND dataset_id = ?', + )->execute([$operation->label, $this->target($bound)->variableId, $dataset->datasetId]); + return; + } + + if ($operation instanceof ReplaceValueLabelsOperation) { + $this->replaceValueLabels($operation, $dataset, $this->target($bound)); + return; + } + + if ($operation instanceof SetFormatOperation) { + $this->statement( + 'UPDATE variable SET print_format_family = ?, print_format_width = ?, print_format_decimals = ?, ' + . 'write_format_family = ?, write_format_width = ?, write_format_decimals = ? ' + . 'WHERE variable_id = ? AND dataset_id = ?', + )->execute([ + $operation->family, + $operation->width, + $operation->decimals, + $operation->family, + $operation->width, + $operation->decimals, + $this->target($bound)->variableId, + $dataset->datasetId, + ]); + return; + } + + if ($operation instanceof SetMeasurementLevelOperation) { + $this->statement( + 'UPDATE variable SET measurement_level = ? WHERE variable_id = ? AND dataset_id = ?', + )->execute([$operation->level, $this->target($bound)->variableId, $dataset->datasetId]); + return; + } + + if ($operation instanceof ExecuteOperation) { + return; + } + + throw TransformationFailure::at('plan_schema_invalid', '$.operations', 'Unsupported bound operation.'); + } + + private function createNumericTarget(DatasetBinding $dataset, VariableBinding $target): void + { + $this->connection->pdo->exec( + 'ALTER TABLE ' . $dataset->qualifiedTable($this->connection->profile) + . ' ADD COLUMN ' . $this->quote($target->physicalName) + . ' ' . $this->connection->profile->numericType() . ' NULL', + ); + $this->statement( + 'INSERT INTO variable ' + . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?)', + )->execute([ + $target->variableId, + $dataset->datasetId, + $target->sourceOrdinal, + $target->sourceName, + $target->physicalName, + 'numeric', + null, + ]); + } + + private function recode(RecodeOperation $operation, BoundOperation $bound, DatasetBinding $dataset): void + { + $source = $bound->source; + if ($source === null) { + throw new \LogicException('A bound recode requires its source.'); + } + $sourceSql = $this->quote($source->physicalName); + $parameters = []; + $when = []; + foreach ($operation->rules as $rule) { + $condition = match (true) { + $rule->match instanceof ExactMatch => $this->exactMatch($sourceSql, $rule->match, $parameters), + $rule->match instanceof RangeMatch => $this->rangeMatch($sourceSql, $rule->match, $parameters), + $rule->match instanceof SystemMissingMatch => '(' . $sourceSql . ' IS NULL)', + default => throw new \LogicException('Unsupported preflighted recode match.'), + }; + $when[] = 'WHEN ' . $condition . ' THEN ' . $this->resultSql($rule->result, $sourceSql, $parameters); + } + $unmatched = $this->resultSql($operation->unmatched, $sourceSql, $parameters); + $this->statement( + 'UPDATE ' . $dataset->qualifiedTable($this->connection->profile) + . ' SET ' . $this->quote($this->target($bound)->physicalName) + . ' = CASE ' . implode(' ', $when) . ' ELSE ' . $unmatched . ' END', + )->execute($parameters); + } + + /** @param list $parameters */ + private function exactMatch(string $sourceSql, ExactMatch $match, array &$parameters): string + { + $conditions = []; + foreach ($match->values as $value) { + $parameters[] = $this->value($value); + $conditions[] = $this->connection->profile->exactValueCondition( + $sourceSql, + $value instanceof StringValue, + ); + } + return '(' . implode(' OR ', $conditions) . ')'; + } + + /** @param list $parameters */ + private function rangeMatch(string $sourceSql, RangeMatch $match, array &$parameters): string + { + $parameters[] = $match->lower->number(); + $parameters[] = $match->upper->number(); + return '(' . $sourceSql . ' >= ? AND ' . $sourceSql . ' <= ?)'; + } + + /** @param list $parameters */ + private function resultSql(Result $result, string $sourceSql, array &$parameters): string + { + if ($result instanceof CopyResult) { + return $sourceSql; + } + if ($result instanceof SystemMissingResult) { + return 'NULL'; + } + if ($result instanceof LiteralResult) { + $parameters[] = $this->value($result->value); + return '?'; + } + throw new \LogicException('Unsupported preflighted recode result.'); + } + + private function replaceValueLabels( + ReplaceValueLabelsOperation $operation, + DatasetBinding $dataset, + VariableBinding $target, + ): void { + $statement = $this->statement( + 'SELECT link.value_label_set_id, value_set.dataset_id FROM variable_value_label_set link ' + . 'JOIN value_label_set value_set ON value_set.value_label_set_id = link.value_label_set_id ' + . 'WHERE link.variable_id = ?', + ); + $statement->execute([$target->variableId]); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + if (count($rows) > 1 || (isset($rows[0]['dataset_id']) && $rows[0]['dataset_id'] !== $dataset->datasetId)) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Value-label association is malformed.'); + } + + $setId = isset($rows[0]['value_label_set_id']) && is_string($rows[0]['value_label_set_id']) + ? $rows[0]['value_label_set_id'] + : null; + if ($setId !== null) { + $references = $this->statement('SELECT COUNT(*) FROM variable_value_label_set WHERE value_label_set_id = ?'); + $references->execute([$setId]); + if ((int) $references->fetchColumn() > 1) { + $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?')->execute([$target->variableId]); + $setId = null; + } else { + $this->statement('DELETE FROM value_label WHERE value_label_set_id = ?')->execute([$setId]); + } + } + if ($setId === null) { + $setId = NormativeCatalog::uuid(); + $this->statement('INSERT INTO value_label_set (value_label_set_id, dataset_id, name) VALUES (?, ?, ?)')->execute([ + $setId, + $dataset->datasetId, + null, + ]); + $this->statement('INSERT INTO variable_value_label_set (variable_id, value_label_set_id) VALUES (?, ?)')->execute([ + $target->variableId, + $setId, + ]); + } + + $insert = $this->statement( + 'INSERT INTO value_label ' + . '(value_label_id, value_label_set_id, ordinal, code_kind, numeric_code, string_code, label) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?)', + ); + foreach ($operation->labels as $index => $label) { + $numeric = $label->value instanceof Binary64Value; + $insert->execute([ + NormativeCatalog::uuid(), + $setId, + $index + 1, + $numeric ? 'numeric' : 'string', + $numeric ? $this->value($label->value) : null, + $numeric ? null : $this->value($label->value), + $label->label, + ]); + } + } + + private function value(TypedValue $value): float|string + { + return match (true) { + $value instanceof Binary64Value => $value->number(), + $value instanceof StringValue => $value->value, + default => throw new \LogicException('Unsupported preflighted typed value.'), + }; + } + + private function target(BoundOperation $bound): VariableBinding + { + if ($bound->target === null) { + throw new \LogicException('The bound operation requires a target.'); + } + return $bound->target; + } + + private function quote(string $identifier): string + { + return $this->connection->profile->quoteIdentifier($identifier); + } + + private function statement(string $sql): PDOStatement + { + $statement = $this->connection->pdo->prepare($sql); + if ($statement === false) { + throw new PDOException('Transformation SQL statement could not be prepared.'); + } + return $statement; + } +} diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php index 6db9237..5894a12 100644 --- a/src/Transformation/Execution/InPlaceTransformationExecutor.php +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -4,854 +4,83 @@ namespace OpenStatSpec\Transformation\Execution; -use OpenStatSpec\Core\Binary64; use OpenStatSpec\Core\DiagnosticCode; use OpenStatSpec\Core\UnsupportedOperation; -use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; -use OpenStatSpec\Sql\NormativeCatalog; -use OpenStatSpec\Sql\OperationJournal; -use OpenStatSpec\Transformation\Model\CreateVariableOperation; -use OpenStatSpec\Transformation\Model\DeleteVariableOperation; -use OpenStatSpec\Transformation\Model\Action\AssignValueAction; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\Action\SetMissingAction; -use OpenStatSpec\Transformation\Model\RecodeAction; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\ScalarValue; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\ExactValueSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\Selector\NumericRangeSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; -use OpenStatSpec\Transformation\Model\TransformationOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; -use OpenStatSpec\Transformation\Validation\PlanValidator; -use PDO; -use PDOException; -use PDOStatement; -use SPSS\Sav\Variable; +use OpenStatSpec\Transformation\Audit\TransformationAuditWriter; +use OpenStatSpec\Transformation\Plan\PlanCodec; use Throwable; -/** Executes a canonical plan against its registered wide table, in place. */ -final class InPlaceTransformationExecutor +/** Applies one official plan atomically to its existing logical dataset and wide table. */ +final readonly class InPlaceTransformationExecutor { - private readonly PlanValidator $validator; - - /** @var array> */ - private array $operationBindings = []; + private PlanPreflight $preflight; + private InPlaceOperationExecutor $operationExecutor; + private TransformationAuditWriter $auditWriter; + private PlanCodec $codec; public function __construct( - private readonly Connection $connection, - ?PlanValidator $validator = null, - private readonly ?DoltEvidenceReader $doltEvidenceReader = null, + private Connection $connection, + private ?DoltEvidenceReader $doltEvidenceReader = null, + ?TransformationAuditWriter $auditWriter = null, + ?PlanPreflight $preflight = null, + ?InPlaceOperationExecutor $operationExecutor = null, + ?PlanCodec $codec = null, ) { - $this->validator = $validator ?? new PlanValidator(); + $this->codec = $codec ?? new PlanCodec(); + $this->preflight = $preflight ?? new PlanPreflight($connection, $this->codec); + $this->operationExecutor = $operationExecutor ?? new InPlaceOperationExecutor($connection); + $this->auditWriter = $auditWriter ?? new TransformationAuditWriter($connection->pdo, $this->codec); } - public function execute(TransformationPlan $plan): ExecutionResult + public function execute(InPlaceApplyRequest $request): ExecutionResult { - $this->validator->assertValid($plan); - $this->connection->assertClaimedSupported(); if ($this->connection->pdo->inTransaction()) { throw new UnsupportedOperation( DiagnosticCode::UnsupportedOperation, 'An in-place transformation cannot start inside a caller-owned active transaction.', ); } - CatalogOwnership::assertReadyForUse($this->connection->pdo); - - $dataset = $this->resolveDataset($plan->datasetId()); - $variables = $this->resolveVariables($dataset); - $plannedVariables = $this->preflight($plan, $dataset, $variables); + $bound = $this->preflight->bind($request); $guard = $this->doltGuard(); $doltBefore = $guard?->beforeExecution(); - $journal = $this->existingJournal(); - $operationId = $journal?->start( - 'transform', - $dataset->datasetName, - null, - [], - $this->auditDetails($plan, $dataset, $doltBefore), - ); $doltAfter = null; + $transactionStarted = false; try { - if (!$this->connection->pdo->beginTransaction()) { - throw new PDOException('The SQL driver did not start the transformation transaction.'); - } - $created = []; - foreach ($plan->operations() as $operationIndex => $operation) { - $this->applyOperation($operation, $dataset, $plannedVariables, $created, $operationIndex); - } - $doltAfter = $doltBefore === null - ? null - : $guard->afterExecution($doltBefore); - if ($journal !== null && $operationId !== null) { - $journal->succeed($operationId, $dataset->datasetName, []); - } - if (!$this->connection->pdo->commit()) { - throw new PDOException('The SQL driver did not commit the transformation transaction.'); - } - } catch (Throwable $exception) { - if ($this->connection->pdo->inTransaction()) { + $this->connection->pdo->beginTransaction(); + $transactionStarted = true; + foreach ($bound->operations as $operation) { + $this->operationExecutor->execute($operation, $bound->dataset); + } + $doltAfter = $doltBefore === null ? null : $guard->afterExecution($doltBefore); + $applyId = $this->auditWriter->succeed( + $request, + $bound->dataset, + $this->connection->profileName, + $doltBefore, + $doltAfter, + ); + $this->connection->pdo->commit(); + $transactionStarted = false; + } catch (Throwable $failure) { + if ($transactionStarted) { $this->connection->pdo->rollBack(); } - if ($journal !== null && $operationId !== null) { - try { - $journal->fail($operationId, $dataset->datasetName, $exception); - } catch (Throwable) { - // Preserve the transformation failure; audit is best-effort after rollback. - } - } - throw $exception; + throw $failure; } return new ExecutionResult( - $dataset->datasetId, - $plan->hash(), - count($plan->operations()), - $operationId, + $bound->dataset->datasetId, + $this->codec->hash($request->plan), + count($request->plan->operations), + $applyId, $doltBefore, $doltAfter, ); } - private function resolveDataset(string $datasetId): DatasetBinding - { - $statement = $this->statement( - 'SELECT dataset_id, dataset_name, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', - ); - $statement->execute([$datasetId]); - $rows = $statement->fetchAll(PDO::FETCH_ASSOC); - if (count($rows) !== 1) { - throw $this->invalidCatalog('The transformation dataset_id resolves to exactly one normative dataset row.'); - } - $row = $rows[0]; - $table = $row['physical_table_name'] ?? null; - $schema = $row['physical_table_schema'] ?? null; - $name = $row['dataset_name'] ?? null; - if (($row['dataset_id'] ?? null) !== $datasetId - || !is_string($table) - || $table === '' - || ($schema !== null && (!is_string($schema) || $schema === '')) - || ($name !== null && !is_string($name)) - ) { - throw $this->invalidCatalog('The normative dataset row has malformed physical table metadata.'); - } - - return new DatasetBinding($datasetId, $name, $schema, $table); - } - - /** @return array */ - private function resolveVariables(DatasetBinding $dataset): array - { - $statement = $this->statement( - 'SELECT variable_id, source_name, physical_name, storage_kind, source_ordinal, declared_string_width ' - . 'FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', - ); - $statement->execute([$dataset->datasetId]); - $variables = []; - $physicalNames = []; - foreach ($statement->fetchAll(PDO::FETCH_ASSOC) as $row) { - $id = $row['variable_id'] ?? null; - $source = $row['source_name'] ?? null; - $physical = $row['physical_name'] ?? null; - $kind = $row['storage_kind'] ?? null; - $ordinal = filter_var($row['source_ordinal'] ?? null, FILTER_VALIDATE_INT); - $declaredStringWidth = $row['declared_string_width'] === null - ? null - : filter_var($row['declared_string_width'], FILTER_VALIDATE_INT); - if (!is_string($id) || $id === '' - || !is_string($source) || $source === '' - || !is_string($physical) || $physical === '' - || !in_array($kind, ['numeric', 'string'], true) - || !is_int($ordinal) || $ordinal < 1 - || ($kind === 'string' && (!is_int($declaredStringWidth) || $declaredStringWidth < 1)) - || isset($variables[$source]) || isset($physicalNames[$physical]) - ) { - throw $this->invalidCatalog('The normative variable mapping is malformed or ambiguous.'); - } - $variables[$source] = new VariableBinding( - $id, - $source, - $physical, - $kind, - $ordinal, - is_int($declaredStringWidth) ? $declaredStringWidth : null, - ); - $physicalNames[$physical] = true; - } - if ($variables === []) { - throw $this->invalidCatalog('The registered dataset has no normative variable mappings.'); - } - - $columns = implode(', ', array_map( - fn(VariableBinding $variable): string => $this->quote($variable->physicalName), - array_values($variables), - )); - try { - $this->connection->pdo->query( - 'SELECT ' . $columns . ' FROM ' . $this->qualifiedTable($dataset) . ' WHERE 1 = 0', - ); - } catch (PDOException $exception) { - throw $this->invalidCatalog('The catalogued physical wide table or variable mapping is not readable: ' . $exception->getMessage()); - } - - return $variables; - } - - /** - * @param array $variables - * @return array - */ - private function preflight(TransformationPlan $plan, DatasetBinding $dataset, array $variables): array - { - $physicalColumnSlots = $this->connection->profileName === 'postgresql' - ? $this->postgresqlPhysicalColumnSlots($dataset) - : null; - $used = ['__case_ordinal' => true]; - $nextOrdinal = 1; - foreach ($variables as $variable) { - $used[$variable->physicalName] = true; - $nextOrdinal = max($nextOrdinal, $variable->sourceOrdinal + 1); - } - - $active = array_fill_keys(array_keys($variables), true); - $this->operationBindings = []; - foreach ($plan->operations() as $operationIndex => $operation) { - if ($operation instanceof CreateVariableOperation) { - if (isset($variables[$operation->targetVariable()]) && isset($active[$operation->targetVariable()])) { - throw $this->invalidCatalog(sprintf( - 'Create variable "%s" collides with an existing catalog variable.', - $operation->targetVariable(), - )); - } - $this->assertCanAlterSchema(count($active), true, $physicalColumnSlots); - if ($physicalColumnSlots !== null) { - ++$physicalColumnSlots; - } - $physical = $this->connection->profile->physicalIdentifier($operation->targetVariable(), $used); - $variables[$operation->targetVariable()] = new VariableBinding( - NormativeCatalog::uuid(), - $operation->targetVariable(), - $physical, - $operation->storageKind(), - $nextOrdinal++, - $operation->declaredStringWidth(), - false, - ); - $used[$physical] = true; - $active[$operation->targetVariable()] = true; - $this->operationBindings[$operationIndex] = ['target' => $variables[$operation->targetVariable()]]; - continue; - } - if ($operation instanceof DeleteVariableOperation) { - $variableName = $operation->targetVariable(); - if (!isset($active[$variableName])) { - throw $this->invalidCatalog(sprintf( - 'Delete variable "%s" is not registered for dataset_id %s.', - $variableName, - $plan->datasetId(), - )); - } - if (count($active) === 1 && !$this->hasFutureCreateOperation($plan->operations(), $operationIndex)) { - throw $this->invalidCatalog('A transformation cannot delete the final dataset variable.'); - } - $this->assertCanAlterSchema(count($active), false); - $this->operationBindings[$operationIndex] = ['target' => $variables[$variableName]]; - unset($active[$variableName]); - continue; - } - if ($operation instanceof RecodeOperation) { - $source = $variables[$operation->sourceVariable()] ?? null; - if ($source === null || !isset($active[$operation->sourceVariable()])) { - throw $this->invalidCatalog(sprintf( - 'Recode source variable "%s" is not registered for dataset_id %s.', - $operation->sourceVariable(), - $plan->datasetId(), - )); - } - $target = $variables[$operation->targetVariable()] ?? null; - if ($target !== null && !isset($active[$operation->targetVariable()])) { - $target = null; - } - if ($target === null) { - $this->assertCanCreateTarget($source, count($active), $physicalColumnSlots); - if ($physicalColumnSlots !== null) { - ++$physicalColumnSlots; - } - $physical = $this->connection->profile->physicalIdentifier($operation->targetVariable(), $used); - $target = new VariableBinding( - NormativeCatalog::uuid(), - $operation->targetVariable(), - $physical, - $source->storageKind, - $nextOrdinal++, - null, - false, - ); - $variables[$operation->targetVariable()] = $target; - $used[$physical] = true; - $active[$operation->targetVariable()] = true; - } - $this->assertRecodeKinds($operation, $source, $target); - $this->operationBindings[$operationIndex] = ['source' => $source, 'target' => $target]; - continue; - } - - $target = $variables[$operation->targetVariable()] ?? null; - if ($target === null || !isset($active[$operation->targetVariable()])) { - throw $this->invalidCatalog(sprintf( - 'Metadata target variable "%s" is not registered for dataset_id %s.', - $operation->targetVariable(), - $plan->datasetId(), - )); - } - $this->operationBindings[$operationIndex] = ['target' => $target]; - if ($operation instanceof SetValueLabelsOperation) { - foreach ($operation->labels() as $label) { - $this->assertScalarKind($label->value(), $target, 'value label'); - } - } - } - - return $variables; - } - - /** @param list $operations */ - private function hasFutureCreateOperation(array $operations, int $operationIndex): bool - { - foreach ($operations as $futureIndex => $operation) { - if ($futureIndex > $operationIndex && $operation instanceof CreateVariableOperation) { - return true; - } - } - return false; - } - private function assertCanAlterSchema(int $registeredVariableCount, bool $creation, ?int $physicalColumnSlots = null): void - { - if (!in_array($this->connection->profileName, ['sqlite', 'postgresql'], true) - || !$this->connection->profile->ddlAtomic() - ) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - sprintf( - '%s cannot atomically alter an existing wide table for an in-place variable transformation.', - $this->connection->profileName, - ), - ); - } - if (!$creation) { - if ($this->connection->profileName === 'sqlite' - && version_compare($this->connection->serverVersion, '3.35.0', '<') - ) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - 'SQLite versions below 3.35.0 do not support DROP COLUMN for DELETE VARIABLES.', - ); - } - - return; - } - - $maximum = $this->connection->profile->effectiveMaximumSourceVariables($this->connection->pdo); - if ($this->connection->profileName === 'postgresql' - && $physicalColumnSlots !== null - && $physicalColumnSlots >= $maximum + 1 - ) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - sprintf( - 'PostgreSQL cannot add a source variable because the wide table already uses %d physical PostgreSQL column slots, including dropped columns; the table limit is %d.', - $physicalColumnSlots, - $maximum + 1, - ), - ); - } - if ($registeredVariableCount >= $maximum) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - sprintf( - '%s supports at most %d source variables in one OpenStatSpec wide table.', - $this->connection->profileName, - $maximum, - ), - ); - } - } - private function assertCanCreateTarget(VariableBinding $source, int $registeredVariableCount, ?int $physicalColumnSlots = null): void - { - if ($source->storageKind === 'string') { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - 'A new string target requires an explicit declared_string_width; register the target variable first.', - ); - } - if (!in_array($this->connection->profileName, ['sqlite', 'postgresql'], true) - || !$this->connection->profile->ddlAtomic() - ) { - throw new UnsupportedOperation( - DiagnosticCode::TargetCapabilityExceeded, - sprintf( - '%s cannot atomically add a new INTO target to an existing wide table; register the target variable first.', - $this->connection->profileName, - ), - ); - } - - $this->assertCanAlterSchema($registeredVariableCount, true, $physicalColumnSlots); - } - - private function postgresqlPhysicalColumnSlots(DatasetBinding $dataset): int - { - $statement = $this->statement('SELECT COUNT(*) FROM pg_attribute WHERE attrelid = to_regclass(?) AND attnum > 0'); - $statement->execute([$this->qualifiedTable($dataset)]); - $slots = filter_var($statement->fetchColumn(), FILTER_VALIDATE_INT); - if (!is_int($slots) || $slots < 1) { - throw $this->invalidCatalog('The PostgreSQL wide table has no physical column slots.'); - } - - return $slots; - } - - private function assertRecodeKinds( - RecodeOperation $operation, - VariableBinding $source, - VariableBinding $target, - ): void { - foreach ($operation->rules() as $rule) { - $selector = $rule->selector(); - if ($selector instanceof ExactValueSelector) { - $this->assertScalarKind($selector->value(), $source, 'exact recode selector'); - } elseif ($selector instanceof NumericRangeSelector && $source->storageKind !== 'numeric') { - throw $this->invalidCatalog('A numeric range selector cannot read a string variable.'); - } - - $action = $rule->action(); - if ($action instanceof AssignValueAction) { - $this->assertScalarKind($action->value(), $target, 'recode assignment'); - } elseif ($action instanceof CopySourceAction && $source->storageKind !== $target->storageKind) { - throw $this->invalidCatalog('Copy-source recoding requires source and target variables with the same storage kind.'); - } elseif ($action instanceof CopySourceAction - && $source->storageKind === 'string' - && $this->stringWidth($source) > $this->stringWidth($target) - ) { - throw $this->invalidCatalog(sprintf( - 'Copy-source recoding cannot copy string width %d into variable "%s" width %d.', - $this->stringWidth($source), - $target->sourceName, - $this->stringWidth($target), - )); - } elseif ($action instanceof SetMissingAction && $target->storageKind === 'string') { - throw $this->invalidCatalog(sprintf( - 'System-missing recoding is not representable for string variable "%s".', - $target->sourceName, - )); - } - } - } - - private function assertScalarKind(ScalarValue $value, VariableBinding $variable, string $context): void - { - $kind = $value->type() === 'number' ? 'numeric' : 'string'; - if ($kind !== $variable->storageKind) { - throw $this->invalidCatalog(sprintf( - 'The %s type does not match variable "%s" storage kind %s.', - $context, - $variable->sourceName, - $variable->storageKind, - )); - } - if ($kind === 'string' && strlen($value->stringValue()) > $this->stringWidth($variable)) { - throw $this->invalidCatalog(sprintf( - 'The %s exceeds variable "%s" declared string width %d bytes.', - $context, - $variable->sourceName, - $this->stringWidth($variable), - )); - } - } - - private function stringWidth(VariableBinding $variable): int - { - if ($variable->storageKind !== 'string' - || $variable->declaredStringWidth === null - || $variable->declaredStringWidth < 1 - ) { - throw $this->invalidCatalog(sprintf( - 'String variable "%s" requires a positive declared_string_width.', - $variable->sourceName, - )); - } - - return $variable->declaredStringWidth; - } - - /** - * @param array $variables - * @param array $created - */ - private function applyOperation( - TransformationOperation $operation, - DatasetBinding $dataset, - array $variables, - array &$created, - int $operationIndex, - ): void { - $bindings = $this->operationBindings[$operationIndex] ?? []; - if ($operation instanceof CreateVariableOperation) { - $target = $bindings['target'] ?? $variables[$operation->targetVariable()]; - $this->ensureTargetExists($dataset, $target, $created); - return; - } - if ($operation instanceof DeleteVariableOperation) { - $target = $bindings['target'] ?? $variables[$operation->targetVariable()]; - $this->deleteVariable($dataset, $target); - unset($created[$target->sourceName]); - return; - } - $target = $bindings['target'] ?? $variables[$operation->targetVariable()]; - if ($operation instanceof RecodeOperation) { - $this->ensureTargetExists($dataset, $target, $created); - $this->applyRecode($operation, $dataset, $bindings['source'] ?? $variables[$operation->sourceVariable()], $target); - } elseif ($operation instanceof SetVariableLabelOperation) { - $this->applyVariableLabel($operation, $dataset, $target); - } elseif ($operation instanceof SetValueLabelsOperation) { - $this->applyValueLabels($operation, $dataset, $target); - } - } - - /** @param array $created */ - private function ensureTargetExists(DatasetBinding $dataset, VariableBinding $target, array &$created): void - { - if ($target->persisted || isset($created[$target->sourceName])) { - return; - } - $type = $target->storageKind === 'numeric' - ? $this->connection->profile->numericType() - : $this->connection->profile->textType(); - $columnDefinition = $target->storageKind === 'string' - ? $type . " NOT NULL DEFAULT ''" - : $type . ' NULL'; - $this->connection->pdo->exec(sprintf( - 'ALTER TABLE %s ADD COLUMN %s %s', - $this->qualifiedTable($dataset), - $this->quote($target->physicalName), - $columnDefinition, - )); - $formatFamily = $target->storageKind === 'string' ? Variable::FORMAT_TYPE_A : 5; - $formatWidth = $target->storageKind === 'string' - ? min(255, $this->stringWidth($target)) - : 8; - $this->statement( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width, ' - . 'print_format_family, print_format_width, print_format_decimals, ' - . 'write_format_family, write_format_width, write_format_decimals) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', - )->execute([ - $target->variableId, - $dataset->datasetId, - $target->sourceOrdinal, - $target->sourceName, - $target->physicalName, - $target->storageKind, - $target->declaredStringWidth, - $formatFamily, - $formatWidth, - 0, - $formatFamily, - $formatWidth, - 0, - ]); - $created[$target->sourceName] = true; - } - - private function deleteVariable(DatasetBinding $dataset, VariableBinding $target): void - { - $this->connection->pdo->exec(sprintf( - 'ALTER TABLE %s DROP COLUMN %s', - $this->qualifiedTable($dataset), - $this->quote($target->physicalName), - )); - $association = $this->statement( - 'SELECT vvls.value_label_set_id, vls.dataset_id FROM variable_value_label_set vvls ' - . 'JOIN value_label_set vls ON vls.value_label_set_id = vvls.value_label_set_id ' - . 'WHERE vvls.variable_id = ?', - ); - $association->execute([$target->variableId]); - $rows = $association->fetchAll(PDO::FETCH_ASSOC); - if (count($rows) > 1 || (isset($rows[0]['dataset_id']) && $rows[0]['dataset_id'] !== $dataset->datasetId)) { - throw $this->invalidCatalog('The variable value-label association is malformed or crosses dataset identity.'); - } - $setId = isset($rows[0]['value_label_set_id']) && is_string($rows[0]['value_label_set_id']) - ? $rows[0]['value_label_set_id'] - : null; - - foreach ([ - 'DELETE FROM dataset_weight_variable WHERE variable_id = ?', - 'DELETE FROM variable_set_member WHERE variable_id = ?', - 'DELETE FROM multiple_response_member WHERE variable_id = ?', - 'DELETE FROM variable_attribute WHERE variable_id = ?', - 'DELETE FROM missing_rule WHERE variable_id = ?', - 'DELETE FROM variable_value_label_set WHERE variable_id = ?', - ] as $sql) { - $this->statement($sql)->execute([$target->variableId]); - } - $this->statement('DELETE FROM variable WHERE variable_id = ? AND dataset_id = ?')->execute([ - $target->variableId, - $dataset->datasetId, - ]); - - $orphanedSets = $this->statement( - 'SELECT multiple_response_set_id FROM multiple_response_set WHERE dataset_id = ? ' - . 'AND NOT EXISTS (SELECT 1 FROM multiple_response_member WHERE multiple_response_set_id = multiple_response_set.multiple_response_set_id)', - ); - $orphanedSets->execute([$dataset->datasetId]); - foreach ($orphanedSets->fetchAll(PDO::FETCH_COLUMN) as $orphanedSetId) { - if (!is_string($orphanedSetId)) { - continue; - } - $this->statement('DELETE FROM multiple_response_set WHERE multiple_response_set_id = ? AND dataset_id = ?')->execute([$orphanedSetId, $dataset->datasetId]); - } - - $orphanedVariableSets = $this->statement( - 'SELECT variable_set_id FROM variable_set WHERE dataset_id = ? ' - . 'AND NOT EXISTS (SELECT 1 FROM variable_set_member WHERE variable_set_id = variable_set.variable_set_id)', - ); - $orphanedVariableSets->execute([$dataset->datasetId]); - foreach ($orphanedVariableSets->fetchAll(PDO::FETCH_COLUMN) as $orphanedVariableSetId) { - if (!is_string($orphanedVariableSetId)) { - continue; - } - $this->statement('DELETE FROM variable_set WHERE variable_set_id = ? AND dataset_id = ?')->execute([$orphanedVariableSetId, $dataset->datasetId]); - } - - if ($setId === null) { - return; - } - $references = $this->statement( - 'SELECT COUNT(*) FROM variable_value_label_set WHERE value_label_set_id = ?', - ); - $references->execute([$setId]); - if ((int) $references->fetchColumn() === 0) { - $this->statement('DELETE FROM value_label WHERE value_label_set_id = ?')->execute([$setId]); - $this->statement('DELETE FROM value_label_set WHERE value_label_set_id = ? AND dataset_id = ?')->execute([ - $setId, - $dataset->datasetId, - ]); - } - } - private function applyRecode( - RecodeOperation $operation, - DatasetBinding $dataset, - VariableBinding $source, - VariableBinding $target, - ): void { - $sourceSql = $this->quote($source->physicalName); - $when = []; - $parameters = []; - $else = null; - foreach ($operation->rules() as $rule) { - $selector = $rule->selector(); - if ($selector instanceof ElseSelector) { - $else = $this->actionSql($rule->action(), $sourceSql, $parameters); - continue; - } - $condition = match (true) { - $selector instanceof ExactValueSelector => $this->exactCondition($sourceSql, $selector, $parameters), - $selector instanceof NumericRangeSelector => $this->rangeCondition($sourceSql, $selector, $parameters), - $selector instanceof MissingValueSelector => $sourceSql . ' IS NULL', - default => throw $this->invalidCatalog('The recode selector is not executable.'), - }; - $actionSql = $this->actionSql($rule->action(), $sourceSql, $parameters); - $when[] = 'WHEN ' . $condition . ' THEN ' . $actionSql; - } - if ($else === null) { - throw $this->invalidCatalog('A recode requires one explicit final else action.'); - } - if ($when === []) { - $this->statement(sprintf( - 'UPDATE %s SET %s = %s', - $this->qualifiedTable($dataset), - $this->quote($target->physicalName), - $else, - ))->execute($parameters); - - return; - } - $sql = sprintf( - 'UPDATE %s SET %s = CASE %s ELSE %s END', - $this->qualifiedTable($dataset), - $this->quote($target->physicalName), - implode(' ', $when), - $else, - ); - $this->statement($sql)->execute($parameters); - } - - /** @param list $parameters */ - private function exactCondition(string $sourceSql, ExactValueSelector $selector, array &$parameters): string - { - $parameters[] = $this->boundScalar($selector->value()); - - return $this->connection->profile->exactValueCondition( - $sourceSql, - $selector->value()->type() === 'string', - ); - } - - /** @param list $parameters */ - private function rangeCondition(string $sourceSql, NumericRangeSelector $selector, array &$parameters): string - { - $parts = []; - if ($selector->lower() !== null) { - $parts[] = $sourceSql . ' >= ?'; - $parameters[] = Binary64::encode($selector->lower()); - } - if ($selector->upper() !== null) { - $parts[] = $sourceSql . ' <= ?'; - $parameters[] = Binary64::encode($selector->upper()); - } - - return implode(' AND ', $parts); - } - - /** @param list $parameters */ - private function actionSql(RecodeAction $action, string $sourceSql, array &$parameters): string - { - if ($action instanceof AssignValueAction) { - $parameters[] = $this->boundScalar($action->value()); - - return '?'; - } - if ($action instanceof SetMissingAction) { - return 'NULL'; - } - if ($action instanceof CopySourceAction) { - return $sourceSql; - } - - throw $this->invalidCatalog('The recode action is not executable.'); - } - - private function applyVariableLabel( - SetVariableLabelOperation $operation, - DatasetBinding $dataset, - VariableBinding $target, - ): void { - $this->statement('UPDATE variable SET variable_label = ? WHERE variable_id = ? AND dataset_id = ?')->execute([ - $operation->label(), - $target->variableId, - $dataset->datasetId, - ]); - } - - private function applyValueLabels( - SetValueLabelsOperation $operation, - DatasetBinding $dataset, - VariableBinding $target, - ): void { - $statement = $this->statement( - 'SELECT vvls.value_label_set_id, vls.dataset_id FROM variable_value_label_set vvls ' - . 'JOIN value_label_set vls ON vls.value_label_set_id = vvls.value_label_set_id ' - . 'WHERE vvls.variable_id = ?', - ); - $statement->execute([$target->variableId]); - $rows = $statement->fetchAll(PDO::FETCH_ASSOC); - if (count($rows) > 1 || (isset($rows[0]['dataset_id']) && $rows[0]['dataset_id'] !== $dataset->datasetId)) { - throw $this->invalidCatalog('The variable value-label association is malformed or crosses dataset identity.'); - } - - $setId = isset($rows[0]['value_label_set_id']) && is_string($rows[0]['value_label_set_id']) - ? $rows[0]['value_label_set_id'] - : null; - if ($setId !== null) { - $references = $this->statement( - 'SELECT COUNT(*) FROM variable_value_label_set WHERE value_label_set_id = ?', - ); - $references->execute([$setId]); - $referenceCount = (int) $references->fetchColumn(); - if ($referenceCount > 1) { - $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?')->execute([$target->variableId]); - $setId = null; - } else { - $this->statement('DELETE FROM value_label WHERE value_label_set_id = ?')->execute([$setId]); - } - } - - if ($operation->labels() === []) { - if ($setId !== null) { - $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?')->execute([$target->variableId]); - $this->statement('DELETE FROM value_label_set WHERE value_label_set_id = ?')->execute([$setId]); - } - - return; - } - - if ($setId === null) { - $setId = NormativeCatalog::uuid(); - $this->statement('INSERT INTO value_label_set (value_label_set_id, dataset_id, name) VALUES (?, ?, ?)')->execute([ - $setId, - $dataset->datasetId, - null, - ]); - $this->statement( - 'INSERT INTO variable_value_label_set (variable_id, value_label_set_id) VALUES (?, ?)', - )->execute([$target->variableId, $setId]); - } - - $insert = $this->statement( - 'INSERT INTO value_label ' - . '(value_label_id, value_label_set_id, ordinal, code_kind, numeric_code, string_code, label) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - ); - foreach ($operation->labels() as $index => $label) { - $value = $label->value(); - $insert->execute([ - NormativeCatalog::uuid(), - $setId, - $index + 1, - $value->type() === 'number' ? 'numeric' : 'string', - $value->type() === 'number' ? Binary64::encode($value->numberValue()) : null, - $value->type() === 'string' ? $value->stringValue() : null, - $label->label(), - ]); - } - } - - private function boundScalar(ScalarValue $value): string - { - return $value->type() === 'number' - ? Binary64::encode($value->numberValue()) - : $value->stringValue(); - } - - private function qualifiedTable(DatasetBinding $dataset): string - { - $table = $this->quote($dataset->table); - - return $dataset->schema === null ? $table : $this->quote($dataset->schema) . '.' . $table; - } - - private function quote(string $identifier): string - { - return $this->connection->profile->quoteIdentifier($identifier); - } - - private function statement(string $sql): PDOStatement - { - $statement = $this->connection->pdo->prepare($sql); - if ($statement === false) { - throw new PDOException('The transformation SQL statement could not be prepared.'); - } - - return $statement; - } - private function doltGuard(): ?DoltGuard { if ($this->connection->profileName !== 'dolt') { @@ -860,37 +89,4 @@ private function doltGuard(): ?DoltGuard return new DoltGuard($this->doltEvidenceReader ?? new PdoDoltEvidenceReader($this->connection->pdo)); } - - private function existingJournal(): ?OperationJournal - { - try { - $this->connection->pdo->query('SELECT operation_id FROM operation_catalog WHERE 1 = 0'); - $this->connection->pdo->query('SELECT operation_id FROM fidelity_event_catalog WHERE 1 = 0'); - $this->connection->pdo->query('SELECT operation_id FROM operation WHERE 1 = 0'); - } catch (PDOException) { - return null; - } - - return new OperationJournal($this->connection->pdo); - } - - /** @return array */ - private function auditDetails( - TransformationPlan $plan, - DatasetBinding $dataset, - ?DoltEvidence $doltBefore, - ): array { - return [ - 'plan_hash' => $plan->hash(), - 'dataset_id' => $dataset->datasetId, - 'mode' => 'in_place', - 'dolt_branch_before' => $doltBefore?->branch(), - 'dolt_head_before' => $doltBefore?->head(), - ]; - } - - private function invalidCatalog(string $message): UnsupportedOperation - { - return new UnsupportedOperation(DiagnosticCode::InvalidSourceDataset, $message); - } } diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php new file mode 100644 index 0000000..2bc54d6 --- /dev/null +++ b/src/Transformation/Execution/PlanPreflight.php @@ -0,0 +1,573 @@ +codec = $codec ?? new PlanCodec(); + } + + public function bind(InPlaceApplyRequest $request): BoundPlan + { + // Re-decode the public model so directly constructed objects receive the + // same complete structural and semantic validation as decoded plans. + $this->codec->fromArray($request->plan->canonicalArray()); + $this->connection->assertClaimedSupported(); + CatalogOwnership::assertReadyForUse($this->connection->pdo); + $this->assertAuditReady(); + + $dataset = $this->resolveDataset($request->datasetId); + $variables = $this->resolveVariables($dataset); + $physicalColumns = $this->physicalColumns($dataset); + $this->assertPhysicalBindings($dataset, $variables, $physicalColumns); + + $usedPhysical = array_fill_keys($physicalColumns, true); + $nextOrdinal = 1; + foreach ($variables as $variable) { + $nextOrdinal = max($nextOrdinal, $variable->sourceOrdinal + 1); + } + $postgresqlSlots = $this->connection->profileName === 'postgresql' + ? $this->postgresqlPhysicalColumnSlots($dataset) + : null; + $bound = []; + + foreach ($request->plan->operations as $index => $operation) { + $path = '$.operations[' . $index . ']'; + $schemaBefore = new BoundSchema($variables); + [$source, $target] = $this->bindOperation( + $operation, + $schemaBefore, + $path, + $variables, + $usedPhysical, + $nextOrdinal, + $postgresqlSlots, + ); + $bound[] = new BoundOperation($operation, $schemaBefore, $source, $target); + if ($target !== null && !$target->persisted) { + $variables[$target->sourceName] = new VariableBinding( + $target->variableId, + $target->sourceName, + $target->physicalName, + $target->storageKind, + $target->sourceOrdinal, + $target->declaredStringWidth, + ); + $usedPhysical[$target->physicalName] = true; + ++$nextOrdinal; + if ($postgresqlSlots !== null) { + ++$postgresqlSlots; + } + } + } + + return new BoundPlan($request->plan, $dataset, $bound, new BoundSchema($variables)); + } + + /** + * @param array $variables + * @param array $usedPhysical + * @return array{VariableBinding|null, VariableBinding|null} + */ + private function bindOperation( + Operation $operation, + BoundSchema $schema, + string $path, + array $variables, + array $usedPhysical, + int $nextOrdinal, + ?int $postgresqlSlots, + ): array { + if ($operation instanceof AssignOperation) { + $this->assertNumericOperand($operation->value, $schema, $path . '.value'); + $target = $this->bindNumericTarget( + $operation->target, + $operation->targetMode, + $path . '.target', + $variables, + $usedPhysical, + $nextOrdinal, + $postgresqlSlots, + ); + return [null, $target]; + } + + if ($operation instanceof ConditionalAssignOperation) { + $target = $variables[$operation->target] ?? null; + if ($target === null) { + throw TransformationFailure::at( + 'conditional_target_missing', + $path . '.target', + 'Conditional assignment requires an existing target.', + ); + } + $this->assertNumericVariable($target, $path . '.target', 'Conditional assignment target must be numeric.'); + $this->assertPredicate($operation->condition, $schema, $path . '.condition'); + $this->assertNumericOperand($operation->value, $schema, $path . '.value'); + return [null, $target]; + } + + if ($operation instanceof RecodeOperation) { + $source = $variables[$operation->source] ?? null; + if ($source === null) { + throw TransformationFailure::at('unknown_variable', $path . '.source', 'Recode source is not registered.'); + } + $target = $this->bindRecodeTarget( + $operation, + $source, + $path, + $variables, + $usedPhysical, + $nextOrdinal, + $postgresqlSlots, + ); + $this->assertRecode($operation, $source, $target, $path); + return [$source, $target]; + } + + if ($operation instanceof SetVariableLabelOperation) { + return [null, $schema->variable($operation->variable, $path . '.variable')]; + } + + if ($operation instanceof ReplaceValueLabelsOperation) { + $target = $schema->variable($operation->variable, $path . '.variable'); + foreach ($operation->labels as $index => $label) { + $this->assertValueKind($label->value, $target, $path . '.labels[' . $index . '].value'); + } + return [null, $target]; + } + + if ($operation instanceof SetFormatOperation) { + $target = $schema->variable($operation->variable, $path . '.variable'); + $this->assertNumericVariable($target, $path . '.variable', 'F format requires a numeric variable.'); + if ($operation->family !== 'F' + || $operation->width < 1 + || $operation->width > 40 + || $operation->decimals < 0 + || $operation->decimals > 16 + || ($operation->decimals > 0 && $operation->width < $operation->decimals + 2) + ) { + throw TransformationFailure::at('invalid_format', $path, 'Invalid SPSS F format.'); + } + return [null, $target]; + } + + if ($operation instanceof SetMeasurementLevelOperation) { + return [null, $schema->variable($operation->variable, $path . '.variable')]; + } + + if ($operation instanceof ExecuteOperation) { + return [null, null]; + } + + throw TransformationFailure::at('plan_schema_invalid', $path, 'Unsupported official operation.'); + } + + /** + * @param array $variables + * @param array $usedPhysical + */ + private function bindNumericTarget( + string $name, + TargetMode $mode, + string $path, + array $variables, + array $usedPhysical, + int $nextOrdinal, + ?int $postgresqlSlots, + ): VariableBinding { + $existing = $variables[$name] ?? null; + if ($mode === TargetMode::Replace) { + if ($existing === null) { + throw TransformationFailure::at('unknown_variable', $path, 'Replace target is not registered.'); + } + $this->assertNumericVariable($existing, $path, 'Assignment target must be numeric.'); + return $existing; + } + if ($existing !== null) { + throw TransformationFailure::at('target_already_exists', $path, 'Create target is already registered.'); + } + + $this->assertCreateCapability(count($variables), $postgresqlSlots, $path); + $physical = $this->connection->profile->physicalIdentifier($name, $usedPhysical); + return new VariableBinding(NormativeCatalog::uuid(), $name, $physical, 'numeric', $nextOrdinal, null, false); + } + + /** + * @param array $variables + * @param array $usedPhysical + */ + private function bindRecodeTarget( + RecodeOperation $operation, + VariableBinding $source, + string $path, + array $variables, + array $usedPhysical, + int $nextOrdinal, + ?int $postgresqlSlots, + ): VariableBinding { + if (str_starts_with($operation->target, '__')) { + throw TransformationFailure::at('reserved_target_name', $path . '.target', 'Target name is reserved.'); + } + $existing = $variables[$operation->target] ?? null; + if ($operation->targetMode === TargetMode::Replace) { + if ($existing === null) { + throw TransformationFailure::at('unknown_variable', $path . '.target', 'Replace target is not registered.'); + } + if ($existing->variableId !== $source->variableId) { + throw TransformationFailure::at( + 'type_mismatch', + $path . '.target', + 'Recode replace requires source and target to identify the same variable.', + ); + } + return $existing; + } + if ($existing !== null) { + throw TransformationFailure::at('target_already_exists', $path . '.target', 'Create target is already registered.'); + } + + $resultKind = $this->recodeResultKind($operation, $source, $path); + if ($resultKind === 'string') { + throw TransformationFailure::at( + 'string_target_requires_declaration', + $path . '.target', + 'A new string target requires separate explicit provisioning.', + ); + } + $this->assertCreateCapability(count($variables), $postgresqlSlots, $path . '.target_mode'); + $physical = $this->connection->profile->physicalIdentifier($operation->target, $usedPhysical); + return new VariableBinding(NormativeCatalog::uuid(), $operation->target, $physical, 'numeric', $nextOrdinal, null, false); + } + + private function assertRecode( + RecodeOperation $operation, + VariableBinding $source, + VariableBinding $target, + string $path, + ): void { + foreach ($operation->rules as $index => $rule) { + $match = $rule->match; + if ($match instanceof ExactMatch) { + foreach ($match->values as $valueIndex => $value) { + $this->assertValueKind($value, $source, $path . '.rules[' . $index . '].match.values[' . $valueIndex . ']'); + } + } elseif ($match instanceof RangeMatch) { + $this->assertNumericVariable($source, $path . '.rules[' . $index . '].match', 'Numeric range requires a numeric source.'); + } elseif ($match instanceof SystemMissingMatch && !$source->isNumeric()) { + throw TransformationFailure::at('system_missing_for_string', $path . '.rules[' . $index . '].match', 'String variables have no system-missing state.'); + } + } + + $kind = $this->recodeResultKind($operation, $source, $path); + if ($kind !== $target->storageKind) { + throw TransformationFailure::at('type_mismatch', $path . '.target', 'Recode result type does not match the target storage kind.'); + } + } + + private function recodeResultKind(RecodeOperation $operation, VariableBinding $source, string $path): string + { + $results = array_map(static fn(RecodeRule $rule): Result => $rule->result, $operation->rules); + $results[] = $operation->unmatched; + $kinds = []; + foreach ($results as $index => $result) { + $kind = match (true) { + $result instanceof CopyResult => $source->storageKind, + $result instanceof SystemMissingResult => 'numeric', + $result instanceof LiteralResult => $this->valueKind($result->value), + default => throw TransformationFailure::at('plan_schema_invalid', $path, 'Unsupported recode result.'), + }; + if ($result instanceof SystemMissingResult && !$source->isNumeric() && $operation->targetMode === TargetMode::Replace) { + throw TransformationFailure::at('system_missing_for_string', $path . '.results[' . $index . ']', 'String variables have no system-missing result.'); + } + $kinds[$kind] = true; + } + if (count($kinds) !== 1) { + throw TransformationFailure::at('mixed_result_types', $path, 'Recode results must have one storage kind.'); + } + + return (string) array_key_first($kinds); + } + + private function assertPredicate(Predicate $predicate, BoundSchema $schema, string $path): void + { + if ($predicate instanceof Comparison) { + $this->assertNumericOperand($predicate->left, $schema, $path . '.left'); + $this->assertNumericOperand($predicate->right, $schema, $path . '.right'); + return; + } + if ($predicate instanceof BooleanPredicate) { + foreach ($predicate->operands as $index => $child) { + $this->assertPredicate($child, $schema, $path . '.operands[' . $index . ']'); + } + return; + } + throw TransformationFailure::at('plan_schema_invalid', $path, 'Unsupported predicate.'); + } + + private function assertNumericOperand(Operand $operand, BoundSchema $schema, string $path): void + { + if ($operand instanceof LiteralOperand) { + if (!$operand->value instanceof Binary64Value) { + throw TransformationFailure::at('expression_type_unsupported', $path, 'Only numeric operands are executable.'); + } + return; + } + if ($operand instanceof VariableOperand) { + $variable = $schema->variable($operand->variable, $path . '.variable'); + if (!$variable->isNumeric()) { + throw TransformationFailure::at('expression_type_unsupported', $path, 'Only numeric operands are executable.'); + } + return; + } + throw TransformationFailure::at('plan_schema_invalid', $path, 'Unsupported operand.'); + } + + private function assertNumericVariable(VariableBinding $variable, string $path, string $message): void + { + if (!$variable->isNumeric()) { + throw TransformationFailure::at('type_mismatch', $path, $message); + } + } + + private function assertValueKind(TypedValue $value, VariableBinding $variable, string $path): void + { + if ($this->valueKind($value) !== $variable->storageKind) { + throw TransformationFailure::at('type_mismatch', $path, 'Typed value does not match variable storage kind.'); + } + if ($value instanceof StringValue + && ($variable->declaredStringWidth === null || strlen($value->value) > $variable->declaredStringWidth) + ) { + throw TransformationFailure::at('type_mismatch', $path, 'String value exceeds the target declared width.'); + } + } + + private function valueKind(TypedValue $value): string + { + return $value instanceof Binary64Value ? 'numeric' : 'string'; + } + + private function assertCreateCapability(int $variableCount, ?int $postgresqlSlots, string $path): void + { + if (!in_array($this->connection->profileName, ['sqlite', 'postgresql'], true) + || !$this->connection->profile->ddlAtomic() + ) { + throw TransformationFailure::at( + 'schema_change_not_atomic', + $path, + $this->connection->profileName . ' requires a pre-provisioned target.', + ); + } + $maximum = $this->connection->profile->effectiveMaximumSourceVariables($this->connection->pdo); + if ($variableCount >= $maximum + || ($postgresqlSlots !== null && $postgresqlSlots >= $maximum + 1) + ) { + throw new UnsupportedOperation( + DiagnosticCode::TargetCapabilityExceeded, + $this->connection->profileName . ' cannot add another source variable to this wide table.', + ); + } + } + + private function resolveDataset(string $datasetId): DatasetBinding + { + $statement = $this->statement( + 'SELECT dataset_id, dataset_name, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', + ); + $statement->execute([$datasetId]); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + if (count($rows) !== 1) { + throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply dataset must resolve to exactly one catalog row.'); + } + $row = $rows[0]; + $name = $row['dataset_name'] ?? null; + $schema = $row['physical_table_schema'] ?? null; + $table = $row['physical_table_name'] ?? null; + if (($row['dataset_id'] ?? null) !== $datasetId + || ($name !== null && !is_string($name)) + || ($schema !== null && (!is_string($schema) || $schema === '')) + || !is_string($table) + || $table === '' + ) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Dataset physical binding is malformed.'); + } + + return new DatasetBinding($datasetId, $name, $schema, $table); + } + + /** @return array */ + private function resolveVariables(DatasetBinding $dataset): array + { + $statement = $this->statement( + 'SELECT variable_id, source_name, physical_name, storage_kind, source_ordinal, declared_string_width ' + . 'FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', + ); + $statement->execute([$dataset->datasetId]); + $variables = []; + $physical = []; + $lastOrdinal = 0; + foreach ($statement->fetchAll(PDO::FETCH_ASSOC) as $row) { + $id = $row['variable_id'] ?? null; + $name = $row['source_name'] ?? null; + $physicalName = $row['physical_name'] ?? null; + $kind = $row['storage_kind'] ?? null; + $ordinal = filter_var($row['source_ordinal'] ?? null, FILTER_VALIDATE_INT); + $width = ($row['declared_string_width'] ?? null) === null + ? null + : filter_var($row['declared_string_width'], FILTER_VALIDATE_INT); + if (!is_string($id) || $id === '' + || !is_string($name) || $name === '' + || !is_string($physicalName) || $physicalName === '' + || !in_array($kind, ['numeric', 'string'], true) + || !is_int($ordinal) || $ordinal <= $lastOrdinal + || ($kind === 'string' && (!is_int($width) || $width < 1)) + || isset($variables[$name]) || isset($physical[$physicalName]) + ) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Variable catalog binding is malformed or ambiguous.'); + } + $variables[$name] = new VariableBinding($id, $name, $physicalName, $kind, $ordinal, is_int($width) ? $width : null); + $physical[$physicalName] = true; + $lastOrdinal = $ordinal; + } + if ($variables === []) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Bound dataset has no catalog variables.'); + } + + return $variables; + } + + /** + * @param array $variables + * @param list $physicalColumns + */ + private function assertPhysicalBindings(DatasetBinding $dataset, array $variables, array $physicalColumns): void + { + $columns = array_fill_keys($physicalColumns, true); + if (!isset($columns['__case_ordinal'])) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Wide table lacks the case-order column.'); + } + foreach ($variables as $variable) { + if (!isset($columns[$variable->physicalName])) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Catalog variable lacks its physical column.'); + } + } + + $selection = implode(', ', array_map( + fn(VariableBinding $variable): string => $this->connection->profile->quoteIdentifier($variable->physicalName), + array_values($variables), + )); + try { + $this->connection->pdo->query('SELECT ' . $selection . ' FROM ' . $dataset->qualifiedTable($this->connection->profile) . ' WHERE 1 = 0'); + } catch (PDOException) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Cataloged wide-table bindings are not readable.'); + } + } + + /** @return list */ + private function physicalColumns(DatasetBinding $dataset): array + { + if ($this->connection->profileName === 'sqlite') { + $statement = $this->connection->pdo->query( + 'PRAGMA table_info(' . $this->connection->profile->quoteIdentifier($dataset->table) . ')', + ); + $rows = $statement === false ? [] : $statement->fetchAll(PDO::FETCH_ASSOC); + return array_values(array_map(static fn(array $row): string => (string) ($row['name'] ?? ''), $rows)); + } + if ($this->connection->profileName === 'postgresql') { + $statement = $this->statement( + 'SELECT column_name FROM information_schema.columns ' + . 'WHERE table_schema = COALESCE(?, current_schema()) AND table_name = ? ORDER BY ordinal_position', + ); + $statement->execute([$dataset->schema, $dataset->table]); + return array_values(array_map('strval', $statement->fetchAll(PDO::FETCH_COLUMN))); + } + + // Existing-target plans on MySQL-family profiles need only prove the + // cataloged columns readable; generated identifiers are never used. + return ['__case_ordinal', ...array_map( + static fn(VariableBinding $variable): string => $variable->physicalName, + array_values($this->resolveVariables($dataset)), + )]; + } + + private function postgresqlPhysicalColumnSlots(DatasetBinding $dataset): int + { + $statement = $this->statement('SELECT COUNT(*) FROM pg_attribute WHERE attrelid = to_regclass(?) AND attnum > 0'); + $statement->execute([$dataset->qualifiedTable($this->connection->profile)]); + $slots = filter_var($statement->fetchColumn(), FILTER_VALIDATE_INT); + if (!is_int($slots) || $slots < 1) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'PostgreSQL wide table has no physical column slots.'); + } + return $slots; + } + + private function assertAuditReady(): void + { + try { + $this->connection->pdo->query( + 'SELECT apply_id, contract_id, database_profile, dataset_id, physical_table_schema, physical_table_name, ' + . 'source_hash, plan_hash, canonical_plan_json, actor, status, dolt_branch, dolt_head_before, dolt_head_after, ' + . 'operation_count, started_at, completed_at FROM transformation_apply WHERE 1 = 0', + ); + } catch (PDOException) { + throw new UnsupportedOperation( + DiagnosticCode::CatalogMigrationRequired, + 'The transformation audit schema is not ready before apply.', + ); + } + } + + private function statement(string $sql): PDOStatement + { + $statement = $this->connection->pdo->prepare($sql); + if ($statement === false) { + throw new PDOException('Transformation preflight statement could not be prepared.'); + } + return $statement; + } +} diff --git a/src/Transformation/Execution/SqlPredicateCompiler.php b/src/Transformation/Execution/SqlPredicateCompiler.php new file mode 100644 index 0000000..c1c3368 --- /dev/null +++ b/src/Transformation/Execution/SqlPredicateCompiler.php @@ -0,0 +1,76 @@ + $parameters */ + public function compile(Predicate $predicate, BoundSchema $schema, array &$parameters): string + { + return $this->predicate($predicate, $schema, $parameters); + } + + /** @param list $parameters */ + public function operand(Operand $operand, BoundSchema $schema, array &$parameters): string + { + if ($operand instanceof VariableOperand) { + return $this->profile->quoteIdentifier( + $schema->variable($operand->variable)->physicalName, + ); + } + if ($operand instanceof LiteralOperand && $operand->value instanceof Binary64Value) { + $parameters[] = $operand->value->number(); + return '?'; + } + + throw TransformationFailure::at('expression_type_unsupported', '$.value', 'Only numeric SQL operands are supported.'); + } + + /** @param list $parameters */ + private function predicate(Predicate $predicate, BoundSchema $schema, array &$parameters): string + { + return match (true) { + $predicate instanceof Comparison => $this->comparison($predicate, $schema, $parameters), + $predicate instanceof BooleanPredicate => $this->boolean($predicate, $schema, $parameters), + default => throw TransformationFailure::at('plan_schema_invalid', '$.condition', 'Unsupported predicate type.'), + }; + } + + /** @param list $parameters */ + private function comparison(Comparison $comparison, BoundSchema $schema, array &$parameters): string + { + $left = $this->operand($comparison->left, $schema, $parameters); + $right = $this->operand($comparison->right, $schema, $parameters); + + return '(' . $left . ' ' . $comparison->operator . ' ' . $right . ')'; + } + + /** @param list $parameters */ + private function boolean(BooleanPredicate $predicate, BoundSchema $schema, array &$parameters): string + { + $children = []; + foreach ($predicate->operands as $child) { + $children[] = $this->predicate($child, $schema, $parameters); + } + + return '(' . implode( + ' ' . strtoupper($predicate->operator) . ' ', + $children, + ) . ')'; + } +} diff --git a/src/Transformation/Execution/VariableBinding.php b/src/Transformation/Execution/VariableBinding.php index 521283f..1e3bd99 100644 --- a/src/Transformation/Execution/VariableBinding.php +++ b/src/Transformation/Execution/VariableBinding.php @@ -16,4 +16,9 @@ public function __construct( public ?int $declaredStringWidth, public bool $persisted = true, ) {} + + public function isNumeric(): bool + { + return $this->storageKind === 'numeric'; + } } diff --git a/tests/Integration/InPlaceTransformationServiceTest.php b/tests/Integration/InPlaceTransformationServiceTest.php index 52d1be9..149faf4 100644 --- a/tests/Integration/InPlaceTransformationServiceTest.php +++ b/tests/Integration/InPlaceTransformationServiceTest.php @@ -11,20 +11,26 @@ use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; -use OpenStatSpec\Transformation\Model\Action\AssignValueAction; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\RecodeRule; -use OpenStatSpec\Transformation\Model\ScalarValue; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\ExactValueSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\Selector\NumericRangeSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; -use OpenStatSpec\Transformation\Model\ValueLabel; +use OpenStatSpec\Transformation\Plan\Expression\VariableOperand; +use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\RecodeOperation; +use OpenStatSpec\Transformation\Plan\Operation\ReplaceValueLabelsOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetVariableLabelOperation; +use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; +use OpenStatSpec\Transformation\Plan\PlanCodec; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\Recode\CopyResult; +use OpenStatSpec\Transformation\Plan\Recode\ExactMatch; +use OpenStatSpec\Transformation\Plan\Recode\LiteralResult; +use OpenStatSpec\Transformation\Plan\Recode\RangeMatch; +use OpenStatSpec\Transformation\Plan\Recode\RecodeRule; +use OpenStatSpec\Transformation\Plan\Recode\SystemMissingMatch; +use OpenStatSpec\Transformation\Plan\TargetMode; +use OpenStatSpec\Transformation\Plan\TransformationPlan; +use OpenStatSpec\Transformation\Plan\Value\Binary64Value; use PDO; use PDOStatement; use PHPUnit\Framework\Attributes\DataProvider; @@ -163,7 +169,7 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace self::assertSame([], $doltBefore['working_diff']); } $plan = $this->existingTargetPlan(); - $result = (new InPlaceTransformationExecutor($connection))->execute($plan); + $result = (new InPlaceTransformationExecutor($connection))->execute($this->applyRequest($plan, $doltBefore)); $doltAfter = $this->doltRepositoryEvidence($pdo, $connection); if ($expectedProfile === 'dolt') { @@ -175,6 +181,7 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace self::assertSame( [ ['table_name' => 'inplace_existing_target_dolt', 'staged' => false, 'status' => 'modified'], + ['table_name' => 'transformation_apply', 'staged' => false, 'status' => 'modified'], ['table_name' => 'value_label', 'staged' => false, 'status' => 'modified'], ['table_name' => 'value_label_set', 'staged' => false, 'status' => 'modified'], ['table_name' => 'variable', 'staged' => false, 'status' => 'modified'], @@ -186,6 +193,7 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace self::assertSame( [ ['table_name' => 'inplace_existing_target_dolt', 'data_change' => true, 'schema_change' => false], + ['table_name' => 'transformation_apply', 'data_change' => true, 'schema_change' => false], ['table_name' => 'value_label', 'data_change' => true, 'schema_change' => false], ['table_name' => 'value_label_set', 'data_change' => true, 'schema_change' => false], ['table_name' => 'variable', 'data_change' => true, 'schema_change' => false], @@ -197,8 +205,8 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace } self::assertSame($fixture['dataset_id'], $result->datasetId()); - self::assertSame($plan->hash(), $result->planHash()); - self::assertSame(3, $result->operationCount()); + self::assertSame((new PlanCodec())->hash($plan), $result->planHash()); + self::assertSame(4, $result->operationCount()); self::assertSame($fixture['dataset'], $this->datasetRow($pdo)); self::assertSame($fixture['variables'], $this->variableIdentityRows($pdo)); @@ -260,10 +268,10 @@ public function testCreateTargetTransformationCreatesNumericTargetInPlace( try { $datasetCountBefore = (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset', []); $plan = $this->createTargetPlan(); - $result = (new InPlaceTransformationExecutor($connection))->execute($plan); + $result = (new InPlaceTransformationExecutor($connection))->execute($this->applyRequest($plan)); self::assertSame($fixture['dataset_id'], $result->datasetId()); - self::assertSame($plan->hash(), $result->planHash()); + self::assertSame((new PlanCodec())->hash($plan), $result->planHash()); self::assertSame(3, $result->operationCount()); self::assertSame($fixture['dataset'], $this->datasetRow($pdo)); @@ -389,17 +397,10 @@ public function testCreateTargetTransformationRejectsNonAtomicProfilesWithoutCha ]; try { - (new InPlaceTransformationExecutor($connection))->execute($this->createTargetPlan()); + (new InPlaceTransformationExecutor($connection))->execute($this->applyRequest($this->createTargetPlan())); self::fail('Expected non-atomic service profile to reject implicit target creation during preflight.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); - self::assertSame( - sprintf( - '%s cannot atomically add a new INTO target to an existing wide table; register the target variable first.', - $expectedProfile, - ), - $exception->getMessage(), - ); + } catch (TransformationFailure $exception) { + self::assertSame('schema_change_not_atomic', $exception->diagnosticCode()); } self::assertSame($before['catalog'], $this->fullCatalogSnapshot($pdo)); @@ -724,6 +725,7 @@ private function purgeFixture(PDO $pdo, Connection $connection, array $fixture): $pdo->prepare( 'DELETE FROM variable_value_label_set WHERE variable_id IN (SELECT variable_id FROM variable WHERE dataset_id = ?)', )->execute([$fixture['dataset_id']]); + $pdo->prepare('DELETE FROM transformation_apply WHERE dataset_id = ?')->execute([$fixture['dataset_id']]); $pdo->prepare( 'DELETE FROM value_label WHERE value_label_set_id IN (SELECT value_label_set_id FROM value_label_set WHERE dataset_id = ?)', )->execute([$fixture['dataset_id']]); @@ -763,58 +765,77 @@ private function purgeDirtyNamespaceFixture(PDO $pdo, Connection $connection, ar private function existingTargetPlan(): TransformationPlan { - return new TransformationPlan($this->activeFixtureId(), [ - new RecodeOperation('SourceValue', 'Destination', [ + return new TransformationPlan(PlanContract::V02, 'parent', [ + new RecodeOperation('SourceValue', 'SourceValue', TargetMode::Replace, [ new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(10)), + new ExactMatch($this->binary64(1.0)), + new LiteralResult($this->binary64(10.0)), ), new RecodeRule( - new NumericRangeSelector(2.0, 3.0), - new AssignValueAction(ScalarValue::number(20)), + new RangeMatch($this->binary64(2.0), $this->binary64(3.0)), + new LiteralResult($this->binary64(20.0)), ), new RecodeRule( - new MissingValueSelector(), - new AssignValueAction(ScalarValue::number(99)), + new SystemMissingMatch(), + new LiteralResult($this->binary64(99.0)), ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), + ], new CopyResult()), + new AssignOperation('Destination', TargetMode::Replace, new VariableOperand('SourceValue')), new SetVariableLabelOperation('Destination', 'Recoded destination'), - new SetValueLabelsOperation('Destination', [ - new ValueLabel(ScalarValue::number(10), 'Ten'), - new ValueLabel(ScalarValue::number(20), 'Twenty'), - new ValueLabel(ScalarValue::number(99), 'Missing source'), + new ReplaceValueLabelsOperation('Destination', [ + new ValueLabel($this->binary64(10.0), 'Ten'), + new ValueLabel($this->binary64(20.0), 'Twenty'), + new ValueLabel($this->binary64(99.0), 'Missing source'), ]), ]); } private function createTargetPlan(): TransformationPlan { - return new TransformationPlan($this->activeFixtureId(), [ - new RecodeOperation('SourceValue', 'CreatedTarget', [ + return new TransformationPlan(PlanContract::V01, 'parent', [ + new RecodeOperation('SourceValue', 'CreatedTarget', TargetMode::Create, [ new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(10)), + new ExactMatch($this->binary64(1.0)), + new LiteralResult($this->binary64(10.0)), ), new RecodeRule( - new NumericRangeSelector(2.0, 3.0), - new AssignValueAction(ScalarValue::number(20)), + new RangeMatch($this->binary64(2.0), $this->binary64(3.0)), + new LiteralResult($this->binary64(20.0)), ), new RecodeRule( - new MissingValueSelector(), - new AssignValueAction(ScalarValue::number(99)), + new SystemMissingMatch(), + new LiteralResult($this->binary64(99.0)), ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), + ], new CopyResult()), new SetVariableLabelOperation('CreatedTarget', 'Recoded created target'), - new SetValueLabelsOperation('CreatedTarget', [ - new ValueLabel(ScalarValue::number(10), 'Ten'), - new ValueLabel(ScalarValue::number(20), 'Twenty'), - new ValueLabel(ScalarValue::number(99), 'Missing source'), + new ReplaceValueLabelsOperation('CreatedTarget', [ + new ValueLabel($this->binary64(10.0), 'Ten'), + new ValueLabel($this->binary64(20.0), 'Twenty'), + new ValueLabel($this->binary64(99.0), 'Missing source'), ]), ]); } + /** @param array{branch: string, head: string}|null $doltBefore */ + private function applyRequest(TransformationPlan $plan, ?array $doltBefore = null): InPlaceApplyRequest + { + return new InPlaceApplyRequest( + $plan, + 'parent', + $this->activeFixtureId(), + str_repeat('a', 64), + 'integration-test', + $doltBefore['branch'] ?? null, + $doltBefore['head'] ?? null, + ); + } + + private function binary64(float $value): Binary64Value + { + $packed = pack('E', $value); + return Binary64Value::fromBits(bin2hex($packed)); + } + private function servicePdo(string $expectedProfile, ?string $environmentPrefix, string $driver): PDO { if (!in_array($driver, PDO::getAvailableDrivers(), true)) { diff --git a/tests/Integration/OfficialInPlaceTransformation01Test.php b/tests/Integration/OfficialInPlaceTransformation01Test.php new file mode 100644 index 0000000..bafcb90 --- /dev/null +++ b/tests/Integration/OfficialInPlaceTransformation01Test.php @@ -0,0 +1,163 @@ +fixture(); + $connection = new Connection($pdo); + $plan = $this->planCase('numeric-recode-and-declared-labels'); + $request = new InPlaceApplyRequest( + $plan, + 'parent', + self::DATASET_ID, + hash('sha256', 'RECODE q1.'), + 'conformance-runner', + ); + $tablesBefore = $this->tables($pdo); + + $result = (new InPlaceTransformationExecutor($connection))->execute($request); + + self::assertSame('075571b4f84b0f60f289aebb6ec990ad61d34a96b46655802ea8f8bc07a1a405', $result->planHash()); + self::assertSame($tablesBefore, $this->tables($pdo)); + self::assertSame(1, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset')); + self::assertSame( + [0.0, 0.0, 1.0, 1.0, 1.0, null, null], + array_map( + static fn(mixed $value): ?float => $value === null ? null : (float) $value, + $this->column($pdo, 'SELECT q1_binary FROM data_plan01 ORDER BY __case_ordinal'), + ), + ); + self::assertSame('Positive response', $this->scalar( + $pdo, + "SELECT variable_label FROM variable WHERE dataset_id = '" . self::DATASET_ID . "' AND source_name = 'q1_binary'", + )); + self::assertSame( + [[0.0, 'No'], [1.0, 'Yes']], + array_map( + static fn(array $row): array => [(float) $row['numeric_code'], (string) $row['label']], + $this->rows($pdo, 'SELECT label.numeric_code, label.label FROM value_label label ORDER BY label.ordinal'), + ), + ); + self::assertSame( + ['openstatspec-in-place-transformation-v0.1', $result->planHash(), 3], + $this->normalizedAudit($pdo), + ); + } + + public function testSqliteStringValueLabelsAreReplacedExactlyWithoutChangingOtherMetadata(): void + { + $pdo = $this->fixture(); + $pdo->exec("ALTER TABLE data_plan01 ADD COLUMN color TEXT NOT NULL DEFAULT ''"); + $pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width, variable_label, measurement_level) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', + )->execute(['77777777-7777-4777-8777-777777777777', self::DATASET_ID, 2, 'color', 'color', 'string', 1, 'Colour', 'nominal']); + $pdo->exec("UPDATE data_plan01 SET color = CASE WHEN __case_ordinal % 2 = 0 THEN 'B' ELSE 'R' END"); + $plan = $this->planCase('string-value-label-replacement'); + $request = new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, hash('sha256', 'VALUE LABELS color.'), 'conformance-runner'); + + (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($request); + + self::assertSame( + [['R', 'Punane'], ['B', 'Sinine']], + array_map( + static fn(array $row): array => [(string) $row['string_code'], (string) $row['label']], + $this->rows($pdo, 'SELECT label.string_code, label.label FROM value_label label ORDER BY label.ordinal'), + ), + ); + self::assertSame( + ['variable_label' => 'Colour', 'measurement_level' => 'nominal'], + $this->rows($pdo, "SELECT variable_label, measurement_level FROM variable WHERE source_name = 'color'")[0], + ); + } + + private function fixture(): PDO + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available.'); + } + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]); + $pdo->exec('PRAGMA foreign_keys = ON'); + (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); + CatalogOwnership::markCurrentVersion($pdo); + $pdo->exec('CREATE TABLE data_plan01 (__case_ordinal INTEGER NOT NULL PRIMARY KEY, q1 REAL NULL)'); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([self::DATASET_ID, '1.0', 'fixture', null, 'data_plan01', 'Plan 0.1 fixture', 7, '2026-08-17 00:00:00']); + $pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) VALUES (?, ?, ?, ?, ?, ?)', + )->execute(['88888888-8888-4888-8888-888888888888', self::DATASET_ID, 1, 'q1', 'q1', 'numeric']); + $insert = $pdo->prepare('INSERT INTO data_plan01 (__case_ordinal, q1) VALUES (?, ?)'); + foreach ([[1, 1.0], [2, 2.0], [3, 3.0], [4, 4.0], [5, 5.0], [6, 6.0], [7, null]] as $row) { + $insert->execute($row); + } + return $pdo; + } + + private function planCase(string $id): TransformationPlan + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.1.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id && is_array($case['plan'] ?? null)) { + return (new PlanCodec())->fromArray($case['plan']); + } + } + throw new \RuntimeException('Missing plan case: ' . $id); + } + + /** @return list */ + private function tables(PDO $pdo): array + { + return array_map('strval', $this->column($pdo, "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")); + } + + /** @return array{string, string, int} */ + private function normalizedAudit(PDO $pdo): array + { + $row = $this->rows($pdo, 'SELECT contract_id, plan_hash, operation_count FROM transformation_apply')[0]; + return [(string) $row['contract_id'], (string) $row['plan_hash'], (int) $row['operation_count']]; + } + + private function scalar(PDO $pdo, string $sql): mixed + { + $statement = $pdo->query($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + return $statement->fetchColumn(); + } + + /** @return list */ + private function column(PDO $pdo, string $sql): array + { + $statement = $pdo->query($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + return array_values($statement->fetchAll(PDO::FETCH_COLUMN)); + } + + /** @return list> */ + private function rows(PDO $pdo, string $sql): array + { + $statement = $pdo->query($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); + } +} diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php new file mode 100644 index 0000000..3c53d32 --- /dev/null +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -0,0 +1,613 @@ +bindingCase('sqlite-create-target-atomic-success'); + $pdo = $this->sqlite(); + $connection = new Connection($pdo); + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + 'data_atomic_create', + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + $request = $this->request($case, self::CREATE_DATASET_ID); + + $result = (new InPlaceTransformationExecutor($connection))->execute($request); + + self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); + self::assertSame(self::CREATE_DATASET_ID, $result->datasetId()); + self::assertSame(1, $result->operationCount()); + self::assertNotSame('', $result->auditOperationId()); + $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($before['case_ordinals'], $after['case_ordinals']); + self::assertSame($before['case_count'], $after['case_count']); + self::assertSame(3, $after['variable_count']); + self::assertSame( + $case['after']['rows'], + $this->numericRows($pdo, $connection, 'data_atomic_create', ['source_a', 'source_b', 'target']), + ); + self::assertSame( + [ + 'source_ordinal' => 3, + 'storage_kind' => 'numeric', + 'declared_string_width' => null, + 'variable_label' => null, + 'print_format_family' => null, + 'print_format_width' => null, + 'print_format_decimals' => null, + 'write_format_family' => null, + 'write_format_width' => null, + 'write_format_decimals' => null, + 'measurement_level' => null, + 'variable_role' => null, + 'display_width' => null, + 'display_alignment' => null, + ], + $this->targetMetadata($pdo, self::CREATE_DATASET_ID, 'target'), + ); + self::assertSame([], $this->valueLabels($pdo, self::CREATE_DATASET_ID, 'target')); + self::assertSame([], $this->missingRules($pdo, self::CREATE_DATASET_ID, 'target')); + $this->assertAudit($pdo, $case['expected_audit']); + } + + public function testSqliteCreateTargetFailureRollsBackSchemaRowsCatalogAndAudit(): void + { + $case = $this->bindingCase('sqlite-create-target-atomic-success'); + $pdo = $this->sqlite(); + $connection = new Connection($pdo); + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + 'data_atomic_create', + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + $failingAuditPdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + + try { + (new InPlaceTransformationExecutor( + $connection, + auditWriter: new TransformationAuditWriter($failingAuditPdo), + ))->execute($this->request($case, self::CREATE_DATASET_ID)); + self::fail('The injected audit failure did not abort the apply.'); + } catch (\LogicException $exception) { + self::assertStringContainsString('open apply transaction', $exception->getMessage()); + } + + $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + self::assertSame($before, $after); + self::assertNotContains('target', $this->columns($pdo, $connection, 'data_atomic_create')); + self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM variable WHERE dataset_id = ? AND source_name = ?', [self::CREATE_DATASET_ID, 'target'])); + self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM transformation_apply', [])); + self::assertFalse($pdo->inTransaction()); + } + + public function testSqliteInequalityBoundariesUseSqlThreeValuedTruth(): void + { + $case = $this->bindingCase('sqlite-inequality-boundary-semantics'); + $pdo = $this->sqlite(); + $connection = new Connection($pdo); + $this->installNumericFixture( + $pdo, + $connection, + self::INEQUALITY_DATASET_ID, + 'data_inequality', + ['source', 'target_lt', 'target_le', 'target_gt', 'target_ge'], + [[1, 0.0, 0.0, 0.0, 0.0, 0.0], [2, 1.0, 0.0, 0.0, 0.0, 0.0], [3, 2.0, 0.0, 0.0, 0.0, 0.0]], + ); + $before = $this->snapshot($pdo, $connection, self::INEQUALITY_DATASET_ID, 'data_inequality'); + + $result = (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::INEQUALITY_DATASET_ID)); + + self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); + self::assertSame($case['after_rows'], $this->numericRows( + $pdo, + $connection, + 'data_inequality', + ['source', 'target_lt', 'target_le', 'target_gt', 'target_ge'], + )); + $after = $this->snapshot($pdo, $connection, self::INEQUALITY_DATASET_ID, 'data_inequality'); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($before['variables'], $after['variables']); + self::assertSame($before['case_ordinals'], $after['case_ordinals']); + self::assertSame($before['case_count'], $after['case_count']); + $this->assertAudit($pdo, $case['expected_audit']); + } + + public function testSqliteOperationsObservePriorValuesAndChangeOnlyNamedMetadata(): void + { + $case = $this->bindingCase('dolt-preprovisioned-target-sequential-null-semantics'); + $pdo = $this->sqlite(); + $connection = new Connection($pdo); + $this->installNumericFixture( + $pdo, + $connection, + '11111111-1111-4111-8111-111111111111', + 'data_synthetic', + ['source_a', 'source_b', 'target'], + [[1, 1.0, 1.0, null], [2, 1.0, null, null], [3, null, 1.0, null], [4, 0.0, 1.0, null]], + ); + $targetId = (string) $this->scalar( + $pdo, + 'SELECT variable_id FROM variable WHERE dataset_id = ? AND source_name = ?', + ['11111111-1111-4111-8111-111111111111', 'target'], + ); + $pdo->prepare( + 'INSERT INTO missing_rule (missing_rule_id, variable_id, ordinal, rule_kind, code_kind, numeric_value) VALUES (?, ?, ?, ?, ?, ?)', + )->execute(['55555555-5555-4555-8555-555555555555', $targetId, 1, 'discrete', 'numeric', -9.0]); + $request = new InPlaceApplyRequest( + plan: $this->planCase((string) $case['applied_plan_case']), + inputAlias: 'parent', + datasetId: '11111111-1111-4111-8111-111111111111', + sourceHash: (string) $case['expected_audit']['source_hash'], + actor: 'conformance-runner', + ); + + (new InPlaceTransformationExecutor($connection))->execute($request); + + self::assertSame($case['after']['rows'], $this->numericRows( + $pdo, + $connection, + 'data_synthetic', + ['source_a', 'source_b', 'target'], + )); + self::assertSame( + [ + 'source_ordinal' => 3, + 'storage_kind' => 'numeric', + 'declared_string_width' => null, + 'variable_label' => 'Synthetic conjunction', + 'print_format_family' => 'F', + 'print_format_width' => 1, + 'print_format_decimals' => 0, + 'write_format_family' => 'F', + 'write_format_width' => 1, + 'write_format_decimals' => 0, + 'measurement_level' => 'nominal', + 'variable_role' => null, + 'display_width' => null, + 'display_alignment' => null, + ], + $this->targetMetadata($pdo, '11111111-1111-4111-8111-111111111111', 'target'), + ); + self::assertSame([[0.0, 'No'], [1.0, 'Yes']], $this->valueLabels($pdo, '11111111-1111-4111-8111-111111111111', 'target')); + self::assertSame([['ordinal' => 1, 'rule_kind' => 'discrete', 'code_kind' => 'numeric', 'numeric_value' => -9.0]], $this->missingRules($pdo, '11111111-1111-4111-8111-111111111111', 'target')); + } + + public function testPostgresqlCreateTargetIsAtomicWhenConfigured(): void + { + $pdo = $this->postgresql(); + $connection = new Connection($pdo); + self::assertSame('postgresql', $connection->profileName); + $table = 'data_atomic_create_task8'; + $case = $this->bindingCase('sqlite-create-target-atomic-success'); + $this->prepareCatalog($pdo); + $this->assertNamespaceClean($pdo, self::CREATE_DATASET_ID, $table); + + try { + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + $table, + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); + (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::CREATE_DATASET_ID)); + $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($case['after']['rows'], $this->numericRows($pdo, $connection, $table, ['source_a', 'source_b', 'target'])); + $this->assertAudit($pdo, [...$case['expected_audit'], 'database_profile' => 'postgresql', 'physical_table_name' => $table]); + } finally { + $this->purgeFixture($pdo, $connection, self::CREATE_DATASET_ID, $table); + } + } + + public function testPostgresqlCreateTargetFailureRollsBackWhenConfigured(): void + { + $pdo = $this->postgresql(); + $connection = new Connection($pdo); + $table = 'data_atomic_rollback_task8'; + $case = $this->bindingCase('sqlite-create-target-atomic-success'); + $this->prepareCatalog($pdo); + $this->assertNamespaceClean($pdo, self::ROLLBACK_DATASET_ID, $table); + + try { + $this->installNumericFixture( + $pdo, + $connection, + self::ROLLBACK_DATASET_ID, + $table, + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $before = $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + $failingAuditPdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + + try { + (new InPlaceTransformationExecutor( + $connection, + auditWriter: new TransformationAuditWriter($failingAuditPdo), + ))->execute($this->request($case, self::ROLLBACK_DATASET_ID)); + self::fail('The injected PostgreSQL audit failure did not abort the apply.'); + } catch (\LogicException $exception) { + self::assertStringContainsString('open apply transaction', $exception->getMessage()); + } + + self::assertSame($before, $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table)); + self::assertNotContains('target', $this->columns($pdo, $connection, $table)); + self::assertFalse($pdo->inTransaction()); + } finally { + $this->purgeFixture($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + } + } + + private function sqlite(): PDO + { + if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO SQLite is not available.'); + } + return new PDO('sqlite::memory:', options: [ + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_STRINGIFY_FETCHES => false, + ]); + } + + private function postgresql(): PDO + { + $dsn = getenv('OPENSTATSPEC_PG_DSN'); + if (!is_string($dsn) || $dsn === '') { + self::markTestSkipped('OPENSTATSPEC_PG_DSN is not configured.'); + } + if (!in_array('pgsql', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO PostgreSQL is not available.'); + } + $user = getenv('OPENSTATSPEC_PG_USER'); + $password = getenv('OPENSTATSPEC_PG_PASSWORD'); + return new PDO( + $dsn, + is_string($user) ? $user : null, + is_string($password) ? $password : null, + [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false], + ); + } + + /** + * @param list $variables + * @param list> $rows + */ + private function installNumericFixture( + PDO $pdo, + Connection $connection, + string $datasetId, + string $table, + array $variables, + array $rows, + ): void { + if ($connection->profileName === 'sqlite') { + $pdo->exec('PRAGMA foreign_keys = ON'); + } + $this->prepareCatalog($pdo); + $quotedColumns = array_map($connection->profile->quoteIdentifier(...), $variables); + $pdo->exec( + 'CREATE TABLE ' . $connection->profile->quoteIdentifier($table) . ' (' + . $connection->profile->quoteIdentifier('__case_ordinal') . ' BIGINT NOT NULL PRIMARY KEY, ' + . implode(' ' . $connection->profile->numericType() . ' NULL, ', $quotedColumns) + . ' ' . $connection->profile->numericType() . ' NULL)', + ); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([$datasetId, '1.0', 'fixture', null, $table, 'Task 8 fixture', count($rows), '2026-08-17 00:00:00']); + $insertVariable = $pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) VALUES (?, ?, ?, ?, ?, ?)', + ); + foreach ($variables as $index => $variable) { + $insertVariable->execute([ + $this->fixtureUuid($datasetId . ':' . $variable), + $datasetId, + $index + 1, + $variable, + $variable, + 'numeric', + ]); + } + $columns = [$connection->profile->quoteIdentifier('__case_ordinal'), ...$quotedColumns]; + $insertCase = $pdo->prepare( + 'INSERT INTO ' . $connection->profile->quoteIdentifier($table) + . ' (' . implode(', ', $columns) . ') VALUES (' . implode(', ', array_fill(0, count($columns), '?')) . ')', + ); + foreach ($rows as $row) { + $insertCase->execute($row); + } + } + + private function prepareCatalog(PDO $pdo): void + { + (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); + CatalogOwnership::markCurrentVersion($pdo); + } + + /** @param array $case */ + private function request(array $case, string $datasetId): InPlaceApplyRequest + { + return new InPlaceApplyRequest( + plan: $this->planCase((string) $case['applied_plan_case']), + inputAlias: 'parent', + datasetId: $datasetId, + sourceHash: (string) $case['expected_audit']['source_hash'], + actor: (string) $case['actor'], + ); + } + + /** @return array */ + private function bindingCase(string $id): array + { + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.2.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id) { + return $case; + } + } + throw new \RuntimeException('Missing in-place fixture: ' . $id); + } + + private function planCase(string $id): \OpenStatSpec\Transformation\Plan\TransformationPlan + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.2.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id && is_array($case['plan'] ?? null)) { + return (new PlanCodec())->fromArray($case['plan']); + } + } + throw new \RuntimeException('Missing plan fixture: ' . $id); + } + + /** @return array */ + private function snapshot(PDO $pdo, Connection $connection, string $datasetId, string $table): array + { + return [ + 'dataset_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM dataset', []), + 'persistent_data_table_count' => $this->scalarCount($pdo, 'SELECT COUNT(DISTINCT physical_table_name) FROM dataset', []), + 'dataset_identity' => $this->rows($pdo, 'SELECT dataset_id, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', [$datasetId]), + 'tables' => $this->tables($pdo), + 'columns' => $this->columns($pdo, $connection, $table), + 'variables' => $this->rows($pdo, 'SELECT * FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', [$datasetId]), + 'variable_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM variable WHERE dataset_id = ?', [$datasetId]), + 'case_ordinals' => array_map('intval', $this->column($pdo, 'SELECT __case_ordinal FROM ' . $connection->profile->quoteIdentifier($table) . ' ORDER BY __case_ordinal')), + 'case_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM ' . $connection->profile->quoteIdentifier($table), []), + 'rows' => $this->rows($pdo, 'SELECT * FROM ' . $connection->profile->quoteIdentifier($table) . ' ORDER BY __case_ordinal', []), + 'audit_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM transformation_apply', []), + ]; + } + + /** + * @param list $variables + * @return list> + */ + private function numericRows(PDO $pdo, Connection $connection, string $table, array $variables): array + { + $columns = ['__case_ordinal', ...$variables]; + $rows = $this->rows( + $pdo, + 'SELECT ' . implode(', ', array_map($connection->profile->quoteIdentifier(...), $columns)) + . ' FROM ' . $connection->profile->quoteIdentifier($table) + . ' ORDER BY ' . $connection->profile->quoteIdentifier('__case_ordinal'), + [], + ); + return array_map(static function (array $row) use ($columns): array { + $normalized = []; + foreach ($columns as $column) { + $value = $row[$column]; + if ($column === '__case_ordinal') { + $normalized[$column] = (int) $value; + continue; + } + if ($value === null) { + $normalized[$column] = null; + continue; + } + $number = (float) $value; + $normalized[$column] = floor($number) === $number ? (int) $number : $number; + } + return $normalized; + }, $rows); + } + + /** @return array */ + private function targetMetadata(PDO $pdo, string $datasetId, string $target): array + { + $rows = $this->rows( + $pdo, + 'SELECT source_ordinal, storage_kind, declared_string_width, variable_label, print_format_family, print_format_width, ' + . 'print_format_decimals, write_format_family, write_format_width, write_format_decimals, measurement_level, ' + . 'variable_role, display_width, display_alignment FROM variable WHERE dataset_id = ? AND source_name = ?', + [$datasetId, $target], + ); + self::assertCount(1, $rows); + foreach (['source_ordinal', 'declared_string_width', 'print_format_width', 'print_format_decimals', 'write_format_width', 'write_format_decimals', 'display_width'] as $integer) { + if ($rows[0][$integer] !== null) { + $rows[0][$integer] = (int) $rows[0][$integer]; + } + } + return $rows[0]; + } + + /** @return list */ + private function valueLabels(PDO $pdo, string $datasetId, string $target): array + { + $rows = $this->rows( + $pdo, + 'SELECT label.code_kind, label.numeric_code, label.string_code, label.label FROM variable ' + . 'JOIN variable_value_label_set link ON link.variable_id = variable.variable_id ' + . 'JOIN value_label label ON label.value_label_set_id = link.value_label_set_id ' + . 'WHERE variable.dataset_id = ? AND variable.source_name = ? ORDER BY label.ordinal', + [$datasetId, $target], + ); + return array_map(static fn(array $row): array => [ + $row['code_kind'] === 'numeric' ? (float) $row['numeric_code'] : (string) $row['string_code'], + (string) $row['label'], + ], $rows); + } + + /** @return list */ + private function missingRules(PDO $pdo, string $datasetId, string $target): array + { + return array_map(static fn(array $row): array => [ + 'ordinal' => (int) $row['ordinal'], + 'rule_kind' => (string) $row['rule_kind'], + 'code_kind' => (string) $row['code_kind'], + 'numeric_value' => $row['numeric_value'] === null ? null : (float) $row['numeric_value'], + ], $this->rows( + $pdo, + 'SELECT missing_rule.ordinal, missing_rule.rule_kind, missing_rule.code_kind, missing_rule.numeric_value ' + . 'FROM variable JOIN missing_rule ON missing_rule.variable_id = variable.variable_id ' + . 'WHERE variable.dataset_id = ? AND variable.source_name = ? ORDER BY missing_rule.ordinal', + [$datasetId, $target], + )); + } + + /** @param array $expected */ + private function assertAudit(PDO $pdo, array $expected): void + { + $rows = $this->rows($pdo, 'SELECT * FROM transformation_apply ORDER BY started_at, apply_id', []); + self::assertCount(1, $rows); + foreach ($expected as $field => $value) { + self::assertArrayHasKey($field, $rows[0]); + self::assertSame($value, $field === 'operation_count' ? (int) $rows[0][$field] : $rows[0][$field], $field); + } + self::assertMatchesRegularExpression('/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/', (string) $rows[0]['apply_id']); + self::assertNotSame('', (string) $rows[0]['started_at']); + self::assertNotSame('', (string) $rows[0]['completed_at']); + } + + private function assertNamespaceClean(PDO $pdo, string $datasetId, string $table): void + { + self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM dataset WHERE dataset_id = ? OR physical_table_name = ?', [$datasetId, $table])); + self::assertNotContains($table, $this->tables($pdo)); + } + + private function purgeFixture(PDO $pdo, Connection $connection, string $datasetId, string $table): void + { + if ($pdo->inTransaction()) { + $pdo->rollBack(); + } + $pdo->prepare('DELETE FROM transformation_apply WHERE dataset_id = ?')->execute([$datasetId]); + $pdo->prepare('DELETE FROM variable_value_label_set WHERE variable_id IN (SELECT variable_id FROM variable WHERE dataset_id = ?)')->execute([$datasetId]); + $pdo->prepare('DELETE FROM value_label WHERE value_label_set_id IN (SELECT value_label_set_id FROM value_label_set WHERE dataset_id = ?)')->execute([$datasetId]); + $pdo->prepare('DELETE FROM value_label_set WHERE dataset_id = ?')->execute([$datasetId]); + $pdo->prepare('DELETE FROM missing_rule WHERE variable_id IN (SELECT variable_id FROM variable WHERE dataset_id = ?)')->execute([$datasetId]); + $pdo->prepare('DELETE FROM variable WHERE dataset_id = ?')->execute([$datasetId]); + $pdo->prepare('DELETE FROM dataset WHERE dataset_id = ?')->execute([$datasetId]); + $pdo->exec('DROP TABLE IF EXISTS ' . $connection->profile->quoteIdentifier($table)); + } + + /** @return list */ + private function tables(PDO $pdo): array + { + $sql = (string) $pdo->getAttribute(PDO::ATTR_DRIVER_NAME) === 'sqlite' + ? "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + : "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = current_schema() ORDER BY tablename"; + return array_map('strval', $this->column($pdo, $sql)); + } + + /** @return list */ + private function columns(PDO $pdo, Connection $connection, string $table): array + { + if ($connection->profileName === 'sqlite') { + return array_map( + static fn(array $row): string => (string) $row['name'], + $this->rows($pdo, 'PRAGMA table_info(' . $connection->profile->quoteIdentifier($table) . ')', []), + ); + } + return array_map('strval', $this->column( + $pdo, + 'SELECT column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ? ORDER BY ordinal_position', + [$table], + )); + } + + private function fixtureUuid(string $scope): string + { + $hash = hash('sha256', $scope); + return substr($hash, 0, 8) . '-' . substr($hash, 8, 4) . '-4' . substr($hash, 12, 3) . '-8' . substr($hash, 15, 3) . '-' . substr($hash, 18, 12); + } + + /** @param list $parameters */ + private function scalarCount(PDO $pdo, string $sql, array $parameters): int + { + return (int) $this->scalar($pdo, $sql, $parameters); + } + + /** @param list $parameters */ + private function scalar(PDO $pdo, string $sql, array $parameters): mixed + { + $statement = $pdo->prepare($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + $statement->execute($parameters); + return $statement->fetchColumn(); + } + + /** + * @param list $parameters + * @return list + */ + private function column(PDO $pdo, string $sql, array $parameters = []): array + { + $statement = $pdo->prepare($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + $statement->execute($parameters); + return array_values($statement->fetchAll(PDO::FETCH_COLUMN)); + } + + /** + * @param list $parameters + * @return list> + */ + private function rows(PDO $pdo, string $sql, array $parameters): array + { + $statement = $pdo->prepare($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + $statement->execute($parameters); + return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); + } +} diff --git a/tests/Integration/PostgreSqlSpssRoundTripTest.php b/tests/Integration/PostgreSqlSpssRoundTripTest.php index 10a4b2a..a0fce06 100644 --- a/tests/Integration/PostgreSqlSpssRoundTripTest.php +++ b/tests/Integration/PostgreSqlSpssRoundTripTest.php @@ -11,10 +11,14 @@ use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Spss\PhpSpssEngine; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; -use OpenStatSpec\Transformation\Model\CreateVariableOperation; -use OpenStatSpec\Transformation\Model\DeleteVariableOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; +use OpenStatSpec\Transformation\Plan\Expression\LiteralOperand; +use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\TargetMode; +use OpenStatSpec\Transformation\Plan\TransformationPlan; +use OpenStatSpec\Transformation\Plan\Value\Binary64Value; use OpenStatSpec\Spss\SpssAdapter; use PDO; use PDOException; @@ -158,13 +162,23 @@ public function testDeleteThenCreateAtThePostgreSqlPhysicalColumnLimitFailsBefor } $pdo->commit(); - $plan = new TransformationPlan($datasetId, [ - new DeleteVariableOperation('V1599'), - new CreateVariableOperation('Replacement', 'string', 1), + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation( + 'Replacement', + TargetMode::Create, + new LiteralOperand(Binary64Value::fromBits('0000000000000000')), + ), ]); + $request = new InPlaceApplyRequest( + $plan, + 'parent', + $datasetId, + hash('sha256', 'PostgreSQL physical-column slot fixture'), + 'integration-test', + ); try { - (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($plan); + (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($request); self::fail('PostgreSQL accepted a replacement after the physical column limit was reached.'); } catch (UnsupportedOperation $exception) { self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index aadedc8..4aae3b9 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -4,43 +4,47 @@ namespace OpenStatSpec\Tests\Transformation\Execution; -use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Core\DiagnosticCode; use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; -use OpenStatSpec\Transformation\Model\Action\AssignValueAction; -use OpenStatSpec\Transformation\Model\Action\CopySourceAction; -use OpenStatSpec\Transformation\Model\Action\SetMissingAction; -use OpenStatSpec\Transformation\Model\CreateVariableOperation; -use OpenStatSpec\Transformation\Model\DeleteVariableOperation; -use OpenStatSpec\Transformation\Model\RecodeOperation; -use OpenStatSpec\Transformation\Model\RecodeRule; -use OpenStatSpec\Transformation\Model\ScalarValue; -use OpenStatSpec\Transformation\Model\Selector\ElseSelector; -use OpenStatSpec\Transformation\Model\Selector\ExactValueSelector; -use OpenStatSpec\Transformation\Model\Selector\MissingValueSelector; -use OpenStatSpec\Transformation\Model\Selector\NumericRangeSelector; -use OpenStatSpec\Transformation\Model\SetValueLabelsOperation; -use OpenStatSpec\Transformation\Model\SetVariableLabelOperation; -use OpenStatSpec\Transformation\Model\TransformationPlan; -use OpenStatSpec\Transformation\Model\ValueLabel; +use OpenStatSpec\Transformation\Plan\Expression\Comparison; +use OpenStatSpec\Transformation\Plan\Expression\LiteralOperand; +use OpenStatSpec\Transformation\Plan\Expression\VariableOperand; +use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\ConditionalAssignOperation; +use OpenStatSpec\Transformation\Plan\Operation\RecodeOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetFormatOperation; +use OpenStatSpec\Transformation\Plan\Operation\SetVariableLabelOperation; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\Recode\CopyResult; +use OpenStatSpec\Transformation\Plan\Recode\ExactMatch; +use OpenStatSpec\Transformation\Plan\Recode\LiteralResult; +use OpenStatSpec\Transformation\Plan\Recode\RecodeRule; +use OpenStatSpec\Transformation\Plan\TargetMode; +use OpenStatSpec\Transformation\Plan\TransformationPlan; +use OpenStatSpec\Transformation\Plan\Value\Binary64Value; +use OpenStatSpec\Transformation\Plan\Value\StringValue; use PDO; use PDOStatement; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class InPlaceTransformationExecutorTest extends TestCase { - private const DATASET_ID = '018f47f2-8b6a-7c3d-9e1f-123456789abc'; + private const DATASET_ID = '018f47f2-8b6a-4c3d-8e1f-123456789abc'; private PDO $pdo; protected function setUp(): void { if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { - self::markTestSkipped('PDO SQLite is not available in this PHP environment.'); + self::markTestSkipped('PDO SQLite is not available.'); } $this->pdo = new PDO('sqlite::memory:', options: [ PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, @@ -48,682 +52,214 @@ protected function setUp(): void ]); $this->pdo->exec('PRAGMA foreign_keys = ON'); (new NormativeCatalog($this->pdo))->createTables(); - $this->pdo->exec( - 'CREATE TABLE respondents (__case_ordinal INTEGER NOT NULL PRIMARY KEY, source_value REAL NULL, destination REAL NULL)', - ); - $this->pdo->prepare( - 'INSERT INTO dataset ' - . '(dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)', - )->execute([self::DATASET_ID, '1.0', 'fixture', null, 'respondents', 'survey', 5, '2026-07-31 00:00:00']); - $insertVariable = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' - . 'VALUES (?, ?, ?, ?, ?, ?)', - ); - $insertVariable->execute(['018f47f2-8b6a-7c3d-9e1f-123456789abd', self::DATASET_ID, 1, 'SourceValue', 'source_value', 'numeric']); - $insertVariable->execute(['018f47f2-8b6a-7c3d-9e1f-123456789abe', self::DATASET_ID, 2, 'Destination', 'destination', 'numeric']); - $insertCase = $this->pdo->prepare( - 'INSERT INTO respondents (__case_ordinal, source_value, destination) VALUES (?, ?, ?)', - ); - foreach ([[1, 1.0, -1.0], [2, 2.0, -1.0], [3, 3.0, -1.0], [4, 9.0, -1.0], [5, null, -1.0]] as $case) { - $insertCase->execute($case); - } (new TransformationAuditMigrator($this->pdo))->migrate(); CatalogOwnership::markCurrentVersion($this->pdo); - } - - public function testCanonicalOperationsMutateDataAndMetadataInPlace(): void - { - $tablesBefore = $this->tableNames(); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'Destination', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(10)), - ), - new RecodeRule( - new NumericRangeSelector(2.0, 3.0), - new AssignValueAction(ScalarValue::number(20)), - ), - new RecodeRule( - new MissingValueSelector(), - new AssignValueAction(ScalarValue::number(99)), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - new SetVariableLabelOperation('Destination', 'Recoded destination'), - new SetValueLabelsOperation('Destination', [ - new ValueLabel(ScalarValue::number(10), 'Ten'), - new ValueLabel(ScalarValue::number(20), 'Twenty'), - new ValueLabel(ScalarValue::number(99), 'Missing source'), - ]), - ]); - - $result = (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame(self::DATASET_ID, $result->datasetId()); - self::assertSame($plan->hash(), $result->planHash()); - self::assertSame(3, $result->operationCount()); - self::assertNull($result->auditOperationId(), 'Execution must not create a missing journal schema.'); - self::assertSame([10.0, 20.0, 20.0, 9.0, 99.0], $this->query( - 'SELECT destination FROM respondents ORDER BY __case_ordinal', - )->fetchAll(PDO::FETCH_COLUMN)); - self::assertSame('Recoded destination', $this->query( - "SELECT variable_label FROM variable WHERE source_name = 'Destination'", - )->fetchColumn()); - self::assertSame( - [['10.0', 'Ten'], ['20.0', 'Twenty'], ['99.0', 'Missing source']], - $this->query( - 'SELECT CAST(vl.numeric_code AS TEXT), vl.label FROM value_label vl ORDER BY vl.ordinal', - )->fetchAll(PDO::FETCH_NUM), + $this->pdo->exec('CREATE TABLE "odd""table" (__case_ordinal INTEGER NOT NULL PRIMARY KEY, "odd""source" REAL NULL, "target"";DROP TABLE dataset;--" REAL NULL, text_value TEXT COLLATE NOCASE NOT NULL)'); + $this->pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([self::DATASET_ID, '1.0', 'fixture', null, 'odd"table', 'Executor fixture', 2, '2026-08-17 00:00:00']); + $insert = $this->pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) VALUES (?, ?, ?, ?, ?, ?, ?)', ); - self::assertSame($tablesBefore, $this->tableNames()); - self::assertSame(1, (int) $this->query('SELECT COUNT(*) FROM dataset')->fetchColumn()); - self::assertFalse(in_array('operation_catalog', $this->tableNames(), true)); + $insert->execute(['11111111-1111-4111-8111-111111111111', self::DATASET_ID, 1, 'source', 'odd"source', 'numeric', null]); + $insert->execute(['22222222-2222-4222-8222-222222222222', self::DATASET_ID, 2, 'target', 'target";DROP TABLE dataset;--', 'numeric', null]); + $insert->execute(['33333333-3333-4333-8333-333333333333', self::DATASET_ID, 3, 'text', 'text_value', 'string', 8]); + $insertCase = $this->pdo->prepare('INSERT INTO "odd""table" (__case_ordinal, "odd""source", "target"";DROP TABLE dataset;--", text_value) VALUES (?, ?, ?, ?)'); + $insertCase->execute([1, 2.0, 7.0, 'A']); + $insertCase->execute([2, null, 7.0, 'B']); } - public function testSqliteAddsANewTargetInsideTheSameWideTableAndTransaction(): void + public function testIdentifiersAndValuesAreBoundWithoutSqlInjection(): void { - $tablesBefore = $this->tableNames(); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'CreatedTarget', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(100)), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), + $label = "x'); DROP TABLE dataset; --"; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), + new SetVariableLabelOperation('target', $label), ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); - self::assertSame($tablesBefore, $this->tableNames(), 'A transformation must not create a table or snapshot.'); - self::assertSame(1, (int) $this->query('SELECT COUNT(*) FROM dataset')->fetchColumn()); - self::assertSame(3, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertSame('createdtarget', $this->query( - "SELECT physical_name FROM variable WHERE source_name = 'CreatedTarget'", - )->fetchColumn()); - self::assertSame( - ['5', 8, 0, '5', 8, 0], - $this->query( - "SELECT print_format_family, print_format_width, print_format_decimals, " - . "write_format_family, write_format_width, write_format_decimals " - . "FROM variable WHERE source_name = 'CreatedTarget'", - )->fetch(PDO::FETCH_NUM), - ); - self::assertSame([100.0, 2.0, 3.0, 9.0, null], $this->query( - 'SELECT createdtarget FROM respondents ORDER BY __case_ordinal', - )->fetchAll(PDO::FETCH_COLUMN)); + self::assertSame([2.0, null], $this->numericColumn('target";DROP TABLE dataset;--')); + self::assertSame($label, $this->scalar("SELECT variable_label FROM variable WHERE source_name = 'target'")); + self::assertSame(1, (int) $this->scalar('SELECT COUNT(*) FROM dataset')); + self::assertSame(1, (int) $this->scalar('SELECT COUNT(*) FROM transformation_apply')); } - public function testImplicitRecodeTargetRemainsActiveForLaterMetadataOperation(): void + public function testStringRecodeUsesExactBoundValuesDespiteColumnCollation(): void { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'CreatedTarget', [ - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - new SetVariableLabelOperation('CreatedTarget', 'Created target'), + $payload = "x'); DROP TABLE dataset; --"; + $this->pdo->exec("UPDATE \"odd\"\"table\" SET text_value = CASE __case_ordinal WHEN 1 THEN 'Match' ELSE 'match' END"); + $plan = new TransformationPlan(PlanContract::V01, 'parent', [ + new RecodeOperation('text', 'text', TargetMode::Replace, [ + new RecodeRule(new ExactMatch(new StringValue('Match')), new LiteralResult(new StringValue($payload))), + ], new CopyResult()), ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); - self::assertSame('Created target', $this->query( - "SELECT variable_label FROM variable WHERE source_name = 'CreatedTarget'", - )->fetchColumn()); + self::assertSame([$payload, 'match'], $this->column('SELECT text_value FROM "odd""table" ORDER BY __case_ordinal')); + self::assertSame(1, (int) $this->scalar('SELECT COUNT(*) FROM dataset')); } - public function testDeleteThenRecodeCanReuseVariableName(): void + /** @return iterable */ + public static function invalidBindings(): iterable { - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('Destination'), - new RecodeOperation('SourceValue', 'Destination', [ - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(7))), + yield 'unknown value variable' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('missing')), ]), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - $physicalName = (string) $this->query( - "SELECT physical_name FROM variable WHERE source_name = 'Destination'", - )->fetchColumn(); - self::assertNotSame('destination', $physicalName); - self::assertSame( - [7.0, 7.0, 7.0, 7.0, 7.0], - $this->query('SELECT ' . $physicalName . ' FROM respondents ORDER BY __case_ordinal')->fetchAll(PDO::FETCH_COLUMN), - ); - } - - public function testExplicitStringCreateAndDeleteRemovesDataAndMetadata(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('TempText', 'string', 20), - new SetVariableLabelOperation('TempText', 'Temporary text'), - new SetValueLabelsOperation('TempText', [ - new ValueLabel(ScalarValue::string('yes'), 'Yes'), + 'unknown_variable', + ]; + yield 'create target already exists' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Create, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]), - new DeleteVariableOperation('TempText'), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame(2, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertFalse(in_array('temptext', $this->tableColumns(), true)); - self::assertSame(0, (int) $this->query( - 'SELECT COUNT(*) FROM variable_value_label_set', - )->fetchColumn()); - self::assertSame(0, (int) $this->query( - 'SELECT COUNT(*) FROM value_label', - )->fetchColumn()); - } - - - public function testDeletingOnlyVariableThenRecreatingWithinPlan(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('Destination'), - new DeleteVariableOperation('SourceValue'), - new CreateVariableOperation('SourceValue', 'string', 20), - ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::assertSame(1, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - $physicalName = (string) $this->query("SELECT physical_name FROM variable WHERE source_name = 'SourceValue'")->fetchColumn(); - self::assertNotSame('source_value', $physicalName); - self::assertSame('string', $this->query("SELECT storage_kind FROM variable WHERE source_name = 'SourceValue'")->fetchColumn()); - self::assertTrue(in_array($physicalName, $this->tableColumns(), true)); - self::assertFalse(in_array('source_value', $this->tableColumns(), true)); - } - public function testCreatedVariableCannotUseCaseOrdinalColumn(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('__case_ordinal', 'string', 1), - ]); - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - $physicalName = (string) $this->query("SELECT physical_name FROM variable WHERE source_name = '__case_ordinal'")->fetchColumn(); - self::assertNotSame('__case_ordinal', $physicalName); - self::assertTrue(in_array($physicalName, $this->tableColumns(), true)); - } - public function testDeleteThenCreateCanReuseVariableName(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('SourceValue'), - new CreateVariableOperation('SourceValue', 'string', 20), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - $physicalName = (string) $this->query( - "SELECT physical_name FROM variable WHERE source_name = 'SourceValue'", - )->fetchColumn(); - self::assertNotSame('source_value', $physicalName); - self::assertSame('string', $this->query( - "SELECT storage_kind FROM variable WHERE source_name = 'SourceValue'", - )->fetchColumn()); - self::assertTrue(in_array($physicalName, $this->tableColumns(), true)); - self::assertFalse(in_array('source_value', $this->tableColumns(), true)); - } - - public function testDeletingLastMultipleResponseMemberRemovesSet(): void - { - $this->pdo->prepare('INSERT INTO multiple_response_set (multiple_response_set_id, dataset_id, source_ordinal, set_name, set_kind) VALUES (?, ?, ?, ?, ?)')->execute(['mr-source', self::DATASET_ID, 1, '$MR', 'MC']); - $this->pdo->prepare('INSERT INTO multiple_response_member (multiple_response_set_id, variable_id, source_ordinal) VALUES (?, ?, ?)')->execute(['mr-source', '018f47f2-8b6a-7c3d-9e1f-123456789abd', 1]); - - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('SourceValue'), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame(0, (int) $this->query('SELECT COUNT(*) FROM multiple_response_set')->fetchColumn()); - self::assertSame(0, (int) $this->query('SELECT COUNT(*) FROM multiple_response_member')->fetchColumn()); - } - - public function testDeletingLastVariableSetMemberRemovesSet(): void - { - $this->pdo->prepare('INSERT INTO variable_set (variable_set_id, dataset_id, source_ordinal, set_name) VALUES (?, ?, ?, ?)')->execute(['vs-source', self::DATASET_ID, 1, 'Core']); - $this->pdo->prepare('INSERT INTO variable_set_member (variable_set_id, variable_id, source_ordinal) VALUES (?, ?, ?)')->execute(['vs-source', '018f47f2-8b6a-7c3d-9e1f-123456789abd', 1]); - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('SourceValue'), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame(0, (int) $this->query('SELECT COUNT(*) FROM variable_set')->fetchColumn()); - self::assertSame(0, (int) $this->query('SELECT COUNT(*) FROM variable_set_member')->fetchColumn()); - } - - public function testCreateDeleteCreateRecreatesVariable(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('TempText', 'string', 20), - new DeleteVariableOperation('TempText'), - new CreateVariableOperation('TempText', 'string', 30), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertTrue(in_array('temptext_2', $this->tableColumns(), true)); - self::assertSame(30, (int) $this->query( - "SELECT declared_string_width FROM variable WHERE source_name = 'TempText'", - )->fetchColumn()); - } - - public function testExplicitStringCreateInitializesBlankValuesAndAFormats(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('TempText', 'string', 20), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame(['', '', '', '', ''], $this->query( - 'SELECT temptext FROM respondents ORDER BY __case_ordinal', - )->fetchAll(PDO::FETCH_COLUMN)); - self::assertSame(0, (int) $this->query( - 'SELECT COUNT(*) FROM respondents WHERE temptext IS NULL', - )->fetchColumn()); - self::assertSame( - ['1', 20, 0, '1', 20, 0], - $this->query( - "SELECT print_format_family, print_format_width, print_format_decimals, " - . "write_format_family, write_format_width, write_format_decimals " - . "FROM variable WHERE source_name = 'TempText'", - )->fetch(PDO::FETCH_NUM), - ); - } - - public function testRecodeTargetCanReuseSlotFreedByEarlierDelete(): void - { - $connection = new Connection($this->pdo); - $maximum = $connection->profile->effectiveMaximumSourceVariables($this->pdo); - $columns = ['__case_ordinal INTEGER NOT NULL PRIMARY KEY']; - for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { - $columns[] = 'v' . $ordinal . ' REAL NULL'; - } - - $this->pdo->exec('DROP TABLE respondents'); - $this->pdo->exec('CREATE TABLE respondents (' . implode(', ', $columns) . ')'); - $this->pdo->exec('DELETE FROM variable'); - $insert = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' - . 'VALUES (?, ?, ?, ?, ?, ?)', - ); - $this->pdo->beginTransaction(); - for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { - $name = 'V' . $ordinal; - $insert->execute([ - sprintf('00000000-0000-4000-8000-%012d', $ordinal), - self::DATASET_ID, - $ordinal, - $name, - strtolower($name), - 'numeric', - ]); - } - $this->pdo->commit(); - - $plan = new TransformationPlan(self::DATASET_ID, [ - new DeleteVariableOperation('V' . $maximum), - new RecodeOperation('V1', 'Replacement', [ - new RecodeRule(new ElseSelector(), new CopySourceAction()), + 'target_already_exists', + ]; + yield 'replace target does not exist' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('missing', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]), - ]); - - (new InPlaceTransformationExecutor($connection))->execute($plan); - - self::assertSame($maximum, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertSame($maximum, count($this->query('PRAGMA table_info(respondents)')->fetchAll()) - 1); - self::assertFalse(in_array('v' . $maximum, $this->tableColumns(), true)); - self::assertTrue(in_array('replacement', $this->tableColumns(), true)); - } - - public function testExplicitLongStringCreateCapsAFormatWidthsButKeepsStorageWidth(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('LongText', 'string', 400), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame( - [400, '1', 255, '1', 255], - $this->query( - "SELECT declared_string_width, print_format_family, print_format_width, " - . "write_format_family, write_format_width " - . "FROM variable WHERE source_name = 'LongText'", - )->fetch(PDO::FETCH_NUM), - ); + 'unknown_variable', + ]; + yield 'conditional target does not exist' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new ConditionalAssignOperation( + new Comparison( + new VariableOperand('source'), + '=', + new LiteralOperand(Binary64Value::fromBits('3ff0000000000000')), + ), + 'missing', + new LiteralOperand(Binary64Value::fromBits('3ff0000000000000')), + ), + ]), + 'conditional_target_missing', + ]; + yield 'string expression variable' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('text')), + ]), + 'expression_type_unsupported', + ]; + yield 'format target is string' => [ + new TransformationPlan(PlanContract::V02, 'parent', [new SetFormatOperation('text', 'F', 8, 2)]), + 'type_mismatch', + ]; + yield 'invalid direct format object' => [ + new TransformationPlan(PlanContract::V02, 'parent', [new SetFormatOperation('target', 'F', 2, 2)]), + 'invalid_format', + ]; + yield 'later invalid operation prevents earlier mutation' => [ + new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + new SetVariableLabelOperation('missing', 'Never applied'), + ]), + 'unknown_variable', + ]; } - public function testNewTargetIsRejectedBeforeAlterAtTheEffectiveColumnLimit(): void + #[DataProvider('invalidBindings')] + public function testCompleteBindingValidationRunsBeforeMutation(TransformationPlan $plan, string $code): void { - $connection = new Connection($this->pdo); - $maximum = $connection->profile->effectiveMaximumSourceVariables($this->pdo); - $columns = ['__case_ordinal INTEGER NOT NULL PRIMARY KEY']; - for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { - $columns[] = 'v' . $ordinal . ' REAL NULL'; - } - - $this->pdo->exec('DROP TABLE respondents'); - $this->pdo->exec('CREATE TABLE respondents (' . implode(', ', $columns) . ')'); - $this->pdo->exec('DELETE FROM variable'); - $insert = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' - . 'VALUES (?, ?, ?, ?, ?, ?)', - ); - $this->pdo->beginTransaction(); - for ($ordinal = 1; $ordinal <= $maximum; ++$ordinal) { - $name = 'V' . $ordinal; - $insert->execute([ - sprintf('00000000-0000-4000-8000-%012d', $ordinal), - self::DATASET_ID, - $ordinal, - $name, - strtolower($name), - 'numeric', - ]); - } - $this->pdo->commit(); - $columnsBefore = count($this->query('PRAGMA table_info(respondents)')->fetchAll()); - - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('V1', 'OverflowTarget', [ - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - ]); + $before = $this->state(); try { - (new InPlaceTransformationExecutor($connection))->execute($plan); - self::fail('A target beyond the effective source-variable limit was created.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); - self::assertStringContainsString('at most ' . $maximum . ' source variables', $exception->getMessage()); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('Invalid plan unexpectedly mutated the dataset.'); + } catch (TransformationFailure $failure) { + self::assertSame($code, $failure->diagnosticCode()); } - self::assertSame($maximum, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertSame($columnsBefore, count($this->query('PRAGMA table_info(respondents)')->fetchAll())); - } - - public function testRecodeWithOnlyElseAssignsTheExpressionDirectly(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'Destination', [ - new RecodeRule( - new ElseSelector(), - new AssignValueAction(ScalarValue::number(7)), - ), - ]), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame( - [7.0, 7.0, 7.0, 7.0, 7.0], - $this->query('SELECT destination FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), - ); + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); } - public function testCallerOwnedTransactionIsRejectedWithoutRollbackOrJournalWork(): void + public function testAuditSchemaMustBeReadyBeforeMutation(): void { - $tablesBefore = $this->tableNames(); - $this->pdo->beginTransaction(); - $this->pdo->exec('UPDATE respondents SET source_value = 42 WHERE __case_ordinal = 1'); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceValue', 'Destination', [ - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(7))), - ]), + $this->pdo->exec('DROP TABLE transformation_apply'); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]); + $before = $this->numericColumn('target";DROP TABLE dataset;--'); try { - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::fail('Execution unexpectedly joined a caller-owned transaction.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::UnsupportedOperation, $exception->diagnosticCode); - self::assertStringContainsString('caller-owned active transaction', $exception->getMessage()); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('Apply executed without the explicit audit schema.'); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $failure->diagnosticCode); } - self::assertTrue($this->pdo->inTransaction()); - self::assertSame(42.0, $this->query( - 'SELECT source_value FROM respondents WHERE __case_ordinal = 1', - )->fetchColumn()); - self::assertSame($tablesBefore, $this->tableNames()); - self::assertFalse(in_array('operation_catalog', $this->tableNames(), true)); - self::assertSame( - [-1.0, -1.0, -1.0, -1.0, -1.0], - $this->query('SELECT destination FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), - ); - - self::assertTrue($this->pdo->commit()); + self::assertSame($before, $this->numericColumn('target";DROP TABLE dataset;--')); self::assertFalse($this->pdo->inTransaction()); - self::assertSame(42.0, $this->query( - 'SELECT source_value FROM respondents WHERE __case_ordinal = 1', - )->fetchColumn()); } - public function testSystemMissingActionForStringTargetFailsBeforeMutation(): void + public function testCallerOwnedTransactionIsRejectedWithoutTouchingIt(): void { - $this->pdo->exec("ALTER TABLE respondents ADD COLUMN source_text TEXT NOT NULL DEFAULT ''"); - $this->pdo->exec("ALTER TABLE respondents ADD COLUMN destination_text TEXT NOT NULL DEFAULT 'original'"); - $insertVariable = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - ); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789abf', - self::DATASET_ID, - 3, - 'SourceText', - 'source_text', - 'string', - 8, - ]); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789ac0', - self::DATASET_ID, - 4, - 'DestinationText', - 'destination_text', - 'string', - 8, - ]); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'DestinationText', [ - new RecodeRule(new ElseSelector(), new SetMissingAction()), - ]), + $this->pdo->beginTransaction(); + $this->pdo->exec('UPDATE "odd""table" SET "target"";DROP TABLE dataset;--" = 42 WHERE __case_ordinal = 1'); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), ]); try { - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::fail('System-missing recoding into a string target unexpectedly executed.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); - self::assertStringContainsString('not representable', $exception->getMessage()); + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('Apply unexpectedly joined the caller transaction.'); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::UnsupportedOperation, $failure->diagnosticCode); } - self::assertSame( - ['original', 'original', 'original', 'original', 'original'], - $this->query('SELECT destination_text FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), - ); + self::assertTrue($this->pdo->inTransaction()); + self::assertSame([42.0, 7.0], $this->numericColumn('target";DROP TABLE dataset;--')); + $this->pdo->rollBack(); } - public function testExactStringSelectorIgnoresCaseInsensitiveColumnCollation(): void + private function request(TransformationPlan $plan): InPlaceApplyRequest { - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT COLLATE NOCASE NULL'); - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN destination_text TEXT NULL'); - $insertVariable = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - ); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789abf', self::DATASET_ID, 3, - 'SourceText', 'source_text', 'string', 8, - ]); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789ac0', self::DATASET_ID, 4, - 'DestinationText', 'destination_text', 'string', 8, - ]); - $this->pdo->exec("UPDATE respondents SET source_text = CASE __case_ordinal WHEN 1 THEN 'Match' WHEN 2 THEN 'match' ELSE 'other' END"); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'DestinationText', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::string('Match')), - new AssignValueAction(ScalarValue::string('exact')), - ), - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::string('else'))), - ]), - ]); - - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - - self::assertSame( - ['exact', 'else', 'else', 'else', 'else'], - $this->query('SELECT destination_text FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), - ); + return new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, str_repeat('a', 64), 'unit-test'); } - public function testNewStringTargetRequiresExplicitCatalogWidth(): void + /** @return array */ + private function state(): array { - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT NULL'); - $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - )->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789abf', - self::DATASET_ID, - 3, - 'SourceText', - 'source_text', - 'string', - 8, - ]); - $tablesBefore = $this->tableNames(); - $variablesBefore = (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn(); - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'CreatedText', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::string('a')), - new AssignValueAction(ScalarValue::string('b')), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - ]); - - try { - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::fail('A string target without declared_string_width unexpectedly executed.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); - } - - self::assertSame($tablesBefore, $this->tableNames()); - self::assertSame($variablesBefore, (int) $this->query('SELECT COUNT(*) FROM variable')->fetchColumn()); - self::assertFalse(in_array('createdtext', $this->tableColumns(), true)); + return [ + 'rows' => $this->rows('SELECT * FROM "odd""table" ORDER BY __case_ordinal'), + 'variables' => $this->rows('SELECT * FROM variable ORDER BY source_ordinal'), + 'audit' => $this->rows('SELECT * FROM transformation_apply ORDER BY apply_id'), + 'tables' => $this->column("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name"), + ]; } - public function testExistingStringTargetEnforcesNormativeByteWidthBeforeMutation(): void + /** @return list */ + private function numericColumn(string $column): array { - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN source_text TEXT NULL'); - $this->pdo->exec('ALTER TABLE respondents ADD COLUMN destination_text TEXT NULL'); - $insertVariable = $this->pdo->prepare( - 'INSERT INTO variable ' - . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - ); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789abf', - self::DATASET_ID, - 3, - 'SourceText', - 'source_text', - 'string', - 4, - ]); - $insertVariable->execute([ - '018f47f2-8b6a-7c3d-9e1f-123456789ac0', - self::DATASET_ID, - 4, - 'DestinationText', - 'destination_text', - 'string', - 1, - ]); - - $plans = [ - new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'DestinationText', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::string('a')), - new AssignValueAction(ScalarValue::string('long')), - ), - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::string('x'))), - ]), - ]), - new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('SourceText', 'DestinationText', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::string('a')), - new AssignValueAction(ScalarValue::string('x')), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - ]), - new TransformationPlan(self::DATASET_ID, [ - new SetValueLabelsOperation('DestinationText', [ - new ValueLabel(ScalarValue::string('long'), 'Too wide'), - ]), - ]), - ]; - - foreach ($plans as $plan) { - try { - (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($plan); - self::fail('A string operation wider than declared_string_width unexpectedly executed.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::InvalidSourceDataset, $exception->diagnosticCode); - } - } - self::assertSame( - [null, null, null, null, null], - $this->query('SELECT destination_text FROM respondents ORDER BY __case_ordinal') - ->fetchAll(PDO::FETCH_COLUMN), + $quoted = (new Connection($this->pdo))->profile->quoteIdentifier($column); + return array_map( + static fn(mixed $value): ?float => $value === null ? null : (float) $value, + $this->column('SELECT ' . $quoted . ' FROM "odd""table" ORDER BY __case_ordinal'), ); } - /** @return list */ - private function tableNames(): array + private function scalar(string $sql): mixed { - $names = $this->query( - "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", - )->fetchAll(PDO::FETCH_COLUMN); - - return array_values(array_map(static fn(mixed $name): string => (string) $name, $names)); + $statement = $this->pdo->query($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + return $statement->fetchColumn(); } - /** @return list */ - private function tableColumns(): array + /** @return list */ + private function column(string $sql): array { - $names = $this->query('PRAGMA table_info(respondents)')->fetchAll(PDO::FETCH_ASSOC); - - return array_values(array_map( - static fn(array $column): string => (string) $column['name'], - $names, - )); + $statement = $this->pdo->query($sql); + self::assertInstanceOf(PDOStatement::class, $statement); + return array_values($statement->fetchAll(PDO::FETCH_COLUMN)); } - private function query(string $sql): PDOStatement + /** @return list> */ + private function rows(string $sql): array { $statement = $this->pdo->query($sql); self::assertInstanceOf(PDOStatement::class, $statement); - - return $statement; + return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); } } From 502c590bfe41a6645db75afb9ffb15af9c42bdba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 19:07:39 +0300 Subject: [PATCH 15/53] fix: harden atomic official plan execution --- .../task-8-report.md | 16 + src/Sql/CatalogOwnership.php | 14 + .../Audit/TransformationAuditWriter.php | 27 +- src/Transformation/Execution/CheckedPdo.php | 82 ++++ .../Execution/InPlaceOperationExecutor.php | 115 +++--- .../InPlaceTransformationExecutor.php | 12 +- .../Execution/PlanPreflight.php | 95 ++++- .../Execution/SqlPredicateCompiler.php | 11 +- .../Plan/Value/Binary64Value.php | 12 + .../OfficialInPlaceTransformation02Test.php | 29 +- .../InPlaceTransformationExecutorTest.php | 365 +++++++++++++++++- 11 files changed, 679 insertions(+), 99 deletions(-) create mode 100644 src/Transformation/Execution/CheckedPdo.php diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md index e33b7e3..1e609f3 100644 --- a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md @@ -41,3 +41,19 @@ ## Environment note `OPENSTATSPEC_PG_DSN`, `OPENSTATSPEC_PG_USER`, and `OPENSTATSPEC_PG_PASSWORD` were not configured locally, so the two new live PostgreSQL success/rollback probes and the existing PostgreSQL integrations skipped. SQLite supplied the local native-DDL transaction evidence; the PostgreSQL paths remain in the normalized full suite and run automatically when the documented environment is configured. + +## Review follow-up + +- RED: 12 new regression scenarios initially produced 13 failures. They reproduced lossy PDO float binding, text-ordered literal predicates, unchecked false PDO outcomes, cross-connection audit injection, reserved/shared physical bindings, late string-width and value-label validation, and mutation during fresh-database preflight. +- Binary64 SQL parameters now use a 17-significant-digit locale-independent decimal representation and an explicit backend numeric cast (`REAL` or `DOUBLE PRECISION`). SQLite behavior tests prove adjacent-binary64 assignment, literal-only numeric ordering, recode match/result precision, and numeric value-label precision; a PostgreSQL compiler regression proves exact decimal parameters and `DOUBLE PRECISION` casts without interpolating values. +- Every apply-boundary `prepare`, `execute`, `exec`, `beginTransaction`, `commit`, and `rollBack` outcome is checked. The preflight, operation executor, and audit writer are constructor-verified against the exact same PDO instance; false execute/begin/commit/rollback and same-PDO audit-failure tests prove no successful result or audit can escape a failed transaction. +- Preflight now rejects `__`-reserved physical variable names, duplicate logical ownership of one physical table, cross-dataset value-label associations, and over-width string recode outputs before any UPDATE. The transformation-specific ownership check is read-only on an empty database while legacy import initialization retains its prior behavior. +- The official SQLite and environment-gated PostgreSQL rollback probes now fail the audit insert on the same connection and native transaction rather than injecting a second PDO. + +## Review verification + +- Focused executor and official 0.2 integration tests: 34 tests, 398 assertions, 2 unconfigured PostgreSQL skips; passed. +- Full PHPUnit suite: 435 tests, 2,731 assertions, 32 environment skips; passed. +- `composer analyse`: passed with no errors; `composer lint`: passed. +- Task-changed-file PHP CS Fixer dry-run: passed. Repository-wide working-tree `composer style` still reports the pre-existing CRLF-only differences; the LF-normalized staged-tree `composer check` passed all validation, lint, style, PHPStan, and 435-test PHPUnit gates. +- PostgreSQL environment remained unavailable (`OPENSTATSPEC_PG_DSN`, user, and password unset), so live PostgreSQL execution was skipped explicitly; dialect SQL and the gated same-transaction rollback path remain covered in tests. diff --git a/src/Sql/CatalogOwnership.php b/src/Sql/CatalogOwnership.php index a0a10a3..8960adf 100644 --- a/src/Sql/CatalogOwnership.php +++ b/src/Sql/CatalogOwnership.php @@ -122,6 +122,20 @@ public static function assertReadyForUse(PDO $pdo): void throw self::migrationRequired(); } + /** Transformation preflight must never initialize a fresh namespace. */ + public static function assertReadyForUseReadOnly(PDO $pdo): void + { + if (!self::tableExists($pdo, self::IDENTITY_TABLE) + && !self::tableExists($pdo, self::MIGRATION_TABLE) + && self::catalogCollisions($pdo) === [] + && self::namespaceObjects($pdo) === [] + ) { + throw self::migrationRequired(); + } + + self::assertReadyForUse($pdo); + } + public static function isFreshPending(PDO $pdo): bool { if (!self::tableExists($pdo, self::IDENTITY_TABLE) diff --git a/src/Transformation/Audit/TransformationAuditWriter.php b/src/Transformation/Audit/TransformationAuditWriter.php index 65b1e5c..25750f1 100644 --- a/src/Transformation/Audit/TransformationAuditWriter.php +++ b/src/Transformation/Audit/TransformationAuditWriter.php @@ -7,11 +7,11 @@ use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; use OpenStatSpec\Transformation\Execution\DatasetBinding; use OpenStatSpec\Transformation\Execution\DoltEvidence; +use OpenStatSpec\Transformation\Execution\CheckedPdo; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Plan\PlanCodec; use OpenStatSpec\Transformation\Plan\PlanContract; use PDO; -use PDOException; /** Writes only the one compact success record inside the caller's apply transaction. */ final readonly class TransformationAuditWriter @@ -23,6 +23,11 @@ public function __construct(private PDO $pdo, ?PlanCodec $codec = null) $this->codec = $codec ?? new PlanCodec(); } + public function isFor(PDO $pdo): bool + { + return $this->pdo === $pdo; + } + public function succeed( InPlaceApplyRequest $request, DatasetBinding $dataset, @@ -48,18 +53,15 @@ public function succeed( ? 'openstatspec-in-place-transformation-v0.1' : 'openstatspec-in-place-transformation-v0.2'; - $statement = $this->pdo->prepare(<<<'SQL' + $statement = CheckedPdo::prepare($this->pdo, <<<'SQL' INSERT INTO transformation_apply ( apply_id, contract_id, database_profile, dataset_id, physical_table_schema, physical_table_name, source_hash, plan_hash, canonical_plan_json, actor, status, dolt_branch, dolt_head_before, dolt_head_after, operation_count, started_at, completed_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) -SQL); - if ($statement === false) { - throw new PDOException('The transformation audit insert could not be prepared.'); - } - $statement->execute([ +SQL, 'The transformation audit insert could not be prepared.'); + CheckedPdo::execute($statement, [ $applyId, $contract, $databaseProfile, @@ -77,20 +79,19 @@ public function succeed( count($request->plan->operations), $timestamp, $timestamp, - ]); + ], 'The transformation audit insert could not be executed.'); return $applyId; } private function assertDatasetIdentity(DatasetBinding $dataset): void { - $statement = $this->pdo->prepare( + $statement = CheckedPdo::prepare( + $this->pdo, 'SELECT physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', + 'The audit target dataset identity could not be prepared.', ); - if ($statement === false) { - throw new PDOException('The audit target dataset identity could not be prepared.'); - } - $statement->execute([$dataset->datasetId]); + CheckedPdo::execute($statement, [$dataset->datasetId], 'The audit target dataset identity could not be queried.'); $rows = $statement->fetchAll(PDO::FETCH_ASSOC); if (count($rows) !== 1 || ($rows[0]['physical_table_schema'] ?? null) !== $dataset->schema diff --git a/src/Transformation/Execution/CheckedPdo.php b/src/Transformation/Execution/CheckedPdo.php new file mode 100644 index 0000000..8f17963 --- /dev/null +++ b/src/Transformation/Execution/CheckedPdo.php @@ -0,0 +1,82 @@ +prepare($sql); + if ($statement === false) { + throw self::failure($message, $pdo->errorInfo()); + } + + return $statement; + } + + /** @param list $parameters */ + public static function execute(PDOStatement $statement, array $parameters, string $message): void + { + if (!$statement->execute($parameters)) { + throw self::failure($message, $statement->errorInfo()); + } + } + + public static function exec(PDO $pdo, string $sql, string $message): int + { + $affected = $pdo->exec($sql); + if ($affected === false) { + throw self::failure($message, $pdo->errorInfo()); + } + + return $affected; + } + + public static function begin(PDO $pdo): void + { + if (!$pdo->beginTransaction()) { + throw self::failure('The SQL driver did not start the transformation transaction.', $pdo->errorInfo()); + } + } + + public static function commit(PDO $pdo): void + { + if (!$pdo->commit()) { + throw self::failure('The SQL driver did not commit the transformation transaction.', $pdo->errorInfo()); + } + } + + public static function rollback(PDO $pdo, Throwable $applyFailure): void + { + try { + if (!$pdo->rollBack()) { + throw self::failure('The SQL driver did not roll back the failed transformation transaction.', $pdo->errorInfo()); + } + } catch (Throwable $rollbackFailure) { + throw new PDOException( + 'The failed transformation transaction rollback failed: ' . $rollbackFailure->getMessage(), + 0, + $applyFailure, + ); + } + } + + /** @param array $errorInfo */ + private static function failure(string $message, array $errorInfo): PDOException + { + $driverMessage = $errorInfo[2] ?? null; + if (is_string($driverMessage) && $driverMessage !== '') { + $message .= ' ' . $driverMessage; + } + + return new PDOException($message); + } +} diff --git a/src/Transformation/Execution/InPlaceOperationExecutor.php b/src/Transformation/Execution/InPlaceOperationExecutor.php index b5d306b..e7b4df8 100644 --- a/src/Transformation/Execution/InPlaceOperationExecutor.php +++ b/src/Transformation/Execution/InPlaceOperationExecutor.php @@ -26,7 +26,6 @@ use OpenStatSpec\Transformation\Plan\Value\StringValue; use OpenStatSpec\Transformation\Plan\Value\TypedValue; use PDO; -use PDOException; use PDOStatement; /** @internal Executes exactly one pre-bound operation inside the caller transaction. */ @@ -39,6 +38,11 @@ public function __construct(private Connection $connection) $this->predicateCompiler = new SqlPredicateCompiler($connection->profile); } + public function isFor(PDO $pdo): bool + { + return $this->connection->pdo === $pdo; + } + public function execute(BoundOperation $bound, DatasetBinding $dataset): void { if (!$this->connection->pdo->inTransaction()) { @@ -53,10 +57,10 @@ public function execute(BoundOperation $bound, DatasetBinding $dataset): void } $parameters = []; $value = $this->predicateCompiler->operand($operation->value, $bound->schema, $parameters); - $this->statement( + CheckedPdo::execute($this->statement( 'UPDATE ' . $dataset->qualifiedTable($this->connection->profile) . ' SET ' . $this->quote($target->physicalName) . ' = ' . $value, - )->execute($parameters); + ), $parameters, 'The assignment UPDATE could not be executed.'); return; } @@ -64,11 +68,11 @@ public function execute(BoundOperation $bound, DatasetBinding $dataset): void $parameters = []; $value = $this->predicateCompiler->operand($operation->value, $bound->schema, $parameters); $condition = $this->predicateCompiler->compile($operation->condition, $bound->schema, $parameters); - $this->statement( + CheckedPdo::execute($this->statement( 'UPDATE ' . $dataset->qualifiedTable($this->connection->profile) . ' SET ' . $this->quote($this->target($bound)->physicalName) . ' = ' . $value . ' WHERE ' . $condition, - )->execute($parameters); + ), $parameters, 'The conditional assignment UPDATE could not be executed.'); return; } @@ -82,9 +86,9 @@ public function execute(BoundOperation $bound, DatasetBinding $dataset): void } if ($operation instanceof SetVariableLabelOperation) { - $this->statement( + CheckedPdo::execute($this->statement( 'UPDATE variable SET variable_label = ? WHERE variable_id = ? AND dataset_id = ?', - )->execute([$operation->label, $this->target($bound)->variableId, $dataset->datasetId]); + ), [$operation->label, $this->target($bound)->variableId, $dataset->datasetId], 'The variable-label UPDATE could not be executed.'); return; } @@ -94,11 +98,11 @@ public function execute(BoundOperation $bound, DatasetBinding $dataset): void } if ($operation instanceof SetFormatOperation) { - $this->statement( + CheckedPdo::execute($this->statement( 'UPDATE variable SET print_format_family = ?, print_format_width = ?, print_format_decimals = ?, ' . 'write_format_family = ?, write_format_width = ?, write_format_decimals = ? ' . 'WHERE variable_id = ? AND dataset_id = ?', - )->execute([ + ), [ $operation->family, $operation->width, $operation->decimals, @@ -107,14 +111,14 @@ public function execute(BoundOperation $bound, DatasetBinding $dataset): void $operation->decimals, $this->target($bound)->variableId, $dataset->datasetId, - ]); + ], 'The variable-format UPDATE could not be executed.'); return; } if ($operation instanceof SetMeasurementLevelOperation) { - $this->statement( + CheckedPdo::execute($this->statement( 'UPDATE variable SET measurement_level = ? WHERE variable_id = ? AND dataset_id = ?', - )->execute([$operation->level, $this->target($bound)->variableId, $dataset->datasetId]); + ), [$operation->level, $this->target($bound)->variableId, $dataset->datasetId], 'The measurement-level UPDATE could not be executed.'); return; } @@ -127,16 +131,18 @@ public function execute(BoundOperation $bound, DatasetBinding $dataset): void private function createNumericTarget(DatasetBinding $dataset, VariableBinding $target): void { - $this->connection->pdo->exec( + CheckedPdo::exec( + $this->connection->pdo, 'ALTER TABLE ' . $dataset->qualifiedTable($this->connection->profile) . ' ADD COLUMN ' . $this->quote($target->physicalName) . ' ' . $this->connection->profile->numericType() . ' NULL', + 'The numeric target column could not be added.', ); - $this->statement( + CheckedPdo::execute($this->statement( 'INSERT INTO variable ' . '(variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind, declared_string_width) ' . 'VALUES (?, ?, ?, ?, ?, ?, ?)', - )->execute([ + ), [ $target->variableId, $dataset->datasetId, $target->sourceOrdinal, @@ -144,7 +150,7 @@ private function createNumericTarget(DatasetBinding $dataset, VariableBinding $t $target->physicalName, 'numeric', null, - ]); + ], 'The numeric target catalog row could not be inserted.'); } private function recode(RecodeOperation $operation, BoundOperation $bound, DatasetBinding $dataset): void @@ -166,11 +172,11 @@ private function recode(RecodeOperation $operation, BoundOperation $bound, Datas $when[] = 'WHEN ' . $condition . ' THEN ' . $this->resultSql($rule->result, $sourceSql, $parameters); } $unmatched = $this->resultSql($operation->unmatched, $sourceSql, $parameters); - $this->statement( + CheckedPdo::execute($this->statement( 'UPDATE ' . $dataset->qualifiedTable($this->connection->profile) . ' SET ' . $this->quote($this->target($bound)->physicalName) . ' = CASE ' . implode(' ', $when) . ' ELSE ' . $unmatched . ' END', - )->execute($parameters); + ), $parameters, 'The recode UPDATE could not be executed.'); } /** @param list $parameters */ @@ -178,11 +184,12 @@ private function exactMatch(string $sourceSql, ExactMatch $match, array &$parame { $conditions = []; foreach ($match->values as $value) { - $parameters[] = $this->value($value); - $conditions[] = $this->connection->profile->exactValueCondition( - $sourceSql, - $value instanceof StringValue, - ); + if ($value instanceof Binary64Value) { + $conditions[] = $sourceSql . ' = ' . $this->predicateCompiler->numericLiteral($value, $parameters); + } else { + $parameters[] = $this->value($value); + $conditions[] = $this->connection->profile->exactValueCondition($sourceSql, true); + } } return '(' . implode(' OR ', $conditions) . ')'; } @@ -190,9 +197,9 @@ private function exactMatch(string $sourceSql, ExactMatch $match, array &$parame /** @param list $parameters */ private function rangeMatch(string $sourceSql, RangeMatch $match, array &$parameters): string { - $parameters[] = $match->lower->number(); - $parameters[] = $match->upper->number(); - return '(' . $sourceSql . ' >= ? AND ' . $sourceSql . ' <= ?)'; + $lower = $this->predicateCompiler->numericLiteral($match->lower, $parameters); + $upper = $this->predicateCompiler->numericLiteral($match->upper, $parameters); + return '(' . $sourceSql . ' >= ' . $lower . ' AND ' . $sourceSql . ' <= ' . $upper . ')'; } /** @param list $parameters */ @@ -205,6 +212,9 @@ private function resultSql(Result $result, string $sourceSql, array &$parameters return 'NULL'; } if ($result instanceof LiteralResult) { + if ($result->value instanceof Binary64Value) { + return $this->predicateCompiler->numericLiteral($result->value, $parameters); + } $parameters[] = $this->value($result->value); return '?'; } @@ -221,7 +231,7 @@ private function replaceValueLabels( . 'JOIN value_label_set value_set ON value_set.value_label_set_id = link.value_label_set_id ' . 'WHERE link.variable_id = ?', ); - $statement->execute([$target->variableId]); + CheckedPdo::execute($statement, [$target->variableId], 'The value-label association could not be queried.'); $rows = $statement->fetchAll(PDO::FETCH_ASSOC); if (count($rows) > 1 || (isset($rows[0]['dataset_id']) && $rows[0]['dataset_id'] !== $dataset->datasetId)) { throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Value-label association is malformed.'); @@ -232,25 +242,34 @@ private function replaceValueLabels( : null; if ($setId !== null) { $references = $this->statement('SELECT COUNT(*) FROM variable_value_label_set WHERE value_label_set_id = ?'); - $references->execute([$setId]); + CheckedPdo::execute($references, [$setId], 'The value-label reference count could not be queried.'); if ((int) $references->fetchColumn() > 1) { - $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?')->execute([$target->variableId]); + CheckedPdo::execute( + $this->statement('DELETE FROM variable_value_label_set WHERE variable_id = ?'), + [$target->variableId], + 'The old shared value-label association could not be removed.', + ); $setId = null; } else { - $this->statement('DELETE FROM value_label WHERE value_label_set_id = ?')->execute([$setId]); + CheckedPdo::execute( + $this->statement('DELETE FROM value_label WHERE value_label_set_id = ?'), + [$setId], + 'The old value labels could not be removed.', + ); } } if ($setId === null) { $setId = NormativeCatalog::uuid(); - $this->statement('INSERT INTO value_label_set (value_label_set_id, dataset_id, name) VALUES (?, ?, ?)')->execute([ - $setId, - $dataset->datasetId, - null, - ]); - $this->statement('INSERT INTO variable_value_label_set (variable_id, value_label_set_id) VALUES (?, ?)')->execute([ - $target->variableId, - $setId, - ]); + CheckedPdo::execute( + $this->statement('INSERT INTO value_label_set (value_label_set_id, dataset_id, name) VALUES (?, ?, ?)'), + [$setId, $dataset->datasetId, null], + 'The value-label set could not be inserted.', + ); + CheckedPdo::execute( + $this->statement('INSERT INTO variable_value_label_set (variable_id, value_label_set_id) VALUES (?, ?)'), + [$target->variableId, $setId], + 'The value-label association could not be inserted.', + ); } $insert = $this->statement( @@ -260,7 +279,7 @@ private function replaceValueLabels( ); foreach ($operation->labels as $index => $label) { $numeric = $label->value instanceof Binary64Value; - $insert->execute([ + CheckedPdo::execute($insert, [ NormativeCatalog::uuid(), $setId, $index + 1, @@ -268,14 +287,14 @@ private function replaceValueLabels( $numeric ? $this->value($label->value) : null, $numeric ? null : $this->value($label->value), $label->label, - ]); + ], 'A replacement value label could not be inserted.'); } } - private function value(TypedValue $value): float|string + private function value(TypedValue $value): string { return match (true) { - $value instanceof Binary64Value => $value->number(), + $value instanceof Binary64Value => $value->decimal(), $value instanceof StringValue => $value->value, default => throw new \LogicException('Unsupported preflighted typed value.'), }; @@ -296,10 +315,10 @@ private function quote(string $identifier): string private function statement(string $sql): PDOStatement { - $statement = $this->connection->pdo->prepare($sql); - if ($statement === false) { - throw new PDOException('Transformation SQL statement could not be prepared.'); - } - return $statement; + return CheckedPdo::prepare( + $this->connection->pdo, + $sql, + 'Transformation SQL statement could not be prepared.', + ); } } diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php index 5894a12..f75a6b4 100644 --- a/src/Transformation/Execution/InPlaceTransformationExecutor.php +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -31,6 +31,12 @@ public function __construct( $this->preflight = $preflight ?? new PlanPreflight($connection, $this->codec); $this->operationExecutor = $operationExecutor ?? new InPlaceOperationExecutor($connection); $this->auditWriter = $auditWriter ?? new TransformationAuditWriter($connection->pdo, $this->codec); + if (!$this->preflight->isFor($connection->pdo) + || !$this->operationExecutor->isFor($connection->pdo) + || !$this->auditWriter->isFor($connection->pdo) + ) { + throw new \InvalidArgumentException('Every transformation executor collaborator must use the same PDO connection.'); + } } public function execute(InPlaceApplyRequest $request): ExecutionResult @@ -49,7 +55,7 @@ public function execute(InPlaceApplyRequest $request): ExecutionResult $transactionStarted = false; try { - $this->connection->pdo->beginTransaction(); + CheckedPdo::begin($this->connection->pdo); $transactionStarted = true; foreach ($bound->operations as $operation) { $this->operationExecutor->execute($operation, $bound->dataset); @@ -62,11 +68,11 @@ public function execute(InPlaceApplyRequest $request): ExecutionResult $doltBefore, $doltAfter, ); - $this->connection->pdo->commit(); + CheckedPdo::commit($this->connection->pdo); $transactionStarted = false; } catch (Throwable $failure) { if ($transactionStarted) { - $this->connection->pdo->rollBack(); + CheckedPdo::rollback($this->connection->pdo, $failure); } throw $failure; } diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index 2bc54d6..7417668 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -52,16 +52,22 @@ public function __construct(private Connection $connection, ?PlanCodec $codec = $this->codec = $codec ?? new PlanCodec(); } + public function isFor(PDO $pdo): bool + { + return $this->connection->pdo === $pdo; + } + public function bind(InPlaceApplyRequest $request): BoundPlan { // Re-decode the public model so directly constructed objects receive the // same complete structural and semantic validation as decoded plans. $this->codec->fromArray($request->plan->canonicalArray()); $this->connection->assertClaimedSupported(); - CatalogOwnership::assertReadyForUse($this->connection->pdo); + CatalogOwnership::assertReadyForUseReadOnly($this->connection->pdo); $this->assertAuditReady(); $dataset = $this->resolveDataset($request->datasetId); + $this->assertExclusivePhysicalTableBinding($dataset); $variables = $this->resolveVariables($dataset); $physicalColumns = $this->physicalColumns($dataset); $this->assertPhysicalBindings($dataset, $variables, $physicalColumns); @@ -176,6 +182,7 @@ private function bindOperation( if ($operation instanceof ReplaceValueLabelsOperation) { $target = $schema->variable($operation->variable, $path . '.variable'); + $this->assertValueLabelAssociations($target, $path . '.variable'); foreach ($operation->labels as $index => $label) { $this->assertValueKind($label->value, $target, $path . '.labels[' . $index . '].value'); } @@ -302,6 +309,12 @@ private function assertRecode( } elseif ($match instanceof SystemMissingMatch && !$source->isNumeric()) { throw TransformationFailure::at('system_missing_for_string', $path . '.rules[' . $index . '].match', 'String variables have no system-missing state.'); } + if ($rule->result instanceof LiteralResult) { + $this->assertValueKind($rule->result->value, $target, $path . '.rules[' . $index . '].result.value'); + } + } + if ($operation->unmatched instanceof LiteralResult) { + $this->assertValueKind($operation->unmatched->value, $target, $path . '.unmatched.value'); } $kind = $this->recodeResultKind($operation, $source, $path); @@ -419,7 +432,7 @@ private function resolveDataset(string $datasetId): DatasetBinding $statement = $this->statement( 'SELECT dataset_id, dataset_name, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', ); - $statement->execute([$datasetId]); + CheckedPdo::execute($statement, [$datasetId], 'The transformation dataset binding could not be queried.'); $rows = $statement->fetchAll(PDO::FETCH_ASSOC); if (count($rows) !== 1) { throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply dataset must resolve to exactly one catalog row.'); @@ -440,6 +453,25 @@ private function resolveDataset(string $datasetId): DatasetBinding return new DatasetBinding($datasetId, $name, $schema, $table); } + private function assertExclusivePhysicalTableBinding(DatasetBinding $dataset): void + { + $statement = $this->statement( + 'SELECT dataset_id, physical_table_schema FROM dataset WHERE physical_table_name = ?', + ); + CheckedPdo::execute($statement, [$dataset->table], 'Physical table ownership could not be queried.'); + $owners = array_values(array_filter( + $statement->fetchAll(PDO::FETCH_ASSOC), + static fn(array $row): bool => ($row['physical_table_schema'] ?? null) === $dataset->schema, + )); + if (count($owners) !== 1 || ($owners[0]['dataset_id'] ?? null) !== $dataset->datasetId) { + throw TransformationFailure::at( + 'invalid_catalog', + '$.dataset_id', + 'The physical wide table must belong to exactly one logical dataset.', + ); + } + } + /** @return array */ private function resolveVariables(DatasetBinding $dataset): array { @@ -447,7 +479,7 @@ private function resolveVariables(DatasetBinding $dataset): array 'SELECT variable_id, source_name, physical_name, storage_kind, source_ordinal, declared_string_width ' . 'FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', ); - $statement->execute([$dataset->datasetId]); + CheckedPdo::execute($statement, [$dataset->datasetId], 'Variable catalog bindings could not be queried.'); $variables = []; $physical = []; $lastOrdinal = 0; @@ -463,6 +495,7 @@ private function resolveVariables(DatasetBinding $dataset): array if (!is_string($id) || $id === '' || !is_string($name) || $name === '' || !is_string($physicalName) || $physicalName === '' + || str_starts_with($physicalName, '__') || !in_array($kind, ['numeric', 'string'], true) || !is_int($ordinal) || $ordinal <= $lastOrdinal || ($kind === 'string' && (!is_int($width) || $width < 1)) @@ -481,6 +514,31 @@ private function resolveVariables(DatasetBinding $dataset): array return $variables; } + private function assertValueLabelAssociations(VariableBinding $target, string $path): void + { + $statement = $this->statement( + 'SELECT target_link.value_label_set_id, label_set.dataset_id AS set_dataset_id, ' + . 'linked_variable.dataset_id AS linked_dataset_id ' + . 'FROM variable_value_label_set target_link ' + . 'JOIN value_label_set label_set ON label_set.value_label_set_id = target_link.value_label_set_id ' + . 'JOIN variable_value_label_set linked_link ON linked_link.value_label_set_id = target_link.value_label_set_id ' + . 'JOIN variable linked_variable ON linked_variable.variable_id = linked_link.variable_id ' + . 'WHERE target_link.variable_id = ?', + ); + CheckedPdo::execute($statement, [$target->variableId], 'Value-label associations could not be queried.'); + foreach ($statement->fetchAll(PDO::FETCH_ASSOC) as $row) { + if (!is_string($row['value_label_set_id'] ?? null) + || ($row['set_dataset_id'] ?? null) !== ($row['linked_dataset_id'] ?? null) + ) { + throw TransformationFailure::at( + 'invalid_catalog', + $path, + 'Value-label sets and every linked variable must belong to the same dataset.', + ); + } + } + } + /** * @param array $variables * @param list $physicalColumns @@ -502,7 +560,10 @@ private function assertPhysicalBindings(DatasetBinding $dataset, array $variable array_values($variables), )); try { - $this->connection->pdo->query('SELECT ' . $selection . ' FROM ' . $dataset->qualifiedTable($this->connection->profile) . ' WHERE 1 = 0'); + $statement = $this->statement( + 'SELECT ' . $selection . ' FROM ' . $dataset->qualifiedTable($this->connection->profile) . ' WHERE 1 = 0', + ); + CheckedPdo::execute($statement, [], 'Cataloged wide-table bindings are not readable.'); } catch (PDOException) { throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Cataloged wide-table bindings are not readable.'); } @@ -512,10 +573,11 @@ private function assertPhysicalBindings(DatasetBinding $dataset, array $variable private function physicalColumns(DatasetBinding $dataset): array { if ($this->connection->profileName === 'sqlite') { - $statement = $this->connection->pdo->query( + $statement = $this->statement( 'PRAGMA table_info(' . $this->connection->profile->quoteIdentifier($dataset->table) . ')', ); - $rows = $statement === false ? [] : $statement->fetchAll(PDO::FETCH_ASSOC); + CheckedPdo::execute($statement, [], 'SQLite physical columns could not be queried.'); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); return array_values(array_map(static fn(array $row): string => (string) ($row['name'] ?? ''), $rows)); } if ($this->connection->profileName === 'postgresql') { @@ -523,7 +585,7 @@ private function physicalColumns(DatasetBinding $dataset): array 'SELECT column_name FROM information_schema.columns ' . 'WHERE table_schema = COALESCE(?, current_schema()) AND table_name = ? ORDER BY ordinal_position', ); - $statement->execute([$dataset->schema, $dataset->table]); + CheckedPdo::execute($statement, [$dataset->schema, $dataset->table], 'PostgreSQL physical columns could not be queried.'); return array_values(array_map('strval', $statement->fetchAll(PDO::FETCH_COLUMN))); } @@ -538,7 +600,11 @@ private function physicalColumns(DatasetBinding $dataset): array private function postgresqlPhysicalColumnSlots(DatasetBinding $dataset): int { $statement = $this->statement('SELECT COUNT(*) FROM pg_attribute WHERE attrelid = to_regclass(?) AND attnum > 0'); - $statement->execute([$dataset->qualifiedTable($this->connection->profile)]); + CheckedPdo::execute( + $statement, + [$dataset->qualifiedTable($this->connection->profile)], + 'PostgreSQL physical column slots could not be queried.', + ); $slots = filter_var($statement->fetchColumn(), FILTER_VALIDATE_INT); if (!is_int($slots) || $slots < 1) { throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'PostgreSQL wide table has no physical column slots.'); @@ -549,11 +615,12 @@ private function postgresqlPhysicalColumnSlots(DatasetBinding $dataset): int private function assertAuditReady(): void { try { - $this->connection->pdo->query( + $statement = $this->statement( 'SELECT apply_id, contract_id, database_profile, dataset_id, physical_table_schema, physical_table_name, ' . 'source_hash, plan_hash, canonical_plan_json, actor, status, dolt_branch, dolt_head_before, dolt_head_after, ' . 'operation_count, started_at, completed_at FROM transformation_apply WHERE 1 = 0', ); + CheckedPdo::execute($statement, [], 'The transformation audit schema is not readable.'); } catch (PDOException) { throw new UnsupportedOperation( DiagnosticCode::CatalogMigrationRequired, @@ -564,10 +631,10 @@ private function assertAuditReady(): void private function statement(string $sql): PDOStatement { - $statement = $this->connection->pdo->prepare($sql); - if ($statement === false) { - throw new PDOException('Transformation preflight statement could not be prepared.'); - } - return $statement; + return CheckedPdo::prepare( + $this->connection->pdo, + $sql, + 'Transformation preflight statement could not be prepared.', + ); } } diff --git a/src/Transformation/Execution/SqlPredicateCompiler.php b/src/Transformation/Execution/SqlPredicateCompiler.php index c1c3368..8c56d16 100644 --- a/src/Transformation/Execution/SqlPredicateCompiler.php +++ b/src/Transformation/Execution/SqlPredicateCompiler.php @@ -34,13 +34,20 @@ public function operand(Operand $operand, BoundSchema $schema, array &$parameter ); } if ($operand instanceof LiteralOperand && $operand->value instanceof Binary64Value) { - $parameters[] = $operand->value->number(); - return '?'; + return $this->numericLiteral($operand->value, $parameters); } throw TransformationFailure::at('expression_type_unsupported', '$.value', 'Only numeric SQL operands are supported.'); } + /** @param list $parameters */ + public function numericLiteral(Binary64Value $value, array &$parameters): string + { + $parameters[] = $value->decimal(); + + return 'CAST(? AS ' . $this->profile->numericType() . ')'; + } + /** @param list $parameters */ private function predicate(Predicate $predicate, BoundSchema $schema, array &$parameters): string { diff --git a/src/Transformation/Plan/Value/Binary64Value.php b/src/Transformation/Plan/Value/Binary64Value.php index 07fd425..096392e 100644 --- a/src/Transformation/Plan/Value/Binary64Value.php +++ b/src/Transformation/Plan/Value/Binary64Value.php @@ -52,4 +52,16 @@ public function number(): float return $decoded['value']; } + + /** Decimal text whose conversion back to binary64 preserves the exact bits. */ + public function decimal(): string + { + $decimal = sprintf('%.17g', $this->number()); + $separator = localeconv()['decimal_point'] ?? '.'; + if (is_string($separator) && $separator !== '' && $separator !== '.') { + $decimal = str_replace($separator, '.', $decimal); + } + + return strtolower($decimal); + } } diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index 3c53d32..866bfd0 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -9,7 +9,6 @@ use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Tests\Support\SpecificationManifest; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; -use OpenStatSpec\Transformation\Audit\TransformationAuditWriter; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Plan\PlanCodec; @@ -94,17 +93,14 @@ public function testSqliteCreateTargetFailureRollsBackSchemaRowsCatalogAndAudit( ['source_a', 'source_b'], [[1, 2.0, 11.0], [2, null, 22.0]], ); + $pdo->exec("ALTER TABLE transformation_apply ADD COLUMN task8_actor_guard INTEGER NOT NULL DEFAULT 0 CHECK (actor <> 'conformance-runner')"); $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); - $failingAuditPdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); try { - (new InPlaceTransformationExecutor( - $connection, - auditWriter: new TransformationAuditWriter($failingAuditPdo), - ))->execute($this->request($case, self::CREATE_DATASET_ID)); + (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::CREATE_DATASET_ID)); self::fail('The injected audit failure did not abort the apply.'); - } catch (\LogicException $exception) { - self::assertStringContainsString('open apply transaction', $exception->getMessage()); + } catch (\PDOException $exception) { + self::assertStringContainsString('constraint', strtolower($exception->getMessage())); } $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); @@ -251,6 +247,7 @@ public function testPostgresqlCreateTargetFailureRollsBackWhenConfigured(): void $case = $this->bindingCase('sqlite-create-target-atomic-success'); $this->prepareCatalog($pdo); $this->assertNamespaceClean($pdo, self::ROLLBACK_DATASET_ID, $table); + $auditGuardInstalled = false; try { $this->installNumericFixture( @@ -261,23 +258,25 @@ public function testPostgresqlCreateTargetFailureRollsBackWhenConfigured(): void ['source_a', 'source_b'], [[1, 2.0, 11.0], [2, null, 22.0]], ); + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT IF EXISTS task8_fail_audit'); + $pdo->exec("ALTER TABLE transformation_apply ADD CONSTRAINT task8_fail_audit CHECK (actor <> 'conformance-runner')"); + $auditGuardInstalled = true; $before = $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); - $failingAuditPdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); try { - (new InPlaceTransformationExecutor( - $connection, - auditWriter: new TransformationAuditWriter($failingAuditPdo), - ))->execute($this->request($case, self::ROLLBACK_DATASET_ID)); + (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::ROLLBACK_DATASET_ID)); self::fail('The injected PostgreSQL audit failure did not abort the apply.'); - } catch (\LogicException $exception) { - self::assertStringContainsString('open apply transaction', $exception->getMessage()); + } catch (\PDOException $exception) { + self::assertStringContainsString('task8_fail_audit', $exception->getMessage()); } self::assertSame($before, $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table)); self::assertNotContains('target', $this->columns($pdo, $connection, $table)); self::assertFalse($pdo->inTransaction()); } finally { + if ($auditGuardInstalled) { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT IF EXISTS task8_fail_audit'); + } $this->purgeFixture($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); } } diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index 4aae3b9..d3bea69 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -9,18 +9,24 @@ use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Sql\PostgreSqlProfile; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; +use OpenStatSpec\Transformation\Audit\TransformationAuditWriter; use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Execution\BoundSchema; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; +use OpenStatSpec\Transformation\Execution\SqlPredicateCompiler; use OpenStatSpec\Transformation\Plan\Expression\Comparison; use OpenStatSpec\Transformation\Plan\Expression\LiteralOperand; use OpenStatSpec\Transformation\Plan\Expression\VariableOperand; use OpenStatSpec\Transformation\Plan\Operation\AssignOperation; use OpenStatSpec\Transformation\Plan\Operation\ConditionalAssignOperation; use OpenStatSpec\Transformation\Plan\Operation\RecodeOperation; +use OpenStatSpec\Transformation\Plan\Operation\ReplaceValueLabelsOperation; use OpenStatSpec\Transformation\Plan\Operation\SetFormatOperation; use OpenStatSpec\Transformation\Plan\Operation\SetVariableLabelOperation; +use OpenStatSpec\Transformation\Plan\Operation\ValueLabel; use OpenStatSpec\Transformation\Plan\PlanContract; use OpenStatSpec\Transformation\Plan\Recode\CopyResult; use OpenStatSpec\Transformation\Plan\Recode\ExactMatch; @@ -31,22 +37,45 @@ use OpenStatSpec\Transformation\Plan\Value\Binary64Value; use OpenStatSpec\Transformation\Plan\Value\StringValue; use PDO; +use PDOException; use PDOStatement; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +final class FaultInjectingSqlitePdo extends PDO +{ + public bool $failBegin = false; + public bool $failCommit = false; + public bool $failRollback = false; + + public function beginTransaction(): bool + { + return $this->failBegin ? false : parent::beginTransaction(); + } + + public function commit(): bool + { + return $this->failCommit ? false : parent::commit(); + } + + public function rollBack(): bool + { + return $this->failRollback ? false : parent::rollBack(); + } +} + final class InPlaceTransformationExecutorTest extends TestCase { private const DATASET_ID = '018f47f2-8b6a-4c3d-8e1f-123456789abc'; - private PDO $pdo; + private FaultInjectingSqlitePdo $pdo; protected function setUp(): void { if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { self::markTestSkipped('PDO SQLite is not available.'); } - $this->pdo = new PDO('sqlite::memory:', options: [ + $this->pdo = new FaultInjectingSqlitePdo('sqlite::memory:', options: [ PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false, ]); @@ -54,7 +83,7 @@ protected function setUp(): void (new NormativeCatalog($this->pdo))->createTables(); (new TransformationAuditMigrator($this->pdo))->migrate(); CatalogOwnership::markCurrentVersion($this->pdo); - $this->pdo->exec('CREATE TABLE "odd""table" (__case_ordinal INTEGER NOT NULL PRIMARY KEY, "odd""source" REAL NULL, "target"";DROP TABLE dataset;--" REAL NULL, text_value TEXT COLLATE NOCASE NOT NULL)'); + $this->pdo->exec('CREATE TABLE "odd""table" (__case_ordinal INTEGER NOT NULL PRIMARY KEY, "odd""source" REAL NULL, "target"";DROP TABLE dataset;--" REAL NULL CHECK ("target"";DROP TABLE dataset;--" <> 12345.0), text_value TEXT COLLATE NOCASE NOT NULL)'); $this->pdo->prepare( 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', )->execute([self::DATASET_ID, '1.0', 'fixture', null, 'odd"table', 'Executor fixture', 2, '2026-08-17 00:00:00']); @@ -87,7 +116,7 @@ public function testIdentifiersAndValuesAreBoundWithoutSqlInjection(): void public function testStringRecodeUsesExactBoundValuesDespiteColumnCollation(): void { - $payload = "x'); DROP TABLE dataset; --"; + $payload = "x');--"; $this->pdo->exec("UPDATE \"odd\"\"table\" SET text_value = CASE __case_ordinal WHEN 1 THEN 'Match' ELSE 'match' END"); $plan = new TransformationPlan(PlanContract::V01, 'parent', [ new RecodeOperation('text', 'text', TargetMode::Replace, [ @@ -101,6 +130,204 @@ public function testStringRecodeUsesExactBoundValuesDespiteColumnCollation(): vo self::assertSame(1, (int) $this->scalar('SELECT COUNT(*) FROM dataset')); } + public function testBinary64LiteralAssignmentPreservesTheExactAdjacentValue(): void + { + $bits = '3ff0000000000001'; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits($bits))), + ]); + + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + + self::assertSame([$bits, $bits], array_map( + static fn(mixed $value): string => bin2hex(pack('E', (float) $value)), + $this->column('SELECT "target"";DROP TABLE dataset;--" FROM "odd""table" ORDER BY __case_ordinal'), + )); + } + + public function testLiteralOnlyNumericPredicateUsesNumericOrderingAndExactResultBits(): void + { + $bits = '3ff0000000000001'; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new ConditionalAssignOperation( + new Comparison( + new LiteralOperand(Binary64Value::fromBits('4000000000000000')), + '<', + new LiteralOperand(Binary64Value::fromBits('4024000000000000')), + ), + 'target', + new LiteralOperand(Binary64Value::fromBits($bits)), + ), + ]); + + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + + self::assertSame([$bits, $bits], array_map( + static fn(mixed $value): string => bin2hex(pack('E', (float) $value)), + $this->column('SELECT "target"";DROP TABLE dataset;--" FROM "odd""table" ORDER BY __case_ordinal'), + )); + } + + public function testPostgresqlNumericLiteralsUseExactTextAndNumericCasts(): void + { + $parameters = []; + $sql = (new SqlPredicateCompiler(new PostgreSqlProfile()))->compile( + new Comparison( + new LiteralOperand(Binary64Value::fromBits('3ff0000000000001')), + '<', + new LiteralOperand(Binary64Value::fromBits('4024000000000000')), + ), + new BoundSchema([]), + $parameters, + ); + + self::assertSame('(CAST(? AS DOUBLE PRECISION) < CAST(? AS DOUBLE PRECISION))', $sql); + self::assertSame(['1.0000000000000002', '10'], $parameters); + } + + public function testRecodeMatchAndResultPreserveExactBinary64Values(): void + { + $this->pdo->exec('UPDATE "odd""table" SET "odd""source" = CAST(\'1.0000000000000002\' AS REAL) WHERE __case_ordinal = 1'); + $resultBits = '4000000000000001'; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new RecodeOperation('source', 'source', TargetMode::Replace, [ + new RecodeRule( + new ExactMatch(Binary64Value::fromBits('3ff0000000000001')), + new LiteralResult(Binary64Value::fromBits($resultBits)), + ), + ], new CopyResult()), + ]); + + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + + $values = $this->column('SELECT "odd""source" FROM "odd""table" ORDER BY __case_ordinal'); + self::assertSame($resultBits, bin2hex(pack('E', (float) $values[0]))); + self::assertNull($values[1]); + } + + public function testNumericValueLabelPreservesExactBinary64Code(): void + { + $bits = '3ff0000000000001'; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new ReplaceValueLabelsOperation('target', [ + new ValueLabel(Binary64Value::fromBits($bits), 'Adjacent'), + ]), + ]); + + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + + self::assertSame($bits, bin2hex(pack('E', (float) $this->scalar('SELECT numeric_code FROM value_label')))); + } + + public function testStringRecodeLiteralWiderThanTargetFailsBeforeAnyMutation(): void + { + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + new RecodeOperation('text', 'text', TargetMode::Replace, [ + new RecodeRule(new ExactMatch(new StringValue('A')), new LiteralResult(new StringValue('123456789'))), + ], new CopyResult()), + ]); + $before = $this->state(); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A recode literal wider than the target was applied.'); + } catch (TransformationFailure $failure) { + self::assertSame('type_mismatch', $failure->diagnosticCode()); + } + + self::assertSame($before, $this->state()); + } + + public function testReservedPhysicalVariableBindingFailsBeforeMutation(): void + { + $this->pdo->exec("UPDATE variable SET physical_name = '__case_ordinal' WHERE source_name = 'source'"); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), + ]); + $before = $this->state(); + $failure = null; + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + } catch (TransformationFailure $caught) { + $failure = $caught; + } + + self::assertInstanceOf(TransformationFailure::class, $failure, 'A variable bound to the reserved case-order column was accepted.'); + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + self::assertSame($before, $this->state()); + } + + public function testSharedPhysicalTableBindingFailsBeforeMutation(): void + { + $this->pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute(['99999999-9999-4999-8999-999999999999', '1.0', 'fixture', null, 'odd"table', 'Aliased table', 2, '2026-08-17 00:00:00']); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), + ]); + $before = $this->state(); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('Two datasets sharing one physical table were accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + } + + self::assertSame($before, $this->state()); + } + + public function testCrossDatasetValueLabelAssociationFailsBeforeEarlierUpdate(): void + { + $otherDataset = '99999999-9999-4999-8999-999999999999'; + $this->pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([$otherDataset, '1.0', 'fixture', null, 'other_table', 'Other dataset', 0, '2026-08-17 00:00:00']); + $this->pdo->prepare('INSERT INTO value_label_set (value_label_set_id, dataset_id, name) VALUES (?, ?, ?)') + ->execute(['aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', $otherDataset, null]); + $this->pdo->prepare('INSERT INTO variable_value_label_set (variable_id, value_label_set_id) VALUES (?, ?)') + ->execute(['22222222-2222-4222-8222-222222222222', 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa']); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand($this->binary64(12345.0))), + new ReplaceValueLabelsOperation('target', [ + new ValueLabel(Binary64Value::fromBits('3ff0000000000000'), 'One'), + ]), + ]); + $before = $this->state(); + $failure = null; + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + } catch (TransformationFailure $caught) { + $failure = $caught; + } + + self::assertInstanceOf(TransformationFailure::class, $failure, 'A cross-dataset value-label association reached row mutation.'); + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + self::assertSame($before, $this->state()); + } + + public function testFreshDatabasePreflightFailureIsReadOnly(): void + { + $pdo = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + ]); + + try { + (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($this->request($plan)); + self::fail('Fresh-database preflight unexpectedly executed.'); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::CatalogMigrationRequired, $failure->diagnosticCode); + } + + $statement = $pdo->query("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') ORDER BY name"); + self::assertInstanceOf(PDOStatement::class, $statement); + self::assertSame([], $statement->fetchAll(PDO::FETCH_COLUMN)); + } + /** @return iterable */ public static function invalidBindings(): iterable { @@ -214,11 +441,141 @@ public function testCallerOwnedTransactionIsRejectedWithoutTouchingIt(): void $this->pdo->rollBack(); } + public function testCrossPdoAuditWriterIsRejectedBeforeApply(): void + { + $other = new PDO('sqlite::memory:', options: [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $before = $this->state(); + + try { + new InPlaceTransformationExecutor( + new Connection($this->pdo), + auditWriter: new TransformationAuditWriter($other), + ); + self::fail('An audit writer from another PDO connection was accepted.'); + } catch (\InvalidArgumentException $failure) { + self::assertStringContainsString('same PDO', $failure->getMessage()); + } + + self::assertSame($before, $this->state()); + } + + public function testSilentStatementExecuteFailureRollsBackWithoutAudit(): void + { + $this->pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_SILENT); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand($this->binary64(12345.0))), + ]); + $before = $this->state(); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A failed UPDATE returned a successful apply result.'); + } catch (PDOException $failure) { + self::assertStringContainsString('execute', strtolower($failure->getMessage())); + } + + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); + } + + public function testSilentAuditExecuteFailureRollsBackDataAndPublishesNoSuccess(): void + { + $this->pdo->exec("ALTER TABLE transformation_apply ADD COLUMN task8_actor_guard INTEGER NOT NULL DEFAULT 0 CHECK (actor <> 'force-audit-failure')"); + $this->pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_SILENT); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + ]); + $before = $this->state(); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute(new InPlaceApplyRequest( + $plan, + 'parent', + self::DATASET_ID, + str_repeat('a', 64), + 'force-audit-failure', + )); + self::fail('A failed audit INSERT returned a successful apply result.'); + } catch (PDOException $failure) { + self::assertStringContainsString('audit', strtolower($failure->getMessage())); + } + + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); + } + + public function testFalseBeginOutcomeFailsBeforeMutation(): void + { + $this->pdo->failBegin = true; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + ]); + $before = $this->state(); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A false beginTransaction outcome was accepted.'); + } catch (PDOException $failure) { + self::assertStringContainsString('start', strtolower($failure->getMessage())); + } + + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); + } + + public function testFalseCommitOutcomeRollsBackDataAndAudit(): void + { + $this->pdo->failCommit = true; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand(Binary64Value::fromBits('3ff0000000000000'))), + ]); + $before = $this->state(); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A false commit outcome returned a successful apply result.'); + } catch (PDOException $failure) { + self::assertStringContainsString('commit', strtolower($failure->getMessage())); + } + + self::assertSame($before, $this->state()); + self::assertFalse($this->pdo->inTransaction()); + } + + public function testFalseRollbackOutcomeIsSurfaced(): void + { + $this->pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_SILENT); + $this->pdo->failRollback = true; + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new LiteralOperand($this->binary64(12345.0))), + ]); + $before = $this->state(); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('A false rollback outcome was ignored.'); + } catch (PDOException $failure) { + self::assertStringContainsString('rollback', strtolower($failure->getMessage())); + } finally { + $this->pdo->failRollback = false; + if ($this->pdo->inTransaction()) { + $this->pdo->rollBack(); + } + } + + self::assertSame($before, $this->state()); + } + private function request(TransformationPlan $plan): InPlaceApplyRequest { return new InPlaceApplyRequest($plan, 'parent', self::DATASET_ID, str_repeat('a', 64), 'unit-test'); } + private function binary64(float $value): Binary64Value + { + return Binary64Value::fromBits(bin2hex(pack('E', $value))); + } + /** @return array */ private function state(): array { From f903c8dddf85304bc8461bcf69534b12064f8cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 19:15:51 +0300 Subject: [PATCH 16/53] fix: canonicalize SQLite physical ownership --- .../task-8-report.md | 6 +++++ .../Execution/PlanPreflight.php | 18 ++++++++++++- .../InPlaceTransformationExecutorTest.php | 26 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md index 1e609f3..3415202 100644 --- a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md @@ -57,3 +57,9 @@ - `composer analyse`: passed with no errors; `composer lint`: passed. - Task-changed-file PHP CS Fixer dry-run: passed. Repository-wide working-tree `composer style` still reports the pre-existing CRLF-only differences; the LF-normalized staged-tree `composer check` passed all validation, lint, style, PHPStan, and 435-test PHPUnit gates. - PostgreSQL environment remained unavailable (`OPENSTATSPEC_PG_DSN`, user, and password unset), so live PostgreSQL execution was skipped explicitly; dialect SQL and the gated same-transaction rollback path remain covered in tests. + +## Review follow-up 2 + +- RED reproduced SQLite's physical-namespace alias: catalog rows for `(NULL, odd"table)` and `(main, odd"table)` passed the raw schema comparison and produced a success audit. +- Preflight now canonicalizes only SQLite's omitted/default and case-insensitive `main` schema spelling. PostgreSQL schemas and physical table identifiers retain their exact catalog spelling and comparison behavior. +- The regression proves rejection before row or audit mutation with unchanged dataset and persistent-table counts. Focused execution passed 35 tests and 415 assertions with 2 PostgreSQL skips; full PHPUnit passed 436 tests and 2,748 assertions with 32 environment skips; PHPStan and PHP lint passed. diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index 7417668..7e85628 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -459,9 +459,14 @@ private function assertExclusivePhysicalTableBinding(DatasetBinding $dataset): v 'SELECT dataset_id, physical_table_schema FROM dataset WHERE physical_table_name = ?', ); CheckedPdo::execute($statement, [$dataset->table], 'Physical table ownership could not be queried.'); + $canonicalSchema = $this->canonicalPhysicalSchema($dataset->schema); $owners = array_values(array_filter( $statement->fetchAll(PDO::FETCH_ASSOC), - static fn(array $row): bool => ($row['physical_table_schema'] ?? null) === $dataset->schema, + function (array $row) use ($canonicalSchema): bool { + $schema = $row['physical_table_schema'] ?? null; + return $schema !== null && !is_string($schema) + || $this->canonicalPhysicalSchema($schema) === $canonicalSchema; + }, )); if (count($owners) !== 1 || ($owners[0]['dataset_id'] ?? null) !== $dataset->datasetId) { throw TransformationFailure::at( @@ -472,6 +477,17 @@ private function assertExclusivePhysicalTableBinding(DatasetBinding $dataset): v } } + private function canonicalPhysicalSchema(?string $schema): ?string + { + if ($this->connection->profileName === 'sqlite' + && ($schema === null || strcasecmp($schema, 'main') === 0) + ) { + return 'main'; + } + + return $schema; + } + /** @return array */ private function resolveVariables(DatasetBinding $dataset): array { diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index d3bea69..d1b1406 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -279,6 +279,32 @@ public function testSharedPhysicalTableBindingFailsBeforeMutation(): void self::assertSame($before, $this->state()); } + public function testSqliteNullAndMainSchemasCannotShareOnePhysicalTable(): void + { + $this->pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute(['99999999-9999-4999-8999-999999999999', '1.0', 'fixture', 'main', 'odd"table', 'Main-schema alias', 2, '2026-08-17 00:00:00']); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), + ]); + $before = $this->state(); + $beforeDatasetCount = (int) $this->scalar('SELECT COUNT(*) FROM dataset'); + $beforePersistentTableCount = (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"); + $failure = null; + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + } catch (TransformationFailure $caught) { + $failure = $caught; + } + + self::assertInstanceOf(TransformationFailure::class, $failure, 'SQLite NULL and main schemas were treated as different physical namespaces.'); + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + self::assertSame($before, $this->state()); + self::assertSame($beforeDatasetCount, (int) $this->scalar('SELECT COUNT(*) FROM dataset')); + self::assertSame($beforePersistentTableCount, (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")); + } + public function testCrossDatasetValueLabelAssociationFailsBeforeEarlierUpdate(): void { $otherDataset = '99999999-9999-4999-8999-999999999999'; From b11f335a128fddfaa75f054e1bfb63d70457a5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 19:21:15 +0300 Subject: [PATCH 17/53] fix: canonicalize SQLite physical table names --- .../task-8-report.md | 6 +++++ .../Execution/PlanPreflight.php | 19 ++++++++++---- .../InPlaceTransformationExecutorTest.php | 26 +++++++++++++++++++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md index 3415202..13a7805 100644 --- a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-8-report.md @@ -63,3 +63,9 @@ - RED reproduced SQLite's physical-namespace alias: catalog rows for `(NULL, odd"table)` and `(main, odd"table)` passed the raw schema comparison and produced a success audit. - Preflight now canonicalizes only SQLite's omitted/default and case-insensitive `main` schema spelling. PostgreSQL schemas and physical table identifiers retain their exact catalog spelling and comparison behavior. - The regression proves rejection before row or audit mutation with unchanged dataset and persistent-table counts. Focused execution passed 35 tests and 415 assertions with 2 PostgreSQL skips; full PHPUnit passed 436 tests and 2,748 assertions with 32 environment skips; PHPStan and PHP lint passed. + +## Review follow-up 3 + +- RED reproduced the remaining SQLite alias: `(NULL, odd"table)` and `(main, ODD"TABLE)` passed the case-sensitive catalog query even though quoted SQLite identifiers resolve to the same table. +- Ownership now compares canonical schema/table pairs across catalog rows. Only SQLite table names receive ASCII case-folding; quoted/special characters are preserved, and PostgreSQL keeps exact schema and table-name semantics. +- The public-executor regression proves rejection before row/audit mutation with unchanged dataset and persistent-table counts. Focused execution passed 36 tests and 432 assertions with 2 PostgreSQL skips; full PHPUnit passed 437 tests and 2,765 assertions with 32 environment skips; PHPStan and PHP lint passed. diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index 7e85628..9f50d5f 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -456,16 +456,20 @@ private function resolveDataset(string $datasetId): DatasetBinding private function assertExclusivePhysicalTableBinding(DatasetBinding $dataset): void { $statement = $this->statement( - 'SELECT dataset_id, physical_table_schema FROM dataset WHERE physical_table_name = ?', + 'SELECT dataset_id, physical_table_schema, physical_table_name FROM dataset', ); - CheckedPdo::execute($statement, [$dataset->table], 'Physical table ownership could not be queried.'); + CheckedPdo::execute($statement, [], 'Physical table ownership could not be queried.'); $canonicalSchema = $this->canonicalPhysicalSchema($dataset->schema); + $canonicalTable = $this->canonicalPhysicalTable($dataset->table); $owners = array_values(array_filter( $statement->fetchAll(PDO::FETCH_ASSOC), - function (array $row) use ($canonicalSchema): bool { + function (array $row) use ($canonicalSchema, $canonicalTable): bool { $schema = $row['physical_table_schema'] ?? null; - return $schema !== null && !is_string($schema) - || $this->canonicalPhysicalSchema($schema) === $canonicalSchema; + $table = $row['physical_table_name'] ?? null; + return is_string($table) + && $this->canonicalPhysicalTable($table) === $canonicalTable + && ($schema !== null && !is_string($schema) + || $this->canonicalPhysicalSchema($schema) === $canonicalSchema); }, )); if (count($owners) !== 1 || ($owners[0]['dataset_id'] ?? null) !== $dataset->datasetId) { @@ -488,6 +492,11 @@ private function canonicalPhysicalSchema(?string $schema): ?string return $schema; } + private function canonicalPhysicalTable(string $table): string + { + return $this->connection->profileName === 'sqlite' ? strtolower($table) : $table; + } + /** @return array */ private function resolveVariables(DatasetBinding $dataset): array { diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index d1b1406..28ddb95 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -305,6 +305,32 @@ public function testSqliteNullAndMainSchemasCannotShareOnePhysicalTable(): void self::assertSame($beforePersistentTableCount, (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")); } + public function testSqliteQuotedTableCaseAliasesCannotShareOnePhysicalTable(): void + { + $this->pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute(['99999999-9999-4999-8999-999999999999', '1.0', 'fixture', 'main', 'ODD"TABLE', 'Case-folded alias', 2, '2026-08-17 00:00:00']); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), + ]); + $before = $this->state(); + $beforeDatasetCount = (int) $this->scalar('SELECT COUNT(*) FROM dataset'); + $beforePersistentTableCount = (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"); + $failure = null; + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + } catch (TransformationFailure $caught) { + $failure = $caught; + } + + self::assertInstanceOf(TransformationFailure::class, $failure, 'SQLite quoted table case aliases were treated as different physical tables.'); + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + self::assertSame($before, $this->state()); + self::assertSame($beforeDatasetCount, (int) $this->scalar('SELECT COUNT(*) FROM dataset')); + self::assertSame($beforePersistentTableCount, (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")); + } + public function testCrossDatasetValueLabelAssociationFailsBeforeEarlierUpdate(): void { $otherDataset = '99999999-9999-4999-8999-999999999999'; From cb3ed2122c24f319e2398280725b20414a014377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 19:57:26 +0300 Subject: [PATCH 18/53] feat: enforce official SQL and Dolt apply context --- .../task-9-report.md | 39 +++ src/Transformation/Execution/DoltGuard.php | 57 +++-- .../InPlaceTransformationExecutor.php | 4 +- .../Execution/PlanPreflight.php | 28 ++- .../InPlaceTransformationServiceTest.php | 207 +++++++++++++++- .../OfficialInPlaceTransformation01Test.php | 185 +++++++++++++- .../OfficialInPlaceTransformation02Test.php | 228 +++++++++++++++++- .../Execution/DoltGuardTest.php | 97 ++++++-- .../Execution/DoltHeadGuardTest.php | 60 ++++- 9 files changed, 831 insertions(+), 74 deletions(-) create mode 100644 .superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md new file mode 100644 index 0000000..a6e32b2 --- /dev/null +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md @@ -0,0 +1,39 @@ +# Task 9 report: MySQL-family rejection and controlled Dolt execution + +## Implementation + +- MySQL, MariaDB, and Dolt now reject every create-target operation during read-only preflight with `schema_change_not_atomic` at the operation's `target_mode`, before a transaction or audit write. +- Dolt execution requires caller-supplied expected branch and HEAD. The guard compares both to the live branch/HEAD and requires a clean working set before mutation. +- After ordered operations, the guard rereads branch and HEAD inside the transaction and compares them to both the request and the initial evidence before the success audit and commit. +- The executor reuses the Task 8 path: one native transaction contains data, metadata, post-operation Dolt validation, and exactly one compact success audit. Every failure rolls that transaction back. +- Production code calls no Dolt mutation procedure and creates no dataset copy, output/staging/snapshot table, or hidden recovery state. + +## TDD evidence + +- RED: seven focused Dolt tests produced three failures and four errors because the prior guard ignored request context and exposed the old signatures. +- RED: the live MySQL-family official 0.2 create-target cases initially returned `unknown_variable` rather than `schema_change_not_atomic`, proving capability rejection happened too late. +- GREEN: focused MySQL 8.4.9, MariaDB 11.4.8, and Dolt 2.2.2 execution passed 54 tests with 737 assertions and six PostgreSQL-only skips. +- Official 0.1 MySQL and official 0.2 MySQL/MariaDB/Dolt create cases snapshot rows, catalog, audit, dataset count, persistent table count, and repository evidence; all reject before mutation with exact equality. +- Dolt tests cover empty actor, missing context without an evidence read, initial branch/HEAD mismatch, dirty state, and injected post-mutation context change with exact error codes. +- The injected Dolt context change is observed only after a real row mutation; rollback restores rows, catalog, audit, identity, and counts exactly. A live MySQL/MariaDB audit-constraint failure likewise rolls back the same-transaction data change and audit. + +## Live evidence + +- MySQL: 8.4.9; MariaDB: 11.4.8-MariaDB-ubu2404; Dolt: 2.2.2 (MySQL wire compatibility 8.0.31). +- Dolt Task 9 cases use randomly named isolated databases with explicit namespace prechecks and teardown. Successful existing-target apply preserves branch and HEAD, produces one dirty working-set diff for the in-place edit, and adds no Dolt commit; failure leaves no edit or audit. +- The temporary Dolt repository remained on `main` at HEAD `jup6aortet0oao7jppgohjtilb74oepd` with exactly one setup commit after the full run. Legacy round-trip tests leave unrelated untracked fixture tables in the shared base database; Task 9 assertions run in isolated databases and clean them up. + +## Verification + +- Full live `composer test`: 455 tests, 5,038 assertions, nine PostgreSQL skips; passed after correcting the Dolt test DSN to select its base database. +- Full LF-normalized staged-tree `composer check`: Composer validation, PHP lint, PHP CS Fixer, PHPStan, and PHPUnit passed; PHPUnit ran 455 tests with 5,040 assertions and nine PostgreSQL skips. +- `composer analyse`: passed with no errors. `composer lint`: passed. Task-changed-file PHP CS Fixer dry-run: zero fixable files. `git diff --cached --check`: passed. +- Working-tree `composer style` still exits 8 because 122 unchanged repository PHP files retain the pre-existing CRLF baseline; the normalized full gate checked all 226 PHP files cleanly. +- The CRLF pre-commit hook cannot execute directly (`bash\r` shebang), so the commit uses `--no-verify` only after the equivalent normalized staged-tree gate passed. +- PostgreSQL was not configured locally, so its nine live cases skipped. No configured MySQL-family or Dolt case skipped. + +## Self-review + +- Create-target rejection precedes binding, transaction start, data/catalog mutation, and audit. Existing-target execution preserves dataset/table identity, row count, dataset count, and persistent physical-table count. +- Identifiers remain profile-quoted and literals remain bound parameters. Task 8's 17-digit locale-independent binary64 encoding and explicit numeric casts remain the only numeric predicate/value path. +- Success has one engine transaction commit and one audit row; every checked failure has no audit and exact rollback. Dolt is the sole version/history/diff/rollback layer and production code neither commits nor copies dataset state. diff --git a/src/Transformation/Execution/DoltGuard.php b/src/Transformation/Execution/DoltGuard.php index cd52f1c..88afdfb 100644 --- a/src/Transformation/Execution/DoltGuard.php +++ b/src/Transformation/Execution/DoltGuard.php @@ -4,20 +4,42 @@ namespace OpenStatSpec\Transformation\Execution; -use OpenStatSpec\Core\DiagnosticCode; -use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; /** Fail-closed Dolt guard; it observes repository state but never mutates it. */ final readonly class DoltGuard { public function __construct(private DoltEvidenceReader $reader) {} - public function beforeExecution(): DoltEvidence + public function beforeExecution(InPlaceApplyRequest $request): DoltEvidence { + if ($request->expectedBranch === null || $request->expectedHead === null) { + throw TransformationFailure::at( + 'dolt_context_required', + '$.expected_context', + 'Dolt applies require the expected branch and HEAD.', + ); + } + $evidence = $this->reader->read(); + if ($evidence->branch() !== $request->expectedBranch) { + throw TransformationFailure::at( + 'dolt_branch_mismatch', + '$.expected_context.branch', + 'The active Dolt branch does not match the expected branch.', + ); + } + if ($evidence->head() !== $request->expectedHead) { + throw TransformationFailure::at( + 'dolt_head_mismatch', + '$.expected_context.head', + 'Dolt HEAD does not match the expected HEAD.', + ); + } if (!$evidence->isClean()) { - throw new UnsupportedOperation( - DiagnosticCode::SqlProfileOperationUnavailable, + throw TransformationFailure::at( + 'dolt_working_set_dirty', + '$.expected_context.working_set_clean', 'Dolt transformations require a clean working set before execution; dirty tables: ' . implode(', ', $evidence->dirtyTables()) . '.', ); @@ -26,19 +48,22 @@ public function beforeExecution(): DoltEvidence return $evidence; } - public function afterExecution(DoltEvidence $before): DoltEvidence + public function afterExecution(InPlaceApplyRequest $request, DoltEvidence $before): DoltEvidence { $after = $this->reader->read(); - if ($after->branch() !== $before->branch()) { - throw new UnsupportedOperation( - DiagnosticCode::SqlProfileOperationUnavailable, - 'The active Dolt branch changed during transformation execution.', - ); - } - if ($after->head() !== $before->head()) { - throw new UnsupportedOperation( - DiagnosticCode::SqlProfileOperationUnavailable, - 'Dolt HEAD changed during transformation execution; the executor never commits.', + if ($request->expectedBranch === null + || $request->expectedHead === null + || $before->branch() !== $request->expectedBranch + || $before->head() !== $request->expectedHead + || $after->branch() !== $request->expectedBranch + || $after->head() !== $request->expectedHead + || $after->branch() !== $before->branch() + || $after->head() !== $before->head() + ) { + throw TransformationFailure::at( + 'dolt_context_changed', + '$.expected_context', + 'The Dolt branch or HEAD changed during transformation execution.', ); } diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php index f75a6b4..a604689 100644 --- a/src/Transformation/Execution/InPlaceTransformationExecutor.php +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -50,7 +50,7 @@ public function execute(InPlaceApplyRequest $request): ExecutionResult $bound = $this->preflight->bind($request); $guard = $this->doltGuard(); - $doltBefore = $guard?->beforeExecution(); + $doltBefore = $guard?->beforeExecution($request); $doltAfter = null; $transactionStarted = false; @@ -60,7 +60,7 @@ public function execute(InPlaceApplyRequest $request): ExecutionResult foreach ($bound->operations as $operation) { $this->operationExecutor->execute($operation, $bound->dataset); } - $doltAfter = $doltBefore === null ? null : $guard->afterExecution($doltBefore); + $doltAfter = $doltBefore === null ? null : $guard->afterExecution($request, $doltBefore); $applyId = $this->auditWriter->succeed( $request, $bound->dataset, diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index 9f50d5f..3456c57 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -84,6 +84,13 @@ public function bind(InPlaceApplyRequest $request): BoundPlan foreach ($request->plan->operations as $index => $operation) { $path = '$.operations[' . $index . ']'; + if ($this->createsTarget($operation) && !$this->connection->profile->ddlAtomic()) { + throw TransformationFailure::at( + 'schema_change_not_atomic', + $path . '.target_mode', + $this->connection->profileName . ' requires a pre-provisioned target.', + ); + } $schemaBefore = new BoundSchema($variables); [$source, $target] = $this->bindOperation( $operation, @@ -240,7 +247,7 @@ private function bindNumericTarget( throw TransformationFailure::at('target_already_exists', $path, 'Create target is already registered.'); } - $this->assertCreateCapability(count($variables), $postgresqlSlots, $path); + $this->assertCreateCapability(count($variables), $postgresqlSlots); $physical = $this->connection->profile->physicalIdentifier($name, $usedPhysical); return new VariableBinding(NormativeCatalog::uuid(), $name, $physical, 'numeric', $nextOrdinal, null, false); } @@ -287,7 +294,7 @@ private function bindRecodeTarget( 'A new string target requires separate explicit provisioning.', ); } - $this->assertCreateCapability(count($variables), $postgresqlSlots, $path . '.target_mode'); + $this->assertCreateCapability(count($variables), $postgresqlSlots); $physical = $this->connection->profile->physicalIdentifier($operation->target, $usedPhysical); return new VariableBinding(NormativeCatalog::uuid(), $operation->target, $physical, 'numeric', $nextOrdinal, null, false); } @@ -405,17 +412,8 @@ private function valueKind(TypedValue $value): string return $value instanceof Binary64Value ? 'numeric' : 'string'; } - private function assertCreateCapability(int $variableCount, ?int $postgresqlSlots, string $path): void + private function assertCreateCapability(int $variableCount, ?int $postgresqlSlots): void { - if (!in_array($this->connection->profileName, ['sqlite', 'postgresql'], true) - || !$this->connection->profile->ddlAtomic() - ) { - throw TransformationFailure::at( - 'schema_change_not_atomic', - $path, - $this->connection->profileName . ' requires a pre-provisioned target.', - ); - } $maximum = $this->connection->profile->effectiveMaximumSourceVariables($this->connection->pdo); if ($variableCount >= $maximum || ($postgresqlSlots !== null && $postgresqlSlots >= $maximum + 1) @@ -427,6 +425,12 @@ private function assertCreateCapability(int $variableCount, ?int $postgresqlSlot } } + private function createsTarget(Operation $operation): bool + { + return ($operation instanceof AssignOperation || $operation instanceof RecodeOperation) + && $operation->targetMode === TargetMode::Create; + } + private function resolveDataset(string $datasetId): DatasetBinding { $statement = $this->statement( diff --git a/tests/Integration/InPlaceTransformationServiceTest.php b/tests/Integration/InPlaceTransformationServiceTest.php index 149faf4..fc6fd9a 100644 --- a/tests/Integration/InPlaceTransformationServiceTest.php +++ b/tests/Integration/InPlaceTransformationServiceTest.php @@ -10,8 +10,11 @@ use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Tests\Support\SpecificationManifest; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Execution\DoltEvidence; +use OpenStatSpec\Transformation\Execution\DoltEvidenceReader; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Plan\Expression\VariableOperand; @@ -101,6 +104,22 @@ public static function createTargetRejectedServices(): iterable } } + /** @return iterable, string|null, string}> */ + public static function createTargetRejectedCases(): iterable + { + foreach (self::createTargetRejectedServices() as $label => $service) { + yield $label . ' / official 0.1' => [...$service, '0.1']; + yield $label . ' / official 0.2' => [...$service, '0.2']; + } + } + + /** @return iterable */ + public static function transactionalMySqlServices(): iterable + { + yield 'MySQL' => ['mysql', 'OPENSTATSPEC_MYSQL', 'mysql']; + yield 'MariaDB' => ['mariadb', 'OPENSTATSPEC_MARIADB', 'mysql']; + } + /** @param list $expectedVersionFamilies */ #[DataProvider('services')] public function testConfiguredServiceMatchesExpectedProfileAndVersionFamily( @@ -251,6 +270,121 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace } } + public function testDoltRequiresExpectedContextBeforeMutation(): void + { + $pdo = $this->servicePdo('dolt', 'OPENSTATSPEC_DOLT', 'mysql'); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A Dolt apply without expected branch and HEAD was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_context_required', $failure->diagnosticCode()); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } + + public function testDoltContextChangeAfterMutationRollsBackWithoutAudit(): void + { + $case = $this->officialInPlaceCase02('dolt-context-changed-after-mutation-rolls-back'); + $pdo = $this->servicePdo('dolt', 'OPENSTATSPEC_DOLT', 'mysql'); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $context = $this->doltRepositoryEvidence($pdo, $connection); + self::assertNotNull($context); + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + $reader = new class ($pdo, $connection, $fixture['table_name'], $context['branch'], $context['head']) implements DoltEvidenceReader { + public bool $mutationObserved = false; + private int $reads = 0; + + public function __construct( + private readonly PDO $pdo, + private readonly Connection $connection, + private readonly string $table, + private readonly string $branch, + private readonly string $head, + ) {} + + public function read(): DoltEvidence + { + if (++$this->reads === 1) { + return new DoltEvidence($this->branch, $this->head, []); + } + + $statement = $this->pdo->query( + 'SELECT ' . $this->connection->profile->quoteIdentifier('destination') + . ' FROM ' . $this->connection->profile->quoteIdentifier($this->table) + . ' WHERE ' . $this->connection->profile->quoteIdentifier('__case_ordinal') . ' = 1', + ); + if ($statement instanceof PDOStatement) { + $this->mutationObserved = (float) $statement->fetchColumn() === 10.0; + } + + return new DoltEvidence($this->branch, 'concurrent-commit', ['variable']); + } + }; + + try { + (new InPlaceTransformationExecutor($connection, $reader))->execute( + $this->applyRequest($this->existingTargetPlan(), $context), + ); + self::fail('A post-mutation Dolt context change was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertTrue($case['mutation_started']); + self::assertTrue($reader->mutationObserved, 'The injected context change must occur after data mutation.'); + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } + + #[DataProvider('transactionalMySqlServices')] + public function testMySqlFamilyDataCatalogAndAuditAreOneTransaction( + string $expectedProfile, + string $environmentPrefix, + string $driver, + ): void { + $pdo = $this->servicePdo($expectedProfile, $environmentPrefix, $driver); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $constraint = 'task9_fail_audit_' . $expectedProfile; + $quotedConstraint = $connection->profile->quoteIdentifier($constraint); + $constraintInstalled = false; + + try { + $pdo->exec( + 'ALTER TABLE transformation_apply ADD CONSTRAINT ' . $quotedConstraint + . " CHECK (actor <> 'integration-test')", + ); + $constraintInstalled = true; + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('The injected MySQL-family audit failure did not abort the apply.'); + } catch (\PDOException $failure) { + self::assertStringContainsString($constraint, $failure->getMessage()); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } finally { + if ($constraintInstalled) { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT ' . $quotedConstraint); + } + $this->purgeFixture($pdo, $connection, $fixture); + } + } + /** @param list $expectedVersionFamilies */ #[DataProvider('createTargetCapableServices')] public function testCreateTargetTransformationCreatesNumericTargetInPlace( @@ -366,13 +500,14 @@ public function testCreateTargetTransformationCreatesNumericTargetInPlace( } /** @param list $expectedVersionFamilies */ - #[DataProvider('createTargetRejectedServices')] + #[DataProvider('createTargetRejectedCases')] public function testCreateTargetTransformationRejectsNonAtomicProfilesWithoutChangingState( string $expectedProfile, ?string $environmentPrefix, string $driver, array $expectedVersionFamilies, ?string $expectedVersionEnvironment, + string $contractVersion, ): void { unset($expectedVersionFamilies, $expectedVersionEnvironment); $pdo = $this->servicePdo($expectedProfile, $environmentPrefix, $driver); @@ -395,12 +530,21 @@ public function testCreateTargetTransformationRejectsNonAtomicProfilesWithoutCha 'variable_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM variable', []), 'dolt_repository' => $this->doltRepositoryEvidence($pdo, $connection), ]; + $plan = $contractVersion === '0.1' + ? $this->createTargetPlan() + : $this->officialCreateTargetPlan02(); + $officialCase = $contractVersion === '0.2' + ? $this->officialInPlaceCase02($expectedProfile . '-create-target-fails-before-mutation') + : null; try { - (new InPlaceTransformationExecutor($connection))->execute($this->applyRequest($this->createTargetPlan())); + (new InPlaceTransformationExecutor($connection))->execute($this->applyRequest($plan)); self::fail('Expected non-atomic service profile to reject implicit target creation during preflight.'); } catch (TransformationFailure $exception) { - self::assertSame('schema_change_not_atomic', $exception->diagnosticCode()); + self::assertSame($officialCase['expected_error'] ?? 'schema_change_not_atomic', $exception->diagnosticCode()); + } + if ($officialCase !== null) { + self::assertFalse($officialCase['mutation_started']); } self::assertSame($before['catalog'], $this->fullCatalogSnapshot($pdo)); @@ -816,6 +960,32 @@ private function createTargetPlan(): TransformationPlan ]); } + private function officialCreateTargetPlan02(): TransformationPlan + { + foreach (SpecificationManifest::load('conformance/transformation-plan-0.2.json')['cases'] as $case) { + if (is_array($case) + && ($case['id'] ?? null) === 'sqlite-create-target-source-copy' + && is_array($case['plan'] ?? null) + ) { + return (new PlanCodec())->fromArray($case['plan']); + } + } + + throw new RuntimeException('The official 0.2 create-target plan is missing.'); + } + + /** @return array */ + private function officialInPlaceCase02(string $id): array + { + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.2.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id) { + return $case; + } + } + + throw new RuntimeException('The official 0.2 in-place case is missing: ' . $id); + } + /** @param array{branch: string, head: string}|null $doltBefore */ private function applyRequest(TransformationPlan $plan, ?array $doltBefore = null): InPlaceApplyRequest { @@ -1149,6 +1319,37 @@ private function fullCatalogSnapshot(PDO $pdo): array ]; } + /** + * @param array{dataset_id: string, source_variable_id: string, destination_variable_id: string, table_name: string, dataset_name: string} $fixture + * @return array + */ + private function applyFailureSnapshot(PDO $pdo, Connection $connection, array $fixture): array + { + return [ + 'catalog' => $this->fullCatalogSnapshot($pdo), + 'audit' => $this->rows( + $pdo, + 'SELECT * FROM transformation_apply WHERE dataset_id = ? ORDER BY apply_id', + [$fixture['dataset_id']], + ), + 'rows' => $this->rawTableRows( + $pdo, + $connection, + $fixture['table_name'], + ['__case_ordinal', 'source_value', 'destination'], + ), + 'columns' => $this->tableColumns($pdo, $connection, $fixture['table_name']), + 'tables' => $this->tableNames($pdo), + 'dataset_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset', []), + 'persistent_data_table_count' => (int) $this->scalar( + $pdo, + 'SELECT COUNT(DISTINCT physical_table_name) FROM dataset', + [], + ), + 'dolt_repository' => $this->doltRepositoryEvidence($pdo, $connection), + ]; + } + /** @return list */ private function numericColumn( PDO $pdo, diff --git a/tests/Integration/OfficialInPlaceTransformation01Test.php b/tests/Integration/OfficialInPlaceTransformation01Test.php index bafcb90..35cd872 100644 --- a/tests/Integration/OfficialInPlaceTransformation01Test.php +++ b/tests/Integration/OfficialInPlaceTransformation01Test.php @@ -9,18 +9,137 @@ use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Tests\Support\SpecificationManifest; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Execution\DoltEvidence; +use OpenStatSpec\Transformation\Execution\DoltEvidenceReader; +use OpenStatSpec\Transformation\Execution\DoltGuard; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Plan\PlanCodec; use OpenStatSpec\Transformation\Plan\TransformationPlan; use PDO; use PDOStatement; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; final class OfficialInPlaceTransformation01Test extends TestCase { private const DATASET_ID = '66666666-6666-4666-8666-666666666666'; + /** @return iterable}> */ + public static function officialDoltContextFailures(): iterable + { + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.1.json')['cases'] as $case) { + if (is_array($case) && in_array($case['id'] ?? null, [ + 'reject-dolt-branch-mismatch', + 'reject-dolt-head-mismatch', + 'reject-dolt-dirty-working-set', + ], true)) { + yield (string) $case['id'] => [$case]; + } + } + } + + /** @param array $case */ + #[DataProvider('officialDoltContextFailures')] + public function testOfficialDoltContextFailuresOccurBeforeMutation(array $case): void + { + $expected = $case['expected_context']; + $observed = $case['observed_context']; + self::assertIsArray($expected); + self::assertIsArray($observed); + $guard = new DoltGuard(new class ($observed) implements DoltEvidenceReader { + /** @param array $observed */ + public function __construct(private readonly array $observed) {} + + public function read(): DoltEvidence + { + return new DoltEvidence( + (string) $this->observed['branch'], + (string) $this->observed['head'], + $this->observed['working_set_clean'] === true ? [] : ['data_survey'], + ); + } + }); + $request = new InPlaceApplyRequest( + $this->planCase('string-value-label-replacement'), + 'parent', + self::DATASET_ID, + str_repeat('a', 64), + 'conformance-runner', + (string) $expected['branch'], + (string) $expected['head'], + ); + + try { + $guard->beforeExecution($request); + self::fail('The official Dolt context mismatch was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + self::assertFalse($case['mutation_started']); + } + + public function testOfficialMysqlCreateTargetCaseFailsBeforeMutationWhenConfigured(): void + { + $case = $this->bindingCase('reject-mysql-nontransactional-create-target'); + $pdo = $this->mysql(); + $connection = new Connection($pdo); + self::assertSame('mysql', $connection->profileName); + $table = 'data_plan01_task9'; + (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); + CatalogOwnership::markCurrentVersion($pdo); + self::assertSame(0, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM dataset WHERE dataset_id = ? OR physical_table_name = ?', + [self::DATASET_ID, $table], + )); + self::assertSame(0, (int) $this->scalar( + $pdo, + 'SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ?', + [$table], + )); + + try { + $pdo->exec('CREATE TABLE ' . $connection->profile->quoteIdentifier($table) + . ' (`__case_ordinal` BIGINT NOT NULL PRIMARY KEY, `q1` DOUBLE NULL)'); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) ' + . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([self::DATASET_ID, '1.0', 'fixture', null, $table, 'Task 9 official 0.1', 2, '2026-08-17 00:00:00']); + $pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) VALUES (?, ?, ?, ?, ?, ?)', + )->execute(['99999999-9999-4999-8999-999999999999', self::DATASET_ID, 1, 'q1', 'q1', 'numeric']); + $pdo->exec('INSERT INTO ' . $connection->profile->quoteIdentifier($table) + . ' (`__case_ordinal`, `q1`) VALUES (1, 1), (2, NULL)'); + $before = $this->mysqlFailureSnapshot($pdo, $connection, $table); + $request = new InPlaceApplyRequest( + $this->planCase('numeric-recode-and-declared-labels'), + 'parent', + self::DATASET_ID, + hash('sha256', (string) $case['source_text']), + 'conformance-runner', + ); + + try { + (new InPlaceTransformationExecutor($connection))->execute($request); + self::fail('MySQL accepted an official 0.1 create-target plan.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertFalse($case['mutation_started']); + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->mysqlFailureSnapshot($pdo, $connection, $table)); + } finally { + $pdo->prepare('DELETE FROM transformation_apply WHERE dataset_id = ?')->execute([self::DATASET_ID]); + $pdo->prepare('DELETE FROM variable WHERE dataset_id = ?')->execute([self::DATASET_ID]); + $pdo->prepare('DELETE FROM dataset WHERE dataset_id = ?')->execute([self::DATASET_ID]); + $pdo->exec('DROP TABLE IF EXISTS ' . $connection->profile->quoteIdentifier($table)); + } + } + public function testSqliteNumericRecodeUsesFirstMatchInclusiveRangeAndSystemMissing(): void { $pdo = $this->fixture(); @@ -115,6 +234,26 @@ private function fixture(): PDO return $pdo; } + private function mysql(): PDO + { + $dsn = getenv('OPENSTATSPEC_MYSQL_DSN'); + if (!is_string($dsn) || $dsn === '') { + self::markTestSkipped('OPENSTATSPEC_MYSQL_DSN is not configured.'); + } + if (!in_array('mysql', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO MySQL is not available.'); + } + $user = getenv('OPENSTATSPEC_MYSQL_USER'); + $password = getenv('OPENSTATSPEC_MYSQL_PASSWORD'); + + return new PDO( + $dsn, + is_string($user) ? $user : null, + is_string($password) ? $password : null, + [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false], + ); + } + private function planCase(string $id): TransformationPlan { foreach (SpecificationManifest::load('conformance/transformation-plan-0.1.json')['cases'] as $case) { @@ -125,6 +264,36 @@ private function planCase(string $id): TransformationPlan throw new \RuntimeException('Missing plan case: ' . $id); } + /** @return array */ + private function bindingCase(string $id): array + { + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.1.json')['cases'] as $case) { + if (is_array($case) && ($case['id'] ?? null) === $id) { + return $case; + } + } + + throw new \RuntimeException('Missing in-place 0.1 case: ' . $id); + } + + /** @return array */ + private function mysqlFailureSnapshot(PDO $pdo, Connection $connection, string $table): array + { + return [ + 'dataset_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset'), + 'persistent_data_table_count' => (int) $this->scalar($pdo, 'SELECT COUNT(DISTINCT physical_table_name) FROM dataset'), + 'dataset' => $this->rows($pdo, 'SELECT * FROM dataset WHERE dataset_id = ?', [self::DATASET_ID]), + 'variables' => $this->rows($pdo, 'SELECT * FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', [self::DATASET_ID]), + 'audit' => $this->rows($pdo, 'SELECT * FROM transformation_apply WHERE dataset_id = ? ORDER BY apply_id', [self::DATASET_ID]), + 'tables' => $this->column($pdo, 'SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() ORDER BY table_name'), + 'rows' => $this->rows( + $pdo, + 'SELECT * FROM ' . $connection->profile->quoteIdentifier($table) + . ' ORDER BY ' . $connection->profile->quoteIdentifier('__case_ordinal'), + ), + ]; + } + /** @return list */ private function tables(PDO $pdo): array { @@ -138,10 +307,12 @@ private function normalizedAudit(PDO $pdo): array return [(string) $row['contract_id'], (string) $row['plan_hash'], (int) $row['operation_count']]; } - private function scalar(PDO $pdo, string $sql): mixed + /** @param list $parameters */ + private function scalar(PDO $pdo, string $sql, array $parameters = []): mixed { - $statement = $pdo->query($sql); + $statement = $pdo->prepare($sql); self::assertInstanceOf(PDOStatement::class, $statement); + $statement->execute($parameters); return $statement->fetchColumn(); } @@ -153,11 +324,15 @@ private function column(PDO $pdo, string $sql): array return array_values($statement->fetchAll(PDO::FETCH_COLUMN)); } - /** @return list> */ - private function rows(PDO $pdo, string $sql): array + /** + * @param list $parameters + * @return list> + */ + private function rows(PDO $pdo, string $sql, array $parameters = []): array { - $statement = $pdo->query($sql); + $statement = $pdo->prepare($sql); self::assertInstanceOf(PDOStatement::class, $statement); + $statement->execute($parameters); return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); } } diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index 866bfd0..bddc5dd 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -9,12 +9,15 @@ use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Tests\Support\SpecificationManifest; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Plan\PlanCodec; use PDO; use PDOStatement; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +use RuntimeException; final class OfficialInPlaceTransformation02Test extends TestCase { @@ -22,6 +25,96 @@ final class OfficialInPlaceTransformation02Test extends TestCase private const INEQUALITY_DATASET_ID = '44444444-4444-4444-8444-444444444444'; private const ROLLBACK_DATASET_ID = '55555555-5555-4555-8555-555555555555'; + /** @var list */ + private array $doltTestDatabases = []; + + protected function tearDown(): void + { + try { + foreach (array_reverse($this->doltTestDatabases) as $fixture) { + $fixture['admin']->exec('DROP DATABASE ' . $this->quoteDoltDatabase($fixture['database'])); + } + } finally { + $this->doltTestDatabases = []; + parent::tearDown(); + } + } + + /** @return iterable */ + public static function nonAtomicCreateCases(): iterable + { + yield 'MySQL' => ['mysql', 'mysql-create-target-fails-before-mutation']; + yield 'MariaDB' => ['mariadb', 'mariadb-create-target-fails-before-mutation']; + yield 'Dolt' => ['dolt', 'dolt-create-target-fails-before-mutation']; + } + + #[DataProvider('nonAtomicCreateCases')] + public function testOfficialMySqlFamilyCreateTargetCasesFailBeforeMutation( + string $expectedProfile, + string $caseId, + ): void { + $case = $this->bindingCase($caseId); + $pdo = $this->mysqlFamily($expectedProfile); + $connection = new Connection($pdo); + self::assertSame($expectedProfile, $connection->profileName); + $table = 'data_task9_create_' . $expectedProfile; + $this->prepareCatalog($pdo); + $this->assertNamespaceClean($pdo, self::CREATE_DATASET_ID, $table); + + try { + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + $table, + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $context = $this->commitAndReadDoltFixtureContext($pdo, $connection); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); + $request = $this->request( + $this->bindingCase('sqlite-create-target-atomic-success'), + self::CREATE_DATASET_ID, + $context['branch'] ?? null, + $context['head'] ?? null, + ); + + try { + (new InPlaceTransformationExecutor($connection))->execute($request); + self::fail('A non-atomic MySQL-family profile accepted an official create-target plan.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertFalse($pdo->inTransaction()); + self::assertFalse($case['mutation_started']); + self::assertSame($before, $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table)); + } finally { + $this->purgeFixture($pdo, $connection, self::CREATE_DATASET_ID, $table); + } + } + + public function testOfficialDoltEmptyActorCaseFailsBeforeMutation(): void + { + $case = $this->bindingCase('dolt-empty-actor-fails-before-mutation'); + + try { + new InPlaceApplyRequest( + $this->planCase('binding-variable-missing-existing-target'), + 'parent', + '11111111-1111-4111-8111-111111111111', + str_repeat('a', 64), + (string) $case['actor'], + 'feature/recode', + 'provisioning-commit', + ); + self::fail('The official empty actor was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + self::assertFalse($case['mutation_started']); + } + public function testSqliteCreateTargetIsAtomicAndPreservesIdentity(): void { $case = $this->bindingCase('sqlite-create-target-atomic-success'); @@ -311,6 +404,61 @@ private function postgresql(): PDO ); } + private function mysqlFamily(string $profile): PDO + { + if (!in_array('mysql', PDO::getAvailableDrivers(), true)) { + self::markTestSkipped('PDO MySQL is not available.'); + } + $prefix = match ($profile) { + 'mysql' => 'OPENSTATSPEC_MYSQL', + 'mariadb' => 'OPENSTATSPEC_MARIADB', + 'dolt' => 'OPENSTATSPEC_DOLT', + default => throw new RuntimeException('Unsupported MySQL-family test profile.'), + }; + $dsn = getenv($prefix . '_DSN'); + if (!is_string($dsn) || $dsn === '') { + self::markTestSkipped($prefix . '_DSN is not configured.'); + } + $user = getenv($prefix . '_USER'); + $password = getenv($prefix . '_PASSWORD'); + $options = [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]; + + if ($profile !== 'dolt') { + return new PDO( + $dsn, + is_string($user) ? $user : null, + is_string($password) ? $password : null, + $options, + ); + } + + $adminUser = getenv($prefix . '_ADMIN_USER'); + $adminPassword = getenv($prefix . '_ADMIN_PASSWORD'); + if (!is_string($adminUser) || $adminUser === '' || !is_string($adminPassword)) { + self::markTestSkipped('Explicit Dolt admin credentials are required for an isolated test database.'); + } + $database = sprintf('openstatspec_t9_%d_%s', getmypid(), bin2hex(random_bytes(6))); + $admin = new PDO($dsn, $adminUser, $adminPassword, $options); + $admin->exec('CREATE DATABASE ' . $this->quoteDoltDatabase($database)); + $this->doltTestDatabases[] = ['admin' => $admin, 'database' => $database]; + if (is_string($user) && $user !== '') { + $quotedUser = $admin->quote($user); + if (!is_string($quotedUser)) { + throw new RuntimeException('Unable to quote the Dolt test user.'); + } + $admin->exec( + 'GRANT ALL PRIVILEGES ON ' . $this->quoteDoltDatabase($database) . ".* TO {$quotedUser}@'%'", + ); + } + + return new PDO( + $this->dsnForDatabase($dsn, $database), + is_string($user) ? $user : null, + is_string($password) ? $password : null, + $options, + ); + } + /** * @param list $variables * @param list> $rows @@ -369,14 +517,20 @@ private function prepareCatalog(PDO $pdo): void } /** @param array $case */ - private function request(array $case, string $datasetId): InPlaceApplyRequest - { + private function request( + array $case, + string $datasetId, + ?string $expectedBranch = null, + ?string $expectedHead = null, + ): InPlaceApplyRequest { return new InPlaceApplyRequest( plan: $this->planCase((string) $case['applied_plan_case']), inputAlias: 'parent', datasetId: $datasetId, sourceHash: (string) $case['expected_audit']['source_hash'], actor: (string) $case['actor'], + expectedBranch: $expectedBranch, + expectedHead: $expectedHead, ); } @@ -540,12 +694,73 @@ private function purgeFixture(PDO $pdo, Connection $connection, string $datasetI $pdo->exec('DROP TABLE IF EXISTS ' . $connection->profile->quoteIdentifier($table)); } + /** @return array{branch: string, head: string}|null */ + private function commitAndReadDoltFixtureContext(PDO $pdo, Connection $connection): ?array + { + if ($connection->profileName !== 'dolt') { + return null; + } + $statement = $pdo->prepare('CALL DOLT_COMMIT(?, ?)'); + self::assertInstanceOf(PDOStatement::class, $statement); + $statement->execute(['-Am', 'Task 9 official create-target fixture']); + $identity = $pdo->query("SELECT active_branch() AS branch_name, dolt_hashof('HEAD') AS head_hash"); + self::assertInstanceOf(PDOStatement::class, $identity); + $row = $identity->fetch(PDO::FETCH_ASSOC); + self::assertIsArray($row); + $status = $pdo->query('SELECT table_name FROM dolt_status'); + self::assertInstanceOf(PDOStatement::class, $status); + self::assertSame([], $status->fetchAll(PDO::FETCH_COLUMN)); + + return ['branch' => (string) $row['branch_name'], 'head' => (string) $row['head_hash']]; + } + + private function quoteDoltDatabase(string $database): string + { + if (strlen($database) > 64 || preg_match('/\A[a-z][a-z0-9_]*\z/D', $database) !== 1) { + throw new RuntimeException('Unsafe Dolt test database name.'); + } + + return '`' . $database . '`'; + } + + private function dsnForDatabase(string $dsn, string $database): string + { + $this->quoteDoltDatabase($database); + if (!str_starts_with(strtolower($dsn), 'mysql:')) { + throw new RuntimeException('Dolt tests require a MySQL PDO DSN.'); + } + $parts = explode(';', substr($dsn, strlen('mysql:'))); + $found = null; + foreach ($parts as $index => $part) { + if (strtolower(trim((string) explode('=', $part, 2)[0])) !== 'dbname') { + continue; + } + if ($found !== null) { + throw new RuntimeException('Dolt DSN contains duplicate dbname settings.'); + } + $found = $index; + } + if ($found === null) { + if (end($parts) === '') { + array_pop($parts); + } + $parts[] = 'dbname=' . $database; + } else { + $parts[$found] = 'dbname=' . $database; + } + + return 'mysql:' . implode(';', $parts); + } + /** @return list */ private function tables(PDO $pdo): array { - $sql = (string) $pdo->getAttribute(PDO::ATTR_DRIVER_NAME) === 'sqlite' - ? "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name" - : "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = current_schema() ORDER BY tablename"; + $sql = match ((string) $pdo->getAttribute(PDO::ATTR_DRIVER_NAME)) { + 'sqlite' => "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + 'pgsql' => "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = current_schema() ORDER BY tablename", + 'mysql' => 'SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() ORDER BY table_name', + default => throw new RuntimeException('Unsupported integration driver.'), + }; return array_map('strval', $this->column($pdo, $sql)); } @@ -558,9 +773,10 @@ private function columns(PDO $pdo, Connection $connection, string $table): array $this->rows($pdo, 'PRAGMA table_info(' . $connection->profile->quoteIdentifier($table) . ')', []), ); } + $schema = $connection->profileName === 'postgresql' ? 'current_schema()' : 'DATABASE()'; return array_map('strval', $this->column( $pdo, - 'SELECT column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ? ORDER BY ordinal_position', + 'SELECT column_name FROM information_schema.columns WHERE table_schema = ' . $schema . ' AND table_name = ? ORDER BY ordinal_position', [$table], )); } diff --git a/tests/Transformation/Execution/DoltGuardTest.php b/tests/Transformation/Execution/DoltGuardTest.php index 52631eb..a17f731 100644 --- a/tests/Transformation/Execution/DoltGuardTest.php +++ b/tests/Transformation/Execution/DoltGuardTest.php @@ -4,15 +4,68 @@ namespace OpenStatSpec\Tests\Transformation\Execution; -use OpenStatSpec\Core\DiagnosticCode; -use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; use OpenStatSpec\Transformation\Execution\DoltEvidence; use OpenStatSpec\Transformation\Execution\DoltEvidenceReader; use OpenStatSpec\Transformation\Execution\DoltGuard; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; +use OpenStatSpec\Transformation\Plan\Operation\ExecuteOperation; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\TransformationPlan; use PHPUnit\Framework\TestCase; final class DoltGuardTest extends TestCase { + public function testItRequiresCallerSuppliedDoltContextBeforeReadingRepositoryState(): void + { + $reader = new class implements DoltEvidenceReader { + public int $reads = 0; + + public function read(): DoltEvidence + { + ++$this->reads; + return new DoltEvidence('feature/recode', 'expected-head', []); + } + }; + $guard = new DoltGuard($reader); + + try { + $guard->beforeExecution($this->request()); + self::fail('A Dolt apply without expected branch and HEAD was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_context_required', $failure->diagnosticCode()); + self::assertSame(0, $reader->reads); + } + } + + public function testItRejectsAnUnexpectedBranchBeforeExecution(): void + { + $guard = new DoltGuard($this->reader( + new DoltEvidence('main', 'expected-head', []), + )); + + try { + $guard->beforeExecution($this->request('feature/recode', 'expected-head')); + self::fail('An unexpected Dolt branch was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_branch_mismatch', $failure->diagnosticCode()); + } + } + + public function testItRejectsAnUnexpectedHeadBeforeExecution(): void + { + $guard = new DoltGuard($this->reader( + new DoltEvidence('feature/recode', 'other-head', []), + )); + + try { + $guard->beforeExecution($this->request('feature/recode', 'expected-head')); + self::fail('An unexpected Dolt HEAD was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_head_mismatch', $failure->diagnosticCode()); + } + } + public function testItRejectsADirtyWorkingSetBeforeExecution(): void { $guard = new DoltGuard($this->reader( @@ -20,12 +73,11 @@ public function testItRejectsADirtyWorkingSetBeforeExecution(): void )); try { - $guard->beforeExecution(); + $guard->beforeExecution($this->request('main', 'abc123')); self::fail('A dirty Dolt working set was accepted.'); - } catch (UnsupportedOperation $exception) { - self::assertSame(DiagnosticCode::SqlProfileOperationUnavailable, $exception->diagnosticCode); - self::assertStringContainsString('clean working set', $exception->getMessage()); - self::assertStringContainsString('respondents', $exception->getMessage()); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_working_set_dirty', $failure->diagnosticCode()); + self::assertStringContainsString('respondents', $failure->getMessage()); } } @@ -36,27 +88,15 @@ public function testItCapturesPostEvidenceWithoutRequiringTheExpectedEditToBeCle new DoltEvidence('main', 'abc123', ['respondents', 'variable']), )); - $before = $guard->beforeExecution(); - $after = $guard->afterExecution($before); + $request = $this->request('main', 'abc123'); + $before = $guard->beforeExecution($request); + $after = $guard->afterExecution($request, $before); self::assertTrue($before->isClean()); self::assertFalse($after->isClean()); self::assertSame(['respondents', 'variable'], $after->dirtyTables()); } - public function testItRejectsBranchOrHeadMutation(): void - { - $guard = new DoltGuard($this->reader( - new DoltEvidence('main', 'abc123', []), - new DoltEvidence('other', 'abc123', ['respondents']), - )); - $before = $guard->beforeExecution(); - - $this->expectException(UnsupportedOperation::class); - $this->expectExceptionMessage('branch changed'); - $guard->afterExecution($before); - } - private function reader(DoltEvidence ...$evidence): DoltEvidenceReader { return new class (array_values($evidence)) implements DoltEvidenceReader { @@ -74,4 +114,17 @@ public function read(): DoltEvidence } }; } + + private function request(?string $branch = null, ?string $head = null): InPlaceApplyRequest + { + return new InPlaceApplyRequest( + new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]), + 'parent', + '11111111-1111-4111-8111-111111111111', + str_repeat('a', 64), + 'conformance-runner', + $branch, + $head, + ); + } } diff --git a/tests/Transformation/Execution/DoltHeadGuardTest.php b/tests/Transformation/Execution/DoltHeadGuardTest.php index 1def689..96bc9ff 100644 --- a/tests/Transformation/Execution/DoltHeadGuardTest.php +++ b/tests/Transformation/Execution/DoltHeadGuardTest.php @@ -4,15 +4,19 @@ namespace OpenStatSpec\Tests\Transformation\Execution; -use OpenStatSpec\Core\UnsupportedOperation; +use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; use OpenStatSpec\Transformation\Execution\DoltEvidence; use OpenStatSpec\Transformation\Execution\DoltEvidenceReader; use OpenStatSpec\Transformation\Execution\DoltGuard; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; +use OpenStatSpec\Transformation\Plan\Operation\ExecuteOperation; +use OpenStatSpec\Transformation\Plan\PlanContract; +use OpenStatSpec\Transformation\Plan\TransformationPlan; use PHPUnit\Framework\TestCase; final class DoltHeadGuardTest extends TestCase { - public function testItRejectsAHeadChangeWithoutMutatingRepositoryState(): void + public function testItRejectsAHeadChangeWithTheOfficialPostMutationDiagnostic(): void { $reader = new class implements DoltEvidenceReader { private int $reads = 0; @@ -20,15 +24,55 @@ public function testItRejectsAHeadChangeWithoutMutatingRepositoryState(): void public function read(): DoltEvidence { return ++$this->reads === 1 - ? new DoltEvidence('main', 'before123', []) - : new DoltEvidence('main', 'after456', ['respondents']); + ? new DoltEvidence('feature/recode', 'expected-head', []) + : new DoltEvidence('feature/recode', 'after456', ['respondents']); } }; $guard = new DoltGuard($reader); - $before = $guard->beforeExecution(); + $request = $this->request(); + $before = $guard->beforeExecution($request); - $this->expectException(UnsupportedOperation::class); - $this->expectExceptionMessage('HEAD changed'); - $guard->afterExecution($before); + try { + $guard->afterExecution($request, $before); + self::fail('A post-mutation Dolt HEAD change was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_context_changed', $failure->diagnosticCode()); + } + } + + public function testItRejectsABranchChangeWithTheOfficialPostMutationDiagnostic(): void + { + $guard = new DoltGuard(new class implements DoltEvidenceReader { + private int $reads = 0; + + public function read(): DoltEvidence + { + return ++$this->reads === 1 + ? new DoltEvidence('feature/recode', 'expected-head', []) + : new DoltEvidence('main', 'expected-head', ['respondents']); + } + }); + $request = $this->request(); + $before = $guard->beforeExecution($request); + + try { + $guard->afterExecution($request, $before); + self::fail('A post-mutation Dolt branch change was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('dolt_context_changed', $failure->diagnosticCode()); + } + } + + private function request(): InPlaceApplyRequest + { + return new InPlaceApplyRequest( + new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]), + 'parent', + '11111111-1111-4111-8111-111111111111', + str_repeat('a', 64), + 'conformance-runner', + 'feature/recode', + 'expected-head', + ); } } From dadb3f139c1babdb6e96be57ab6871b472a08c69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 21:01:38 +0300 Subject: [PATCH 19/53] fix: harden MySQL transformation preflight --- .../task-9-report.md | 13 +- .../Execution/PlanPreflight.php | 50 ++ .../InPlaceTransformationServiceTest.php | 262 +++++++- .../OfficialInPlaceTransformation01Test.php | 608 ++++++++++++++---- 4 files changed, 792 insertions(+), 141 deletions(-) diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md index a6e32b2..8754c04 100644 --- a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md @@ -6,12 +6,17 @@ - Dolt execution requires caller-supplied expected branch and HEAD. The guard compares both to the live branch/HEAD and requires a clean working set before mutation. - After ordered operations, the guard rereads branch and HEAD inside the transaction and compares them to both the request and the initial evidence before the success audit and commit. - The executor reuses the Task 8 path: one native transaction contains data, metadata, post-operation Dolt validation, and exactly one compact success audit. Every failure rolls that transaction back. +- MySQL/MariaDB preflight now queries parameterized `information_schema.tables` metadata for the bound wide table and the six apply-participating catalog/audit tables. It requires exactly one `BASE TABLE` row using InnoDB for each relation and fails closed on MyISAM, unknown/NULL, missing, duplicate, view, or unreadable metadata before transaction start. +- The engine rule is limited to the `mysql` and `mariadb` profiles. Dolt keeps its independent transaction contract and is not treated as InnoDB merely because it uses the MySQL wire protocol. - Production code calls no Dolt mutation procedure and creates no dataset copy, output/staging/snapshot table, or hidden recovery state. ## TDD evidence - RED: seven focused Dolt tests produced three failures and four errors because the prior guard ignored request context and exposed the old signatures. - RED: the live MySQL-family official 0.2 create-target cases initially returned `unknown_variable` rather than `schema_change_not_atomic`, proving capability rejection happened too late. +- Review RED: 18 live MySQL/MariaDB storage-engine cases produced 15 acceptance failures; three malformed-relation cases were already rejected by namespace ownership. The accepted cases included a MyISAM bound wide table reaching an injected audit failure, all six apply-participating catalog/audit tables using MyISAM, and missing wide-table metadata on MySQL. +- Review GREEN: all 18 engine cases passed with 228 assertions. Exact before/after snapshots prove no data, catalog, audit, dataset-count, or physical-table-count change and no open transaction. +- Manifest-gate RED: the prior backend gate covered only one of the six official 0.1 case IDs; a clean inventory test failed on the other five IDs. GREEN runs every manifest row through an exhaustive ID dispatcher backed by the live MySQL or Dolt executor and passed 6 tests with 334 assertions. - GREEN: focused MySQL 8.4.9, MariaDB 11.4.8, and Dolt 2.2.2 execution passed 54 tests with 737 assertions and six PostgreSQL-only skips. - Official 0.1 MySQL and official 0.2 MySQL/MariaDB/Dolt create cases snapshot rows, catalog, audit, dataset count, persistent table count, and repository evidence; all reject before mutation with exact equality. - Dolt tests cover empty actor, missing context without an evidence read, initial branch/HEAD mismatch, dirty state, and injected post-mutation context change with exact error codes. @@ -20,13 +25,15 @@ ## Live evidence - MySQL: 8.4.9; MariaDB: 11.4.8-MariaDB-ubu2404; Dolt: 2.2.2 (MySQL wire compatibility 8.0.31). +- The final focused live gate ran all six official 0.1 cases and all eleven official 0.2 cases, plus engine and Dolt guard regressions: 74 tests, 1,266 assertions, and six PostgreSQL-only skips. No configured MySQL, MariaDB, or Dolt case skipped. - Dolt Task 9 cases use randomly named isolated databases with explicit namespace prechecks and teardown. Successful existing-target apply preserves branch and HEAD, produces one dirty working-set diff for the in-place edit, and adds no Dolt commit; failure leaves no edit or audit. -- The temporary Dolt repository remained on `main` at HEAD `jup6aortet0oao7jppgohjtilb74oepd` with exactly one setup commit after the full run. Legacy round-trip tests leave unrelated untracked fixture tables in the shared base database; Task 9 assertions run in isolated databases and clean them up. +- The authoritative temporary Dolt repository remained on `main` at HEAD `dvba843qo8iqg962au54meed7degbeok` with exactly one setup commit after the full run. Legacy round-trip tests leave unrelated untracked fixture tables in the shared base database; Task 9 assertions run in isolated databases and clean them up. ## Verification -- Full live `composer test`: 455 tests, 5,038 assertions, nine PostgreSQL skips; passed after correcting the Dolt test DSN to select its base database. -- Full LF-normalized staged-tree `composer check`: Composer validation, PHP lint, PHP CS Fixer, PHPStan, and PHPUnit passed; PHPUnit ran 455 tests with 5,040 assertions and nine PostgreSQL skips. +- Full live `composer test`: 475 tests, 5,565 assertions, nine PostgreSQL skips; passed with MySQL 8.4.9, MariaDB 11.4.8, and Dolt 2.2.2 configured. +- Full LF-normalized staged-tree `composer check`: Composer validation, PHP lint, PHP CS Fixer over 226 files, PHPStan, and PHPUnit passed; PHPUnit ran 475 tests with 5,565 assertions and nine PostgreSQL skips. +- The first normalized run reached Composer's default 300-second process timeout during a legacy Dolt dictionary test. Its interrupted fixture was confined to the temporary Dolt volume; that named container/volume was recreated from a clean one-commit repository, and the authoritative full check passed with `COMPOSER_PROCESS_TIMEOUT=900`. - `composer analyse`: passed with no errors. `composer lint`: passed. Task-changed-file PHP CS Fixer dry-run: zero fixable files. `git diff --cached --check`: passed. - Working-tree `composer style` still exits 8 because 122 unchanged repository PHP files retain the pre-existing CRLF baseline; the normalized full gate checked all 226 PHP files cleanly. - The CRLF pre-commit hook cannot execute directly (`bash\r` shebang), so the commit uses `--no-verify` only after the equivalent normalized staged-tree gate passed. diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index 3456c57..15f45e2 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -67,6 +67,7 @@ public function bind(InPlaceApplyRequest $request): BoundPlan $this->assertAuditReady(); $dataset = $this->resolveDataset($request->datasetId); + $this->assertMySqlFamilyTransactionalEngines($dataset); $this->assertExclusivePhysicalTableBinding($dataset); $variables = $this->resolveVariables($dataset); $physicalColumns = $this->physicalColumns($dataset); @@ -658,6 +659,55 @@ private function assertAuditReady(): void } } + private function assertMySqlFamilyTransactionalEngines(DatasetBinding $dataset): void + { + if (!in_array($this->connection->profileName, ['mysql', 'mariadb'], true)) { + return; + } + + $relations = [ + [$dataset->schema, $dataset->table], + [null, 'dataset'], + [null, 'variable'], + [null, 'value_label_set'], + [null, 'value_label'], + [null, 'variable_value_label_set'], + [null, 'transformation_apply'], + ]; + foreach ($relations as [$schema, $table]) { + try { + $statement = $this->statement( + 'SELECT table_type, engine FROM information_schema.tables ' + . 'WHERE table_schema = COALESCE(?, DATABASE()) AND table_name = ?', + ); + CheckedPdo::execute( + $statement, + [$schema, $table], + 'MySQL-family storage-engine metadata could not be queried.', + ); + $rows = $statement->fetchAll(PDO::FETCH_ASSOC); + } catch (PDOException $failure) { + throw new UnsupportedOperation( + DiagnosticCode::SqlProfileOperationUnavailable, + 'MySQL-family transactional storage-engine metadata is unavailable: ' . $failure->getMessage(), + ); + } + + $row = isset($rows[0]) ? array_change_key_case($rows[0], CASE_LOWER) : []; + if (count($rows) !== 1 + || !is_string($row['table_type'] ?? null) + || strcasecmp($row['table_type'], 'BASE TABLE') !== 0 + || !is_string($row['engine'] ?? null) + || strcasecmp($row['engine'], 'InnoDB') !== 0 + ) { + throw new UnsupportedOperation( + DiagnosticCode::SqlProfileOperationUnavailable, + 'MySQL-family atomic apply requires every participating data, catalog, and audit table to use InnoDB.', + ); + } + } + } + private function statement(string $sql): PDOStatement { return CheckedPdo::prepare( diff --git a/tests/Integration/InPlaceTransformationServiceTest.php b/tests/Integration/InPlaceTransformationServiceTest.php index fc6fd9a..002956d 100644 --- a/tests/Integration/InPlaceTransformationServiceTest.php +++ b/tests/Integration/InPlaceTransformationServiceTest.php @@ -53,7 +53,7 @@ final class InPlaceTransformationServiceTest extends TestCase private ?array $activeFixture = null; /** @var list */ - private array $doltTestDatabases = []; + private array $isolatedTestDatabases = []; private ?string $doltDatabaseRunId = null; @@ -62,13 +62,13 @@ final class InPlaceTransformationServiceTest extends TestCase protected function tearDown(): void { try { - foreach (array_reverse($this->doltTestDatabases) as $testDatabase) { + foreach (array_reverse($this->isolatedTestDatabases) as $testDatabase) { $testDatabase['admin']->exec( 'DROP DATABASE ' . $this->quoteDoltDatabaseName($testDatabase['database']), ); } } finally { - $this->doltTestDatabases = []; + $this->isolatedTestDatabases = []; $this->activeFixture = null; parent::tearDown(); } @@ -120,6 +120,16 @@ public static function transactionalMySqlServices(): iterable yield 'MariaDB' => ['mariadb', 'OPENSTATSPEC_MARIADB', 'mysql']; } + /** @return iterable */ + public static function mySqlFamilyCatalogAndAuditEngineCases(): iterable + { + foreach (self::transactionalMySqlServices() as $service => $configuration) { + foreach (['dataset', 'variable', 'value_label_set', 'value_label', 'variable_value_label_set', 'transformation_apply'] as $table) { + yield $service . ' / ' . $table => [...$configuration, $table]; + } + } + } + /** @param list $expectedVersionFamilies */ #[DataProvider('services')] public function testConfiguredServiceMatchesExpectedProfileAndVersionFamily( @@ -385,6 +395,161 @@ public function testMySqlFamilyDataCatalogAndAuditAreOneTransaction( } } + #[DataProvider('transactionalMySqlServices')] + public function testMySqlFamilyRejectsMyIsamWideTableBeforeForcedAuditFailure( + string $expectedProfile, + string $environmentPrefix, + string $driver, + ): void { + unset($driver); + $pdo = $this->isolatedMySqlFamilyPdo($expectedProfile, $environmentPrefix); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $constraint = 'task9_nontransactional_wide_' . $expectedProfile; + $quotedConstraint = $connection->profile->quoteIdentifier($constraint); + + try { + $pdo->exec( + 'ALTER TABLE ' . $connection->profile->quoteIdentifier($fixture['table_name']) . ' ENGINE=MyISAM', + ); + $pdo->exec( + 'ALTER TABLE transformation_apply ADD CONSTRAINT ' . $quotedConstraint + . " CHECK (actor <> 'integration-test')", + ); + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A MyISAM-bound wide table was accepted for an atomic apply.'); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::SqlProfileOperationUnavailable, $failure->diagnosticCode); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } finally { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT ' . $quotedConstraint); + $this->purgeFixture($pdo, $connection, $fixture); + } + } + + #[DataProvider('mySqlFamilyCatalogAndAuditEngineCases')] + public function testMySqlFamilyRejectsNonTransactionalCatalogAndAuditEngines( + string $expectedProfile, + string $environmentPrefix, + string $driver, + string $table, + ): void { + unset($driver); + $pdo = $this->isolatedMySqlFamilyPdo($expectedProfile, $environmentPrefix); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $this->dropMySqlForeignKeys($pdo, $connection); + $this->dropMySqlSecondaryIndexes($pdo, $connection, $table); + $pdo->exec('ALTER TABLE ' . $connection->profile->quoteIdentifier($table) . ' ENGINE=MyISAM'); + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A non-transactional catalog or audit table was accepted: ' . $table); + } catch (UnsupportedOperation $failure) { + self::assertSame(DiagnosticCode::SqlProfileOperationUnavailable, $failure->diagnosticCode); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } + + #[DataProvider('transactionalMySqlServices')] + public function testMySqlFamilyRejectsMissingWideTableEngineMetadata( + string $expectedProfile, + string $environmentPrefix, + string $driver, + ): void { + unset($driver); + $pdo = $this->isolatedMySqlFamilyPdo($expectedProfile, $environmentPrefix); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $quotedTable = $connection->profile->quoteIdentifier($fixture['table_name']); + $rows = $this->rawTableRows( + $pdo, + $connection, + $fixture['table_name'], + ['__case_ordinal', 'source_value', 'destination'], + ); + $pdo->exec('DROP TABLE ' . $quotedTable); + $pdo->exec( + 'CREATE TEMPORARY TABLE ' . $quotedTable . ' (' + . '`__case_ordinal` BIGINT NOT NULL PRIMARY KEY, ' + . '`source_value` DOUBLE NULL, `destination` DOUBLE NULL) ENGINE=InnoDB', + ); + $insert = $pdo->prepare( + 'INSERT INTO ' . $quotedTable . ' (`__case_ordinal`, `source_value`, `destination`) VALUES (?, ?, ?)', + ); + foreach ($rows as $row) { + $insert->execute([$row['__case_ordinal'], $row['source_value'], $row['destination']]); + } + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A table missing from storage-engine metadata was accepted.'); + } catch (UnsupportedOperation $failure) { + self::assertContains($failure->diagnosticCode, [ + DiagnosticCode::SqlProfileOperationUnavailable, + DiagnosticCode::CatalogNamespaceCollision, + ]); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } + + #[DataProvider('transactionalMySqlServices')] + public function testMySqlFamilyRejectsUnknownViewEngineMetadata( + string $expectedProfile, + string $environmentPrefix, + string $driver, + ): void { + unset($driver); + $pdo = $this->isolatedMySqlFamilyPdo($expectedProfile, $environmentPrefix); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + $quotedTable = $connection->profile->quoteIdentifier($fixture['table_name']); + $backing = $fixture['table_name'] . '_backing'; + $quotedBacking = $connection->profile->quoteIdentifier($backing); + $pdo->exec('RENAME TABLE ' . $quotedTable . ' TO ' . $quotedBacking); + $pdo->exec('CREATE VIEW ' . $quotedTable . ' AS SELECT * FROM ' . $quotedBacking); + $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); + + try { + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('A view with NULL storage-engine metadata was accepted.'); + } catch (UnsupportedOperation $failure) { + self::assertContains($failure->diagnosticCode, [ + DiagnosticCode::SqlProfileOperationUnavailable, + DiagnosticCode::CatalogNamespaceCollision, + ]); + } + + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); + } finally { + $pdo->exec('DROP VIEW IF EXISTS ' . $quotedTable); + $pdo->exec('RENAME TABLE ' . $quotedBacking . ' TO ' . $quotedTable); + $this->purgeFixture($pdo, $connection, $fixture); + } + } + /** @param list $expectedVersionFamilies */ #[DataProvider('createTargetCapableServices')] public function testCreateTargetTransformationCreatesNumericTargetInPlace( @@ -1069,6 +1234,95 @@ private function servicePdo(string $expectedProfile, ?string $environmentPrefix, ); } + private function isolatedMySqlFamilyPdo(string $expectedProfile, string $environmentPrefix): PDO + { + $dsn = getenv($environmentPrefix . '_DSN'); + $adminUser = getenv($environmentPrefix . '_ADMIN_USER'); + $adminPassword = getenv($environmentPrefix . '_ADMIN_PASSWORD'); + if (!is_string($dsn) || $dsn === '') { + self::markTestSkipped($environmentPrefix . '_DSN is not configured.'); + } + if (!is_string($adminUser) || $adminUser === '' || !is_string($adminPassword)) { + self::markTestSkipped( + $environmentPrefix . '_ADMIN_USER and ' . $environmentPrefix + . '_ADMIN_PASSWORD must be explicitly configured for isolated engine tests.', + ); + } + $targetUser = getenv($environmentPrefix . '_USER'); + $targetPassword = getenv($environmentPrefix . '_PASSWORD'); + $options = [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]; + $database = $this->newDoltDatabaseName(); + $admin = new PDO($dsn, $adminUser, $adminPassword, $options); + $admin->exec('CREATE DATABASE ' . $this->quoteDoltDatabaseName($database)); + $this->isolatedTestDatabases[] = ['admin' => $admin, 'database' => $database]; + if (is_string($targetUser) && $targetUser !== '') { + $quotedTargetUser = $admin->quote($targetUser); + if (!is_string($quotedTargetUser)) { + throw new RuntimeException('Unable to quote the MySQL-family target user.'); + } + $admin->exec( + 'GRANT ALL PRIVILEGES ON ' . $this->quoteDoltDatabaseName($database) + . ".* TO {$quotedTargetUser}@'%'", + ); + } + $pdo = new PDO( + $this->dsnForDoltDatabase($dsn, $database), + is_string($targetUser) ? $targetUser : null, + is_string($targetPassword) ? $targetPassword : null, + $options, + ); + self::assertSame($expectedProfile, (new Connection($pdo))->profileName); + self::assertSame($database, $this->scalar($pdo, 'SELECT DATABASE()', [])); + + return $pdo; + } + + private function dropMySqlForeignKeys(PDO $pdo, Connection $connection): void + { + $constraints = $this->rows( + $pdo, + 'SELECT table_name, constraint_name FROM information_schema.referential_constraints ' + . 'WHERE constraint_schema = DATABASE() ORDER BY table_name, constraint_name', + [], + ); + foreach ($constraints as $constraint) { + $constraint = array_change_key_case($constraint, CASE_LOWER); + $table = $constraint['table_name'] ?? null; + $name = $constraint['constraint_name'] ?? null; + if (!is_string($table) || $table === '' || !is_string($name) || $name === '') { + throw new RuntimeException('MySQL-family foreign-key metadata is malformed.'); + } + $pdo->exec( + 'ALTER TABLE ' . $connection->profile->quoteIdentifier($table) + . ' DROP FOREIGN KEY ' . $connection->profile->quoteIdentifier($name), + ); + } + } + + private function dropMySqlSecondaryIndexes(PDO $pdo, Connection $connection, string $table): void + { + $indexRows = $this->rows( + $pdo, + 'SELECT index_name FROM information_schema.statistics ' + . "WHERE table_schema = DATABASE() AND table_name = ? AND index_name <> 'PRIMARY' ORDER BY index_name", + [$table], + ); + $indexes = []; + foreach ($indexRows as $row) { + $row = array_change_key_case($row, CASE_LOWER); + if (!is_string($row['index_name'] ?? null) || $row['index_name'] === '') { + throw new RuntimeException('MySQL-family index metadata is malformed.'); + } + $indexes[$row['index_name']] = true; + } + foreach (array_keys($indexes) as $index) { + $pdo->exec( + 'ALTER TABLE ' . $connection->profile->quoteIdentifier($table) + . ' DROP INDEX ' . $connection->profile->quoteIdentifier($index), + ); + } + } + /** @return array{user: string, password: string}|null */ private function doltAdminCredentials(string|false $user, string|false $password): ?array { @@ -1128,7 +1382,7 @@ private function isolatedDoltPdo( $options, ); $admin->exec('CREATE DATABASE ' . $this->quoteDoltDatabaseName($database)); - $this->doltTestDatabases[] = ['admin' => $admin, 'database' => $database]; + $this->isolatedTestDatabases[] = ['admin' => $admin, 'database' => $database]; if ($targetUser !== null && $targetUser !== '') { $quotedTargetUser = $admin->quote($targetUser); if (!is_string($quotedTargetUser)) { diff --git a/tests/Integration/OfficialInPlaceTransformation01Test.php b/tests/Integration/OfficialInPlaceTransformation01Test.php index 35cd872..3f1fb6b 100644 --- a/tests/Integration/OfficialInPlaceTransformation01Test.php +++ b/tests/Integration/OfficialInPlaceTransformation01Test.php @@ -4,15 +4,16 @@ namespace OpenStatSpec\Tests\Integration; +use OpenStatSpec\Frontend\Spss\Request\InputSchema; +use OpenStatSpec\Frontend\Spss\Request\InputVariable; +use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; +use OpenStatSpec\Frontend\Spss\SpssCompiler; use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; use OpenStatSpec\Tests\Support\SpecificationManifest; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; -use OpenStatSpec\Transformation\Execution\DoltEvidence; -use OpenStatSpec\Transformation\Execution\DoltEvidenceReader; -use OpenStatSpec\Transformation\Execution\DoltGuard; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Plan\PlanCodec; @@ -21,123 +22,53 @@ use PDOStatement; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +use RuntimeException; final class OfficialInPlaceTransformation01Test extends TestCase { private const DATASET_ID = '66666666-6666-4666-8666-666666666666'; - /** @return iterable}> */ - public static function officialDoltContextFailures(): iterable + /** @var list */ + private array $isolatedTestDatabases = []; + + protected function tearDown(): void { - foreach (SpecificationManifest::load('conformance/in-place-transformation-0.1.json')['cases'] as $case) { - if (is_array($case) && in_array($case['id'] ?? null, [ - 'reject-dolt-branch-mismatch', - 'reject-dolt-head-mismatch', - 'reject-dolt-dirty-working-set', - ], true)) { - yield (string) $case['id'] => [$case]; + try { + foreach (array_reverse($this->isolatedTestDatabases) as $fixture) { + $fixture['admin']->exec('DROP DATABASE ' . $this->quoteDatabase($fixture['database'])); } + } finally { + $this->isolatedTestDatabases = []; + parent::tearDown(); } } - /** @param array $case */ - #[DataProvider('officialDoltContextFailures')] - public function testOfficialDoltContextFailuresOccurBeforeMutation(array $case): void + /** @return iterable}> */ + public static function officialBackendCases(): iterable { - $expected = $case['expected_context']; - $observed = $case['observed_context']; - self::assertIsArray($expected); - self::assertIsArray($observed); - $guard = new DoltGuard(new class ($observed) implements DoltEvidenceReader { - /** @param array $observed */ - public function __construct(private readonly array $observed) {} - - public function read(): DoltEvidence - { - return new DoltEvidence( - (string) $this->observed['branch'], - (string) $this->observed['head'], - $this->observed['working_set_clean'] === true ? [] : ['data_survey'], - ); + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.1.json')['cases'] as $case) { + if (is_array($case) && is_string($case['id'] ?? null)) { + yield $case['id'] => [$case]; } - }); - $request = new InPlaceApplyRequest( - $this->planCase('string-value-label-replacement'), - 'parent', - self::DATASET_ID, - str_repeat('a', 64), - 'conformance-runner', - (string) $expected['branch'], - (string) $expected['head'], - ); - - try { - $guard->beforeExecution($request); - self::fail('The official Dolt context mismatch was accepted.'); - } catch (TransformationFailure $failure) { - self::assertSame($case['expected_error'], $failure->diagnosticCode()); } - self::assertFalse($case['mutation_started']); } - public function testOfficialMysqlCreateTargetCaseFailsBeforeMutationWhenConfigured(): void + /** @param array $case */ + #[DataProvider('officialBackendCases')] + public function testEveryOfficialBackendManifestCaseRuns(array $case): void { - $case = $this->bindingCase('reject-mysql-nontransactional-create-target'); - $pdo = $this->mysql(); - $connection = new Connection($pdo); - self::assertSame('mysql', $connection->profileName); - $table = 'data_plan01_task9'; - (new NormativeCatalog($pdo))->createTables(); - (new TransformationAuditMigrator($pdo))->migrate(); - CatalogOwnership::markCurrentVersion($pdo); - self::assertSame(0, (int) $this->scalar( - $pdo, - 'SELECT COUNT(*) FROM dataset WHERE dataset_id = ? OR physical_table_name = ?', - [self::DATASET_ID, $table], - )); - self::assertSame(0, (int) $this->scalar( - $pdo, - 'SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ?', - [$table], - )); - - try { - $pdo->exec('CREATE TABLE ' . $connection->profile->quoteIdentifier($table) - . ' (`__case_ordinal` BIGINT NOT NULL PRIMARY KEY, `q1` DOUBLE NULL)'); - $pdo->prepare( - 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) ' - . 'VALUES (?, ?, ?, ?, ?, ?, ?, ?)', - )->execute([self::DATASET_ID, '1.0', 'fixture', null, $table, 'Task 9 official 0.1', 2, '2026-08-17 00:00:00']); - $pdo->prepare( - 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) VALUES (?, ?, ?, ?, ?, ?)', - )->execute(['99999999-9999-4999-8999-999999999999', self::DATASET_ID, 1, 'q1', 'q1', 'numeric']); - $pdo->exec('INSERT INTO ' . $connection->profile->quoteIdentifier($table) - . ' (`__case_ordinal`, `q1`) VALUES (1, 1), (2, NULL)'); - $before = $this->mysqlFailureSnapshot($pdo, $connection, $table); - $request = new InPlaceApplyRequest( - $this->planCase('numeric-recode-and-declared-labels'), - 'parent', - self::DATASET_ID, - hash('sha256', (string) $case['source_text']), - 'conformance-runner', - ); + $id = $case['id'] ?? null; + self::assertIsString($id); - try { - (new InPlaceTransformationExecutor($connection))->execute($request); - self::fail('MySQL accepted an official 0.1 create-target plan.'); - } catch (TransformationFailure $failure) { - self::assertSame($case['expected_error'], $failure->diagnosticCode()); - } - - self::assertFalse($case['mutation_started']); - self::assertFalse($pdo->inTransaction()); - self::assertSame($before, $this->mysqlFailureSnapshot($pdo, $connection, $table)); - } finally { - $pdo->prepare('DELETE FROM transformation_apply WHERE dataset_id = ?')->execute([self::DATASET_ID]); - $pdo->prepare('DELETE FROM variable WHERE dataset_id = ?')->execute([self::DATASET_ID]); - $pdo->prepare('DELETE FROM dataset WHERE dataset_id = ?')->execute([self::DATASET_ID]); - $pdo->exec('DROP TABLE IF EXISTS ' . $connection->profile->quoteIdentifier($table)); - } + match ($id) { + 'mysql-recode-and-labels-preserve-dataset-and-table-identity', + 'dolt-recode-and-labels-preserve-controlled-context' => $this->assertOfficialBackendSuccess($case), + 'reject-dolt-branch-mismatch', + 'reject-dolt-head-mismatch', + 'reject-dolt-dirty-working-set' => $this->assertOfficialDoltContextFailure($case), + 'reject-mysql-nontransactional-create-target' => $this->assertOfficialMySqlCreateRejection($case), + default => throw new RuntimeException('Unhandled official in-place 0.1 case: ' . $id), + }; } public function testSqliteNumericRecodeUsesFirstMatchInclusiveRangeAndSystemMissing(): void @@ -234,24 +165,451 @@ private function fixture(): PDO return $pdo; } - private function mysql(): PDO + /** @param array $case */ + private function assertOfficialBackendSuccess(array $case): void { - $dsn = getenv('OPENSTATSPEC_MYSQL_DSN'); + $profile = $case['database_profile'] ?? null; + $beforeContract = $case['before'] ?? null; + $afterContract = $case['after'] ?? null; + self::assertContains($profile, ['mysql', 'dolt']); + self::assertIsArray($beforeContract); + self::assertIsArray($afterContract); + $database = $profile === 'mysql' ? (string) $beforeContract['physical_table_schema'] : null; + $pdo = $this->isolatedBackendPdo((string) $profile, $database); + $connection = new Connection($pdo); + $fixture = $this->installOfficialBackendFixture($pdo, $connection, $case); + $context = $this->backendContext($pdo, $connection); + $before = $this->officialBackendSnapshot($pdo, $connection, $fixture); + $compiled = $this->compileOfficialSource((string) $case['source_text']); + $request = new InPlaceApplyRequest( + $compiled->plan, + 'parent', + $fixture['dataset_id'], + $compiled->sourceHash, + 'conformance-runner', + $context['branch'] ?? null, + $context['head'] ?? null, + ); + + $result = (new InPlaceTransformationExecutor($connection))->execute($request); + $after = $this->officialBackendSnapshot($pdo, $connection, $fixture); + + self::assertSame($beforeContract['dataset_id'], $result->datasetId()); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($before['case_count'], $after['case_count']); + self::assertSame($before['case_ordinals'], $after['case_ordinals']); + self::assertSame([0.0, 0.0, 1.0, 4.0], $after['score']); + self::assertSame($afterContract['variable_label'], $after['variable_label']); + self::assertSame($afterContract['value_labels'], $after['value_labels']); + self::assertCount(1, $after['audit']); + $audit = $after['audit'][0]; + foreach ($case['required_audit_fields'] as $field) { + self::assertArrayHasKey($field, $audit); + } + self::assertSame('openstatspec-in-place-transformation-v0.1', $audit['contract_id']); + self::assertSame($profile, $audit['database_profile']); + self::assertSame($fixture['dataset_id'], $audit['dataset_id']); + self::assertSame($fixture['schema'], $audit['physical_table_schema']); + self::assertSame($fixture['table'], $audit['physical_table_name']); + self::assertSame($compiled->sourceHash, $audit['source_hash']); + self::assertSame($result->planHash(), $audit['plan_hash']); + self::assertSame('conformance-runner', $audit['actor']); + self::assertSame('succeeded', $audit['status']); + self::assertSame($result->operationCount(), (int) $audit['operation_count']); + $this->assertNoForbiddenArtifactTables($after['tables']); + + if ($profile === 'dolt') { + if ($context === null) { + throw new RuntimeException('Dolt conformance execution did not capture repository context.'); + } + self::assertNotNull($before['repository']); + self::assertNotNull($after['repository']); + self::assertSame($before['repository']['branch'], $after['repository']['branch']); + self::assertSame($before['repository']['head'], $after['repository']['head']); + self::assertSame($before['repository']['history'], $after['repository']['history']); + self::assertNotSame([], $after['repository']['status']); + self::assertContains($fixture['table'], array_column($after['repository']['status'], 'table_name')); + self::assertFalse((bool) $afterContract['dolt_commit_performed']); + self::assertSame($context['branch'], $audit['dolt_branch']); + self::assertSame($context['head'], $audit['dolt_head_before']); + self::assertSame($context['head'], $audit['dolt_head_after']); + } else { + self::assertNull($audit['dolt_branch']); + self::assertNull($audit['dolt_head_before']); + self::assertNull($audit['dolt_head_after']); + } + } + + /** @param array $case */ + private function assertOfficialDoltContextFailure(array $case): void + { + $pdo = $this->isolatedBackendPdo('dolt'); + $connection = new Connection($pdo); + $fixture = $this->installOfficialBackendFixture($pdo, $connection, $case); + $context = $this->backendContext($pdo, $connection); + self::assertNotNull($context); + $id = (string) $case['id']; + if ($id === 'reject-dolt-dirty-working-set') { + $pdo->exec( + 'UPDATE ' . $this->qualifiedBackendTable($connection, $fixture['schema'], $fixture['table']) + . ' SET `score` = 99 WHERE `__case_ordinal` = 1', + ); + } + $before = $this->officialBackendSnapshot($pdo, $connection, $fixture); + $source = $this->successfulSourceText(); + $compiled = $this->compileOfficialSource($source); + $expectedBranch = $context['branch']; + $expectedHead = $context['head']; + if ($id === 'reject-dolt-branch-mismatch') { + $expectedBranch = (string) $case['expected_context']['branch']; + } elseif ($id === 'reject-dolt-head-mismatch') { + $expectedHead = (string) $case['observed_context']['head']; + } + + try { + (new InPlaceTransformationExecutor($connection))->execute(new InPlaceApplyRequest( + $compiled->plan, + 'parent', + $fixture['dataset_id'], + hash('sha256', $source), + 'conformance-runner', + $expectedBranch, + $expectedHead, + )); + self::fail('An official Dolt context failure was accepted: ' . $id); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertFalse($case['mutation_started']); + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->officialBackendSnapshot($pdo, $connection, $fixture)); + } + + /** @param array $case */ + private function assertOfficialMySqlCreateRejection(array $case): void + { + $pdo = $this->isolatedBackendPdo('mysql'); + $connection = new Connection($pdo); + $fixture = $this->installOfficialBackendFixture($pdo, $connection, $case); + $before = $this->officialBackendSnapshot($pdo, $connection, $fixture); + $compiled = $this->compileOfficialSource((string) $case['source_text']); + + try { + (new InPlaceTransformationExecutor($connection))->execute(new InPlaceApplyRequest( + $compiled->plan, + 'parent', + $fixture['dataset_id'], + $compiled->sourceHash, + 'conformance-runner', + )); + self::fail('The official MySQL create-target rejection was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertFalse($case['mutation_started']); + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->officialBackendSnapshot($pdo, $connection, $fixture)); + } + + private function compileOfficialSource(string $source): \OpenStatSpec\Frontend\Spss\SpssCompilationResult + { + return (new SpssCompiler())->compile(new SpssFrontendRequest( + SpssFrontendRequest::CONTRACT, + 'parent', + new InputSchema([new InputVariable('score', 'numeric')]), + $source, + )); + } + + private function successfulSourceText(): string + { + return (string) $this->bindingCase('mysql-recode-and-labels-preserve-dataset-and-table-identity')['source_text']; + } + + private function isolatedBackendPdo(string $profile, ?string $preferredDatabase = null): PDO + { + $prefix = match ($profile) { + 'mysql' => 'OPENSTATSPEC_MYSQL', + 'dolt' => 'OPENSTATSPEC_DOLT', + default => throw new RuntimeException('Unsupported official 0.1 backend profile.'), + }; + $dsn = getenv($prefix . '_DSN'); + $adminUser = getenv($prefix . '_ADMIN_USER'); + $adminPassword = getenv($prefix . '_ADMIN_PASSWORD'); if (!is_string($dsn) || $dsn === '') { - self::markTestSkipped('OPENSTATSPEC_MYSQL_DSN is not configured.'); + self::markTestSkipped($prefix . '_DSN is not configured.'); } - if (!in_array('mysql', PDO::getAvailableDrivers(), true)) { - self::markTestSkipped('PDO MySQL is not available.'); + if (!is_string($adminUser) || $adminUser === '' || !is_string($adminPassword)) { + self::markTestSkipped($prefix . ' explicit admin credentials are required for isolated official tests.'); } - $user = getenv('OPENSTATSPEC_MYSQL_USER'); - $password = getenv('OPENSTATSPEC_MYSQL_PASSWORD'); - - return new PDO( - $dsn, - is_string($user) ? $user : null, - is_string($password) ? $password : null, - [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false], + $targetUser = getenv($prefix . '_USER'); + $targetPassword = getenv($prefix . '_PASSWORD'); + $options = [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]; + $database = $preferredDatabase ?? sprintf( + 'openstatspec_t9_v01_%d_%s', + getmypid(), + bin2hex(random_bytes(6)), ); + $quotedDatabase = $this->quoteDatabase($database); + $admin = new PDO($dsn, $adminUser, $adminPassword, $options); + self::assertSame(0, (int) $this->scalar( + $admin, + 'SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = ?', + [$database], + )); + $admin->exec('CREATE DATABASE ' . $quotedDatabase); + $this->isolatedTestDatabases[] = ['admin' => $admin, 'database' => $database]; + if (is_string($targetUser) && $targetUser !== '') { + $quotedTargetUser = $admin->quote($targetUser); + if (!is_string($quotedTargetUser)) { + throw new RuntimeException('Unable to quote the official backend test user.'); + } + $admin->exec('GRANT ALL PRIVILEGES ON ' . $quotedDatabase . ".* TO {$quotedTargetUser}@'%'"); + } + + $pdo = new PDO( + $this->dsnForDatabase($dsn, $database), + is_string($targetUser) ? $targetUser : null, + is_string($targetPassword) ? $targetPassword : null, + $options, + ); + self::assertSame($profile, (new Connection($pdo))->profileName); + self::assertSame($database, $this->scalar($pdo, 'SELECT DATABASE()')); + + return $pdo; + } + + /** + * @param array $case + * @return array{dataset_id: string, schema: string|null, table: string} + */ + private function installOfficialBackendFixture(PDO $pdo, Connection $connection, array $case): array + { + $before = is_array($case['before'] ?? null) ? $case['before'] : []; + $observed = is_array($case['observed_context'] ?? null) ? $case['observed_context'] : []; + if ($connection->profileName === 'dolt') { + $branch = is_string($observed['branch'] ?? null) + ? $observed['branch'] + : (is_string($before['dolt_branch'] ?? null) ? $before['dolt_branch'] : 'main'); + $active = (string) $this->scalar($pdo, 'SELECT active_branch()'); + if ($branch !== $active) { + $checkout = $pdo->prepare('CALL DOLT_CHECKOUT(?, ?)'); + self::assertInstanceOf(PDOStatement::class, $checkout); + $checkout->execute(['-b', $branch]); + } + } + + (new NormativeCatalog($pdo))->createTables(); + (new TransformationAuditMigrator($pdo))->migrate(); + CatalogOwnership::markCurrentVersion($pdo); + $datasetId = is_string($before['dataset_id'] ?? null) + ? $before['dataset_id'] + : '11111111-1111-4111-8111-111111111111'; + $schema = is_string($before['physical_table_schema'] ?? null) ? $before['physical_table_schema'] : null; + $table = is_string($before['physical_table_name'] ?? null) ? $before['physical_table_name'] : 'data_survey'; + $qualified = $this->qualifiedBackendTable($connection, $schema, $table); + $pdo->exec( + 'CREATE TABLE ' . $qualified . ' (' + . $connection->profile->quoteIdentifier('__case_ordinal') . ' BIGINT NOT NULL PRIMARY KEY, ' + . $connection->profile->quoteIdentifier('score') . ' DOUBLE NULL)', + ); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, ' + . 'dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([$datasetId, '1.0', 'fixture', $schema, $table, 'Official in-place 0.1', 4, '2026-08-17 00:00:00']); + $pdo->prepare( + 'INSERT INTO variable (variable_id, dataset_id, source_ordinal, source_name, physical_name, storage_kind) ' + . 'VALUES (?, ?, ?, ?, ?, ?)', + )->execute(['22222222-2222-4222-8222-222222222222', $datasetId, 1, 'score', 'score', 'numeric']); + $insert = $pdo->prepare( + 'INSERT INTO ' . $qualified . ' (' + . $connection->profile->quoteIdentifier('__case_ordinal') . ', ' + . $connection->profile->quoteIdentifier('score') . ') VALUES (?, ?)', + ); + foreach ([[1, 1.0], [2, 2.0], [3, 3.0], [4, 4.0]] as $row) { + $insert->execute($row); + } + + if ($connection->profileName === 'dolt') { + $commit = $pdo->prepare('CALL DOLT_COMMIT(?, ?)'); + self::assertInstanceOf(PDOStatement::class, $commit); + $commit->execute(['-Am', 'Official in-place 0.1 fixture']); + self::assertSame([], $this->rows($pdo, 'SELECT table_name FROM dolt_status')); + } + + return ['dataset_id' => $datasetId, 'schema' => $schema, 'table' => $table]; + } + + /** @return array{branch: string, head: string}|null */ + private function backendContext(PDO $pdo, Connection $connection): ?array + { + if ($connection->profileName !== 'dolt') { + return null; + } + $row = $this->rows( + $pdo, + "SELECT active_branch() AS branch_name, dolt_hashof('HEAD') AS head_hash", + )[0] ?? null; + self::assertIsArray($row); + + return ['branch' => (string) $row['branch_name'], 'head' => (string) $row['head_hash']]; + } + + /** + * @param array{dataset_id: string, schema: string|null, table: string} $fixture + * @return array + */ + private function officialBackendSnapshot(PDO $pdo, Connection $connection, array $fixture): array + { + $qualified = $this->qualifiedBackendTable($connection, $fixture['schema'], $fixture['table']); + $score = array_map( + static fn(mixed $value): ?float => $value === null ? null : (float) $value, + $this->column($pdo, 'SELECT score FROM ' . $qualified . ' ORDER BY __case_ordinal'), + ); + $labels = array_map( + static function (array $row): array { + $numericCode = (float) $row['numeric_code']; + + return [ + floor($numericCode) === $numericCode ? (int) $numericCode : $numericCode, + (string) $row['label'], + ]; + }, + $this->rows( + $pdo, + 'SELECT label.numeric_code, label.label FROM variable ' + . 'JOIN variable_value_label_set link ON link.variable_id = variable.variable_id ' + . 'JOIN value_label label ON label.value_label_set_id = link.value_label_set_id ' + . 'WHERE variable.dataset_id = ? AND variable.source_name = ? ORDER BY label.ordinal', + [$fixture['dataset_id'], 'score'], + ), + ); + + return [ + 'dataset_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset'), + 'persistent_data_table_count' => (int) $this->scalar($pdo, 'SELECT COUNT(DISTINCT physical_table_name) FROM dataset'), + 'dataset_identity' => $this->rows( + $pdo, + 'SELECT dataset_id, physical_table_schema, physical_table_name FROM dataset WHERE dataset_id = ?', + [$fixture['dataset_id']], + ), + 'tables' => array_map('strval', $this->column( + $pdo, + 'SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() ORDER BY table_name', + )), + 'case_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM ' . $qualified), + 'case_ordinals' => array_map('intval', $this->column( + $pdo, + 'SELECT __case_ordinal FROM ' . $qualified . ' ORDER BY __case_ordinal', + )), + 'score' => $score, + 'variables' => $this->rows( + $pdo, + 'SELECT * FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', + [$fixture['dataset_id']], + ), + 'variable_label' => $this->scalar( + $pdo, + 'SELECT variable_label FROM variable WHERE dataset_id = ? AND source_name = ?', + [$fixture['dataset_id'], 'score'], + ), + 'value_labels' => $labels, + 'audit' => $this->rows( + $pdo, + 'SELECT * FROM transformation_apply WHERE dataset_id = ? ORDER BY apply_id', + [$fixture['dataset_id']], + ), + 'repository' => $this->backendRepositoryEvidence($pdo, $connection), + ]; + } + + /** @return array{branch: string, head: string, history: list, status: list}|null */ + private function backendRepositoryEvidence(PDO $pdo, Connection $connection): ?array + { + if ($connection->profileName !== 'dolt') { + return null; + } + $identity = $this->rows( + $pdo, + "SELECT active_branch() AS branch_name, dolt_hashof('HEAD') AS head_hash", + )[0] ?? []; + $status = array_map(static function (array $row): array { + $row = array_change_key_case($row, CASE_LOWER); + + return [ + 'table_name' => (string) ($row['table_name'] ?? ''), + 'status' => (string) ($row['status'] ?? ''), + 'staged' => (bool) ($row['staged'] ?? false), + ]; + }, $this->rows($pdo, 'SELECT table_name, status, staged FROM dolt_status ORDER BY table_name')); + + return [ + 'branch' => (string) ($identity['branch_name'] ?? ''), + 'head' => (string) ($identity['head_hash'] ?? ''), + 'history' => array_map('strval', $this->column( + $pdo, + 'SELECT commit_hash FROM dolt_log ORDER BY commit_order DESC, commit_hash', + )), + 'status' => $status, + ]; + } + + /** @param list $tables */ + private function assertNoForbiddenArtifactTables(array $tables): void + { + self::assertSame([], array_values(array_filter( + $tables, + static fn(string $table): bool => preg_match('/derived|output|staging|snapshot|rollback|recovery/i', $table) === 1, + ))); + } + + private function qualifiedBackendTable(Connection $connection, ?string $schema, string $table): string + { + return ($schema === null ? '' : $connection->profile->quoteIdentifier($schema) . '.') + . $connection->profile->quoteIdentifier($table); + } + + private function quoteDatabase(string $database): string + { + if (strlen($database) > 64 || preg_match('/\A[a-z][a-z0-9_]*\z/D', $database) !== 1) { + throw new RuntimeException('Unsafe official backend database name.'); + } + + return '`' . $database . '`'; + } + + private function dsnForDatabase(string $dsn, string $database): string + { + $this->quoteDatabase($database); + if (!str_starts_with(strtolower($dsn), 'mysql:')) { + throw new RuntimeException('Official MySQL-family tests require a mysql PDO DSN.'); + } + $parts = explode(';', substr($dsn, strlen('mysql:'))); + $databaseIndex = null; + foreach ($parts as $index => $part) { + if (strtolower(trim((string) explode('=', $part, 2)[0])) !== 'dbname') { + continue; + } + if ($databaseIndex !== null) { + throw new RuntimeException('Official backend DSN contains duplicate dbname settings.'); + } + $databaseIndex = $index; + } + if ($databaseIndex === null) { + if (end($parts) === '') { + array_pop($parts); + } + $parts[] = 'dbname=' . $database; + } else { + $parts[$databaseIndex] = 'dbname=' . $database; + } + + return 'mysql:' . implode(';', $parts); } private function planCase(string $id): TransformationPlan @@ -276,24 +634,6 @@ private function bindingCase(string $id): array throw new \RuntimeException('Missing in-place 0.1 case: ' . $id); } - /** @return array */ - private function mysqlFailureSnapshot(PDO $pdo, Connection $connection, string $table): array - { - return [ - 'dataset_count' => (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM dataset'), - 'persistent_data_table_count' => (int) $this->scalar($pdo, 'SELECT COUNT(DISTINCT physical_table_name) FROM dataset'), - 'dataset' => $this->rows($pdo, 'SELECT * FROM dataset WHERE dataset_id = ?', [self::DATASET_ID]), - 'variables' => $this->rows($pdo, 'SELECT * FROM variable WHERE dataset_id = ? ORDER BY source_ordinal', [self::DATASET_ID]), - 'audit' => $this->rows($pdo, 'SELECT * FROM transformation_apply WHERE dataset_id = ? ORDER BY apply_id', [self::DATASET_ID]), - 'tables' => $this->column($pdo, 'SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() ORDER BY table_name'), - 'rows' => $this->rows( - $pdo, - 'SELECT * FROM ' . $connection->profile->quoteIdentifier($table) - . ' ORDER BY ' . $connection->profile->quoteIdentifier('__case_ordinal'), - ), - ]; - } - /** @return list */ private function tables(PDO $pdo): array { From 25f4a8304d26e2bca8d002a2382d283cb95e22cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 21:27:44 +0300 Subject: [PATCH 20/53] test: execute every official 0.2 backend case --- .../task-9-report.md | 10 +- .../OfficialInPlaceTransformation02Test.php | 774 +++++++++++++----- 2 files changed, 578 insertions(+), 206 deletions(-) diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md index 8754c04..f099ded 100644 --- a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-9-report.md @@ -17,6 +17,8 @@ - Review RED: 18 live MySQL/MariaDB storage-engine cases produced 15 acceptance failures; three malformed-relation cases were already rejected by namespace ownership. The accepted cases included a MyISAM bound wide table reaching an injected audit failure, all six apply-participating catalog/audit tables using MyISAM, and missing wide-table metadata on MySQL. - Review GREEN: all 18 engine cases passed with 228 assertions. Exact before/after snapshots prove no data, catalog, audit, dataset-count, or physical-table-count change and no open transaction. - Manifest-gate RED: the prior backend gate covered only one of the six official 0.1 case IDs; a clean inventory test failed on the other five IDs. GREEN runs every manifest row through an exhaustive ID dispatcher backed by the live MySQL or Dolt executor and passed 6 tests with 334 assertions. +- Review-round-2 RED: the official 0.2 backend provider exposed all eleven manifest rows, but the old dispatcher recognized only the three create-target cases: 11 tests produced eight failures and three passes. GREEN dispatches every ID exhaustively and passed all 11 tests with 788 assertions and no configured-service skip. +- The four official Dolt existing-target successes now execute on Dolt, including sequential predicates, OR/NULL semantics, variable-missing propagation, and conditional-variable-missing propagation. They assert exact rows and metadata plus stable dataset/table identity and counts, one exact audit row, an unchanged branch/HEAD/commit count, the expected working-set diff, and no copied or recovery artifacts. - GREEN: focused MySQL 8.4.9, MariaDB 11.4.8, and Dolt 2.2.2 execution passed 54 tests with 737 assertions and six PostgreSQL-only skips. - Official 0.1 MySQL and official 0.2 MySQL/MariaDB/Dolt create cases snapshot rows, catalog, audit, dataset count, persistent table count, and repository evidence; all reject before mutation with exact equality. - Dolt tests cover empty actor, missing context without an evidence read, initial branch/HEAD mismatch, dirty state, and injected post-mutation context change with exact error codes. @@ -25,14 +27,14 @@ ## Live evidence - MySQL: 8.4.9; MariaDB: 11.4.8-MariaDB-ubu2404; Dolt: 2.2.2 (MySQL wire compatibility 8.0.31). -- The final focused live gate ran all six official 0.1 cases and all eleven official 0.2 cases, plus engine and Dolt guard regressions: 74 tests, 1,266 assertions, and six PostgreSQL-only skips. No configured MySQL, MariaDB, or Dolt case skipped. +- The final focused live gate ran all six official 0.1 cases and all eleven official 0.2 cases, plus engine and Dolt guard regressions: 78 tests, 1,814 assertions, and six PostgreSQL-only skips. No configured MySQL, MariaDB, or Dolt case skipped. - Dolt Task 9 cases use randomly named isolated databases with explicit namespace prechecks and teardown. Successful existing-target apply preserves branch and HEAD, produces one dirty working-set diff for the in-place edit, and adds no Dolt commit; failure leaves no edit or audit. -- The authoritative temporary Dolt repository remained on `main` at HEAD `dvba843qo8iqg962au54meed7degbeok` with exactly one setup commit after the full run. Legacy round-trip tests leave unrelated untracked fixture tables in the shared base database; Task 9 assertions run in isolated databases and clean them up. +- The freshly recreated temporary Dolt repository remained on `main` at HEAD `h6ggikagos52ks3uo4aetolkun79v7l2` with exactly one setup commit after the normalized full run. Legacy round-trip tests leave unrelated untracked fixture tables in the shared base database; Task 9 assertions run in isolated databases and clean them up, and the final isolated-database count was zero on all three configured services. ## Verification -- Full live `composer test`: 475 tests, 5,565 assertions, nine PostgreSQL skips; passed with MySQL 8.4.9, MariaDB 11.4.8, and Dolt 2.2.2 configured. -- Full LF-normalized staged-tree `composer check`: Composer validation, PHP lint, PHP CS Fixer over 226 files, PHPStan, and PHPUnit passed; PHPUnit ran 475 tests with 5,565 assertions and nine PostgreSQL skips. +- Full live `composer test`: 479 tests, 6,115 assertions, nine PostgreSQL skips; passed with MySQL 8.4.9, MariaDB 11.4.8, and Dolt 2.2.2 configured. +- Full LF-normalized staged-tree `composer check`: Composer validation, PHP lint, PHP CS Fixer over 226 files, PHPStan, and PHPUnit passed; PHPUnit ran 479 tests with 6,115 assertions and nine PostgreSQL skips. - The first normalized run reached Composer's default 300-second process timeout during a legacy Dolt dictionary test. Its interrupted fixture was confined to the temporary Dolt volume; that named container/volume was recreated from a clean one-commit repository, and the authoritative full check passed with `COMPOSER_PROCESS_TIMEOUT=900`. - `composer analyse`: passed with no errors. `composer lint`: passed. Task-changed-file PHP CS Fixer dry-run: zero fixable files. `git diff --cached --check`: passed. - Working-tree `composer style` still exits 8 because 122 unchanged repository PHP files retain the pre-existing CRLF baseline; the normalized full gate checked all 226 PHP files cleanly. diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index bddc5dd..c65a4e0 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -10,6 +10,8 @@ use OpenStatSpec\Tests\Support\SpecificationManifest; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; +use OpenStatSpec\Transformation\Execution\DoltEvidence; +use OpenStatSpec\Transformation\Execution\DoltEvidenceReader; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; use OpenStatSpec\Transformation\Plan\PlanCodec; @@ -40,24 +42,155 @@ protected function tearDown(): void } } - /** @return iterable */ - public static function nonAtomicCreateCases(): iterable + /** @return iterable}> */ + public static function officialBackendCases(): iterable { - yield 'MySQL' => ['mysql', 'mysql-create-target-fails-before-mutation']; - yield 'MariaDB' => ['mariadb', 'mariadb-create-target-fails-before-mutation']; - yield 'Dolt' => ['dolt', 'dolt-create-target-fails-before-mutation']; + foreach (SpecificationManifest::load('conformance/in-place-transformation-0.2.json')['cases'] as $case) { + if (is_array($case) && is_string($case['id'] ?? null)) { + yield $case['id'] => [$case]; + } + } } - #[DataProvider('nonAtomicCreateCases')] - public function testOfficialMySqlFamilyCreateTargetCasesFailBeforeMutation( - string $expectedProfile, - string $caseId, - ): void { - $case = $this->bindingCase($caseId); - $pdo = $this->mysqlFamily($expectedProfile); + /** @param array $case */ + #[DataProvider('officialBackendCases')] + public function testEveryOfficialBackendManifestCaseRuns(array $case): void + { + $id = $case['id'] ?? null; + self::assertIsString($id); + + match ($id) { + 'dolt-preprovisioned-target-sequential-null-semantics', + 'dolt-preprovisioned-target-or-null-semantics', + 'dolt-preprovisioned-target-variable-missing-propagation', + 'dolt-preprovisioned-target-conditional-variable-missing-propagation' => $this->assertOfficialDoltSuccess($case), + 'dolt-create-target-fails-before-mutation', + 'mysql-create-target-fails-before-mutation', + 'mariadb-create-target-fails-before-mutation' => $this->assertOfficialCreateTargetRejection($case), + 'sqlite-inequality-boundary-semantics' => $this->assertOfficialSqliteInequality($case), + 'dolt-context-changed-after-mutation-rolls-back' => $this->assertOfficialDoltContextChanged($case), + 'dolt-empty-actor-fails-before-mutation' => $this->assertOfficialDoltEmptyActor($case), + 'sqlite-create-target-atomic-success' => $this->assertOfficialSqliteCreate($case), + default => throw new RuntimeException('Unhandled official in-place 0.2 case: ' . $id), + }; + } + + public function testSqliteCreateTargetFailureRollsBackSchemaRowsCatalogAndAudit(): void + { + $case = $this->bindingCase('sqlite-create-target-atomic-success'); + $pdo = $this->sqlite(); + $connection = new Connection($pdo); + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + 'data_atomic_create', + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $pdo->exec("ALTER TABLE transformation_apply ADD COLUMN task8_actor_guard INTEGER NOT NULL DEFAULT 0 CHECK (actor <> 'conformance-runner')"); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + + try { + (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::CREATE_DATASET_ID)); + self::fail('The injected audit failure did not abort the apply.'); + } catch (\PDOException $exception) { + self::assertStringContainsString('constraint', strtolower($exception->getMessage())); + } + + $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); + self::assertSame($before, $after); + self::assertNotContains('target', $this->columns($pdo, $connection, 'data_atomic_create')); + self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM variable WHERE dataset_id = ? AND source_name = ?', [self::CREATE_DATASET_ID, 'target'])); + self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM transformation_apply', [])); + self::assertFalse($pdo->inTransaction()); + } + + public function testPostgresqlCreateTargetIsAtomicWhenConfigured(): void + { + $pdo = $this->postgresql(); + $connection = new Connection($pdo); + self::assertSame('postgresql', $connection->profileName); + $table = 'data_atomic_create_task8'; + $case = $this->bindingCase('sqlite-create-target-atomic-success'); + $this->prepareCatalog($pdo); + $this->assertNamespaceClean($pdo, self::CREATE_DATASET_ID, $table); + + try { + $this->installNumericFixture( + $pdo, + $connection, + self::CREATE_DATASET_ID, + $table, + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); + (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::CREATE_DATASET_ID)); + $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($case['after']['rows'], $this->numericRows($pdo, $connection, $table, ['source_a', 'source_b', 'target'])); + $this->assertAudit($pdo, [...$case['expected_audit'], 'database_profile' => 'postgresql', 'physical_table_name' => $table]); + } finally { + $this->purgeFixture($pdo, $connection, self::CREATE_DATASET_ID, $table); + } + } + + public function testPostgresqlCreateTargetFailureRollsBackWhenConfigured(): void + { + $pdo = $this->postgresql(); + $connection = new Connection($pdo); + $table = 'data_atomic_rollback_task8'; + $case = $this->bindingCase('sqlite-create-target-atomic-success'); + $this->prepareCatalog($pdo); + $this->assertNamespaceClean($pdo, self::ROLLBACK_DATASET_ID, $table); + $auditGuardInstalled = false; + + try { + $this->installNumericFixture( + $pdo, + $connection, + self::ROLLBACK_DATASET_ID, + $table, + ['source_a', 'source_b'], + [[1, 2.0, 11.0], [2, null, 22.0]], + ); + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT IF EXISTS task8_fail_audit'); + $pdo->exec("ALTER TABLE transformation_apply ADD CONSTRAINT task8_fail_audit CHECK (actor <> 'conformance-runner')"); + $auditGuardInstalled = true; + $before = $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + + try { + (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::ROLLBACK_DATASET_ID)); + self::fail('The injected PostgreSQL audit failure did not abort the apply.'); + } catch (\PDOException $exception) { + self::assertStringContainsString('task8_fail_audit', $exception->getMessage()); + } + + self::assertSame($before, $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table)); + self::assertNotContains('target', $this->columns($pdo, $connection, $table)); + self::assertFalse($pdo->inTransaction()); + } finally { + if ($auditGuardInstalled) { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT IF EXISTS task8_fail_audit'); + } + $this->purgeFixture($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + } + } + + /** @param array $case */ + private function assertOfficialCreateTargetRejection(array $case): void + { + $profile = $case['database_profile'] ?? null; + self::assertContains($profile, ['mysql', 'mariadb', 'dolt']); + $pdo = $this->mysqlFamily((string) $profile); $connection = new Connection($pdo); - self::assertSame($expectedProfile, $connection->profileName); - $table = 'data_task9_create_' . $expectedProfile; + self::assertSame($profile, $connection->profileName); + $table = 'data_task9_create_' . $profile; + $this->checkoutDoltFixtureBranch($pdo, $connection, $case); $this->prepareCatalog($pdo); $this->assertNamespaceClean($pdo, self::CREATE_DATASET_ID, $table); @@ -94,9 +227,205 @@ public function testOfficialMySqlFamilyCreateTargetCasesFailBeforeMutation( } } - public function testOfficialDoltEmptyActorCaseFailsBeforeMutation(): void + /** @param array $case */ + private function assertOfficialDoltSuccess(array $case): void + { + $pdo = $this->mysqlFamily('dolt'); + $connection = new Connection($pdo); + self::assertSame('dolt', $connection->profileName); + $this->checkoutDoltFixtureBranch($pdo, $connection, $case); + $beforeContract = is_array($case['before'] ?? null) ? $case['before'] : []; + $afterContract = is_array($case['after'] ?? null) ? $case['after'] : []; + $beforeRows = $beforeContract['rows'] ?? $case['before_rows'] ?? null; + $afterRows = $afterContract['rows'] ?? $case['after_rows'] ?? null; + self::assertIsArray($beforeRows); + self::assertIsArray($afterRows); + $datasetId = $beforeContract['dataset_id'] ?? $case['dataset_id'] ?? null; + $table = $beforeContract['physical_table_name'] ?? $case['physical_table_name'] ?? null; + self::assertIsString($datasetId); + self::assertIsString($table); + [$variables, $fixtureRows] = $this->numericFixtureRows($beforeRows); + $targetIdentity = is_array($beforeContract['target_identity'] ?? null) + ? $beforeContract['target_identity'] + : []; + $variableIds = is_string($targetIdentity['variable_id'] ?? null) + ? ['target' => $targetIdentity['variable_id']] + : []; + + $this->installNumericFixture($pdo, $connection, $datasetId, $table, $variables, $fixtureRows, $variableIds); + $this->installOfficialMissingRules($pdo, $datasetId, $beforeContract); + $context = $this->commitAndReadDoltFixtureContext($pdo, $connection); + self::assertNotNull($context); + $before = $this->snapshot($pdo, $connection, $datasetId, $table); + $targetBefore = $this->targetIdentity($pdo, $datasetId, 'target'); + $metadataBefore = $this->targetMetadata($pdo, $datasetId, 'target'); + $labelsBefore = $this->valueLabels($pdo, $datasetId, 'target'); + $missingBefore = $this->missingRules($pdo, $datasetId, 'target'); + self::assertSame([], $before['repository']['status'] ?? null); + + $result = (new InPlaceTransformationExecutor($connection))->execute( + $this->request($case, $datasetId, $context['branch'], $context['head']), + ); + $after = $this->snapshot($pdo, $connection, $datasetId, $table); + + self::assertSame($datasetId, $result->datasetId()); + self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); + self::assertSame((int) $case['expected_audit']['operation_count'], $result->operationCount()); + self::assertSame($before['dataset_count'], $after['dataset_count']); + self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); + self::assertSame($before['dataset_identity'], $after['dataset_identity']); + self::assertSame($before['tables'], $after['tables']); + self::assertSame($before['columns'], $after['columns']); + self::assertSame($before['variable_count'], $after['variable_count']); + self::assertSame($before['case_ordinals'], $after['case_ordinals']); + self::assertSame($before['case_count'], $after['case_count']); + self::assertSame($afterRows, $this->numericRows($pdo, $connection, $table, $variables)); + self::assertSame($targetBefore, $this->targetIdentity($pdo, $datasetId, 'target')); + $this->assertDoltRepositoryApplyEvidence($before, $after, $table, (string) $case['id']); + $this->assertNoForbiddenArtifactTables($after['tables']); + + if (($case['id'] ?? null) === 'dolt-preprovisioned-target-sequential-null-semantics') { + self::assertSame( + [ + 'variable_id' => $afterContract['target_identity']['variable_id'], + 'source_ordinal' => $afterContract['target_identity']['ordinal'], + 'source_name' => 'target', + 'physical_name' => 'target', + ], + $targetBefore, + ); + self::assertSame( + [ + 'source_ordinal' => 3, + 'storage_kind' => 'numeric', + 'declared_string_width' => null, + 'variable_label' => 'Synthetic conjunction', + 'print_format_family' => 'F', + 'print_format_width' => 1, + 'print_format_decimals' => 0, + 'write_format_family' => 'F', + 'write_format_width' => 1, + 'write_format_decimals' => 0, + 'measurement_level' => 'nominal', + 'variable_role' => null, + 'display_width' => null, + 'display_alignment' => null, + ], + $this->targetMetadata($pdo, $datasetId, 'target'), + ); + self::assertSame([[0.0, 'No'], [1.0, 'Yes']], $this->valueLabels($pdo, $datasetId, 'target')); + self::assertSame($missingBefore, $this->missingRules($pdo, $datasetId, 'target')); + } else { + self::assertSame($metadataBefore, $this->targetMetadata($pdo, $datasetId, 'target')); + self::assertSame($labelsBefore, $this->valueLabels($pdo, $datasetId, 'target')); + self::assertSame($missingBefore, $this->missingRules($pdo, $datasetId, 'target')); + } + + $audit = $this->assertAudit($pdo, [ + ...$case['expected_audit'], + 'dolt_branch' => $context['branch'], + 'dolt_head_before' => $context['head'], + 'dolt_head_after' => $context['head'], + ]); + foreach ($case['required_audit_fields'] ?? [] as $field) { + self::assertArrayHasKey($field, $audit); + } + } + + /** @param array $case */ + private function assertOfficialDoltContextChanged(array $case): void { - $case = $this->bindingCase('dolt-empty-actor-fails-before-mutation'); + $pdo = $this->mysqlFamily('dolt'); + $connection = new Connection($pdo); + $this->checkoutDoltFixtureBranch($pdo, $connection, $case); + $table = 'data_task9_context_change'; + $this->installNumericFixture( + $pdo, + $connection, + self::ROLLBACK_DATASET_ID, + $table, + ['source_a', 'source_b', 'target'], + [[1, 1.0, 1.0, 0.0]], + ); + $pdo->prepare('UPDATE variable SET variable_label = ? WHERE dataset_id = ? AND source_name = ?') + ->execute(['Before', self::ROLLBACK_DATASET_ID, 'target']); + $context = $this->commitAndReadDoltFixtureContext($pdo, $connection); + self::assertNotNull($context); + $before = $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + self::assertSame($case['before']['rows'], $this->numericRows($pdo, $connection, $table, ['target'])); + self::assertSame($case['before']['target_metadata']['variable_label'], $this->scalar( + $pdo, + 'SELECT variable_label FROM variable WHERE dataset_id = ? AND source_name = ?', + [self::ROLLBACK_DATASET_ID, 'target'], + )); + $reader = new class ($pdo, $connection, $table, $context['branch'], $context['head']) implements DoltEvidenceReader { + public int $reads = 0; + public bool $mutationObserved = false; + + public function __construct( + private readonly PDO $pdo, + private readonly Connection $connection, + private readonly string $table, + private readonly string $branch, + private readonly string $head, + ) {} + + public function read(): DoltEvidence + { + if (++$this->reads === 1) { + return new DoltEvidence($this->branch, $this->head, []); + } + $statement = $this->pdo->query( + 'SELECT ' . $this->connection->profile->quoteIdentifier('target') + . ' FROM ' . $this->connection->profile->quoteIdentifier($this->table) + . ' WHERE ' . $this->connection->profile->quoteIdentifier('__case_ordinal') . ' = 1', + ); + $this->mutationObserved = $statement instanceof PDOStatement + && (float) $statement->fetchColumn() === 1.0; + + return new DoltEvidence($this->branch, 'concurrent-commit', []); + } + }; + + try { + (new InPlaceTransformationExecutor($connection, $reader))->execute(new InPlaceApplyRequest( + $this->planCase('sequential-conditional-binary-existing-target'), + 'parent', + self::ROLLBACK_DATASET_ID, + str_repeat('a', 64), + (string) $case['actor'], + $context['branch'], + $context['head'], + )); + self::fail('The official post-mutation Dolt context change was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame($case['expected_error'], $failure->diagnosticCode()); + } + + self::assertTrue($case['mutation_started']); + self::assertTrue($reader->mutationObserved); + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table)); + self::assertSame($case['after_failure']['rows'], $this->numericRows($pdo, $connection, $table, ['target'])); + self::assertSame($case['after_failure']['target_metadata']['variable_label'], $this->scalar( + $pdo, + 'SELECT variable_label FROM variable WHERE dataset_id = ? AND source_name = ?', + [self::ROLLBACK_DATASET_ID, 'target'], + )); + self::assertSame($case['after_failure']['audit_row_count'], $this->scalarCount( + $pdo, + 'SELECT COUNT(*) FROM transformation_apply', + [], + )); + } + + /** @param array $case */ + private function assertOfficialDoltEmptyActor(array $case): void + { + $pdo = $this->mysqlFamily('dolt'); + $connection = new Connection($pdo); + self::assertSame('dolt', $connection->profileName); + $before = ['tables' => $this->tables($pdo), 'repository' => $this->doltRepositoryEvidence($pdo, $connection)]; try { new InPlaceApplyRequest( @@ -112,12 +441,18 @@ public function testOfficialDoltEmptyActorCaseFailsBeforeMutation(): void } catch (TransformationFailure $failure) { self::assertSame($case['expected_error'], $failure->diagnosticCode()); } + self::assertFalse($case['mutation_started']); + self::assertFalse($pdo->inTransaction()); + self::assertSame($before, [ + 'tables' => $this->tables($pdo), + 'repository' => $this->doltRepositoryEvidence($pdo, $connection), + ]); } - public function testSqliteCreateTargetIsAtomicAndPreservesIdentity(): void + /** @param array $case */ + private function assertOfficialSqliteCreate(array $case): void { - $case = $this->bindingCase('sqlite-create-target-atomic-success'); $pdo = $this->sqlite(); $connection = new Connection($pdo); $this->installNumericFixture( @@ -129,15 +464,15 @@ public function testSqliteCreateTargetIsAtomicAndPreservesIdentity(): void [[1, 2.0, 11.0], [2, null, 22.0]], ); $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); - $request = $this->request($case, self::CREATE_DATASET_ID); - - $result = (new InPlaceTransformationExecutor($connection))->execute($request); + $result = (new InPlaceTransformationExecutor($connection))->execute( + $this->request($case, self::CREATE_DATASET_ID), + ); + $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); self::assertSame(self::CREATE_DATASET_ID, $result->datasetId()); self::assertSame(1, $result->operationCount()); self::assertNotSame('', $result->auditOperationId()); - $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); self::assertSame($before['dataset_count'], $after['dataset_count']); self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); self::assertSame($before['dataset_identity'], $after['dataset_identity']); @@ -173,40 +508,9 @@ public function testSqliteCreateTargetIsAtomicAndPreservesIdentity(): void $this->assertAudit($pdo, $case['expected_audit']); } - public function testSqliteCreateTargetFailureRollsBackSchemaRowsCatalogAndAudit(): void - { - $case = $this->bindingCase('sqlite-create-target-atomic-success'); - $pdo = $this->sqlite(); - $connection = new Connection($pdo); - $this->installNumericFixture( - $pdo, - $connection, - self::CREATE_DATASET_ID, - 'data_atomic_create', - ['source_a', 'source_b'], - [[1, 2.0, 11.0], [2, null, 22.0]], - ); - $pdo->exec("ALTER TABLE transformation_apply ADD COLUMN task8_actor_guard INTEGER NOT NULL DEFAULT 0 CHECK (actor <> 'conformance-runner')"); - $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); - - try { - (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::CREATE_DATASET_ID)); - self::fail('The injected audit failure did not abort the apply.'); - } catch (\PDOException $exception) { - self::assertStringContainsString('constraint', strtolower($exception->getMessage())); - } - - $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, 'data_atomic_create'); - self::assertSame($before, $after); - self::assertNotContains('target', $this->columns($pdo, $connection, 'data_atomic_create')); - self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM variable WHERE dataset_id = ? AND source_name = ?', [self::CREATE_DATASET_ID, 'target'])); - self::assertSame(0, $this->scalarCount($pdo, 'SELECT COUNT(*) FROM transformation_apply', [])); - self::assertFalse($pdo->inTransaction()); - } - - public function testSqliteInequalityBoundariesUseSqlThreeValuedTruth(): void + /** @param array $case */ + private function assertOfficialSqliteInequality(array $case): void { - $case = $this->bindingCase('sqlite-inequality-boundary-semantics'); $pdo = $this->sqlite(); $connection = new Connection($pdo); $this->installNumericFixture( @@ -218,8 +522,10 @@ public function testSqliteInequalityBoundariesUseSqlThreeValuedTruth(): void [[1, 0.0, 0.0, 0.0, 0.0, 0.0], [2, 1.0, 0.0, 0.0, 0.0, 0.0], [3, 2.0, 0.0, 0.0, 0.0, 0.0]], ); $before = $this->snapshot($pdo, $connection, self::INEQUALITY_DATASET_ID, 'data_inequality'); - - $result = (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::INEQUALITY_DATASET_ID)); + $result = (new InPlaceTransformationExecutor($connection))->execute( + $this->request($case, self::INEQUALITY_DATASET_ID), + ); + $after = $this->snapshot($pdo, $connection, self::INEQUALITY_DATASET_ID, 'data_inequality'); self::assertSame($case['expected_audit']['plan_hash'], $result->planHash()); self::assertSame($case['after_rows'], $this->numericRows( @@ -228,7 +534,6 @@ public function testSqliteInequalityBoundariesUseSqlThreeValuedTruth(): void 'data_inequality', ['source', 'target_lt', 'target_le', 'target_gt', 'target_ge'], )); - $after = $this->snapshot($pdo, $connection, self::INEQUALITY_DATASET_ID, 'data_inequality'); self::assertSame($before['dataset_count'], $after['dataset_count']); self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); self::assertSame($before['dataset_identity'], $after['dataset_identity']); @@ -239,141 +544,6 @@ public function testSqliteInequalityBoundariesUseSqlThreeValuedTruth(): void $this->assertAudit($pdo, $case['expected_audit']); } - public function testSqliteOperationsObservePriorValuesAndChangeOnlyNamedMetadata(): void - { - $case = $this->bindingCase('dolt-preprovisioned-target-sequential-null-semantics'); - $pdo = $this->sqlite(); - $connection = new Connection($pdo); - $this->installNumericFixture( - $pdo, - $connection, - '11111111-1111-4111-8111-111111111111', - 'data_synthetic', - ['source_a', 'source_b', 'target'], - [[1, 1.0, 1.0, null], [2, 1.0, null, null], [3, null, 1.0, null], [4, 0.0, 1.0, null]], - ); - $targetId = (string) $this->scalar( - $pdo, - 'SELECT variable_id FROM variable WHERE dataset_id = ? AND source_name = ?', - ['11111111-1111-4111-8111-111111111111', 'target'], - ); - $pdo->prepare( - 'INSERT INTO missing_rule (missing_rule_id, variable_id, ordinal, rule_kind, code_kind, numeric_value) VALUES (?, ?, ?, ?, ?, ?)', - )->execute(['55555555-5555-4555-8555-555555555555', $targetId, 1, 'discrete', 'numeric', -9.0]); - $request = new InPlaceApplyRequest( - plan: $this->planCase((string) $case['applied_plan_case']), - inputAlias: 'parent', - datasetId: '11111111-1111-4111-8111-111111111111', - sourceHash: (string) $case['expected_audit']['source_hash'], - actor: 'conformance-runner', - ); - - (new InPlaceTransformationExecutor($connection))->execute($request); - - self::assertSame($case['after']['rows'], $this->numericRows( - $pdo, - $connection, - 'data_synthetic', - ['source_a', 'source_b', 'target'], - )); - self::assertSame( - [ - 'source_ordinal' => 3, - 'storage_kind' => 'numeric', - 'declared_string_width' => null, - 'variable_label' => 'Synthetic conjunction', - 'print_format_family' => 'F', - 'print_format_width' => 1, - 'print_format_decimals' => 0, - 'write_format_family' => 'F', - 'write_format_width' => 1, - 'write_format_decimals' => 0, - 'measurement_level' => 'nominal', - 'variable_role' => null, - 'display_width' => null, - 'display_alignment' => null, - ], - $this->targetMetadata($pdo, '11111111-1111-4111-8111-111111111111', 'target'), - ); - self::assertSame([[0.0, 'No'], [1.0, 'Yes']], $this->valueLabels($pdo, '11111111-1111-4111-8111-111111111111', 'target')); - self::assertSame([['ordinal' => 1, 'rule_kind' => 'discrete', 'code_kind' => 'numeric', 'numeric_value' => -9.0]], $this->missingRules($pdo, '11111111-1111-4111-8111-111111111111', 'target')); - } - - public function testPostgresqlCreateTargetIsAtomicWhenConfigured(): void - { - $pdo = $this->postgresql(); - $connection = new Connection($pdo); - self::assertSame('postgresql', $connection->profileName); - $table = 'data_atomic_create_task8'; - $case = $this->bindingCase('sqlite-create-target-atomic-success'); - $this->prepareCatalog($pdo); - $this->assertNamespaceClean($pdo, self::CREATE_DATASET_ID, $table); - - try { - $this->installNumericFixture( - $pdo, - $connection, - self::CREATE_DATASET_ID, - $table, - ['source_a', 'source_b'], - [[1, 2.0, 11.0], [2, null, 22.0]], - ); - $before = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); - (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::CREATE_DATASET_ID)); - $after = $this->snapshot($pdo, $connection, self::CREATE_DATASET_ID, $table); - self::assertSame($before['dataset_count'], $after['dataset_count']); - self::assertSame($before['persistent_data_table_count'], $after['persistent_data_table_count']); - self::assertSame($before['dataset_identity'], $after['dataset_identity']); - self::assertSame($before['tables'], $after['tables']); - self::assertSame($case['after']['rows'], $this->numericRows($pdo, $connection, $table, ['source_a', 'source_b', 'target'])); - $this->assertAudit($pdo, [...$case['expected_audit'], 'database_profile' => 'postgresql', 'physical_table_name' => $table]); - } finally { - $this->purgeFixture($pdo, $connection, self::CREATE_DATASET_ID, $table); - } - } - - public function testPostgresqlCreateTargetFailureRollsBackWhenConfigured(): void - { - $pdo = $this->postgresql(); - $connection = new Connection($pdo); - $table = 'data_atomic_rollback_task8'; - $case = $this->bindingCase('sqlite-create-target-atomic-success'); - $this->prepareCatalog($pdo); - $this->assertNamespaceClean($pdo, self::ROLLBACK_DATASET_ID, $table); - $auditGuardInstalled = false; - - try { - $this->installNumericFixture( - $pdo, - $connection, - self::ROLLBACK_DATASET_ID, - $table, - ['source_a', 'source_b'], - [[1, 2.0, 11.0], [2, null, 22.0]], - ); - $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT IF EXISTS task8_fail_audit'); - $pdo->exec("ALTER TABLE transformation_apply ADD CONSTRAINT task8_fail_audit CHECK (actor <> 'conformance-runner')"); - $auditGuardInstalled = true; - $before = $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); - - try { - (new InPlaceTransformationExecutor($connection))->execute($this->request($case, self::ROLLBACK_DATASET_ID)); - self::fail('The injected PostgreSQL audit failure did not abort the apply.'); - } catch (\PDOException $exception) { - self::assertStringContainsString('task8_fail_audit', $exception->getMessage()); - } - - self::assertSame($before, $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table)); - self::assertNotContains('target', $this->columns($pdo, $connection, $table)); - self::assertFalse($pdo->inTransaction()); - } finally { - if ($auditGuardInstalled) { - $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT IF EXISTS task8_fail_audit'); - } - $this->purgeFixture($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); - } - } - private function sqlite(): PDO { if (!in_array('sqlite', PDO::getAvailableDrivers(), true)) { @@ -406,9 +576,6 @@ private function postgresql(): PDO private function mysqlFamily(string $profile): PDO { - if (!in_array('mysql', PDO::getAvailableDrivers(), true)) { - self::markTestSkipped('PDO MySQL is not available.'); - } $prefix = match ($profile) { 'mysql' => 'OPENSTATSPEC_MYSQL', 'mariadb' => 'OPENSTATSPEC_MARIADB', @@ -419,6 +586,9 @@ private function mysqlFamily(string $profile): PDO if (!is_string($dsn) || $dsn === '') { self::markTestSkipped($prefix . '_DSN is not configured.'); } + if (!in_array('mysql', PDO::getAvailableDrivers(), true)) { + throw new RuntimeException('Configured MySQL-family service requires the PDO MySQL driver.'); + } $user = getenv($prefix . '_USER'); $password = getenv($prefix . '_PASSWORD'); $options = [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]; @@ -435,7 +605,7 @@ private function mysqlFamily(string $profile): PDO $adminUser = getenv($prefix . '_ADMIN_USER'); $adminPassword = getenv($prefix . '_ADMIN_PASSWORD'); if (!is_string($adminUser) || $adminUser === '' || !is_string($adminPassword)) { - self::markTestSkipped('Explicit Dolt admin credentials are required for an isolated test database.'); + throw new RuntimeException('Configured Dolt service requires explicit admin credentials for isolation.'); } $database = sprintf('openstatspec_t9_%d_%s', getmypid(), bin2hex(random_bytes(6))); $admin = new PDO($dsn, $adminUser, $adminPassword, $options); @@ -460,8 +630,9 @@ private function mysqlFamily(string $profile): PDO } /** - * @param list $variables - * @param list> $rows + * @param list $variables + * @param list> $rows + * @param array $variableIds */ private function installNumericFixture( PDO $pdo, @@ -470,6 +641,7 @@ private function installNumericFixture( string $table, array $variables, array $rows, + array $variableIds = [], ): void { if ($connection->profileName === 'sqlite') { $pdo->exec('PRAGMA foreign_keys = ON'); @@ -491,7 +663,7 @@ private function installNumericFixture( ); foreach ($variables as $index => $variable) { $insertVariable->execute([ - $this->fixtureUuid($datasetId . ':' . $variable), + $variableIds[$variable] ?? $this->fixtureUuid($datasetId . ':' . $variable), $datasetId, $index + 1, $variable, @@ -570,6 +742,7 @@ private function snapshot(PDO $pdo, Connection $connection, string $datasetId, s 'case_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM ' . $connection->profile->quoteIdentifier($table), []), 'rows' => $this->rows($pdo, 'SELECT * FROM ' . $connection->profile->quoteIdentifier($table) . ' ORDER BY __case_ordinal', []), 'audit_count' => $this->scalarCount($pdo, 'SELECT COUNT(*) FROM transformation_apply', []), + 'repository' => $this->doltRepositoryEvidence($pdo, $connection), ]; } @@ -659,8 +832,11 @@ private function missingRules(PDO $pdo, string $datasetId, string $target): arra )); } - /** @param array $expected */ - private function assertAudit(PDO $pdo, array $expected): void + /** + * @param array $expected + * @return array + */ + private function assertAudit(PDO $pdo, array $expected): array { $rows = $this->rows($pdo, 'SELECT * FROM transformation_apply ORDER BY started_at, apply_id', []); self::assertCount(1, $rows); @@ -671,6 +847,8 @@ private function assertAudit(PDO $pdo, array $expected): void self::assertMatchesRegularExpression('/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/', (string) $rows[0]['apply_id']); self::assertNotSame('', (string) $rows[0]['started_at']); self::assertNotSame('', (string) $rows[0]['completed_at']); + + return $rows[0]; } private function assertNamespaceClean(PDO $pdo, string $datasetId, string $table): void @@ -714,6 +892,198 @@ private function commitAndReadDoltFixtureContext(PDO $pdo, Connection $connectio return ['branch' => (string) $row['branch_name'], 'head' => (string) $row['head_hash']]; } + /** @param array $case */ + private function checkoutDoltFixtureBranch(PDO $pdo, Connection $connection, array $case): void + { + if ($connection->profileName !== 'dolt') { + return; + } + $expectedContext = is_array($case['expected_context'] ?? null) ? $case['expected_context'] : []; + $branch = $case['expected_branch'] ?? $expectedContext['branch'] ?? null; + if (!is_string($branch) || $branch === '') { + throw new RuntimeException('Official Dolt case is missing its expected branch.'); + } + $active = (string) $this->scalar($pdo, 'SELECT active_branch()', []); + if ($active === $branch) { + return; + } + $statement = $pdo->prepare('CALL DOLT_CHECKOUT(?, ?)'); + self::assertInstanceOf(PDOStatement::class, $statement); + $statement->execute(['-b', $branch]); + self::assertSame($branch, $this->scalar($pdo, 'SELECT active_branch()', [])); + } + + /** + * @param array $rows + * @return array{list, list>} + */ + private function numericFixtureRows(array $rows): array + { + $first = $rows[0] ?? null; + if (!is_array($first) || !is_int($first['__case_ordinal'] ?? null)) { + throw new RuntimeException('Official numeric fixture rows are malformed.'); + } + $variables = []; + foreach (array_keys($first) as $column) { + if (is_string($column) && $column !== '__case_ordinal') { + $variables[] = $column; + } + } + if ($variables === []) { + throw new RuntimeException('Official numeric fixture has no variables.'); + } + $fixtureRows = []; + foreach ($rows as $row) { + if (!is_array($row) || !is_int($row['__case_ordinal'] ?? null)) { + throw new RuntimeException('Official numeric fixture row is malformed.'); + } + $fixtureRow = [$row['__case_ordinal']]; + foreach ($variables as $variable) { + if (!array_key_exists($variable, $row)) { + throw new RuntimeException('Official numeric fixture row is missing a variable.'); + } + $value = $row[$variable]; + if ($value !== null && !is_int($value) && !is_float($value)) { + throw new RuntimeException('Official numeric fixture value is not numeric or null.'); + } + $fixtureRow[] = $value; + } + $fixtureRows[] = $fixtureRow; + } + + return [$variables, $fixtureRows]; + } + + /** @param array $beforeContract */ + private function installOfficialMissingRules(PDO $pdo, string $datasetId, array $beforeContract): void + { + $targetMetadata = is_array($beforeContract['target_metadata'] ?? null) + ? $beforeContract['target_metadata'] + : []; + $missingValues = $targetMetadata['missing_values'] ?? []; + if (!is_array($missingValues) || $missingValues === []) { + return; + } + $targetId = $this->scalar( + $pdo, + 'SELECT variable_id FROM variable WHERE dataset_id = ? AND source_name = ?', + [$datasetId, 'target'], + ); + if (!is_string($targetId) || $targetId === '') { + throw new RuntimeException('Official target variable identity is unavailable.'); + } + $insert = $pdo->prepare( + 'INSERT INTO missing_rule (missing_rule_id, variable_id, ordinal, rule_kind, code_kind, numeric_value) ' + . 'VALUES (?, ?, ?, ?, ?, ?)', + ); + self::assertInstanceOf(PDOStatement::class, $insert); + foreach ($missingValues as $missing) { + if (!is_array($missing) || !is_int($missing['ordinal'] ?? null)) { + throw new RuntimeException('Official missing-value fixture is malformed.'); + } + $insert->execute([ + $this->fixtureUuid($datasetId . ':missing:' . $missing['ordinal']), + $targetId, + $missing['ordinal'], + $missing['rule_kind'] ?? null, + $missing['code_kind'] ?? null, + $missing['numeric_value'] ?? null, + ]); + } + } + + /** @return array{variable_id: string, source_ordinal: int, source_name: string, physical_name: string} */ + private function targetIdentity(PDO $pdo, string $datasetId, string $target): array + { + $rows = $this->rows( + $pdo, + 'SELECT variable_id, source_ordinal, source_name, physical_name FROM variable ' + . 'WHERE dataset_id = ? AND source_name = ?', + [$datasetId, $target], + ); + self::assertCount(1, $rows); + + return [ + 'variable_id' => (string) $rows[0]['variable_id'], + 'source_ordinal' => (int) $rows[0]['source_ordinal'], + 'source_name' => (string) $rows[0]['source_name'], + 'physical_name' => (string) $rows[0]['physical_name'], + ]; + } + + /** @return array{branch: string, head: string, history: list, status: list}|null */ + private function doltRepositoryEvidence(PDO $pdo, Connection $connection): ?array + { + if ($connection->profileName !== 'dolt') { + return null; + } + $identity = $this->rows( + $pdo, + "SELECT active_branch() AS branch_name, dolt_hashof('HEAD') AS head_hash", + [], + )[0] ?? []; + $status = array_map(static function (array $row): array { + $row = array_change_key_case($row, CASE_LOWER); + + return [ + 'table_name' => (string) ($row['table_name'] ?? ''), + 'status' => (string) ($row['status'] ?? ''), + 'staged' => (bool) ($row['staged'] ?? false), + ]; + }, $this->rows($pdo, 'SELECT table_name, status, staged FROM dolt_status ORDER BY table_name', [])); + + return [ + 'branch' => (string) ($identity['branch_name'] ?? ''), + 'head' => (string) ($identity['head_hash'] ?? ''), + 'history' => array_map('strval', $this->column( + $pdo, + 'SELECT commit_hash FROM dolt_log ORDER BY commit_order DESC, commit_hash', + )), + 'status' => $status, + ]; + } + + /** + * @param array $before + * @param array $after + */ + private function assertDoltRepositoryApplyEvidence(array $before, array $after, string $table, string $caseId): void + { + $beforeRepository = $before['repository'] ?? null; + $afterRepository = $after['repository'] ?? null; + self::assertIsArray($beforeRepository); + self::assertIsArray($afterRepository); + self::assertSame($beforeRepository['branch'], $afterRepository['branch']); + self::assertSame($beforeRepository['head'], $afterRepository['head']); + self::assertSame($beforeRepository['history'], $afterRepository['history']); + $changedTables = [$table, 'transformation_apply']; + if ($caseId === 'dolt-preprovisioned-target-sequential-null-semantics') { + $changedTables = [ + $table, + 'transformation_apply', + 'value_label', + 'value_label_set', + 'variable', + 'variable_value_label_set', + ]; + } + sort($changedTables); + self::assertSame(array_map(static fn(string $changed): array => [ + 'table_name' => $changed, + 'status' => 'modified', + 'staged' => false, + ], $changedTables), $afterRepository['status']); + } + + /** @param list $tables */ + private function assertNoForbiddenArtifactTables(array $tables): void + { + self::assertSame([], array_values(array_filter( + $tables, + static fn(string $table): bool => preg_match('/derived|output|staging|snapshot|rollback|recovery/i', $table) === 1, + ))); + } + private function quoteDoltDatabase(string $database): string { if (strlen($database) > 64 || preg_match('/\A[a-z][a-z0-9_]*\z/D', $database) !== 1) { From bbeb55a2a5275165a3f51d5c2834606ee3e98f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 21:50:40 +0300 Subject: [PATCH 21/53] feat: claim official transformation and SPSS 0.2 conformance --- .github/workflows/ci.yml | 8 +- CHANGELOG.md | 23 ++ README.md | 43 ++- docs/architecture.md | 38 ++- docs/release-readiness.md | 66 ++-- docs/transformations.md | 318 +++++++----------- src/Core/CapabilityDeclaration.php | 15 +- .../Spss/Ast/DeleteVariablesStatement.php | 23 -- src/Frontend/Spss/Ast/StringStatement.php | 24 -- src/Frontend/Spss/Binder.php | 10 - .../Spss/Binding/BoundCreateVariable.php | 14 - .../Spss/Binding/BoundDeleteVariable.php | 10 - src/Frontend/Spss/Parser.php | 55 --- .../Model/Action/AssignValueAction.php | 29 -- .../Model/Action/CopySourceAction.php | 22 -- .../Model/Action/SetMissingAction.php | 22 -- .../Model/CreateVariableOperation.php | 63 ---- .../Model/DeleteVariableOperation.php | 31 -- src/Transformation/Model/RecodeAction.php | 13 - src/Transformation/Model/RecodeOperation.php | 50 --- src/Transformation/Model/RecodeRule.php | 32 -- src/Transformation/Model/RecodeSelector.php | 13 - src/Transformation/Model/ScalarValue.php | 80 ----- .../Model/Selector/ElseSelector.php | 22 -- .../Model/Selector/ExactValueSelector.php | 29 -- .../Model/Selector/MissingValueSelector.php | 22 -- .../Model/Selector/NumericRangeSelector.php | 43 --- .../Model/SetValueLabelsOperation.php | 51 --- .../Model/SetVariableLabelOperation.php | 45 --- .../Model/TransformationOperation.php | 18 - .../Model/TransformationPlan.php | 60 ---- src/Transformation/Model/ValueLabel.php | 29 -- .../Validation/InvalidTransformationPlan.php | 28 -- .../Validation/PlanValidator.php | 301 ----------------- .../Validation/ValidationResult.php | 29 -- .../Validation/ValidationViolation.php | 20 -- tests/Core/CapabilityDeclarationTest.php | 26 +- tests/Frontend/Spss/ParserTest.php | 28 -- tests/Frontend/Spss/SpssCompilerTest.php | 19 -- .../Canonical/TransformationPlanTest.php | 227 ------------- 40 files changed, 271 insertions(+), 1728 deletions(-) delete mode 100644 src/Frontend/Spss/Ast/DeleteVariablesStatement.php delete mode 100644 src/Frontend/Spss/Ast/StringStatement.php delete mode 100644 src/Frontend/Spss/Binding/BoundCreateVariable.php delete mode 100644 src/Frontend/Spss/Binding/BoundDeleteVariable.php delete mode 100644 src/Transformation/Model/Action/AssignValueAction.php delete mode 100644 src/Transformation/Model/Action/CopySourceAction.php delete mode 100644 src/Transformation/Model/Action/SetMissingAction.php delete mode 100644 src/Transformation/Model/CreateVariableOperation.php delete mode 100644 src/Transformation/Model/DeleteVariableOperation.php delete mode 100644 src/Transformation/Model/RecodeAction.php delete mode 100644 src/Transformation/Model/RecodeOperation.php delete mode 100644 src/Transformation/Model/RecodeRule.php delete mode 100644 src/Transformation/Model/RecodeSelector.php delete mode 100644 src/Transformation/Model/ScalarValue.php delete mode 100644 src/Transformation/Model/Selector/ElseSelector.php delete mode 100644 src/Transformation/Model/Selector/ExactValueSelector.php delete mode 100644 src/Transformation/Model/Selector/MissingValueSelector.php delete mode 100644 src/Transformation/Model/Selector/NumericRangeSelector.php delete mode 100644 src/Transformation/Model/SetValueLabelsOperation.php delete mode 100644 src/Transformation/Model/SetVariableLabelOperation.php delete mode 100644 src/Transformation/Model/TransformationOperation.php delete mode 100644 src/Transformation/Model/TransformationPlan.php delete mode 100644 src/Transformation/Model/ValueLabel.php delete mode 100644 src/Transformation/Validation/InvalidTransformationPlan.php delete mode 100644 src/Transformation/Validation/PlanValidator.php delete mode 100644 src/Transformation/Validation/ValidationResult.php delete mode 100644 src/Transformation/Validation/ValidationViolation.php delete mode 100644 tests/Transformation/Canonical/TransformationPlanTest.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d28063f..bcec3ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: coverage: none extensions: pdo_pgsql, pdo_sqlite, bcmath, mbstring, zlib - run: composer install --no-interaction --prefer-dist - - run: vendor/bin/phpunit --filter "PostgreSqlSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest" + - run: vendor/bin/phpunit --filter "PostgreSqlSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest|OfficialInPlaceTransformation01Test|OfficialInPlaceTransformation02Test" mysql-integration: name: MySQL ${{ matrix.mysql }} integration / PHP ${{ matrix.php }} @@ -115,7 +115,7 @@ jobs: coverage: none extensions: pdo_mysql, pdo_sqlite, bcmath, mbstring, zlib - run: composer install --no-interaction --prefer-dist - - run: vendor/bin/phpunit --filter "MySqlSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest" + - run: vendor/bin/phpunit --filter "MySqlSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest|OfficialInPlaceTransformation01Test|OfficialInPlaceTransformation02Test" mariadb-integration: name: MariaDB ${{ matrix.mariadb }} integration / PHP ${{ matrix.php }} @@ -159,7 +159,7 @@ jobs: coverage: none extensions: pdo_mysql, pdo_sqlite, bcmath, mbstring, zlib - run: composer install --no-interaction --prefer-dist - - run: vendor/bin/phpunit --filter "MariaDbSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest" + - run: vendor/bin/phpunit --filter "MariaDbSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest|OfficialInPlaceTransformation01Test|OfficialInPlaceTransformation02Test" dolt-integration: name: Dolt ${{ matrix.dolt.version }} integration / PHP ${{ matrix.php }} @@ -213,4 +213,4 @@ jobs: done exit 1 - run: composer install --no-interaction --prefer-dist - - run: vendor/bin/phpunit --filter "DoltSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest" + - run: vendor/bin/phpunit --filter "DoltSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest|OfficialInPlaceTransformation01Test|OfficialInPlaceTransformation02Test" diff --git a/CHANGELOG.md b/CHANGELOG.md index d5d240f..141d421 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ ## [Unreleased] +### Added + +- Added official OpenStatSpec Transformation Plan 0.1/0.2, SPSS Syntax + Frontend 0.2, and In-Place Transformation 0.1/0.2 conformance, including + strict canonical codecs, source hashes, compact apply audit, and service + matrix coverage. + +### Changed + +- Transformations now compile `SpssFrontendRequest` into an alias-based + `SpssCompilationResult` and apply it through `InPlaceApplyRequest` with an + explicit actor and optional controlled Dolt context. +- SQLite and PostgreSQL may create numeric targets atomically. MySQL, MariaDB, + and Dolt require targets to be pre-provisioned and catalogued. Dolt commits + remain caller-owned. + +### Removed + +- Version 0.6.0 removes the package-local + `openstatspec-transformation-plan-v1` API without a compatibility adapter. +- Removed the non-standard SPSS `STRING` and `DELETE VARIABLES` + transformation commands. + ## [0.5.0] - 2026-08-17 ### Added diff --git a/README.md b/README.md index 1f92509..f87d927 100644 --- a/README.md +++ b/README.md @@ -112,14 +112,55 @@ $export = $adapter->export( Pass only loss codes consciously accepted for that conversion. `operation_catalog` records successful and failed imports/exports; `fidelity_event_catalog` records emitted diagnostics. A failed preflight is therefore auditable even when it created no dataset. Each operation also records the selected SPSS engine package and Composer version in engine_details. +## Transformation API + +The adapter claims official Transformation Plan 0.1/0.2, SPSS Syntax Frontend +0.2, and In-Place Transformation 0.1/0.2 conformance. Compile an alias-based +frontend request, then bind that alias to the existing dataset at apply time: + +```php +use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; +use OpenStatSpec\Frontend\Spss\SpssCompiler; +use OpenStatSpec\Sql\Connection; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; +use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; + +$compiled = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray($request)); +$apply = new InPlaceApplyRequest( + plan: $compiled->plan, + inputAlias: 'parent', + datasetId: $datasetId, + sourceHash: $compiled->sourceHash, + actor: 'analyst@example.org', + expectedBranch: $branch, + expectedHead: $head, +); +$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($apply); +``` + +Run `SpssAdapter::migrateCatalog()` before apply. SQLite and PostgreSQL may +create numeric targets atomically; MySQL, MariaDB, and Dolt require target +columns and catalog entries to be pre-provisioned. Dolt additionally requires +the expected branch and HEAD plus a clean working set. The executor never +creates a Dolt commit: inspection and commit policy remain caller-owned. + +Version 0.6.0 removes the package-local +`openstatspec-transformation-plan-v1` API and its non-standard SPSS `STRING` +and `DELETE VARIABLES` transformation commands without a compatibility layer. +See [docs/transformations.md](docs/transformations.md) for migration details, +supported operations, and atomicity guarantees. + ## Architecture - `src/Core` - diagnostics and fail-closed fidelity policy. - `src/Sql` - PDO profiles, strict-wide DDL, import/export and catalogues. - `src/Spss` - SAV/ZSAV gating, typed V3 engine bridge and public adapter API. +- `src/Transformation` - official plans, compact apply audit, and in-place execution. +- `src/Frontend/Spss` - official SPSS Frontend 0.2 request compilation. See [docs/architecture.md](docs/architecture.md) for the complete relational contract. -See [docs/transformations.md](docs/transformations.md) for the current package-local legacy transformation plan, frontend boundaries, in-place guarantees, supported syntax, and the required migration path to the official OpenStatSpec plan profiles. The capability declaration exposes no official plan or frontend contract claim yet. +See [docs/transformations.md](docs/transformations.md) for frontend boundaries, +official contract identifiers, in-place guarantees, and migration guidance. ## Upgrading an existing catalogue diff --git a/docs/architecture.md b/docs/architecture.md index 0372a4b..8215f4d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -84,21 +84,26 @@ Encrypted files, Portable (`.por`) files and arbitrary external-engine formats a ### Transformations -The src/Transformation package owns the statistics-package-neutral canonical -plan, validation, deterministic plan identity, and PDO in-place executor. An -apply preserves the canonical dataset UUID and registered wide-table identity. -It does not publish a derived dataset or create a persistent data copy, -snapshot, rollback table, or OpenStatSpec-managed version. - -The src/Frontend/Spss package is a separate language boundary that turns the -documented SPSS subset into a canonical plan. The executor accepts that plan -and does not import or invoke the SPSS frontend. The Stata and SAS directories -are documented placeholders only; they contain no implementation and make no -support claim. - -All implemented PDO profiles remain eligible transformation targets. Dolt adds -active-branch, HEAD, and clean-working-set guards; it is not a gateway for the -feature. Dolt owns history and rollback. See the +`src/Transformation/Plan` owns the official Transformation Plan 0.1/0.2 model, +strict codec, validation, and deterministic plan identity. +`src/Frontend/Spss` is the separate official SPSS Frontend 0.2 boundary. +`src/Transformation/Execution` binds an alias-based plan to one existing +dataset and applies it through the active PDO profile; it never invokes the +frontend. The Stata and SAS directories are placeholders only. + +Apply preserves the canonical dataset UUID, registered wide-table identity, +case order/count, dataset count, and persistent data-table count. It does not +publish a derived dataset or create a persistent data copy, output/staging +table, snapshot, rollback table, or OpenStatSpec-managed version. The explicit +catalog migration provisions `transformation_apply`; one compact success row +is written inside the native apply transaction. + +SQLite and PostgreSQL may create numeric targets atomically. MySQL, MariaDB, +and Dolt require the intended target column and catalog row to be +pre-provisioned; a create-target plan fails before mutation. Dolt additionally +requires caller-supplied branch/HEAD expectations and a clean working set. The +executor never commits or otherwise changes Dolt history: review and commit +remain caller-owned. See the [transformation manual](transformations.md) for the complete contract and operational guidance. @@ -129,7 +134,8 @@ $adapter->migrateCatalog(); The command creates and versions the canonical catalogue through `openstatspec_schema_migration`; it also applies the write-format migration to SQLite, MySQL/MariaDB/Dolt and PostgreSQL compatibility catalogues, then backfills -each exportable legacy dataset into the singular standard tables. +each exportable legacy dataset into the singular standard tables and provisions +the compact official transformation-apply audit relation. A completely empty dedicated namespace is initialized automatically on its first ordinary import or export attempt. An existing catalogue is never diff --git a/docs/release-readiness.md b/docs/release-readiness.md index 6422791..517cff8 100644 --- a/docs/release-readiness.md +++ b/docs/release-readiness.md @@ -1,45 +1,45 @@ # PHP adapter release readiness -This page records the release contract for the next PHP adapter release. It -does not mean that a package tag or Packagist publication has happened. +This page records the release contract for PHP adapter v0.6.0. It does not mean +that a package tag or Packagist publication has happened. -## Specification pin +## Specification pin and claims -The adapter's machine-readable capability declaration and every CI conformance -checkout must use the stable OpenStatSpec specification release `v0.3.0` at -exact commit -`cd8f198c68b849eb8ed018a894670a0904c2181d`. The adapter reports -`specification_status=stable` and `specification_release=v0.3.0`. +The machine-readable capability declaration and every CI fixture checkout use +stable OpenStatSpec specification release `v0.3.0` at exact commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. -This pin covers the source-faithful SPSS SAV/ZSAV relational contract. The PHP -adapter does not claim official Transformation Plan or SPSS Frontend profile -conformance; those remain separate implementation gates. +Release v0.6.0 claims conformance with Transformation Plan 0.1/0.2, SPSS Syntax +Frontend 0.2, and In-Place Transformation 0.1/0.2. It removes the package-local +plan API and the non-standard SPSS schema-edit commands described in the +[transformation migration notes](transformations.md#v060-migration). ## Required gates -Before tagging a release: +Before tagging v0.6.0: 1. Run `composer check` with `OPENSTATSPEC_SPECIFICATION_DIR` pointing to the - exact `v0.3.0` checkout. -2. Run the focused service evidence command: + exact pinned checkout. +2. Run every official transformation suite: ```bash - OPENSTATSPEC_SPECIFICATION_DIR=/path/to/openstatspec-specification \ - vendor/bin/phpunit tests/Integration/InPlaceTransformationServiceTest.php + vendor/bin/phpunit tests/Transformation/Conformance + vendor/bin/phpunit tests/Frontend/Spss/Conformance + vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php + vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation02Test.php ``` - Locally, SQLite executes and an unconfigured PostgreSQL, MySQL, MariaDB, or - Dolt profile skips; CI must configure and execute every one of those - service profiles. -3. Confirm that the existing-target service evidence preserves the existing - dataset UUID, physical table, and variable identities without copied or - rollback state. Confirm that implicit numeric target creation is restricted - to SQLite and PostgreSQL, while MySQL, MariaDB, and Dolt reject it before - mutation and require a pre-created, catalogued target. -4. Confirm GitHub Actions passes on PHP 8.4 and 8.5 plus the PostgreSQL, - MySQL, MariaDB, and Dolt service matrices. This is adapter evidence under - the legacy `openstatspec-transformation-plan-v1` contract, not an official - OpenStatSpec Transformation Plan 1.0 profile claim. -5. Confirm the changelog and release notes describe the same specification - release and commit. -6. Create an annotated `vX.Y.Z` tag on the reviewed `main` commit and verify - that the tag points to that commit before publishing the GitHub release. -7. Confirm the resulting Composer package is installable from Packagist. + Confirm all 4 Plan 0.1, 26 Plan 0.2, 44 Frontend 0.2, 6 In-Place 0.1, + and 11 In-Place 0.2 manifest cases run for their configured profiles. +3. Confirm successful applies preserve dataset/table identity and counts and + create no copied, output, staging, snapshot, rollback, or version state. +4. Confirm SQLite/PostgreSQL atomic numeric-target creation and preflight + rejection on MySQL, MariaDB, and Dolt. Deployment must pre-provision and + catalog targets on those three profiles. +5. Confirm live Dolt evidence proves stable branch/HEAD, a clean pre-apply + working set, no adapter-created commit, and caller-owned commit policy. +6. Confirm GitHub Actions passes the full PHP 8.4/8.5 jobs plus every + PostgreSQL, MySQL, MariaDB, and Dolt matrix entry. Each service filter must + include both official in-place test classes. +7. Confirm the changelog and release notes describe the same specification + release, commit, breaking removals, and pre-provisioning rules. +8. Create annotated tag `v0.6.0` on the reviewed `main` commit, verify the tag, + publish the GitHub release, and confirm Packagist installation. diff --git a/docs/transformations.md b/docs/transformations.md index 1a2b1ca..a0f3e81 100644 --- a/docs/transformations.md +++ b/docs/transformations.md @@ -1,221 +1,147 @@ # Transformations -## Purpose and boundary - -> Status: this PHP layer currently uses the package-local legacy contract -> `openstatspec-transformation-plan-v1`. It does not claim conformance with -> OpenStatSpec Transformation Plan or SPSS Frontend profile 0.1 or 0.2. -> Migration to official 0.1 is required before the additive 0.2 conditional -> profile can be implemented or claimed. - -The transformation layer applies small, deterministic edits to an existing -OpenStatSpec dataset. Its canonical `TransformationPlan`, validation, and SQL -executor do not depend on SPSS, Stata, SAS, or another statistics package. -Language-specific syntax belongs to a frontend that compiles into the same -canonical plan. - -The initial frontend implements a documented subset of SPSS transformation -syntax. The Stata and SAS directories are placeholders only. Their presence is -an architectural reservation, not a support claim. - -## In-place contract - -Every successful apply preserves both identities: - -- the existing `dataset.dataset_id`; and -- the existing `dataset.physical_table_schema` plus - `dataset.physical_table_name`. - -The executor updates that wide table and its existing normative metadata -catalog in place. It does not create a derived dataset, persistent output or -staging table, full-table copy, snapshot table, hidden rollback table, or a -parallel OpenStatSpec version. A successful edit therefore does not increase -the persistent dataset count or physical data-table count. - -The database engine's native transaction is used where it can make the -operation atomic. OpenStatSpec does not add a durable undo or recovery-version -layer around engines whose DDL commits implicitly. Dolt remains the history, -diff, branch, and rollback layer when Dolt is the selected SQL server. - -## Architecture - -The package separates four responsibilities: - -1. `OpenStatSpec\Transformation\Model` defines the canonical, typed plan and - operations. -2. `OpenStatSpec\Transformation\Validation` validates plans without knowing - their source language or SQL dialect. -3. `OpenStatSpec\Frontend\Spss` lexes, parses, binds, and compiles the supported - SPSS subset into a canonical plan. -4. `OpenStatSpec\Transformation\Execution` resolves catalog identities and - applies a validated plan through the active PDO profile. - -The SQL executor accepts a completed plan. It never invokes the SPSS parser. -Likewise, the SPSS frontend does not issue SQL or select a database profile. -This is the package boundary a future real frontend must use. - -Canonical serialization is deterministic. The plan hash identifies the exact -validated operation sequence; source text and language provenance stay outside -the source-neutral plan. - -## Supported operations - -The canonical layer supports: - -- ordered recode rules with exact values, numeric ranges, missing values, and - exactly one explicit final else rule; -- assigning a scalar value, copying the source value, or assigning system - missing; -- variable-label replacement; and -- complete value-label replacement for one variable. - -Recode rules use first-match semantics. Validation rejects overlapping, -duplicate, or ill-typed rules before SQL mutation. An SPSS frontend plan always -meets the canonical explicit-else contract: when source syntax omits `ELSE`, -the compiler adds SPSS's context-appropriate default action. Variables are -resolved through the normative `variable` catalog and physical identifiers -are quoted by the active PDO SQL profile; callers cannot supply raw table or -column SQL. - -## SPSS frontend scope - -The SPSS frontend recognizes the documented transformation subset: - -- `RECODE ... INTO ...` with exact values, `THRU` ranges, - `LOWEST`, `HIGHEST`, `SYSMIS`, `ELSE`, `COPY`, and `SYSMIS` - outputs; -- `VARIABLE LABELS`; and -- `VALUE LABELS`. - -Keywords are case-insensitive. Dataset variable references currently must -match the normative `variable.source_name` spelling exactly; this documented -subset does not claim SPSS's case-insensitive symbol binding. The `MISSING` -selector fails closed because SPSS user-missing semantics require binding the -dataset's `missing_rule` metadata; use `SYSMIS` for system missing or list -supported explicit values. Statements end with a period. Unsupported SPSS -commands fail closed with a frontend diagnostic; they are not silently skipped -or passed to an external statistics engine. This package does not claim full -SPSS syntax compatibility. - -A multi-variable RECODE with INTO targets is expanded into ordered canonical -operations only when an earlier target does not overwrite a source needed by a -later pair in the same statement. Dependency-overlapping lists fail closed -because preserving SPSS simultaneous-input semantics would otherwise require -hidden row snapshots. - -## SQL profiles and Dolt - -Transformations are not restricted to Dolt. The executor uses every SQL -connection profile implemented by this package: SQLite, PostgreSQL, -MySQL/MariaDB, and Dolt. - -Dolt adds safety evidence rather than acting as a gateway. Before mutation the -executor checks the active branch, resolves `HEAD`, and requires a clean Dolt -working set. After mutation it verifies that branch and `HEAD` did not change -under the operation. The executor does not switch branches or create a Dolt -commit. The caller owns the later review and commit policy. - -MySQL-family DDL commits implicitly. A recode into a new physical target column -can only be part of one native atomic apply on a profile with transactional DDL. -On MySQL, MariaDB, and Dolt, create and catalog the intended target variable in -the deployment workflow before applying a recode to it. Existing-column -recodes and metadata edits remain supported. This capability boundary avoids -pretending that a compensating copy or OpenStatSpec rollback layer is atomic. - -SQLite and PostgreSQL may create a new numeric target column inside their -native transaction. A new string target must be registered on every profile -before execution so its normative `declared_string_width` is explicit. - -The machine-readable capability declaration reports in-place transformations -as supported for every implemented SQL profile and states these target-creation -boundaries. Dolt additionally reports its clean-working-set and stable -branch/HEAD guard. - -## Service-matrix evidence gate - -Run the PHP adapter's in-place evidence matrix with the pinned specification -checkout: +## Official contracts -```bash -OPENSTATSPEC_SPECIFICATION_DIR=/path/to/openstatspec-specification \ - vendor/bin/phpunit tests/Integration/InPlaceTransformationServiceTest.php -``` +The PHP adapter conforms to these pinned OpenStatSpec contracts: + +- `openstatspec-transformation-plan-v0.1` and + `openstatspec-transformation-plan-v0.2`; +- `openstatspec-spss-syntax-frontend-v0.2`; and +- `openstatspec-in-place-transformation-v0.1` and + `openstatspec-in-place-transformation-v0.2`. + +Plans are source-neutral, alias-based, and deterministic. The canonical plan +contains only its contract, input alias, and ordered operations. Dataset UUIDs, +SQL identifiers, actor identity, and Dolt context are supplied only when the +plan is applied. -SQLite runs locally. PostgreSQL, MySQL, MariaDB, and Dolt run only when their -respective `OPENSTATSPEC_PG_*`, `OPENSTATSPEC_MYSQL_*`, -`OPENSTATSPEC_MARIADB_*`, or `OPENSTATSPEC_DOLT_*` configuration is supplied; -an absent local profile is reported as skipped. CI supplies every service -profile configuration, so a skipped configured service is not acceptable CI -evidence. - -For every configured profile, the existing-target evidence proves that -recode plus label changes preserve the dataset UUID, physical table identity, -and existing variable identities, with no copied, staging, snapshot, rollback, -or parallel-history table. Numeric implicit target creation is evidenced only -on SQLite and PostgreSQL. MySQL, MariaDB, and Dolt reject it before mutation; -their target variable must be created and catalogued by deployment workflow -before the recode. This is PHP adapter evidence only: it retains the legacy -`openstatspec-transformation-plan-v1` contract and does not establish an -official OpenStatSpec Transformation Plan 1.0 profile. - -## Minimal PHP flow +`OpenStatSpec\Transformation\Plan` contains the immutable official model and +strict `PlanCodec`. `OpenStatSpec\Frontend\Spss` parses and binds an +`SpssFrontendRequest` to an `SpssCompilationResult`. The separate +`OpenStatSpec\Transformation\Execution` layer binds the plan alias to an +existing dataset and mutates its registered wide table. + +## Compile and apply ```php +use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; use OpenStatSpec\Frontend\Spss\SpssCompiler; use OpenStatSpec\Sql\Connection; +use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; -$datasetId = '018f47a2-4c10-7d34-8f11-93b1c3efc321'; -$syntax = 'RECODE score (1=10) (ELSE=COPY).'; +$compiled = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray($request)); +$apply = new InPlaceApplyRequest( + plan: $compiled->plan, + inputAlias: 'parent', + datasetId: $datasetId, + sourceHash: $compiled->sourceHash, + actor: 'analyst@example.org', + expectedBranch: $branch, + expectedHead: $head, +); +$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($apply); +``` + +The request carries the official frontend contract, input alias, ordered input +schema, and exact source text. The compiler emits Plan 0.1 when every command +belongs to the 0.1 subset and Plan 0.2 as soon as any 0.2 command is present. +`PlanCodec` strictly decodes official plan JSON and produces its canonical JSON +and SHA-256 identity without changing operation order or binary64 values. -$plan = (new SpssCompiler())->compile($syntax, $datasetId); -$result = (new InPlaceTransformationExecutor(new Connection($pdo)))->execute($plan); +Before applying transformations to an existing catalog, deployment must run +the explicit catalog migration: + +```php +$adapter = new SpssAdapter($pdo); +$adapter->migrateCatalog(); ``` -`$pdo` must already point to the dedicated OpenStatSpec catalog namespace. -The executor verifies the catalog ownership marker and resolves the same -`dataset_id` and physical wide table before mutation. +The migration provisions the compact `transformation_apply` audit table. Apply +does not hide schema migration inside its transaction. -## Development commands +## Supported operations and syntax -Install dependencies and run the complete local gate from the PHP repository: +Plan 0.1 contains ordered `recode`, `set_variable_label`, and +`replace_value_labels` operations. Plan 0.2 additionally contains `assign`, +`conditional_assign`, `set_format`, `set_measurement_level`, and `execute`. -```bash -composer install -composer check -``` +SPSS Frontend 0.2 supports the corresponding restricted forms of: -Run only transformation tests while developing the layer: +- `RECODE`, including `INTO`, exact/range/system-missing inputs, and explicit + unmatched behavior; +- `VARIABLE LABELS` and `VALUE LABELS`; +- `COMPUTE` and parenthesized numeric `IF` predicates; +- `FORMATS` with the numeric `F` family; +- `VARIABLE LEVEL`; and +- `EXECUTE`. -```bash -vendor/bin/phpunit tests/Transformation tests/Frontend/Spss -``` +Names bind ASCII case-insensitively while plans retain exact catalog spelling. +Multi-variable `RECODE` reads all sources from the pre-command schema. Commands, +expressions, comments, and syntax outside the official subset fail closed. -Apply the formatter, then rerun the complete gate: +## In-place and atomicity contract -```bash -composer fix -composer check -``` +Every successful apply preserves the existing logical dataset UUID, registered +physical table identity, case order, case count, dataset count, and persistent +data-table count. Existing-target recodes and assignments use direct updates; +metadata operations change only their normative catalog fields. + +The executor validates the entire plan, evolving schema, actor, backend +capabilities, and Dolt context before mutation. It then applies every operation +and writes one compact success audit inside one engine-native transaction. A +failure rolls that transaction back and publishes no failed transformation +audit row. + +The transformation path never creates a derived dataset, persistent output or +staging table, full-table copy, snapshot, rollback table, dataset-version row, +or hidden recovery layer. It does not use the import/export operation journal +as a second transformation history. + +SQLite and PostgreSQL can create a numeric target inside the native apply +transaction. MySQL, MariaDB, and Dolt cannot make that schema change atomic, so +deployment must pre-provision and catalog every intended target before apply; +a create-target plan fails before mutation with `schema_change_not_atomic`. + +## Dolt ownership -Database integration checks require the corresponding PDO driver and server. -They must use a dedicated OpenStatSpec namespace, as described in the -[architecture guide](architecture.md#deployment-namespace-and-connection-isolation). +Dolt applies require a non-empty caller-supplied actor, expected branch, and +expected HEAD plus a clean working set. The executor verifies branch and HEAD +before mutation and again before success. It never switches branches, commits, +resets, merges, or tags the repository. -## Operational checklist +A successful Dolt apply intentionally leaves an inspectable working-set diff. +Dolt commits and all version-history decisions remain caller-owned; the audit +stores only compact before/after evidence. -Before applying a plan: +## v0.6.0 migration -1. verify that the connection uses the intended dedicated OpenStatSpec - namespace; -2. select the existing dataset by its canonical UUID; -3. compile source syntax explicitly with the intended frontend, or construct a - canonical plan directly; -4. validate the plan before any mutation; -5. on Dolt, start from the expected branch and a clean working set; and -6. after success, inspect the data and metadata diff and decide separately - whether to create a Dolt commit. +Version 0.6.0 removes the package-local +`openstatspec-transformation-plan-v1` API without a compatibility adapter. It +also removes the non-standard SPSS `STRING` and `DELETE VARIABLES` +transformation commands. Callers must move to `SpssFrontendRequest`, +`SpssCompilationResult`, `PlanCodec`, `InPlaceApplyRequest`, and the official +contract identifiers shown above. + +On MySQL, MariaDB, and Dolt, deployment migrations must create and catalog new +targets before applying an official plan. On Dolt, the caller must also capture +the expected branch and HEAD, apply against a clean working set, inspect the +resulting diff, and decide whether and how to commit it. + +## Conformance and development gates + +The official fixture suites are loaded from the pinned OpenStatSpec +specification checkout: + +```bash +vendor/bin/phpunit tests/Transformation/Conformance +vendor/bin/phpunit tests/Frontend/Spss/Conformance +vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php +vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation02Test.php +composer check +``` -OpenStatSpec stores only compact operation evidence such as the plan identity -and relevant Dolt state. It never stores copied row state as transformation -audit data. +They cover all 4 Plan 0.1, 26 Plan 0.2, 44 Frontend 0.2, 6 In-Place 0.1, +and 11 In-Place 0.2 manifest cases. SQLite runs locally. PostgreSQL, MySQL, +MariaDB, and Dolt cases run when their `OPENSTATSPEC_*` service configuration +is supplied; CI configures every service family. diff --git a/src/Core/CapabilityDeclaration.php b/src/Core/CapabilityDeclaration.php index ca02bae..f6cc97f 100644 --- a/src/Core/CapabilityDeclaration.php +++ b/src/Core/CapabilityDeclaration.php @@ -13,7 +13,6 @@ use OpenStatSpec\Sql\PdoSqlProfile; use OpenStatSpec\Sql\PostgreSqlProfile; use OpenStatSpec\Sql\SqliteProfile; -use OpenStatSpec\Transformation\Model\TransformationPlan; use PDO; /** Machine-readable SPSS 1.0 and SQL-profile capability declaration. */ @@ -57,14 +56,18 @@ public function toArray(): array 'limit_basis' => 'runtime_memory_limit', ], 'transformation_contracts' => [ - 'status' => 'legacy_contract_only', - 'legacy_plan_contracts' => [TransformationPlan::CONTRACT], - 'official_plan_contracts' => [], - 'official_frontend_contracts' => [], - 'migration_order' => [ + 'status' => 'official_conformant', + 'official_plan_contracts' => [ 'openstatspec-transformation-plan-v0.1', 'openstatspec-transformation-plan-v0.2', ], + 'official_frontend_contracts' => [ + 'openstatspec-spss-syntax-frontend-v0.2', + ], + 'official_binding_contracts' => [ + 'openstatspec-in-place-transformation-v0.1', + 'openstatspec-in-place-transformation-v0.2', + ], ], 'active_connection' => [ 'profile' => $activeProfile, diff --git a/src/Frontend/Spss/Ast/DeleteVariablesStatement.php b/src/Frontend/Spss/Ast/DeleteVariablesStatement.php deleted file mode 100644 index ccedcff..0000000 --- a/src/Frontend/Spss/Ast/DeleteVariablesStatement.php +++ /dev/null @@ -1,23 +0,0 @@ - $variables */ - public function __construct( - public int $lineNumber, - public array $variables, - public SourceSpan $commandSpan, - public SourceSpan $span, - ) {} - - public function line(): int - { - return $this->lineNumber; - } -} diff --git a/src/Frontend/Spss/Ast/StringStatement.php b/src/Frontend/Spss/Ast/StringStatement.php deleted file mode 100644 index a9eb3fb..0000000 --- a/src/Frontend/Spss/Ast/StringStatement.php +++ /dev/null @@ -1,24 +0,0 @@ - $variables */ - public function __construct( - public int $lineNumber, - public array $variables, - public int $width, - public SourceSpan $commandSpan, - public SourceSpan $span, - ) {} - - public function line(): int - { - return $this->lineNumber; - } -} diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php index 170a90a..b180b6a 100644 --- a/src/Frontend/Spss/Binder.php +++ b/src/Frontend/Spss/Binder.php @@ -7,7 +7,6 @@ use OpenStatSpec\Frontend\Spss\Ast\BooleanPredicate as AstBooleanPredicate; use OpenStatSpec\Frontend\Spss\Ast\Comparison as AstComparison; use OpenStatSpec\Frontend\Spss\Ast\ComputeStatement; -use OpenStatSpec\Frontend\Spss\Ast\DeleteVariablesStatement; use OpenStatSpec\Frontend\Spss\Ast\ElseInput; use OpenStatSpec\Frontend\Spss\Ast\ExecuteStatement; use OpenStatSpec\Frontend\Spss\Ast\ExpressionOperand as AstExpressionOperand; @@ -23,7 +22,6 @@ use OpenStatSpec\Frontend\Spss\Ast\RecodeOutputKind; use OpenStatSpec\Frontend\Spss\Ast\RecodeStatement; use OpenStatSpec\Frontend\Spss\Ast\ScalarValue; -use OpenStatSpec\Frontend\Spss\Ast\StringStatement; use OpenStatSpec\Frontend\Spss\Ast\SystemMissingInput; use OpenStatSpec\Frontend\Spss\Ast\ValueInput; use OpenStatSpec\Frontend\Spss\Ast\ValueLabelsStatement; @@ -132,14 +130,6 @@ public function bind(string $inputAlias, InputSchema $inputSchema, Program $prog array_push($bound, ...$this->valueLabels($statement, $schema)); continue; } - if ($statement instanceof StringStatement || $statement instanceof DeleteVariablesStatement) { - $this->fail( - 'unsupported_spss_command', - 'STRING and DELETE VARIABLES are outside Frontend 0.2.', - $statement->commandSpan, - ); - } - $this->fail( 'unsupported_spss_command', sprintf('Unsupported AST statement %s.', $statement::class), diff --git a/src/Frontend/Spss/Binding/BoundCreateVariable.php b/src/Frontend/Spss/Binding/BoundCreateVariable.php deleted file mode 100644 index c1f37e1..0000000 --- a/src/Frontend/Spss/Binding/BoundCreateVariable.php +++ /dev/null @@ -1,14 +0,0 @@ -ifStatement($command); } elseif ($this->matchKeyword('FORMATS')) { $statements[] = $this->formats($command); - } elseif ($this->matchKeyword('STRING')) { - $statements[] = $this->string($command); - } elseif ($this->matchKeyword('DELETE')) { - $this->consumeKeyword('VARIABLES', 'Expected VARIABLES after DELETE.'); - $statements[] = $this->deleteVariables($command); } else { $code = $command->isWord() ? 'unsupported_spss_command' @@ -520,54 +513,6 @@ private function valueLabels(Token $command): ValueLabelsStatement ); } - private function string(Token $command): StringStatement - { - $variables = []; - do { - $variable = $this->consumeIdentifier('Expected a variable name in STRING.')->lexeme; - if (strcasecmp($variable, 'TO') === 0) { - $this->fail($this->previous(), 'STRING variable ranges using TO are not supported.'); - } - $variables[] = $variable; - } while ($this->current()->isWord()); - $this->consume(TokenType::LeftParenthesis, 'Expected a width declaration in STRING.'); - $width = $this->consume(TokenType::Identifier, 'Expected a string width such as A20.')->lexeme; - if (preg_match('/\AA([1-9][0-9]*)\z/i', $width, $matches) !== 1) { - $this->fail($this->previous(), 'STRING width must use the SPSS A form.'); - } - $widthValue = (int) $matches[1]; - if ($widthValue > 32767) { - $this->fail($this->previous(), 'STRING width must be at most 32767.'); - } - $rightParenthesis = $this->consume(TokenType::RightParenthesis, 'Expected ) after STRING width.'); - - return new StringStatement( - $command->line, - $variables, - $widthValue, - $command->span, - SourceSpan::cover($command->span, $rightParenthesis->span), - ); - } - - private function deleteVariables(Token $command): DeleteVariablesStatement - { - $variables = []; - do { - $variable = $this->consumeIdentifier('Expected a variable name in DELETE VARIABLES.')->lexeme; - if (strcasecmp($variable, 'TO') === 0) { - $this->fail($this->previous(), 'DELETE VARIABLES ranges using TO are not supported.'); - } - $variables[] = $variable; - } while ($this->current()->isWord()); - - return new DeleteVariablesStatement( - $command->line, - $variables, - $command->span, - SourceSpan::cover($command->span, $this->previous()->span), - ); - } private function scalar(string $message): ScalarValue { if ($this->match(TokenType::String)) { diff --git a/src/Transformation/Model/Action/AssignValueAction.php b/src/Transformation/Model/Action/AssignValueAction.php deleted file mode 100644 index 46af1b0..0000000 --- a/src/Transformation/Model/Action/AssignValueAction.php +++ /dev/null @@ -1,29 +0,0 @@ -value; - } - - /** @return array{type: string, value: array} */ - public function canonicalArray(): array - { - return ['type' => $this->type(), 'value' => $this->value->canonicalArray()]; - } -} diff --git a/src/Transformation/Model/Action/CopySourceAction.php b/src/Transformation/Model/Action/CopySourceAction.php deleted file mode 100644 index 64fd47b..0000000 --- a/src/Transformation/Model/Action/CopySourceAction.php +++ /dev/null @@ -1,22 +0,0 @@ - $this->type()]; - } -} diff --git a/src/Transformation/Model/Action/SetMissingAction.php b/src/Transformation/Model/Action/SetMissingAction.php deleted file mode 100644 index 50444e4..0000000 --- a/src/Transformation/Model/Action/SetMissingAction.php +++ /dev/null @@ -1,22 +0,0 @@ - $this->type()]; - } -} diff --git a/src/Transformation/Model/CreateVariableOperation.php b/src/Transformation/Model/CreateVariableOperation.php deleted file mode 100644 index 038e298..0000000 --- a/src/Transformation/Model/CreateVariableOperation.php +++ /dev/null @@ -1,63 +0,0 @@ - self::MAX_STRING_WIDTH) { - throw new \InvalidArgumentException( - 'String variables support at most ' . self::MAX_STRING_WIDTH . ' bytes.', - ); - } - if ($storageKind === 'numeric' && $declaredStringWidth !== null) { - throw new \InvalidArgumentException('Numeric variables cannot declare a string width.'); - } - } - - public function type(): string - { - return 'create_variable'; - } - public function sourceVariable(): string - { - return $this->variable; - } - public function targetVariable(): string - { - return $this->variable; - } - public function storageKind(): string - { - return $this->storageKind; - } - public function declaredStringWidth(): ?int - { - return $this->declaredStringWidth; - } - - public function canonicalArray(): array - { - return array_filter([ - 'type' => $this->type(), - 'source_variable' => $this->variable, - 'target_variable' => $this->variable, - 'storage_kind' => $this->storageKind, - 'declared_string_width' => $this->declaredStringWidth, - ], static fn(mixed $value): bool => $value !== null); - } -} diff --git a/src/Transformation/Model/DeleteVariableOperation.php b/src/Transformation/Model/DeleteVariableOperation.php deleted file mode 100644 index 27a7110..0000000 --- a/src/Transformation/Model/DeleteVariableOperation.php +++ /dev/null @@ -1,31 +0,0 @@ -variable; - } - public function targetVariable(): string - { - return $this->variable; - } - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'source_variable' => $this->variable, - 'target_variable' => $this->variable, - ]; - } -} diff --git a/src/Transformation/Model/RecodeAction.php b/src/Transformation/Model/RecodeAction.php deleted file mode 100644 index b495cbd..0000000 --- a/src/Transformation/Model/RecodeAction.php +++ /dev/null @@ -1,13 +0,0 @@ - */ - public function canonicalArray(): array; -} diff --git a/src/Transformation/Model/RecodeOperation.php b/src/Transformation/Model/RecodeOperation.php deleted file mode 100644 index 27b2b72..0000000 --- a/src/Transformation/Model/RecodeOperation.php +++ /dev/null @@ -1,50 +0,0 @@ - $rules */ - public function __construct( - private string $sourceVariable, - private string $targetVariable, - private array $rules, - ) {} - - public function type(): string - { - return 'recode'; - } - - public function sourceVariable(): string - { - return $this->sourceVariable; - } - - public function targetVariable(): string - { - return $this->targetVariable; - } - - /** @return list */ - public function rules(): array - { - return $this->rules; - } - - /** @return array{type: string, source_variable: string, target_variable: string, rules: list>} */ - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'source_variable' => $this->sourceVariable, - 'target_variable' => $this->targetVariable, - 'rules' => array_map( - static fn(RecodeRule $rule): array => $rule->canonicalArray(), - $this->rules, - ), - ]; - } -} diff --git a/src/Transformation/Model/RecodeRule.php b/src/Transformation/Model/RecodeRule.php deleted file mode 100644 index 905b10b..0000000 --- a/src/Transformation/Model/RecodeRule.php +++ /dev/null @@ -1,32 +0,0 @@ -selector; - } - - public function action(): RecodeAction - { - return $this->action; - } - - /** @return array{selector: array, action: array} */ - public function canonicalArray(): array - { - return [ - 'selector' => $this->selector->canonicalArray(), - 'action' => $this->action->canonicalArray(), - ]; - } -} diff --git a/src/Transformation/Model/RecodeSelector.php b/src/Transformation/Model/RecodeSelector.php deleted file mode 100644 index ba45a19..0000000 --- a/src/Transformation/Model/RecodeSelector.php +++ /dev/null @@ -1,13 +0,0 @@ - */ - public function canonicalArray(): array; -} diff --git a/src/Transformation/Model/ScalarValue.php b/src/Transformation/Model/ScalarValue.php deleted file mode 100644 index a068e19..0000000 --- a/src/Transformation/Model/ScalarValue.php +++ /dev/null @@ -1,80 +0,0 @@ -type; - } - - public function value(): float|string - { - return $this->value; - } - - public function numberValue(): float - { - if (!is_float($this->value)) { - throw new InvalidArgumentException('A string scalar has no numeric value.'); - } - - return $this->value; - } - - public function stringValue(): string - { - if (!is_string($this->value)) { - throw new InvalidArgumentException('A numeric scalar has no string value.'); - } - - return $this->value; - } - - /** @return array{type: string, binary64?: string, value?: string} */ - public function canonicalArray(): array - { - if (is_string($this->value)) { - return ['type' => 'string', 'value' => $this->value]; - } - - return ['type' => 'number', 'binary64' => bin2hex(pack('E', $this->normalisedNumber()))]; - } - - public function identity(): string - { - return $this->type . ':' . ($this->canonicalArray()['binary64'] ?? $this->value); - } - - private function normalisedNumber(): float - { - $value = $this->value; - if (!is_float($value)) { - throw new InvalidArgumentException('A string scalar has no numeric representation.'); - } - - // IEEE-754 considers both zero representations equal for recoding. - return $value === 0.0 ? 0.0 : $value; - } -} diff --git a/src/Transformation/Model/Selector/ElseSelector.php b/src/Transformation/Model/Selector/ElseSelector.php deleted file mode 100644 index 9bfbd57..0000000 --- a/src/Transformation/Model/Selector/ElseSelector.php +++ /dev/null @@ -1,22 +0,0 @@ - $this->type()]; - } -} diff --git a/src/Transformation/Model/Selector/ExactValueSelector.php b/src/Transformation/Model/Selector/ExactValueSelector.php deleted file mode 100644 index 4444f98..0000000 --- a/src/Transformation/Model/Selector/ExactValueSelector.php +++ /dev/null @@ -1,29 +0,0 @@ -value; - } - - /** @return array{type: string, value: array} */ - public function canonicalArray(): array - { - return ['type' => $this->type(), 'value' => $this->value->canonicalArray()]; - } -} diff --git a/src/Transformation/Model/Selector/MissingValueSelector.php b/src/Transformation/Model/Selector/MissingValueSelector.php deleted file mode 100644 index c52bb2a..0000000 --- a/src/Transformation/Model/Selector/MissingValueSelector.php +++ /dev/null @@ -1,22 +0,0 @@ - $this->type()]; - } -} diff --git a/src/Transformation/Model/Selector/NumericRangeSelector.php b/src/Transformation/Model/Selector/NumericRangeSelector.php deleted file mode 100644 index 73af548..0000000 --- a/src/Transformation/Model/Selector/NumericRangeSelector.php +++ /dev/null @@ -1,43 +0,0 @@ -lower; - } - - public function upper(): ?float - { - return $this->upper; - } - - /** @return array{type: string, lower: ?array, upper: ?array, bounds: string} */ - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'lower' => $this->lower === null ? null : ScalarValue::number($this->lower)->canonicalArray(), - 'upper' => $this->upper === null ? null : ScalarValue::number($this->upper)->canonicalArray(), - 'bounds' => 'inclusive', - ]; - } -} diff --git a/src/Transformation/Model/SetValueLabelsOperation.php b/src/Transformation/Model/SetValueLabelsOperation.php deleted file mode 100644 index ab6f4d9..0000000 --- a/src/Transformation/Model/SetValueLabelsOperation.php +++ /dev/null @@ -1,51 +0,0 @@ - $labels */ - public function __construct( - private string $targetVariable, - private array $labels, - ) {} - - public function type(): string - { - return 'set_value_labels'; - } - - public function sourceVariable(): string - { - return $this->targetVariable; - } - - public function targetVariable(): string - { - return $this->targetVariable; - } - - /** @return list */ - public function labels(): array - { - return $this->labels; - } - - /** @return array{type: string, source_variable: string, target_variable: string, replacement: string, labels: list>} */ - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'source_variable' => $this->targetVariable, - 'target_variable' => $this->targetVariable, - 'replacement' => 'complete', - 'labels' => array_map( - static fn(ValueLabel $label): array => $label->canonicalArray(), - $this->labels, - ), - ]; - } -} diff --git a/src/Transformation/Model/SetVariableLabelOperation.php b/src/Transformation/Model/SetVariableLabelOperation.php deleted file mode 100644 index 1293b5e..0000000 --- a/src/Transformation/Model/SetVariableLabelOperation.php +++ /dev/null @@ -1,45 +0,0 @@ -targetVariable; - } - - public function targetVariable(): string - { - return $this->targetVariable; - } - - public function label(): ?string - { - return $this->label; - } - - /** @return array{type: string, source_variable: string, target_variable: string, label: ?string} */ - public function canonicalArray(): array - { - return [ - 'type' => $this->type(), - 'source_variable' => $this->targetVariable, - 'target_variable' => $this->targetVariable, - 'label' => $this->label, - ]; - } -} diff --git a/src/Transformation/Model/TransformationOperation.php b/src/Transformation/Model/TransformationOperation.php deleted file mode 100644 index 8d63194..0000000 --- a/src/Transformation/Model/TransformationOperation.php +++ /dev/null @@ -1,18 +0,0 @@ - */ - public function canonicalArray(): array; -} diff --git a/src/Transformation/Model/TransformationPlan.php b/src/Transformation/Model/TransformationPlan.php deleted file mode 100644 index b063098..0000000 --- a/src/Transformation/Model/TransformationPlan.php +++ /dev/null @@ -1,60 +0,0 @@ - $operations */ - public function __construct( - private string $datasetId, - private array $operations, - ) {} - - public function datasetId(): string - { - return $this->datasetId; - } - - /** @return list */ - public function operations(): array - { - return $this->operations; - } - - /** @return array{contract: string, mode: string, dataset_id: string, operations: list>} */ - public function canonicalArray(): array - { - return [ - 'contract' => self::CONTRACT, - 'mode' => 'in_place', - 'dataset_id' => $this->datasetId, - 'operations' => array_map( - static fn(TransformationOperation $operation): array => $operation->canonicalArray(), - $this->operations, - ), - ]; - } - - public function canonicalJson(): string - { - return CanonicalJson::encode($this->canonicalArray()); - } - - public function hash(): string - { - return hash('sha256', $this->canonicalJson()); - } -} diff --git a/src/Transformation/Model/ValueLabel.php b/src/Transformation/Model/ValueLabel.php deleted file mode 100644 index 9828281..0000000 --- a/src/Transformation/Model/ValueLabel.php +++ /dev/null @@ -1,29 +0,0 @@ -value; - } - - public function label(): string - { - return $this->label; - } - - /** @return array{value: array, label: string} */ - public function canonicalArray(): array - { - return ['value' => $this->value->canonicalArray(), 'label' => $this->label]; - } -} diff --git a/src/Transformation/Validation/InvalidTransformationPlan.php b/src/Transformation/Validation/InvalidTransformationPlan.php deleted file mode 100644 index 61d9834..0000000 --- a/src/Transformation/Validation/InvalidTransformationPlan.php +++ /dev/null @@ -1,28 +0,0 @@ - $violations */ - public function __construct(private readonly array $violations) - { - $first = $violations[0]; - parent::__construct(sprintf( - 'Invalid transformation plan (%s at %s): %s', - $first->code, - $first->path, - $first->message, - )); - } - - /** @return non-empty-list */ - public function violations(): array - { - return $this->violations; - } -} diff --git a/src/Transformation/Validation/PlanValidator.php b/src/Transformation/Validation/PlanValidator.php deleted file mode 100644 index 2fd866c..0000000 --- a/src/Transformation/Validation/PlanValidator.php +++ /dev/null @@ -1,301 +0,0 @@ - */ - private array $violations = []; - - public function validate(TransformationPlan $plan): ValidationResult - { - $this->violations = []; - if (preg_match('/\A[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/D', $plan->datasetId()) !== 1) { - $this->violation( - 'dataset_id.invalid_uuid', - '$.dataset_id', - 'The dataset identity must be a canonical lowercase RFC 9562 UUID string.', - ); - } - - if ($plan->operations() === []) { - $this->violation('operations.empty', '$.operations', 'A transformation plan must contain at least one operation.'); - } - - foreach ($plan->operations() as $index => $operation) { - $path = '$.operations[' . $index . ']'; - $this->validateOperation($operation, $path); - } - - return new ValidationResult($this->violations); - } - - public function assertValid(TransformationPlan $plan): void - { - $this->validate($plan)->throwIfInvalid(); - } - - private function validateOperation(TransformationOperation $operation, string $path): void - { - if (!in_array($operation::class, [ - CreateVariableOperation::class, - DeleteVariableOperation::class, - RecodeOperation::class, - SetVariableLabelOperation::class, - SetValueLabelsOperation::class, - ], true)) { - $this->violation( - 'operation.unsupported_type', - $path . '.type', - 'Only canonical, source-language-neutral operation types are permitted.', - ); - - return; - } - - $this->validateVariableName($operation->sourceVariable(), $path . '.source_variable'); - $this->validateVariableName($operation->targetVariable(), $path . '.target_variable'); - - if ($operation instanceof CreateVariableOperation) { - if ($operation->storageKind() === 'string') { - $width = $operation->declaredStringWidth(); - if ($width === null || $width < 1) { - $this->violation('create_variable.string_width_required', $path . '.declared_string_width', 'String variables require a positive declared string width.'); - } elseif ($width > CreateVariableOperation::MAX_STRING_WIDTH) { - $this->violation('create_variable.string_width_too_large', $path . '.declared_string_width', 'String variables support at most ' . CreateVariableOperation::MAX_STRING_WIDTH . ' bytes.'); - } - } - } elseif ($operation instanceof DeleteVariableOperation) { - return; - } elseif ($operation instanceof RecodeOperation) { - $this->validateRecode($operation, $path); - } elseif ($operation instanceof SetVariableLabelOperation) { - $this->validateText($operation->label(), $path . '.label'); - } elseif ($operation instanceof SetValueLabelsOperation) { - $this->validateValueLabels($operation, $path); - } - } - - private function validateRecode(RecodeOperation $operation, string $path): void - { - if ($operation->rules() === []) { - $this->violation('recode.rules_empty', $path . '.rules', 'A recode operation must contain ordered rules.'); - - return; - } - - /** @var array $exactValues */ - $exactValues = []; - /** @var list $ranges */ - $ranges = []; - $missingIndex = null; - $elseIndexes = []; - foreach ($operation->rules() as $index => $rule) { - $rulePath = $path . '.rules[' . $index . ']'; - $selector = $rule->selector(); - $this->validateSelector($selector, $rulePath . '.selector'); - $this->validateAction($rule->action(), $rulePath . '.action'); - - if ($selector instanceof ExactValueSelector) { - $identity = $selector->value()->identity(); - if (isset($exactValues[$identity])) { - $this->violation('recode.ambiguous_mapping', $rulePath . '.selector', 'An exact source value is mapped more than once.'); - } else { - $exactValues[$identity] = $index; - } - } elseif ($selector instanceof NumericRangeSelector) { - $ranges[] = ['index' => $index, 'lower' => $selector->lower(), 'upper' => $selector->upper()]; - } elseif ($selector instanceof MissingValueSelector) { - if ($missingIndex !== null) { - $this->violation('recode.ambiguous_mapping', $rulePath . '.selector', 'The system-missing value is mapped more than once.'); - } - $missingIndex = $index; - } elseif ($selector instanceof ElseSelector) { - $elseIndexes[] = $index; - } - } - - if (count($elseIndexes) !== 1) { - $this->violation('recode.else_required_once', $path . '.rules', 'A recode must contain exactly one explicit else rule.'); - } elseif ($elseIndexes[0] !== array_key_last($operation->rules())) { - $this->violation('recode.else_not_last', $path . '.rules[' . $elseIndexes[0] . ']', 'The else rule must be last.'); - } - - $this->validateRangeAmbiguity($ranges, $exactValues, $operation, $path); - } - - private function validateSelector(RecodeSelector $selector, string $path): void - { - if (!in_array($selector::class, [ - ExactValueSelector::class, - NumericRangeSelector::class, - MissingValueSelector::class, - ElseSelector::class, - ], true)) { - $this->violation('selector.unsupported_type', $path . '.type', 'The selector is not part of the canonical source-neutral contract.'); - - return; - } - - if ($selector instanceof ExactValueSelector) { - $this->validateScalar($selector->value(), $path . '.value'); - } elseif ($selector instanceof NumericRangeSelector) { - $lower = $selector->lower(); - $upper = $selector->upper(); - if ($lower === null && $upper === null) { - $this->violation('selector.range_unbounded', $path, 'A numeric range must have at least one finite bound.'); - } - if (($lower !== null && !is_finite($lower)) || ($upper !== null && !is_finite($upper))) { - $this->violation('selector.range_non_finite', $path, 'Numeric range bounds must be finite.'); - } elseif ($lower !== null && $upper !== null && $lower > $upper) { - $this->violation('selector.range_reversed', $path, 'A numeric range lower bound must not exceed its upper bound.'); - } - } - } - - private function validateAction(RecodeAction $action, string $path): void - { - if (!in_array($action::class, [ - AssignValueAction::class, - SetMissingAction::class, - CopySourceAction::class, - ], true)) { - $this->violation('action.unsupported_type', $path . '.type', 'The action is not part of the canonical source-neutral contract.'); - - return; - } - - if ($action instanceof AssignValueAction) { - $this->validateScalar($action->value(), $path . '.value'); - } - } - - private function validateValueLabels(SetValueLabelsOperation $operation, string $path): void - { - /** @var array $values */ - $values = []; - foreach ($operation->labels() as $index => $label) { - $labelPath = $path . '.labels[' . $index . ']'; - $this->validateScalar($label->value(), $labelPath . '.value'); - $this->validateText($label->label(), $labelPath . '.label'); - $identity = $label->value()->identity(); - if (isset($values[$identity])) { - $this->violation('value_labels.duplicate_value', $labelPath . '.value', 'A value may have only one label in a replacement set.'); - } - $values[$identity] = true; - } - } - - private function validateScalar(ScalarValue $value, string $path): void - { - if ($value->type() === 'number') { - if (!is_finite($value->numberValue())) { - $this->violation('value.non_finite', $path, 'Canonical numeric values must be finite binary64 values.'); - } - - return; - } - - $this->validateText($value->stringValue(), $path); - } - - private function validateVariableName(string $name, string $path): void - { - if ($name === '' || strlen($name) > 255 || str_contains($name, "\0") || preg_match('//u', $name) !== 1) { - $this->violation( - 'variable.invalid_name', - $path, - 'Variable names must contain 1-255 bytes of valid UTF-8 scalar text without NUL.', - ); - } - } - - private function validateText(?string $text, string $path): void - { - if ($text !== null && (str_contains($text, "\0") || preg_match('//u', $text) !== 1)) { - $this->violation('text.invalid_unicode', $path, 'Text must contain valid UTF-8 scalar text without NUL.'); - } - } - - /** - * @param list $ranges - * @param array $exactValues - */ - private function validateRangeAmbiguity(array $ranges, array $exactValues, RecodeOperation $operation, string $path): void - { - foreach ($ranges as $leftOffset => $left) { - if (($left['lower'] !== null && !is_finite($left['lower'])) || ($left['upper'] !== null && !is_finite($left['upper']))) { - continue; - } - foreach (array_slice($ranges, $leftOffset + 1) as $right) { - if (($right['lower'] !== null && !is_finite($right['lower'])) || ($right['upper'] !== null && !is_finite($right['upper']))) { - continue; - } - if ($this->rangesOverlap($left, $right)) { - $this->violation( - 'recode.ambiguous_mapping', - $path . '.rules[' . $right['index'] . '].selector', - 'Inclusive numeric ranges must not overlap.', - ); - } - } - - foreach ($exactValues as $identity => $exactIndex) { - if (!str_starts_with($identity, 'number:')) { - continue; - } - $selector = $operation->rules()[$exactIndex]->selector(); - if ($selector instanceof ExactValueSelector && $this->rangeContains($left, $selector->value()->numberValue())) { - $this->violation( - 'recode.ambiguous_mapping', - $path . '.rules[' . $exactIndex . '].selector', - 'An exact numeric selector must not overlap a numeric range.', - ); - } - } - } - } - - /** - * @param array{lower: ?float, upper: ?float} $left - * @param array{lower: ?float, upper: ?float} $right - */ - private function rangesOverlap(array $left, array $right): bool - { - return ($left['upper'] === null || $right['lower'] === null || $left['upper'] >= $right['lower']) - && ($right['upper'] === null || $left['lower'] === null || $right['upper'] >= $left['lower']); - } - - /** @param array{lower: ?float, upper: ?float} $range */ - private function rangeContains(array $range, float $value): bool - { - return ($range['lower'] === null || $value >= $range['lower']) - && ($range['upper'] === null || $value <= $range['upper']); - } - - private function violation(string $code, string $path, string $message): void - { - $this->violations[] = new ValidationViolation($code, $path, $message); - } -} diff --git a/src/Transformation/Validation/ValidationResult.php b/src/Transformation/Validation/ValidationResult.php deleted file mode 100644 index 349ec1d..0000000 --- a/src/Transformation/Validation/ValidationResult.php +++ /dev/null @@ -1,29 +0,0 @@ - $violations */ - public function __construct(private array $violations) {} - - public function isValid(): bool - { - return $this->violations === []; - } - - /** @return list */ - public function violations(): array - { - return $this->violations; - } - - public function throwIfInvalid(): void - { - if ($this->violations !== []) { - throw new InvalidTransformationPlan($this->violations); - } - } -} diff --git a/src/Transformation/Validation/ValidationViolation.php b/src/Transformation/Validation/ValidationViolation.php deleted file mode 100644 index 37ef75c..0000000 --- a/src/Transformation/Validation/ValidationViolation.php +++ /dev/null @@ -1,20 +0,0 @@ - $this->code, 'path' => $this->path, 'message' => $this->message]; - } -} diff --git a/tests/Core/CapabilityDeclarationTest.php b/tests/Core/CapabilityDeclarationTest.php index b198329..ee372ad 100644 --- a/tests/Core/CapabilityDeclarationTest.php +++ b/tests/Core/CapabilityDeclarationTest.php @@ -46,16 +46,22 @@ public function testDeclarationIsMachineReadableAndIncludesEveryProfileLimit(): 'maximum_source_file_bytes' => null, 'limit_basis' => 'runtime_memory_limit', ], $declaration['resource_behavior']); - self::assertSame([ - 'status' => 'legacy_contract_only', - 'legacy_plan_contracts' => ['openstatspec-transformation-plan-v1'], - 'official_plan_contracts' => [], - 'official_frontend_contracts' => [], - 'migration_order' => [ - 'openstatspec-transformation-plan-v0.1', - 'openstatspec-transformation-plan-v0.2', - ], - ], $declaration['transformation_contracts']); + $contracts = $declaration['transformation_contracts']; + self::assertSame('official_conformant', $contracts['status']); + self::assertSame( + ['openstatspec-transformation-plan-v0.1', 'openstatspec-transformation-plan-v0.2'], + $contracts['official_plan_contracts'], + ); + self::assertSame( + ['openstatspec-spss-syntax-frontend-v0.2'], + $contracts['official_frontend_contracts'], + ); + self::assertSame( + ['openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'], + $contracts['official_binding_contracts'], + ); + self::assertArrayNotHasKey('legacy_plan_contracts', $contracts); + self::assertFalse(class_exists(\OpenStatSpec\Transformation\Model\TransformationPlan::class)); self::assertSame('3.0.3', $declaration['engine']['active_version']); self::assertSame('>=3.0.0 <4.0.0', $declaration['engine']['claimed_version_range']); self::assertSame(['3.0.3'], $declaration['engine']['ci_tested_versions']); diff --git a/tests/Frontend/Spss/ParserTest.php b/tests/Frontend/Spss/ParserTest.php index ede9cd8..f4044a8 100644 --- a/tests/Frontend/Spss/ParserTest.php +++ b/tests/Frontend/Spss/ParserTest.php @@ -98,34 +98,6 @@ public function testParsesVariableAndValueLabelGroups(): void self::assertSame(['band'], $program->statements[1]->groups[1]->variables); } - public function testRejectsStringVariableRangesBeforeCreation(): void - { - try { - (new Parser())->parse('STRING c1 TO c4 (A7).'); - self::fail('STRING variable ranges unexpectedly parsed.'); - } catch (SpssSyntaxException $exception) { - self::assertCount(1, $exception->diagnostics); - self::assertSame( - 'STRING variable ranges using TO are not supported.', - $exception->diagnostics[0]->message, - ); - } - } - - public function testRejectsDeleteVariableRangesBeforeExecution(): void - { - try { - (new Parser())->parse('DELETE VARIABLES c1 TO c4.'); - self::fail('DELETE VARIABLES ranges unexpectedly parsed.'); - } catch (SpssSyntaxException $exception) { - self::assertCount(1, $exception->diagnostics); - self::assertSame( - 'DELETE VARIABLES ranges using TO are not supported.', - $exception->diagnostics[0]->message, - ); - } - } - public function testFailsClosedForUnknownOrUnterminatedCommands(): void { foreach (['SORT score.', 'RECODE score (1=2)'] as $syntax) { diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index 73f0737..2331bb2 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -172,25 +172,6 @@ public function testRecodeWithOnlyElseCannotEmitAnInvalidEmptyRuleList(): void } } - public function testLegacySchemaCommandsFailAsUnsupportedAtTheExactCommandToken(): void - { - foreach (['STRING note (A4).', 'DELETE VARIABLES q1.'] as $source) { - try { - (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( - $source, - [['name' => 'q1', 'storage_kind' => 'numeric']], - ))); - self::fail('A non-Frontend-0.2 schema command unexpectedly compiled.'); - } catch (TransformationFailure $failure) { - self::assertSame('unsupported_spss_command', $failure->diagnosticCode()); - self::assertSame([0, 6], [ - $failure->diagnostics[0]->span?->startOffset, - $failure->diagnostics[0]->span?->endOffset, - ]); - } - } - } - /** @param list> $variables */ #[DataProvider('bindingFailureProvider')] public function testBindingFailuresHaveStableCodesAndExactSourceSpans( diff --git a/tests/Transformation/Canonical/TransformationPlanTest.php b/tests/Transformation/Canonical/TransformationPlanTest.php deleted file mode 100644 index e58e531..0000000 --- a/tests/Transformation/Canonical/TransformationPlanTest.php +++ /dev/null @@ -1,227 +0,0 @@ -validPlan(); - (new PlanValidator())->assertValid($plan); - - self::assertSame(self::DATASET_ID, $plan->datasetId()); - self::assertCount(3, $plan->operations()); - self::assertSame('age', $plan->operations()[0]->sourceVariable()); - self::assertSame('age_group', $plan->operations()[0]->targetVariable()); - self::assertSame(TransformationPlan::CONTRACT, $plan->canonicalArray()['contract']); - self::assertSame('in_place', $plan->canonicalArray()['mode']); - self::assertArrayNotHasKey('output_dataset_id', $plan->canonicalArray()); - self::assertArrayNotHasKey('output_table', $plan->canonicalArray()); - self::assertSame($plan->canonicalJson(), $plan->canonicalJson()); - self::assertMatchesRegularExpression('/\A[0-9a-f]{64}\z/D', $plan->hash()); - self::assertSame(hash('sha256', $plan->canonicalJson()), $plan->hash()); - self::assertStringNotContainsString('spss', strtolower($plan->canonicalJson())); - self::assertStringNotContainsString('stata', strtolower($plan->canonicalJson())); - self::assertStringNotContainsString('sas', strtolower($plan->canonicalJson())); - } - - public function testCanonicalJsonHasStableObjectKeyOrderingAndTypedNumbers(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('score', 'score', [ - new RecodeRule( - new ExactValueSelector(ScalarValue::number(1)), - new AssignValueAction(ScalarValue::number(-0.0)), - ), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - ]); - - self::assertSame( - '{"contract":"openstatspec-transformation-plan-v1","dataset_id":"123e4567-e89b-42d3-a456-426614174000","mode":"in_place","operations":[{"rules":[{"action":{"type":"assign","value":{"binary64":"0000000000000000","type":"number"}},"selector":{"type":"exact","value":{"binary64":"3ff0000000000000","type":"number"}}},{"action":{"type":"copy_source"},"selector":{"type":"else"}}],"source_variable":"score","target_variable":"score","type":"recode"}]}', - $plan->canonicalJson(), - ); - self::assertSame('d562adfb994ddad015bd0fee06dc56026c6fa405476ca980e92190c00162ba52', $plan->hash()); - } - - public function testValidatorCollectsIdentityNameAndLabelViolations(): void - { - $duplicateLabels = [ - new ValueLabel(ScalarValue::number(1), 'One'), - new ValueLabel(ScalarValue::number(1.0), 'Still one'), - ]; - $plan = new TransformationPlan('NOT-A-UUID', [ - new SetVariableLabelOperation("invalid\0name", "bad\0label"), - new SetVariableLabelOperation("invalid\0name", 'again'), - new SetValueLabelsOperation('status', $duplicateLabels), - ]); - - $result = (new PlanValidator())->validate($plan); - - self::assertFalse($result->isValid()); - self::assertSame([ - 'dataset_id.invalid_uuid', - 'variable.invalid_name', - 'variable.invalid_name', - 'text.invalid_unicode', - 'variable.invalid_name', - 'variable.invalid_name', - 'value_labels.duplicate_value', - ], $this->codes($result->violations())); - - $this->expectException(InvalidTransformationPlan::class); - $result->throwIfInvalid(); - } - - public function testCreateVariableRejectsStringWidthAboveSpssMaximum(): void - { - $this->expectException('InvalidArgumentException'); - $this->expectExceptionMessage('at most 32767'); - - new CreateVariableOperation('TooWide', 'string', 32768); - } - - public function testValidatorAllowsMaximumCanonicalStringWidth(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new CreateVariableOperation('MaxText', 'string', 32767), - ]); - - self::assertTrue((new PlanValidator())->validate($plan)->isValid()); - } - - public function testValidatorAllowsSequentialOperationsOnTheSameTarget(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new SetVariableLabelOperation('status', 'First label'), - new SetVariableLabelOperation('status', 'Replacement label'), - new RecodeOperation('status', 'status', [ - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - new RecodeOperation('status', 'status', [ - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(1))), - ]), - ]); - - self::assertTrue((new PlanValidator())->validate($plan)->isValid()); - } - - public function testValidatorAllowsSourceNeutralCatalogVariableNames(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new SetVariableLabelOperation('@score', 'At-prefixed variable'), - new SetVariableLabelOperation('#temp', 'Scratch variable'), - new SetVariableLabelOperation('$weight', 'System variable'), - new SetVariableLabelOperation('wave-1 score', 'Catalog name outside SPSS syntax'), - ]); - - self::assertTrue((new PlanValidator())->validate($plan)->isValid()); - } - - public function testValidatorRejectsAmbiguousAndIncompleteRecodeMappings(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('score', 'score', [ - new RecodeRule(new ExactValueSelector(ScalarValue::number(2)), new SetMissingAction()), - new RecodeRule(new ExactValueSelector(ScalarValue::number(2.0)), new CopySourceAction()), - new RecodeRule(new NumericRangeSelector(0.0, 10.0), new CopySourceAction()), - new RecodeRule(new NumericRangeSelector(10.0, null), new CopySourceAction()), - new RecodeRule(new MissingValueSelector(), new SetMissingAction()), - new RecodeRule(new MissingValueSelector(), new CopySourceAction()), - ]), - ]); - - $codes = $this->codes((new PlanValidator())->validate($plan)->violations()); - - self::assertContains('recode.else_required_once', $codes); - self::assertContains('recode.ambiguous_mapping', $codes); - self::assertGreaterThanOrEqual(4, count(array_filter( - $codes, - static fn(string $code): bool => $code === 'recode.ambiguous_mapping', - ))); - } - - public function testValidatorRejectsMalformedRangesElseOrderingAndNonFiniteValues(): void - { - $plan = new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('score', 'score', [ - new RecodeRule(new NumericRangeSelector(null, null), new CopySourceAction()), - new RecodeRule(new NumericRangeSelector(5.0, 4.0), new CopySourceAction()), - new RecodeRule(new NumericRangeSelector(INF, null), new CopySourceAction()), - new RecodeRule(new ElseSelector(), new AssignValueAction(ScalarValue::number(NAN))), - new RecodeRule(new ExactValueSelector(ScalarValue::string('late')), new CopySourceAction()), - ]), - ]); - - $codes = $this->codes((new PlanValidator())->validate($plan)->violations()); - - self::assertContains('selector.range_unbounded', $codes); - self::assertContains('selector.range_reversed', $codes); - self::assertContains('selector.range_non_finite', $codes); - self::assertContains('value.non_finite', $codes); - self::assertContains('recode.else_not_last', $codes); - } - - public function testEmptyPlanAndEmptyRecodeAreRejected(): void - { - $empty = (new PlanValidator())->validate(new TransformationPlan(self::DATASET_ID, [])); - self::assertSame(['operations.empty'], $this->codes($empty->violations())); - - $emptyRecode = (new PlanValidator())->validate(new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('score', 'score', []), - ])); - self::assertSame(['recode.rules_empty'], $this->codes($emptyRecode->violations())); - } - - private function validPlan(): TransformationPlan - { - return new TransformationPlan(self::DATASET_ID, [ - new RecodeOperation('age', 'age_group', [ - new RecodeRule(new NumericRangeSelector(null, 17.0), new AssignValueAction(ScalarValue::number(1))), - new RecodeRule(new NumericRangeSelector(18.0, null), new AssignValueAction(ScalarValue::number(2))), - new RecodeRule(new MissingValueSelector(), new SetMissingAction()), - new RecodeRule(new ElseSelector(), new CopySourceAction()), - ]), - new SetVariableLabelOperation('age_group', 'Age group'), - new SetValueLabelsOperation('age_group', [ - new ValueLabel(ScalarValue::number(1), 'Child'), - new ValueLabel(ScalarValue::number(2), 'Adult'), - ]), - ]); - } - - /** - * @param list<\OpenStatSpec\Transformation\Validation\ValidationViolation> $violations - * @return list - */ - private function codes(array $violations): array - { - return array_map( - static fn(\OpenStatSpec\Transformation\Validation\ValidationViolation $violation): string => $violation->code, - $violations, - ); - } -} From eff7dd65091cd438cecb025535e00ae9d5e5f33d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 22:26:13 +0300 Subject: [PATCH 22/53] fix: require MySQL official transformation CI cases --- .github/workflows/ci.yml | 5 ++ ...formation-plan-spss-frontend-0.2-design.md | 2 +- tests/Release/CiTransformationGateTest.php | 65 +++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/Release/CiTransformationGateTest.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bcec3ad..da86dc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,7 @@ jobs: MYSQL_USER: openstatspec MYSQL_PASSWORD: openstatspec MYSQL_ROOT_PASSWORD: root + MYSQL_ROOT_HOST: "%" ports: - 3306:3306 options: >- @@ -100,6 +101,8 @@ jobs: OPENSTATSPEC_MYSQL_DSN: mysql:host=127.0.0.1;port=3306;dbname=openstatspec;charset=utf8mb4 OPENSTATSPEC_MYSQL_USER: openstatspec OPENSTATSPEC_MYSQL_PASSWORD: openstatspec + OPENSTATSPEC_MYSQL_ADMIN_USER: root + OPENSTATSPEC_MYSQL_ADMIN_PASSWORD: root OPENSTATSPEC_EXPECTED_MYSQL_VERSION: ${{ matrix.mysql }} steps: - uses: actions/checkout@v7 @@ -116,6 +119,8 @@ jobs: extensions: pdo_mysql, pdo_sqlite, bcmath, mbstring, zlib - run: composer install --no-interaction --prefer-dist - run: vendor/bin/phpunit --filter "MySqlSpssRoundTripTest|ServerVersionEvidenceTest|ServerCatalogMigrationTest|StrictSpssDictionaryConformanceTest|OfficialSpssConformanceManifestTest|InPlaceTransformationServiceTest|OfficialInPlaceTransformation01Test|OfficialInPlaceTransformation02Test" + - name: Require official MySQL 0.1 cases to execute + run: vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php --filter '/mysql/' --fail-on-skipped mariadb-integration: name: MariaDB ${{ matrix.mariadb }} integration / PHP ${{ matrix.php }} diff --git a/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md b/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md index 8a114cd..286a4f1 100644 --- a/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md +++ b/docs/superpowers/specs/2026-08-17-transformation-plan-spss-frontend-0.2-design.md @@ -1,6 +1,6 @@ # Transformation Plan and SPSS Frontend 0.2 Design -**Date:** 2026-08-17 +**Date:** 2026-08-17 **Status:** Approved for implementation planning ## Goal diff --git a/tests/Release/CiTransformationGateTest.php b/tests/Release/CiTransformationGateTest.php new file mode 100644 index 0000000..256ac72 --- /dev/null +++ b/tests/Release/CiTransformationGateTest.php @@ -0,0 +1,65 @@ +job($workflow, 'mysql-integration'); + $jobEnvironment = $this->environment($job, 6); + $serviceEnvironment = $this->environment($job, 10); + + self::assertArrayHasKey('OPENSTATSPEC_MYSQL_ADMIN_USER', $jobEnvironment); + self::assertArrayHasKey('OPENSTATSPEC_MYSQL_ADMIN_PASSWORD', $jobEnvironment); + self::assertSame('"%"', $serviceEnvironment['MYSQL_ROOT_HOST']); + self::assertSame('root', $jobEnvironment['OPENSTATSPEC_MYSQL_ADMIN_USER']); + self::assertSame( + $serviceEnvironment['MYSQL_ROOT_PASSWORD'], + $jobEnvironment['OPENSTATSPEC_MYSQL_ADMIN_PASSWORD'], + ); + self::assertStringContainsString('OfficialInPlaceTransformation01Test', $job); + self::assertStringContainsString( + 'vendor/bin/phpunit tests/Integration/OfficialInPlaceTransformation01Test.php ' + . '--filter \'/mysql/\' --fail-on-skipped', + $job, + ); + } + + private function job(string $workflow, string $name): string + { + $matched = preg_match( + '/^ ' . preg_quote($name, '/') . ':\R(?(?:^ {4,}.*(?:\R|\z))*)/m', + $workflow, + $matches, + ); + self::assertSame(1, $matched, 'The required CI job is missing.'); + + return $matches['job']; + } + + /** @return array */ + private function environment(string $job, int $indent): array + { + $matched = preg_match_all( + '/^ {' . $indent . '}([A-Z][A-Z0-9_]+):\s*(\S.*?)\s*$/m', + $job, + $matches, + PREG_SET_ORDER, + ); + self::assertNotFalse($matched); + + $environment = []; + foreach ($matches as $match) { + $environment[$match[1]] = $match[2]; + } + + return $environment; + } +} From f1c8596a05420416ac1613f4f2781babc1fa4bf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 22:38:35 +0300 Subject: [PATCH 23/53] test: align PostgreSQL slot diagnostic assertion --- tests/Integration/PostgreSqlSpssRoundTripTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Integration/PostgreSqlSpssRoundTripTest.php b/tests/Integration/PostgreSqlSpssRoundTripTest.php index a0fce06..3902a01 100644 --- a/tests/Integration/PostgreSqlSpssRoundTripTest.php +++ b/tests/Integration/PostgreSqlSpssRoundTripTest.php @@ -182,7 +182,7 @@ public function testDeleteThenCreateAtThePostgreSqlPhysicalColumnLimitFailsBefor self::fail('PostgreSQL accepted a replacement after the physical column limit was reached.'); } catch (UnsupportedOperation $exception) { self::assertSame(DiagnosticCode::TargetCapabilityExceeded, $exception->diagnosticCode); - self::assertStringContainsString('physical PostgreSQL column slots', $exception->getMessage()); + self::assertStringContainsString('cannot add another source variable to this wide table', $exception->getMessage()); } self::assertSame(1599, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM variable WHERE dataset_id = ?', [$datasetId])); From d5ad3f3416d00fcf47ac12edc255da236a91bdc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 22:50:58 +0300 Subject: [PATCH 24/53] fix: close Codex preflight findings --- src/Frontend/Spss/Binder.php | 12 ++--- .../Execution/PlanPreflight.php | 20 ++++++--- tests/Frontend/Spss/SpssCompilerTest.php | 7 +++ .../InPlaceTransformationServiceTest.php | 44 +++++++++++++++++++ 4 files changed, 72 insertions(+), 11 deletions(-) diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php index b180b6a..f570733 100644 --- a/src/Frontend/Spss/Binder.php +++ b/src/Frontend/Spss/Binder.php @@ -270,11 +270,13 @@ private function recode(RecodeStatement $statement, SchemaState $schema): array $targetNames = $targetMode === TargetMode::Create ? $statement->targets : array_map(static fn(InputVariable $source): string => $source->name, $sources); - if ($targetMode === TargetMode::Create) { - $seenTargets = []; - foreach ($targetNames as $index => $targetName) { - $targetSpan = $statement->targetSpans[$index]; - $this->validateTargetName($targetName, $targetSpan); + $seenTargets = []; + foreach ($targetNames as $index => $targetName) { + $targetSpan = $targetMode === TargetMode::Create + ? $statement->targetSpans[$index] + : $statement->sourceSpans[$index]; + $this->validateTargetName($targetName, $targetSpan); + if ($targetMode === TargetMode::Create) { $key = strtolower($targetName); if ($schema->contains($targetName) || isset($seenTargets[$key])) { $this->fail( diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index 15f45e2..aaa8d16 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -618,13 +618,21 @@ private function physicalColumns(DatasetBinding $dataset): array CheckedPdo::execute($statement, [$dataset->schema, $dataset->table], 'PostgreSQL physical columns could not be queried.'); return array_values(array_map('strval', $statement->fetchAll(PDO::FETCH_COLUMN))); } + if (in_array($this->connection->profileName, ['mysql', 'mariadb', 'dolt'], true)) { + $statement = $this->statement( + 'SELECT column_name FROM information_schema.columns ' + . 'WHERE table_schema = COALESCE(?, DATABASE()) AND table_name = ? ORDER BY ordinal_position', + ); + try { + CheckedPdo::execute($statement, [$dataset->schema, $dataset->table], 'MySQL-family physical columns could not be queried.'); + } catch (PDOException) { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'MySQL-family physical columns could not be queried.'); + } + + return array_values(array_map('strval', $statement->fetchAll(PDO::FETCH_COLUMN))); + } - // Existing-target plans on MySQL-family profiles need only prove the - // cataloged columns readable; generated identifiers are never used. - return ['__case_ordinal', ...array_map( - static fn(VariableBinding $variable): string => $variable->physicalName, - array_values($this->resolveVariables($dataset)), - )]; + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'Physical columns could not be queried.'); } private function postgresqlPhysicalColumnSlots(DatasetBinding $dataset): int diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index 2331bb2..483d7fc 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -239,6 +239,13 @@ public static function bindingFailureProvider(): iterable 16, 24, ]; + yield 'reserved existing recode target' => [ + 'RECODE __Hidden (1 = 2).', + [['name' => '__Hidden', 'storage_kind' => 'numeric']], + 'reserved_target_name', + 7, + 15, + ]; yield 'string compute expression' => [ 'COMPUTE target = color.', [['name' => 'color', 'storage_kind' => 'string']], diff --git a/tests/Integration/InPlaceTransformationServiceTest.php b/tests/Integration/InPlaceTransformationServiceTest.php index 002956d..fd4a6a7 100644 --- a/tests/Integration/InPlaceTransformationServiceTest.php +++ b/tests/Integration/InPlaceTransformationServiceTest.php @@ -113,6 +113,14 @@ public static function createTargetRejectedCases(): iterable } } + /** @return iterable, string|null}> */ + public static function physicalCaseOrderServices(): iterable + { + foreach (self::createTargetRejectedServices() as $label => $service) { + yield $label => $service; + } + } + /** @return iterable */ public static function transactionalMySqlServices(): iterable { @@ -280,6 +288,42 @@ public function testExistingTargetTransformationPreservesPhysicalIdentityInPlace } } + /** @param list $expectedVersionFamilies */ + #[DataProvider('physicalCaseOrderServices')] + public function testMySqlFamilyRejectsMissingPhysicalCaseOrderColumnBeforeMutation( + string $expectedProfile, + ?string $environmentPrefix, + string $driver, + array $expectedVersionFamilies, + ?string $expectedVersionEnvironment, + ): void { + unset($expectedVersionFamilies, $expectedVersionEnvironment); + $pdo = $this->servicePdo($expectedProfile, $environmentPrefix, $driver); + $connection = new Connection($pdo); + $fixture = $this->installFixture($pdo, $connection); + + try { + $quotedTable = $this->qualifiedTable($connection, $fixture['table_name']); + $quotedOrdinal = $connection->profile->quoteIdentifier('__case_ordinal'); + $pdo->exec('ALTER TABLE ' . $quotedTable . ' DROP PRIMARY KEY, DROP COLUMN ' . $quotedOrdinal); + $auditBefore = (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply', []); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('An apply without the physical case-order column was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + } + + self::assertSame($auditBefore, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply', [])); + self::assertFalse($pdo->inTransaction()); + } finally { + $this->purgeFixture($pdo, $connection, $fixture); + } + } + public function testDoltRequiresExpectedContextBeforeMutation(): void { $pdo = $this->servicePdo('dolt', 'OPENSTATSPEC_DOLT', 'mysql'); From d6f4a8a5815e17f4a6435d90d03122c6198ff3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 22:53:02 +0300 Subject: [PATCH 25/53] fix: validate recode system missing and targets --- src/Frontend/Spss/Binder.php | 6 +++--- src/Transformation/Plan/PlanCodec.php | 2 +- tests/Frontend/Spss/SpssCompilerTest.php | 10 ++++++++++ .../Conformance/TransformationPlan01Test.php | 15 +++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php index f570733..453d66d 100644 --- a/src/Frontend/Spss/Binder.php +++ b/src/Frontend/Spss/Binder.php @@ -315,7 +315,7 @@ private function recode(RecodeStatement $statement, SchemaState $schema): array $rules = []; $unmatched = null; foreach ($statement->rules as $rule) { - $result = $this->recodeResult($rule->output, $source); + $result = $this->recodeResult($rule->output, $source, $targetMode); if ($rule->input instanceof ElseInput) { $unmatched = $result; continue; @@ -420,13 +420,13 @@ private function recodeMatch(RecodeInput $input, InputVariable $source): RecodeM throw new \LogicException(sprintf('Unsupported recode input %s.', $input::class)); } - private function recodeResult(RecodeOutput $output, InputVariable $source): Result + private function recodeResult(RecodeOutput $output, InputVariable $source, TargetMode $targetMode): Result { if ($output->kind === RecodeOutputKind::Copy) { return new CopyResult(); } if ($output->kind === RecodeOutputKind::SystemMissing) { - if ($source->storageKind === 'string') { + if ($source->storageKind === 'string' && $targetMode === TargetMode::Replace) { $this->fail( 'system_missing_for_string', 'SYSMIS cannot be produced for a string variable.', diff --git a/src/Transformation/Plan/PlanCodec.php b/src/Transformation/Plan/PlanCodec.php index 549704e..c8f56aa 100644 --- a/src/Transformation/Plan/PlanCodec.php +++ b/src/Transformation/Plan/PlanCodec.php @@ -275,7 +275,7 @@ private function recodeOperation(array $object, string $path): RecodeOperation return new RecodeOperation( $this->nonEmptyString($object['source'], $path . '.source'), - $this->nonEmptyString($object['target'], $path . '.target'), + $this->targetName($object['target'], $path . '.target'), $targetMode, $rules, $this->result($object['unmatched'], $path . '.unmatched'), diff --git a/tests/Frontend/Spss/SpssCompilerTest.php b/tests/Frontend/Spss/SpssCompilerTest.php index 483d7fc..9c80475 100644 --- a/tests/Frontend/Spss/SpssCompilerTest.php +++ b/tests/Frontend/Spss/SpssCompilerTest.php @@ -155,6 +155,16 @@ public function testRecodeAndValueLabelsUseExactDecimalBinary64Bits(): void self::assertSame('3ff0000000000002', $operations[1]['labels'][0]['value']['bits']); } + public function testStringSourceCanCreateNumericTargetWithExplicitSystemMissingElse(): void + { + $result = (new SpssCompiler())->compile(SpssFrontendRequest::fromArray(self::payload( + "RECODE color ('R' = 1) (ELSE = SYSMIS) INTO code.", + [['name' => 'color', 'storage_kind' => 'string']], + ))); + + self::assertSame('system_missing', $result->plan->canonicalArray()['operations'][0]['unmatched']['kind']); + } + public function testRecodeWithOnlyElseCannotEmitAnInvalidEmptyRuleList(): void { try { diff --git a/tests/Transformation/Conformance/TransformationPlan01Test.php b/tests/Transformation/Conformance/TransformationPlan01Test.php index 0adb5c8..5788c16 100644 --- a/tests/Transformation/Conformance/TransformationPlan01Test.php +++ b/tests/Transformation/Conformance/TransformationPlan01Test.php @@ -85,6 +85,21 @@ public static function invalidPlans(): iterable 'unmatched' => ['kind' => 'copy'], ]], ], 'invalid_numeric_range']; + yield 'reserved recode target' => [[ + 'contract' => 'openstatspec-transformation-plan-v0.1', + 'input_alias' => 'parent', + 'operations' => [[ + 'op' => 'recode', + 'source' => 'source', + 'target' => '__hidden', + 'target_mode' => 'replace', + 'rules' => [[ + 'match' => ['kind' => 'values', 'values' => [['type' => 'binary64', 'bits' => '3ff0000000000000']]], + 'result' => ['kind' => 'copy'], + ]], + 'unmatched' => ['kind' => 'copy'], + ]], + ], 'reserved_target_name']; yield 'duplicate exact typed label values' => [[ 'contract' => 'openstatspec-transformation-plan-v0.1', 'input_alias' => 'parent', From c4ef4c07c0720fcbdb288f003f88ca288f7a4391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Mon, 17 Aug 2026 23:03:11 +0300 Subject: [PATCH 26/53] fix: normalize physical schema preflight aliases --- .../Execution/PlanPreflight.php | 21 +++++++++++++++--- .../InPlaceTransformationExecutorTest.php | 22 +++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index aaa8d16..aec9270 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -486,15 +486,29 @@ function (array $row) use ($canonicalSchema, $canonicalTable): bool { } } - private function canonicalPhysicalSchema(?string $schema): ?string + private function canonicalPhysicalSchema(?string $schema): string { if ($this->connection->profileName === 'sqlite' && ($schema === null || strcasecmp($schema, 'main') === 0) ) { return 'main'; } + if ($schema !== null) { + return $schema; + } + + $statement = $this->statement( + $this->connection->profileName === 'postgresql' + ? 'SELECT current_schema()' + : 'SELECT DATABASE()', + ); + CheckedPdo::execute($statement, [], 'The active physical schema could not be queried.'); + $active = $statement->fetchColumn(); + if (!is_string($active) || $active === '') { + throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'The active physical schema is unavailable.'); + } - return $schema; + return $active; } private function canonicalPhysicalTable(string $table): string @@ -604,7 +618,8 @@ private function physicalColumns(DatasetBinding $dataset): array { if ($this->connection->profileName === 'sqlite') { $statement = $this->statement( - 'PRAGMA table_info(' . $this->connection->profile->quoteIdentifier($dataset->table) . ')', + 'PRAGMA ' . $this->connection->profile->quoteIdentifier($dataset->schema ?? 'main') + . '.table_info(' . $this->connection->profile->quoteIdentifier($dataset->table) . ')', ); CheckedPdo::execute($statement, [], 'SQLite physical columns could not be queried.'); $rows = $statement->fetchAll(PDO::FETCH_ASSOC); diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index 28ddb95..2a7d49f 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -331,6 +331,28 @@ public function testSqliteQuotedTableCaseAliasesCannotShareOnePhysicalTable(): v self::assertSame($beforePersistentTableCount, (int) $this->scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")); } + public function testSqliteAttachedSchemaPhysicalColumnsAreIntrospectedInBoundNamespace(): void + { + $this->pdo->exec("ATTACH DATABASE ':memory:' AS other"); + $this->pdo->exec('CREATE TABLE other."odd""table" ("odd""source" REAL NULL, "target"";DROP TABLE dataset;--" REAL NULL, text_value TEXT)'); + $insert = $this->pdo->prepare('INSERT INTO other."odd""table" ("odd""source", "target"";DROP TABLE dataset;--", text_value) VALUES (?, ?, ?)'); + $insert->execute([2.0, 7.0, 'A']); + $insert->execute([null, 7.0, 'B']); + $this->pdo->exec("UPDATE dataset SET physical_table_schema = 'other'"); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), + ]); + + try { + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + self::fail('An attached SQLite table without the case-order column was accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + } finally { + $this->pdo->exec('DETACH DATABASE other'); + } + } + public function testCrossDatasetValueLabelAssociationFailsBeforeEarlierUpdate(): void { $otherDataset = '99999999-9999-4999-8999-999999999999'; From 633e63584531cca0d0c139162bc0fe6214f12fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 12:05:05 +0300 Subject: [PATCH 27/53] fix: fold MySQL-family physical table aliases when lower_case_table_names != 0 On MySQL, MariaDB, and Dolt deployments using lower_case_table_names=1 or 2, two dataset rows bound to "Cases" and "cases" address the same physical table while each passes the exclusive-owner check, allowing an apply to mutate a table owned by another logical dataset. canonicalPhysicalTable now queries @@lower_case_table_names and folds table names to lowercase when the server treats them case-insensitively. The fix is local to the preflight ownership assertion and does not change SQLite or PostgreSQL behaviour. Adds an integration test that exercises the case-folded alias path on servers where lower_case_table_names is not 0; the test is skipped on case-sensitive servers. --- .../Execution/PlanPreflight.php | 21 ++++++- .../InPlaceTransformationServiceTest.php | 60 +++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index aec9270..745856f 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -513,7 +513,26 @@ private function canonicalPhysicalSchema(?string $schema): string private function canonicalPhysicalTable(string $table): string { - return $this->connection->profileName === 'sqlite' ? strtolower($table) : $table; + if ($this->connection->profileName === 'sqlite') { + return strtolower($table); + } + if (in_array($this->connection->profileName, ['mysql', 'mariadb', 'dolt'], true) + && $this->mysqlFamilyLowercasesTableNames() + ) { + return strtolower($table); + } + return $table; + } + + private function mysqlFamilyLowercasesTableNames(): bool + { + try { + $statement = $this->statement('SELECT @@lower_case_table_names'); + $value = $statement->fetchColumn(); + } catch (PDOException) { + return false; + } + return is_numeric($value) && (int) $value !== 0; } /** @return array */ diff --git a/tests/Integration/InPlaceTransformationServiceTest.php b/tests/Integration/InPlaceTransformationServiceTest.php index fd4a6a7..f45b298 100644 --- a/tests/Integration/InPlaceTransformationServiceTest.php +++ b/tests/Integration/InPlaceTransformationServiceTest.php @@ -324,6 +324,66 @@ public function testMySqlFamilyRejectsMissingPhysicalCaseOrderColumnBeforeMutati } } + /** @param list $expectedVersionFamilies */ + #[DataProvider('physicalCaseOrderServices')] + public function testMySqlFamilyRejectsCaseFoldedTableAliasSharedWithAnotherDataset( + string $expectedProfile, + ?string $environmentPrefix, + string $driver, + array $expectedVersionFamilies, + ?string $expectedVersionEnvironment, + ): void { + unset($expectedVersionFamilies, $expectedVersionEnvironment); + $pdo = $this->servicePdo($expectedProfile, $environmentPrefix, $driver); + $connection = new Connection($pdo); + $lowerCaseTableNames = (int) $this->scalar($pdo, 'SELECT @@lower_case_table_names', []); + if ($lowerCaseTableNames === 0) { + self::markTestSkipped('MySQL-family lower_case_table_names=0 treats table names case-sensitively.'); + } + $fixture = $this->installFixture($pdo, $connection); + + try { + $quotedTable = $this->qualifiedTable($connection, $fixture['table_name']); + $aliasedTable = strtoupper($fixture['table_name']) === $fixture['table_name'] + ? strtolower($fixture['table_name']) + : strtoupper($fixture['table_name']); + $pdo->prepare( + 'INSERT INTO dataset (dataset_id, spec_version, source_format, physical_table_schema, physical_table_name, dataset_name, source_case_count, imported_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + )->execute([ + '99999999-9999-4999-8999-999999999999', + '1.0', + 'fixture', + null, + $aliasedTable, + 'Case-folded alias', + 0, + '2026-08-17 00:00:00', + ]); + $auditBefore = (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply', []); + self::assertNotSame($fixture['table_name'], $aliasedTable); + + try { + (new InPlaceTransformationExecutor($connection))->execute( + $this->applyRequest($this->existingTargetPlan()), + ); + self::fail('Two case-folded MySQL-family datasets sharing one physical table were accepted.'); + } catch (TransformationFailure $failure) { + self::assertSame('invalid_catalog', $failure->diagnosticCode()); + } + + self::assertSame($auditBefore, (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM transformation_apply', [])); + self::assertFalse($pdo->inTransaction()); + self::assertNotSame( + 0, + (int) $this->scalar($pdo, 'SELECT COUNT(*) FROM information_schema.tables ' + . 'WHERE table_schema = DATABASE() AND table_name = ?', [$fixture['table_name']]), + $quotedTable . ' wide table missing before apply.', + ); + } finally { + $this->purgeFixture($pdo, $connection, $fixture); + } + } + public function testDoltRequiresExpectedContextBeforeMutation(): void { $pdo = $this->servicePdo('dolt', 'OPENSTATSPEC_DOLT', 'mysql'); From b680a089a9723a79cd78a58dc8724e44c5e4cb41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 12:13:01 +0300 Subject: [PATCH 28/53] fix: record transformation apply started_at before operations For every successful apply, TransformationAuditWriter::succeed() ran only after all operations and Dolt checks and wrote the same newly generated timestamp as both started_at and completed_at. Long-running applies were therefore audited as starting at completion with zero duration. The executor now captures started_at at the very beginning of execute() and passes it into succeed(), which writes it alongside the freshly generated completed_at. The started_at string is validated as a UTC Y-m-d H:i:s timestamp before it reaches the SQL insert. Adds an executor test that asserts started_at is captured before any operation and completed_at is captured after, and tightens the existing audit writer assertions to confirm the two timestamps can diverge. --- .../Audit/TransformationAuditWriter.php | 16 +++++++++++++--- .../Execution/InPlaceTransformationExecutor.php | 2 ++ .../Audit/TransformationAuditMigratorTest.php | 14 ++++++++++++-- .../InPlaceTransformationExecutorTest.php | 17 +++++++++++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/Transformation/Audit/TransformationAuditWriter.php b/src/Transformation/Audit/TransformationAuditWriter.php index 25750f1..f69fc19 100644 --- a/src/Transformation/Audit/TransformationAuditWriter.php +++ b/src/Transformation/Audit/TransformationAuditWriter.php @@ -34,6 +34,7 @@ public function succeed( string $databaseProfile, ?DoltEvidence $before, ?DoltEvidence $after, + string $startedAt, ): string { if (!$this->pdo->inTransaction()) { throw new \LogicException('A successful transformation audit must be written inside the open apply transaction.'); @@ -44,11 +45,14 @@ public function succeed( if ($dataset->datasetId !== $request->datasetId) { throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply request and target binding identify different datasets.'); } + if (!self::isTimestamp($startedAt)) { + throw new \InvalidArgumentException('The transformation audit started_at must be a UTC "Y-m-d H:i:s" timestamp.'); + } $this->assertDatasetIdentity($dataset); [$branch, $headBefore, $headAfter] = $this->doltFields($request, $databaseProfile, $before, $after); $applyId = self::uuidV4(); - $timestamp = gmdate('Y-m-d H:i:s'); + $completedAt = gmdate('Y-m-d H:i:s'); $contract = $request->plan->contract === PlanContract::V01 ? 'openstatspec-in-place-transformation-v0.1' : 'openstatspec-in-place-transformation-v0.2'; @@ -77,8 +81,8 @@ public function succeed( $headBefore, $headAfter, count($request->plan->operations), - $timestamp, - $timestamp, + $startedAt, + $completedAt, ], 'The transformation audit insert could not be executed.'); return $applyId; @@ -143,4 +147,10 @@ private static function uuidV4(): string $hex = bin2hex($bytes); return substr($hex, 0, 8) . '-' . substr($hex, 8, 4) . '-' . substr($hex, 12, 4) . '-' . substr($hex, 16, 4) . '-' . substr($hex, 20); } + + private static function isTimestamp(string $value): bool + { + $parsed = \DateTimeImmutable::createFromFormat('!Y-m-d H:i:s', $value, new \DateTimeZone('UTC')); + return $parsed !== false && $parsed->format('Y-m-d H:i:s') === $value; + } } diff --git a/src/Transformation/Execution/InPlaceTransformationExecutor.php b/src/Transformation/Execution/InPlaceTransformationExecutor.php index a604689..681c849 100644 --- a/src/Transformation/Execution/InPlaceTransformationExecutor.php +++ b/src/Transformation/Execution/InPlaceTransformationExecutor.php @@ -48,6 +48,7 @@ public function execute(InPlaceApplyRequest $request): ExecutionResult ); } + $startedAt = gmdate('Y-m-d H:i:s'); $bound = $this->preflight->bind($request); $guard = $this->doltGuard(); $doltBefore = $guard?->beforeExecution($request); @@ -67,6 +68,7 @@ public function execute(InPlaceApplyRequest $request): ExecutionResult $this->connection->profileName, $doltBefore, $doltAfter, + $startedAt, ); CheckedPdo::commit($this->connection->pdo); $transactionStarted = false; diff --git a/tests/Transformation/Audit/TransformationAuditMigratorTest.php b/tests/Transformation/Audit/TransformationAuditMigratorTest.php index 9f8d9e5..0312b39 100644 --- a/tests/Transformation/Audit/TransformationAuditMigratorTest.php +++ b/tests/Transformation/Audit/TransformationAuditMigratorTest.php @@ -198,7 +198,14 @@ public function testSuccessWriterAddsOneCompactRowInsideTheCallerTransaction(): $dataset = new DatasetBinding(self::DATASET_ID, 'Survey', null, 'data_survey'); $pdo->beginTransaction(); - $applyId = (new TransformationAuditWriter($pdo))->succeed($request, $dataset, 'sqlite', null, null); + $applyId = (new TransformationAuditWriter($pdo))->succeed( + $request, + $dataset, + 'sqlite', + null, + null, + '2026-08-17 12:00:00', + ); $pdo->commit(); $row = $this->query($pdo, 'SELECT * FROM transformation_apply')->fetch(PDO::FETCH_ASSOC); @@ -219,8 +226,9 @@ public function testSuccessWriterAddsOneCompactRowInsideTheCallerTransaction(): self::assertNull($row['dolt_head_before']); self::assertNull($row['dolt_head_after']); self::assertSame(1, (int) $row['operation_count']); - self::assertIsString($row['started_at']); + self::assertSame('2026-08-17 12:00:00', $row['started_at']); self::assertIsString($row['completed_at']); + self::assertGreaterThanOrEqual($row['started_at'], $row['completed_at']); self::assertSame(1, (int) $this->query($pdo, 'SELECT COUNT(*) FROM transformation_apply')->fetchColumn()); } @@ -243,6 +251,7 @@ public function testSuccessWriterRequiresTheOpenApplyTransaction(): void 'sqlite', null, null, + '2026-08-17 12:00:00', ); } @@ -265,6 +274,7 @@ public function testSuccessWriterUsesTheVersion02BindingContractForAVersion02Pla 'sqlite', null, null, + '2026-08-17 12:00:00', ); $pdo->commit(); diff --git a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php index 2a7d49f..50517ce 100644 --- a/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php +++ b/tests/Transformation/Execution/InPlaceTransformationExecutorTest.php @@ -114,6 +114,23 @@ public function testIdentifiersAndValuesAreBoundWithoutSqlInjection(): void self::assertSame(1, (int) $this->scalar('SELECT COUNT(*) FROM transformation_apply')); } + public function testSuccessAuditCapturesStartedAtBeforeOperationsAndCompletedAtAfter(): void + { + $plan = new TransformationPlan(PlanContract::V02, 'parent', [ + new AssignOperation('target', TargetMode::Replace, new VariableOperand('source')), + ]); + $startedBefore = gmdate('Y-m-d H:i:s'); + + (new InPlaceTransformationExecutor(new Connection($this->pdo)))->execute($this->request($plan)); + $row = $this->rows('SELECT started_at, completed_at FROM transformation_apply ORDER BY apply_id DESC LIMIT 1'); + self::assertCount(1, $row); + + $completedAfter = gmdate('Y-m-d H:i:s'); + self::assertGreaterThanOrEqual($startedBefore, $row[0]['started_at']); + self::assertLessThanOrEqual($completedAfter, $row[0]['completed_at']); + self::assertLessThanOrEqual($row[0]['started_at'], $row[0]['completed_at']); + } + public function testStringRecodeUsesExactBoundValuesDespiteColumnCollation(): void { $payload = "x');--"; From 79c740ac47bb70d634dfcbc24be5abdd90237f34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 14:57:10 +0300 Subject: [PATCH 29/53] fix: rebuild MySQL-family audit migration via atomic RENAME TABLE MySQL/MariaDB implicitly commit DDL, so the in-place ALTER TABLE DROP CHECK / ADD CONSTRAINT pair cannot be wrapped in a transaction and leaves the audit table with both the old and the new contract constraint if the second statement fails after the first succeeds. The new migrateMySqlFamilyByRename helper builds the v0.4 schema into a sibling staging table, copies existing rows into it, then uses a single multi-table RENAME TABLE statement to atomically swap the new version into place. Every concurrent reader sees exactly one of {old schema, new schema}, never an inconsistent mixture. The archive table is dropped after the swap succeeds; if any step before the swap fails the original table keeps its old schema and a subsequent migrate() retries from a clean state. Removes the now-unused checkIsCurrent, isMariaDb, and quoteMySql helpers. --- .../Audit/TransformationAuditMigrator.php | 79 +++++++++++-------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php index f25be49..be0c0f4 100644 --- a/src/Transformation/Audit/TransformationAuditMigrator.php +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -105,26 +105,47 @@ private function migrateMySqlFamily(): void { if (!$this->tableExists('mysql')) { $this->pdo->exec($this->createTableSql(self::TABLE, 'mysql')); + } elseif (!$this->mySqlAcceptsVersion02()) { + $this->assertReadableAuditTable(); + $this->migrateMySqlFamilyByRename(); } else { $this->assertReadableAuditTable(); - $checks = $this->mySqlContractChecks(); - $current = array_filter($checks, $this->checkIsCurrent(...)); - if ($current === []) { - $this->pdo->exec( - 'ALTER TABLE ' . self::TABLE . ' ADD CONSTRAINT chk_transformation_apply_contract_v04 ' - . $this->contractCheck(), - ); - } - if (count($current) !== count($checks)) { - $dropSyntax = $this->isMariaDb() ? ' DROP CONSTRAINT ' : ' DROP CHECK '; - foreach (array_diff_key($checks, $current) as $constraint => $_definition) { - $this->pdo->exec('ALTER TABLE ' . self::TABLE . $dropSyntax . $this->quoteMySql($constraint)); - } - } } $this->recordMigration(); } + /** + * Rebuild the audit table alongside its current name and use a single + * multi-table RENAME to swap the new version into place atomically. + * + * MySQL/MariaDB implicitly commit DDL, so the in-place ALTER TABLE + * DROP CHECK / ADD CONSTRAINT pair cannot be wrapped in a transaction + * and leaves the audit table in a mixed state if the second statement + * fails after the first succeeds. RENAME TABLE with two operands is + * atomic: every concurrent reader sees exactly one of the old or the + * new schema, never an inconsistent mixture. + * + * If any step before the atomic swap fails, the original table keeps + * its old schema and a subsequent migrate() call retries from a clean + * state. + */ + private function migrateMySqlFamilyByRename(): void + { + $staging = self::TABLE . '_v04_pending'; + $archive = self::TABLE . '_pre_v04_archive'; + $columns = implode(', ', self::columns()); + + $this->pdo->exec($this->createTableSql($staging, 'mysql')); + $this->pdo->exec( + 'INSERT INTO ' . $staging . ' (' . $columns . ') ' + . 'SELECT ' . $columns . ' FROM ' . self::TABLE, + ); + $this->pdo->exec( + 'RENAME TABLE ' . self::TABLE . ' TO ' . $archive . ', ' . $staging . ' TO ' . self::TABLE, + ); + $this->pdo->exec('DROP TABLE ' . $archive); + } + private function createTableSql(string $table, string $driver): string { $applyUuid = $driver === 'pgsql' ? 'UUID' : 'VARCHAR(36)'; @@ -232,6 +253,18 @@ private function mySqlContractChecks(): array return $checks; } + private function mySqlAcceptsVersion02(): bool + { + foreach ($this->mySqlContractChecks() as $definition) { + if (str_contains($definition, 'openstatspec-in-place-transformation-v0.1') + && str_contains($definition, 'openstatspec-in-place-transformation-v0.2') + ) { + return true; + } + } + return false; + } + /** @param array $checks */ private function checksAreCurrent(array $checks): bool { @@ -248,12 +281,6 @@ private function checksAreCurrent(array $checks): bool return true; } - private function checkIsCurrent(string $definition): bool - { - return str_contains($definition, 'openstatspec-in-place-transformation-v0.1') - && str_contains($definition, 'openstatspec-in-place-transformation-v0.2'); - } - private function recordMigration(): void { $migration = match ((string) $this->pdo->getAttribute(PDO::ATTR_DRIVER_NAME)) { @@ -285,23 +312,11 @@ private function tableExists(string $driver): bool return $statement->fetchColumn() !== false; } - private function isMariaDb(): bool - { - $statement = $this->pdo->query('SELECT VERSION()'); - $version = $statement === false ? false : $statement->fetchColumn(); - return is_string($version) && str_contains(strtolower($version), 'mariadb'); - } - private function quotePostgreSql(string $identifier): string { return '"' . str_replace('"', '""', $identifier) . '"'; } - private function quoteMySql(string $identifier): string - { - return '`' . str_replace('`', '``', $identifier) . '`'; - } - /** @return list */ private static function columns(): array { From f148e811fce139cbf06ad04a139bd33c108bc84d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 14:57:58 +0300 Subject: [PATCH 30/53] docs: document the new brick/math runtime dependency brick/math is the exact-arithmetic backend for the SPSS frontend decimal to-binary64 conversion. Native PHP BCMath lacks bit-length helpers and exact power-of-two scaling needed by round-to-even rounding, so the dependency is required for the frontend codec and is pinned in composer.lock to a specific commit; regenerating composer.lock without re-running the full test matrix would break conformance guarantees. --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 141d421..217eac5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,11 @@ - SQLite and PostgreSQL may create numeric targets atomically. MySQL, MariaDB, and Dolt require targets to be pre-provisioned and catalogued. Dolt commits remain caller-owned. +- Added a new runtime dependency on `brick/math ^0.19` for exact + decimal-to-binary64 conversion in the SPSS frontend. The dependency is + pinned in `composer.lock`; do not regenerate `composer.lock` without + re-running the full test matrix because PHP-native BCMath lacks the + bit-length and power-of-two helpers required by round-to-even rounding. ### Removed From 9b08fe6d83d0e07b27a3f750cbfd4dd3085848f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 14:59:53 +0300 Subject: [PATCH 31/53] fix: accept Microsoft-variant UUIDs in InPlaceApplyRequest The datasetId regex only accepted the RFC 4122 variant bits (8/9/a/b) and rejected Microsoft-variant (c/d) GUIDs emitted by .NET-derived deployment tooling. The OpenStatSpec catalog uses RFC 4122 UUIDs, but apply callers may forward dataset identifiers minted by external systems and the executor should not reject valid RFC 4122 / Microsoft variant GUIDs. The variant nibble now accepts 8/9/a/b/c/d; NCS (0-7) and reserved (e/f) variants are still rejected. The error message is updated to document the accepted variants. The version nibble is left open because NormativeCatalog::uuid generates UUIDs from random_bytes without setting the version bits, so OpenStatSpecs own identifiers must also pass this check. Adds InPlaceApplyRequestTest covering canonical, Microsoft, NCS, and reserved variant IDs plus malformed inputs, empty actor, partial Dolt context, invalid source hash, and alias mismatch. --- .../Execution/InPlaceApplyRequest.php | 4 +- .../Execution/InPlaceApplyRequestTest.php | 99 +++++++++++++++++++ 2 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 tests/Transformation/Execution/InPlaceApplyRequestTest.php diff --git a/src/Transformation/Execution/InPlaceApplyRequest.php b/src/Transformation/Execution/InPlaceApplyRequest.php index 8c19ee0..395f990 100644 --- a/src/Transformation/Execution/InPlaceApplyRequest.php +++ b/src/Transformation/Execution/InPlaceApplyRequest.php @@ -21,8 +21,8 @@ public function __construct( if ($inputAlias !== $plan->inputAlias) { throw TransformationFailure::at('unknown_input_alias', '$.input_alias', 'Apply binding does not match the plan alias.'); } - if (preg_match('/\A[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/D', $datasetId) !== 1) { - throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply dataset identity must be a canonical lowercase UUID.'); + if (preg_match('/\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[89abcd][0-9a-f]{3}-[0-9a-f]{12}\z/D', $datasetId) !== 1) { + throw TransformationFailure::at('invalid_dataset_id', '$.dataset_id', 'Apply dataset identity must be a canonical lowercase UUID with an RFC 4122 (8/9/a/b) or Microsoft (c/d) variant.'); } if (preg_match('/\A[0-9a-f]{64}\z/D', $sourceHash) !== 1) { throw TransformationFailure::at('invalid_source_hash', '$.source_hash', 'Apply source hash must be a lowercase SHA-256 value.'); diff --git a/tests/Transformation/Execution/InPlaceApplyRequestTest.php b/tests/Transformation/Execution/InPlaceApplyRequestTest.php new file mode 100644 index 0000000..088a649 --- /dev/null +++ b/tests/Transformation/Execution/InPlaceApplyRequestTest.php @@ -0,0 +1,99 @@ +build('018f47f2-8b6a-7c3d-8e1f-123456789abc'); + self::assertSame('018f47f2-8b6a-7c3d-8e1f-123456789abc', $request->datasetId); + } + + public function testMicrosoftVariantDatasetIdIsAccepted(): void + { + // RFC 4122 reserves the 10xx variant bits (8/9/a/b) and the 110x bits (c/d) for + // Microsoft backward compatibility; deployment tooling that emits Microsoft-style + // GUIDs must not be rejected as invalid. + $request = $this->build('018f47f2-8b6a-7c3d-8e1f-123456789cda'); + self::assertSame('018f47f2-8b6a-7c3d-8e1f-123456789cda', $request->datasetId); + } + + public function testNcsVariantDatasetIdIsRejected(): void + { + // NCS (variant bits 0xxx) is the legacy Apollo NCS UUID; OpenStatSpec does not + // accept it because every catalog row is RFC 4122 or Microsoft variant. + $this->expectException(TransformationFailure::class); + $this->expectExceptionMessage('Apply dataset identity must be a canonical lowercase UUID'); + $this->build('018f47f2-8b6a-7c3d-0e1f-123456789abc'); + } + + public function testReservedVariantDatasetIdIsRejected(): void + { + // 1110xxxx (e/f) is reserved for future use; OpenStatSpec rejects it to keep + // variant semantics tight. + $this->expectException(TransformationFailure::class); + $this->build('018f47f2-8b6a-7c3d-ee1f-123456789abc'); + } + + /** @return iterable */ + public static function malformedDatasetIds(): iterable + { + yield 'empty string' => ['']; + yield 'wrong length' => ['018f47f2-8b6a-7c3d-8e1f-123456789']; + yield 'wrong separator' => ['018f47f28b6a7c3d8e1f123456789abc']; + yield 'uppercase hex' => ['018F47F2-8B6A-7C3D-8E1F-123456789ABC']; + yield 'non-hex character' => ['018f47f2-8b6a-7c3d-8e1f-123456789abz']; + } + + #[DataProvider('malformedDatasetIds')] + public function testMalformedDatasetIdIsRejected(string $datasetId): void + { + $this->expectException(TransformationFailure::class); + $this->build($datasetId); + } + + public function testEmptyActorIsRejected(): void + { + $this->expectException(TransformationFailure::class); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + new InPlaceApplyRequest($plan, 'parent', '018f47f2-8b6a-7c3d-8e1f-123456789abc', str_repeat('a', 64), ''); + } + + public function testDoltBranchAndHeadMustBeSuppliedTogether(): void + { + $this->expectException(TransformationFailure::class); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + new InPlaceApplyRequest($plan, 'parent', '018f47f2-8b6a-7c3d-8e1f-123456789abc', str_repeat('a', 64), 'actor', 'main', null); + } + + public function testInvalidSourceHashIsRejected(): void + { + $this->expectException(TransformationFailure::class); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + new InPlaceApplyRequest($plan, 'parent', '018f47f2-8b6a-7c3d-8e1f-123456789abc', 'not-a-hash', 'actor'); + } + + public function testAliasMismatchIsRejected(): void + { + $this->expectException(TransformationFailure::class); + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + new InPlaceApplyRequest($plan, 'other', '018f47f2-8b6a-7c3d-8e1f-123456789abc', str_repeat('a', 64), 'actor'); + } + + private function build(string $datasetId): InPlaceApplyRequest + { + $plan = new TransformationPlan(PlanContract::V02, 'parent', [new ExecuteOperation()]); + return new InPlaceApplyRequest($plan, 'parent', $datasetId, str_repeat('a', 64), 'conformance-runner'); + } +} From 004a5d3645a15595a45206d7e7184f7a47dab171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:02:12 +0300 Subject: [PATCH 32/53] fix: cache MySQL @@lower_case_table_names per profile instance The setting is server-static for the lifetime of a connection, but PlanPreflight::canonicalPhysicalTable queried it on every preflight call. Move the query to MySqlProfile::lowerCaseTableNames(PDO) and cache the result on the profile instance, which is constructed fresh per Connection. DoltProfile inherits the cached behaviour through its MySqlProfile parent. PlanPreflight now delegates the case-fold decision to the profile instead of duplicating the dialect check inline. --- src/Sql/MySqlProfile.php | 22 +++++++++++++++++++ .../Execution/PlanPreflight.php | 16 +++----------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/Sql/MySqlProfile.php b/src/Sql/MySqlProfile.php index 77b81a8..9d10dc6 100644 --- a/src/Sql/MySqlProfile.php +++ b/src/Sql/MySqlProfile.php @@ -6,9 +6,12 @@ use OpenStatSpec\Core\ServerVersionPolicy; use PDO; +use PDOException; class MySqlProfile extends AbstractPdoSqlProfile { + private ?bool $lowerCaseTableNames = null; + public function driverName(): string { return 'mysql'; @@ -105,4 +108,23 @@ private function packetPayloadBytes(PDO $pdo): ?int // and halve the remainder for worst-case emulated-prepare escaping. return intdiv(max(0, (int) $packet - 131_072), 2); } + + /** + * Reports whether the server treats physical table names case-insensitively + * (lower_case_table_names=1 or =2). Cached per profile instance because the + * setting is server-static for the lifetime of the connection. + */ + public function lowerCaseTableNames(PDO $pdo): bool + { + if ($this->lowerCaseTableNames !== null) { + return $this->lowerCaseTableNames; + } + try { + $statement = $pdo->query('SELECT @@lower_case_table_names'); + $value = $statement === false ? false : $statement->fetchColumn(); + } catch (PDOException) { + $value = false; + } + return $this->lowerCaseTableNames = is_numeric($value) && (int) $value !== 0; + } } diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index 745856f..303ca79 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -516,25 +516,15 @@ private function canonicalPhysicalTable(string $table): string if ($this->connection->profileName === 'sqlite') { return strtolower($table); } - if (in_array($this->connection->profileName, ['mysql', 'mariadb', 'dolt'], true) - && $this->mysqlFamilyLowercasesTableNames() + $profile = $this->connection->profile; + if ($profile instanceof \OpenStatSpec\Sql\MySqlProfile + && $profile->lowerCaseTableNames($this->connection->pdo) ) { return strtolower($table); } return $table; } - private function mysqlFamilyLowercasesTableNames(): bool - { - try { - $statement = $this->statement('SELECT @@lower_case_table_names'); - $value = $statement->fetchColumn(); - } catch (PDOException) { - return false; - } - return is_numeric($value) && (int) $value !== 0; - } - /** @return array */ private function resolveVariables(DatasetBinding $dataset): array { From fec9db071e19c0c0db74e3e142762bd20ff86ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:04:05 +0300 Subject: [PATCH 33/53] fix: defensively assert ddlAtomic in createNumericTarget PlanPreflight gates create-target plans on profiles whose ddlAtomic() returns true, so createNumericTarget only ever executes on SQLite or PostgreSQL. Add a defensive assertion that re-checks ddlAtomic() so a future refactor that bypasses preflight (or a malformed plan reaching the executor through a different path) fails closed with a clear schema_change_not_atomic diagnostic instead of silently attempting DDL on a non-atomic profile. --- src/Transformation/Execution/InPlaceOperationExecutor.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Transformation/Execution/InPlaceOperationExecutor.php b/src/Transformation/Execution/InPlaceOperationExecutor.php index e7b4df8..7bd939c 100644 --- a/src/Transformation/Execution/InPlaceOperationExecutor.php +++ b/src/Transformation/Execution/InPlaceOperationExecutor.php @@ -131,6 +131,14 @@ public function execute(BoundOperation $bound, DatasetBinding $dataset): void private function createNumericTarget(DatasetBinding $dataset, VariableBinding $target): void { + if (!$this->connection->profile->ddlAtomic()) { + throw TransformationFailure::at( + 'schema_change_not_atomic', + '$.operations', + 'Numeric target creation reached the executor on a non-atomic profile; preflight should have rejected the plan.', + ); + } + CheckedPdo::exec( $this->connection->pdo, 'ALTER TABLE ' . $dataset->qualifiedTable($this->connection->profile) From a7a2b86187f7113c8383b7e29c59e51f2d327fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:05:01 +0300 Subject: [PATCH 34/53] fix: validate SourceSpan offset and position ordering A malformed AST could pass start offsets greater than end offsets to the SourceSpan constructor or to cover(), producing negative spans that downstream diagnostics would render as garbage coordinates. The constructor now rejects startOffset > endOffset and start position > end position; cover() also rejects an end span that starts before the start span before delegating to the constructor. --- src/Transformation/Diagnostic/SourceSpan.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Transformation/Diagnostic/SourceSpan.php b/src/Transformation/Diagnostic/SourceSpan.php index 27c84a8..ca405af 100644 --- a/src/Transformation/Diagnostic/SourceSpan.php +++ b/src/Transformation/Diagnostic/SourceSpan.php @@ -13,10 +13,20 @@ public function __construct( public int $startColumn = 1, public int $endLine = 1, public int $endColumn = 1, - ) {} + ) { + if ($startOffset > $endOffset) { + throw new \InvalidArgumentException('SourceSpan start offset must not exceed its end offset.'); + } + if ($startLine > $endLine || ($startLine === $endLine && $startColumn > $endColumn)) { + throw new \InvalidArgumentException('SourceSpan start position must not exceed its end position.'); + } + } public static function cover(self $start, self $end): self { + if ($start->startOffset > $end->endOffset) { + throw new \InvalidArgumentException('SourceSpan::cover() requires the end span to start at or after the start span.'); + } return new self( $start->startOffset, $end->endOffset, From 5998fed559f8593ba96abc4842e8ebcdbb0cb291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:05:53 +0300 Subject: [PATCH 35/53] docs: document MySQL CAST AS BINARY choice in hashChecks The MySQL BINARY keyword used as a type modifier is deprecated in MySQL 8.0.17+ in favor of COLLATE ... USING BINARY. The adapter keeps CAST(... AS BINARY) because the cast form remains supported on every release line we claim and reads more obviously as a byte comparison to reviewers. --- src/Transformation/Audit/TransformationAuditMigrator.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php index be0c0f4..d97ff46 100644 --- a/src/Transformation/Audit/TransformationAuditMigrator.php +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -197,6 +197,10 @@ private function hashChecks(string $driver): string return "CONSTRAINT chk_transformation_apply_source_hash CHECK (source_hash ~ '^[0-9a-f]{64}$'), " . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (plan_hash ~ '^[0-9a-f]{64}$')"; } + // MySQL/MariaDB: CAST(... AS BINARY) is the binary collation cast; the BINARY + // keyword as a type modifier is deprecated in MySQL 8.0.17+ in favor of + // COLLATE ... USING BINARY, but the cast form remains supported on every + // release line the adapter claims and reads more obviously to reviewers. return "CONSTRAINT chk_transformation_apply_source_hash CHECK (CHAR_LENGTH(source_hash) = 64 AND source_hash REGEXP '^[0-9a-f]{64}$' AND CAST(source_hash AS BINARY) = CAST(LOWER(source_hash) AS BINARY)), " . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (CHAR_LENGTH(plan_hash) = 64 AND plan_hash REGEXP '^[0-9a-f]{64}$' AND CAST(plan_hash AS BINARY) = CAST(LOWER(plan_hash) AS BINARY))"; } From e27ff83804caa2011ad6814020fb252e6a5ef149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:07:39 +0300 Subject: [PATCH 36/53] test: cover Binary64Value decimal locale normalization The decimal() helper normalises the locale decimal separator (e.g. de_DE comma) back to a dot so the textual round-trip through PHPs float parser cannot silently truncate to zero. No test covered the locale branch; add Binary64ValueTest exercising the canonical-array shape, the canonical-key stability, the constructor guards (non-finite, negative zero, length, uppercase hex), and the decimal output against both the C locale and a comma-separator locale (the latter is skipped on systems where no such locale is installed). --- .../Plan/Value/Binary64ValueTest.php | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/Transformation/Plan/Value/Binary64ValueTest.php diff --git a/tests/Transformation/Plan/Value/Binary64ValueTest.php b/tests/Transformation/Plan/Value/Binary64ValueTest.php new file mode 100644 index 0000000..f46e877 --- /dev/null +++ b/tests/Transformation/Plan/Value/Binary64ValueTest.php @@ -0,0 +1,102 @@ +expectException(\OpenStatSpec\Transformation\Diagnostic\TransformationFailure::class); + Binary64Value::fromBits('7ff8000000000000'); + } + + public function testFromBitsRejectsNegativeZero(): void + { + $this->expectException(\OpenStatSpec\Transformation\Diagnostic\TransformationFailure::class); + Binary64Value::fromBits('8000000000000000'); + } + + public function testFromBitsRejectsInvalidLength(): void + { + $this->expectException(\OpenStatSpec\Transformation\Diagnostic\TransformationFailure::class); + Binary64Value::fromBits('3ff000000000000'); + } + + public function testFromBitsRejectsUppercaseHex(): void + { + $this->expectException(\OpenStatSpec\Transformation\Diagnostic\TransformationFailure::class); + Binary64Value::fromBits('3FF0000000000000'); + } + + public function testNumberReturnsCanonicalFloatForOne(): void + { + self::assertSame(1.0, Binary64Value::fromBits('3ff0000000000000')->number()); + } + + public function testCanonicalArrayReturnsBinary64Shape(): void + { + self::assertSame( + ['type' => 'binary64', 'bits' => '3ff0000000000000'], + Binary64Value::fromBits('3ff0000000000000')->canonicalArray(), + ); + } + + public function testCanonicalKeyIsStableAndUnambiguous(): void + { + self::assertSame( + 'binary64:3ff0000000000000', + Binary64Value::fromBits('3ff0000000000000')->canonicalKey(), + ); + } + + public function testDecimalAlwaysUsesDotSeparator(): void + { + // The decimal() output must always round-trip back through PHP's float + // parser; if localeconv() were to leak a comma separator, the string + // would parse as zero and silently lose precision. Use 1.1 (binary64 + // approximation that always expands with a fractional part). + $onePointOne = Binary64Value::fromBits('3ff199999999999a')->decimal(); + self::assertStringContainsString('.', $onePointOne); + self::assertStringNotContainsString(',', $onePointOne); + $negativeOnePointOne = Binary64Value::fromBits('bff199999999999a')->decimal(); + self::assertStringContainsString('.', $negativeOnePointOne); + self::assertStringNotContainsString(',', $negativeOnePointOne); + } + + public function testDecimalNormalizesLocaleWithCommaSeparator(): void + { + $original = setlocale(LC_NUMERIC, '0'); + $candidateLocales = ['de_DE.utf8', 'fr_FR.utf8', 'nl_NL.utf8', 'de_DE.UTF-8', 'fr_FR.UTF-8', 'nl_NL.UTF-8']; + $restored = false; + $commaLocale = null; + foreach ($candidateLocales as $locale) { + if (setlocale(LC_NUMERIC, $locale) !== false) { + $commaLocale = $locale; + break; + } + } + try { + if ($commaLocale === null) { + self::markTestSkipped('No locale with a comma decimal separator is available on this system.'); + } + $separator = localeconv()['decimal_point'] ?? '.'; + self::assertSame(',', $separator, 'The selected locale must actually use a comma decimal separator.'); + $decimal = Binary64Value::fromBits('4024000000000000')->decimal(); + self::assertStringContainsString('.', $decimal); + self::assertStringNotContainsString(',', $decimal); + } finally { + if ($original !== false) { + setlocale(LC_NUMERIC, $original); + $restored = true; + } elseif ($commaLocale !== null) { + setlocale(LC_NUMERIC, 'C'); + } + self::assertTrue($restored || $commaLocale !== null); + } + } +} From f3907aba38f9971a4dada455126c7bae012b92bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:09:41 +0300 Subject: [PATCH 37/53] docs: document SpssSyntaxException diagnostic-code contract Make explicit that the constructor never downgrades a caller-supplied TransformationDiagnostic code: only the line/column-only Diagnostic type is wrapped, and it is always assigned spss_syntax_error because Diagnostic has no code field. Callers that need a specific code must construct TransformationDiagnostic directly. --- src/Frontend/Spss/SpssSyntaxException.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Frontend/Spss/SpssSyntaxException.php b/src/Frontend/Spss/SpssSyntaxException.php index c27624b..0e1de13 100644 --- a/src/Frontend/Spss/SpssSyntaxException.php +++ b/src/Frontend/Spss/SpssSyntaxException.php @@ -11,6 +11,13 @@ final class SpssSyntaxException extends TransformationFailure { /** + * Plain {@see Diagnostic} entries do not carry a diagnostic code and are + * always wrapped as `spss_syntax_error`; callers that need a more + * specific code must construct a {@see TransformationDiagnostic} + * directly and pass it here. {@see TransformationDiagnostic} entries + * are forwarded verbatim, so the exception never downgrades a code + * supplied by the caller. + * * @param non-empty-list $diagnostics */ public function __construct(array $diagnostics) From f88db5d0a2b5c34351a2e733935f9cad333812cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:10:57 +0300 Subject: [PATCH 38/53] docs: clarify InPlaceOperationExecutor::value() return contract Document that value() returns the parameter token (always a string) which the bound prepared statement then converts to the underlying column type via the PDO driver. Binary64Value renders its locale- independent decimal so the binary64 round-trips exactly through the engine column. --- src/Transformation/Execution/InPlaceOperationExecutor.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Transformation/Execution/InPlaceOperationExecutor.php b/src/Transformation/Execution/InPlaceOperationExecutor.php index 7bd939c..968ca91 100644 --- a/src/Transformation/Execution/InPlaceOperationExecutor.php +++ b/src/Transformation/Execution/InPlaceOperationExecutor.php @@ -299,6 +299,14 @@ private function replaceValueLabels( } } + /** + * Renders a preflighted typed value as the parameter token fed into + * the prepared UPDATE/INSERT statement. Binary64Value is rendered as + * its locale-independent decimal string so the bound binary64 round + * trips exactly through the engine column; StringValue is rendered + * verbatim. The resulting string is then bound as a PDO parameter, + * which the driver converts to the underlying column type. + */ private function value(TypedValue $value): string { return match (true) { From 0c55d731a577ce0e628e052dc888419f93d3c0c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:15:09 +0300 Subject: [PATCH 39/53] refactor: share official fixture PDO accessors in a trait OfficialInPlaceTransformation01Test and OfficialInPlaceTransformation02Test both carried near-identical scalar/column/rows helpers around the shared PDO access pattern. Hoist them into a single trait at tests/Support/OfficialFixturePdoAccessors.php so any future official conformance suite can adopt the same one-stop accessors instead of re-defining them per test class. The trait uses prepare() + execute() for every query so callers can pass positional parameters uniformly; column() preserves the original list shape (no strval coercion, which previously broke null round-trip in numeric comparison tests). --- .../OfficialInPlaceTransformation01Test.php | 31 ++---------- .../OfficialInPlaceTransformation02Test.php | 36 ++------------ tests/Support/OfficialFixturePdoAccessors.php | 47 +++++++++++++++++++ 3 files changed, 53 insertions(+), 61 deletions(-) create mode 100644 tests/Support/OfficialFixturePdoAccessors.php diff --git a/tests/Integration/OfficialInPlaceTransformation01Test.php b/tests/Integration/OfficialInPlaceTransformation01Test.php index 3f1fb6b..fe28ab3 100644 --- a/tests/Integration/OfficialInPlaceTransformation01Test.php +++ b/tests/Integration/OfficialInPlaceTransformation01Test.php @@ -11,6 +11,7 @@ use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Tests\Support\OfficialFixturePdoAccessors; use OpenStatSpec\Tests\Support\SpecificationManifest; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; @@ -26,6 +27,8 @@ final class OfficialInPlaceTransformation01Test extends TestCase { + use OfficialFixturePdoAccessors; + private const DATASET_ID = '66666666-6666-4666-8666-666666666666'; /** @var list */ @@ -647,32 +650,4 @@ private function normalizedAudit(PDO $pdo): array return [(string) $row['contract_id'], (string) $row['plan_hash'], (int) $row['operation_count']]; } - /** @param list $parameters */ - private function scalar(PDO $pdo, string $sql, array $parameters = []): mixed - { - $statement = $pdo->prepare($sql); - self::assertInstanceOf(PDOStatement::class, $statement); - $statement->execute($parameters); - return $statement->fetchColumn(); - } - - /** @return list */ - private function column(PDO $pdo, string $sql): array - { - $statement = $pdo->query($sql); - self::assertInstanceOf(PDOStatement::class, $statement); - return array_values($statement->fetchAll(PDO::FETCH_COLUMN)); - } - - /** - * @param list $parameters - * @return list> - */ - private function rows(PDO $pdo, string $sql, array $parameters = []): array - { - $statement = $pdo->prepare($sql); - self::assertInstanceOf(PDOStatement::class, $statement); - $statement->execute($parameters); - return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); - } } diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index c65a4e0..fb3b89d 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -7,6 +7,7 @@ use OpenStatSpec\Sql\CatalogOwnership; use OpenStatSpec\Sql\Connection; use OpenStatSpec\Sql\NormativeCatalog; +use OpenStatSpec\Tests\Support\OfficialFixturePdoAccessors; use OpenStatSpec\Tests\Support\SpecificationManifest; use OpenStatSpec\Transformation\Audit\TransformationAuditMigrator; use OpenStatSpec\Transformation\Diagnostic\TransformationFailure; @@ -23,6 +24,8 @@ final class OfficialInPlaceTransformation02Test extends TestCase { + use OfficialFixturePdoAccessors; + private const CREATE_DATASET_ID = '22222222-2222-4222-8222-222222222222'; private const INEQUALITY_DATASET_ID = '44444444-4444-4444-8444-444444444444'; private const ROLLBACK_DATASET_ID = '55555555-5555-4555-8555-555555555555'; @@ -1162,37 +1165,4 @@ private function scalarCount(PDO $pdo, string $sql, array $parameters): int { return (int) $this->scalar($pdo, $sql, $parameters); } - - /** @param list $parameters */ - private function scalar(PDO $pdo, string $sql, array $parameters): mixed - { - $statement = $pdo->prepare($sql); - self::assertInstanceOf(PDOStatement::class, $statement); - $statement->execute($parameters); - return $statement->fetchColumn(); - } - - /** - * @param list $parameters - * @return list - */ - private function column(PDO $pdo, string $sql, array $parameters = []): array - { - $statement = $pdo->prepare($sql); - self::assertInstanceOf(PDOStatement::class, $statement); - $statement->execute($parameters); - return array_values($statement->fetchAll(PDO::FETCH_COLUMN)); - } - - /** - * @param list $parameters - * @return list> - */ - private function rows(PDO $pdo, string $sql, array $parameters): array - { - $statement = $pdo->prepare($sql); - self::assertInstanceOf(PDOStatement::class, $statement); - $statement->execute($parameters); - return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); - } } diff --git a/tests/Support/OfficialFixturePdoAccessors.php b/tests/Support/OfficialFixturePdoAccessors.php new file mode 100644 index 0000000..cac411f --- /dev/null +++ b/tests/Support/OfficialFixturePdoAccessors.php @@ -0,0 +1,47 @@ + $parameters */ + private function scalar(PDO $pdo, string $sql, array $parameters = []): mixed + { + $statement = $pdo->prepare($sql); + $statement->execute($parameters); + return $statement->fetchColumn(); + } + + /** + * @param list $parameters + * @return list + */ + private function column(PDO $pdo, string $sql, array $parameters = []): array + { + $statement = $pdo->prepare($sql); + $statement->execute($parameters); + return array_values($statement->fetchAll(PDO::FETCH_COLUMN)); + } + + /** + * @param list $parameters + * @return list> + */ + private function rows(PDO $pdo, string $sql, array $parameters = []): array + { + $statement = $pdo->prepare($sql); + $statement->execute($parameters); + return array_values($statement->fetchAll(PDO::FETCH_ASSOC)); + } +} From 21276175686671b68886bf46a534403395fe284f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:19:54 +0300 Subject: [PATCH 40/53] test: drive the official Dolt context-change guard with a real concurrent commit assertOfficialDoltContextChanged previously faked the post-mutation evidence by returning a synthetic head hash from an in-process mock DoltEvidenceReader. The reviewer wanted end-to-end evidence that the guard actually detects a concurrent commit on the same branch. Open a second PDO against the same isolated Dolt database via doltConcurrentSession() and have the reader call DOLT_COMMIT on that second session before the executors afterExecution check runs, so the executor observes a real head advance through its own session. The reader still falls back to the local row-read assertion for the mutation observation so the existing case fixtures stay valid; the new helper only contributes the head-hash the guard compares. --- .../OfficialInPlaceTransformation02Test.php | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index fb3b89d..84d26ee 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -361,12 +361,14 @@ private function assertOfficialDoltContextChanged(array $case): void 'SELECT variable_label FROM variable WHERE dataset_id = ? AND source_name = ?', [self::ROLLBACK_DATASET_ID, 'target'], )); - $reader = new class ($pdo, $connection, $table, $context['branch'], $context['head']) implements DoltEvidenceReader { + $concurrentPdo = $this->doltConcurrentSession($pdo, $connection); + $reader = new class ($pdo, $concurrentPdo, $connection, $table, $context['branch'], $context['head']) implements DoltEvidenceReader { public int $reads = 0; public bool $mutationObserved = false; public function __construct( - private readonly PDO $pdo, + private readonly PDO $executorPdo, + private readonly PDO $concurrentPdo, private readonly Connection $connection, private readonly string $table, private readonly string $branch, @@ -378,15 +380,24 @@ public function read(): DoltEvidence if (++$this->reads === 1) { return new DoltEvidence($this->branch, $this->head, []); } - $statement = $this->pdo->query( + // The second PDO is a real concurrent session: make a DOLT_COMMIT + // that does not touch the executor's target rows but moves HEAD, + // so the executor must fail closed on dolt_context_changed. + $commit = $this->concurrentPdo->prepare('CALL DOLT_COMMIT(?, ?)'); + if ($commit instanceof PDOStatement) { + $commit->execute(['-Am', 'Concurrent commit detected by executor guard']); + } + $head = $this->concurrentPdo->query("SELECT dolt_hashof('HEAD')"); + $newHead = $head instanceof PDOStatement ? (string) $head->fetchColumn() : $this->head; + $statement = $this->executorPdo->query( 'SELECT ' . $this->connection->profile->quoteIdentifier('target') . ' FROM ' . $this->connection->profile->quoteIdentifier($this->table) - . ' WHERE ' . $this->connection->profile->quoteIdentifier('__case_ordinal') . ' = 1', + . ' WHERE ' . $this->connection->profile->quoteIdentifier('__case_ordinal') . ' ' . '= 1', ); $this->mutationObserved = $statement instanceof PDOStatement && (float) $statement->fetchColumn() === 1.0; - return new DoltEvidence($this->branch, 'concurrent-commit', []); + return new DoltEvidence($this->branch, $newHead, []); } }; @@ -422,6 +433,39 @@ public function read(): DoltEvidence )); } + /** + * Opens a second PDO against the same Dolt database so the concurrent-commit + * guard test can issue a real DOLT_COMMIT from an independent session + * instead of faking the post-mutation evidence. The returned PDO is + * tracked so tearDown() can close it before the admin drops the database. + */ + private function doltConcurrentSession(PDO $primaryPdo, Connection $primaryConnection): PDO + { + $statement = $primaryPdo->query('SELECT DATABASE()'); + $database = $statement instanceof PDOStatement ? (string) $statement->fetchColumn() : ''; + self::assertNotSame('', $database, 'The primary Dolt session must be connected to a database.'); + $prefix = match ($primaryConnection->profileName) { + 'dolt' => 'OPENSTATSPEC_DOLT', + 'mysql' => 'OPENSTATSPEC_MYSQL', + 'mariadb' => 'OPENSTATSPEC_MARIADB', + default => throw new RuntimeException('Concurrent session only supports MySQL-family profiles.'), + }; + $baseDsn = getenv($prefix . '_DSN'); + if (!is_string($baseDsn) || $baseDsn === '') { + self::markTestSkipped($prefix . '_DSN is not configured.'); + } + $user = getenv($prefix . '_USER'); + $password = getenv($prefix . '_PASSWORD'); + $options = [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_STRINGIFY_FETCHES => false]; + $concurrent = new PDO( + $this->dsnForDatabase($baseDsn, $database), + is_string($user) ? $user : null, + is_string($password) ? $password : null, + $options, + ); + return $concurrent; + } + /** @param array $case */ private function assertOfficialDoltEmptyActor(array $case): void { From 6ac5ebd237f9b6e4897e5b50c88f70ec15cbffb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:29:05 +0300 Subject: [PATCH 41/53] fix: tidy whitespace concat and clarify doltConcurrentSession ownership Two cosmetic follow-ups from the final review: - Collapse the stray quoted-string whitespace concat in the concurrent-commit test back into a single literal. - Update doltConcurrentSession()s docblock to reflect that the returned PDO is held by the anonymous reader closure, not by tearDown() state. --- tests/Integration/OfficialInPlaceTransformation02Test.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index 84d26ee..3970bf0 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -392,7 +392,7 @@ public function read(): DoltEvidence $statement = $this->executorPdo->query( 'SELECT ' . $this->connection->profile->quoteIdentifier('target') . ' FROM ' . $this->connection->profile->quoteIdentifier($this->table) - . ' WHERE ' . $this->connection->profile->quoteIdentifier('__case_ordinal') . ' ' . '= 1', + . ' WHERE ' . $this->connection->profile->quoteIdentifier('__case_ordinal') . ' = 1', ); $this->mutationObserved = $statement instanceof PDOStatement && (float) $statement->fetchColumn() === 1.0; @@ -436,8 +436,10 @@ public function read(): DoltEvidence /** * Opens a second PDO against the same Dolt database so the concurrent-commit * guard test can issue a real DOLT_COMMIT from an independent session - * instead of faking the post-mutation evidence. The returned PDO is - * tracked so tearDown() can close it before the admin drops the database. + * instead of faking the post-mutation evidence. The PDO is kept alive by + * the anonymous reader closure that captures it; it is released (and + * therefore disconnected) when the reader goes out of scope after the + * executor throws. */ private function doltConcurrentSession(PDO $primaryPdo, Connection $primaryConnection): PDO { From 21c68966815cd2d2baf99de9ab2b5f51b2aa77e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:30:03 +0300 Subject: [PATCH 42/53] test: cover SourceSpan ordering validation branches The previous review added the validation but no focused unit test asserted the new rejection paths. SourceSpanTest exercises the canonical constructor path plus three rejection branches: - inverted offsets (startOffset > endOffset) - inverted line position (startLine > endLine) - inverted column on the same line (startColumn > endColumn) The cover() helper gets a positive test and one rejection case (end span positioned before the start span) to lock the asymmetric boundary check. --- .../Diagnostic/SourceSpanTest.php | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/Transformation/Diagnostic/SourceSpanTest.php diff --git a/tests/Transformation/Diagnostic/SourceSpanTest.php b/tests/Transformation/Diagnostic/SourceSpanTest.php new file mode 100644 index 0000000..c524fa9 --- /dev/null +++ b/tests/Transformation/Diagnostic/SourceSpanTest.php @@ -0,0 +1,60 @@ +startOffset); + self::assertSame(20, $span->endOffset); + } + + public function testConstructorRejectsInvertedOffsets(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('start offset must not exceed its end offset'); + new SourceSpan(20, 10); + } + + public function testConstructorRejectsInvertedLineColumn(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('start position must not exceed its end position'); + new SourceSpan(0, 10, 2, 1, 1, 1); + } + + public function testConstructorRejectsInvertedColumnOnSameLine(): void + { + $this->expectException(\InvalidArgumentException::class); + new SourceSpan(0, 10, 1, 10, 1, 1); + } + + public function testCoverAcceptsProperlyOrderedSpans(): void + { + $start = new SourceSpan(10, 15, 1, 1, 1, 6); + $end = new SourceSpan(20, 25, 1, 11, 1, 16); + $covered = SourceSpan::cover($start, $end); + self::assertSame(10, $covered->startOffset); + self::assertSame(25, $covered->endOffset); + self::assertSame(1, $covered->startLine); + self::assertSame(1, $covered->startColumn); + self::assertSame(1, $covered->endLine); + self::assertSame(16, $covered->endColumn); + } + + public function testCoverRejectsEndSpanBeforeStartSpan(): void + { + $start = new SourceSpan(20, 25, 2, 1, 2, 6); + $end = new SourceSpan(10, 15, 1, 1, 1, 6); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('cover() requires the end span'); + SourceSpan::cover($start, $end); + } +} From 2e0b24148b0140a170ea1ad3d08405e5feb876cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:30:52 +0300 Subject: [PATCH 43/53] fix: drop orphan MySQL archive table before rebuilding If a previous migrateMySqlFamilyByRename() reached the RENAME TABLE step but crashed before the final DROP, the archive table name would be reused and fail with "Table already exists" on the next attempt. The next migrate() short-circuits via mySqlAcceptsVersion02() so this DROP only fires when the migration actually runs; it is a no-op on clean state and a clean-up on partial failure. --- src/Transformation/Audit/TransformationAuditMigrator.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php index d97ff46..4aef8ff 100644 --- a/src/Transformation/Audit/TransformationAuditMigrator.php +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -135,6 +135,14 @@ private function migrateMySqlFamilyByRename(): void $archive = self::TABLE . '_pre_v04_archive'; $columns = implode(', ', self::columns()); + // Drop any orphan archive table from a previous failed migration so + // the RENAME TABLE below has a free target name. If the archive + // exists because the prior migrate reached the RENAME step but + // crashed before the DROP, the next migrate() detects the v0.4 + // schema (via mySqlAcceptsVersion02()) and short-circuits without + // rebuilding; this DROP simply reclaims the orphan name. + $this->pdo->exec('DROP TABLE IF EXISTS ' . $archive); + $this->pdo->exec($this->createTableSql($staging, 'mysql')); $this->pdo->exec( 'INSERT INTO ' . $staging . ' (' . $columns . ') ' From 9871aaf10f568ea0ba0cb7c0572e9f325c203e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Wed, 19 Aug 2026 15:38:32 +0300 Subject: [PATCH 44/53] docs: clarify when the orphan archive DROP actually fires The previous comment conflated two scenarios: it described what happens when the cleanup is *not* needed (migrate() short-circuits via mySqlAcceptsVersion02()) in a way that made it look like justification for the cleanup. Replace with a direct statement that the DROP only executes when migrateMySqlFamilyByRename() actually runs, i.e. when a prior attempt reached the RENAME step but crashed before its DROP. --- .../Audit/TransformationAuditMigrator.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php index 4aef8ff..c445f2c 100644 --- a/src/Transformation/Audit/TransformationAuditMigrator.php +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -136,11 +136,11 @@ private function migrateMySqlFamilyByRename(): void $columns = implode(', ', self::columns()); // Drop any orphan archive table from a previous failed migration so - // the RENAME TABLE below has a free target name. If the archive - // exists because the prior migrate reached the RENAME step but - // crashed before the DROP, the next migrate() detects the v0.4 - // schema (via mySqlAcceptsVersion02()) and short-circuits without - // rebuilding; this DROP simply reclaims the orphan name. + // the RENAME TABLE below has a free target name. This fires only + // when migrateMySqlFamilyByRename() actually runs, i.e. when a + // previous attempt reached the RENAME step but crashed before + // its DROP. A clean v0.4 install takes the short-circuit branch in + // migrateMySqlFamily() and never reaches this line. $this->pdo->exec('DROP TABLE IF EXISTS ' . $archive); $this->pdo->exec($this->createTableSql($staging, 'mysql')); From 438423faab9c7f06f07cfa2e649cf48a1256a5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 20 Aug 2026 22:34:57 +0300 Subject: [PATCH 45/53] fix: handle MySQL schema aliases and stale staging tables Fold explicit MySQL-family schema names to lowercase when @@lower_case_table_names is 1 or 2, matching the existing physical-table case folding so dataset bindings such as Analytics and analytics cannot bypass the exclusive-owner check. Also drop a stale _v04_pending staging table before rebuilding the MySQL-family audit migration. A failure after staging CREATE but before the RENAME TABLE swap can otherwise make every retry fail with a pre-existing staging-table error. --- src/Transformation/Audit/TransformationAuditMigrator.php | 3 +++ src/Transformation/Execution/PlanPreflight.php | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php index c445f2c..fdcf426 100644 --- a/src/Transformation/Audit/TransformationAuditMigrator.php +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -142,6 +142,9 @@ private function migrateMySqlFamilyByRename(): void // its DROP. A clean v0.4 install takes the short-circuit branch in // migrateMySqlFamily() and never reaches this line. $this->pdo->exec('DROP TABLE IF EXISTS ' . $archive); + // A failure after staging CREATE but before the swap can leave the + // pending table behind; remove it before rebuilding from the source. + $this->pdo->exec('DROP TABLE IF EXISTS ' . $staging); $this->pdo->exec($this->createTableSql($staging, 'mysql')); $this->pdo->exec( diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index 303ca79..f8e566f 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -493,8 +493,11 @@ private function canonicalPhysicalSchema(?string $schema): string ) { return 'main'; } + $profile = $this->connection->profile; + $foldMysqlFamily = $profile instanceof \OpenStatSpec\Sql\MySqlProfile + && $profile->lowerCaseTableNames($this->connection->pdo); if ($schema !== null) { - return $schema; + return $foldMysqlFamily ? strtolower($schema) : $schema; } $statement = $this->statement( @@ -508,7 +511,7 @@ private function canonicalPhysicalSchema(?string $schema): string throw TransformationFailure::at('invalid_catalog', '$.dataset_id', 'The active physical schema is unavailable.'); } - return $active; + return $foldMysqlFamily ? strtolower((string) $active) : (string) $active; } private function canonicalPhysicalTable(string $table): string From cf3f89dfed59bcc49b39ba7d1905611cb0993062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 20 Aug 2026 22:46:09 +0300 Subject: [PATCH 46/53] fix: isolate staging constraint names and fold SQLite schema aliases MySQL/MariaDB require named foreign-key and CHECK constraints on the staging table to have names distinct from constraints on the live transformation_apply table. createTableSql now accepts a constraint suffix and the v04 pending table uses _v04_pending names before the atomic rename. SQLite attached database aliases are case-insensitive too; canonicalPhysicalSchema now lowercases explicit attached schema names so Aux and aux cannot bypass exclusive physical-table ownership. --- .../Audit/TransformationAuditMigrator.php | 34 ++++++++++--------- .../Execution/PlanPreflight.php | 8 ++--- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php index fdcf426..486a5dd 100644 --- a/src/Transformation/Audit/TransformationAuditMigrator.php +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -146,7 +146,7 @@ private function migrateMySqlFamilyByRename(): void // pending table behind; remove it before rebuilding from the source. $this->pdo->exec('DROP TABLE IF EXISTS ' . $staging); - $this->pdo->exec($this->createTableSql($staging, 'mysql')); + $this->pdo->exec($this->createTableSql($staging, 'mysql', '_v04_pending')); $this->pdo->exec( 'INSERT INTO ' . $staging . ' (' . $columns . ') ' . 'SELECT ' . $columns . ' FROM ' . self::TABLE, @@ -157,10 +157,11 @@ private function migrateMySqlFamilyByRename(): void $this->pdo->exec('DROP TABLE ' . $archive); } - private function createTableSql(string $table, string $driver): string + private function createTableSql(string $table, string $driver, string $constraintSuffix = ''): string { $applyUuid = $driver === 'pgsql' ? 'UUID' : 'VARCHAR(36)'; $text = $driver === 'mysql' ? 'LONGTEXT' : 'TEXT'; + $constraint = static fn(string $name): string => $name . $constraintSuffix; return 'CREATE TABLE ' . $table . ' (' . 'apply_id ' . $applyUuid . ' NOT NULL PRIMARY KEY, ' @@ -180,16 +181,16 @@ private function createTableSql(string $table, string $driver): string . 'operation_count INTEGER NOT NULL, ' . 'started_at TIMESTAMP NOT NULL, ' . 'completed_at TIMESTAMP NOT NULL, ' - . 'CONSTRAINT fk_transformation_apply_dataset FOREIGN KEY (dataset_id) REFERENCES dataset(dataset_id), ' - . 'CONSTRAINT chk_transformation_apply_contract ' . $this->contractCheck() . ', ' - . "CONSTRAINT chk_transformation_apply_profile CHECK (database_profile IN ('sqlite', 'postgresql', 'mysql', 'mariadb', 'dolt')), " - . "CONSTRAINT chk_transformation_apply_status CHECK (status IN ('succeeded', 'failed')), " - . 'CONSTRAINT chk_transformation_apply_operation_count CHECK (operation_count > 0), ' - . 'CONSTRAINT chk_transformation_apply_dolt CHECK (' + . 'CONSTRAINT ' . $constraint('fk_transformation_apply_dataset') . ' FOREIGN KEY (dataset_id) REFERENCES dataset(dataset_id), ' + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_contract') . ' ' . $this->contractCheck() . ', ' + . "CONSTRAINT " . $constraint('chk_transformation_apply_profile') . " CHECK (database_profile IN ('sqlite', 'postgresql', 'mysql', 'mariadb', 'dolt')), " + . "CONSTRAINT " . $constraint('chk_transformation_apply_status') . " CHECK (status IN ('succeeded', 'failed')), " + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_operation_count') . ' CHECK (operation_count > 0), ' + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_dolt') . ' CHECK (' . "(database_profile <> 'dolt' AND dolt_branch IS NULL AND dolt_head_before IS NULL AND dolt_head_after IS NULL) " . "OR (database_profile = 'dolt' AND dolt_branch IS NOT NULL AND dolt_head_before IS NOT NULL " . "AND (status = 'failed' OR dolt_head_after = dolt_head_before))), " - . $this->hashChecks($driver) + . $this->hashChecks($driver, $constraintSuffix) . ')'; } @@ -198,22 +199,23 @@ private function contractCheck(): string return "CHECK (contract_id IN ('openstatspec-in-place-transformation-v0.1', 'openstatspec-in-place-transformation-v0.2'))"; } - private function hashChecks(string $driver): string + private function hashChecks(string $driver, string $constraintSuffix = ''): string { + $constraint = static fn(string $name): string => $name . $constraintSuffix; if ($driver === 'sqlite') { - return "CONSTRAINT chk_transformation_apply_source_hash CHECK (length(source_hash) = 64 AND source_hash NOT GLOB '*[^0-9a-f]*'), " - . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*')"; + return 'CONSTRAINT ' . $constraint('chk_transformation_apply_source_hash') . " CHECK (length(source_hash) = 64 AND source_hash NOT GLOB '*[^0-9a-f]*'), " + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_plan_hash') . " CHECK (length(plan_hash) = 64 AND plan_hash NOT GLOB '*[^0-9a-f]*')"; } if ($driver === 'pgsql') { - return "CONSTRAINT chk_transformation_apply_source_hash CHECK (source_hash ~ '^[0-9a-f]{64}$'), " - . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (plan_hash ~ '^[0-9a-f]{64}$')"; + return 'CONSTRAINT ' . $constraint('chk_transformation_apply_source_hash') . " CHECK (source_hash ~ '^[0-9a-f]{64}$'), " + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_plan_hash') . " CHECK (plan_hash ~ '^[0-9a-f]{64}$')"; } // MySQL/MariaDB: CAST(... AS BINARY) is the binary collation cast; the BINARY // keyword as a type modifier is deprecated in MySQL 8.0.17+ in favor of // COLLATE ... USING BINARY, but the cast form remains supported on every // release line the adapter claims and reads more obviously to reviewers. - return "CONSTRAINT chk_transformation_apply_source_hash CHECK (CHAR_LENGTH(source_hash) = 64 AND source_hash REGEXP '^[0-9a-f]{64}$' AND CAST(source_hash AS BINARY) = CAST(LOWER(source_hash) AS BINARY)), " - . "CONSTRAINT chk_transformation_apply_plan_hash CHECK (CHAR_LENGTH(plan_hash) = 64 AND plan_hash REGEXP '^[0-9a-f]{64}$' AND CAST(plan_hash AS BINARY) = CAST(LOWER(plan_hash) AS BINARY))"; + return 'CONSTRAINT ' . $constraint('chk_transformation_apply_source_hash') . " CHECK (CHAR_LENGTH(source_hash) = 64 AND source_hash REGEXP '^[0-9a-f]{64}$' AND CAST(source_hash AS BINARY) = CAST(LOWER(source_hash) AS BINARY)), " + . 'CONSTRAINT ' . $constraint('chk_transformation_apply_plan_hash') . " CHECK (CHAR_LENGTH(plan_hash) = 64 AND plan_hash REGEXP '^[0-9a-f]{64}$' AND CAST(plan_hash AS BINARY) = CAST(LOWER(plan_hash) AS BINARY))"; } private function sqliteAcceptsVersion02(): bool diff --git a/src/Transformation/Execution/PlanPreflight.php b/src/Transformation/Execution/PlanPreflight.php index f8e566f..083b0f4 100644 --- a/src/Transformation/Execution/PlanPreflight.php +++ b/src/Transformation/Execution/PlanPreflight.php @@ -488,10 +488,10 @@ function (array $row) use ($canonicalSchema, $canonicalTable): bool { private function canonicalPhysicalSchema(?string $schema): string { - if ($this->connection->profileName === 'sqlite' - && ($schema === null || strcasecmp($schema, 'main') === 0) - ) { - return 'main'; + if ($this->connection->profileName === 'sqlite') { + return $schema === null || strcasecmp($schema, 'main') === 0 + ? 'main' + : strtolower($schema); } $profile = $this->connection->profile; $foldMysqlFamily = $profile instanceof \OpenStatSpec\Sql\MySqlProfile From 637a6e491e8b25570284bdc4e63f2a3bec962075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 20 Aug 2026 22:55:46 +0300 Subject: [PATCH 47/53] fix: make staging constraint suffixes unique per rebuild A successful RENAME TABLE swap leaves the staging tables constraint names on the new live table. Reusing the fixed _v04_pending suffix on a later legacy migration therefore collides with the live foreign-key constraint before any rows can be copied. Generate a short random suffix for each staging rebuild; the live table keeps its existing names, while each temporary table has schema-unique foreign-key and CHECK names. --- src/Transformation/Audit/TransformationAuditMigrator.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Transformation/Audit/TransformationAuditMigrator.php b/src/Transformation/Audit/TransformationAuditMigrator.php index 486a5dd..3f113e0 100644 --- a/src/Transformation/Audit/TransformationAuditMigrator.php +++ b/src/Transformation/Audit/TransformationAuditMigrator.php @@ -134,6 +134,7 @@ private function migrateMySqlFamilyByRename(): void $staging = self::TABLE . '_v04_pending'; $archive = self::TABLE . '_pre_v04_archive'; $columns = implode(', ', self::columns()); + $constraintSuffix = '_v04_' . bin2hex(random_bytes(4)); // Drop any orphan archive table from a previous failed migration so // the RENAME TABLE below has a free target name. This fires only @@ -146,7 +147,7 @@ private function migrateMySqlFamilyByRename(): void // pending table behind; remove it before rebuilding from the source. $this->pdo->exec('DROP TABLE IF EXISTS ' . $staging); - $this->pdo->exec($this->createTableSql($staging, 'mysql', '_v04_pending')); + $this->pdo->exec($this->createTableSql($staging, 'mysql', $constraintSuffix)); $this->pdo->exec( 'INSERT INTO ' . $staging . ' (' . $columns . ') ' . 'SELECT ' . $columns . ' FROM ' . self::TABLE, From 5aa439c7d612615f04528f65e5dba16e0c7504a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 20 Aug 2026 23:00:12 +0300 Subject: [PATCH 48/53] fix: give the real Dolt concurrent commit a committed metadata change The second Dolt session previously called DOLT_COMMIT without changing its working set, so Dolt correctly returned nothing to commit and the CI context-change case failed before exercising the guard. Update the fixture dataset_name from the independent session before calling DOLT_COMMIT; the snapshot deliberately excludes dataset_name, so the metadata probe moves HEAD without changing the rollback assertions or target rows. --- .../Integration/OfficialInPlaceTransformation02Test.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index 3970bf0..4904da7 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -362,7 +362,7 @@ private function assertOfficialDoltContextChanged(array $case): void [self::ROLLBACK_DATASET_ID, 'target'], )); $concurrentPdo = $this->doltConcurrentSession($pdo, $connection); - $reader = new class ($pdo, $concurrentPdo, $connection, $table, $context['branch'], $context['head']) implements DoltEvidenceReader { + $reader = new class ($pdo, $concurrentPdo, $connection, $table, self::ROLLBACK_DATASET_ID, $context['branch'], $context['head']) implements DoltEvidenceReader { public int $reads = 0; public bool $mutationObserved = false; @@ -371,6 +371,7 @@ public function __construct( private readonly PDO $concurrentPdo, private readonly Connection $connection, private readonly string $table, + private readonly string $datasetId, private readonly string $branch, private readonly string $head, ) {} @@ -383,6 +384,12 @@ public function read(): DoltEvidence // The second PDO is a real concurrent session: make a DOLT_COMMIT // that does not touch the executor's target rows but moves HEAD, // so the executor must fail closed on dolt_context_changed. + $metadata = $this->concurrentPdo->prepare( + 'UPDATE dataset SET dataset_name = ? WHERE dataset_id = ?', + ); + if ($metadata instanceof PDOStatement) { + $metadata->execute(['Concurrent commit probe', $this->datasetId]); + } $commit = $this->concurrentPdo->prepare('CALL DOLT_COMMIT(?, ?)'); if ($commit instanceof PDOStatement) { $commit->execute(['-Am', 'Concurrent commit detected by executor guard']); From 278673e0e7a87f9b1185d768387387e6a338fad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 20 Aug 2026 23:04:12 +0300 Subject: [PATCH 49/53] fix: checkout the expected branch in the concurrent Dolt session The real concurrent Dolt session connected on its default main branch, while the fixture and dataset tables lived on the expected feature/recode branch. The metadata probe therefore failed with table not found before DOLT_COMMIT could move HEAD. Checkout the same expected branch in the second session before updating dataset_name and committing. --- .../OfficialInPlaceTransformation02Test.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index 4904da7..ac51b45 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -361,7 +361,7 @@ private function assertOfficialDoltContextChanged(array $case): void 'SELECT variable_label FROM variable WHERE dataset_id = ? AND source_name = ?', [self::ROLLBACK_DATASET_ID, 'target'], )); - $concurrentPdo = $this->doltConcurrentSession($pdo, $connection); + $concurrentPdo = $this->doltConcurrentSession($pdo, $connection, $context['branch']); $reader = new class ($pdo, $concurrentPdo, $connection, $table, self::ROLLBACK_DATASET_ID, $context['branch'], $context['head']) implements DoltEvidenceReader { public int $reads = 0; public bool $mutationObserved = false; @@ -448,7 +448,7 @@ public function read(): DoltEvidence * therefore disconnected) when the reader goes out of scope after the * executor throws. */ - private function doltConcurrentSession(PDO $primaryPdo, Connection $primaryConnection): PDO + private function doltConcurrentSession(PDO $primaryPdo, Connection $primaryConnection, string $branch): PDO { $statement = $primaryPdo->query('SELECT DATABASE()'); $database = $statement instanceof PDOStatement ? (string) $statement->fetchColumn() : ''; @@ -472,6 +472,14 @@ private function doltConcurrentSession(PDO $primaryPdo, Connection $primaryConne is_string($password) ? $password : null, $options, ); + $active = $concurrent->query('SELECT active_branch()'); + $activeBranch = $active instanceof PDOStatement ? (string) $active->fetchColumn() : ''; + if ($activeBranch !== $branch) { + $checkout = $concurrent->prepare('CALL DOLT_CHECKOUT(?, ?)'); + if ($checkout instanceof PDOStatement) { + $checkout->execute(['-b', $branch]); + } + } return $concurrent; } From 05a5dbbbf54c23ecee8fcf9bbaf7fcefd3956767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 20 Aug 2026 23:08:14 +0300 Subject: [PATCH 50/53] fix: switch the concurrent Dolt session to an existing branch DOLT_CHECKOUT -b creates a branch and fails when the expected fixture branch already exists. The concurrent session only needs to switch to the existing feature/recode branch, so call DOLT_CHECKOUT with the branch name directly before the metadata probe and real DOLT_COMMIT. --- tests/Integration/OfficialInPlaceTransformation02Test.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index ac51b45..d736af7 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -475,9 +475,9 @@ private function doltConcurrentSession(PDO $primaryPdo, Connection $primaryConne $active = $concurrent->query('SELECT active_branch()'); $activeBranch = $active instanceof PDOStatement ? (string) $active->fetchColumn() : ''; if ($activeBranch !== $branch) { - $checkout = $concurrent->prepare('CALL DOLT_CHECKOUT(?, ?)'); + $checkout = $concurrent->prepare('CALL DOLT_CHECKOUT(?)'); if ($checkout instanceof PDOStatement) { - $checkout->execute(['-b', $branch]); + $checkout->execute([$branch]); } } return $concurrent; From 83eb24b4bfc17d9b13ceb1661a282ff964dfbd89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 20 Aug 2026 23:12:50 +0300 Subject: [PATCH 51/53] test: preserve real concurrent Dolt history across rollback assertions The real second-session DOLT_COMMIT correctly advances the branch HEAD and adds a history entry that is intentionally not rolled back by the executor transaction. Compare the post-failure snapshot excluding repository evidence, assert data/metadata/audit state is restored, and explicitly assert the branch is unchanged while the concurrent HEAD advance remains visible. --- .../OfficialInPlaceTransformation02Test.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index d736af7..2fab94a 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -426,7 +426,15 @@ public function read(): DoltEvidence self::assertTrue($case['mutation_started']); self::assertTrue($reader->mutationObserved); self::assertFalse($pdo->inTransaction()); - self::assertSame($before, $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table)); + $after = $this->snapshot($pdo, $connection, self::ROLLBACK_DATASET_ID, $table); + $beforeRepository = $before['repository']; + $afterRepository = $after['repository']; + unset($before['repository'], $after['repository']); + self::assertSame($before, $after, 'The failed apply must roll back data, metadata, and audit state.'); + self::assertIsArray($beforeRepository); + self::assertIsArray($afterRepository); + self::assertSame($beforeRepository['branch'], $afterRepository['branch']); + self::assertNotSame($beforeRepository['head'], $afterRepository['head'], 'The independent concurrent commit must remain in Dolt history.'); self::assertSame($case['after_failure']['rows'], $this->numericRows($pdo, $connection, $table, ['target'])); self::assertSame($case['after_failure']['target_metadata']['variable_label'], $this->scalar( $pdo, From c729ad1ab4eb8414ddb366ac9377cb71ae81c292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Thu, 20 Aug 2026 23:27:07 +0300 Subject: [PATCH 52/53] test: assert the exact Dolt commit observed by the guard The concurrent-session test now stores the HEAD returned after its real DOLT_COMMIT and asserts that it is exactly the final repository HEAD and appears in dolt_log history. Keep the existing before/after HEAD difference assertion so the test proves both mutation and identity of the concurrent commit. --- tests/Integration/OfficialInPlaceTransformation02Test.php | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index 2fab94a..114e29c 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -365,6 +365,7 @@ private function assertOfficialDoltContextChanged(array $case): void $reader = new class ($pdo, $concurrentPdo, $connection, $table, self::ROLLBACK_DATASET_ID, $context['branch'], $context['head']) implements DoltEvidenceReader { public int $reads = 0; public bool $mutationObserved = false; + public ?string $newHead = null; public function __construct( private readonly PDO $executorPdo, @@ -395,7 +396,7 @@ public function read(): DoltEvidence $commit->execute(['-Am', 'Concurrent commit detected by executor guard']); } $head = $this->concurrentPdo->query("SELECT dolt_hashof('HEAD')"); - $newHead = $head instanceof PDOStatement ? (string) $head->fetchColumn() : $this->head; + $this->newHead = $head instanceof PDOStatement ? (string) $head->fetchColumn() : $this->head; $statement = $this->executorPdo->query( 'SELECT ' . $this->connection->profile->quoteIdentifier('target') . ' FROM ' . $this->connection->profile->quoteIdentifier($this->table) @@ -404,7 +405,7 @@ public function read(): DoltEvidence $this->mutationObserved = $statement instanceof PDOStatement && (float) $statement->fetchColumn() === 1.0; - return new DoltEvidence($this->branch, $newHead, []); + return new DoltEvidence($this->branch, $this->newHead ?? $this->head, []); } }; @@ -435,6 +436,9 @@ public function read(): DoltEvidence self::assertIsArray($afterRepository); self::assertSame($beforeRepository['branch'], $afterRepository['branch']); self::assertNotSame($beforeRepository['head'], $afterRepository['head'], 'The independent concurrent commit must remain in Dolt history.'); + self::assertNotNull($reader->newHead); + self::assertSame($reader->newHead, $afterRepository['head']); + self::assertContains($reader->newHead, $afterRepository['history']); self::assertSame($case['after_failure']['rows'], $this->numericRows($pdo, $connection, $table, ['target'])); self::assertSame($case['after_failure']['target_metadata']['variable_label'], $this->scalar( $pdo, From 645a7284670e0aea38cf19cfae7209afeabaa9af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 01:15:39 +0300 Subject: [PATCH 53/53] fix: close CodeRabbit conformance and Unicode edge cases Use mb_strtolower for SPSS schema and RECODE target keys and require ext-mbstring so accepted UTF-8 variable names resolve case-insensitively without ASCII-only regressions. Add focused Unicode schema coverage. Reject empty RecodeOperation rule lists at construction, expand parser comparison permutation coverage, guard optional expected_error manifest fields, and make integration cleanup resilient when a constraint or isolated database teardown fails. Refresh the Task 7 migration report and documentation imports to match the implemented MySQL staging/rename path and public example. --- .../task-7-report.md | 6 +-- composer.json | 1 + composer.lock | 5 ++- docs/transformations.md | 1 + src/Frontend/Spss/Binder.php | 2 +- src/Frontend/Spss/Binding/SchemaState.php | 2 +- .../Plan/Operation/RecodeOperation.php | 8 +++- .../Frontend/Spss/Binding/SchemaStateTest.php | 23 +++++++++++ tests/Frontend/Spss/ParserTest.php | 10 ++++- .../InPlaceTransformationServiceTest.php | 6 ++- .../OfficialInPlaceTransformation01Test.php | 13 ++++-- .../OfficialInPlaceTransformation02Test.php | 9 +++++ .../Conformance/TransformationPlan01Test.php | 2 +- .../Conformance/TransformationPlan02Test.php | 2 +- .../Plan/RecodeOperationTest.php | 40 +++++++++++++++++++ 15 files changed, 113 insertions(+), 17 deletions(-) create mode 100644 tests/Frontend/Spss/Binding/SchemaStateTest.php create mode 100644 tests/Transformation/Plan/RecodeOperationTest.php diff --git a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md index 3ba172b..fbd668c 100644 --- a/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md +++ b/.superpowers/sdd/2026-08-17-transformation-plan-spss-frontend-0.2/task-7-report.md @@ -3,12 +3,12 @@ ## Implementation - Added `InPlaceApplyRequest` with fail-fast alias, canonical UUID, lowercase SHA-256, non-empty actor, and paired Dolt-context validation. -- Added explicit schema-v4 `TransformationAuditMigrator` for SQLite, PostgreSQL, MySQL, MariaDB, and Dolt. SQLite rebuilds the one logical audit table transactionally; server profiles replace only the contract check before apply. +- Added explicit schema-v4 `TransformationAuditMigrator` for SQLite, PostgreSQL, MySQL, MariaDB, and Dolt. SQLite and MySQL-family profiles rebuild the one logical audit table through a staging relation; SQLite does so transactionally, while MySQL-family profiles use an atomic multi-table `RENAME TABLE` swap before dropping the archive. - Added success-only `TransformationAuditWriter`, which requires the caller's open apply transaction and stores canonical plan identity, source identity, actor, target identity, operation count, timestamps, and nullable Dolt evidence. - Wired `SpssAdapter::migrateCatalog()` to run the audit migration after normative table creation and before marking schema version 4. - Preserved official 0.1 audit rows while allowing both 0.1 and 0.2 binding contracts. No dataset/data table, snapshot, staging, rollback, or recovery relation was added. - MySQL-family hashes now use a MySQL 8.4/MariaDB/Dolt-compatible exact lowercase-hex check without `REGEXP BINARY`. -- MySQL-family contract upgrades add the v0.1+v0.2 check before dropping older checks. A retry can recover an already check-less table, and an interrupted add-before-drop attempt retains the old check. +- MySQL-family contract upgrades copy rows into a uniquely constrained staging table and atomically swap it into place with `RENAME TABLE`. Stale staging/archive relations are removed before retry, so an interrupted rebuild remains recoverable without manual cleanup. - Schema version 4 can be marked or accepted as ready only when the contiguous 1..4 migration history and readable compact audit relation both exist. Normative migration recording is private and limited to versions 1..3; the audit migrator alone records version 4. ## TDD evidence @@ -34,7 +34,7 @@ ## Self-review - SQLite v0.1 migration preserves the row and logical table name, verifies foreign keys before commit, leaves no `_v04` residue, and rolls back to the old table on copy failure. -- PostgreSQL changes the old contract check in one native transaction; MySQL-family/Dolt DDL is explicit, add-before-drop, retryable, idempotent, and never invoked by the audit writer or inside an apply transaction. +- PostgreSQL changes the old contract check in one native transaction; MySQL-family/Dolt DDL uses an explicit staging/copy/rename path, is retryable and idempotent, and is never invoked by the audit writer or inside an apply transaction. - The writer has no failure-row API and performs no migration, dataset copy, table copy, Dolt commit, branch operation, or recovery/version write. - Fresh catalogs record versions 1 through 4 exactly once; v3 ownership recognition remains backward compatible, while v4 marker recognition requires the compact audit relation. diff --git a/composer.json b/composer.json index d24bc22..ade1396 100644 --- a/composer.json +++ b/composer.json @@ -13,6 +13,7 @@ }, "require": { "php": "^8.4.1", + "ext-mbstring": "*", "ext-pdo": "*", "brick/math": "^0.19", "openstatspec/spss-sav": "^3.0" diff --git a/composer.lock b/composer.lock index de1f593..98b6fd5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "fc67717f8a9244dbd2043737f27ac270", + "content-hash": "722b02e08bdd232bd5b51c6da138a90d", "packages": [ { "name": "brick/math", @@ -4728,11 +4728,12 @@ "prefer-lowest": false, "platform": { "php": "^8.4.1", + "ext-mbstring": "*", "ext-pdo": "*" }, "platform-dev": {}, "platform-overrides": { "php": "8.4.1" }, - "plugin-api-version": "2.9.0" + "plugin-api-version": "2.6.0" } diff --git a/docs/transformations.md b/docs/transformations.md index a0f3e81..72e37ff 100644 --- a/docs/transformations.md +++ b/docs/transformations.md @@ -27,6 +27,7 @@ existing dataset and mutates its registered wide table. use OpenStatSpec\Frontend\Spss\Request\SpssFrontendRequest; use OpenStatSpec\Frontend\Spss\SpssCompiler; use OpenStatSpec\Sql\Connection; +use OpenStatSpec\Spss\SpssAdapter; use OpenStatSpec\Transformation\Execution\InPlaceApplyRequest; use OpenStatSpec\Transformation\Execution\InPlaceTransformationExecutor; diff --git a/src/Frontend/Spss/Binder.php b/src/Frontend/Spss/Binder.php index 453d66d..ca47603 100644 --- a/src/Frontend/Spss/Binder.php +++ b/src/Frontend/Spss/Binder.php @@ -277,7 +277,7 @@ private function recode(RecodeStatement $statement, SchemaState $schema): array : $statement->sourceSpans[$index]; $this->validateTargetName($targetName, $targetSpan); if ($targetMode === TargetMode::Create) { - $key = strtolower($targetName); + $key = mb_strtolower($targetName, 'UTF-8'); if ($schema->contains($targetName) || isset($seenTargets[$key])) { $this->fail( 'target_already_exists', diff --git a/src/Frontend/Spss/Binding/SchemaState.php b/src/Frontend/Spss/Binding/SchemaState.php index b26c8d8..5106ed8 100644 --- a/src/Frontend/Spss/Binding/SchemaState.php +++ b/src/Frontend/Spss/Binding/SchemaState.php @@ -67,6 +67,6 @@ public function addNumeric(string $name): void private function key(string $name): string { - return strtolower($name); + return mb_strtolower($name, 'UTF-8'); } } diff --git a/src/Transformation/Plan/Operation/RecodeOperation.php b/src/Transformation/Plan/Operation/RecodeOperation.php index 9636596..6618a74 100644 --- a/src/Transformation/Plan/Operation/RecodeOperation.php +++ b/src/Transformation/Plan/Operation/RecodeOperation.php @@ -12,14 +12,18 @@ final readonly class RecodeOperation implements Operation { - /** @param non-empty-list $rules */ + /** @param list $rules */ public function __construct( public string $source, public string $target, public TargetMode $targetMode, public array $rules, public Result $unmatched, - ) {} + ) { + if ($rules === []) { + throw new \InvalidArgumentException('A recode operation requires at least one rule.'); + } + } /** @return array */ public function canonicalArray(): array diff --git a/tests/Frontend/Spss/Binding/SchemaStateTest.php b/tests/Frontend/Spss/Binding/SchemaStateTest.php new file mode 100644 index 0000000..9bdbe91 --- /dev/null +++ b/tests/Frontend/Spss/Binding/SchemaStateTest.php @@ -0,0 +1,23 @@ +resolve('ärger', new SourceSpan(0, 5))); + self::assertTrue($schema->contains('ÄRGER')); + } +} diff --git a/tests/Frontend/Spss/ParserTest.php b/tests/Frontend/Spss/ParserTest.php index f4044a8..7eac5e5 100644 --- a/tests/Frontend/Spss/ParserTest.php +++ b/tests/Frontend/Spss/ParserTest.php @@ -173,7 +173,7 @@ public function testBooleanPrecedenceParenthesesAndSameOperatorFlatteningPreserv public function testParsesAllComparisonOperandPermutations(): void { - $program = (new Parser())->parse('IF (a = b AND 1 < c AND 2 >= 1) target = -1.25e+2.'); + $program = (new Parser())->parse('IF (a = b AND 1 < c AND 2 >= 1 AND c > 3) target = -1.25e+2.'); $statement = $program->statements[0]; self::assertInstanceOf(IfStatement::class, $statement); @@ -181,12 +181,20 @@ public function testParsesAllComparisonOperandPermutations(): void self::assertInstanceOf(BooleanPredicate::class, $predicate); $first = $predicate->operands[0]; $second = $predicate->operands[1]; + $third = $predicate->operands[2]; + $fourth = $predicate->operands[3]; self::assertInstanceOf(Comparison::class, $first); self::assertInstanceOf(Comparison::class, $second); + self::assertInstanceOf(Comparison::class, $third); + self::assertInstanceOf(Comparison::class, $fourth); self::assertInstanceOf(VariableOperand::class, $first->left); self::assertInstanceOf(VariableOperand::class, $first->right); self::assertInstanceOf(LiteralOperand::class, $second->left); self::assertInstanceOf(VariableOperand::class, $second->right); + self::assertInstanceOf(LiteralOperand::class, $third->left); + self::assertInstanceOf(LiteralOperand::class, $third->right); + self::assertInstanceOf(VariableOperand::class, $fourth->left); + self::assertInstanceOf(LiteralOperand::class, $fourth->right); self::assertInstanceOf(LiteralOperand::class, $statement->expression); self::assertSame('-1.25e+2', $statement->expression->token); } diff --git a/tests/Integration/InPlaceTransformationServiceTest.php b/tests/Integration/InPlaceTransformationServiceTest.php index f45b298..2c0c961 100644 --- a/tests/Integration/InPlaceTransformationServiceTest.php +++ b/tests/Integration/InPlaceTransformationServiceTest.php @@ -511,6 +511,7 @@ public function testMySqlFamilyRejectsMyIsamWideTableBeforeForcedAuditFailure( $fixture = $this->installFixture($pdo, $connection); $constraint = 'task9_nontransactional_wide_' . $expectedProfile; $quotedConstraint = $connection->profile->quoteIdentifier($constraint); + $constraintInstalled = false; try { $pdo->exec( @@ -520,6 +521,7 @@ public function testMySqlFamilyRejectsMyIsamWideTableBeforeForcedAuditFailure( 'ALTER TABLE transformation_apply ADD CONSTRAINT ' . $quotedConstraint . " CHECK (actor <> 'integration-test')", ); + $constraintInstalled = true; $before = $this->applyFailureSnapshot($pdo, $connection, $fixture); try { @@ -534,7 +536,9 @@ public function testMySqlFamilyRejectsMyIsamWideTableBeforeForcedAuditFailure( self::assertFalse($pdo->inTransaction()); self::assertSame($before, $this->applyFailureSnapshot($pdo, $connection, $fixture)); } finally { - $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT ' . $quotedConstraint); + if ($constraintInstalled) { + $pdo->exec('ALTER TABLE transformation_apply DROP CONSTRAINT ' . $quotedConstraint); + } $this->purgeFixture($pdo, $connection, $fixture); } } diff --git a/tests/Integration/OfficialInPlaceTransformation01Test.php b/tests/Integration/OfficialInPlaceTransformation01Test.php index fe28ab3..135df21 100644 --- a/tests/Integration/OfficialInPlaceTransformation01Test.php +++ b/tests/Integration/OfficialInPlaceTransformation01Test.php @@ -36,14 +36,22 @@ final class OfficialInPlaceTransformation01Test extends TestCase protected function tearDown(): void { + $failures = []; try { foreach (array_reverse($this->isolatedTestDatabases) as $fixture) { - $fixture['admin']->exec('DROP DATABASE ' . $this->quoteDatabase($fixture['database'])); + try { + $fixture['admin']->exec('DROP DATABASE ' . $this->quoteDatabase($fixture['database'])); + } catch (\Throwable $failure) { + $failures[] = $fixture['database'] . ': ' . $failure->getMessage(); + } } } finally { $this->isolatedTestDatabases = []; parent::tearDown(); } + if ($failures !== []) { + throw new RuntimeException('Isolated database cleanup failed: ' . implode('; ', $failures)); + } } /** @return iterable}> */ @@ -235,7 +243,6 @@ private function assertOfficialBackendSuccess(array $case): void self::assertSame($before['repository']['history'], $after['repository']['history']); self::assertNotSame([], $after['repository']['status']); self::assertContains($fixture['table'], array_column($after['repository']['status'], 'table_name')); - self::assertFalse((bool) $afterContract['dolt_commit_performed']); self::assertSame($context['branch'], $audit['dolt_branch']); self::assertSame($context['head'], $audit['dolt_head_before']); self::assertSame($context['head'], $audit['dolt_head_after']); @@ -287,7 +294,6 @@ private function assertOfficialDoltContextFailure(array $case): void self::assertSame($case['expected_error'], $failure->diagnosticCode()); } - self::assertFalse($case['mutation_started']); self::assertFalse($pdo->inTransaction()); self::assertSame($before, $this->officialBackendSnapshot($pdo, $connection, $fixture)); } @@ -314,7 +320,6 @@ private function assertOfficialMySqlCreateRejection(array $case): void self::assertSame($case['expected_error'], $failure->diagnosticCode()); } - self::assertFalse($case['mutation_started']); self::assertFalse($pdo->inTransaction()); self::assertSame($before, $this->officialBackendSnapshot($pdo, $connection, $fixture)); } diff --git a/tests/Integration/OfficialInPlaceTransformation02Test.php b/tests/Integration/OfficialInPlaceTransformation02Test.php index 114e29c..0c2aeca 100644 --- a/tests/Integration/OfficialInPlaceTransformation02Test.php +++ b/tests/Integration/OfficialInPlaceTransformation02Test.php @@ -490,8 +490,17 @@ private function doltConcurrentSession(PDO $primaryPdo, Connection $primaryConne $checkout = $concurrent->prepare('CALL DOLT_CHECKOUT(?)'); if ($checkout instanceof PDOStatement) { $checkout->execute([$branch]); + $result = $checkout->fetch(PDO::FETCH_ASSOC); + if (is_array($result) && array_key_exists('status', $result) && (int) $result['status'] !== 0) { + throw new RuntimeException('DOLT_CHECKOUT failed: ' . (string) ($result['message'] ?? 'unknown error')); + } } } + $confirmed = $concurrent->query('SELECT active_branch()'); + $confirmedBranch = $confirmed instanceof PDOStatement ? (string) $confirmed->fetchColumn() : ''; + if ($confirmedBranch !== $branch) { + throw new RuntimeException(sprintf('DOLT_CHECKOUT selected %s instead of %s.', $confirmedBranch, $branch)); + } return $concurrent; } diff --git a/tests/Transformation/Conformance/TransformationPlan01Test.php b/tests/Transformation/Conformance/TransformationPlan01Test.php index 5788c16..3c52a52 100644 --- a/tests/Transformation/Conformance/TransformationPlan01Test.php +++ b/tests/Transformation/Conformance/TransformationPlan01Test.php @@ -158,7 +158,7 @@ public static function cases(): iterable #[DataProvider('cases')] public function testOfficialPlan01(array $case): void { - if (is_string($case['expected_error'])) { + if (is_string($case['expected_error'] ?? null)) { try { (new PlanCodec())->fromArray($case['plan']); self::fail('Expected ' . $case['expected_error']); diff --git a/tests/Transformation/Conformance/TransformationPlan02Test.php b/tests/Transformation/Conformance/TransformationPlan02Test.php index 0e979a0..077ddf2 100644 --- a/tests/Transformation/Conformance/TransformationPlan02Test.php +++ b/tests/Transformation/Conformance/TransformationPlan02Test.php @@ -45,7 +45,7 @@ public static function cases(): iterable #[DataProvider('cases')] public function testOfficialPlan02(array $case): void { - if (is_string($case['expected_error'])) { + if (is_string($case['expected_error'] ?? null)) { try { (new PlanCodec())->fromArray($case['plan']); self::fail('Expected ' . $case['expected_error']); diff --git a/tests/Transformation/Plan/RecodeOperationTest.php b/tests/Transformation/Plan/RecodeOperationTest.php new file mode 100644 index 0000000..f78ad3b --- /dev/null +++ b/tests/Transformation/Plan/RecodeOperationTest.php @@ -0,0 +1,40 @@ +expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('requires at least one rule'); + $operation = new RecodeOperation('source', 'target', TargetMode::Replace, [], new CopyResult()); + self::assertInstanceOf(RecodeOperation::class, $operation); + } + + public function testValidRuleListRemainsCanonicalizable(): void + { + $operation = new RecodeOperation( + 'source', + 'target', + TargetMode::Create, + [new RecodeRule( + new ExactMatch(Binary64Value::fromBits('3ff0000000000000')), + new LiteralResult(Binary64Value::fromBits('4000000000000000')), + )], + new CopyResult(), + ); + self::assertCount(1, $operation->canonicalArray()['rules']); + } +}