Support verified legacy SPSS locales

Author: TonisOrmisson

THIRD_PARTY_NOTICES.md

@@ -5,7 +5,7 @@ SPSS engine is a separate third-party distribution:
 
 | Component | Version | Applicable terms |
 | --- | --- | --- |
-| pyspssio | TonisOrmisson/pyspssio pinned commit 446af0c | MIT for the Python wrapper, except for its IBM I/O Module files |
+| pyspssio | TonisOrmisson/pyspssio pinned commit 6a0f9fa | MIT for the Python wrapper, except for its IBM I/O Module files |
 | IBM I/O Modules for IBM SPSS Statistics Data Files | bundled by the pinned pyspssio fork | IBM International License Agreement for Non-Warranted Programs and the accompanying License Information / REDIST files |
 
 pyspssio ships the IBM licence, redistribution list, and third-party notices

docs/sav-profile.md

@@ -42,7 +42,7 @@ of this boundary. It records the pinned source commit and installed engine versi
 | Print and write formats independently | Supported as raw IBM I/O tuples | The adapter stores both tuples separately and writes them without collapsing either value. |
 | Variable sets | Supported through raw IBM I/O | The adapter stores source sets in the extension catalog and writes them through the raw dictionary setter. Invalid target definitions fail before data are written. |
 | Legacy compatible variable names | Supported through the strict dictionary bridge | The adapter stores the short-name mapping and rewrites both the fixed variable record and long-name extension before export. |
-| Source encoding | UTF-8 fidelity only | UTF-8 is supported. A legacy code page is retained in metadata, but export requires `source-encoding-not-preserved` because the writer has no legacy-code-page preservation contract. |
+| Source encoding | UTF-8 by default; legacy code page with an explicit OS locale | The caller supplies legacy_locale and the writer verifies that the emitted encoding matches the stored source encoding. Without it, export fails before output creation. |
 | Custom attributes with one value | Supported | File and variable scalar attributes round trip through the normalized attribute catalog. |
 | Custom-attribute value arrays | Supported through raw IBM I/O | The adapter represents an array as IBM SPSS `Name[1]`, `Name[2]`, … members and reconstructs the ordered array in the normalized catalog. |
 

pyproject.toml

@@ -10,7 +10,7 @@ readme = "README.md"
 requires-python = ">=3.11"
 license = {text = "Apache-2.0"}
 authors = [{name = "OpenStatSpec"}]
-dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "pyspssio @ git+https://github.com/TonisOrmisson/pyspssio.git@446af0c"]
+dependencies = ["SQLAlchemy>=2.0", "pandas>=2.2", "pyspssio @ git+https://github.com/TonisOrmisson/pyspssio.git@6a0f9fa"]
 
 [project.optional-dependencies]
 dev = ["pytest>=8"]

src/openstatspec/api.py

@@ -32,7 +32,7 @@ def capability_matrix() -> Mapping[str, Any]:
             "documents": "supported",
             "source_encoding": {
                 "utf8": "supported",
-                "legacy_code_pages": "fail-closed-on-export",
+                "legacy_code_pages": "requires-explicit-legacy-locale",
             },
             "measurement_level": "supported", "user_missing_rules": "supported",
             "multiple_response_sets": "supported",

src/openstatspec/cli.py

@@ -25,13 +25,14 @@ def main(argv: Sequence[str] | None = None) -> int:
     exporter.add_argument("--dataset-id", required=True)
     exporter.add_argument("--output", required=True)
     exporter.add_argument("--allow-loss", action="append", default=[])
+    exporter.add_argument("--legacy-locale", help="OS locale for a non-UTF-8 source encoding")
     args = parser.parse_args(argv)
     if args.command == "capabilities":
         result = capability_matrix()
     elif args.command == "import":
         result = import_sav(args.source, database_url=args.database_url, dataset_id=args.dataset_id)
     elif args.command == "export":
-        result = export_sav(database_url=args.database_url, dataset_id=args.dataset_id, destination=args.output, allow_loss=args.allow_loss)
+        result = export_sav(database_url=args.database_url, dataset_id=args.dataset_id, destination=args.output, allow_loss=args.allow_loss, legacy_locale=args.legacy_locale)
     elif args.command == "inspect":
         result = inspect(args.source)
     else:

src/openstatspec/spss/__init__.py

@@ -15,4 +15,8 @@ def import_dataset(source: str | Path, *, database_url: Any, dataset_id: str, **
 
 
 def export_dataset(*, database_url: Any, dataset_id: str, destination: str | Path, **options: Any) -> dict[str, Any]:
-    return export_sav_dataset(database_url=str(database_url), dataset_id=dataset_id, destination=destination, allow_loss=tuple(options.get("allow_loss", ())) )
+    return export_sav_dataset(
+        database_url=str(database_url), dataset_id=dataset_id, destination=destination,
+        allow_loss=tuple(options.get("allow_loss", ())),
+        legacy_locale=options.get("legacy_locale"),
+    )

src/openstatspec/spss/sav.py

@@ -52,7 +52,7 @@ def engine_identity() -> dict[str, str]:
     return {
         "package": "pyspssio",
         "distribution": "TonisOrmisson/pyspssio",
-        "pinned_commit": "446af0c",
+        "pinned_commit": "6a0f9fa",
         "installed_version": str(pyspssio.__version__),
     }
 
@@ -255,7 +255,7 @@ def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str
 
 def export_sav_dataset(
     *, database_url: str, dataset_id: str, destination: str | Path,
-    allow_loss: tuple[str, ...] = (),
+    allow_loss: tuple[str, ...] = (), legacy_locale: str | None = None,
 ) -> dict[str, Any]:
     destination_path = Path(destination)
     if destination_path.suffix.lower() not in {".sav", ".zsav"}:
@@ -266,9 +266,15 @@ def export_sav_dataset(
         case_weight_variable=dataset.get("case_weight_variable"),
         multiple_response_sets=dataset.get("multiple_response_sets"),
     )
+    persisted_events = read_fidelity_events(database_url=database_url, dataset_id=dataset_id)
+    if legacy_locale is not None:
+        persisted_events = tuple(
+            event for event in persisted_events
+            if event["code"] != "source-encoding-not-preserved"
+        )
     loss_report = _merge_loss_reports(
-        _export_loss_report(dataset, variables),
-        read_fidelity_events(database_url=database_url, dataset_id=dataset_id),
+        _export_loss_report(dataset, variables, legacy_locale=legacy_locale),
+        persisted_events,
     )
     rejected = [event["code"] for event in loss_report if event["code"] not in allow_loss]
     if rejected:
@@ -287,7 +293,9 @@ def export_sav_dataset(
         columns=[variable["source_name"] for variable in variables],
     )
     try:
-        _write_with_dictionary_bridge(destination_path, frame, dataset, variables)
+        _write_with_dictionary_bridge(
+            destination_path, frame, dataset, variables, legacy_locale=legacy_locale,
+        )
     except Exception:
         destination_path.unlink(missing_ok=True)
         raise
@@ -298,7 +306,7 @@ def export_sav_dataset(
         allowed_fidelity_events=tuple(
             event for event in loss_report if event["code"] in allow_loss
         ),
-        operation_details={"engine": engine_identity()},
+        operation_details={"engine": engine_identity(), "legacy_locale": legacy_locale},
     )
     return {
         "dataset_id": dataset_id,
@@ -310,6 +318,7 @@ def export_sav_dataset(
 
 def _write_with_dictionary_bridge(
     destination: Path, frame: pd.DataFrame, dataset: dict[str, Any], variables: list[dict[str, Any]],
+    *, legacy_locale: str | None = None,
 ) -> None:
     """Write a dictionary through pyspssio and preserve its document record.
 
@@ -320,17 +329,27 @@ def _write_with_dictionary_bridge(
     """
     metadata = _writer_metadata(dataset, variables)
     documents = _json_load(dataset.get("documents"), [])
+    source_encoding = str(dataset.get("source_encoding") or "UTF-8")
+    legacy_output = _is_non_utf8_encoding(source_encoding) and legacy_locale is not None
     with ExitStack() as stack:
         document_source = None
         if documents:
             temporary_directory = Path(stack.enter_context(TemporaryDirectory()))
             temporary_source = temporary_directory / "documents.sav"
             pyspssio.write_sav(
                 str(temporary_source), pd.DataFrame({"document_source": [0.0]}),
+                unicode=not legacy_output, locale=legacy_locale,
             )
-            write_document_lines(temporary_source, documents, encoding="UTF-8")
+            temporary_encoding = str(pyspssio.read_metadata(str(temporary_source)).get("encoding") or "UTF-8")
+            _require_matching_legacy_encoding(source_encoding, temporary_encoding, legacy_output)
+            write_document_lines(temporary_source, documents, encoding=temporary_encoding)
             document_source = stack.enter_context(pyspssio.Reader(str(temporary_source), mode="r"))
-        writer = stack.enter_context(pyspssio.Writer(str(destination), mode="w"))
+        writer = stack.enter_context(
+            pyspssio.Writer(
+                str(destination), mode="w", unicode=not legacy_output, locale=legacy_locale,
+            )
+        )
+        _require_matching_legacy_encoding(source_encoding, str(writer.file_encoding), legacy_output)
         writer.compression = 2 if destination.suffix.lower() == ".zsav" else 1
         writer.file_label = str(dataset.get("file_label") or "")
         for variable in variables:
@@ -461,17 +480,34 @@ def _engine_loss_report(metadata: dict[str, Any]) -> dict[str, dict[str, Any]]:
 
 
 def _export_loss_report(
-    dataset: dict[str, Any], variables: list[dict[str, Any]],
+    dataset: dict[str, Any], variables: list[dict[str, Any]], *, legacy_locale: str | None,
 ) -> tuple[dict[str, Any], ...]:
     events: list[dict[str, Any]] = []
-    if _is_non_utf8_encoding(dataset.get("source_encoding")):
+    if _is_non_utf8_encoding(dataset.get("source_encoding")) and legacy_locale is None:
         events.append({
             "code": "source-encoding-not-preserved",
-            "detail": "The pyspssio writer has no source-encoding preservation contract for this legacy code page.",
+            "detail": "Export requires an explicit legacy_locale for this non-UTF-8 source encoding.",
             "details": {"source_encoding": dataset.get("source_encoding")},
         })
     return tuple(events)
 
+
+def _require_matching_legacy_encoding(
+    source_encoding: str, emitted_encoding: str, legacy_output: bool,
+) -> None:
+    if legacy_output and _normalize_encoding(source_encoding) != _normalize_encoding(emitted_encoding):
+        raise UnsupportedOperationError(
+            "Configured legacy locale emitted "
+            + emitted_encoding
+            + " instead of required source encoding "
+            + source_encoding
+            + "."
+        )
+
+
+def _normalize_encoding(value: str) -> str:
+    return str(value).strip().replace("_", "-").replace("WINDOWS-", "CP").upper()
+
 def _merge_loss_reports(*reports: tuple[dict[str, Any], ...]) -> tuple[dict[str, Any], ...]:
     events: dict[str, dict[str, Any]] = {}
     for report in reports:

tests/test_cli.py

@@ -17,7 +17,7 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N
     assert openstatspec.cli.main(["inspect", str(source)]) == 0
     inspected = json.loads(capsys.readouterr().out)
     assert inspected["source_format"] == "SAV"
-    assert inspected["engine"]["pinned_commit"] == "446af0c"
+    assert inspected["engine"]["pinned_commit"] == "6a0f9fa"
     assert inspected["source_sha256"]
     assert inspected["loss_report"] == []
     assert imported["case_count"] == 1
@@ -37,7 +37,7 @@ def test_cli_import_inspect_validate_and_export_emit_json(tmp_path, capsys) -> N
 
 def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) -> None:
     matrix = openstatspec.capability_matrix()
-    assert matrix["engine"]["pinned_commit"] == "446af0c"
+    assert matrix["engine"]["pinned_commit"] == "6a0f9fa"
 
     assert matrix["spss"] == {
         "values": "supported",
@@ -49,7 +49,7 @@ def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) ->
         "documents": "supported",
         "source_encoding": {
             "utf8": "supported",
-            "legacy_code_pages": "fail-closed-on-export",
+            "legacy_code_pages": "requires-explicit-legacy-locale",
         },
         "measurement_level": "supported",
         "user_missing_rules": "supported",

tests/test_loss_reports.py

@@ -9,6 +9,7 @@
 from openstatspec.core import UnsupportedOperationError
 from openstatspec.sql.wide import create_wide_dataset
 from openstatspec.spss.raw_dictionary import write_compatible_names
+from openstatspec.spss import sav as sav_module
 
 
 _REQUIRED_ENGINE_LOSS = []
@@ -27,7 +28,7 @@ def test_persisted_import_fidelity_events_require_consent_after_reopen(tmp_path)
     connection = sqlite3.connect(database_path)
     assert {row[0] for row in connection.execute("select code from fidelity_event_catalog")} == set(_REQUIRED_ENGINE_LOSS)
     import_details = json.loads(connection.execute("select details from operation_catalog order by created_at limit 1").fetchone()[0])
-    assert import_details["engine"]["pinned_commit"] == "446af0c"
+    assert import_details["engine"]["pinned_commit"] == "6a0f9fa"
 
     openstatspec.export_sav(database_url=database, dataset_id="persisted", destination=blocked)
     assert blocked.exists()
@@ -75,6 +76,49 @@ def test_non_utf8_source_encoding_is_explicit_export_loss(tmp_path) -> None:
     metadata = pyspssio.read_metadata(str(destination))
     assert metadata["encoding"] == "UTF-8"
 
+
+def test_explicit_legacy_locale_selects_the_single_engine_route(tmp_path, monkeypatch) -> None:
+    database_path = tmp_path / "legacy-locale.sqlite"
+    database = f"sqlite:///{database_path}"
+    destination = tmp_path / "legacy-locale.sav"
+    create_wide_dataset(
+        database_url=database, dataset_id="legacy-locale", source_name="legacy.sav",
+        source_format="SAV", source_encoding="WINDOWS-1252", rows=[{"name": "Muller"}],
+        variables=[{
+            "ordinal": 1, "source_name": "name", "physical_name": "name", "storage_kind": "string",
+            "string_width": 8, "label": "", "format": "A8", "measure": "nominal", "role": None,
+            "alignment": None, "display_width": 8, "attributes": "{}", "compat_name": None,
+            "value_labels": "{}", "missing_ranges": "[]",
+        }],
+        fidelity_events=[{
+            "code": "source-encoding-not-preserved",
+            "detail": "Legacy encoding needs a locale.",
+            "details": {"source_encoding": "WINDOWS-1252"},
+        }],
+    )
+    observed = {}
+
+    def writer(destination_path, frame, dataset, variables, *, legacy_locale=None):
+        observed["locale"] = legacy_locale
+        observed["encoding"] = dataset["source_encoding"]
+        observed["values"] = frame["name"].tolist()
+
+    monkeypatch.setattr(sav_module, "_write_with_dictionary_bridge", writer)
+    result = openstatspec.export_sav(
+        database_url=database, dataset_id="legacy-locale", destination=destination,
+        legacy_locale="en_US.cp1252",
+    )
+    assert result.diagnostics == ()
+    assert observed == {
+        "locale": "en_US.cp1252", "encoding": "WINDOWS-1252", "values": ["Muller"],
+    }
+
+
+def test_legacy_locale_must_emit_the_exact_source_encoding() -> None:
+    sav_module._require_matching_legacy_encoding("WINDOWS-1252", "CP1252", True)
+    with pytest.raises(UnsupportedOperationError, match="instead of required"):
+        sav_module._require_matching_legacy_encoding("WINDOWS-1252", "UTF-8", True)
+
 @pytest.mark.parametrize("suffix", [".sav", ".zsav"])
 def test_compatible_variable_name_round_trips_from_current_sql_catalog(tmp_path, suffix: str) -> None:
     source = tmp_path / f"compat-source{suffix}"