Preserve SAV missing values and file metadata

Author: TonisOrmisson

src/openstatspec/spss/sav.py

@@ -16,6 +16,7 @@ def _variables(meta: Any, names: list[str]) -> list[dict[str, Any]]:
     labels = dict(getattr(meta, "column_names_to_labels", {}) or {})
     formats = dict(getattr(meta, "original_variable_types", {}) or {})
     measures = dict(getattr(meta, "variable_measure", {}) or {})
+    alignments = dict(getattr(meta, "variable_alignment", {}) or {})
     displays = dict(getattr(meta, "variable_display_width", {}) or {})
     value_labels = dict(getattr(meta, "variable_value_labels", {}) or {})
     missing_ranges = dict(getattr(meta, "missing_ranges", {}) or {})
@@ -28,7 +29,8 @@ def _variables(meta: Any, names: list[str]) -> list[dict[str, Any]]:
             "physical_name": physical_name(source_name, used), "storage_kind": kind,
             "string_width": widths.get(source_name) if kind == "string" else None,
             "label": labels.get(source_name, "") or "", "format": formats.get(source_name),
-            "measure": measures.get(source_name), "display_width": displays.get(source_name),
+            "measure": measures.get(source_name), "alignment": alignments.get(source_name),
+            "display_width": displays.get(source_name),
             "value_labels": json.dumps(value_labels.get(source_name, {}), sort_keys=True),
             "missing_ranges": json.dumps(missing_ranges.get(source_name, []), default=str),
         })
@@ -69,6 +71,8 @@ def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str
         database_url=database_url, dataset_id=dataset_id, source_name=source_path.name,
         source_format=source_path.suffix[1:].upper(), rows=_rows(frame, variables),
         variables=variables, file_label=getattr(meta, "file_label", "") or "",
+        source_encoding=getattr(meta, "file_encoding", None),
+        documents=json.dumps(list(getattr(meta, "notes", []) or [])),
     )
     return {**result, "loss_report": LossReport().events}
 
@@ -87,6 +91,10 @@ def export_sav_dataset(*, database_url: str, dataset_id: str, destination: str |
     formats = {item["source_name"]: item["format"] for item in variables if item["format"]}
     measures = {item["source_name"]: item["measure"] for item in variables if item["measure"]}
     displays = {item["source_name"]: item["display_width"] for item in variables if item["display_width"]}
+    missing_ranges = {
+        item["source_name"]: json.loads(item["missing_ranges"])
+        for item in variables if item["missing_ranges"] != "[]"
+    }
     value_labels = {
         item["source_name"]: {(float(key) if item["storage_kind"] == "numeric" else key): value for key, value in json.loads(item["value_labels"]).items()}
         for item in variables if item["value_labels"] != "{}"
@@ -95,10 +103,11 @@ def export_sav_dataset(*, database_url: str, dataset_id: str, destination: str |
         frame, destination_path, file_label=dataset["file_label"], column_labels=labels,
         variable_format=formats, variable_measure=measures,
         variable_display_width=displays, variable_value_labels=value_labels,
+        missing_ranges=missing_ranges, note=json.loads(dataset["documents"]),
     )
     return {
         "dataset_id": dataset_id, "destination": str(destination_path),
-        "loss_report": ({"code": "missing-ranges-not-exported", "detail": "The initial writer does not preserve user-missing ranges."},),
+        "loss_report": LossReport().events,
     }
 
 

src/openstatspec/sql/wide.py

@@ -16,8 +16,10 @@ def catalog(metadata: MetaData) -> tuple[Table, Table]:
         Column("data_table", String(255), nullable=False, unique=True),
         Column("source_format", String(16), nullable=False),
         Column("source_name", Text, nullable=False),
+        Column("source_encoding", String(128)),
         Column("case_count", BigInteger, nullable=False),
         Column("file_label", Text, nullable=False, default=""),
+        Column("documents", Text, nullable=False, default="[]"),
     )
     variables = Table(
         "variable_catalog", metadata,
@@ -30,6 +32,7 @@ def catalog(metadata: MetaData) -> tuple[Table, Table]:
         Column("label", Text, nullable=False, default=""),
         Column("format", String(64)),
         Column("measure", String(32)),
+        Column("alignment", String(32)),
         Column("display_width", Integer),
         Column("value_labels", Text, nullable=False, default="{}"),
         Column("missing_ranges", Text, nullable=False, default="[]"),
@@ -56,6 +59,7 @@ def data_table_name(dataset_id: str) -> str:
 def create_wide_dataset(
     *, database_url: str, dataset_id: str, source_name: str, source_format: str,
     rows: Iterable[Mapping[str, Any]], variables: list[dict[str, Any]], file_label: str = "",
+    source_encoding: str | None = None, documents: str = "[]",
 ) -> dict[str, Any]:
     engine = create_engine(database_url)
     metadata = MetaData()
@@ -74,7 +78,8 @@ def create_wide_dataset(
         materialized = [{"__case_ordinal": ordinal, **row} for ordinal, row in enumerate(rows, start=1)]
         connection.execute(insert(datasets).values(
             dataset_id=dataset_id, data_table=data_table.name, source_format=source_format,
-            source_name=source_name, case_count=len(materialized), file_label=file_label,
+            source_name=source_name, source_encoding=source_encoding, case_count=len(materialized),
+            file_label=file_label, documents=documents,
         ))
         connection.execute(insert(variable_catalog), [dict(dataset_id=dataset_id, **item) for item in variables])
         if materialized:

tests/test_sav_sqlite.py

@@ -1,3 +1,4 @@
+import json
 import sqlite3
 
 import openstatspec
@@ -18,6 +19,8 @@ def test_sav_round_trip_uses_one_wide_table_and_catalog(tmp_path) -> None:
         variable_value_labels={"age": {34.0: "thirty-four"}},
         variable_measure={"age": "scale", "name": "nominal"},
         variable_format={"age": "F8.0", "name": "A12"},
+        missing_ranges={"age": [34.0]},
+        note=["Fixture note"],
     )
 
     imported = openstatspec.import_sav(source, database_url=database, dataset_id="tiny")
@@ -31,11 +34,14 @@ def test_sav_round_trip_uses_one_wide_table_and_catalog(tmp_path) -> None:
     assert connection.execute("select __case_ordinal, age, name from data_tiny order by __case_ordinal").fetchall() == [
         (1, 34.0, "Ada"), (2, None, "")
     ]
+    assert connection.execute("select source_encoding, documents from dataset_catalog").fetchone() == ("UTF-8", json.dumps(["Fixture note"]))
 
     openstatspec.export_sav(database_url=database, dataset_id="tiny", destination=exported)
     frame, meta = pyreadstat.read_sav(exported, user_missing=True)
     assert frame["age"].iloc[0] == 34.0
     assert pd.isna(frame["age"].iloc[1])
     assert frame["name"].tolist() == ["Ada", ""]
     assert meta.column_labels == ["Age", "Name"]
+    assert meta.missing_ranges == {"age": [{"lo": 34.0, "hi": 34.0}]}
+    assert meta.notes == ["Fixture note"]