Skip to content

Commit 501f178

Browse files
committed
Expand SAV semantic conformance fixtures
1 parent afc5dd7 commit 501f178

3 files changed

Lines changed: 192 additions & 7 deletions

File tree

‎src/openstatspec/sql/wide.py‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
"""SQLite reference SQL profile for the strict OpenStatSpec wide-table contract."""
22

33
import json
4+
import math
45
import re
6+
import sys
57
from datetime import UTC, datetime
68
from uuid import uuid4
79
from collections.abc import Iterable, Mapping
@@ -198,6 +200,18 @@ def missing_rule_catalog(metadata: MetaData) -> Table:
198200

199201

200202
def _typed_endpoint(value: Any) -> tuple[str, float | None, str | None]:
203+
"""Encode typed values without losing SPSS LOWEST/HIGHEST sentinels.
204+
205+
ReadStat exposes a LOWEST range bound as NaN and HIGHEST as positive
206+
infinity. Storing either in a SQL binary64 column is not portable, so the
207+
catalog records an explicit endpoint type and reconstructs the ReadStat
208+
writer input on export.
209+
"""
210+
if isinstance(value, float):
211+
if math.isnan(value):
212+
return "lowest", None, None
213+
if math.isinf(value):
214+
return ("highest" if value > 0 else "lowest"), None, None
201215
if isinstance(value, bool):
202216
return "text", None, str(value)
203217
if isinstance(value, (int, float)):
@@ -382,7 +396,12 @@ def create_wide_dataset(
382396

383397

384398
def _endpoint_from_row(row: Mapping[str, Any], *, prefix: str) -> Any:
385-
return row[f"{prefix}_numeric"] if row[f"{prefix}_type"] == "numeric" else row[f"{prefix}_text"]
399+
endpoint_type = row[f"{prefix}_type"]
400+
if endpoint_type == "lowest":
401+
return -sys.float_info.max
402+
if endpoint_type == "highest":
403+
return sys.float_info.max
404+
return row[f"{prefix}_numeric"] if endpoint_type == "numeric" else row[f"{prefix}_text"]
386405

387406

388407
def read_wide_dataset(*, database_url: str, dataset_id: str) -> tuple[dict[str, Any], list[dict[str, Any]], list[dict[str, Any]]]:

‎tests/conformance.py‎

Lines changed: 127 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,119 @@
1-
"""Semantic comparison helpers for supported SAV/ZSAV round trips."""
1+
"""Reusable fixtures and semantic comparators for supported SAV/ZSAV round trips.
22
3+
These helpers deliberately compare only metadata that the Python adapter and
4+
its current SAV writer expose. Attributes, variable sets, and
5+
multiple-response sets are not asserted here because they are explicitly
6+
reported as unsupported fidelity losses.
7+
"""
8+
9+
import math
10+
import sys
311
from pathlib import Path
412
from typing import Any
513

614
import pandas as pd
715
import pyreadstat
16+
from pandas.testing import assert_frame_equal
17+
18+
19+
def write_supported_semantics_fixture(destination: str | Path) -> dict[str, Any]:
20+
"""Write a fixture spanning every currently supported dictionary semantic.
21+
22+
-sys.float_info.max and sys.float_info.max are how ReadStat represents
23+
SPSS LOWEST and HIGHEST missing-range endpoints on write. ReadStat exposes
24+
them as NaN and +inf when the SAV/ZSAV file is read back; the comparator
25+
canonicalizes those markers.
26+
"""
27+
destination = Path(destination)
28+
long_text = "Õ🙂漢字" * 90
29+
frame = pd.DataFrame({
30+
"lowest_range": [-2.0, -1.0, 0.0, 7.0],
31+
"highest_range": [-99.0, 0.0, 1.0, 42.0],
32+
"code": [3.0, 1.0, 2.0, 1.0],
33+
"status": ["NA", "DK", "ok", ""],
34+
"comment": [long_text, "", "näide", long_text],
35+
"interview_date": [23123.0, 23124.0, 23125.0, 23126.0],
36+
})
37+
pyreadstat.write_sav(
38+
frame,
39+
destination,
40+
compress=destination.suffix.lower() == ".zsav",
41+
file_label="OpenStatSpec supported-semantics fixture",
42+
column_labels={
43+
"lowest_range": "LOWEST-style missing range",
44+
"highest_range": "HIGHEST-style missing range",
45+
"code": "Ordered numeric code",
46+
"status": "String missing code",
47+
"comment": "Long UTF-8 comment",
48+
"interview_date": "SPSS numeric date",
49+
},
50+
variable_value_labels={
51+
# Deliberately non-sorted insertion order is part of the fixture.
52+
"code": {3.0: "third", 1.0: "first", 2.0: "second"},
53+
"status": {"DK": "don't know", "NA": "not answered", "ok": "valid"},
54+
},
55+
missing_ranges={
56+
"lowest_range": [{"lo": -sys.float_info.max, "hi": -1.0}, 7.0],
57+
"highest_range": [{"lo": 1.0, "hi": sys.float_info.max}, -99.0],
58+
"status": ["NA", "DK"],
59+
},
60+
variable_measure={
61+
"lowest_range": "scale",
62+
"highest_range": "scale",
63+
"code": "ordinal",
64+
"status": "nominal",
65+
"comment": "nominal",
66+
"interview_date": "scale",
67+
},
68+
variable_display_width={
69+
"lowest_range": 12,
70+
"highest_range": 12,
71+
"code": 8,
72+
"status": 12,
73+
"comment": 48,
74+
"interview_date": 11,
75+
},
76+
variable_format={
77+
"lowest_range": "F12.1",
78+
"highest_range": "F12.1",
79+
"code": "F8.0",
80+
"status": "A8",
81+
"comment": "A1024",
82+
"interview_date": "DATE11",
83+
},
84+
note=["First ordered document", "Teine dokument: Õ🙂"],
85+
)
86+
return {"long_text": long_text}
87+
88+
89+
def _canonical_missing_endpoint(value: Any, *, endpoint: str) -> Any:
90+
"""Normalize ReadStat's public sentinels for SPSS LOWEST/HIGHEST."""
91+
if isinstance(value, float):
92+
if math.isnan(value):
93+
return "<LOWEST>" if endpoint == "lo" else "<SYSTEM-NAN>"
94+
if math.isinf(value):
95+
return "<HIGHEST>" if value > 0 else "<LOWEST>"
96+
return value
97+
98+
99+
def _canonical_missing_ranges(metadata: Any) -> dict[str, tuple[tuple[Any, Any], ...]]:
100+
result: dict[str, tuple[tuple[Any, Any], ...]] = {}
101+
for variable, rules in (getattr(metadata, "missing_ranges", {}) or {}).items():
102+
result[variable] = tuple(
103+
(
104+
_canonical_missing_endpoint(rule.get("lo"), endpoint="lo"),
105+
_canonical_missing_endpoint(rule.get("hi"), endpoint="hi"),
106+
)
107+
for rule in rules
108+
)
109+
return result
110+
111+
112+
def _ordered_value_labels(metadata: Any) -> dict[str, tuple[tuple[Any, str], ...]]:
113+
return {
114+
variable: tuple((value, str(label)) for value, label in labels.items())
115+
for variable, labels in (getattr(metadata, "variable_value_labels", {}) or {}).items()
116+
}
8117

9118

10119
def compare_sav_semantics(source: str | Path, exported: str | Path) -> dict[str, Any]:
@@ -13,11 +122,24 @@ def compare_sav_semantics(source: str | Path, exported: str | Path) -> dict[str,
13122
failures: list[str] = []
14123
if list(source_frame.columns) != list(exported_frame.columns):
15124
failures.append("variable-order")
16-
if not source_frame.equals(exported_frame):
125+
try:
126+
assert_frame_equal(source_frame, exported_frame, check_dtype=True, check_like=False)
127+
except AssertionError:
17128
failures.append("values-or-case-order")
18-
for attribute in ("column_labels", "original_variable_types", "variable_measure", "missing_ranges", "notes", "file_label"):
129+
for attribute in (
130+
"column_labels",
131+
"original_variable_types",
132+
"variable_measure",
133+
"variable_display_width",
134+
"variable_storage_width",
135+
"notes",
136+
"file_label",
137+
"file_encoding",
138+
):
19139
if getattr(source_meta, attribute, None) != getattr(exported_meta, attribute, None):
20140
failures.append(attribute)
21-
if getattr(source_meta, "variable_value_labels", {}) != getattr(exported_meta, "variable_value_labels", {}):
141+
if _canonical_missing_ranges(source_meta) != _canonical_missing_ranges(exported_meta):
142+
failures.append("missing_ranges")
143+
if _ordered_value_labels(source_meta) != _ordered_value_labels(exported_meta):
22144
failures.append("variable_value_labels")
23-
return {"equivalent": not failures, "differences": failures}
145+
return {"equivalent": not failures, "differences": failures}

‎tests/test_sav_sqlite.py‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import hashlib
22
import pytest
33
import json
4-
from conformance import compare_sav_semantics
4+
from conformance import compare_sav_semantics, write_supported_semantics_fixture
55
import sqlite3
66

77
import openstatspec
@@ -192,3 +192,47 @@ def test_legacy_json_metadata_remains_exportable_after_catalog_migration(tmp_pat
192192
assert metadata.notes == ["legacy document"]
193193
assert metadata.variable_value_labels == {"code": {1.0: "one"}}
194194
assert metadata.missing_ranges == {"code": [{"lo": 2.0, "hi": 2.0}]}
195+
196+
197+
@pytest.mark.parametrize("suffix", [".sav", ".zsav"])
198+
def test_supported_spss_semantics_fixture_round_trips_through_sqlite(tmp_path, suffix: str) -> None:
199+
# Attributes, variable sets, and MR sets are explicit fidelity losses, not
200+
# currently supported round-trip semantics.
201+
source = tmp_path / f"supported{suffix}"
202+
database_path = tmp_path / f"supported-{suffix[1:]}.sqlite"
203+
destination = tmp_path / f"supported-roundtrip{suffix}"
204+
expected = write_supported_semantics_fixture(source)
205+
dataset_id = f"supported-{suffix[1:]}"
206+
207+
result = openstatspec.import_sav(
208+
source, database_url=f"sqlite:///{database_path}", dataset_id=dataset_id,
209+
)
210+
assert result["case_count"] == 4
211+
assert openstatspec.validate(
212+
database_url=f"sqlite:///{database_path}", dataset_id=dataset_id
213+
)["valid"] is True
214+
215+
connection = sqlite3.connect(database_path)
216+
assert connection.execute("select source_format, source_encoding from dataset_catalog where dataset_id = ?", (dataset_id,)).fetchone() == (suffix[1:].upper(), "UTF-8")
217+
data_table = "data_supported_sav" if suffix == ".sav" else "data_supported_zsav"
218+
assert connection.execute(
219+
f"select comment from {data_table} order by __case_ordinal"
220+
).fetchall()[0] == (expected["long_text"],)
221+
# Typed normalized catalog values retain insertion order and string-vs-number identity.
222+
assert connection.execute(
223+
"select ordinal, numeric_value, text_value, label from value_label_catalog "
224+
"where dataset_id = ? and variable_ordinal = 3 order by ordinal",
225+
(dataset_id,),
226+
).fetchall() == [
227+
(1, 3.0, None, "third"),
228+
(2, 1.0, None, "first"),
229+
(3, 2.0, None, "second"),
230+
]
231+
232+
openstatspec.export_sav(
233+
database_url=f"sqlite:///{database_path}",
234+
dataset_id=dataset_id,
235+
destination=destination,
236+
allow_loss=["unobservable-source-dictionary-features"],
237+
)
238+
assert compare_sav_semantics(source, destination) == {"equivalent": True, "differences": []}

0 commit comments

Comments
 (0)