1- """Semantic comparison helpers for supported SAV/ZSAV round trips."""
1+ """Reusable fixtures and semantic comparators for supported SAV/ZSAV round trips.
22
3+ These helpers deliberately compare only metadata that the Python adapter and
4+ its current SAV writer expose. Attributes, variable sets, and
5+ multiple-response sets are not asserted here because they are explicitly
6+ reported as unsupported fidelity losses.
7+ """
8+
9+ import math
10+ import sys
311from pathlib import Path
412from typing import Any
513
614import pandas as pd
715import pyreadstat
16+ from pandas .testing import assert_frame_equal
17+
18+
19+ def write_supported_semantics_fixture (destination : str | Path ) -> dict [str , Any ]:
20+ """Write a fixture spanning every currently supported dictionary semantic.
21+
22+ -sys.float_info.max and sys.float_info.max are how ReadStat represents
23+ SPSS LOWEST and HIGHEST missing-range endpoints on write. ReadStat exposes
24+ them as NaN and +inf when the SAV/ZSAV file is read back; the comparator
25+ canonicalizes those markers.
26+ """
27+ destination = Path (destination )
28+ long_text = "Õ🙂漢字" * 90
29+ frame = pd .DataFrame ({
30+ "lowest_range" : [- 2.0 , - 1.0 , 0.0 , 7.0 ],
31+ "highest_range" : [- 99.0 , 0.0 , 1.0 , 42.0 ],
32+ "code" : [3.0 , 1.0 , 2.0 , 1.0 ],
33+ "status" : ["NA" , "DK" , "ok" , "" ],
34+ "comment" : [long_text , "" , "näide" , long_text ],
35+ "interview_date" : [23123.0 , 23124.0 , 23125.0 , 23126.0 ],
36+ })
37+ pyreadstat .write_sav (
38+ frame ,
39+ destination ,
40+ compress = destination .suffix .lower () == ".zsav" ,
41+ file_label = "OpenStatSpec supported-semantics fixture" ,
42+ column_labels = {
43+ "lowest_range" : "LOWEST-style missing range" ,
44+ "highest_range" : "HIGHEST-style missing range" ,
45+ "code" : "Ordered numeric code" ,
46+ "status" : "String missing code" ,
47+ "comment" : "Long UTF-8 comment" ,
48+ "interview_date" : "SPSS numeric date" ,
49+ },
50+ variable_value_labels = {
51+ # Deliberately non-sorted insertion order is part of the fixture.
52+ "code" : {3.0 : "third" , 1.0 : "first" , 2.0 : "second" },
53+ "status" : {"DK" : "don't know" , "NA" : "not answered" , "ok" : "valid" },
54+ },
55+ missing_ranges = {
56+ "lowest_range" : [{"lo" : - sys .float_info .max , "hi" : - 1.0 }, 7.0 ],
57+ "highest_range" : [{"lo" : 1.0 , "hi" : sys .float_info .max }, - 99.0 ],
58+ "status" : ["NA" , "DK" ],
59+ },
60+ variable_measure = {
61+ "lowest_range" : "scale" ,
62+ "highest_range" : "scale" ,
63+ "code" : "ordinal" ,
64+ "status" : "nominal" ,
65+ "comment" : "nominal" ,
66+ "interview_date" : "scale" ,
67+ },
68+ variable_display_width = {
69+ "lowest_range" : 12 ,
70+ "highest_range" : 12 ,
71+ "code" : 8 ,
72+ "status" : 12 ,
73+ "comment" : 48 ,
74+ "interview_date" : 11 ,
75+ },
76+ variable_format = {
77+ "lowest_range" : "F12.1" ,
78+ "highest_range" : "F12.1" ,
79+ "code" : "F8.0" ,
80+ "status" : "A8" ,
81+ "comment" : "A1024" ,
82+ "interview_date" : "DATE11" ,
83+ },
84+ note = ["First ordered document" , "Teine dokument: Õ🙂" ],
85+ )
86+ return {"long_text" : long_text }
87+
88+
89+ def _canonical_missing_endpoint (value : Any , * , endpoint : str ) -> Any :
90+ """Normalize ReadStat's public sentinels for SPSS LOWEST/HIGHEST."""
91+ if isinstance (value , float ):
92+ if math .isnan (value ):
93+ return "<LOWEST>" if endpoint == "lo" else "<SYSTEM-NAN>"
94+ if math .isinf (value ):
95+ return "<HIGHEST>" if value > 0 else "<LOWEST>"
96+ return value
97+
98+
99+ def _canonical_missing_ranges (metadata : Any ) -> dict [str , tuple [tuple [Any , Any ], ...]]:
100+ result : dict [str , tuple [tuple [Any , Any ], ...]] = {}
101+ for variable , rules in (getattr (metadata , "missing_ranges" , {}) or {}).items ():
102+ result [variable ] = tuple (
103+ (
104+ _canonical_missing_endpoint (rule .get ("lo" ), endpoint = "lo" ),
105+ _canonical_missing_endpoint (rule .get ("hi" ), endpoint = "hi" ),
106+ )
107+ for rule in rules
108+ )
109+ return result
110+
111+
112+ def _ordered_value_labels (metadata : Any ) -> dict [str , tuple [tuple [Any , str ], ...]]:
113+ return {
114+ variable : tuple ((value , str (label )) for value , label in labels .items ())
115+ for variable , labels in (getattr (metadata , "variable_value_labels" , {}) or {}).items ()
116+ }
8117
9118
10119def compare_sav_semantics (source : str | Path , exported : str | Path ) -> dict [str , Any ]:
@@ -13,11 +122,24 @@ def compare_sav_semantics(source: str | Path, exported: str | Path) -> dict[str,
13122 failures : list [str ] = []
14123 if list (source_frame .columns ) != list (exported_frame .columns ):
15124 failures .append ("variable-order" )
16- if not source_frame .equals (exported_frame ):
125+ try :
126+ assert_frame_equal (source_frame , exported_frame , check_dtype = True , check_like = False )
127+ except AssertionError :
17128 failures .append ("values-or-case-order" )
18- for attribute in ("column_labels" , "original_variable_types" , "variable_measure" , "missing_ranges" , "notes" , "file_label" ):
129+ for attribute in (
130+ "column_labels" ,
131+ "original_variable_types" ,
132+ "variable_measure" ,
133+ "variable_display_width" ,
134+ "variable_storage_width" ,
135+ "notes" ,
136+ "file_label" ,
137+ "file_encoding" ,
138+ ):
19139 if getattr (source_meta , attribute , None ) != getattr (exported_meta , attribute , None ):
20140 failures .append (attribute )
21- if getattr (source_meta , "variable_value_labels" , {}) != getattr (exported_meta , "variable_value_labels" , {}):
141+ if _canonical_missing_ranges (source_meta ) != _canonical_missing_ranges (exported_meta ):
142+ failures .append ("missing_ranges" )
143+ if _ordered_value_labels (source_meta ) != _ordered_value_labels (exported_meta ):
22144 failures .append ("variable_value_labels" )
23- return {"equivalent" : not failures , "differences" : failures }
145+ return {"equivalent" : not failures , "differences" : failures }
0 commit comments