Skip to content

Commit d4cc506

Browse files
committed
Preserve SPSS compatible variable names
1 parent bacc892 commit d4cc506

9 files changed

Lines changed: 218 additions & 92 deletions

File tree

‎docs/sav-profile.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ of this boundary. It records the pinned source commit and installed engine versi
4141
| Ordered document text | Supported through a strict type-6 dictionary bridge plus the pinned fork | Import stores normalized document rows; export creates a temporary UTF-8 SAV source and has IBM I/O copy the records into SAV or ZSAV without invalidating ZSAV dictionary offsets. |
4242
| Print and write formats independently | Supported as raw IBM I/O tuples | The adapter stores both tuples separately and writes them without collapsing either value. |
4343
| Variable sets | Supported through raw IBM I/O | The adapter stores source sets in the extension catalog and writes them through the raw dictionary setter. Invalid target definitions fail before data are written. |
44-
| Legacy compatible variable names | Fail-closed on export | A compatible name that differs from the long source name is retained in the catalog, but cannot be set through the public writer. Export requires `compatible-variable-name-not-exportable`. |
44+
| Legacy compatible variable names | Supported through the strict dictionary bridge | The adapter stores the short-name mapping and rewrites both the fixed variable record and long-name extension before export. |
4545
| Source encoding | UTF-8 fidelity only | UTF-8 is supported. A legacy code page is retained in metadata, but export requires `source-encoding-not-preserved` because the writer has no legacy-code-page preservation contract. |
4646
| Custom attributes with one value | Supported | File and variable scalar attributes round trip through the normalized attribute catalog. |
4747
| Custom-attribute value arrays | Supported through raw IBM I/O | The adapter represents an array as IBM SPSS `Name[1]`, `Name[2]`, … members and reconstructs the ordered array in the normalized catalog. |

‎src/openstatspec/api.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ def capability_matrix() -> Mapping[str, Any]:
3838
"multiple_response_sets": "supported",
3939
"variable_alignment": "supported",
4040
"variable_sets": "supported",
41-
"compatible_variable_names": "fail-closed-on-export",
41+
"compatible_variable_names": "supported",
4242
"custom_attributes": {
4343
"scalar_values": "supported",
4444
"ordered_value_arrays": "supported",

‎src/openstatspec/spss/raw_dictionary.py‎

Lines changed: 154 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
"""Small, fail-closed helpers for the SPSS type-6 document record.
1+
"""Small, fail-closed helpers for selected raw SPSS dictionary records.
22
33
This is not a second SAV engine: pyspssio still owns values and the normal
44
dictionary. IBM I/O exposes copying document records but not their text.
@@ -51,14 +51,155 @@ def write_document_lines(path: str | Path, lines: Iterable[str], *, encoding: st
5151
if len(documents) > 1:
5252
raise RawDictionaryError("SAV dictionary contains more than one document record.")
5353
replacement = _document_record(lines, encoding=encoding, byte_order=byte_order)
54+
terminator = next(record for record in records if record.record_type == 999)
5455
if documents:
5556
current = documents[0]
56-
target.write_bytes(data[: current.start] + replacement + data[current.end :])
57+
updated = data[: current.start] + replacement + data[current.end :]
58+
else:
59+
extension = next((record for record in records if record.record_type == 7), None)
60+
insertion = extension.start if extension else terminator.start
61+
updated = data[:insertion] + replacement + data[insertion:]
62+
target.write_bytes(_shift_zsav_offsets(
63+
updated,
64+
original_data_start=terminator.end,
65+
delta=len(updated) - len(data),
66+
byte_order=byte_order,
67+
))
68+
69+
70+
def write_compatible_names(
71+
path: str | Path, names: dict[str, str], *, encoding: str,
72+
) -> None:
73+
"""Set exact legacy short names without changing long source variable names.
74+
75+
IBM I/O has no compatible-name setter. The standard long-name extension
76+
and the fixed 8-byte variable records contain the complete mapping, so this
77+
narrowly rewrites only those two dictionary locations. Every requested
78+
source name must have a long-name record; otherwise the operation fails
79+
instead of silently deriving another name.
80+
"""
81+
if not names:
5782
return
58-
extension = next((record for record in records if record.record_type == 7), None)
83+
target = Path(path)
84+
data = target.read_bytes()
85+
byte_order, records = _records(data)
5986
terminator = next(record for record in records if record.record_type == 999)
60-
insertion = extension.start if extension else terminator.start
61-
target.write_bytes(data[:insertion] + replacement + data[insertion:])
87+
long_names = next(
88+
(
89+
record for record in records
90+
if record.record_type == 7 and _int(data, record.start + 4, byte_order) == 13
91+
),
92+
None,
93+
)
94+
if long_names is None:
95+
raise RawDictionaryError("SAV dictionary has no long-variable-name record.")
96+
payload_start = long_names.start + 16
97+
payload = data[payload_start : long_names.end]
98+
pairs = _long_name_pairs(payload, encoding)
99+
replacements: dict[str, str] = {}
100+
updated_pairs: list[tuple[str, str]] = []
101+
unresolved = set(names)
102+
for short_name, long_name in pairs:
103+
replacement = names.get(long_name)
104+
if replacement is None:
105+
updated_pairs.append((short_name, long_name))
106+
continue
107+
replacements[short_name] = _validated_compatible_name(replacement)
108+
updated_pairs.append((replacements[short_name], long_name))
109+
unresolved.remove(long_name)
110+
if unresolved:
111+
missing = ", ".join(sorted(unresolved))
112+
raise RawDictionaryError(
113+
"Compatible-name update requires a long-name record for: " + missing
114+
)
115+
116+
mutable = bytearray(data)
117+
for record in records:
118+
if record.record_type != 2:
119+
continue
120+
short_name = bytes(mutable[record.start + 24 : record.start + 32]).decode(encoding).rstrip(" ")
121+
replacement = replacements.get(short_name)
122+
if replacement is not None:
123+
mutable[record.start + 24 : record.start + 32] = replacement.encode("ascii").ljust(8, b" ")
124+
125+
new_payload = b" ".join(
126+
short_name.encode("ascii") + b"=" + long_name.encode(encoding)
127+
for short_name, long_name in updated_pairs
128+
)
129+
header = bytearray(mutable[long_names.start : long_names.start + 16])
130+
header[12:16] = _pack(len(new_payload), byte_order)
131+
updated = (
132+
bytes(mutable[: long_names.start]) + bytes(header) + new_payload
133+
+ bytes(mutable[long_names.end :])
134+
)
135+
target.write_bytes(_shift_zsav_offsets(
136+
updated,
137+
original_data_start=terminator.end,
138+
delta=len(updated) - len(data),
139+
byte_order=byte_order,
140+
))
141+
142+
143+
def _long_name_pairs(payload: bytes, encoding: str) -> list[tuple[str, str]]:
144+
pairs: list[tuple[str, str]] = []
145+
for raw_pair in payload.split(b" "):
146+
try:
147+
raw_short, raw_long = raw_pair.split(b"=", maxsplit=1)
148+
short_name = raw_short.decode("ascii")
149+
long_name = raw_long.decode(encoding)
150+
except (UnicodeDecodeError, ValueError) as error:
151+
raise RawDictionaryError("Invalid SAV long-variable-name record.") from error
152+
pairs.append((short_name, long_name))
153+
return pairs
154+
155+
156+
def _validated_compatible_name(value: str) -> str:
157+
encoded = str(value).encode("ascii")
158+
if not 1 <= len(encoded) <= 8:
159+
raise RawDictionaryError("SPSS compatible variable names must contain one to eight ASCII bytes.")
160+
first = encoded[:1]
161+
rest = encoded[1:]
162+
if not (first.isalpha() or first == b"@"):
163+
raise RawDictionaryError("SPSS compatible variable names must start with an ASCII letter or @.")
164+
if any(character not in b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.$#"
165+
for character in rest):
166+
raise RawDictionaryError("SPSS compatible variable name contains an unsupported character.")
167+
return encoded.decode("ascii")
168+
169+
170+
def _shift_zsav_offsets(
171+
data: bytes, *, original_data_start: int, delta: int, byte_order: str,
172+
) -> bytes:
173+
"""Shift all absolute ZSAV data offsets after a dictionary-size rewrite."""
174+
if not delta or data[:4] != b"$FL3":
175+
return data
176+
adjusted = bytearray(data)
177+
data_start = original_data_start + delta
178+
_need(adjusted, data_start, 24)
179+
original_zheader = _int64(adjusted, data_start, byte_order)
180+
original_ztrailer = _int64(adjusted, data_start + 8, byte_order)
181+
trailer_length = _int64(adjusted, data_start + 16, byte_order)
182+
if original_zheader != original_data_start:
183+
raise RawDictionaryError("Unexpected ZSAV data-header offset.")
184+
_set_int64(adjusted, data_start, original_zheader + delta, byte_order)
185+
_set_int64(adjusted, data_start + 8, original_ztrailer + delta, byte_order)
186+
trailer_start = original_ztrailer + delta
187+
if trailer_length < 24:
188+
raise RawDictionaryError("Invalid ZSAV data-trailer length.")
189+
_need(adjusted, trailer_start, 24)
190+
block_count = _int(adjusted, trailer_start + 20, byte_order)
191+
if block_count < 0 or trailer_length != 24 + (block_count * 24):
192+
raise RawDictionaryError("Invalid ZSAV data-trailer block metadata.")
193+
_need(adjusted, trailer_start + 24, block_count * 24)
194+
for index in range(block_count):
195+
descriptor = trailer_start + 24 + (index * 24)
196+
_set_int64(adjusted, descriptor, _int64(adjusted, descriptor, byte_order) + delta, byte_order)
197+
_set_int64(
198+
adjusted, descriptor + 8,
199+
_int64(adjusted, descriptor + 8, byte_order) + delta,
200+
byte_order,
201+
)
202+
return bytes(adjusted)
62203

63204

64205
def _document_record(lines: Iterable[str], *, encoding: str, byte_order: str) -> bytes:
@@ -160,6 +301,14 @@ def _int(data: bytes, offset: int, byte_order: str) -> int:
160301
return int.from_bytes(data[offset : offset + 4], byte_order, signed=True)
161302

162303

304+
def _int64(data: bytes, offset: int, byte_order: str) -> int:
305+
return int.from_bytes(data[offset : offset + 8], byte_order, signed=True)
306+
307+
308+
def _set_int64(data: bytearray, offset: int, value: int, byte_order: str) -> None:
309+
data[offset : offset + 8] = int(value).to_bytes(8, byte_order, signed=True)
310+
311+
163312
def _round4(value: int) -> int:
164313
return (value + 3) & ~3
165314

‎src/openstatspec/spss/sav.py‎

Lines changed: 18 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,12 @@
1818
import pandas as pd
1919
import pyspssio
2020

21-
from .raw_dictionary import RawDictionaryError, read_document_lines, write_document_lines
21+
from .raw_dictionary import (
22+
RawDictionaryError,
23+
read_document_lines,
24+
write_compatible_names,
25+
write_document_lines,
26+
)
2227
from .dictionary import (
2328
attribute_pairs,
2429
attribute_values,
@@ -190,7 +195,7 @@ def inspect_sav(source: str | Path) -> dict[str, Any]:
190195
metadata, loss_report = _dictionary(source_path)
191196
names = list(metadata.get("var_names") or [])
192197
variables = _variables(metadata, names)
193-
loss_report = _merge_loss_reports(tuple(loss_report.values()), _compat_name_loss_report(variables))
198+
loss_report = _merge_loss_reports(tuple(loss_report.values()))
194199
return {
195200
"source_format": source_path.suffix[1:].upper(),
196201
"engine": engine_identity(),
@@ -220,7 +225,7 @@ def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str
220225
# frame read's values where a future pyspssio version exposes more there.
221226
metadata = {**dictionary, **metadata}
222227
variables = _variables(metadata, list(frame.columns))
223-
loss_report = _merge_loss_reports(tuple(loss_report.values()), _compat_name_loss_report(variables))
228+
loss_report = _merge_loss_reports(tuple(loss_report.values()))
224229
result = create_wide_dataset(
225230
database_url=database_url,
226231
dataset_id=dataset_id,
@@ -364,6 +369,16 @@ def _write_with_dictionary_bridge(
364369
writer.copy_documents_from(document_source)
365370
writer.commit_header()
366371
writer.write_data(frame)
372+
write_compatible_names(
373+
destination,
374+
{
375+
str(variable["source_name"]): str(variable["compat_name"])
376+
for variable in variables
377+
if variable.get("compat_name")
378+
and str(variable["compat_name"]).casefold() != str(variable["source_name"]).casefold()
379+
},
380+
encoding="UTF-8",
381+
)
367382

368383
def _catalog_format(variable: dict[str, Any], key: str) -> tuple[int, int, int]:
369384
encoded = variable.get(key)
@@ -445,37 +460,6 @@ def _engine_loss_report(metadata: dict[str, Any]) -> dict[str, dict[str, Any]]:
445460
return events
446461

447462

448-
def _compat_name_loss_report(variables: list[dict[str, Any]]) -> tuple[dict[str, Any], ...]:
449-
"""Report legacy SPSS compatible names the writer cannot set explicitly.
450-
451-
A compatible name is meaningful only when it differs from the long source
452-
variable name. ``pyspssio`` exposes these names while reading, but its
453-
public writer metadata has no ``var_compat_names`` input. It may derive a
454-
name today, but that is not a preservation contract: export must therefore
455-
require an explicit, auditable acceptance rather than silently rederive or
456-
rename the value.
457-
"""
458-
events: list[dict[str, Any]] = []
459-
for variable in variables:
460-
source_name = str(variable["source_name"])
461-
compat_name = variable.get("compat_name")
462-
if compat_name is None or str(compat_name).casefold() == source_name.casefold():
463-
continue
464-
events.append({
465-
"code": "compatible-variable-name-not-exportable",
466-
"detail": (
467-
"pyspssio exposes the source compatible variable name but its public "
468-
"writer API cannot set or guarantee preservation of that name."
469-
),
470-
"details": {
471-
"source_name": source_name,
472-
"compatible_name": str(compat_name),
473-
"physical_name": str(variable["physical_name"]),
474-
},
475-
})
476-
return tuple(events)
477-
478-
479463
def _export_loss_report(
480464
dataset: dict[str, Any], variables: list[dict[str, Any]],
481465
) -> tuple[dict[str, Any], ...]:
@@ -486,10 +470,8 @@ def _export_loss_report(
486470
"detail": "The pyspssio writer has no source-encoding preservation contract for this legacy code page.",
487471
"details": {"source_encoding": dataset.get("source_encoding")},
488472
})
489-
events.extend(_compat_name_loss_report(variables))
490473
return tuple(events)
491474

492-
493475
def _merge_loss_reports(*reports: tuple[dict[str, Any], ...]) -> tuple[dict[str, Any], ...]:
494476
events: dict[str, dict[str, Any]] = {}
495477
for report in reports:

‎tests/test_cli.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def test_capability_matrix_is_public_and_cli_matches_engine_boundary(capsys) ->
5656
"multiple_response_sets": "supported",
5757
"variable_alignment": "supported",
5858
"variable_sets": "supported",
59-
"compatible_variable_names": "fail-closed-on-export",
59+
"compatible_variable_names": "supported",
6060
"custom_attributes": {
6161
"scalar_values": "supported",
6262
"ordered_value_arrays": "supported",

‎tests/test_document_round_trip.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,11 @@
66
import pytest
77

88
import openstatspec
9-
from openstatspec.spss.raw_dictionary import read_document_lines, write_document_lines
9+
from openstatspec.spss.raw_dictionary import (
10+
read_document_lines,
11+
write_compatible_names,
12+
write_document_lines,
13+
)
1014

1115

1216
def _source_with_documents(path: Path, lines: list[str]) -> None:
@@ -65,3 +69,20 @@ def test_document_lines_import_from_zsav_and_export_to_sav(tmp_path: Path) -> No
6569
)
6670
assert read_document_lines(destination, encoding="UTF-8") == expected
6771
assert pyspssio.read_sav(str(destination))[0]["answer"].tolist() == [1.0, 2.0]
72+
73+
74+
def test_document_and_compatible_name_round_trip_to_zsav(tmp_path: Path) -> None:
75+
source = tmp_path / "source.sav"
76+
destination = tmp_path / "destination.zsav"
77+
database = f"sqlite:///{tmp_path / 'combined.sqlite'}"
78+
source_name = "long_variable_name"
79+
pyspssio.write_sav(str(source), pd.DataFrame({source_name: [7.0]}))
80+
write_document_lines(source, ["Combined dictionary fixture."], encoding="UTF-8")
81+
write_compatible_names(source, {source_name: "ANSWER"}, encoding="UTF-8")
82+
83+
openstatspec.import_sav(source, database_url=database, dataset_id="combined")
84+
openstatspec.export_sav(database_url=database, dataset_id="combined", destination=destination)
85+
86+
assert read_document_lines(destination, encoding="UTF-8") == ["Combined dictionary fixture."]
87+
assert pyspssio.read_metadata(str(destination))["var_compat_names"][source_name] == "ANSWER"
88+
assert pyspssio.read_sav(str(destination))[0][source_name].tolist() == [7.0]

0 commit comments

Comments
 (0)