Skip to content

Commit a899eb7

Browse files
Fix SPSS VLS compatible names (#3)
Fix SPSS VLS compatible names
2 parents 034ce47 + 8b40b32 commit a899eb7

5 files changed

Lines changed: 424 additions & 35 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ SPSS profile.
1212
- Import of unencrypted SAV and ZSAV sources into one dedicated wide SQL table
1313
and its metadata catalog.
1414
- Export of supported dataset semantics to SAV and ZSAV.
15+
- Consistent legacy compatible-name rewriting across type-2, subtype-13, and
16+
VLS subtype-14 records, with fail-closed SAV/ZSAV validation.
1517
- SQLite, PostgreSQL, MySQL, and MariaDB profiles, including service-backed
1618
PostgreSQL 17/18, MySQL 8.4/9.7, and MariaDB 11.4/11.8/12.3 CI coverage.
1719
- Preflight checks for target profile limits, atomic imports, validation, a

‎docs/sav-profile.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ of this boundary. It records the pinned source commit and installed engine versi
4141
| Ordered document text | Supported through a strict type-6 dictionary bridge plus the pinned fork | Import stores normalized document rows; export creates a temporary UTF-8 SAV source and has IBM I/O copy the records into SAV or ZSAV without invalidating ZSAV dictionary offsets. |
4242
| Print and write formats independently | Supported as raw IBM I/O tuples | The adapter stores both tuples separately and writes them without collapsing either value. |
4343
| Variable sets | Supported through raw IBM I/O | The adapter stores source sets in the extension catalog and writes them through the raw dictionary setter. Invalid target definitions fail before data are written. |
44-
| Legacy compatible variable names | Supported through the strict dictionary bridge | The adapter stores the short-name mapping and rewrites both the fixed variable record and long-name extension before export. |
44+
| Legacy compatible variable names | Supported through the strict dictionary bridge | The adapter atomically rewrites and reparses type-2, subtype-13, and VLS subtype-14 names as one consistent SAV/ZSAV dictionary; malformed or duplicate subtype-14 entries fail closed. |
4545
| Source encoding | UTF-8 by default; legacy code page with an explicit OS locale | The caller supplies legacy_locale and the writer verifies that the emitted encoding matches the stored source encoding. Without it, export fails before output creation. |
4646
| Custom attributes with one value | Supported | File and variable scalar attributes round trip through the normalized attribute catalog. |
4747
| Custom-attribute value arrays | Supported through raw IBM I/O | The adapter represents an array as IBM SPSS `Name[1]`, `Name[2]`, … members and reconstructs the ordered array in the normalized catalog. |

‎src/openstatspec/spss/raw_dictionary.py‎

Lines changed: 259 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@
55
"""
66
from __future__ import annotations
77

8+
import os
9+
import stat
810
from dataclasses import dataclass
911
from pathlib import Path
12+
from tempfile import NamedTemporaryFile
1013
from typing import Iterable
1114

1215

@@ -70,32 +73,31 @@ def write_document_lines(path: str | Path, lines: Iterable[str], *, encoding: st
7073
def write_compatible_names(
7174
path: str | Path, names: dict[str, str], *, encoding: str,
7275
) -> None:
73-
"""Set exact legacy short names without changing long source variable names.
76+
"""Set exact legacy short names consistently in type-2/13/14 records.
7477
75-
IBM I/O has no compatible-name setter. The standard long-name extension
76-
and the fixed 8-byte variable records contain the complete mapping, so this
77-
narrowly rewrites only those two dictionary locations. Every requested
78-
source name must have a long-name record; otherwise the operation fails
79-
instead of silently deriving another name.
78+
IBM I/O has no compatible-name setter. Every requested source name must
79+
have a long-name record, and every very-long-string key must agree with the
80+
type-2 and subtype-13 records. The fully rebuilt dictionary is reparsed
81+
before an atomic replacement is published.
8082
"""
8183
if not names:
8284
return
8385
target = Path(path)
8486
data = target.read_bytes()
8587
byte_order, records = _records(data)
8688
terminator = next(record for record in records if record.record_type == 999)
87-
long_names = next(
88-
(
89-
record for record in records
90-
if record.record_type == 7 and _int(data, record.start + 4, byte_order) == 13
91-
),
92-
None,
93-
)
94-
if long_names is None:
89+
long_name_records = [
90+
record for record in records
91+
if record.record_type == 7 and _int(data, record.start + 4, byte_order) == 13
92+
]
93+
if not long_name_records:
9594
raise RawDictionaryError("SAV dictionary has no long-variable-name record.")
96-
payload_start = long_names.start + 16
97-
payload = data[payload_start : long_names.end]
98-
pairs = _long_name_pairs(payload, encoding)
95+
if len(long_name_records) != 1:
96+
raise RawDictionaryError("SAV dictionary has more than one long-variable-name record.")
97+
long_names = long_name_records[0]
98+
if _int(data, long_names.start + 8, byte_order) != 1:
99+
raise RawDictionaryError("Invalid SAV long-variable-name record dimensions.")
100+
pairs = _long_name_pairs(data[long_names.start + 16 : long_names.end], encoding)
99101
replacements: dict[str, str] = {}
100102
updated_pairs: list[tuple[str, str]] = []
101103
unresolved = set(names)
@@ -112,6 +114,44 @@ def write_compatible_names(
112114
raise RawDictionaryError(
113115
"Compatible-name update requires a long-name record for: " + missing
114116
)
117+
updated_short_names = [short_name.casefold() for short_name, _ in updated_pairs]
118+
if len(updated_short_names) != len(set(updated_short_names)):
119+
raise RawDictionaryError("Compatible-name update would create duplicate short names.")
120+
121+
type_2_names = _type_2_names(data, records, byte_order)
122+
for short_name in replacements:
123+
if type_2_names.count(short_name) != 1:
124+
raise RawDictionaryError(
125+
f"Expected exactly one type-2 record for compatible name {short_name!r}."
126+
)
127+
128+
long_name_by_short = {short_name: long_name for short_name, long_name in pairs}
129+
vls_records = [
130+
record for record in records
131+
if record.record_type == 7 and _int(data, record.start + 4, byte_order) == 14
132+
]
133+
parsed_vls: dict[int, tuple[list[tuple[str, bytes]], bool]] = {}
134+
seen_vls: set[str] = set()
135+
vls_replacements: dict[str, str] = {}
136+
vls_source_names: set[str] = set()
137+
for record in vls_records:
138+
if _int(data, record.start + 8, byte_order) != 1:
139+
raise RawDictionaryError("Invalid SAV very-long-string record dimensions.")
140+
entries, trailing_tab = _very_long_string_entries(data[record.start + 16 : record.end])
141+
parsed_vls[record.start] = (entries, trailing_tab)
142+
for short_name, _ in entries:
143+
normalized = short_name.casefold()
144+
if normalized in seen_vls:
145+
raise RawDictionaryError("Duplicate SAV very-long-string entry.")
146+
seen_vls.add(normalized)
147+
if short_name not in long_name_by_short or type_2_names.count(short_name) != 1:
148+
raise RawDictionaryError(
149+
"SAV type-2/subtype-13/subtype-14 names are inconsistent."
150+
)
151+
replacement = replacements.get(short_name)
152+
if replacement is not None:
153+
vls_replacements[short_name] = replacement
154+
vls_source_names.add(long_name_by_short[short_name])
115155

116156
mutable = bytearray(data)
117157
for record in records:
@@ -122,22 +162,50 @@ def write_compatible_names(
122162
if replacement is not None:
123163
mutable[record.start + 24 : record.start + 32] = replacement.encode("ascii").ljust(8, b" ")
124164

125-
new_payload = b" ".join(
165+
new_payload = b"\t".join(
126166
short_name.encode("ascii") + b"=" + long_name.encode(encoding)
127167
for short_name, long_name in updated_pairs
128168
)
129-
header = bytearray(mutable[long_names.start : long_names.start + 16])
130-
header[12:16] = _pack(len(new_payload), byte_order)
131-
updated = (
132-
bytes(mutable[: long_names.start]) + bytes(header) + new_payload
133-
+ bytes(mutable[long_names.end :])
134-
)
135-
target.write_bytes(_shift_zsav_offsets(
169+
replacement_records = {
170+
long_names.start: _extension_record(
171+
mutable, long_names, new_payload, byte_order=byte_order,
172+
),
173+
}
174+
for record in vls_records:
175+
entries, trailing_tab = parsed_vls[record.start]
176+
rewritten = [
177+
(vls_replacements.get(short_name, short_name), width)
178+
for short_name, width in entries
179+
]
180+
vls_payload = _very_long_string_payload(rewritten, trailing_tab=trailing_tab)
181+
replacement_records[record.start] = _extension_record(
182+
mutable, record, vls_payload, byte_order=byte_order,
183+
)
184+
185+
chunks: list[bytes] = []
186+
cursor = 0
187+
for record in sorted(
188+
(record for record in records if record.start in replacement_records),
189+
key=lambda item: item.start,
190+
):
191+
chunks.append(bytes(mutable[cursor : record.start]))
192+
chunks.append(replacement_records[record.start])
193+
cursor = record.end
194+
chunks.append(bytes(mutable[cursor:]))
195+
updated = b"".join(chunks)
196+
updated = _shift_zsav_offsets(
136197
updated,
137198
original_data_start=terminator.end,
138199
delta=len(updated) - len(data),
139200
byte_order=byte_order,
140-
))
201+
)
202+
_assert_compatible_name_consistency(
203+
updated,
204+
requested=names,
205+
vls_source_names=vls_source_names,
206+
encoding=encoding,
207+
)
208+
_atomic_write_bytes(target, updated)
141209

142210

143211
def write_extended_mrset_labels(
@@ -212,18 +280,181 @@ def write_extended_mrset_labels(
212280

213281

214282
def _long_name_pairs(payload: bytes, encoding: str) -> list[tuple[str, str]]:
283+
if not payload:
284+
raise RawDictionaryError("Empty SAV long-variable-name record.")
215285
pairs: list[tuple[str, str]] = []
216-
for raw_pair in payload.split(b" "):
286+
short_names: set[str] = set()
287+
long_names: set[str] = set()
288+
for raw_pair in payload.split(b"\t"):
217289
try:
218290
raw_short, raw_long = raw_pair.split(b"=", maxsplit=1)
219-
short_name = raw_short.decode("ascii")
291+
short_name = _dictionary_short_name(raw_short)
220292
long_name = raw_long.decode(encoding)
221293
except (UnicodeDecodeError, ValueError) as error:
222294
raise RawDictionaryError("Invalid SAV long-variable-name record.") from error
295+
normalized_short = short_name.casefold()
296+
normalized_long = long_name.casefold()
297+
if not long_name or normalized_short in short_names or normalized_long in long_names:
298+
raise RawDictionaryError("Duplicate or empty SAV long-variable-name entry.")
299+
short_names.add(normalized_short)
300+
long_names.add(normalized_long)
223301
pairs.append((short_name, long_name))
224302
return pairs
225303

226304

305+
def _very_long_string_entries(payload: bytes) -> tuple[list[tuple[str, bytes]], bool]:
306+
"""Parse subtype-14 entries without accepting ambiguous separators or widths."""
307+
if not payload:
308+
raise RawDictionaryError("Empty SAV very-long-string record.")
309+
raw_entries = payload.split(b"\t")
310+
trailing_tab = raw_entries[-1] == b""
311+
if trailing_tab:
312+
raw_entries.pop()
313+
if not raw_entries or any(not entry for entry in raw_entries):
314+
raise RawDictionaryError("Invalid SAV very-long-string separators.")
315+
entries: list[tuple[str, bytes]] = []
316+
for raw_entry in raw_entries:
317+
if not raw_entry.endswith(b"\x00"):
318+
raise RawDictionaryError("SAV very-long-string entry is missing its NUL terminator.")
319+
try:
320+
raw_short, raw_width = raw_entry[:-1].split(b"=", maxsplit=1)
321+
short_name = _dictionary_short_name(raw_short)
322+
if not raw_width or not raw_width.isdigit():
323+
raise ValueError
324+
width = int(raw_width.decode("ascii"))
325+
except (UnicodeDecodeError, ValueError) as error:
326+
raise RawDictionaryError("Invalid SAV very-long-string entry.") from error
327+
if not 256 <= width <= 32767:
328+
raise RawDictionaryError("SAV very-long-string width is outside 256..32767.")
329+
entries.append((short_name, raw_width))
330+
return entries, trailing_tab
331+
332+
333+
def _very_long_string_payload(
334+
entries: Iterable[tuple[str, bytes]], *, trailing_tab: bool,
335+
) -> bytes:
336+
payload = b"\t".join(
337+
short_name.encode("ascii") + b"=" + width + b"\x00"
338+
for short_name, width in entries
339+
)
340+
return payload + (b"\t" if trailing_tab else b"")
341+
342+
343+
def _dictionary_short_name(raw_name: bytes) -> str:
344+
try:
345+
name = raw_name.decode("ascii")
346+
except UnicodeDecodeError as error:
347+
raise RawDictionaryError("SAV dictionary short name is not ASCII.") from error
348+
allowed_first = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz@#$"
349+
allowed_rest = allowed_first + b"0123456789_."
350+
if not 1 <= len(raw_name) <= 8 or raw_name[:1] not in allowed_first:
351+
raise RawDictionaryError("Invalid SAV dictionary short name.")
352+
if any(character not in allowed_rest for character in raw_name[1:]):
353+
raise RawDictionaryError("Invalid SAV dictionary short name.")
354+
return name
355+
356+
357+
def _type_2_names(data: bytes, records: list[_Record], byte_order: str) -> list[str]:
358+
names: list[str] = []
359+
for record in records:
360+
if record.record_type != 2 or _int(data, record.start + 4, byte_order) < 0:
361+
continue
362+
raw_name = data[record.start + 24 : record.start + 32].rstrip(b" ")
363+
names.append(_dictionary_short_name(raw_name))
364+
return names
365+
366+
367+
def _extension_record(
368+
data: bytes, record: _Record, payload: bytes, *, byte_order: str,
369+
) -> bytes:
370+
header = bytearray(data[record.start : record.start + 16])
371+
header[8:12] = _pack(1, byte_order)
372+
header[12:16] = _pack(len(payload), byte_order)
373+
return bytes(header) + payload
374+
375+
376+
def _assert_compatible_name_consistency(
377+
data: bytes,
378+
*,
379+
requested: dict[str, str],
380+
vls_source_names: set[str],
381+
encoding: str,
382+
) -> None:
383+
byte_order, records = _records(data)
384+
type_2_names = _type_2_names(data, records, byte_order)
385+
normalized_type_2 = [name.casefold() for name in type_2_names]
386+
long_name_records = [
387+
record for record in records
388+
if record.record_type == 7 and _int(data, record.start + 4, byte_order) == 13
389+
]
390+
if len(long_name_records) != 1:
391+
raise RawDictionaryError("Rewritten SAV has an inconsistent long-variable-name record.")
392+
long_name_record = long_name_records[0]
393+
if _int(data, long_name_record.start + 8, byte_order) != 1:
394+
raise RawDictionaryError("Rewritten SAV has invalid long-variable-name dimensions.")
395+
pairs = _long_name_pairs(
396+
data[long_name_record.start + 16 : long_name_record.end], encoding,
397+
)
398+
pair_by_long = {long_name: short_name for short_name, long_name in pairs}
399+
normalized_pair_names = {short_name.casefold() for short_name, _ in pairs}
400+
for short_name, _ in pairs:
401+
if normalized_type_2.count(short_name.casefold()) != 1:
402+
raise RawDictionaryError("Rewritten type-2 and subtype-13 names are inconsistent.")
403+
404+
vls_names: list[str] = []
405+
for record in records:
406+
if record.record_type != 7 or _int(data, record.start + 4, byte_order) != 14:
407+
continue
408+
if _int(data, record.start + 8, byte_order) != 1:
409+
raise RawDictionaryError("Rewritten SAV has invalid very-long-string dimensions.")
410+
entries, _ = _very_long_string_entries(data[record.start + 16 : record.end])
411+
vls_names.extend(short_name for short_name, _ in entries)
412+
normalized_vls = [name.casefold() for name in vls_names]
413+
if len(normalized_vls) != len(set(normalized_vls)):
414+
raise RawDictionaryError("Rewritten SAV has duplicate very-long-string entries.")
415+
for short_name in normalized_vls:
416+
if short_name not in normalized_pair_names or normalized_type_2.count(short_name) != 1:
417+
raise RawDictionaryError(
418+
"Rewritten type-2/subtype-13/subtype-14 names are inconsistent."
419+
)
420+
421+
for source_name, requested_name in requested.items():
422+
replacement = _validated_compatible_name(requested_name)
423+
if pair_by_long.get(source_name) != replacement:
424+
raise RawDictionaryError("Requested compatible name is absent after dictionary rewrite.")
425+
if normalized_type_2.count(replacement.casefold()) != 1:
426+
raise RawDictionaryError("Requested compatible name is inconsistent with type-2 records.")
427+
if (
428+
source_name in vls_source_names
429+
and normalized_vls.count(replacement.casefold()) != 1
430+
):
431+
raise RawDictionaryError("Requested compatible name is inconsistent with subtype-14.")
432+
433+
434+
def _atomic_write_bytes(target: Path, data: bytes) -> None:
435+
"""Publish validated dictionary bytes without exposing a partial replacement."""
436+
mode = stat.S_IMODE(target.stat().st_mode)
437+
temporary: Path | None = None
438+
try:
439+
with NamedTemporaryFile(
440+
mode="wb",
441+
dir=target.parent,
442+
prefix=f".{target.name}.",
443+
suffix=".tmp",
444+
delete=False,
445+
) as handle:
446+
temporary = Path(handle.name)
447+
handle.write(data)
448+
handle.flush()
449+
os.fsync(handle.fileno())
450+
os.chmod(temporary, mode)
451+
os.replace(temporary, target)
452+
temporary = None
453+
finally:
454+
if temporary is not None:
455+
temporary.unlink(missing_ok=True)
456+
457+
227458
def _validated_compatible_name(value: str) -> str:
228459
encoded = str(value).encode("ascii")
229460
if not 1 <= len(encoded) <= 8:

0 commit comments

Comments
 (0)