Catalog source table and ReadStat storage types

Author: TonisOrmisson

src/openstatspec/spss/sav.py

@@ -23,12 +23,14 @@ def _variables(meta: Any, names: list[str]) -> list[dict[str, Any]]:
     value_labels = dict(getattr(meta, "variable_value_labels", {}) or {})
     missing_ranges = dict(getattr(meta, "missing_ranges", {}) or {})
     widths = dict(getattr(meta, "variable_storage_width", {}) or {})
+    readstat_types = dict(getattr(meta, "readstat_variable_types", {}) or {})
     result = []
     for ordinal, source_name in enumerate(names, start=1):
         kind = "string" if str(formats.get(source_name, "")).upper().startswith("A") else "numeric"
         result.append({
             "ordinal": ordinal, "source_name": source_name,
             "physical_name": physical_name(source_name, used), "storage_kind": kind,
+            "readstat_storage_type": readstat_types.get(source_name),
             "string_width": widths.get(source_name) if kind == "string" else None,
             "label": labels.get(source_name, "") or "", "format": formats.get(source_name),
             "measure": measures.get(source_name), "alignment": alignments.get(source_name),
@@ -79,6 +81,7 @@ def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str
         source_format=source_path.suffix[1:].upper(), rows=_rows(frame, variables),
         variables=variables, file_label=getattr(meta, "file_label", "") or "",
         source_encoding=getattr(meta, "file_encoding", None),
+        source_table_name=getattr(meta, "table_name", None),
         source_sha256=_sha256(source_path),
         source_created_at=_iso_datetime(getattr(meta, "creation_time", None)),
         source_modified_at=_iso_datetime(getattr(meta, "modification_time", None)),

src/openstatspec/sql/wide.py

@@ -18,6 +18,7 @@ def catalog(metadata: MetaData) -> tuple[Table, Table]:
         Column("data_table", String(255), nullable=False, unique=True),
         Column("source_format", String(16), nullable=False),
         Column("source_name", Text, nullable=False),
+        Column("source_table_name", Text),
         Column("source_sha256", String(64), nullable=False),
         Column("source_created_at", String(40)),
         Column("source_modified_at", String(40)),
@@ -35,6 +36,7 @@ def catalog(metadata: MetaData) -> tuple[Table, Table]:
         Column("source_name", String(255), nullable=False),
         Column("physical_name", String(255), nullable=False),
         Column("storage_kind", String(16), nullable=False),
+        Column("readstat_storage_type", String(32)),
         Column("string_width", Integer),
         Column("label", Text, nullable=False, default=""),
         Column("format", String(64)),
@@ -96,6 +98,7 @@ def create_wide_dataset(
     *, database_url: str, dataset_id: str, source_name: str, source_format: str,
     rows: Iterable[Mapping[str, Any]], variables: list[dict[str, Any]], file_label: str = "",
     source_encoding: str | None = None, documents: str = "[]",
+    source_table_name: str | None = None,
     source_sha256: str = "",
     source_created_at: str | None = None, source_modified_at: str | None = None,
     imported_at: str = "",
@@ -124,6 +127,7 @@ def create_wide_dataset(
         connection.execute(insert(datasets).values(
             dataset_id=dataset_id, data_table=data_table.name, source_format=source_format,
             source_name=source_name, source_encoding=source_encoding, case_count=len(materialized),
+            source_table_name=source_table_name,
             source_sha256=source_sha256,
             source_created_at=source_created_at, source_modified_at=source_modified_at,
             imported_at=imported_at,