Record source artifact hash during import

Author: TonisOrmisson

src/openstatspec/spss/sav.py

@@ -1,5 +1,6 @@
 """Initial unencrypted SAV/ZSAV adapter using pyreadstat."""
 
+import hashlib
 import json
 import math
 from pathlib import Path
@@ -72,6 +73,7 @@ def import_sav_dataset(*, source: str | Path, database_url: str, dataset_id: str
         source_format=source_path.suffix[1:].upper(), rows=_rows(frame, variables),
         variables=variables, file_label=getattr(meta, "file_label", "") or "",
         source_encoding=getattr(meta, "file_encoding", None),
+        source_sha256=_sha256(source_path),
         documents=json.dumps(list(getattr(meta, "notes", []) or [])),
         multiple_response_sets=json.dumps(dict(getattr(meta, "mr_sets", {}) or {}), default=str),
     )
@@ -114,6 +116,14 @@ def export_sav_dataset(*, database_url: str, dataset_id: str, destination: str |
 
 
 
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as source:
+        for chunk in iter(lambda: source.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
 def _import_loss_report(meta: Any) -> tuple[dict[str, str], ...]:
     if getattr(meta, "mr_sets", {}) or {}:
         return ({"code": "multiple-response-sets-not-exportable", "detail": "Multiple-response sets are catalogued but pyreadstat cannot write them back to SAV."},)

src/openstatspec/sql/wide.py

@@ -17,6 +17,7 @@ def catalog(metadata: MetaData) -> tuple[Table, Table]:
         Column("data_table", String(255), nullable=False, unique=True),
         Column("source_format", String(16), nullable=False),
         Column("source_name", Text, nullable=False),
+        Column("source_sha256", String(64), nullable=False),
         Column("source_encoding", String(128)),
         Column("case_count", BigInteger, nullable=False),
         Column("file_label", Text, nullable=False, default=""),
@@ -62,6 +63,7 @@ def create_wide_dataset(
     *, database_url: str, dataset_id: str, source_name: str, source_format: str,
     rows: Iterable[Mapping[str, Any]], variables: list[dict[str, Any]], file_label: str = "",
     source_encoding: str | None = None, documents: str = "[]",
+    source_sha256: str = "",
     multiple_response_sets: str = "{}",
 ) -> dict[str, Any]:
     profile = profile_for_url(database_url)
@@ -84,6 +86,7 @@ def create_wide_dataset(
         connection.execute(insert(datasets).values(
             dataset_id=dataset_id, data_table=data_table.name, source_format=source_format,
             source_name=source_name, source_encoding=source_encoding, case_count=len(materialized),
+            source_sha256=source_sha256,
             file_label=file_label, documents=documents,
             multiple_response_sets=multiple_response_sets,
         ))

tests/test_sav_sqlite.py

@@ -1,3 +1,4 @@
+import hashlib
 import json
 import sqlite3
 
@@ -35,6 +36,7 @@ def test_sav_round_trip_uses_one_wide_table_and_catalog(tmp_path) -> None:
         (1, 34.0, "Ada"), (2, None, "")
     ]
     assert connection.execute("select source_encoding, documents from dataset_catalog").fetchone() == ("UTF-8", json.dumps(["Fixture note"]))
+    assert connection.execute("select source_sha256 from dataset_catalog").fetchone() == (hashlib.sha256(source.read_bytes()).hexdigest(),)
 
     openstatspec.export_sav(database_url=database, dataset_id="tiny", destination=exported)
     frame, meta = pyreadstat.read_sav(exported, user_missing=True)