Skip to content

Commit ed8fb60

Browse files
committed
feat(seed): take created_at/updated_at from git instead of the clock
With ids fixed, a second dump of unchanged data still rewrote every page in 10 of 12 collections. The only difference was the two timestamps: the database is rebuilt for each dump, so they recorded when the dump ran, not when the record changed. brands and gpus were the only stable collections because their schemas do not expose them. One `git log` pass over data/ maps each record to its first and last commit time. Outside a git checkout the map is empty and the model defaults stand. Refs #1
1 parent c34820e commit ed8fb60

2 files changed

Lines changed: 139 additions & 14 deletions

File tree

‎app/seed.py‎

Lines changed: 73 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@
1717

1818
import hashlib
1919
import json
20-
from datetime import date
20+
import subprocess
21+
from datetime import date, datetime
2122
from pathlib import Path
2223
from typing import Any
2324

@@ -39,6 +40,43 @@
3940
DATA_DIR = get_data_root()
4041

4142

43+
NUL = "\x00" # git log record separator, so a commit line cannot look like a path
44+
45+
46+
def _git_timestamps(data_dir: Path) -> dict[str, tuple[datetime, datetime]]:
47+
"""Map each record path to (first commit, last commit) times.
48+
49+
`created_at`/`updated_at` used to be stamped with "now" at seed time, so
50+
every dump rewrote every page with a timestamp that only said when the dump
51+
ran. Git already knows when a record appeared and when it last changed, and
52+
those answers do not move between runs.
53+
54+
One `git log` pass over the whole data tree; an empty map (no git, shallow
55+
clone) leaves the model defaults in place.
56+
"""
57+
try:
58+
log = subprocess.run(
59+
# --relative keeps paths relative to data_dir, matching __path.
60+
["git", "log", "--reverse", "--no-renames", "--relative",
61+
"--format=%x00%cI", "--name-only", "--diff-filter=AM", "--", "."],
62+
cwd=data_dir, check=True, capture_output=True, text=True, encoding="utf-8",
63+
).stdout
64+
except (OSError, subprocess.CalledProcessError):
65+
return {}
66+
67+
stamps: dict[str, tuple[datetime, datetime]] = {}
68+
when: datetime | None = None
69+
for line in log.splitlines():
70+
if line.startswith(NUL):
71+
when = datetime.fromisoformat(line[1:])
72+
continue
73+
if when is None or not line.endswith(".json"):
74+
continue
75+
first, _ = stamps.get(line, (when, when))
76+
stamps[line] = (first, when)
77+
return stamps
78+
79+
4280
def _load_dir(subdir: Path) -> list[dict[str, Any]]:
4381
if not subdir.exists():
4482
return []
@@ -52,6 +90,7 @@ def _load_dir(subdir: Path) -> list[dict[str, Any]]:
5290
for key, value in list(record.items()):
5391
if key.endswith("_date") and isinstance(value, str):
5492
record[key] = date.fromisoformat(value)
93+
record["__path"] = path.relative_to(subdir.parent).as_posix()
5594
items.append(record)
5695
return items
5796

@@ -82,6 +121,26 @@ def _with_id(obj: Any, taken: set[int]) -> Any:
82121
return obj
83122

84123

124+
def _stamp(obj: Any, path: str | None, stamps: dict[str, tuple[datetime, datetime]]) -> Any:
125+
"""Replace the seed-time timestamps with the record's git history."""
126+
times = stamps.get(path or "")
127+
if times:
128+
obj.created_at, obj.updated_at = times
129+
return obj
130+
131+
132+
def _row(
133+
model: Any,
134+
record: dict[str, Any],
135+
taken: set[int],
136+
stamps: dict[str, tuple[datetime, datetime]],
137+
**fks: Any,
138+
) -> Any:
139+
"""Build one row: resolved FKs, stable id, git-derived timestamps."""
140+
path = record.pop("__path", None)
141+
return _stamp(_with_id(model(**fks, **record), taken), path, stamps)
142+
143+
85144
def _existing_slugs(session: Session, model: type[SQLModel]) -> set[str]:
86145
rows = session.exec(select(model)).all()
87146
return {row.slug for row in rows} # type: ignore[attr-defined] # all data models have slug
@@ -91,6 +150,7 @@ def seed(session: Session, data_dir: Path = DATA_DIR) -> dict[str, int]:
91150
"""Idempotently insert seed data. Returns counts of newly inserted rows."""
92151
# Ids assigned in this run; only used to break hash collisions.
93152
taken: set[int] = set()
153+
stamps = _git_timestamps(data_dir)
94154
counts = {
95155
"brands": 0,
96156
"socs": 0,
@@ -114,7 +174,7 @@ def seed(session: Session, data_dir: Path = DATA_DIR) -> dict[str, int]:
114174
# `categories` lives in the JSON for browsing/validation only — the Brand
115175
# table model does not (yet) carry it, so drop before construction.
116176
record.pop("categories", None)
117-
session.add(_with_id(Brand(**record), taken))
177+
session.add(_row(Brand, record, taken, stamps))
118178
counts["brands"] += 1
119179
session.commit()
120180

@@ -131,7 +191,7 @@ def seed(session: Session, data_dir: Path = DATA_DIR) -> dict[str, int]:
131191
raise ValueError(
132192
f"SoC '{record['slug']}' references unknown brand '{manufacturer}'"
133193
)
134-
session.add(_with_id(SoC(manufacturer_id=manufacturer_id, **record), taken))
194+
session.add(_row(SoC, record, taken, stamps, manufacturer_id=manufacturer_id))
135195
counts["socs"] += 1
136196
session.commit()
137197

@@ -154,8 +214,7 @@ def seed(session: Session, data_dir: Path = DATA_DIR) -> dict[str, int]:
154214
raise ValueError(
155215
f"Smartphone '{record['slug']}' references unknown SoC '{soc_slug}'"
156216
)
157-
phone = Smartphone(brand_id=brand_id, soc_id=soc_id, **record)
158-
session.add(_with_id(phone, taken))
217+
session.add(_row(Smartphone, record, taken, stamps, brand_id=brand_id, soc_id=soc_id))
159218
counts["smartphones"] += 1
160219
session.commit()
161220

@@ -178,8 +237,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N
178237
f"{subdir.rstrip('s').title()} '{record['slug']}' "
179238
f"references unknown SoC '{soc_slug}'"
180239
)
181-
device = model(brand_id=brand_id, soc_id=soc_id, **record)
182-
session.add(_with_id(device, taken))
240+
session.add(_row(model, record, taken, stamps, brand_id=brand_id, soc_id=soc_id))
183241
counts[count_key] += 1
184242
session.commit()
185243

@@ -198,7 +256,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N
198256
raise ValueError(
199257
f"GPU '{record['slug']}' references unknown brand '{manufacturer}'"
200258
)
201-
session.add(_with_id(DiscreteGPU(manufacturer_id=manufacturer_id, **record), taken))
259+
session.add(_row(DiscreteGPU, record, taken, stamps, manufacturer_id=manufacturer_id))
202260
counts["gpus"] += 1
203261
session.commit()
204262

@@ -213,7 +271,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N
213271
raise ValueError(
214272
f"CPU '{record['slug']}' references unknown brand '{manufacturer}'"
215273
)
216-
session.add(_with_id(CPU(manufacturer_id=manufacturer_id, **record), taken))
274+
session.add(_row(CPU, record, taken, stamps, manufacturer_id=manufacturer_id))
217275
counts["cpus"] += 1
218276
session.commit()
219277

@@ -242,8 +300,9 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N
242300
raise ValueError(
243301
f"Laptop '{record['slug']}' references unknown GPU '{gpu_slug}'"
244302
)
245-
laptop = Laptop(brand_id=brand_id, cpu_id=cpu_id, gpu_id=gpu_id, **record)
246-
session.add(_with_id(laptop, taken))
303+
session.add(
304+
_row(Laptop, record, taken, stamps, brand_id=brand_id, cpu_id=cpu_id, gpu_id=gpu_id)
305+
)
247306
counts["laptops"] += 1
248307
session.commit()
249308

@@ -258,7 +317,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N
258317
raise ValueError(
259318
f"Monitor '{record['slug']}' references unknown brand '{brand_slug}'"
260319
)
261-
session.add(_with_id(Monitor(brand_id=brand_id, **record), taken))
320+
session.add(_row(Monitor, record, taken, stamps, brand_id=brand_id))
262321
counts["monitors"] += 1
263322
session.commit()
264323

@@ -267,7 +326,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N
267326
for record in _load_dir(data_dir / "software"):
268327
if record["slug"] in software_slugs:
269328
continue
270-
session.add(_with_id(Software(**record), taken))
329+
session.add(_row(Software, record, taken, stamps))
271330
counts["software"] += 1
272331
session.commit()
273332

@@ -276,7 +335,7 @@ def seed_mobile_devices(subdir: str, model: type[SQLModel], count_key: str) -> N
276335
for record in _load_dir(data_dir / "website"):
277336
if record["slug"] in website_slugs:
278337
continue
279-
session.add(_with_id(Website(**record), taken))
338+
session.add(_row(Website, record, taken, stamps))
280339
counts["websites"] += 1
281340
session.commit()
282341

‎tests/unit/test_git_timestamps.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
"""Timestamps come from git, so a re-dump of unchanged data is byte-identical."""
2+
3+
from __future__ import annotations
4+
5+
import json
6+
import os
7+
import subprocess
8+
from pathlib import Path
9+
10+
from app.seed import _git_timestamps
11+
12+
13+
def _git(repo: Path, *args: str, when: str | None = None) -> None:
14+
env = {
15+
"GIT_AUTHOR_NAME": "t",
16+
"GIT_AUTHOR_EMAIL": "t@example.com",
17+
"GIT_COMMITTER_NAME": "t",
18+
"GIT_COMMITTER_EMAIL": "t@example.com",
19+
"PATH": os.environ["PATH"],
20+
}
21+
if when:
22+
# _git_timestamps reads committer dates, so both must be pinned.
23+
env["GIT_AUTHOR_DATE"] = env["GIT_COMMITTER_DATE"] = when
24+
subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True, env=env)
25+
26+
27+
def _repo(tmp_path: Path) -> Path:
28+
data = tmp_path / "data"
29+
(data / "cpu").mkdir(parents=True)
30+
_git(tmp_path, "init", "-q")
31+
return data
32+
33+
34+
def _commit(repo: Path, rel: str, payload: dict, when: str) -> None:
35+
path = repo / rel
36+
path.parent.mkdir(parents=True, exist_ok=True)
37+
path.write_text(json.dumps(payload), encoding="utf-8")
38+
_git(repo.parent, "add", "-A")
39+
_git(repo.parent, "commit", "-q", "-m", rel, when=when)
40+
41+
42+
def test_created_and_updated_track_first_and_last_commit(tmp_path):
43+
data = _repo(tmp_path)
44+
_commit(data, "cpu/a.json", {"slug": "a"}, "2026-01-02T03:04:05+00:00")
45+
_commit(data, "cpu/b.json", {"slug": "b"}, "2026-02-02T03:04:05+00:00")
46+
_commit(data, "cpu/a.json", {"slug": "a", "cores": 8}, "2026-03-02T03:04:05+00:00")
47+
48+
stamps = _git_timestamps(data)
49+
created_a, updated_a = stamps["cpu/a.json"]
50+
created_b, updated_b = stamps["cpu/b.json"]
51+
52+
assert created_a < updated_a # edited later
53+
assert created_b == updated_b # written once
54+
assert created_a < created_b # a came first
55+
56+
57+
def test_repeated_reads_agree(tmp_path):
58+
data = _repo(tmp_path)
59+
_commit(data, "cpu/a.json", {"slug": "a"}, "2026-01-02T03:04:05+00:00")
60+
assert _git_timestamps(data) == _git_timestamps(data)
61+
62+
63+
def test_outside_a_git_repo_returns_nothing(tmp_path):
64+
plain = tmp_path / "plain"
65+
(plain / "cpu").mkdir(parents=True)
66+
assert _git_timestamps(plain) == {}

0 commit comments

Comments
 (0)