Land the #15 lineup stack on main

.gitignore

@@ -2,6 +2,7 @@
 .env
 config.json
 *.xmltv
+!testdata/xmlguide_golden.xmltv
 tmdb_cache.json
 tvlogo_cache.json
 guide_cache.json

CLAUDE.md

@@ -27,13 +27,13 @@ Run `go test ./...` for the setup, configuration, and provider-client tests.
 
 ## Architecture
 
-The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. Runtime orchestration lives in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`.
+The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. The grid download loop and guide assembly live in `scrape/` (`scrape.Fetch`), which is importable by other programs and does no enrichment or env reads (the default `web.Client` it uses does keep `grid_cache/` on disk). Enrichment, persistence, and runtime orchestration live in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`.
 
 **Data flow:**
 
 1. `/setup` uses `web.ProviderClient` to discover Gracenote lineups by country and postal code. `appconfig.Store` persists the selected non-secret source in `config.json`; complete legacy `GN_*` settings can bootstrap it.
-2. `web.Client.GetDataByTime` fetches 6-hour grid slices from the GraceNote API (`tvlistings.gracenote.com/api/grid`) — 56 slots for 14 days. A 5-second sleep separates requests. Raw JSON types live in `web/web.go`.
-3. `guide.ConvertChannel` / `guide.ConvertEvent` translate the raw JSON into `guide.TVGuide` (internal canonical types). The `guide.tmpl` template renders these to XMLTV. `index.html`, `setup.html`, and `guide.tmpl` are embedded at build time via `//go:embed`.
+2. `scrape.Fetch` drives `web.Client.GetDataByTimeContext` over 6-hour grid slices from the GraceNote API (`tvlistings.gracenote.com/api/grid`) — 56 slots for 14 days. A 5-second sleep separates requests. Failed slots are skipped; if every slot fails the scrape returns `scrape.ErrNoData` and the previous guide stays live. Raw JSON types live in `web/web.go`.
+3. `guide.ConvertChannel` / `guide.ConvertEvent` translate the raw JSON into `guide.TVGuide` (internal canonical types). `TVGuide.Channels` is deduplicated by station for XMLTV; `TVGuide.Lineup` retains every provider position (number plus station) and is served by `/api/lineup.json`. `TVGuide.Source` records the lineup the guide came from. The `guide.tmpl` template renders these to XMLTV. `index.html`, `setup.html`, and `guide.tmpl` are embedded at build time via `//go:embed`.
 4. `tmdb.Client.Lookup` enriches programs (poster images, ratings, overview, year) via TMDB search API. Deduplicates by `(title, isMovie)` before hitting the API. Rate-limited to ~4 req/sec.
 5. `tvlogo.Client.Resolve` replaces Gracenote channel icons with verified PNGs from `github.com/tv-logo/tv-logos`. Generates candidate URL slugs from callsign/affiliate name and HEAD-checks each (rate-limited to ~5 req/sec).
 6. `fixDeadImageURLs` rewrites `zap2it.tmsimg.com` → `tmsimg.com` for broken Gracenote image URLs.
@@ -43,7 +43,7 @@ The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV l
 
 | Cache | File | TTL |
 |---|---|---|
-| Guide (in-memory + disk) | `guide_cache.json` | 4h (startup skip) / 24h (rescrape) |
+| Guide (in-memory + disk) | `guide_cache.json` | 4h (startup skip) / 24h (rescrape); carries a schema version, older caches are rebuilt once |
 | TMDB lookups | `tmdb_cache.json` | 7 days |
 | TV logo HEAD checks | `tvlogo_cache.json` | persisted, no expiry |
 | Image proxy | `image_cache/` dir | indefinite (per-URL SHA256 key) |

README.md

@@ -118,24 +118,88 @@ A saved `CONFIG_PATH` selection takes precedence over legacy `GN_*` settings. De
 | `POST /api/setup/provider` | Save the selected provider and queue a fresh guide |
 | `GET /xmlguide.xmltv` | XMLTV guide data (point your DVR here) |
 | `GET /api/guide.json` | Guide data as JSON |
+| `GET /api/lineup.json` | Every channel position in the active provider lineup as JSON (see below) |
 | `GET /` | The Grid — built-in web UI |
 | `GET /img?url=...` | Image proxy with local cache |
 | `GET /api/livetv/config` | Returns `{"enabled":true/false}` — whether Jellyfin live TV is configured |
 | `GET /api/livetv/channels` | Proxies Jellyfin channel list (requires `JELLYFIN_URL` and `JELLYFIN_API_KEY`) |
 | `GET /api/livetv/tune?id=<channelId>` | Starts a live stream for the given channel and returns an HLS URL |
 | `POST /api/livetv/stop` | Forwards a playback-stop notification to Jellyfin to end a live stream |
 
+### Lineup JSON
+
+`/api/lineup.json` describes the provider lineup itself rather than the schedule: one entry per channel number, never collapsed, so a station carried at two numbers appears twice. It returns `503` with a `Retry-After` header until the first guide has been built.
+
+```json
+{
+  "generated": "2026-09-13T04:10:22Z",
+  "source": {
+    "providerName": "Local Over the Air Broadcast",
+    "providerType": "OTA",
+    "location": "",
+    "lineupId": "USA-lineupId-DEFAULT",
+    "headendId": "lineupId",
+    "postalCode": "13490",
+    "country": "USA",
+    "device": "-",
+    "language": "en-us"
+  },
+  "positions": [
+    {
+      "number": "2.1",
+      "stationId": "53158",
+      "placementId": "531580",
+      "callSign": "WKTVDT",
+      "affiliate": "NATIONAL BROADCASTING COMPANY",
+      "affiliateCallSign": "",
+      "filters": ["sports", "news"],
+      "logoUrl": "https://raw.githubusercontent.com/tv-logo/tv-logos/main/countries/united-states/nbc-us.png"
+    }
+  ]
+}
+```
+
+`filters` are Gracenote's own station tags with the `filter-` prefix removed. `placementId` is Gracenote's row identifier and is not stable across scrapes; use `number` plus `stationId` to identify a position. `providerName`, `providerType`, and `location` are filled from the saved setup when it matches the lineup the guide was built from.
+
 ## The Grid
 
 The server includes a built-in retro-styled TV guide web UI at the root URL. If no provider is configured, `/` redirects to `/setup`. Once configured, the guide auto-scrolls through your channel lineup and shows program details, posters, and metadata. Handy for a quick glance at what's on without opening your DVR app.
 
 ![The Grid](https://gist.githubusercontent.com/daniel-widrick/2c52c4d023ffe75d163b4eff58263c77/raw/demo.gif)
 
+## Using as a Library
+
+The grid download and conversion live in the `scrape` package, so another Go program can fetch a lineup's listings without running the server:
+
+```go
+import (
+    "context"
+    "fmt"
+
+    "github.com/daniel-widrick/GraceNoteScraper/scrape"
+    "github.com/daniel-widrick/GraceNoteScraper/web"
+)
+
+func main() {
+    prefs := web.Preferences{Country: "USA", ZipCode: "13490", Headend: "lineupId", LineupId: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us"}
+    g, err := scrape.Fetch(context.Background(), prefs, scrape.Options{Days: 1})
+    if err != nil {
+        panic(err)
+    }
+    fmt.Printf("%d stations, %d lineup positions, %d programs\n", len(g.Channels), len(g.Lineup), len(g.Programs))
+}
+```
+
+`Fetch` returns a `guide.TVGuide`: `Channels` deduplicated by station, `Lineup` with every position, `Programs`, and `Source`. It pauses five seconds between grid requests by default and skips slots that fail; it returns `scrape.ErrNoData` only when every slot failed. Logo and TMDB enrichment are not part of the package. Lineup discovery by postal code is available through `web.NewProviderClient().FindProviders`.
+
+The default grid client is `web.NewClient`, which retries failed requests and keeps a raw-response cache in a `grid_cache/` directory under the working directory. Pass your own `Options.Fetcher` to change that.
+
 ## Project Structure
 
 ```
 appconfig/       Persisted non-secret provider configuration
-main.go          Entry point, HTTP server, scraper, image proxy
+main.go          Entry point, HTTP server, enrichment, image proxy
+scrape/          Grid download loop and guide assembly (importable)
 guide/           GraceNote data types and XMLTV conversion
 web/             HTTP client for GraceNote API
 tmdb/            TMDB client and cache

guide/guide.go

@@ -2,25 +2,124 @@ package guide
 
 import (
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
+	"time"
 	"unicode"
 
 	"github.com/daniel-widrick/GraceNoteScraper/web"
 )
 
 type TVGuide struct {
+	// Channels is the XMLTV view: one entry per Gracenote station.
 	Channels []Channel
 	Programs []Program
+	// Lineup retains every provider position, so a station carried at two
+	// channel numbers appears twice. It is never collapsed.
+	Lineup []LineupPosition
+	// Source records which provider lineup produced this guide.
+	Source Source
+}
+
+// Source identifies the Gracenote lineup a guide was built from.
+type Source struct {
+	Country     string
+	PostalCode  string
+	HeadendID   string
+	LineupID    string
+	Device      string
+	Language    string
+	GeneratedAt time.Time
+}
+
+// SourceFromPreferences copies the request preferences into a Source.
+func SourceFromPreferences(p web.Preferences, generatedAt time.Time) Source {
+	return Source{
+		Country:     p.Country,
+		PostalCode:  p.ZipCode,
+		HeadendID:   p.Headend,
+		LineupID:    p.LineupId,
+		Device:      p.Device,
+		Language:    p.Language,
+		GeneratedAt: generatedAt,
+	}
+}
+
+// LineupPosition is one channel number in a provider lineup.
+type LineupPosition struct {
+	ChannelNo         string
+	StationID         string
+	PlacementID       string // Gracenote row id; carried for fidelity, not a stable key
+	CallSign          string
+	Affiliate         string
+	AffiliateCallSign string
+	Filters           []string
+	LogoURL           string
+}
+
+// Key identifies a position across grid slices: the same station at the same
+// number is one position no matter how many responses it appears in.
+func (p LineupPosition) Key() string {
+	return p.ChannelNo + "|" + p.StationID
+}
+
+// ConvertLineupPosition converts a JSON channel row to a lineup position.
+func ConvertLineupPosition(ch web.JSONChannel) LineupPosition {
+	return LineupPosition{
+		ChannelNo:         ch.ChannelNo,
+		StationID:         ch.ChannelID,
+		PlacementID:       ch.ID,
+		CallSign:          ch.CallSign,
+		Affiliate:         ch.AffiliateName,
+		AffiliateCallSign: normalizeNull(ch.AffiliateCallSign),
+		Filters:           stripFilterPrefixes(ch.StationFilters),
+		LogoURL:           gracenoteIconURL(ch.Thumbnail),
+	}
+}
+
+// ChannelNumberLess orders channel numbers numerically where both parse
+// (so "2.1" < "10" < "100"), places numeric numbers before non-numeric ones,
+// and falls back to string order. Equal numbers compare as strings so the
+// ordering is strict.
+func ChannelNumberLess(a, b string) bool {
+	af, errA := strconv.ParseFloat(strings.TrimSpace(a), 64)
+	bf, errB := strconv.ParseFloat(strings.TrimSpace(b), 64)
+	switch {
+	case errA == nil && errB == nil:
+		if af != bf {
+			return af < bf
+		}
+		return a < b
+	case errA == nil:
+		return true
+	case errB == nil:
+		return false
+	default:
+		return a < b
+	}
+}
+
+// SortLineup orders positions by channel number, then station ID, in place.
+func SortLineup(positions []LineupPosition) {
+	sort.SliceStable(positions, func(i, j int) bool {
+		if positions[i].ChannelNo != positions[j].ChannelNo {
+			return ChannelNumberLess(positions[i].ChannelNo, positions[j].ChannelNo)
+		}
+		return positions[i].StationID < positions[j].StationID
+	})
 }
 
 type Channel struct {
-	ID           string
-	DisplayNames []DisplayName
-	IconURL      string
-	CallSign     string // internal, not in template
-	Affiliate    string // internal, not in template
-	ChannelNo    string // internal, not in template
+	ID                string
+	DisplayNames      []DisplayName
+	IconURL           string
+	CallSign          string   // internal, not in template
+	Affiliate         string   // internal, not in template
+	ChannelNo         string   // internal, not in template
+	PlacementID       string   // internal, not in template; Gracenote row id, not a stable key
+	AffiliateCallSign string   // internal, not in template
+	Filters           []string // internal, not in template; Gracenote station filters, prefix stripped
 }
 
 type DisplayName struct {
@@ -45,6 +144,10 @@ type Program struct {
 	Country         string
 	EpisodeNumbers  []EpisodeNumber
 	Categories      []Category
+	Filters         []string // internal, not in template; raw Gracenote event filters, prefix stripped
+	TMSID           string   // internal, not in template
+	ReleaseYear     string   // internal, not in template
+	Generic         bool     // internal, not in template
 	New             bool
 	Premiere        bool
 	PreviouslyShown bool
@@ -95,21 +198,26 @@ func formatXMLTVTime(iso string) string {
 	return s
 }
 
+// gracenoteIconURL builds an absolute icon URL from a Gracenote thumbnail
+// path: strip leading slashes, strip query params, prepend http://
+func gracenoteIconURL(thumbnail string) string {
+	if thumbnail == "" {
+		return ""
+	}
+	raw := thumbnail
+	if idx := strings.Index(raw, "?"); idx >= 0 {
+		raw = raw[:idx]
+	}
+	raw = strings.TrimLeft(raw, "/")
+	if raw == "" {
+		return ""
+	}
+	return "http://" + raw
+}
+
 // converts a JSON channel to a template Channel struct.
 func ConvertChannel(ch web.JSONChannel) Channel {
-	// Build icon URL: strip leading slashes, strip query params, prepend http://
-	iconURL := ""
-	if ch.Thumbnail != "" {
-		raw := ch.Thumbnail
-		// Strip query string
-		if idx := strings.Index(raw, "?"); idx >= 0 {
-			raw = raw[:idx]
-		}
-		raw = strings.TrimLeft(raw, "/")
-		if raw != "" {
-			iconURL = "http://" + raw
-		}
-	}
+	iconURL := gracenoteIconURL(ch.Thumbnail)
 
 	return Channel{
 		ID: ch.ChannelID,
@@ -119,13 +227,37 @@ func ConvertChannel(ch web.JSONChannel) Channel {
 			{Name: xmlEscape(ch.CallSign)},
 			{Name: xmlEscape(titleCase(ch.AffiliateName))},
 		},
-		IconURL:   iconURL,
-		CallSign:  ch.CallSign,
-		Affiliate: ch.AffiliateName,
-		ChannelNo: ch.ChannelNo,
+		IconURL:           iconURL,
+		CallSign:          ch.CallSign,
+		Affiliate:         ch.AffiliateName,
+		ChannelNo:         ch.ChannelNo,
+		PlacementID:       ch.ID,
+		AffiliateCallSign: normalizeNull(ch.AffiliateCallSign),
+		Filters:           stripFilterPrefixes(ch.StationFilters),
 	}
 }
 
+// normalizeNull maps Gracenote's literal "null" string to an empty value.
+func normalizeNull(s string) string {
+	if strings.EqualFold(strings.TrimSpace(s), "null") {
+		return ""
+	}
+	return s
+}
+
+// stripFilterPrefixes turns Gracenote filter tags such as "filter-sports" into
+// "sports". A nil input stays nil so callers can distinguish absent from empty.
+func stripFilterPrefixes(filters []string) []string {
+	if filters == nil {
+		return nil
+	}
+	out := make([]string, 0, len(filters))
+	for _, f := range filters {
+		out = append(out, strings.TrimPrefix(f, "filter-"))
+	}
+	return out
+}
+
 // converts a JSON event to a template Program struct.
 func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program {
 	season := 0
@@ -163,10 +295,13 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program {
 	// URL
 	programURL := "https://tvlistings.gracenote.com//overview.html?programSeriesId=" + ev.SeriesID + "&amp;tmsId=" + ev.Program.ID
 
+	// Raw Gracenote filters, kept separately so consumers can tell them apart
+	// from the Series and Finale labels added below.
+	filters := stripFilterPrefixes(ev.Filter)
+
 	// Categories from filter array (strip "filter-" prefix)
 	var categories []Category
-	for _, f := range ev.Filter {
-		name := strings.TrimPrefix(f, "filter-")
+	for _, name := range filters {
 		categories = append(categories, Category{Name: name, Lang: lang})
 	}
 
@@ -266,6 +401,10 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program {
 		Country:         country,
 		EpisodeNumbers:  episodeNumbers,
 		Categories:      categories,
+		Filters:         filters,
+		TMSID:           ev.Program.TmsID,
+		ReleaseYear:     string(ev.Program.ReleaseYear),
+		Generic:         bool(ev.Program.IsGeneric),
 		New:             isNew,
 		Premiere:        isPremiere,
 		PreviouslyShown: !isNew,