Move the grid download loop into an importable scrape package

CLAUDE.md

@@ -27,12 +27,12 @@ Run `go test ./...` for the setup, configuration, and provider-client tests.
 
 ## Architecture
 
-The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. Runtime orchestration lives in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`.
+The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. The grid download loop and guide assembly live in `scrape/` (`scrape.Fetch`), which is importable by other programs and does no enrichment or env reads (the default `web.Client` it uses does keep `grid_cache/` on disk). Enrichment, persistence, and runtime orchestration live in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`.
 
 **Data flow:**
 
 1. `/setup` uses `web.ProviderClient` to discover Gracenote lineups by country and postal code. `appconfig.Store` persists the selected non-secret source in `config.json`; complete legacy `GN_*` settings can bootstrap it.
-2. `web.Client.GetDataByTime` fetches 6-hour grid slices from the GraceNote API (`tvlistings.gracenote.com/api/grid`) — 56 slots for 14 days. A 5-second sleep separates requests. Raw JSON types live in `web/web.go`.
+2. `scrape.Fetch` drives `web.Client.GetDataByTimeContext` over 6-hour grid slices from the GraceNote API (`tvlistings.gracenote.com/api/grid`) — 56 slots for 14 days. A 5-second sleep separates requests. Failed slots are skipped; if every slot fails the scrape returns `scrape.ErrNoData` and the previous guide stays live. Raw JSON types live in `web/web.go`.
 3. `guide.ConvertChannel` / `guide.ConvertEvent` translate the raw JSON into `guide.TVGuide` (internal canonical types). `TVGuide.Channels` is deduplicated by station for XMLTV; `TVGuide.Lineup` retains every provider position (number plus station) and is served by `/api/lineup.json`. `TVGuide.Source` records the lineup the guide came from. The `guide.tmpl` template renders these to XMLTV. `index.html`, `setup.html`, and `guide.tmpl` are embedded at build time via `//go:embed`.
 4. `tmdb.Client.Lookup` enriches programs (poster images, ratings, overview, year) via TMDB search API. Deduplicates by `(title, isMovie)` before hitting the API. Rate-limited to ~4 req/sec.
 5. `tvlogo.Client.Resolve` replaces Gracenote channel icons with verified PNGs from `github.com/tv-logo/tv-logos`. Generates candidate URL slugs from callsign/affiliate name and HEAD-checks each (rate-limited to ~5 req/sec).

README.md

@@ -167,11 +167,39 @@ The server includes a built-in retro-styled TV guide web UI at the root URL. If
 
 ![The Grid](https://gist.githubusercontent.com/daniel-widrick/2c52c4d023ffe75d163b4eff58263c77/raw/demo.gif)
 
+## Using as a Library
+
+The grid download and conversion live in the `scrape` package, so another Go program can fetch a lineup's listings without running the server:
+
+```go
+import (
+    "context"
+    "fmt"
+
+    "github.com/daniel-widrick/GraceNoteScraper/scrape"
+    "github.com/daniel-widrick/GraceNoteScraper/web"
+)
+
+func main() {
+    prefs := web.Preferences{Country: "USA", ZipCode: "13490", Headend: "lineupId", LineupId: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us"}
+    g, err := scrape.Fetch(context.Background(), prefs, scrape.Options{Days: 1})
+    if err != nil {
+        panic(err)
+    }
+    fmt.Printf("%d stations, %d lineup positions, %d programs\n", len(g.Channels), len(g.Lineup), len(g.Programs))
+}
+```
+
+`Fetch` returns a `guide.TVGuide`: `Channels` deduplicated by station, `Lineup` with every position, `Programs`, and `Source`. It pauses five seconds between grid requests by default and skips slots that fail; it returns `scrape.ErrNoData` only when every slot failed. Logo and TMDB enrichment are not part of the package. Lineup discovery by postal code is available through `web.NewProviderClient().FindProviders`.
+
+The default grid client is `web.NewClient`, which retries failed requests and keeps a raw-response cache in a `grid_cache/` directory under the working directory. Pass your own `Options.Fetcher` to change that.
+
 ## Project Structure
 
 ```
 appconfig/       Persisted non-secret provider configuration
-main.go          Entry point, HTTP server, scraper, image proxy
+main.go          Entry point, HTTP server, enrichment, image proxy
+scrape/          Grid download loop and guide assembly (importable)
 guide/           GraceNote data types and XMLTV conversion
 web/             HTTP client for GraceNote API
 tmdb/            TMDB client and cache

main.go

@@ -21,12 +21,14 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"text/template"
 	"time"
 
 	"github.com/daniel-widrick/GraceNoteScraper/appconfig"
 	"github.com/daniel-widrick/GraceNoteScraper/guide"
+	"github.com/daniel-widrick/GraceNoteScraper/scrape"
 	"github.com/daniel-widrick/GraceNoteScraper/tmdb"
 	"github.com/daniel-widrick/GraceNoteScraper/tvlogo"
 	"github.com/daniel-widrick/GraceNoteScraper/util"
@@ -276,70 +278,32 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch
 			}
 		}
 	}
-	client := web.NewClient(pref)
 
-	now := time.Now().UTC()
-	midnight := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
-	endTime := midnight.Add(14 * 24 * time.Hour)
-
-	channelMap := make(map[string]guide.Channel)
-	lineupMap := make(map[string]guide.LineupPosition)
-	eventMap := make(map[string]bool)
-	var programs []guide.Program
-
-	totalSlots := int(endTime.Sub(midnight) / (6 * time.Hour))
-	slot := 0
-	for t := midnight; t.Before(endTime); t = t.Add(6 * time.Hour) {
-		if sourceCurrent != nil && !sourceCurrent() {
-			return nil, errScrapeSourceChanged
-		}
-		slot++
-		ts := t.Unix()
-		report(scrapeProgressUpdate{Stage: "gracenote", Message: fmt.Sprintf("Downloading guide data (%d of %d)", slot, totalSlots), Completed: slot - 1, Total: totalSlots, Channels: len(channelMap), Programs: len(programs)})
-		log.Printf("Fetching grid %d/%d for time=%d (%s)", slot, totalSlots, ts, t.Format(time.RFC3339))
-
-		grid, err := client.GetDataByTime(ts)
-		if err != nil {
-			log.Printf("Error fetching grid at %d: %v", ts, err)
-			continue
-		}
-
-		for _, ch := range grid.Channels {
-			if _, exists := channelMap[ch.ChannelID]; !exists {
-				channelMap[ch.ChannelID] = guide.ConvertChannel(ch)
-			}
-			position := guide.ConvertLineupPosition(ch)
-			if _, exists := lineupMap[position.Key()]; !exists {
-				lineupMap[position.Key()] = position
-			}
-
-			for _, ev := range ch.Events {
-				dedupKey := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime
-				if eventMap[dedupKey] {
-					continue
-				}
-				eventMap[dedupKey] = true
-				programs = append(programs, guide.ConvertEvent(ev, ch.ChannelID, pref.Language, pref.Country))
-			}
-		}
-
-		log.Printf("Channels so far: %d, Events so far: %d", len(channelMap), len(programs))
-		report(scrapeProgressUpdate{Stage: "gracenote", Message: fmt.Sprintf("Downloaded guide data (%d of %d)", slot, totalSlots), Completed: slot, Total: totalSlots, Channels: len(channelMap), Programs: len(programs)})
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	sourceChanged := watchSourceChange(ctx, cancel, sourceCurrent)
 
-		if t.Add(6 * time.Hour).Before(endTime) {
-			time.Sleep(5 * time.Second)
+	opts := newScrapeOptions(pref)
+	opts.Progress = func(p scrape.Progress) {
+		message := fmt.Sprintf("Downloading guide data (%d of %d)", p.Slot, p.TotalSlots)
+		if p.Phase == scrape.PhaseFetched {
+			message = fmt.Sprintf("Downloaded guide data (%d of %d)", p.Slot, p.TotalSlots)
 		}
+		report(scrapeProgressUpdate{Stage: "gracenote", Message: message, Completed: p.Completed, Total: p.TotalSlots, Channels: p.Channels, Programs: p.Programs})
 	}
-
-	var channels []guide.Channel
-	for _, ch := range channelMap {
-		channels = append(channels, ch)
+	fetched, err := scrape.Fetch(ctx, pref, opts)
+	if err != nil {
+		if sourceChanged() {
+			return nil, errScrapeSourceChanged
+		}
+		return nil, err
 	}
-	lineup := make([]guide.LineupPosition, 0, len(lineupMap))
-	for _, position := range lineupMap {
-		lineup = append(lineup, position)
+	if sourceChanged() {
+		return nil, errScrapeSourceChanged
 	}
-	guide.SortLineup(lineup)
+	channels := fetched.Channels
+	lineup := fetched.Lineup
+	programs := fetched.Programs
 
 	logoClient := tvlogo.NewClient(pref.Country, "tvlogo_cache.json")
 	if logoClient != nil {
@@ -363,7 +327,7 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch
 		Channels: channels,
 		Programs: programs,
 		Lineup:   lineup,
-		Source:   guide.SourceFromPreferences(pref, time.Now().UTC()),
+		Source:   fetched.Source,
 	}
 
 	if channelFilter != nil {
@@ -399,6 +363,52 @@ func renderXMLTV(w io.Writer, tvGuide *guide.TVGuide) error {
 	return nil
 }
 
+// newScrapeOptions builds the production scrape configuration. Tests replace
+// it to inject a fake grid fetcher and remove the inter-slot delay.
+var newScrapeOptions = func(pref web.Preferences) scrape.Options {
+	return scrape.Options{Fetcher: web.NewClient(pref)}
+}
+
+// watchSourceChange cancels ctx as soon as sourceCurrent reports false, so a
+// lineup change during the grid download stops the scrape promptly. The
+// returned func reports whether a change has been observed, re-checking live
+// so a change between the last tick and the call is not missed.
+func watchSourceChange(ctx context.Context, cancel context.CancelFunc, sourceCurrent func() bool) func() bool {
+	if sourceCurrent == nil {
+		return func() bool { return false }
+	}
+	var changed atomic.Bool
+	check := func() bool {
+		if changed.Load() {
+			return true
+		}
+		if !sourceCurrent() {
+			changed.Store(true)
+			cancel()
+			return true
+		}
+		return false
+	}
+	if check() {
+		return check
+	}
+	go func() {
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				if check() {
+					return
+				}
+			}
+		}
+	}()
+	return check
+}
+
 func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error {
 	log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs))
 

run_scrape_test.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log"
+	"os"
+	"sync/atomic"
+	"testing"
+
+	"github.com/daniel-widrick/GraceNoteScraper/guide"
+	"github.com/daniel-widrick/GraceNoteScraper/scrape"
+	"github.com/daniel-widrick/GraceNoteScraper/web"
+)
+
+type countingFetcher struct {
+	calls atomic.Int32
+	grid  *web.GridResponse
+	err   error
+}
+
+func (f *countingFetcher) GetDataByTimeContext(ctx context.Context, t int64) (*web.GridResponse, error) {
+	f.calls.Add(1)
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	if f.err != nil {
+		return nil, f.err
+	}
+	return f.grid, nil
+}
+
+// useFakeScrape swaps the production scrape options for a fake fetcher with
+// one day of slots and no delay. The preferences use a country tvlogo does not
+// support so no logo lookups reach the network.
+func useFakeScrape(t *testing.T, f *countingFetcher) web.Preferences {
+	t.Helper()
+	t.Chdir(t.TempDir())
+	original := newScrapeOptions
+	newScrapeOptions = func(web.Preferences) scrape.Options {
+		return scrape.Options{Fetcher: f, Days: 1, SlotDelay: scrape.NoDelay, Logger: log.New(io.Discard, "", 0)}
+	}
+	t.Cleanup(func() { newScrapeOptions = original })
+	return web.Preferences{Country: "ZZZ", ZipCode: "00000", Headend: "h", LineupId: "L", Device: "-", Language: "en-us"}
+}
+
+func TestRunScrapeBuildsLineupAndPersists(t *testing.T) {
+	f := &countingFetcher{grid: &web.GridResponse{Channels: []web.JSONChannel{
+		{ChannelID: "s1", ID: "s10", ChannelNo: "2.1", CallSign: "AAA", AffiliateName: "A Net", StationFilters: []string{"filter-news"},
+			Events: []web.JSONEvent{{StartTime: "2026-09-13T00:00:00Z", EndTime: "2026-09-13T01:00:00Z", Duration: "60", SeriesID: "SH1", Program: web.JSONProgram{ID: "EP1", Title: "one"}}}},
+		{ChannelID: "s1", ID: "s199", ChannelNo: "1002", CallSign: "AAA", AffiliateName: "A Net"},
+	}}}
+	pref := useFakeScrape(t, f)
+
+	var persisted *guide.TVGuide
+	persister := func(g *guide.TVGuide) (bool, error) {
+		persisted = g
+		return true, persistGuideFiles(g, "fp")
+	}
+	var updates []scrapeProgressUpdate
+	got, err := runScrape(pref, nil, "http://base:8080", nil, "fp", func() bool { return true }, persister, func(u scrapeProgressUpdate) { updates = append(updates, u) })
+	if err != nil {
+		t.Fatalf("runScrape: %v", err)
+	}
+	if f.calls.Load() != 4 {
+		t.Errorf("grid fetches = %d, want 4 for one day", f.calls.Load())
+	}
+	if persisted != got {
+		t.Error("persister should receive the returned guide")
+	}
+	if len(got.Channels) != 1 || len(got.Lineup) != 2 || len(got.Programs) != 1 {
+		t.Fatalf("channels/lineup/programs = %d/%d/%d", len(got.Channels), len(got.Lineup), len(got.Programs))
+	}
+	if got.Lineup[0].LogoURL != "" {
+		t.Errorf("no logo source available, got %q", got.Lineup[0].LogoURL)
+	}
+	if got.Source.LineupID != "L" {
+		t.Errorf("source = %+v", got.Source)
+	}
+	if _, err := os.Stat("xmlguide.xmltv"); err != nil {
+		t.Errorf("xmlguide.xmltv not written: %v", err)
+	}
+	stages := map[string]bool{}
+	for _, u := range updates {
+		stages[u.Stage] = true
+	}
+	for _, want := range []string{"gracenote", "logos", "saving"} {
+		if !stages[want] {
+			t.Errorf("missing progress stage %q (got %v)", want, stages)
+		}
+	}
+	if stages["tmdb"] {
+		t.Error("no TMDB client configured, so no tmdb stage should be reported")
+	}
+	var gracenote []scrapeProgressUpdate
+	for _, u := range updates {
+		if u.Stage == "gracenote" {
+			gracenote = append(gracenote, u)
+		}
+	}
+	if len(gracenote) != 8 || gracenote[0].Total != 4 || gracenote[7].Completed != 4 {
+		t.Errorf("gracenote progress = %d updates, first %+v, last %+v", len(gracenote), gracenote[0], gracenote[len(gracenote)-1])
+	}
+}
+
+func TestRunScrapeReportsSourceChange(t *testing.T) {
+	f := &countingFetcher{grid: &web.GridResponse{Channels: []web.JSONChannel{{ChannelID: "s1", ChannelNo: "2"}}}}
+	pref := useFakeScrape(t, f)
+
+	var seen atomic.Int32
+	sourceCurrent := func() bool {
+		// Current for the initial check, changed afterwards.
+		return seen.Add(1) == 1
+	}
+	persisterCalled := false
+	_, err := runScrape(pref, nil, "", nil, "fp", sourceCurrent, func(*guide.TVGuide) (bool, error) {
+		persisterCalled = true
+		return true, nil
+	})
+	if !errors.Is(err, errScrapeSourceChanged) {
+		t.Fatalf("err = %v, want errScrapeSourceChanged", err)
+	}
+	if persisterCalled {
+		t.Error("persister must not run after a source change")
+	}
+}
+
+func TestRunScrapeSurfacesTotalFailure(t *testing.T) {
+	f := &countingFetcher{err: errors.New("gracenote down")}
+	pref := useFakeScrape(t, f)
+	_, err := runScrape(pref, nil, "", nil, "fp", nil, func(*guide.TVGuide) (bool, error) {
+		t.Fatal("persister must not run with no data")
+		return false, nil
+	})
+	if !errors.Is(err, scrape.ErrNoData) {
+		t.Fatalf("err = %v, want ErrNoData", err)
+	}
+}