From 4e1c228f9d9d7c692b884dd04415b01440de7ea6 Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:29:42 -0400 Subject: [PATCH 1/7] test: add XMLTV golden test rendered from a grid fixture Factor template rendering out of persistGuideFiles into renderXMLTV so tests can render to a buffer. The golden file is generated from the current conversion code before any model changes. --- main.go | 18 +- web/testdata/grid_sample.json | 380 ++++++++++++++++++++++++++++++++++ xmltv_golden_test.go | 90 ++++++++ 3 files changed, 482 insertions(+), 6 deletions(-) create mode 100644 web/testdata/grid_sample.json create mode 100644 xmltv_golden_test.go diff --git a/main.go b/main.go index 69dc22e..92e930d 100644 --- a/main.go +++ b/main.go @@ -320,14 +320,20 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch return tvGuide, nil } -func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { - log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs)) - - // Parse embedded template +// renderXMLTV writes the guide as XMLTV using the embedded template. +func renderXMLTV(w io.Writer, tvGuide *guide.TVGuide) error { tmpl, err := template.ParseFS(guideTmplFS, "guide.tmpl") if err != nil { return fmt.Errorf("failed to parse template: %w", err) } + if err := tmpl.Execute(w, tvGuide); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + return nil +} + +func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { + log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs)) // Atomic write: write to temp file, then rename tmpFile, err := os.CreateTemp(".", "xmlguide-*.tmp") @@ -336,10 +342,10 @@ func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { } tmpName := tmpFile.Name() - if err := tmpl.Execute(tmpFile, tvGuide); err != nil { + if err := renderXMLTV(tmpFile, tvGuide); err != nil { tmpFile.Close() os.Remove(tmpName) - return fmt.Errorf("failed to execute template: %w", err) + return err } if err := tmpFile.Close(); err != nil { os.Remove(tmpName) diff --git a/web/testdata/grid_sample.json b/web/testdata/grid_sample.json new file mode 100644 index 0000000..bae4ef2 --- /dev/null +++ b/web/testdata/grid_sample.json @@ -0,0 +1,380 @@ +{ + "channels": [ + { + "affiliateCallSign": "null", + "affiliateName": "NATIONAL BROADCASTING COMPANY", + "callSign": "WKTVDT", + "channelId": "53158", + "channelNo": "2.1", + "events": [ + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "210", + "endTime": "2026-09-13T03:00:00Z", + "filter": [ + "filter-sports" + ], + "flag": [ + "Live", + "New" + ], + "program": { + "episode": null, + "episodeTitle": "Iowa State at Iowa", + "id": "EP062700990343", + "isGeneric": "0", + "releaseYear": 2019, + "season": null, + "seriesId": "SH06270099", + "shortDesc": "The Iowa State Cyclones take on the No. 21 Iowa Hawkeyes at ", + "title": "College Football", + "tmsId": "EP062700990343" + }, + "rating": null, + "seriesId": "SH06270099", + "startTime": "2026-09-12T23:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p53158_placeholder" + }, + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "29", + "endTime": "2026-09-13T03:29:00Z", + "filter": [ + "filter-news" + ], + "flag": [ + "New" + ], + "program": { + "episode": null, + "episodeTitle": null, + "id": "SH007933000000", + "isGeneric": "0", + "releaseYear": "2021", + "season": null, + "seriesId": "SH00793300", + "shortDesc": "Stay informed with the latest breaking news and headlines.", + "title": "NewsChannel 2 at 11", + "tmsId": "SH007933000000" + }, + "rating": null, + "seriesId": "SH00793300", + "startTime": "2026-09-13T03:00:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p53158_placeholder" + }, + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "94", + "endTime": "2026-09-13T05:03:00Z", + "filter": [], + "flag": [], + "program": { + "episode": "14", + "episodeTitle": "Ryan Gosling; Gorillaz", + "id": "EP000037101609", + "isGeneric": "0", + "releaseYear": null, + "season": "51", + "seriesId": "SH00003710", + "shortDesc": "Host Ryan Gosling; Gorillaz perform.", + "title": "Saturday Night Live", + "tmsId": "EP000037101609" + }, + "rating": "TV-14", + "seriesId": "SH00003710", + "startTime": "2026-09-13T03:29:00Z", + "tags": [ + "Stereo", + "CC" + ], + "thumbnail": "p53158_placeholder" + } + ], + "id": "531580", + "stationFilters": [ + "filter-sports", + "filter-news" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/53158.png?w=55" + }, + { + "affiliateCallSign": "null", + "affiliateName": "BOUNCE TV", + "callSign": "WSYRDT3", + "channelId": "84540", + "channelNo": "9.3", + "events": [ + { + "callSign": "WSYRDT3", + "channelNo": "9.3", + "duration": "150", + "endTime": "2026-09-13T01:30:00Z", + "filter": [ + "filter-movie" + ], + "flag": [], + "program": { + "episode": null, + "episodeTitle": null, + "id": "MV015399200000", + "isGeneric": "1", + "releaseYear": "2021", + "season": null, + "seriesId": "MV01539920", + "shortDesc": "Rescued as a child by the legendary assassin Moody, Anna is ", + "title": "The Prot\u00e9g\u00e9", + "tmsId": "MV015399200000" + }, + "rating": "TV-14", + "seriesId": "MV01539920", + "startTime": "2026-09-12T23:00:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p84540_placeholder" + }, + { + "callSign": "WSYRDT3", + "channelNo": "9.3", + "duration": "120", + "endTime": "2026-09-13T03:30:00Z", + "filter": [ + "filter-movie" + ], + "flag": [], + "program": { + "episode": null, + "episodeTitle": null, + "id": "MV002273030000", + "isGeneric": 0, + "releaseYear": "2009", + "season": null, + "seriesId": "MV00227303", + "shortDesc": "A rogue (Rain) enlists the help of the Europol agent (Naomie", + "title": "Ninja Assassin", + "tmsId": "MV002273030000" + }, + "rating": "TV-14", + "seriesId": "MV00227303", + "startTime": "2026-09-13T01:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p84540_placeholder" + }, + { + "callSign": "WSYRDT3", + "channelNo": "9.3", + "duration": "120", + "endTime": "2026-09-13T05:30:00Z", + "filter": [ + "filter-movie" + ], + "flag": [], + "program": { + "episode": null, + "episodeTitle": null, + "id": "MV003756000000", + "isGeneric": "0", + "releaseYear": "2012", + "season": null, + "seriesId": "MV00375600", + "shortDesc": "After his partner is killed, a veteran hit man (Sylvester St", + "title": "Bullet to the Head", + "tmsId": "MV003756000000" + }, + "rating": "TV-14", + "seriesId": "MV00375600", + "startTime": "2026-09-13T03:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p84540_placeholder" + } + ], + "id": "8454012", + "stationFilters": [ + "filter-movie" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/84540.png?w=55" + }, + { + "affiliateCallSign": "null", + "affiliateName": "PBS KIDS HD", + "callSign": "WCNYDT4", + "channelId": "49216", + "channelNo": "24.4", + "events": [ + { + "callSign": "WCNYDT4", + "channelNo": "24.4", + "duration": "30", + "endTime": "2026-09-13T00:30:00Z", + "filter": [ + "filter-family" + ], + "flag": [ + "New", + "Finale" + ], + "program": { + "episode": "30", + "episodeTitle": "The Square Dance; The Process", + "id": "EP052844640100", + "isGeneric": "0", + "releaseYear": null, + "season": "1", + "seriesId": "SH05284464", + "shortDesc": "Dylan realizes he doesn't love square-dancing; Carl teaches ", + "title": "Carl the Collector", + "tmsId": "EP052844640100" + }, + "rating": "TV-Y", + "seriesId": "SH05284464", + "startTime": "2026-09-13T00:00:00Z", + "tags": [ + "Stereo", + "CC" + ], + "thumbnail": "p49216_placeholder" + }, + { + "callSign": "WCNYDT4", + "channelNo": "24.4", + "duration": "30", + "endTime": "2026-09-13T01:00:00Z", + "filter": [ + "filter-family" + ], + "flag": [ + "Premiere" + ], + "program": { + "episode": "31", + "episodeTitle": "The Bark Banquet; Carl's Collection of To-Dos", + "id": "EP052844640101", + "isGeneric": "0", + "releaseYear": null, + "season": "1", + "seriesId": "SH05284464", + "shortDesc": "Carl wants to start his own family tradition after Sheldon i", + "title": "Carl the Collector", + "tmsId": "EP052844640101" + }, + "rating": "TV-Y", + "seriesId": "SH05284464", + "startTime": "2026-09-13T00:30:00Z", + "tags": [ + "Stereo", + "CC" + ], + "thumbnail": "p49216_placeholder" + }, + { + "callSign": "WCNYDT4", + "channelNo": "24.4", + "duration": "30", + "endTime": "2026-09-13T01:30:00Z", + "filter": [ + "filter-family" + ], + "flag": [], + "program": { + "episode": "6", + "episodeTitle": "Buzz and the Tree", + "id": "EP003618800155", + "isGeneric": "0", + "releaseYear": null, + "season": "13", + "seriesId": "SH00361880", + "shortDesc": "It's Arbor Day in Cyberspace, but Buzz is more interested in", + "title": "Cyberchase", + "tmsId": "EP003618800155" + }, + "rating": "TV-Y", + "seriesId": "SH00361880", + "startTime": "2026-09-13T01:00:00Z", + "tags": [ + "Stereo", + "CC", + "DVS" + ], + "thumbnail": "p49216_placeholder" + } + ], + "id": "4921632", + "stationFilters": [ + "filter-family" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/49216.png?w=55" + }, + { + "affiliateCallSign": "null", + "affiliateName": "NATIONAL BROADCASTING COMPANY", + "callSign": "WKTVDT", + "channelId": "53158", + "channelNo": "1002", + "events": [ + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "210", + "endTime": "2026-09-13T03:00:00Z", + "filter": [ + "filter-sports" + ], + "flag": [ + "Live", + "New" + ], + "program": { + "episode": null, + "episodeTitle": "Iowa State at Iowa", + "id": "EP062700990343", + "isGeneric": "0", + "releaseYear": null, + "season": null, + "seriesId": "SH06270099", + "shortDesc": "The Iowa State Cyclones take on the No. 21 Iowa Hawkeyes at ", + "title": "College Football", + "tmsId": "EP062700990343" + }, + "rating": null, + "seriesId": "SH06270099", + "startTime": "2026-09-12T23:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p53158_placeholder" + } + ], + "id": "5315899", + "stationFilters": [ + "filter-sports", + "filter-news" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/53158.png?w=55" + } + ] +} \ No newline at end of file diff --git a/xmltv_golden_test.go b/xmltv_golden_test.go new file mode 100644 index 0000000..d4f67e1 --- /dev/null +++ b/xmltv_golden_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "os" + "path/filepath" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/guide" + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +var updateGolden = flag.Bool("update", false, "rewrite testdata/xmlguide_golden.xmltv from the current conversion path") + +const ( + gridFixturePath = "web/testdata/grid_sample.json" + xmltvGoldenPath = "testdata/xmlguide_golden.xmltv" + fixtureLanguage = "en-us" + fixtureCountry = "USA" +) + +// loadGridFixture decodes the shared grid sample used by the golden and +// conversion tests. +func loadGridFixture(t *testing.T) web.GridResponse { + t.Helper() + data, err := os.ReadFile(gridFixturePath) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + var grid web.GridResponse + if err := json.Unmarshal(data, &grid); err != nil { + t.Fatalf("decode fixture: %v", err) + } + return grid +} + +// guideFromGrid mirrors the scrape loop's conversion: channels deduplicated by +// station ID in first-seen order, events deduplicated by station, start, and end. +func guideFromGrid(grid web.GridResponse) *guide.TVGuide { + seenChannel := make(map[string]bool) + seenEvent := make(map[string]bool) + g := &guide.TVGuide{} + for _, ch := range grid.Channels { + if !seenChannel[ch.ChannelID] { + seenChannel[ch.ChannelID] = true + g.Channels = append(g.Channels, guide.ConvertChannel(ch)) + } + for _, ev := range ch.Events { + key := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime + if seenEvent[key] { + continue + } + seenEvent[key] = true + g.Programs = append(g.Programs, guide.ConvertEvent(ev, ch.ChannelID, fixtureLanguage, fixtureCountry)) + } + } + return g +} + +// TestXMLTVGolden guards the promise that model changes never alter XMLTV +// output. Regenerate deliberately with: go test -run TestXMLTVGolden -update +func TestXMLTVGolden(t *testing.T) { + g := guideFromGrid(loadGridFixture(t)) + + var rendered bytes.Buffer + if err := renderXMLTV(&rendered, g); err != nil { + t.Fatalf("render: %v", err) + } + + if *updateGolden { + if err := os.MkdirAll(filepath.Dir(xmltvGoldenPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(xmltvGoldenPath, rendered.Bytes(), 0644); err != nil { + t.Fatal(err) + } + t.Logf("wrote %s (%d bytes)", xmltvGoldenPath, rendered.Len()) + return + } + + want, err := os.ReadFile(xmltvGoldenPath) + if err != nil { + t.Fatalf("read golden (run with -update to create it): %v", err) + } + if !bytes.Equal(want, rendered.Bytes()) { + t.Fatalf("XMLTV output changed from golden.\nIf this is intended, run: go test -run TestXMLTVGolden -update\n--- got ---\n%s", rendered.String()) + } +} From 3c7fda516a8110bcf532839968ad215643117aa1 Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:30:52 -0400 Subject: [PATCH 2/7] Keep raw Gracenote fields on channels and programs Carry the row id, affiliate callsign, and station filters through to guide.Channel, and the TMS id, release year, generic flag, and raw event filters through to guide.Program. Categories are unchanged; Filters holds the Gracenote list before Series and Finale are appended. releaseYear and isGeneric use tolerant decoders so an unexpected scalar encoding cannot fail an entire six-hour grid. The guide cache gains a schema version. An older cache is rebuilt once so the new fields are populated. --- guide/guide.go | 62 ++++++++++++++++---- guide/guide_test.go | 115 +++++++++++++++++++++++++++++++++++++ main.go | 12 +++- setup_test.go | 19 +++++++ web/flex.go | 76 +++++++++++++++++++++++++ web/web.go | 32 +++++++---- web/web_decode_test.go | 125 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 416 insertions(+), 25 deletions(-) create mode 100644 guide/guide_test.go create mode 100644 web/flex.go create mode 100644 web/web_decode_test.go diff --git a/guide/guide.go b/guide/guide.go index 821f5f7..d052037 100644 --- a/guide/guide.go +++ b/guide/guide.go @@ -15,12 +15,15 @@ type TVGuide struct { } type Channel struct { - ID string - DisplayNames []DisplayName - IconURL string - CallSign string // internal, not in template - Affiliate string // internal, not in template - ChannelNo string // internal, not in template + ID string + DisplayNames []DisplayName + IconURL string + CallSign string // internal, not in template + Affiliate string // internal, not in template + ChannelNo string // internal, not in template + PlacementID string // internal, not in template; Gracenote row id, not a stable key + AffiliateCallSign string // internal, not in template + Filters []string // internal, not in template; Gracenote station filters, prefix stripped } type DisplayName struct { @@ -45,6 +48,10 @@ type Program struct { Country string EpisodeNumbers []EpisodeNumber Categories []Category + Filters []string // internal, not in template; raw Gracenote event filters, prefix stripped + TMSID string // internal, not in template + ReleaseYear string // internal, not in template + Generic bool // internal, not in template New bool Premiere bool PreviouslyShown bool @@ -119,13 +126,37 @@ func ConvertChannel(ch web.JSONChannel) Channel { {Name: xmlEscape(ch.CallSign)}, {Name: xmlEscape(titleCase(ch.AffiliateName))}, }, - IconURL: iconURL, - CallSign: ch.CallSign, - Affiliate: ch.AffiliateName, - ChannelNo: ch.ChannelNo, + IconURL: iconURL, + CallSign: ch.CallSign, + Affiliate: ch.AffiliateName, + ChannelNo: ch.ChannelNo, + PlacementID: ch.ID, + AffiliateCallSign: normalizeNull(ch.AffiliateCallSign), + Filters: stripFilterPrefixes(ch.StationFilters), } } +// normalizeNull maps Gracenote's literal "null" string to an empty value. +func normalizeNull(s string) string { + if strings.EqualFold(strings.TrimSpace(s), "null") { + return "" + } + return s +} + +// stripFilterPrefixes turns Gracenote filter tags such as "filter-sports" into +// "sports". A nil input stays nil so callers can distinguish absent from empty. +func stripFilterPrefixes(filters []string) []string { + if filters == nil { + return nil + } + out := make([]string, 0, len(filters)) + for _, f := range filters { + out = append(out, strings.TrimPrefix(f, "filter-")) + } + return out +} + // converts a JSON event to a template Program struct. func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program { season := 0 @@ -163,10 +194,13 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program { // URL programURL := "https://tvlistings.gracenote.com//overview.html?programSeriesId=" + ev.SeriesID + "&tmsId=" + ev.Program.ID + // Raw Gracenote filters, kept separately so consumers can tell them apart + // from the Series and Finale labels added below. + filters := stripFilterPrefixes(ev.Filter) + // Categories from filter array (strip "filter-" prefix) var categories []Category - for _, f := range ev.Filter { - name := strings.TrimPrefix(f, "filter-") + for _, name := range filters { categories = append(categories, Category{Name: name, Lang: lang}) } @@ -266,6 +300,10 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program { Country: country, EpisodeNumbers: episodeNumbers, Categories: categories, + Filters: filters, + TMSID: ev.Program.TmsID, + ReleaseYear: string(ev.Program.ReleaseYear), + Generic: bool(ev.Program.IsGeneric), New: isNew, Premiere: isPremiere, PreviouslyShown: !isNew, diff --git a/guide/guide_test.go b/guide/guide_test.go new file mode 100644 index 0000000..852cde4 --- /dev/null +++ b/guide/guide_test.go @@ -0,0 +1,115 @@ +package guide + +import ( + "reflect" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +func sampleChannel() web.JSONChannel { + return web.JSONChannel{ + ChannelID: "53158", + ID: "531580", + ChannelNo: "2.1", + CallSign: "WKTVDT", + AffiliateName: "NATIONAL BROADCASTING COMPANY", + AffiliateCallSign: "null", + StationFilters: []string{"filter-sports", "filter-news"}, + Thumbnail: "//images.example.invalid/station/53158.png?w=55", + } +} + +func TestConvertChannelCarriesRawFields(t *testing.T) { + ch := ConvertChannel(sampleChannel()) + + if ch.ID != "53158" || ch.ChannelNo != "2.1" || ch.CallSign != "WKTVDT" { + t.Fatalf("basic fields wrong: %+v", ch) + } + if ch.PlacementID != "531580" { + t.Errorf("PlacementID = %q", ch.PlacementID) + } + if ch.AffiliateCallSign != "" { + t.Errorf(`"null" affiliate callsign should normalize to empty, got %q`, ch.AffiliateCallSign) + } + if !reflect.DeepEqual(ch.Filters, []string{"sports", "news"}) { + t.Errorf("Filters = %v", ch.Filters) + } + if ch.IconURL != "http://images.example.invalid/station/53158.png" { + t.Errorf("IconURL = %q", ch.IconURL) + } + // Existing XMLTV-facing fields must be untouched by the additions. + wantNames := []DisplayName{{"2.1 WKTVDT"}, {"2.1"}, {"WKTVDT"}, {"NATIONAL BROADCASTING COMPANY"}} + if !reflect.DeepEqual(ch.DisplayNames, wantNames) { + t.Errorf("DisplayNames = %v", ch.DisplayNames) + } +} + +func TestConvertChannelKeepsRealAffiliateCallSign(t *testing.T) { + in := sampleChannel() + in.AffiliateCallSign = "NBC" + in.StationFilters = nil + ch := ConvertChannel(in) + if ch.AffiliateCallSign != "NBC" { + t.Errorf("AffiliateCallSign = %q", ch.AffiliateCallSign) + } + if ch.Filters != nil { + t.Errorf("nil station filters should stay nil, got %v", ch.Filters) + } +} + +func sampleEvent() web.JSONEvent { + season, episode, title := "3", "7", "The One" + return web.JSONEvent{ + StartTime: "2026-07-25T07:00:00Z", + EndTime: "2026-07-25T08:00:00Z", + Duration: "60", + SeriesID: "SH06270099", + Flag: []string{"New", "Finale"}, + Filter: []string{"filter-sports"}, + Program: web.JSONProgram{ + ID: "EP062700990343", + TmsID: "EP062700990343", + Title: "Morning Business", + EpisodeTitle: &title, + Season: &season, + Episode: &episode, + ReleaseYear: "2019", + IsGeneric: true, + }, + } +} + +func TestConvertEventSeparatesRawFiltersFromCategories(t *testing.T) { + p := ConvertEvent(sampleEvent(), "53158", "en-us", "USA") + + if !reflect.DeepEqual(p.Filters, []string{"sports"}) { + t.Errorf("Filters = %v", p.Filters) + } + // Categories keep the historical shape: raw filters, then Series (from an + // episode number), then Finale (from the flag). + want := []Category{{"sports", "en-us"}, {"Series", "en-us"}, {"Finale", "en-us"}} + if !reflect.DeepEqual(p.Categories, want) { + t.Errorf("Categories = %v, want %v", p.Categories, want) + } + if p.TMSID != "EP062700990343" || p.ReleaseYear != "2019" || !p.Generic { + t.Errorf("program metadata = tms %q year %q generic %v", p.TMSID, p.ReleaseYear, p.Generic) + } + if !p.New || p.PreviouslyShown { + t.Errorf("flag handling changed: new=%v previouslyShown=%v", p.New, p.PreviouslyShown) + } +} + +func TestConvertEventWithoutFilters(t *testing.T) { + ev := sampleEvent() + ev.Filter = nil + ev.Flag = nil + ev.Program.Season, ev.Program.Episode = nil, nil + p := ConvertEvent(ev, "53158", "en", "USA") + if p.Filters != nil { + t.Errorf("Filters should be nil, got %v", p.Filters) + } + if len(p.Categories) != 0 { + t.Errorf("Categories should be empty, got %v", p.Categories) + } +} diff --git a/main.go b/main.go index 92e930d..156e69b 100644 --- a/main.go +++ b/main.go @@ -369,7 +369,13 @@ func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { const guideCachePath = "guide_cache.json" +// guideCacheVersion is bumped whenever the persisted guide shape gains fields +// that a scrape must populate. An older cache is rebuilt rather than served +// with empty fields. +const guideCacheVersion = 2 + type guideCache struct { + Version int `json:"version"` SavedAt time.Time `json:"saved_at"` SourceFingerprint string `json:"source_fingerprint"` Guide guide.TVGuide `json:"guide"` @@ -377,7 +383,7 @@ type guideCache struct { // saveGuideCache persists the TVGuide to a JSON file. func saveGuideCache(g *guide.TVGuide, sourceFingerprint string) { - data, err := json.Marshal(guideCache{SavedAt: time.Now(), SourceFingerprint: sourceFingerprint, Guide: *g}) + data, err := json.Marshal(guideCache{Version: guideCacheVersion, SavedAt: time.Now(), SourceFingerprint: sourceFingerprint, Guide: *g}) if err != nil { log.Printf("guide cache: failed to marshal: %v", err) return @@ -401,6 +407,10 @@ func loadGuideCache(maxAge time.Duration, sourceFingerprint string) (*guide.TVGu log.Printf("guide cache: corrupt, ignoring: %v", err) return nil, 0, false } + if c.Version != guideCacheVersion { + log.Printf("guide cache: schema version %d, want %d; rebuilding", c.Version, guideCacheVersion) + return nil, 0, false + } if c.SourceFingerprint != sourceFingerprint { log.Println("guide cache: source changed, ignoring cached guide") return nil, 0, false diff --git a/setup_test.go b/setup_test.go index 7651721..ba03181 100644 --- a/setup_test.go +++ b/setup_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" "path/filepath" "strings" "testing" @@ -226,3 +227,21 @@ func TestGuideCacheRequiresMatchingSource(t *testing.T) { t.Fatal("cache did not load for matching source") } } + +func TestGuideCacheRequiresCurrentSchemaVersion(t *testing.T) { + t.Chdir(t.TempDir()) + + // A cache written by an older build has no version field. + legacy := []byte(`{"saved_at":"` + time.Now().UTC().Format(time.RFC3339) + `","source_fingerprint":"src","guide":{"Channels":[],"Programs":[]}}`) + if err := os.WriteFile(guideCachePath, legacy, 0644); err != nil { + t.Fatal(err) + } + if _, _, ok := loadGuideCache(time.Hour, "src"); ok { + t.Fatal("legacy cache without a version loaded") + } + + saveGuideCache(&guide.TVGuide{}, "src") + if _, _, ok := loadGuideCache(time.Hour, "src"); !ok { + t.Fatal("current-version cache did not load") + } +} diff --git a/web/flex.go b/web/flex.go new file mode 100644 index 0000000..ad88a65 --- /dev/null +++ b/web/flex.go @@ -0,0 +1,76 @@ +package web + +import ( + "bytes" + "encoding/json" + "strconv" + "strings" +) + +// FlexString decodes a JSON string, number, or null into a string. Gracenote +// is inconsistent about scalar encodings, and a decode error on one optional +// field would otherwise discard an entire six-hour grid. +type FlexString string + +func (f *FlexString) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *f = "" + return nil + } + if data[0] == '"' { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + *f = FlexString(s) + return nil + } + var n json.Number + if err := json.Unmarshal(data, &n); err == nil { + *f = FlexString(n.String()) + return nil + } + var b bool + if err := json.Unmarshal(data, &b); err == nil { + *f = FlexString(strconv.FormatBool(b)) + return nil + } + *f = "" + return nil +} + +// FlexBool decodes a JSON bool, a 0/1 number, a "0"/"1"/"true"/"false" string, +// or null into a bool. Anything unrecognized decodes as false rather than +// failing the surrounding document. +type FlexBool bool + +func (f *FlexBool) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *f = false + return nil + } + var b bool + if err := json.Unmarshal(data, &b); err == nil { + *f = FlexBool(b) + return nil + } + var n json.Number + if err := json.Unmarshal(data, &n); err == nil { + *f = FlexBool(n.String() != "0" && n.String() != "0.0") + return nil + } + var s string + if err := json.Unmarshal(data, &s); err == nil { + switch strings.ToLower(strings.TrimSpace(s)) { + case "1", "true", "yes": + *f = true + default: + *f = false + } + return nil + } + *f = false + return nil +} diff --git a/web/web.go b/web/web.go index c4160e3..1d7e1d6 100644 --- a/web/web.go +++ b/web/web.go @@ -32,12 +32,17 @@ type GridResponse struct { } type JSONChannel struct { - ChannelID string `json:"channelId"` - ChannelNo string `json:"channelNo"` - CallSign string `json:"callSign"` - AffiliateName string `json:"affiliateName"` - Thumbnail string `json:"thumbnail"` - Events []JSONEvent `json:"events"` + ChannelID string `json:"channelId"` + // ID is the station ID with the row index appended. It identifies a row + // within one response, not a stable lineup placement. + ID string `json:"id"` + ChannelNo string `json:"channelNo"` + CallSign string `json:"callSign"` + AffiliateName string `json:"affiliateName"` + AffiliateCallSign string `json:"affiliateCallSign"` + StationFilters []string `json:"stationFilters"` + Thumbnail string `json:"thumbnail"` + Events []JSONEvent `json:"events"` } type JSONEvent struct { @@ -54,12 +59,15 @@ type JSONEvent struct { } type JSONProgram struct { - ID string `json:"id"` - Title string `json:"title"` - EpisodeTitle *string `json:"episodeTitle"` - ShortDesc *string `json:"shortDesc"` - Season *string `json:"season"` - Episode *string `json:"episode"` + ID string `json:"id"` + TmsID string `json:"tmsId"` + Title string `json:"title"` + EpisodeTitle *string `json:"episodeTitle"` + ShortDesc *string `json:"shortDesc"` + Season *string `json:"season"` + Episode *string `json:"episode"` + ReleaseYear FlexString `json:"releaseYear"` + IsGeneric FlexBool `json:"isGeneric"` } type Preferences struct { diff --git a/web/web_decode_test.go b/web/web_decode_test.go new file mode 100644 index 0000000..e1af3b7 --- /dev/null +++ b/web/web_decode_test.go @@ -0,0 +1,125 @@ +package web + +import ( + "encoding/json" + "os" + "testing" +) + +func TestGridFixtureDecodesRawFields(t *testing.T) { + data, err := os.ReadFile("testdata/grid_sample.json") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + var grid GridResponse + if err := json.Unmarshal(data, &grid); err != nil { + t.Fatalf("decode fixture: %v", err) + } + if len(grid.Channels) != 4 { + t.Fatalf("channels = %d, want 4", len(grid.Channels)) + } + + first := grid.Channels[0] + if first.ID != "531580" || first.ChannelID != "53158" { + t.Fatalf("row id/station id = %q/%q", first.ID, first.ChannelID) + } + if first.AffiliateCallSign != "null" { + t.Fatalf("affiliateCallSign should decode verbatim, got %q", first.AffiliateCallSign) + } + if len(first.StationFilters) != 2 || first.StationFilters[0] != "filter-sports" { + t.Fatalf("stationFilters = %v", first.StationFilters) + } + + // Same station at a second position decodes as its own row. + last := grid.Channels[3] + if last.ChannelID != first.ChannelID || last.ChannelNo == first.ChannelNo || last.ID == first.ID { + t.Fatalf("duplicate position not preserved: %+v", last) + } + + ev := first.Events[0].Program + if ev.TmsID == "" { + t.Fatal("tmsId missing") + } + if ev.ReleaseYear != "2019" { + t.Fatalf("numeric releaseYear = %q, want 2019", ev.ReleaseYear) + } + if first.Events[1].Program.ReleaseYear != "2021" { + t.Fatalf("string releaseYear = %q, want 2021", first.Events[1].Program.ReleaseYear) + } + if first.Events[2].Program.ReleaseYear != "" { + t.Fatalf("null releaseYear = %q, want empty", first.Events[2].Program.ReleaseYear) + } + second := grid.Channels[1] + if !bool(second.Events[0].Program.IsGeneric) { + t.Fatal(`isGeneric "1" should decode true`) + } + if bool(second.Events[1].Program.IsGeneric) { + t.Fatal("isGeneric 0 should decode false") + } +} + +func TestFlexString(t *testing.T) { + cases := []struct { + in string + want string + }{ + {`"2019"`, "2019"}, + {`2019`, "2019"}, + {`2019.0`, "2019.0"}, + {`null`, ""}, + {`true`, "true"}, + {`""`, ""}, + {`[1]`, ""}, + {`{"a":1}`, ""}, + } + for _, c := range cases { + var doc struct { + V FlexString `json:"v"` + } + if err := json.Unmarshal([]byte(`{"v":`+c.in+`}`), &doc); err != nil { + t.Fatalf("%s: unexpected error %v", c.in, err) + } + if string(doc.V) != c.want { + t.Errorf("%s: got %q want %q", c.in, doc.V, c.want) + } + } + var doc struct { + V FlexString `json:"v"` + } + if err := json.Unmarshal([]byte(`{"v":"unterminated}`), &doc); err == nil { + t.Fatal("malformed JSON should still error") + } +} + +func TestFlexBool(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {`true`, true}, + {`false`, false}, + {`1`, true}, + {`0`, false}, + {`2`, true}, + {`"1"`, true}, + {`"0"`, false}, + {`"true"`, true}, + {`"TRUE"`, true}, + {`"false"`, false}, + {`"yes"`, true}, + {`"maybe"`, false}, + {`null`, false}, + {`[true]`, false}, + } + for _, c := range cases { + var doc struct { + V FlexBool `json:"v"` + } + if err := json.Unmarshal([]byte(`{"v":`+c.in+`}`), &doc); err != nil { + t.Fatalf("%s: unexpected error %v", c.in, err) + } + if bool(doc.V) != c.want { + t.Errorf("%s: got %v want %v", c.in, doc.V, c.want) + } + } +} From b28ab1f69f4d886eac0c1a49f52828e9eba3ac2f Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:34:37 -0400 Subject: [PATCH 3/7] test: commit the XMLTV golden file *.xmltv is ignored repo-wide, which kept the golden out of the first commit. Allowlist it explicitly. --- .gitignore | 1 + testdata/xmlguide_golden.xmltv | 167 +++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 testdata/xmlguide_golden.xmltv diff --git a/.gitignore b/.gitignore index a78349e..9c1b9fa 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .env config.json *.xmltv +!testdata/xmlguide_golden.xmltv tmdb_cache.json tvlogo_cache.json guide_cache.json diff --git a/testdata/xmlguide_golden.xmltv b/testdata/xmlguide_golden.xmltv new file mode 100644 index 0000000..8944ec8 --- /dev/null +++ b/testdata/xmlguide_golden.xmltv @@ -0,0 +1,167 @@ + + + + + 2.1 WKTVDT + 2.1 + WKTVDT + NATIONAL BROADCASTING COMPANY + + + + 9.3 WSYRDT3 + 9.3 + WSYRDT3 + BOUNCE TV + + + + 24.4 WCNYDT4 + 24.4 + WCNYDT4 + PBS KIDS HD + + + + College Football + Iowa State at Iowa + The Iowa State Cyclones take on the No. 21 Iowa Hawkeyes at + 210 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH06270099&tmsId=EP062700990343 + USA + EP06270099.0343 + sports + + + + + NewsChannel 2 at 11 + Stay informed with the latest breaking news and headlines. + 29 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH00793300&tmsId=SH007933000000 + USA + SH00793300.0000 + news + + + + + Saturday Night Live + Ryan Gosling; Gorillaz + Host Ryan Gosling; Gorillaz perform. + 94 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH00003710&tmsId=EP000037101609 + USA + S51E14 + 50.13 + EP00003710.1609 + Series + + + TV-14 + + + The Protégé + Rescued as a child by the legendary assassin Moody, Anna is + 150 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=MV01539920&tmsId=MV015399200000 + USA + MV01539920.0000 + movie + + + TV-14 + + + Ninja Assassin + A rogue (Rain) enlists the help of the Europol agent (Naomie + 120 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=MV00227303&tmsId=MV002273030000 + USA + MV00227303.0000 + movie + + + TV-14 + + + Bullet to the Head + After his partner is killed, a veteran hit man (Sylvester St + 120 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=MV00375600&tmsId=MV003756000000 + USA + MV00375600.0000 + movie + + + TV-14 + + + Carl the Collector + The Square Dance; The Process + Dylan realizes he doesn't love square-dancing; Carl teaches + 30 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH05284464&tmsId=EP052844640100 + USA + S01E30 + 0.29 + EP05284464.0100 + family + Series + Finale + + + TV-Y + + + Carl the Collector + The Bark Banquet; Carl's Collection of To-Dos + Carl wants to start his own family tradition after Sheldon i + 30 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH05284464&tmsId=EP052844640101 + USA + S01E31 + 0.30 + EP05284464.0101 + family + Series + + + + TV-Y + + + Cyberchase + Buzz and the Tree + It's Arbor Day in Cyberspace, but Buzz is more interested in + 30 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH00361880&tmsId=EP003618800155 + USA + S13E06 + 12.5 + EP00361880.0155 + family + Series + + + TV-Y + + From 57eac5b78f2787543f4afa78fbc17b0bac08457e Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:32:50 -0400 Subject: [PATCH 4/7] Add a lineup model that retains every provider position TVGuide gains Lineup, a list of channel positions keyed by number and station that is never collapsed, and Source, which records the lineup the guide was built from. Resolved station logos are copied onto each position, the image proxy rewrite covers them, and the Jellyfin channel filter applies to them by channel number. ChannelNumberLess moves into the guide package with a strict ordering: numeric numbers first, ties broken by string, then non-numeric. --- guide/guide.go | 127 +++++++++++++++++++++++++++++++++++++++----- guide/guide_test.go | 82 ++++++++++++++++++++++++++++ lineup_test.go | 85 +++++++++++++++++++++++++++++ main.go | 90 +++++++++++++++++++++---------- 4 files changed, 342 insertions(+), 42 deletions(-) create mode 100644 lineup_test.go diff --git a/guide/guide.go b/guide/guide.go index d052037..b502c4e 100644 --- a/guide/guide.go +++ b/guide/guide.go @@ -2,16 +2,112 @@ package guide import ( "fmt" + "sort" "strconv" "strings" + "time" "unicode" "github.com/daniel-widrick/GraceNoteScraper/web" ) type TVGuide struct { + // Channels is the XMLTV view: one entry per Gracenote station. Channels []Channel Programs []Program + // Lineup retains every provider position, so a station carried at two + // channel numbers appears twice. It is never collapsed. + Lineup []LineupPosition + // Source records which provider lineup produced this guide. + Source Source +} + +// Source identifies the Gracenote lineup a guide was built from. +type Source struct { + Country string + PostalCode string + HeadendID string + LineupID string + Device string + Language string + GeneratedAt time.Time +} + +// SourceFromPreferences copies the request preferences into a Source. +func SourceFromPreferences(p web.Preferences, generatedAt time.Time) Source { + return Source{ + Country: p.Country, + PostalCode: p.ZipCode, + HeadendID: p.Headend, + LineupID: p.LineupId, + Device: p.Device, + Language: p.Language, + GeneratedAt: generatedAt, + } +} + +// LineupPosition is one channel number in a provider lineup. +type LineupPosition struct { + ChannelNo string + StationID string + PlacementID string // Gracenote row id; carried for fidelity, not a stable key + CallSign string + Affiliate string + AffiliateCallSign string + Filters []string + LogoURL string +} + +// Key identifies a position across grid slices: the same station at the same +// number is one position no matter how many responses it appears in. +func (p LineupPosition) Key() string { + return p.ChannelNo + "|" + p.StationID +} + +// ConvertLineupPosition converts a JSON channel row to a lineup position. +func ConvertLineupPosition(ch web.JSONChannel) LineupPosition { + return LineupPosition{ + ChannelNo: ch.ChannelNo, + StationID: ch.ChannelID, + PlacementID: ch.ID, + CallSign: ch.CallSign, + Affiliate: ch.AffiliateName, + AffiliateCallSign: normalizeNull(ch.AffiliateCallSign), + Filters: stripFilterPrefixes(ch.StationFilters), + LogoURL: gracenoteIconURL(ch.Thumbnail), + } +} + +// ChannelNumberLess orders channel numbers numerically where both parse +// (so "2.1" < "10" < "100"), places numeric numbers before non-numeric ones, +// and falls back to string order. Equal numbers compare as strings so the +// ordering is strict. +func ChannelNumberLess(a, b string) bool { + af, errA := strconv.ParseFloat(strings.TrimSpace(a), 64) + bf, errB := strconv.ParseFloat(strings.TrimSpace(b), 64) + switch { + case errA == nil && errB == nil: + if af != bf { + return af < bf + } + return a < b + case errA == nil: + return true + case errB == nil: + return false + default: + return a < b + } +} + +// SortLineup orders positions by channel number, then station ID, in place. +func SortLineup(positions []LineupPosition) { + sort.SliceStable(positions, func(i, j int) bool { + if positions[i].ChannelNo != positions[j].ChannelNo { + return ChannelNumberLess(positions[i].ChannelNo, positions[j].ChannelNo) + } + return positions[i].StationID < positions[j].StationID + }) } type Channel struct { @@ -102,21 +198,26 @@ func formatXMLTVTime(iso string) string { return s } +// gracenoteIconURL builds an absolute icon URL from a Gracenote thumbnail +// path: strip leading slashes, strip query params, prepend http:// +func gracenoteIconURL(thumbnail string) string { + if thumbnail == "" { + return "" + } + raw := thumbnail + if idx := strings.Index(raw, "?"); idx >= 0 { + raw = raw[:idx] + } + raw = strings.TrimLeft(raw, "/") + if raw == "" { + return "" + } + return "http://" + raw +} + // converts a JSON channel to a template Channel struct. func ConvertChannel(ch web.JSONChannel) Channel { - // Build icon URL: strip leading slashes, strip query params, prepend http:// - iconURL := "" - if ch.Thumbnail != "" { - raw := ch.Thumbnail - // Strip query string - if idx := strings.Index(raw, "?"); idx >= 0 { - raw = raw[:idx] - } - raw = strings.TrimLeft(raw, "/") - if raw != "" { - iconURL = "http://" + raw - } - } + iconURL := gracenoteIconURL(ch.Thumbnail) return Channel{ ID: ch.ChannelID, diff --git a/guide/guide_test.go b/guide/guide_test.go index 852cde4..4274f9d 100644 --- a/guide/guide_test.go +++ b/guide/guide_test.go @@ -3,6 +3,7 @@ package guide import ( "reflect" "testing" + "time" "github.com/daniel-widrick/GraceNoteScraper/web" ) @@ -113,3 +114,84 @@ func TestConvertEventWithoutFilters(t *testing.T) { t.Errorf("Categories should be empty, got %v", p.Categories) } } + +func TestConvertLineupPosition(t *testing.T) { + in := sampleChannel() + p := ConvertLineupPosition(in) + want := LineupPosition{ + ChannelNo: "2.1", + StationID: "53158", + PlacementID: "531580", + CallSign: "WKTVDT", + Affiliate: "NATIONAL BROADCASTING COMPANY", + AffiliateCallSign: "", + Filters: []string{"sports", "news"}, + LogoURL: "http://images.example.invalid/station/53158.png", + } + if !reflect.DeepEqual(p, want) { + t.Fatalf("got %+v\nwant %+v", p, want) + } + if p.Key() != "2.1|53158" { + t.Errorf("Key = %q", p.Key()) + } + other := in + other.ChannelNo = "1002" + other.ID = "5315899" + if ConvertLineupPosition(other).Key() == p.Key() { + t.Error("same station at a different number must be a distinct position") + } +} + +func TestChannelNumberLess(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"2.1", "10", true}, + {"10", "2.1", false}, + {"10", "100", true}, + {"9", "10", true}, + {"2.1", "2.10", true}, // equal numerically, string order breaks the tie + {"2.10", "2.1", false}, + {"5", "5", false}, + {"12", "ABC", true}, // numeric before non-numeric + {"ABC", "12", false}, + {"ABC", "ABD", true}, + } + for _, c := range cases { + if got := ChannelNumberLess(c.a, c.b); got != c.want { + t.Errorf("ChannelNumberLess(%q, %q) = %v, want %v", c.a, c.b, got, c.want) + } + } +} + +func TestSortLineup(t *testing.T) { + positions := []LineupPosition{ + {ChannelNo: "100", StationID: "b"}, + {ChannelNo: "ZZ", StationID: "z"}, + {ChannelNo: "2.1", StationID: "a"}, + {ChannelNo: "10", StationID: "c"}, + {ChannelNo: "10", StationID: "a"}, + {ChannelNo: "9", StationID: "d"}, + } + SortLineup(positions) + var got []string + for _, p := range positions { + got = append(got, p.ChannelNo+"/"+p.StationID) + } + want := []string{"2.1/a", "9/d", "10/a", "10/c", "100/b", "ZZ/z"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("order = %v, want %v", got, want) + } +} + +func TestSourceFromPreferences(t *testing.T) { + at := time.Date(2026, 9, 13, 0, 0, 0, 0, time.UTC) + src := SourceFromPreferences(web.Preferences{ + Country: "USA", ZipCode: "13490", Headend: "lineupId", LineupId: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us", + }, at) + want := Source{Country: "USA", PostalCode: "13490", HeadendID: "lineupId", LineupID: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us", GeneratedAt: at} + if src != want { + t.Fatalf("got %+v want %+v", src, want) + } +} diff --git a/lineup_test.go b/lineup_test.go new file mode 100644 index 0000000..2a22296 --- /dev/null +++ b/lineup_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "net/url" + "reflect" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/guide" +) + +func sampleLineupGuide() *guide.TVGuide { + return &guide.TVGuide{ + Channels: []guide.Channel{ + {ID: "s1", ChannelNo: "2", IconURL: "http://logos.example/s1.png", DisplayNames: []guide.DisplayName{{Name: "2 AAA"}, {Name: "2"}, {Name: "AAA"}}}, + {ID: "s2", ChannelNo: "5", IconURL: "", DisplayNames: []guide.DisplayName{{Name: "5 BBB"}, {Name: "5"}, {Name: "BBB"}}}, + }, + Programs: []guide.Program{ + {Channel: "s1", Title: "one", IconSrc: "http://img.example/a.jpg", Images: []guide.Image{{URL: "http://img.example/b.jpg"}}}, + {Channel: "s2", Title: "two"}, + }, + Lineup: []guide.LineupPosition{ + {ChannelNo: "2", StationID: "s1"}, + {ChannelNo: "5", StationID: "s2"}, + {ChannelNo: "1002", StationID: "s1"}, // same station, second position + }, + Source: guide.Source{LineupID: "L"}, + } +} + +func TestPropagateChannelLogosCopiesByStation(t *testing.T) { + g := sampleLineupGuide() + propagateChannelLogos(g.Channels, g.Lineup) + + if g.Lineup[0].LogoURL != "http://logos.example/s1.png" || g.Lineup[2].LogoURL != "http://logos.example/s1.png" { + t.Fatalf("both positions of s1 should carry its logo: %+v", g.Lineup) + } + if g.Lineup[1].LogoURL != "" { + t.Fatalf("s2 has no logo, got %q", g.Lineup[1].LogoURL) + } +} + +func TestRewriteImageURLsCoversLineup(t *testing.T) { + g := sampleLineupGuide() + propagateChannelLogos(g.Channels, g.Lineup) + rewriteImageURLs("http://host:8080/", g.Channels, g.Lineup, g.Programs) + + want := "http://host:8080/img?url=" + url.QueryEscape("http://logos.example/s1.png") + if g.Channels[0].IconURL != want { + t.Errorf("channel icon = %q, want %q", g.Channels[0].IconURL, want) + } + if g.Lineup[0].LogoURL != want || g.Lineup[2].LogoURL != want { + t.Errorf("lineup logos = %q / %q, want %q", g.Lineup[0].LogoURL, g.Lineup[2].LogoURL, want) + } + if g.Lineup[1].LogoURL != "" || g.Channels[1].IconURL != "" { + t.Error("empty URLs must stay empty") + } + if g.Programs[0].IconSrc != "http://host:8080/img?url="+url.QueryEscape("http://img.example/a.jpg") { + t.Errorf("program icon = %q", g.Programs[0].IconSrc) + } + if g.Programs[0].Images[0].URL != "http://host:8080/img?url="+url.QueryEscape("http://img.example/b.jpg") { + t.Errorf("program image = %q", g.Programs[0].Images[0].URL) + } +} + +func TestFilterGuideChannelsFiltersLineup(t *testing.T) { + g := sampleLineupGuide() + filtered := filterGuideChannels(g, map[string]bool{"2": true, "1002": true}) + + if len(filtered.Channels) != 1 || filtered.Channels[0].ID != "s1" { + t.Fatalf("channels = %+v", filtered.Channels) + } + if len(filtered.Programs) != 1 || filtered.Programs[0].Title != "one" { + t.Fatalf("programs = %+v", filtered.Programs) + } + var numbers []string + for _, p := range filtered.Lineup { + numbers = append(numbers, p.ChannelNo) + } + if !reflect.DeepEqual(numbers, []string{"2", "1002"}) { + t.Fatalf("lineup numbers = %v", numbers) + } + if filtered.Source != g.Source { + t.Fatal("source not carried through filter") + } +} diff --git a/main.go b/main.go index 156e69b..8f9424f 100644 --- a/main.go +++ b/main.go @@ -145,7 +145,7 @@ func guideToJSON(g *guide.TVGuide) APIGuide { // Sort channels by number (numeric sort) sort.Slice(channels, func(i, j int) bool { - return channelNumberLess(channels[i].Number, channels[j].Number) + return guide.ChannelNumberLess(channels[i].Number, channels[j].Number) }) return APIGuide{ @@ -154,18 +154,6 @@ func guideToJSON(g *guide.TVGuide) APIGuide { } } -// channelNumberLess compares channel numbers numerically where possible. -func channelNumberLess(a, b string) bool { - // Try to parse as float for numeric comparison (handles "5.1", "12", etc.) - var ai, bi float64 - _, errA := fmt.Sscanf(a, "%f", &ai) - _, errB := fmt.Sscanf(b, "%f", &bi) - if errA == nil && errB == nil { - return ai < bi - } - return a < b -} - // xmltvTimeToISO converts "20250225200000 +0000" → "2025-02-25T20:00:00Z" func xmltvTimeToISO(xmltvTime string) string { xmltvTime = strings.TrimSpace(xmltvTime) @@ -214,6 +202,7 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch endTime := midnight.Add(14 * 24 * time.Hour) channelMap := make(map[string]guide.Channel) + lineupMap := make(map[string]guide.LineupPosition) eventMap := make(map[string]bool) var programs []guide.Program @@ -238,6 +227,10 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch if _, exists := channelMap[ch.ChannelID]; !exists { channelMap[ch.ChannelID] = guide.ConvertChannel(ch) } + position := guide.ConvertLineupPosition(ch) + if _, exists := lineupMap[position.Key()]; !exists { + lineupMap[position.Key()] = position + } for _, ev := range ch.Events { dedupKey := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime @@ -261,6 +254,11 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch for _, ch := range channelMap { channels = append(channels, ch) } + lineup := make([]guide.LineupPosition, 0, len(lineupMap)) + for _, position := range lineupMap { + lineup = append(lineup, position) + } + guide.SortLineup(lineup) logoClient := tvlogo.NewClient(pref.Country, "tvlogo_cache.json") if logoClient != nil { @@ -268,6 +266,7 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch } report(scrapeProgressUpdate{Stage: "logos", Message: "Matching channel logos", Channels: len(channels), Programs: len(programs)}) enrichChannelIcons(logoClient, channels) + propagateChannelLogos(channels, lineup) enrichProgramThumbnails(tmdbClient, programs, func(completed, total int) { report(scrapeProgressUpdate{Stage: "tmdb", Message: fmt.Sprintf("Enriching program titles (%d of %d)", completed, total), Completed: completed, Total: total, Channels: len(channels), Programs: len(programs)}) }) @@ -275,28 +274,15 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch // Rewrite image URLs to go through the local proxy if baseURL != "" { - proxy := strings.TrimRight(baseURL, "/") + "/img?url=" - for i := range channels { - if channels[i].IconURL != "" { - channels[i].IconURL = proxy + neturl.QueryEscape(channels[i].IconURL) - } - } - for i := range programs { - if programs[i].IconSrc != "" { - programs[i].IconSrc = proxy + neturl.QueryEscape(programs[i].IconSrc) - } - for j := range programs[i].Images { - if programs[i].Images[j].URL != "" { - programs[i].Images[j].URL = proxy + neturl.QueryEscape(programs[i].Images[j].URL) - } - } - } + rewriteImageURLs(baseURL, channels, lineup, programs) log.Printf("Rewrote image URLs with base %s", baseURL) } tvGuide := &guide.TVGuide{ Channels: channels, Programs: programs, + Lineup: lineup, + Source: guide.SourceFromPreferences(pref, time.Now().UTC()), } if channelFilter != nil { @@ -974,9 +960,18 @@ func filterGuideChannels(g *guide.TVGuide, allowed map[string]bool) *guide.TVGui } } + var lineup []guide.LineupPosition + for _, position := range g.Lineup { + if allowed[position.ChannelNo] { + lineup = append(lineup, position) + } + } + return &guide.TVGuide{ Channels: channels, Programs: programs, + Lineup: lineup, + Source: g.Source, } } @@ -1289,6 +1284,43 @@ func fixDeadImageURLs(programs []guide.Program) { // resolves channel logos from the tv-logo/tv-logos repo, // replacing dead Gracenote icon URLs with verified GitHub-hosted PNGs. +// rewriteImageURLs routes every image URL through the local /img proxy. +func rewriteImageURLs(baseURL string, channels []guide.Channel, lineup []guide.LineupPosition, programs []guide.Program) { + proxy := strings.TrimRight(baseURL, "/") + "/img?url=" + rewrite := func(u string) string { + if u == "" { + return "" + } + return proxy + neturl.QueryEscape(u) + } + for i := range channels { + channels[i].IconURL = rewrite(channels[i].IconURL) + } + for i := range lineup { + lineup[i].LogoURL = rewrite(lineup[i].LogoURL) + } + for i := range programs { + programs[i].IconSrc = rewrite(programs[i].IconSrc) + for j := range programs[i].Images { + programs[i].Images[j].URL = rewrite(programs[i].Images[j].URL) + } + } +} + +// propagateChannelLogos copies each station's resolved icon onto every lineup +// position carrying that station, so logo resolution stays keyed by station. +func propagateChannelLogos(channels []guide.Channel, lineup []guide.LineupPosition) { + byStation := make(map[string]string, len(channels)) + for _, ch := range channels { + byStation[ch.ID] = ch.IconURL + } + for i := range lineup { + if logo, ok := byStation[lineup[i].StationID]; ok { + lineup[i].LogoURL = logo + } + } +} + func enrichChannelIcons(client *tvlogo.Client, channels []guide.Channel) { if client == nil { return From d9de55093a1eec19705d842e966f515f5101d0d8 Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:34:12 -0400 Subject: [PATCH 5/7] Serve the provider lineup at /api/lineup.json One entry per channel position, sorted by number, with station metadata, Gracenote station filters, and the resolved logo. Provider name, type, and location come from the saved setup when it matches the lineup the guide was built from. Returns 503 with Retry-After until the first guide exists, matching /api/guide.json. --- CLAUDE.md | 4 +- README.md | 36 ++++++++ lineup_handler_test.go | 195 +++++++++++++++++++++++++++++++++++++++++ main.go | 100 +++++++++++++++++++++ 4 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 lineup_handler_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 9dc0268..1a9735e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,7 +33,7 @@ The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV l 1. `/setup` uses `web.ProviderClient` to discover Gracenote lineups by country and postal code. `appconfig.Store` persists the selected non-secret source in `config.json`; complete legacy `GN_*` settings can bootstrap it. 2. `web.Client.GetDataByTime` fetches 6-hour grid slices from the GraceNote API (`tvlistings.gracenote.com/api/grid`) — 56 slots for 14 days. A 5-second sleep separates requests. Raw JSON types live in `web/web.go`. -3. `guide.ConvertChannel` / `guide.ConvertEvent` translate the raw JSON into `guide.TVGuide` (internal canonical types). The `guide.tmpl` template renders these to XMLTV. `index.html`, `setup.html`, and `guide.tmpl` are embedded at build time via `//go:embed`. +3. `guide.ConvertChannel` / `guide.ConvertEvent` translate the raw JSON into `guide.TVGuide` (internal canonical types). `TVGuide.Channels` is deduplicated by station for XMLTV; `TVGuide.Lineup` retains every provider position (number plus station) and is served by `/api/lineup.json`. `TVGuide.Source` records the lineup the guide came from. The `guide.tmpl` template renders these to XMLTV. `index.html`, `setup.html`, and `guide.tmpl` are embedded at build time via `//go:embed`. 4. `tmdb.Client.Lookup` enriches programs (poster images, ratings, overview, year) via TMDB search API. Deduplicates by `(title, isMovie)` before hitting the API. Rate-limited to ~4 req/sec. 5. `tvlogo.Client.Resolve` replaces Gracenote channel icons with verified PNGs from `github.com/tv-logo/tv-logos`. Generates candidate URL slugs from callsign/affiliate name and HEAD-checks each (rate-limited to ~5 req/sec). 6. `fixDeadImageURLs` rewrites `zap2it.tmsimg.com` → `tmsimg.com` for broken Gracenote image URLs. @@ -43,7 +43,7 @@ The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV l | Cache | File | TTL | |---|---|---| -| Guide (in-memory + disk) | `guide_cache.json` | 4h (startup skip) / 24h (rescrape) | +| Guide (in-memory + disk) | `guide_cache.json` | 4h (startup skip) / 24h (rescrape); carries a schema version, older caches are rebuilt once | | TMDB lookups | `tmdb_cache.json` | 7 days | | TV logo HEAD checks | `tvlogo_cache.json` | persisted, no expiry | | Image proxy | `image_cache/` dir | indefinite (per-URL SHA256 key) | diff --git a/README.md b/README.md index ae97626..cb7664c 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ A saved `CONFIG_PATH` selection takes precedence over legacy `GN_*` settings. De | `POST /api/setup/provider` | Save the selected provider and queue a fresh guide | | `GET /xmlguide.xmltv` | XMLTV guide data (point your DVR here) | | `GET /api/guide.json` | Guide data as JSON | +| `GET /api/lineup.json` | Every channel position in the active provider lineup as JSON (see below) | | `GET /` | The Grid — built-in web UI | | `GET /img?url=...` | Image proxy with local cache | | `GET /api/livetv/config` | Returns `{"enabled":true/false}` — whether Jellyfin live TV is configured | @@ -125,6 +126,41 @@ A saved `CONFIG_PATH` selection takes precedence over legacy `GN_*` settings. De | `GET /api/livetv/tune?id=` | Starts a live stream for the given channel and returns an HLS URL | | `POST /api/livetv/stop` | Forwards a playback-stop notification to Jellyfin to end a live stream | +### Lineup JSON + +`/api/lineup.json` describes the provider lineup itself rather than the schedule: one entry per channel number, never collapsed, so a station carried at two numbers appears twice. It returns `503` with a `Retry-After` header until the first guide has been built. + +```json +{ + "generated": "2026-09-13T04:10:22Z", + "source": { + "providerName": "Local Over the Air Broadcast", + "providerType": "OTA", + "location": "", + "lineupId": "USA-lineupId-DEFAULT", + "headendId": "lineupId", + "postalCode": "13490", + "country": "USA", + "device": "-", + "language": "en-us" + }, + "positions": [ + { + "number": "2.1", + "stationId": "53158", + "placementId": "531580", + "callSign": "WKTVDT", + "affiliate": "NATIONAL BROADCASTING COMPANY", + "affiliateCallSign": "", + "filters": ["sports", "news"], + "logoUrl": "https://raw.githubusercontent.com/tv-logo/tv-logos/main/countries/united-states/nbc-us.png" + } + ] +} +``` + +`filters` are Gracenote's own station tags with the `filter-` prefix removed. `placementId` is Gracenote's row identifier and is not stable across scrapes; use `number` plus `stationId` to identify a position. `providerName`, `providerType`, and `location` are filled from the saved setup when it matches the lineup the guide was built from. + ## The Grid The server includes a built-in retro-styled TV guide web UI at the root URL. If no provider is configured, `/` redirects to `/setup`. Once configured, the guide auto-scrolls through your channel lineup and shows program details, posters, and metadata. Handy for a quick glance at what's on without opening your DVR app. diff --git a/lineup_handler_test.go b/lineup_handler_test.go new file mode 100644 index 0000000..5e4b17c --- /dev/null +++ b/lineup_handler_test.go @@ -0,0 +1,195 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "sort" + "testing" + "time" + + "github.com/daniel-widrick/GraceNoteScraper/appconfig" + "github.com/daniel-widrick/GraceNoteScraper/guide" +) + +func lineupTestConfig() appconfig.Config { + return appconfig.Config{ + Version: appconfig.CurrentVersion, + Gracenote: appconfig.GracenoteConfig{ + Country: "USA", PostalCode: "13490", Language: "en-us", ProviderType: "OTA", Device: "-", + LineupID: "USA-lineupId-DEFAULT", ProviderName: "Local Over the Air Broadcast", Location: "Utica", HeadendID: "lineupId", + }, + } +} + +func lineupTestGuide() *guide.TVGuide { + return &guide.TVGuide{ + Lineup: []guide.LineupPosition{ + {ChannelNo: "10", StationID: "s10", PlacementID: "s100", CallSign: "TEN", Affiliate: "Ten Net", Filters: []string{"news"}, LogoURL: "http://logo/ten.png"}, + {ChannelNo: "2.1", StationID: "s2", PlacementID: "s20", CallSign: "TWO", Affiliate: "Two Net", AffiliateCallSign: "TW"}, + {ChannelNo: "1002", StationID: "s2", PlacementID: "s299", CallSign: "TWO", Affiliate: "Two Net", AffiliateCallSign: "TW"}, + }, + Source: guide.Source{ + Country: "USA", PostalCode: "13490", HeadendID: "lineupId", LineupID: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us", + GeneratedAt: time.Date(2026, 9, 13, 4, 10, 22, 0, time.UTC), + }, + } +} + +func newLineupServer(t *testing.T, g *guide.TVGuide, save bool) http.Handler { + t.Helper() + store, err := appconfig.LoadStore(filepath.Join(t.TempDir(), "config.json")) + if err != nil { + t.Fatalf("LoadStore: %v", err) + } + if save { + if err := store.Save(lineupTestConfig()); err != nil { + t.Fatalf("Save: %v", err) + } + } + state := &GuideState{} + state.Update(g) + return handleLineupJSON(state, store) +} + +func TestLineupJSONUnavailableBeforeFirstGuide(t *testing.T) { + rec := httptest.NewRecorder() + newLineupServer(t, nil, true).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d", rec.Code) + } + if rec.Header().Get("Retry-After") != "30" { + t.Fatalf("Retry-After = %q", rec.Header().Get("Retry-After")) + } +} + +func TestLineupJSONResponse(t *testing.T) { + rec := httptest.NewRecorder() + newLineupServer(t, lineupTestGuide(), true).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body = %s", rec.Code, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type = %q", ct) + } + if rec.Header().Get("Access-Control-Allow-Origin") != "*" { + t.Error("CORS header missing") + } + + var got APILineup + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + if got.Generated != "2026-09-13T04:10:22Z" { + t.Errorf("generated = %q", got.Generated) + } + wantSource := APILineupSource{ + ProviderName: "Local Over the Air Broadcast", ProviderType: "OTA", Location: "Utica", + LineupID: "USA-lineupId-DEFAULT", HeadendID: "lineupId", PostalCode: "13490", Country: "USA", Device: "-", Language: "en-us", + } + if got.Source != wantSource { + t.Errorf("source = %+v\nwant %+v", got.Source, wantSource) + } + + var numbers []string + for _, p := range got.Positions { + numbers = append(numbers, p.Number) + } + if !reflect.DeepEqual(numbers, []string{"2.1", "10", "1002"}) { + t.Errorf("positions not sorted by number: %v", numbers) + } + ten := got.Positions[1] + want := APILineupPosition{Number: "10", StationID: "s10", PlacementID: "s100", CallSign: "TEN", Affiliate: "Ten Net", Filters: []string{"news"}, LogoURL: "http://logo/ten.png"} + if !reflect.DeepEqual(ten, want) { + t.Errorf("position = %+v\nwant %+v", ten, want) + } + if got.Positions[0].StationID != "s2" || got.Positions[2].StationID != "s2" { + t.Error("the same station at two numbers must appear twice") + } +} + +func TestLineupJSONKeySetIsStable(t *testing.T) { + rec := httptest.NewRecorder() + newLineupServer(t, lineupTestGuide(), true).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + + var raw map[string]json.RawMessage + if err := json.Unmarshal(rec.Body.Bytes(), &raw); err != nil { + t.Fatal(err) + } + if keys := sortedKeys(raw); !reflect.DeepEqual(keys, []string{"generated", "positions", "source"}) { + t.Errorf("top-level keys = %v", keys) + } + var source map[string]json.RawMessage + if err := json.Unmarshal(raw["source"], &source); err != nil { + t.Fatal(err) + } + if keys := sortedKeys(source); !reflect.DeepEqual(keys, []string{"country", "device", "headendId", "language", "lineupId", "location", "postalCode", "providerName", "providerType"}) { + t.Errorf("source keys = %v", keys) + } + var positions []map[string]json.RawMessage + if err := json.Unmarshal(raw["positions"], &positions); err != nil { + t.Fatal(err) + } + // Position with filters carries the filters key; one without omits it. + withFilters := sortedKeys(positions[1]) + if !reflect.DeepEqual(withFilters, []string{"affiliate", "affiliateCallSign", "callSign", "filters", "logoUrl", "number", "placementId", "stationId"}) { + t.Errorf("position keys = %v", withFilters) + } + if _, ok := positions[0]["filters"]; ok { + t.Error("filters should be omitted when empty") + } + if _, ok := positions[0]["logoUrl"]; !ok { + t.Error("logoUrl must always be present, even when empty") + } +} + +func TestLineupJSONSourceNamingRequiresMatchingConfig(t *testing.T) { + // No saved configuration at all. + rec := httptest.NewRecorder() + newLineupServer(t, lineupTestGuide(), false).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + var got APILineup + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Source.ProviderName != "" || got.Source.ProviderType != "" || got.Source.Location != "" { + t.Errorf("provider naming should be blank without config: %+v", got.Source) + } + if got.Source.LineupID != "USA-lineupId-DEFAULT" { + t.Errorf("guide source fields must still be present: %+v", got.Source) + } + + // Saved configuration describes a different lineup than the guide. + g := lineupTestGuide() + g.Source.LineupID = "USA-OTHER-DEFAULT" + rec = httptest.NewRecorder() + newLineupServer(t, g, true).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/lineup.json", nil)) + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got.Source.ProviderName != "" { + t.Errorf("provider name leaked across lineups: %+v", got.Source) + } + if got.Source.LineupID != "USA-OTHER-DEFAULT" { + t.Errorf("lineup id should come from the guide: %+v", got.Source) + } +} + +func TestLineupJSONDoesNotMutateGuideOrder(t *testing.T) { + g := lineupTestGuide() + before := append([]guide.LineupPosition(nil), g.Lineup...) + lineupToJSON(g, lineupTestConfig(), true) + if !reflect.DeepEqual(before, g.Lineup) { + t.Fatal("lineupToJSON must sort a copy, not the live guide") + } +} + +func sortedKeys(m map[string]json.RawMessage) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/main.go b/main.go index 8f9424f..0197102 100644 --- a/main.go +++ b/main.go @@ -87,6 +87,37 @@ type APIProgram struct { Description string `json:"description,omitempty"` } +// APILineup is the response shape of /api/lineup.json: every provider +// position for the active lineup, plus the source it came from. +type APILineup struct { + Generated string `json:"generated"` + Source APILineupSource `json:"source"` + Positions []APILineupPosition `json:"positions"` +} + +type APILineupSource struct { + ProviderName string `json:"providerName"` + ProviderType string `json:"providerType"` + Location string `json:"location"` + LineupID string `json:"lineupId"` + HeadendID string `json:"headendId"` + PostalCode string `json:"postalCode"` + Country string `json:"country"` + Device string `json:"device"` + Language string `json:"language"` +} + +type APILineupPosition struct { + Number string `json:"number"` + StationID string `json:"stationId"` + PlacementID string `json:"placementId"` + CallSign string `json:"callSign"` + Affiliate string `json:"affiliate"` + AffiliateCallSign string `json:"affiliateCallSign"` + Filters []string `json:"filters,omitempty"` + LogoURL string `json:"logoUrl"` +} + // ---------- Conversion ---------- // guideToJSON converts a TVGuide into the simplified JSON API format. @@ -154,6 +185,56 @@ func guideToJSON(g *guide.TVGuide) APIGuide { } } +// lineupToJSON converts a guide's lineup into the API shape. Provider naming +// comes from the saved configuration only when it describes the same source +// the guide was built from. +func lineupToJSON(g *guide.TVGuide, config appconfig.Config, configured bool) APILineup { + positions := make([]guide.LineupPosition, len(g.Lineup)) + copy(positions, g.Lineup) + guide.SortLineup(positions) + + out := APILineup{ + Generated: g.Source.GeneratedAt.UTC().Format(time.RFC3339), + Source: APILineupSource{ + LineupID: g.Source.LineupID, + HeadendID: g.Source.HeadendID, + PostalCode: g.Source.PostalCode, + Country: g.Source.Country, + Device: g.Source.Device, + Language: g.Source.Language, + }, + Positions: make([]APILineupPosition, 0, len(positions)), + } + if g.Source.GeneratedAt.IsZero() { + out.Generated = time.Now().UTC().Format(time.RFC3339) + } + if configured && sourceMatchesConfig(g.Source, config) { + out.Source.ProviderName = config.Gracenote.ProviderName + out.Source.ProviderType = config.Gracenote.ProviderType + out.Source.Location = config.Gracenote.Location + } + for _, p := range positions { + out.Positions = append(out.Positions, APILineupPosition{ + Number: p.ChannelNo, + StationID: p.StationID, + PlacementID: p.PlacementID, + CallSign: p.CallSign, + Affiliate: p.Affiliate, + AffiliateCallSign: p.AffiliateCallSign, + Filters: p.Filters, + LogoURL: p.LogoURL, + }) + } + return out +} + +// sourceMatchesConfig reports whether a guide was built from the configured lineup. +func sourceMatchesConfig(src guide.Source, config appconfig.Config) bool { + p := config.Preferences() + return src.LineupID == p.LineupId && src.HeadendID == p.Headend && src.PostalCode == p.ZipCode && + src.Country == p.Country && src.Device == p.Device && src.Language == p.Language +} + // xmltvTimeToISO converts "20250225200000 +0000" → "2025-02-25T20:00:00Z" func xmltvTimeToISO(xmltvTime string) string { xmltvTime = strings.TrimSpace(xmltvTime) @@ -607,6 +688,24 @@ func handleGuideJSON(state *GuideState) http.HandlerFunc { } } +func handleLineupJSON(state *GuideState, store *appconfig.Store) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + g := state.Get() + if g == nil { + w.Header().Set("Retry-After", "30") + http.Error(w, "Guide is being generated", http.StatusServiceUnavailable) + return + } + + config, configured, _ := store.Get() + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Access-Control-Allow-Origin", "*") + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + enc.Encode(lineupToJSON(g, config, configured)) + } +} + // ---------- Image proxy ---------- const imageCacheDir = "image_cache" @@ -1115,6 +1214,7 @@ func main() { mux.HandleFunc("/api/setup/status", setupHandlers.handleScrapeStatus) mux.HandleFunc("/xmlguide.xmltv", handleXMLTV(state)) mux.HandleFunc("/api/guide.json", handleGuideJSON(state)) + mux.HandleFunc("/api/lineup.json", handleLineupJSON(state, configStore)) mux.HandleFunc("/img", handleImage) mux.HandleFunc("/api/livetv/config", handleLiveTVConfig(jellyfinURL, jellyfinAPIKey)) if jellyfinURL != "" && jellyfinAPIKey != "" { From 5b579149d16b8ea5c73e1a55046c373e9273deea Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:37:57 -0400 Subject: [PATCH 6/7] Move the grid download loop into an importable scrape package scrape.Fetch(ctx, prefs, opts) downloads every six-hour slot for a lineup and assembles a guide.TVGuide: stations deduplicated in first-seen order, every lineup position retained, programs deduplicated. It does no enrichment, file I/O, or environment reads. main.runScrape now calls it, mapping progress to the setup status and cancelling the context when the active lineup changes mid-download. Behavior change: when every slot fails, the scrape is now an error (scrape.ErrNoData) instead of writing an empty guide, so the previous guide stays live and the 15-minute retry applies. Channel order in the XMLTV output is now first-seen rather than map order. --- CLAUDE.md | 4 +- README.md | 28 ++++- main.go | 128 +++++++++++--------- run_scrape_test.go | 139 ++++++++++++++++++++++ scrape/scrape.go | 188 +++++++++++++++++++++++++++++ scrape/scrape_test.go | 271 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 696 insertions(+), 62 deletions(-) create mode 100644 run_scrape_test.go create mode 100644 scrape/scrape.go create mode 100644 scrape/scrape_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 1a9735e..0198e9a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,12 +27,12 @@ Run `go test ./...` for the setup, configuration, and provider-client tests. ## Architecture -The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. Runtime orchestration lives in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`. +The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. The grid download loop and guide assembly live in `scrape/` (`scrape.Fetch`), which is importable by other programs and does no enrichment, file I/O, or env reads. Enrichment, persistence, and runtime orchestration live in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`. **Data flow:** 1. `/setup` uses `web.ProviderClient` to discover Gracenote lineups by country and postal code. `appconfig.Store` persists the selected non-secret source in `config.json`; complete legacy `GN_*` settings can bootstrap it. -2. `web.Client.GetDataByTime` fetches 6-hour grid slices from the GraceNote API (`tvlistings.gracenote.com/api/grid`) — 56 slots for 14 days. A 5-second sleep separates requests. Raw JSON types live in `web/web.go`. +2. `scrape.Fetch` drives `web.Client.GetDataByTimeContext` over 6-hour grid slices from the GraceNote API (`tvlistings.gracenote.com/api/grid`) — 56 slots for 14 days. A 5-second sleep separates requests. Failed slots are skipped; if every slot fails the scrape returns `scrape.ErrNoData` and the previous guide stays live. Raw JSON types live in `web/web.go`. 3. `guide.ConvertChannel` / `guide.ConvertEvent` translate the raw JSON into `guide.TVGuide` (internal canonical types). `TVGuide.Channels` is deduplicated by station for XMLTV; `TVGuide.Lineup` retains every provider position (number plus station) and is served by `/api/lineup.json`. `TVGuide.Source` records the lineup the guide came from. The `guide.tmpl` template renders these to XMLTV. `index.html`, `setup.html`, and `guide.tmpl` are embedded at build time via `//go:embed`. 4. `tmdb.Client.Lookup` enriches programs (poster images, ratings, overview, year) via TMDB search API. Deduplicates by `(title, isMovie)` before hitting the API. Rate-limited to ~4 req/sec. 5. `tvlogo.Client.Resolve` replaces Gracenote channel icons with verified PNGs from `github.com/tv-logo/tv-logos`. Generates candidate URL slugs from callsign/affiliate name and HEAD-checks each (rate-limited to ~5 req/sec). diff --git a/README.md b/README.md index cb7664c..7242a99 100644 --- a/README.md +++ b/README.md @@ -167,11 +167,37 @@ The server includes a built-in retro-styled TV guide web UI at the root URL. If ![The Grid](https://gist.githubusercontent.com/daniel-widrick/2c52c4d023ffe75d163b4eff58263c77/raw/demo.gif) +## Using as a Library + +The grid download and conversion live in the `scrape` package, so another Go program can fetch a lineup's listings without running the server: + +```go +import ( + "context" + "fmt" + + "github.com/daniel-widrick/GraceNoteScraper/scrape" + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +func main() { + prefs := web.Preferences{Country: "USA", ZipCode: "13490", Headend: "lineupId", LineupId: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us"} + g, err := scrape.Fetch(context.Background(), prefs, scrape.Options{Days: 1}) + if err != nil { + panic(err) + } + fmt.Printf("%d stations, %d lineup positions, %d programs\n", len(g.Channels), len(g.Lineup), len(g.Programs)) +} +``` + +`Fetch` returns a `guide.TVGuide`: `Channels` deduplicated by station, `Lineup` with every position, `Programs`, and `Source`. It pauses five seconds between grid requests by default and skips slots that fail; it returns `scrape.ErrNoData` only when every slot failed. Logo and TMDB enrichment are not part of the package. Lineup discovery by postal code is available through `web.NewProviderClient().FindProviders`. + ## Project Structure ``` appconfig/ Persisted non-secret provider configuration -main.go Entry point, HTTP server, scraper, image proxy +main.go Entry point, HTTP server, enrichment, image proxy +scrape/ Grid download loop and guide assembly (importable) guide/ GraceNote data types and XMLTV conversion web/ HTTP client for GraceNote API tmdb/ TMDB client and cache diff --git a/main.go b/main.go index 0197102..d0643ad 100644 --- a/main.go +++ b/main.go @@ -21,12 +21,14 @@ import ( "sort" "strings" "sync" + "sync/atomic" "syscall" "text/template" "time" "github.com/daniel-widrick/GraceNoteScraper/appconfig" "github.com/daniel-widrick/GraceNoteScraper/guide" + "github.com/daniel-widrick/GraceNoteScraper/scrape" "github.com/daniel-widrick/GraceNoteScraper/tmdb" "github.com/daniel-widrick/GraceNoteScraper/tvlogo" "github.com/daniel-widrick/GraceNoteScraper/util" @@ -276,70 +278,32 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch } } } - client := web.NewClient(pref) - now := time.Now().UTC() - midnight := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) - endTime := midnight.Add(14 * 24 * time.Hour) - - channelMap := make(map[string]guide.Channel) - lineupMap := make(map[string]guide.LineupPosition) - eventMap := make(map[string]bool) - var programs []guide.Program - - totalSlots := int(endTime.Sub(midnight) / (6 * time.Hour)) - slot := 0 - for t := midnight; t.Before(endTime); t = t.Add(6 * time.Hour) { - if sourceCurrent != nil && !sourceCurrent() { - return nil, errScrapeSourceChanged - } - slot++ - ts := t.Unix() - report(scrapeProgressUpdate{Stage: "gracenote", Message: fmt.Sprintf("Downloading guide data (%d of %d)", slot, totalSlots), Completed: slot - 1, Total: totalSlots, Channels: len(channelMap), Programs: len(programs)}) - log.Printf("Fetching grid %d/%d for time=%d (%s)", slot, totalSlots, ts, t.Format(time.RFC3339)) - - grid, err := client.GetDataByTime(ts) - if err != nil { - log.Printf("Error fetching grid at %d: %v", ts, err) - continue - } - - for _, ch := range grid.Channels { - if _, exists := channelMap[ch.ChannelID]; !exists { - channelMap[ch.ChannelID] = guide.ConvertChannel(ch) - } - position := guide.ConvertLineupPosition(ch) - if _, exists := lineupMap[position.Key()]; !exists { - lineupMap[position.Key()] = position - } - - for _, ev := range ch.Events { - dedupKey := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime - if eventMap[dedupKey] { - continue - } - eventMap[dedupKey] = true - programs = append(programs, guide.ConvertEvent(ev, ch.ChannelID, pref.Language, pref.Country)) - } - } - - log.Printf("Channels so far: %d, Events so far: %d", len(channelMap), len(programs)) - report(scrapeProgressUpdate{Stage: "gracenote", Message: fmt.Sprintf("Downloaded guide data (%d of %d)", slot, totalSlots), Completed: slot, Total: totalSlots, Channels: len(channelMap), Programs: len(programs)}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sourceChanged := watchSourceChange(ctx, cancel, sourceCurrent) - if t.Add(6 * time.Hour).Before(endTime) { - time.Sleep(5 * time.Second) + opts := newScrapeOptions(pref) + opts.Progress = func(p scrape.Progress) { + message := fmt.Sprintf("Downloading guide data (%d of %d)", p.Slot, p.TotalSlots) + if p.Phase == scrape.PhaseFetched { + message = fmt.Sprintf("Downloaded guide data (%d of %d)", p.Slot, p.TotalSlots) } + report(scrapeProgressUpdate{Stage: "gracenote", Message: message, Completed: p.Completed, Total: p.TotalSlots, Channels: p.Channels, Programs: p.Programs}) } - - var channels []guide.Channel - for _, ch := range channelMap { - channels = append(channels, ch) + fetched, err := scrape.Fetch(ctx, pref, opts) + if err != nil { + if sourceChanged() { + return nil, errScrapeSourceChanged + } + return nil, err } - lineup := make([]guide.LineupPosition, 0, len(lineupMap)) - for _, position := range lineupMap { - lineup = append(lineup, position) + if sourceChanged() { + return nil, errScrapeSourceChanged } - guide.SortLineup(lineup) + channels := fetched.Channels + lineup := fetched.Lineup + programs := fetched.Programs logoClient := tvlogo.NewClient(pref.Country, "tvlogo_cache.json") if logoClient != nil { @@ -363,7 +327,7 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch Channels: channels, Programs: programs, Lineup: lineup, - Source: guide.SourceFromPreferences(pref, time.Now().UTC()), + Source: fetched.Source, } if channelFilter != nil { @@ -399,6 +363,52 @@ func renderXMLTV(w io.Writer, tvGuide *guide.TVGuide) error { return nil } +// newScrapeOptions builds the production scrape configuration. Tests replace +// it to inject a fake grid fetcher and remove the inter-slot delay. +var newScrapeOptions = func(pref web.Preferences) scrape.Options { + return scrape.Options{Fetcher: web.NewClient(pref)} +} + +// watchSourceChange cancels ctx as soon as sourceCurrent reports false, so a +// lineup change during the grid download stops the scrape promptly. The +// returned func reports whether a change has been observed, re-checking live +// so a change between the last tick and the call is not missed. +func watchSourceChange(ctx context.Context, cancel context.CancelFunc, sourceCurrent func() bool) func() bool { + if sourceCurrent == nil { + return func() bool { return false } + } + var changed atomic.Bool + check := func() bool { + if changed.Load() { + return true + } + if !sourceCurrent() { + changed.Store(true) + cancel() + return true + } + return false + } + if check() { + return check + } + go func() { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if check() { + return + } + } + } + }() + return check +} + func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs)) diff --git a/run_scrape_test.go b/run_scrape_test.go new file mode 100644 index 0000000..75c1a8e --- /dev/null +++ b/run_scrape_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "context" + "errors" + "io" + "log" + "os" + "sync/atomic" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/guide" + "github.com/daniel-widrick/GraceNoteScraper/scrape" + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +type countingFetcher struct { + calls atomic.Int32 + grid *web.GridResponse + err error +} + +func (f *countingFetcher) GetDataByTimeContext(ctx context.Context, t int64) (*web.GridResponse, error) { + f.calls.Add(1) + if err := ctx.Err(); err != nil { + return nil, err + } + if f.err != nil { + return nil, f.err + } + return f.grid, nil +} + +// useFakeScrape swaps the production scrape options for a fake fetcher with +// one day of slots and no delay. The preferences use a country tvlogo does not +// support so no logo lookups reach the network. +func useFakeScrape(t *testing.T, f *countingFetcher) web.Preferences { + t.Helper() + t.Chdir(t.TempDir()) + original := newScrapeOptions + newScrapeOptions = func(web.Preferences) scrape.Options { + return scrape.Options{Fetcher: f, Days: 1, SlotDelay: scrape.NoDelay, Logger: log.New(io.Discard, "", 0)} + } + t.Cleanup(func() { newScrapeOptions = original }) + return web.Preferences{Country: "ZZZ", ZipCode: "00000", Headend: "h", LineupId: "L", Device: "-", Language: "en-us"} +} + +func TestRunScrapeBuildsLineupAndPersists(t *testing.T) { + f := &countingFetcher{grid: &web.GridResponse{Channels: []web.JSONChannel{ + {ChannelID: "s1", ID: "s10", ChannelNo: "2.1", CallSign: "AAA", AffiliateName: "A Net", StationFilters: []string{"filter-news"}, + Events: []web.JSONEvent{{StartTime: "2026-09-13T00:00:00Z", EndTime: "2026-09-13T01:00:00Z", Duration: "60", SeriesID: "SH1", Program: web.JSONProgram{ID: "EP1", Title: "one"}}}}, + {ChannelID: "s1", ID: "s199", ChannelNo: "1002", CallSign: "AAA", AffiliateName: "A Net"}, + }}} + pref := useFakeScrape(t, f) + + var persisted *guide.TVGuide + persister := func(g *guide.TVGuide) (bool, error) { + persisted = g + return true, persistGuideFiles(g, "fp") + } + var updates []scrapeProgressUpdate + got, err := runScrape(pref, nil, "http://base:8080", nil, "fp", func() bool { return true }, persister, func(u scrapeProgressUpdate) { updates = append(updates, u) }) + if err != nil { + t.Fatalf("runScrape: %v", err) + } + if f.calls.Load() != 4 { + t.Errorf("grid fetches = %d, want 4 for one day", f.calls.Load()) + } + if persisted != got { + t.Error("persister should receive the returned guide") + } + if len(got.Channels) != 1 || len(got.Lineup) != 2 || len(got.Programs) != 1 { + t.Fatalf("channels/lineup/programs = %d/%d/%d", len(got.Channels), len(got.Lineup), len(got.Programs)) + } + if got.Lineup[0].LogoURL != "" { + t.Errorf("no logo source available, got %q", got.Lineup[0].LogoURL) + } + if got.Source.LineupID != "L" { + t.Errorf("source = %+v", got.Source) + } + if _, err := os.Stat("xmlguide.xmltv"); err != nil { + t.Errorf("xmlguide.xmltv not written: %v", err) + } + stages := map[string]bool{} + for _, u := range updates { + stages[u.Stage] = true + } + for _, want := range []string{"gracenote", "logos", "saving"} { + if !stages[want] { + t.Errorf("missing progress stage %q (got %v)", want, stages) + } + } + if stages["tmdb"] { + t.Error("no TMDB client configured, so no tmdb stage should be reported") + } + var gracenote []scrapeProgressUpdate + for _, u := range updates { + if u.Stage == "gracenote" { + gracenote = append(gracenote, u) + } + } + if len(gracenote) != 8 || gracenote[0].Total != 4 || gracenote[7].Completed != 4 { + t.Errorf("gracenote progress = %d updates, first %+v, last %+v", len(gracenote), gracenote[0], gracenote[len(gracenote)-1]) + } +} + +func TestRunScrapeReportsSourceChange(t *testing.T) { + f := &countingFetcher{grid: &web.GridResponse{Channels: []web.JSONChannel{{ChannelID: "s1", ChannelNo: "2"}}}} + pref := useFakeScrape(t, f) + + var seen atomic.Int32 + sourceCurrent := func() bool { + // Current for the initial check, changed afterwards. + return seen.Add(1) == 1 + } + persisterCalled := false + _, err := runScrape(pref, nil, "", nil, "fp", sourceCurrent, func(*guide.TVGuide) (bool, error) { + persisterCalled = true + return true, nil + }) + if !errors.Is(err, errScrapeSourceChanged) { + t.Fatalf("err = %v, want errScrapeSourceChanged", err) + } + if persisterCalled { + t.Error("persister must not run after a source change") + } +} + +func TestRunScrapeSurfacesTotalFailure(t *testing.T) { + f := &countingFetcher{err: errors.New("gracenote down")} + pref := useFakeScrape(t, f) + _, err := runScrape(pref, nil, "", nil, "fp", nil, func(*guide.TVGuide) (bool, error) { + t.Fatal("persister must not run with no data") + return false, nil + }) + if !errors.Is(err, scrape.ErrNoData) { + t.Fatalf("err = %v, want ErrNoData", err) + } +} diff --git a/scrape/scrape.go b/scrape/scrape.go new file mode 100644 index 0000000..0ee481b --- /dev/null +++ b/scrape/scrape.go @@ -0,0 +1,188 @@ +// Package scrape fetches a Gracenote lineup's listings and converts them into +// a guide.TVGuide. It performs no enrichment, file I/O, or environment reads, +// so it can be imported by other programs. +package scrape + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/daniel-widrick/GraceNoteScraper/guide" + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +const ( + // SlotDuration is the window Gracenote serves per grid request. + SlotDuration = 6 * time.Hour + // DefaultDays is how far ahead a scrape reaches when Options.Days is zero. + DefaultDays = 14 + // DefaultSlotDelay is the pause between grid requests when + // Options.SlotDelay is zero. Gracenote is an unofficial API; be polite. + DefaultSlotDelay = 5 * time.Second +) + +// ErrNoData is returned when every grid slot failed, so there is nothing to +// build a guide from. A partial failure is not an error; failed slots are +// skipped and reported through Options.Progress. +var ErrNoData = errors.New("scrape: no grid slot returned data") + +// GridFetcher retrieves one six-hour grid. *web.Client satisfies it. +type GridFetcher interface { + GetDataByTimeContext(ctx context.Context, t int64) (*web.GridResponse, error) +} + +// Phase says whether a Progress report precedes or follows a slot fetch. +type Phase int + +const ( + // PhaseFetching is reported just before a slot is requested. + PhaseFetching Phase = iota + // PhaseFetched is reported after a slot succeeded or failed. + PhaseFetched +) + +// Progress describes one step of a scrape. +type Progress struct { + Phase Phase + Slot int // 1-based index of the slot being reported + TotalSlots int // total slots for this scrape + SlotTime time.Time // start of the six-hour window + Completed int // slots finished so far, successful or not + Channels int // distinct stations collected so far + Programs int // programs collected so far + Err error // non-nil on PhaseFetched when the slot failed and was skipped +} + +// Options tunes a Fetch. The zero value is the production configuration. +type Options struct { + Days int // days of listings to fetch; default DefaultDays + SlotDelay time.Duration // pause between slots; default DefaultSlotDelay (tests use a negative value for none) + Fetcher GridFetcher // grid source; default web.NewClient(prefs) + Now func() time.Time // clock; default time.Now + Progress func(Progress) // optional per-slot callback + Logger *log.Logger // default log.Default() +} + +// NoDelay is a SlotDelay value that disables the inter-slot pause. +const NoDelay = -1 + +func (o Options) withDefaults(prefs web.Preferences) Options { + if o.Days <= 0 { + o.Days = DefaultDays + } + if o.SlotDelay == 0 { + o.SlotDelay = DefaultSlotDelay + } + if o.Fetcher == nil { + o.Fetcher = web.NewClient(prefs) + } + if o.Now == nil { + o.Now = time.Now + } + if o.Logger == nil { + o.Logger = log.Default() + } + return o +} + +// Slots returns the six-hour window start times a Fetch will request for the +// given clock and day count, beginning at UTC midnight of the current day. +func Slots(now time.Time, days int) []time.Time { + if days <= 0 { + days = DefaultDays + } + now = now.UTC() + start := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) + end := start.Add(time.Duration(days) * 24 * time.Hour) + slots := make([]time.Time, 0, days*int(24*time.Hour/SlotDuration)) + for t := start; t.Before(end); t = t.Add(SlotDuration) { + slots = append(slots, t) + } + return slots +} + +// Fetch downloads every grid slot for prefs and returns the assembled guide. +// Channels are deduplicated by station in first-seen order, Lineup keeps every +// position, and Programs are deduplicated by station, start, and end. +func Fetch(ctx context.Context, prefs web.Preferences, opts Options) (*guide.TVGuide, error) { + opts = opts.withDefaults(prefs) + slots := Slots(opts.Now(), opts.Days) + + channelIndex := make(map[string]int) + lineupIndex := make(map[string]struct{}) + eventIndex := make(map[string]struct{}) + var channels []guide.Channel + var lineup []guide.LineupPosition + var programs []guide.Program + succeeded := 0 + + report := func(p Progress) { + if opts.Progress != nil { + opts.Progress(p) + } + } + + for i, slotTime := range slots { + if err := ctx.Err(); err != nil { + return nil, err + } + slot := i + 1 + report(Progress{Phase: PhaseFetching, Slot: slot, TotalSlots: len(slots), SlotTime: slotTime, Completed: i, Channels: len(channels), Programs: len(programs)}) + opts.Logger.Printf("Fetching grid %d/%d for time=%d (%s)", slot, len(slots), slotTime.Unix(), slotTime.Format(time.RFC3339)) + + grid, err := opts.Fetcher.GetDataByTimeContext(ctx, slotTime.Unix()) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + opts.Logger.Printf("Error fetching grid at %d: %v", slotTime.Unix(), err) + report(Progress{Phase: PhaseFetched, Slot: slot, TotalSlots: len(slots), SlotTime: slotTime, Completed: slot, Channels: len(channels), Programs: len(programs), Err: err}) + } else { + succeeded++ + for _, ch := range grid.Channels { + if _, seen := channelIndex[ch.ChannelID]; !seen { + channelIndex[ch.ChannelID] = len(channels) + channels = append(channels, guide.ConvertChannel(ch)) + } + position := guide.ConvertLineupPosition(ch) + if _, seen := lineupIndex[position.Key()]; !seen { + lineupIndex[position.Key()] = struct{}{} + lineup = append(lineup, position) + } + for _, ev := range ch.Events { + key := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime + if _, seen := eventIndex[key]; seen { + continue + } + eventIndex[key] = struct{}{} + programs = append(programs, guide.ConvertEvent(ev, ch.ChannelID, prefs.Language, prefs.Country)) + } + } + opts.Logger.Printf("Channels so far: %d, Events so far: %d", len(channels), len(programs)) + report(Progress{Phase: PhaseFetched, Slot: slot, TotalSlots: len(slots), SlotTime: slotTime, Completed: slot, Channels: len(channels), Programs: len(programs)}) + } + + if slot < len(slots) && opts.SlotDelay > 0 { + select { + case <-time.After(opts.SlotDelay): + case <-ctx.Done(): + return nil, ctx.Err() + } + } + } + + if succeeded == 0 { + return nil, fmt.Errorf("%w (%d slots attempted)", ErrNoData, len(slots)) + } + + guide.SortLineup(lineup) + return &guide.TVGuide{ + Channels: channels, + Programs: programs, + Lineup: lineup, + Source: guide.SourceFromPreferences(prefs, opts.Now().UTC()), + }, nil +} diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go new file mode 100644 index 0000000..ab8c207 --- /dev/null +++ b/scrape/scrape_test.go @@ -0,0 +1,271 @@ +package scrape + +import ( + "context" + "errors" + "io" + "log" + "sync" + "testing" + "time" + + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +var fixedNow = time.Date(2026, 9, 13, 15, 42, 0, 0, time.UTC) + +func testPrefs() web.Preferences { + return web.Preferences{Country: "USA", ZipCode: "13490", Headend: "lineupId", LineupId: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us"} +} + +// fakeFetcher serves canned grids keyed by slot time and records every call. +type fakeFetcher struct { + mu sync.Mutex + grids map[int64]*web.GridResponse + errs map[int64]error + always *web.GridResponse + calls []int64 + block chan struct{} // when set, the first call blocks until closed +} + +func (f *fakeFetcher) GetDataByTimeContext(ctx context.Context, t int64) (*web.GridResponse, error) { + f.mu.Lock() + f.calls = append(f.calls, t) + block := f.block + f.block = nil + f.mu.Unlock() + if block != nil { + select { + case <-block: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if err, ok := f.errs[t]; ok { + return nil, err + } + if g, ok := f.grids[t]; ok { + return g, nil + } + if f.always != nil { + return f.always, nil + } + return &web.GridResponse{}, nil +} + +func channel(station, number string, events ...web.JSONEvent) web.JSONChannel { + return web.JSONChannel{ChannelID: station, ID: station + "0", ChannelNo: number, CallSign: "C" + station, AffiliateName: "Net " + station, Events: events} +} + +func event(start, end, title string) web.JSONEvent { + return web.JSONEvent{StartTime: start, EndTime: end, Duration: "60", SeriesID: "SH1", Program: web.JSONProgram{ID: "EP1", Title: title}} +} + +func quietOptions(f *fakeFetcher) Options { + return Options{Days: 1, SlotDelay: NoDelay, Fetcher: f, Now: func() time.Time { return fixedNow }, Logger: log.New(io.Discard, "", 0)} +} + +func TestSlotsAreMidnightAlignedSixHourWindows(t *testing.T) { + slots := Slots(fixedNow, 2) + if len(slots) != 8 { + t.Fatalf("slots = %d, want 8", len(slots)) + } + if !slots[0].Equal(time.Date(2026, 9, 13, 0, 0, 0, 0, time.UTC)) { + t.Errorf("first slot = %s", slots[0]) + } + for i := 1; i < len(slots); i++ { + if slots[i].Sub(slots[i-1]) != SlotDuration { + t.Errorf("slot %d not 6h after previous", i) + } + } + if len(Slots(fixedNow, 0)) != DefaultDays*4 { + t.Errorf("zero days should default to %d days", DefaultDays) + } +} + +func TestFetchRequestsEverySlotAndAssemblesGuide(t *testing.T) { + f := &fakeFetcher{always: &web.GridResponse{Channels: []web.JSONChannel{ + channel("s1", "2.1", event("2026-09-13T00:00:00Z", "2026-09-13T01:00:00Z", "a")), + channel("s2", "5", event("2026-09-13T00:00:00Z", "2026-09-13T02:00:00Z", "b")), + channel("s1", "1002"), // same station at a second number + }}} + opts := quietOptions(f) + opts.Days = 2 + + g, err := Fetch(context.Background(), testPrefs(), opts) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if len(f.calls) != 8 { + t.Fatalf("calls = %d, want 8", len(f.calls)) + } + if f.calls[0] != time.Date(2026, 9, 13, 0, 0, 0, 0, time.UTC).Unix() { + t.Errorf("first request time = %d", f.calls[0]) + } + if len(g.Channels) != 2 { + t.Errorf("channels = %d, want 2 (deduplicated by station)", len(g.Channels)) + } + if g.Channels[0].ID != "s1" || g.Channels[1].ID != "s2" { + t.Errorf("channels should keep first-seen order: %v %v", g.Channels[0].ID, g.Channels[1].ID) + } + if len(g.Lineup) != 3 { + t.Fatalf("lineup = %d, want 3 (never collapsed)", len(g.Lineup)) + } + if g.Lineup[0].ChannelNo != "2.1" || g.Lineup[1].ChannelNo != "5" || g.Lineup[2].ChannelNo != "1002" { + t.Errorf("lineup order = %s %s %s", g.Lineup[0].ChannelNo, g.Lineup[1].ChannelNo, g.Lineup[2].ChannelNo) + } + if len(g.Programs) != 2 { + t.Errorf("programs = %d, want 2 (same events across 8 slots deduplicated)", len(g.Programs)) + } + if g.Source.LineupID != "USA-lineupId-DEFAULT" || g.Source.PostalCode != "13490" || !g.Source.GeneratedAt.Equal(fixedNow) { + t.Errorf("source = %+v", g.Source) + } +} + +func TestFetchKeepsDistinctEventsAcrossSlots(t *testing.T) { + slots := Slots(fixedNow, 1) + f := &fakeFetcher{grids: map[int64]*web.GridResponse{ + slots[0].Unix(): {Channels: []web.JSONChannel{channel("s1", "2", event("2026-09-13T00:00:00Z", "2026-09-13T06:00:00Z", "morning"))}}, + slots[1].Unix(): {Channels: []web.JSONChannel{channel("s1", "2", event("2026-09-13T06:00:00Z", "2026-09-13T12:00:00Z", "midday"))}}, + }} + g, err := Fetch(context.Background(), testPrefs(), quietOptions(f)) + if err != nil { + t.Fatal(err) + } + if len(g.Programs) != 2 { + t.Fatalf("programs = %d, want 2", len(g.Programs)) + } +} + +func TestFetchSkipsFailedSlotAndReportsIt(t *testing.T) { + slots := Slots(fixedNow, 1) + boom := errors.New("upstream timeout") + f := &fakeFetcher{ + always: &web.GridResponse{Channels: []web.JSONChannel{channel("s1", "2")}}, + errs: map[int64]error{slots[1].Unix(): boom}, + } + var reports []Progress + opts := quietOptions(f) + opts.Progress = func(p Progress) { reports = append(reports, p) } + + g, err := Fetch(context.Background(), testPrefs(), opts) + if err != nil { + t.Fatalf("partial failure must not be an error: %v", err) + } + if len(g.Channels) != 1 { + t.Fatalf("channels = %d", len(g.Channels)) + } + if len(reports) != 8 { + t.Fatalf("progress reports = %d, want 2 per slot", len(reports)) + } + failed := reports[3] // slot 2, PhaseFetched + if failed.Phase != PhaseFetched || failed.Slot != 2 || !errors.Is(failed.Err, boom) { + t.Errorf("failed slot report = %+v", failed) + } + for i, r := range reports { + if r.TotalSlots != 4 { + t.Errorf("report %d TotalSlots = %d", i, r.TotalSlots) + } + if i > 0 && r.Completed < reports[i-1].Completed { + t.Errorf("Completed went backwards at report %d", i) + } + } + last := reports[len(reports)-1] + if last.Completed != 4 || last.Slot != 4 || last.Phase != PhaseFetched { + t.Errorf("last report = %+v", last) + } +} + +func TestFetchReturnsErrNoDataWhenEverySlotFails(t *testing.T) { + f := &fakeFetcher{errs: map[int64]error{}} + for _, s := range Slots(fixedNow, 1) { + f.errs[s.Unix()] = errors.New("down") + } + _, err := Fetch(context.Background(), testPrefs(), quietOptions(f)) + if !errors.Is(err, ErrNoData) { + t.Fatalf("err = %v, want ErrNoData", err) + } +} + +func TestFetchStopsPromptlyWhenContextCancelledDuringDelay(t *testing.T) { + f := &fakeFetcher{always: &web.GridResponse{Channels: []web.JSONChannel{channel("s1", "2")}}} + opts := quietOptions(f) + opts.SlotDelay = time.Minute + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan error, 1) + go func() { + _, err := Fetch(ctx, testPrefs(), opts) + done <- err + }() + // Let the first slot complete and the delay begin, then cancel. + deadline := time.After(2 * time.Second) + for { + f.mu.Lock() + n := len(f.calls) + f.mu.Unlock() + if n >= 1 { + break + } + select { + case <-deadline: + t.Fatal("first slot never requested") + default: + time.Sleep(5 * time.Millisecond) + } + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Fetch did not return after cancellation") + } + if len(f.calls) != 1 { + t.Errorf("calls after cancel = %d, want 1", len(f.calls)) + } +} + +func TestFetchStopsWhenContextCancelledDuringRequest(t *testing.T) { + f := &fakeFetcher{always: &web.GridResponse{}, block: make(chan struct{})} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := Fetch(ctx, testPrefs(), quietOptions(f)) + done <- err + }() + time.Sleep(20 * time.Millisecond) + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Fetch did not return") + } +} + +func TestFetchHonoursAlreadyCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + f := &fakeFetcher{} + if _, err := Fetch(ctx, testPrefs(), quietOptions(f)); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v", err) + } + if len(f.calls) != 0 { + t.Errorf("no requests should be made, got %d", len(f.calls)) + } +} + +func TestOptionsDefaults(t *testing.T) { + o := Options{}.withDefaults(testPrefs()) + if o.Days != DefaultDays || o.SlotDelay != DefaultSlotDelay || o.Fetcher == nil || o.Now == nil || o.Logger == nil { + t.Fatalf("defaults not applied: %+v", o) + } + if _, ok := o.Fetcher.(*web.Client); !ok { + t.Fatalf("default fetcher should be *web.Client, got %T", o.Fetcher) + } +} From 5fc179248670bb987416e600adea52c2bc491e9e Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:39:38 -0400 Subject: [PATCH 7/7] docs: note the default grid client's on-disk cache --- CLAUDE.md | 2 +- README.md | 2 ++ scrape/scrape.go | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0198e9a..69f072a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ Run `go test ./...` for the setup, configuration, and provider-client tests. ## Architecture -The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. The grid download loop and guide assembly live in `scrape/` (`scrape.Fetch`), which is importable by other programs and does no enrichment, file I/O, or env reads. Enrichment, persistence, and runtime orchestration live in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`. +The binary is a single Go process that scrapes GraceNote/TMS for 14 days of TV listings and serves the data as XMLTV over HTTP. The grid download loop and guide assembly live in `scrape/` (`scrape.Fetch`), which is importable by other programs and does no enrichment or env reads (the default `web.Client` it uses does keep `grid_cache/` on disk). Enrichment, persistence, and runtime orchestration live in `main.go`; setup handlers live in `setup.go` and persisted configuration lives in `appconfig/`. **Data flow:** diff --git a/README.md b/README.md index 7242a99..cec7897 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ func main() { `Fetch` returns a `guide.TVGuide`: `Channels` deduplicated by station, `Lineup` with every position, `Programs`, and `Source`. It pauses five seconds between grid requests by default and skips slots that fail; it returns `scrape.ErrNoData` only when every slot failed. Logo and TMDB enrichment are not part of the package. Lineup discovery by postal code is available through `web.NewProviderClient().FindProviders`. +The default grid client is `web.NewClient`, which retries failed requests and keeps a raw-response cache in a `grid_cache/` directory under the working directory. Pass your own `Options.Fetcher` to change that. + ## Project Structure ``` diff --git a/scrape/scrape.go b/scrape/scrape.go index 0ee481b..cf863d5 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -1,6 +1,7 @@ // Package scrape fetches a Gracenote lineup's listings and converts them into -// a guide.TVGuide. It performs no enrichment, file I/O, or environment reads, -// so it can be imported by other programs. +// a guide.TVGuide. It performs no enrichment or environment reads, so it can +// be imported by other programs. The default fetcher, web.NewClient, keeps its +// own raw-response cache in ./grid_cache; supply Options.Fetcher to avoid it. package scrape import (