From 4e1c228f9d9d7c692b884dd04415b01440de7ea6 Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:29:42 -0400 Subject: [PATCH 1/3] test: add XMLTV golden test rendered from a grid fixture Factor template rendering out of persistGuideFiles into renderXMLTV so tests can render to a buffer. The golden file is generated from the current conversion code before any model changes. --- main.go | 18 +- web/testdata/grid_sample.json | 380 ++++++++++++++++++++++++++++++++++ xmltv_golden_test.go | 90 ++++++++ 3 files changed, 482 insertions(+), 6 deletions(-) create mode 100644 web/testdata/grid_sample.json create mode 100644 xmltv_golden_test.go diff --git a/main.go b/main.go index 69dc22e..92e930d 100644 --- a/main.go +++ b/main.go @@ -320,14 +320,20 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch return tvGuide, nil } -func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { - log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs)) - - // Parse embedded template +// renderXMLTV writes the guide as XMLTV using the embedded template. +func renderXMLTV(w io.Writer, tvGuide *guide.TVGuide) error { tmpl, err := template.ParseFS(guideTmplFS, "guide.tmpl") if err != nil { return fmt.Errorf("failed to parse template: %w", err) } + if err := tmpl.Execute(w, tvGuide); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + return nil +} + +func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { + log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs)) // Atomic write: write to temp file, then rename tmpFile, err := os.CreateTemp(".", "xmlguide-*.tmp") @@ -336,10 +342,10 @@ func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { } tmpName := tmpFile.Name() - if err := tmpl.Execute(tmpFile, tvGuide); err != nil { + if err := renderXMLTV(tmpFile, tvGuide); err != nil { tmpFile.Close() os.Remove(tmpName) - return fmt.Errorf("failed to execute template: %w", err) + return err } if err := tmpFile.Close(); err != nil { os.Remove(tmpName) diff --git a/web/testdata/grid_sample.json b/web/testdata/grid_sample.json new file mode 100644 index 0000000..bae4ef2 --- /dev/null +++ b/web/testdata/grid_sample.json @@ -0,0 +1,380 @@ +{ + "channels": [ + { + "affiliateCallSign": "null", + "affiliateName": "NATIONAL BROADCASTING COMPANY", + "callSign": "WKTVDT", + "channelId": "53158", + "channelNo": "2.1", + "events": [ + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "210", + "endTime": "2026-09-13T03:00:00Z", + "filter": [ + "filter-sports" + ], + "flag": [ + "Live", + "New" + ], + "program": { + "episode": null, + "episodeTitle": "Iowa State at Iowa", + "id": "EP062700990343", + "isGeneric": "0", + "releaseYear": 2019, + "season": null, + "seriesId": "SH06270099", + "shortDesc": "The Iowa State Cyclones take on the No. 21 Iowa Hawkeyes at ", + "title": "College Football", + "tmsId": "EP062700990343" + }, + "rating": null, + "seriesId": "SH06270099", + "startTime": "2026-09-12T23:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p53158_placeholder" + }, + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "29", + "endTime": "2026-09-13T03:29:00Z", + "filter": [ + "filter-news" + ], + "flag": [ + "New" + ], + "program": { + "episode": null, + "episodeTitle": null, + "id": "SH007933000000", + "isGeneric": "0", + "releaseYear": "2021", + "season": null, + "seriesId": "SH00793300", + "shortDesc": "Stay informed with the latest breaking news and headlines.", + "title": "NewsChannel 2 at 11", + "tmsId": "SH007933000000" + }, + "rating": null, + "seriesId": "SH00793300", + "startTime": "2026-09-13T03:00:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p53158_placeholder" + }, + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "94", + "endTime": "2026-09-13T05:03:00Z", + "filter": [], + "flag": [], + "program": { + "episode": "14", + "episodeTitle": "Ryan Gosling; Gorillaz", + "id": "EP000037101609", + "isGeneric": "0", + "releaseYear": null, + "season": "51", + "seriesId": "SH00003710", + "shortDesc": "Host Ryan Gosling; Gorillaz perform.", + "title": "Saturday Night Live", + "tmsId": "EP000037101609" + }, + "rating": "TV-14", + "seriesId": "SH00003710", + "startTime": "2026-09-13T03:29:00Z", + "tags": [ + "Stereo", + "CC" + ], + "thumbnail": "p53158_placeholder" + } + ], + "id": "531580", + "stationFilters": [ + "filter-sports", + "filter-news" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/53158.png?w=55" + }, + { + "affiliateCallSign": "null", + "affiliateName": "BOUNCE TV", + "callSign": "WSYRDT3", + "channelId": "84540", + "channelNo": "9.3", + "events": [ + { + "callSign": "WSYRDT3", + "channelNo": "9.3", + "duration": "150", + "endTime": "2026-09-13T01:30:00Z", + "filter": [ + "filter-movie" + ], + "flag": [], + "program": { + "episode": null, + "episodeTitle": null, + "id": "MV015399200000", + "isGeneric": "1", + "releaseYear": "2021", + "season": null, + "seriesId": "MV01539920", + "shortDesc": "Rescued as a child by the legendary assassin Moody, Anna is ", + "title": "The Prot\u00e9g\u00e9", + "tmsId": "MV015399200000" + }, + "rating": "TV-14", + "seriesId": "MV01539920", + "startTime": "2026-09-12T23:00:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p84540_placeholder" + }, + { + "callSign": "WSYRDT3", + "channelNo": "9.3", + "duration": "120", + "endTime": "2026-09-13T03:30:00Z", + "filter": [ + "filter-movie" + ], + "flag": [], + "program": { + "episode": null, + "episodeTitle": null, + "id": "MV002273030000", + "isGeneric": 0, + "releaseYear": "2009", + "season": null, + "seriesId": "MV00227303", + "shortDesc": "A rogue (Rain) enlists the help of the Europol agent (Naomie", + "title": "Ninja Assassin", + "tmsId": "MV002273030000" + }, + "rating": "TV-14", + "seriesId": "MV00227303", + "startTime": "2026-09-13T01:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p84540_placeholder" + }, + { + "callSign": "WSYRDT3", + "channelNo": "9.3", + "duration": "120", + "endTime": "2026-09-13T05:30:00Z", + "filter": [ + "filter-movie" + ], + "flag": [], + "program": { + "episode": null, + "episodeTitle": null, + "id": "MV003756000000", + "isGeneric": "0", + "releaseYear": "2012", + "season": null, + "seriesId": "MV00375600", + "shortDesc": "After his partner is killed, a veteran hit man (Sylvester St", + "title": "Bullet to the Head", + "tmsId": "MV003756000000" + }, + "rating": "TV-14", + "seriesId": "MV00375600", + "startTime": "2026-09-13T03:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p84540_placeholder" + } + ], + "id": "8454012", + "stationFilters": [ + "filter-movie" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/84540.png?w=55" + }, + { + "affiliateCallSign": "null", + "affiliateName": "PBS KIDS HD", + "callSign": "WCNYDT4", + "channelId": "49216", + "channelNo": "24.4", + "events": [ + { + "callSign": "WCNYDT4", + "channelNo": "24.4", + "duration": "30", + "endTime": "2026-09-13T00:30:00Z", + "filter": [ + "filter-family" + ], + "flag": [ + "New", + "Finale" + ], + "program": { + "episode": "30", + "episodeTitle": "The Square Dance; The Process", + "id": "EP052844640100", + "isGeneric": "0", + "releaseYear": null, + "season": "1", + "seriesId": "SH05284464", + "shortDesc": "Dylan realizes he doesn't love square-dancing; Carl teaches ", + "title": "Carl the Collector", + "tmsId": "EP052844640100" + }, + "rating": "TV-Y", + "seriesId": "SH05284464", + "startTime": "2026-09-13T00:00:00Z", + "tags": [ + "Stereo", + "CC" + ], + "thumbnail": "p49216_placeholder" + }, + { + "callSign": "WCNYDT4", + "channelNo": "24.4", + "duration": "30", + "endTime": "2026-09-13T01:00:00Z", + "filter": [ + "filter-family" + ], + "flag": [ + "Premiere" + ], + "program": { + "episode": "31", + "episodeTitle": "The Bark Banquet; Carl's Collection of To-Dos", + "id": "EP052844640101", + "isGeneric": "0", + "releaseYear": null, + "season": "1", + "seriesId": "SH05284464", + "shortDesc": "Carl wants to start his own family tradition after Sheldon i", + "title": "Carl the Collector", + "tmsId": "EP052844640101" + }, + "rating": "TV-Y", + "seriesId": "SH05284464", + "startTime": "2026-09-13T00:30:00Z", + "tags": [ + "Stereo", + "CC" + ], + "thumbnail": "p49216_placeholder" + }, + { + "callSign": "WCNYDT4", + "channelNo": "24.4", + "duration": "30", + "endTime": "2026-09-13T01:30:00Z", + "filter": [ + "filter-family" + ], + "flag": [], + "program": { + "episode": "6", + "episodeTitle": "Buzz and the Tree", + "id": "EP003618800155", + "isGeneric": "0", + "releaseYear": null, + "season": "13", + "seriesId": "SH00361880", + "shortDesc": "It's Arbor Day in Cyberspace, but Buzz is more interested in", + "title": "Cyberchase", + "tmsId": "EP003618800155" + }, + "rating": "TV-Y", + "seriesId": "SH00361880", + "startTime": "2026-09-13T01:00:00Z", + "tags": [ + "Stereo", + "CC", + "DVS" + ], + "thumbnail": "p49216_placeholder" + } + ], + "id": "4921632", + "stationFilters": [ + "filter-family" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/49216.png?w=55" + }, + { + "affiliateCallSign": "null", + "affiliateName": "NATIONAL BROADCASTING COMPANY", + "callSign": "WKTVDT", + "channelId": "53158", + "channelNo": "1002", + "events": [ + { + "callSign": "WKTVDT", + "channelNo": "2.1", + "duration": "210", + "endTime": "2026-09-13T03:00:00Z", + "filter": [ + "filter-sports" + ], + "flag": [ + "Live", + "New" + ], + "program": { + "episode": null, + "episodeTitle": "Iowa State at Iowa", + "id": "EP062700990343", + "isGeneric": "0", + "releaseYear": null, + "season": null, + "seriesId": "SH06270099", + "shortDesc": "The Iowa State Cyclones take on the No. 21 Iowa Hawkeyes at ", + "title": "College Football", + "tmsId": "EP062700990343" + }, + "rating": null, + "seriesId": "SH06270099", + "startTime": "2026-09-12T23:30:00Z", + "tags": [ + "CC" + ], + "thumbnail": "p53158_placeholder" + } + ], + "id": "5315899", + "stationFilters": [ + "filter-sports", + "filter-news" + ], + "stationGenres": [ + false + ], + "thumbnail": "//images.example.invalid/station/53158.png?w=55" + } + ] +} \ No newline at end of file diff --git a/xmltv_golden_test.go b/xmltv_golden_test.go new file mode 100644 index 0000000..d4f67e1 --- /dev/null +++ b/xmltv_golden_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "os" + "path/filepath" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/guide" + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +var updateGolden = flag.Bool("update", false, "rewrite testdata/xmlguide_golden.xmltv from the current conversion path") + +const ( + gridFixturePath = "web/testdata/grid_sample.json" + xmltvGoldenPath = "testdata/xmlguide_golden.xmltv" + fixtureLanguage = "en-us" + fixtureCountry = "USA" +) + +// loadGridFixture decodes the shared grid sample used by the golden and +// conversion tests. +func loadGridFixture(t *testing.T) web.GridResponse { + t.Helper() + data, err := os.ReadFile(gridFixturePath) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + var grid web.GridResponse + if err := json.Unmarshal(data, &grid); err != nil { + t.Fatalf("decode fixture: %v", err) + } + return grid +} + +// guideFromGrid mirrors the scrape loop's conversion: channels deduplicated by +// station ID in first-seen order, events deduplicated by station, start, and end. +func guideFromGrid(grid web.GridResponse) *guide.TVGuide { + seenChannel := make(map[string]bool) + seenEvent := make(map[string]bool) + g := &guide.TVGuide{} + for _, ch := range grid.Channels { + if !seenChannel[ch.ChannelID] { + seenChannel[ch.ChannelID] = true + g.Channels = append(g.Channels, guide.ConvertChannel(ch)) + } + for _, ev := range ch.Events { + key := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime + if seenEvent[key] { + continue + } + seenEvent[key] = true + g.Programs = append(g.Programs, guide.ConvertEvent(ev, ch.ChannelID, fixtureLanguage, fixtureCountry)) + } + } + return g +} + +// TestXMLTVGolden guards the promise that model changes never alter XMLTV +// output. Regenerate deliberately with: go test -run TestXMLTVGolden -update +func TestXMLTVGolden(t *testing.T) { + g := guideFromGrid(loadGridFixture(t)) + + var rendered bytes.Buffer + if err := renderXMLTV(&rendered, g); err != nil { + t.Fatalf("render: %v", err) + } + + if *updateGolden { + if err := os.MkdirAll(filepath.Dir(xmltvGoldenPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(xmltvGoldenPath, rendered.Bytes(), 0644); err != nil { + t.Fatal(err) + } + t.Logf("wrote %s (%d bytes)", xmltvGoldenPath, rendered.Len()) + return + } + + want, err := os.ReadFile(xmltvGoldenPath) + if err != nil { + t.Fatalf("read golden (run with -update to create it): %v", err) + } + if !bytes.Equal(want, rendered.Bytes()) { + t.Fatalf("XMLTV output changed from golden.\nIf this is intended, run: go test -run TestXMLTVGolden -update\n--- got ---\n%s", rendered.String()) + } +} From 3c7fda516a8110bcf532839968ad215643117aa1 Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:30:52 -0400 Subject: [PATCH 2/3] Keep raw Gracenote fields on channels and programs Carry the row id, affiliate callsign, and station filters through to guide.Channel, and the TMS id, release year, generic flag, and raw event filters through to guide.Program. Categories are unchanged; Filters holds the Gracenote list before Series and Finale are appended. releaseYear and isGeneric use tolerant decoders so an unexpected scalar encoding cannot fail an entire six-hour grid. The guide cache gains a schema version. An older cache is rebuilt once so the new fields are populated. --- guide/guide.go | 62 ++++++++++++++++---- guide/guide_test.go | 115 +++++++++++++++++++++++++++++++++++++ main.go | 12 +++- setup_test.go | 19 +++++++ web/flex.go | 76 +++++++++++++++++++++++++ web/web.go | 32 +++++++---- web/web_decode_test.go | 125 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 416 insertions(+), 25 deletions(-) create mode 100644 guide/guide_test.go create mode 100644 web/flex.go create mode 100644 web/web_decode_test.go diff --git a/guide/guide.go b/guide/guide.go index 821f5f7..d052037 100644 --- a/guide/guide.go +++ b/guide/guide.go @@ -15,12 +15,15 @@ type TVGuide struct { } type Channel struct { - ID string - DisplayNames []DisplayName - IconURL string - CallSign string // internal, not in template - Affiliate string // internal, not in template - ChannelNo string // internal, not in template + ID string + DisplayNames []DisplayName + IconURL string + CallSign string // internal, not in template + Affiliate string // internal, not in template + ChannelNo string // internal, not in template + PlacementID string // internal, not in template; Gracenote row id, not a stable key + AffiliateCallSign string // internal, not in template + Filters []string // internal, not in template; Gracenote station filters, prefix stripped } type DisplayName struct { @@ -45,6 +48,10 @@ type Program struct { Country string EpisodeNumbers []EpisodeNumber Categories []Category + Filters []string // internal, not in template; raw Gracenote event filters, prefix stripped + TMSID string // internal, not in template + ReleaseYear string // internal, not in template + Generic bool // internal, not in template New bool Premiere bool PreviouslyShown bool @@ -119,13 +126,37 @@ func ConvertChannel(ch web.JSONChannel) Channel { {Name: xmlEscape(ch.CallSign)}, {Name: xmlEscape(titleCase(ch.AffiliateName))}, }, - IconURL: iconURL, - CallSign: ch.CallSign, - Affiliate: ch.AffiliateName, - ChannelNo: ch.ChannelNo, + IconURL: iconURL, + CallSign: ch.CallSign, + Affiliate: ch.AffiliateName, + ChannelNo: ch.ChannelNo, + PlacementID: ch.ID, + AffiliateCallSign: normalizeNull(ch.AffiliateCallSign), + Filters: stripFilterPrefixes(ch.StationFilters), } } +// normalizeNull maps Gracenote's literal "null" string to an empty value. +func normalizeNull(s string) string { + if strings.EqualFold(strings.TrimSpace(s), "null") { + return "" + } + return s +} + +// stripFilterPrefixes turns Gracenote filter tags such as "filter-sports" into +// "sports". A nil input stays nil so callers can distinguish absent from empty. +func stripFilterPrefixes(filters []string) []string { + if filters == nil { + return nil + } + out := make([]string, 0, len(filters)) + for _, f := range filters { + out = append(out, strings.TrimPrefix(f, "filter-")) + } + return out +} + // converts a JSON event to a template Program struct. func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program { season := 0 @@ -163,10 +194,13 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program { // URL programURL := "https://tvlistings.gracenote.com//overview.html?programSeriesId=" + ev.SeriesID + "&tmsId=" + ev.Program.ID + // Raw Gracenote filters, kept separately so consumers can tell them apart + // from the Series and Finale labels added below. + filters := stripFilterPrefixes(ev.Filter) + // Categories from filter array (strip "filter-" prefix) var categories []Category - for _, f := range ev.Filter { - name := strings.TrimPrefix(f, "filter-") + for _, name := range filters { categories = append(categories, Category{Name: name, Lang: lang}) } @@ -266,6 +300,10 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program { Country: country, EpisodeNumbers: episodeNumbers, Categories: categories, + Filters: filters, + TMSID: ev.Program.TmsID, + ReleaseYear: string(ev.Program.ReleaseYear), + Generic: bool(ev.Program.IsGeneric), New: isNew, Premiere: isPremiere, PreviouslyShown: !isNew, diff --git a/guide/guide_test.go b/guide/guide_test.go new file mode 100644 index 0000000..852cde4 --- /dev/null +++ b/guide/guide_test.go @@ -0,0 +1,115 @@ +package guide + +import ( + "reflect" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/web" +) + +func sampleChannel() web.JSONChannel { + return web.JSONChannel{ + ChannelID: "53158", + ID: "531580", + ChannelNo: "2.1", + CallSign: "WKTVDT", + AffiliateName: "NATIONAL BROADCASTING COMPANY", + AffiliateCallSign: "null", + StationFilters: []string{"filter-sports", "filter-news"}, + Thumbnail: "//images.example.invalid/station/53158.png?w=55", + } +} + +func TestConvertChannelCarriesRawFields(t *testing.T) { + ch := ConvertChannel(sampleChannel()) + + if ch.ID != "53158" || ch.ChannelNo != "2.1" || ch.CallSign != "WKTVDT" { + t.Fatalf("basic fields wrong: %+v", ch) + } + if ch.PlacementID != "531580" { + t.Errorf("PlacementID = %q", ch.PlacementID) + } + if ch.AffiliateCallSign != "" { + t.Errorf(`"null" affiliate callsign should normalize to empty, got %q`, ch.AffiliateCallSign) + } + if !reflect.DeepEqual(ch.Filters, []string{"sports", "news"}) { + t.Errorf("Filters = %v", ch.Filters) + } + if ch.IconURL != "http://images.example.invalid/station/53158.png" { + t.Errorf("IconURL = %q", ch.IconURL) + } + // Existing XMLTV-facing fields must be untouched by the additions. + wantNames := []DisplayName{{"2.1 WKTVDT"}, {"2.1"}, {"WKTVDT"}, {"NATIONAL BROADCASTING COMPANY"}} + if !reflect.DeepEqual(ch.DisplayNames, wantNames) { + t.Errorf("DisplayNames = %v", ch.DisplayNames) + } +} + +func TestConvertChannelKeepsRealAffiliateCallSign(t *testing.T) { + in := sampleChannel() + in.AffiliateCallSign = "NBC" + in.StationFilters = nil + ch := ConvertChannel(in) + if ch.AffiliateCallSign != "NBC" { + t.Errorf("AffiliateCallSign = %q", ch.AffiliateCallSign) + } + if ch.Filters != nil { + t.Errorf("nil station filters should stay nil, got %v", ch.Filters) + } +} + +func sampleEvent() web.JSONEvent { + season, episode, title := "3", "7", "The One" + return web.JSONEvent{ + StartTime: "2026-07-25T07:00:00Z", + EndTime: "2026-07-25T08:00:00Z", + Duration: "60", + SeriesID: "SH06270099", + Flag: []string{"New", "Finale"}, + Filter: []string{"filter-sports"}, + Program: web.JSONProgram{ + ID: "EP062700990343", + TmsID: "EP062700990343", + Title: "Morning Business", + EpisodeTitle: &title, + Season: &season, + Episode: &episode, + ReleaseYear: "2019", + IsGeneric: true, + }, + } +} + +func TestConvertEventSeparatesRawFiltersFromCategories(t *testing.T) { + p := ConvertEvent(sampleEvent(), "53158", "en-us", "USA") + + if !reflect.DeepEqual(p.Filters, []string{"sports"}) { + t.Errorf("Filters = %v", p.Filters) + } + // Categories keep the historical shape: raw filters, then Series (from an + // episode number), then Finale (from the flag). + want := []Category{{"sports", "en-us"}, {"Series", "en-us"}, {"Finale", "en-us"}} + if !reflect.DeepEqual(p.Categories, want) { + t.Errorf("Categories = %v, want %v", p.Categories, want) + } + if p.TMSID != "EP062700990343" || p.ReleaseYear != "2019" || !p.Generic { + t.Errorf("program metadata = tms %q year %q generic %v", p.TMSID, p.ReleaseYear, p.Generic) + } + if !p.New || p.PreviouslyShown { + t.Errorf("flag handling changed: new=%v previouslyShown=%v", p.New, p.PreviouslyShown) + } +} + +func TestConvertEventWithoutFilters(t *testing.T) { + ev := sampleEvent() + ev.Filter = nil + ev.Flag = nil + ev.Program.Season, ev.Program.Episode = nil, nil + p := ConvertEvent(ev, "53158", "en", "USA") + if p.Filters != nil { + t.Errorf("Filters should be nil, got %v", p.Filters) + } + if len(p.Categories) != 0 { + t.Errorf("Categories should be empty, got %v", p.Categories) + } +} diff --git a/main.go b/main.go index 92e930d..156e69b 100644 --- a/main.go +++ b/main.go @@ -369,7 +369,13 @@ func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error { const guideCachePath = "guide_cache.json" +// guideCacheVersion is bumped whenever the persisted guide shape gains fields +// that a scrape must populate. An older cache is rebuilt rather than served +// with empty fields. +const guideCacheVersion = 2 + type guideCache struct { + Version int `json:"version"` SavedAt time.Time `json:"saved_at"` SourceFingerprint string `json:"source_fingerprint"` Guide guide.TVGuide `json:"guide"` @@ -377,7 +383,7 @@ type guideCache struct { // saveGuideCache persists the TVGuide to a JSON file. func saveGuideCache(g *guide.TVGuide, sourceFingerprint string) { - data, err := json.Marshal(guideCache{SavedAt: time.Now(), SourceFingerprint: sourceFingerprint, Guide: *g}) + data, err := json.Marshal(guideCache{Version: guideCacheVersion, SavedAt: time.Now(), SourceFingerprint: sourceFingerprint, Guide: *g}) if err != nil { log.Printf("guide cache: failed to marshal: %v", err) return @@ -401,6 +407,10 @@ func loadGuideCache(maxAge time.Duration, sourceFingerprint string) (*guide.TVGu log.Printf("guide cache: corrupt, ignoring: %v", err) return nil, 0, false } + if c.Version != guideCacheVersion { + log.Printf("guide cache: schema version %d, want %d; rebuilding", c.Version, guideCacheVersion) + return nil, 0, false + } if c.SourceFingerprint != sourceFingerprint { log.Println("guide cache: source changed, ignoring cached guide") return nil, 0, false diff --git a/setup_test.go b/setup_test.go index 7651721..ba03181 100644 --- a/setup_test.go +++ b/setup_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" "path/filepath" "strings" "testing" @@ -226,3 +227,21 @@ func TestGuideCacheRequiresMatchingSource(t *testing.T) { t.Fatal("cache did not load for matching source") } } + +func TestGuideCacheRequiresCurrentSchemaVersion(t *testing.T) { + t.Chdir(t.TempDir()) + + // A cache written by an older build has no version field. + legacy := []byte(`{"saved_at":"` + time.Now().UTC().Format(time.RFC3339) + `","source_fingerprint":"src","guide":{"Channels":[],"Programs":[]}}`) + if err := os.WriteFile(guideCachePath, legacy, 0644); err != nil { + t.Fatal(err) + } + if _, _, ok := loadGuideCache(time.Hour, "src"); ok { + t.Fatal("legacy cache without a version loaded") + } + + saveGuideCache(&guide.TVGuide{}, "src") + if _, _, ok := loadGuideCache(time.Hour, "src"); !ok { + t.Fatal("current-version cache did not load") + } +} diff --git a/web/flex.go b/web/flex.go new file mode 100644 index 0000000..ad88a65 --- /dev/null +++ b/web/flex.go @@ -0,0 +1,76 @@ +package web + +import ( + "bytes" + "encoding/json" + "strconv" + "strings" +) + +// FlexString decodes a JSON string, number, or null into a string. Gracenote +// is inconsistent about scalar encodings, and a decode error on one optional +// field would otherwise discard an entire six-hour grid. +type FlexString string + +func (f *FlexString) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *f = "" + return nil + } + if data[0] == '"' { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + *f = FlexString(s) + return nil + } + var n json.Number + if err := json.Unmarshal(data, &n); err == nil { + *f = FlexString(n.String()) + return nil + } + var b bool + if err := json.Unmarshal(data, &b); err == nil { + *f = FlexString(strconv.FormatBool(b)) + return nil + } + *f = "" + return nil +} + +// FlexBool decodes a JSON bool, a 0/1 number, a "0"/"1"/"true"/"false" string, +// or null into a bool. Anything unrecognized decodes as false rather than +// failing the surrounding document. +type FlexBool bool + +func (f *FlexBool) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + *f = false + return nil + } + var b bool + if err := json.Unmarshal(data, &b); err == nil { + *f = FlexBool(b) + return nil + } + var n json.Number + if err := json.Unmarshal(data, &n); err == nil { + *f = FlexBool(n.String() != "0" && n.String() != "0.0") + return nil + } + var s string + if err := json.Unmarshal(data, &s); err == nil { + switch strings.ToLower(strings.TrimSpace(s)) { + case "1", "true", "yes": + *f = true + default: + *f = false + } + return nil + } + *f = false + return nil +} diff --git a/web/web.go b/web/web.go index c4160e3..1d7e1d6 100644 --- a/web/web.go +++ b/web/web.go @@ -32,12 +32,17 @@ type GridResponse struct { } type JSONChannel struct { - ChannelID string `json:"channelId"` - ChannelNo string `json:"channelNo"` - CallSign string `json:"callSign"` - AffiliateName string `json:"affiliateName"` - Thumbnail string `json:"thumbnail"` - Events []JSONEvent `json:"events"` + ChannelID string `json:"channelId"` + // ID is the station ID with the row index appended. It identifies a row + // within one response, not a stable lineup placement. + ID string `json:"id"` + ChannelNo string `json:"channelNo"` + CallSign string `json:"callSign"` + AffiliateName string `json:"affiliateName"` + AffiliateCallSign string `json:"affiliateCallSign"` + StationFilters []string `json:"stationFilters"` + Thumbnail string `json:"thumbnail"` + Events []JSONEvent `json:"events"` } type JSONEvent struct { @@ -54,12 +59,15 @@ type JSONEvent struct { } type JSONProgram struct { - ID string `json:"id"` - Title string `json:"title"` - EpisodeTitle *string `json:"episodeTitle"` - ShortDesc *string `json:"shortDesc"` - Season *string `json:"season"` - Episode *string `json:"episode"` + ID string `json:"id"` + TmsID string `json:"tmsId"` + Title string `json:"title"` + EpisodeTitle *string `json:"episodeTitle"` + ShortDesc *string `json:"shortDesc"` + Season *string `json:"season"` + Episode *string `json:"episode"` + ReleaseYear FlexString `json:"releaseYear"` + IsGeneric FlexBool `json:"isGeneric"` } type Preferences struct { diff --git a/web/web_decode_test.go b/web/web_decode_test.go new file mode 100644 index 0000000..e1af3b7 --- /dev/null +++ b/web/web_decode_test.go @@ -0,0 +1,125 @@ +package web + +import ( + "encoding/json" + "os" + "testing" +) + +func TestGridFixtureDecodesRawFields(t *testing.T) { + data, err := os.ReadFile("testdata/grid_sample.json") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + var grid GridResponse + if err := json.Unmarshal(data, &grid); err != nil { + t.Fatalf("decode fixture: %v", err) + } + if len(grid.Channels) != 4 { + t.Fatalf("channels = %d, want 4", len(grid.Channels)) + } + + first := grid.Channels[0] + if first.ID != "531580" || first.ChannelID != "53158" { + t.Fatalf("row id/station id = %q/%q", first.ID, first.ChannelID) + } + if first.AffiliateCallSign != "null" { + t.Fatalf("affiliateCallSign should decode verbatim, got %q", first.AffiliateCallSign) + } + if len(first.StationFilters) != 2 || first.StationFilters[0] != "filter-sports" { + t.Fatalf("stationFilters = %v", first.StationFilters) + } + + // Same station at a second position decodes as its own row. + last := grid.Channels[3] + if last.ChannelID != first.ChannelID || last.ChannelNo == first.ChannelNo || last.ID == first.ID { + t.Fatalf("duplicate position not preserved: %+v", last) + } + + ev := first.Events[0].Program + if ev.TmsID == "" { + t.Fatal("tmsId missing") + } + if ev.ReleaseYear != "2019" { + t.Fatalf("numeric releaseYear = %q, want 2019", ev.ReleaseYear) + } + if first.Events[1].Program.ReleaseYear != "2021" { + t.Fatalf("string releaseYear = %q, want 2021", first.Events[1].Program.ReleaseYear) + } + if first.Events[2].Program.ReleaseYear != "" { + t.Fatalf("null releaseYear = %q, want empty", first.Events[2].Program.ReleaseYear) + } + second := grid.Channels[1] + if !bool(second.Events[0].Program.IsGeneric) { + t.Fatal(`isGeneric "1" should decode true`) + } + if bool(second.Events[1].Program.IsGeneric) { + t.Fatal("isGeneric 0 should decode false") + } +} + +func TestFlexString(t *testing.T) { + cases := []struct { + in string + want string + }{ + {`"2019"`, "2019"}, + {`2019`, "2019"}, + {`2019.0`, "2019.0"}, + {`null`, ""}, + {`true`, "true"}, + {`""`, ""}, + {`[1]`, ""}, + {`{"a":1}`, ""}, + } + for _, c := range cases { + var doc struct { + V FlexString `json:"v"` + } + if err := json.Unmarshal([]byte(`{"v":`+c.in+`}`), &doc); err != nil { + t.Fatalf("%s: unexpected error %v", c.in, err) + } + if string(doc.V) != c.want { + t.Errorf("%s: got %q want %q", c.in, doc.V, c.want) + } + } + var doc struct { + V FlexString `json:"v"` + } + if err := json.Unmarshal([]byte(`{"v":"unterminated}`), &doc); err == nil { + t.Fatal("malformed JSON should still error") + } +} + +func TestFlexBool(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {`true`, true}, + {`false`, false}, + {`1`, true}, + {`0`, false}, + {`2`, true}, + {`"1"`, true}, + {`"0"`, false}, + {`"true"`, true}, + {`"TRUE"`, true}, + {`"false"`, false}, + {`"yes"`, true}, + {`"maybe"`, false}, + {`null`, false}, + {`[true]`, false}, + } + for _, c := range cases { + var doc struct { + V FlexBool `json:"v"` + } + if err := json.Unmarshal([]byte(`{"v":`+c.in+`}`), &doc); err != nil { + t.Fatalf("%s: unexpected error %v", c.in, err) + } + if bool(doc.V) != c.want { + t.Errorf("%s: got %v want %v", c.in, doc.V, c.want) + } + } +} From b28ab1f69f4d886eac0c1a49f52828e9eba3ac2f Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:34:37 -0400 Subject: [PATCH 3/3] test: commit the XMLTV golden file *.xmltv is ignored repo-wide, which kept the golden out of the first commit. Allowlist it explicitly. --- .gitignore | 1 + testdata/xmlguide_golden.xmltv | 167 +++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 testdata/xmlguide_golden.xmltv diff --git a/.gitignore b/.gitignore index a78349e..9c1b9fa 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .env config.json *.xmltv +!testdata/xmlguide_golden.xmltv tmdb_cache.json tvlogo_cache.json guide_cache.json diff --git a/testdata/xmlguide_golden.xmltv b/testdata/xmlguide_golden.xmltv new file mode 100644 index 0000000..8944ec8 --- /dev/null +++ b/testdata/xmlguide_golden.xmltv @@ -0,0 +1,167 @@ + + + + + 2.1 WKTVDT + 2.1 + WKTVDT + NATIONAL BROADCASTING COMPANY + + + + 9.3 WSYRDT3 + 9.3 + WSYRDT3 + BOUNCE TV + + + + 24.4 WCNYDT4 + 24.4 + WCNYDT4 + PBS KIDS HD + + + + College Football + Iowa State at Iowa + The Iowa State Cyclones take on the No. 21 Iowa Hawkeyes at + 210 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH06270099&tmsId=EP062700990343 + USA + EP06270099.0343 + sports + + + + + NewsChannel 2 at 11 + Stay informed with the latest breaking news and headlines. + 29 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH00793300&tmsId=SH007933000000 + USA + SH00793300.0000 + news + + + + + Saturday Night Live + Ryan Gosling; Gorillaz + Host Ryan Gosling; Gorillaz perform. + 94 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH00003710&tmsId=EP000037101609 + USA + S51E14 + 50.13 + EP00003710.1609 + Series + + + TV-14 + + + The Protégé + Rescued as a child by the legendary assassin Moody, Anna is + 150 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=MV01539920&tmsId=MV015399200000 + USA + MV01539920.0000 + movie + + + TV-14 + + + Ninja Assassin + A rogue (Rain) enlists the help of the Europol agent (Naomie + 120 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=MV00227303&tmsId=MV002273030000 + USA + MV00227303.0000 + movie + + + TV-14 + + + Bullet to the Head + After his partner is killed, a veteran hit man (Sylvester St + 120 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=MV00375600&tmsId=MV003756000000 + USA + MV00375600.0000 + movie + + + TV-14 + + + Carl the Collector + The Square Dance; The Process + Dylan realizes he doesn't love square-dancing; Carl teaches + 30 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH05284464&tmsId=EP052844640100 + USA + S01E30 + 0.29 + EP05284464.0100 + family + Series + Finale + + + TV-Y + + + Carl the Collector + The Bark Banquet; Carl's Collection of To-Dos + Carl wants to start his own family tradition after Sheldon i + 30 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH05284464&tmsId=EP052844640101 + USA + S01E31 + 0.30 + EP05284464.0101 + family + Series + + + + TV-Y + + + Cyberchase + Buzz and the Tree + It's Arbor Day in Cyberspace, but Buzz is more interested in + 30 + en-us + + https://tvlistings.gracenote.com//overview.html?programSeriesId=SH00361880&tmsId=EP003618800155 + USA + S13E06 + 12.5 + EP00361880.0155 + family + Series + + + TV-Y + +