From 57eac5b78f2787543f4afa78fbc17b0bac08457e Mon Sep 17 00:00:00 2001 From: Daniel Widrick Date: Sat, 12 Sep 2026 23:32:50 -0400 Subject: [PATCH] Add a lineup model that retains every provider position TVGuide gains Lineup, a list of channel positions keyed by number and station that is never collapsed, and Source, which records the lineup the guide was built from. Resolved station logos are copied onto each position, the image proxy rewrite covers them, and the Jellyfin channel filter applies to them by channel number. ChannelNumberLess moves into the guide package with a strict ordering: numeric numbers first, ties broken by string, then non-numeric. --- guide/guide.go | 127 +++++++++++++++++++++++++++++++++++++++----- guide/guide_test.go | 82 ++++++++++++++++++++++++++++ lineup_test.go | 85 +++++++++++++++++++++++++++++ main.go | 90 +++++++++++++++++++++---------- 4 files changed, 342 insertions(+), 42 deletions(-) create mode 100644 lineup_test.go diff --git a/guide/guide.go b/guide/guide.go index d052037..b502c4e 100644 --- a/guide/guide.go +++ b/guide/guide.go @@ -2,16 +2,112 @@ package guide import ( "fmt" + "sort" "strconv" "strings" + "time" "unicode" "github.com/daniel-widrick/GraceNoteScraper/web" ) type TVGuide struct { + // Channels is the XMLTV view: one entry per Gracenote station. Channels []Channel Programs []Program + // Lineup retains every provider position, so a station carried at two + // channel numbers appears twice. It is never collapsed. + Lineup []LineupPosition + // Source records which provider lineup produced this guide. + Source Source +} + +// Source identifies the Gracenote lineup a guide was built from. +type Source struct { + Country string + PostalCode string + HeadendID string + LineupID string + Device string + Language string + GeneratedAt time.Time +} + +// SourceFromPreferences copies the request preferences into a Source. +func SourceFromPreferences(p web.Preferences, generatedAt time.Time) Source { + return Source{ + Country: p.Country, + PostalCode: p.ZipCode, + HeadendID: p.Headend, + LineupID: p.LineupId, + Device: p.Device, + Language: p.Language, + GeneratedAt: generatedAt, + } +} + +// LineupPosition is one channel number in a provider lineup. +type LineupPosition struct { + ChannelNo string + StationID string + PlacementID string // Gracenote row id; carried for fidelity, not a stable key + CallSign string + Affiliate string + AffiliateCallSign string + Filters []string + LogoURL string +} + +// Key identifies a position across grid slices: the same station at the same +// number is one position no matter how many responses it appears in. +func (p LineupPosition) Key() string { + return p.ChannelNo + "|" + p.StationID +} + +// ConvertLineupPosition converts a JSON channel row to a lineup position. +func ConvertLineupPosition(ch web.JSONChannel) LineupPosition { + return LineupPosition{ + ChannelNo: ch.ChannelNo, + StationID: ch.ChannelID, + PlacementID: ch.ID, + CallSign: ch.CallSign, + Affiliate: ch.AffiliateName, + AffiliateCallSign: normalizeNull(ch.AffiliateCallSign), + Filters: stripFilterPrefixes(ch.StationFilters), + LogoURL: gracenoteIconURL(ch.Thumbnail), + } +} + +// ChannelNumberLess orders channel numbers numerically where both parse +// (so "2.1" < "10" < "100"), places numeric numbers before non-numeric ones, +// and falls back to string order. Equal numbers compare as strings so the +// ordering is strict. +func ChannelNumberLess(a, b string) bool { + af, errA := strconv.ParseFloat(strings.TrimSpace(a), 64) + bf, errB := strconv.ParseFloat(strings.TrimSpace(b), 64) + switch { + case errA == nil && errB == nil: + if af != bf { + return af < bf + } + return a < b + case errA == nil: + return true + case errB == nil: + return false + default: + return a < b + } +} + +// SortLineup orders positions by channel number, then station ID, in place. +func SortLineup(positions []LineupPosition) { + sort.SliceStable(positions, func(i, j int) bool { + if positions[i].ChannelNo != positions[j].ChannelNo { + return ChannelNumberLess(positions[i].ChannelNo, positions[j].ChannelNo) + } + return positions[i].StationID < positions[j].StationID + }) } type Channel struct { @@ -102,21 +198,26 @@ func formatXMLTVTime(iso string) string { return s } +// gracenoteIconURL builds an absolute icon URL from a Gracenote thumbnail +// path: strip leading slashes, strip query params, prepend http:// +func gracenoteIconURL(thumbnail string) string { + if thumbnail == "" { + return "" + } + raw := thumbnail + if idx := strings.Index(raw, "?"); idx >= 0 { + raw = raw[:idx] + } + raw = strings.TrimLeft(raw, "/") + if raw == "" { + return "" + } + return "http://" + raw +} + // converts a JSON channel to a template Channel struct. func ConvertChannel(ch web.JSONChannel) Channel { - // Build icon URL: strip leading slashes, strip query params, prepend http:// - iconURL := "" - if ch.Thumbnail != "" { - raw := ch.Thumbnail - // Strip query string - if idx := strings.Index(raw, "?"); idx >= 0 { - raw = raw[:idx] - } - raw = strings.TrimLeft(raw, "/") - if raw != "" { - iconURL = "http://" + raw - } - } + iconURL := gracenoteIconURL(ch.Thumbnail) return Channel{ ID: ch.ChannelID, diff --git a/guide/guide_test.go b/guide/guide_test.go index 852cde4..4274f9d 100644 --- a/guide/guide_test.go +++ b/guide/guide_test.go @@ -3,6 +3,7 @@ package guide import ( "reflect" "testing" + "time" "github.com/daniel-widrick/GraceNoteScraper/web" ) @@ -113,3 +114,84 @@ func TestConvertEventWithoutFilters(t *testing.T) { t.Errorf("Categories should be empty, got %v", p.Categories) } } + +func TestConvertLineupPosition(t *testing.T) { + in := sampleChannel() + p := ConvertLineupPosition(in) + want := LineupPosition{ + ChannelNo: "2.1", + StationID: "53158", + PlacementID: "531580", + CallSign: "WKTVDT", + Affiliate: "NATIONAL BROADCASTING COMPANY", + AffiliateCallSign: "", + Filters: []string{"sports", "news"}, + LogoURL: "http://images.example.invalid/station/53158.png", + } + if !reflect.DeepEqual(p, want) { + t.Fatalf("got %+v\nwant %+v", p, want) + } + if p.Key() != "2.1|53158" { + t.Errorf("Key = %q", p.Key()) + } + other := in + other.ChannelNo = "1002" + other.ID = "5315899" + if ConvertLineupPosition(other).Key() == p.Key() { + t.Error("same station at a different number must be a distinct position") + } +} + +func TestChannelNumberLess(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"2.1", "10", true}, + {"10", "2.1", false}, + {"10", "100", true}, + {"9", "10", true}, + {"2.1", "2.10", true}, // equal numerically, string order breaks the tie + {"2.10", "2.1", false}, + {"5", "5", false}, + {"12", "ABC", true}, // numeric before non-numeric + {"ABC", "12", false}, + {"ABC", "ABD", true}, + } + for _, c := range cases { + if got := ChannelNumberLess(c.a, c.b); got != c.want { + t.Errorf("ChannelNumberLess(%q, %q) = %v, want %v", c.a, c.b, got, c.want) + } + } +} + +func TestSortLineup(t *testing.T) { + positions := []LineupPosition{ + {ChannelNo: "100", StationID: "b"}, + {ChannelNo: "ZZ", StationID: "z"}, + {ChannelNo: "2.1", StationID: "a"}, + {ChannelNo: "10", StationID: "c"}, + {ChannelNo: "10", StationID: "a"}, + {ChannelNo: "9", StationID: "d"}, + } + SortLineup(positions) + var got []string + for _, p := range positions { + got = append(got, p.ChannelNo+"/"+p.StationID) + } + want := []string{"2.1/a", "9/d", "10/a", "10/c", "100/b", "ZZ/z"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("order = %v, want %v", got, want) + } +} + +func TestSourceFromPreferences(t *testing.T) { + at := time.Date(2026, 9, 13, 0, 0, 0, 0, time.UTC) + src := SourceFromPreferences(web.Preferences{ + Country: "USA", ZipCode: "13490", Headend: "lineupId", LineupId: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us", + }, at) + want := Source{Country: "USA", PostalCode: "13490", HeadendID: "lineupId", LineupID: "USA-lineupId-DEFAULT", Device: "-", Language: "en-us", GeneratedAt: at} + if src != want { + t.Fatalf("got %+v want %+v", src, want) + } +} diff --git a/lineup_test.go b/lineup_test.go new file mode 100644 index 0000000..2a22296 --- /dev/null +++ b/lineup_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "net/url" + "reflect" + "testing" + + "github.com/daniel-widrick/GraceNoteScraper/guide" +) + +func sampleLineupGuide() *guide.TVGuide { + return &guide.TVGuide{ + Channels: []guide.Channel{ + {ID: "s1", ChannelNo: "2", IconURL: "http://logos.example/s1.png", DisplayNames: []guide.DisplayName{{Name: "2 AAA"}, {Name: "2"}, {Name: "AAA"}}}, + {ID: "s2", ChannelNo: "5", IconURL: "", DisplayNames: []guide.DisplayName{{Name: "5 BBB"}, {Name: "5"}, {Name: "BBB"}}}, + }, + Programs: []guide.Program{ + {Channel: "s1", Title: "one", IconSrc: "http://img.example/a.jpg", Images: []guide.Image{{URL: "http://img.example/b.jpg"}}}, + {Channel: "s2", Title: "two"}, + }, + Lineup: []guide.LineupPosition{ + {ChannelNo: "2", StationID: "s1"}, + {ChannelNo: "5", StationID: "s2"}, + {ChannelNo: "1002", StationID: "s1"}, // same station, second position + }, + Source: guide.Source{LineupID: "L"}, + } +} + +func TestPropagateChannelLogosCopiesByStation(t *testing.T) { + g := sampleLineupGuide() + propagateChannelLogos(g.Channels, g.Lineup) + + if g.Lineup[0].LogoURL != "http://logos.example/s1.png" || g.Lineup[2].LogoURL != "http://logos.example/s1.png" { + t.Fatalf("both positions of s1 should carry its logo: %+v", g.Lineup) + } + if g.Lineup[1].LogoURL != "" { + t.Fatalf("s2 has no logo, got %q", g.Lineup[1].LogoURL) + } +} + +func TestRewriteImageURLsCoversLineup(t *testing.T) { + g := sampleLineupGuide() + propagateChannelLogos(g.Channels, g.Lineup) + rewriteImageURLs("http://host:8080/", g.Channels, g.Lineup, g.Programs) + + want := "http://host:8080/img?url=" + url.QueryEscape("http://logos.example/s1.png") + if g.Channels[0].IconURL != want { + t.Errorf("channel icon = %q, want %q", g.Channels[0].IconURL, want) + } + if g.Lineup[0].LogoURL != want || g.Lineup[2].LogoURL != want { + t.Errorf("lineup logos = %q / %q, want %q", g.Lineup[0].LogoURL, g.Lineup[2].LogoURL, want) + } + if g.Lineup[1].LogoURL != "" || g.Channels[1].IconURL != "" { + t.Error("empty URLs must stay empty") + } + if g.Programs[0].IconSrc != "http://host:8080/img?url="+url.QueryEscape("http://img.example/a.jpg") { + t.Errorf("program icon = %q", g.Programs[0].IconSrc) + } + if g.Programs[0].Images[0].URL != "http://host:8080/img?url="+url.QueryEscape("http://img.example/b.jpg") { + t.Errorf("program image = %q", g.Programs[0].Images[0].URL) + } +} + +func TestFilterGuideChannelsFiltersLineup(t *testing.T) { + g := sampleLineupGuide() + filtered := filterGuideChannels(g, map[string]bool{"2": true, "1002": true}) + + if len(filtered.Channels) != 1 || filtered.Channels[0].ID != "s1" { + t.Fatalf("channels = %+v", filtered.Channels) + } + if len(filtered.Programs) != 1 || filtered.Programs[0].Title != "one" { + t.Fatalf("programs = %+v", filtered.Programs) + } + var numbers []string + for _, p := range filtered.Lineup { + numbers = append(numbers, p.ChannelNo) + } + if !reflect.DeepEqual(numbers, []string{"2", "1002"}) { + t.Fatalf("lineup numbers = %v", numbers) + } + if filtered.Source != g.Source { + t.Fatal("source not carried through filter") + } +} diff --git a/main.go b/main.go index 156e69b..8f9424f 100644 --- a/main.go +++ b/main.go @@ -145,7 +145,7 @@ func guideToJSON(g *guide.TVGuide) APIGuide { // Sort channels by number (numeric sort) sort.Slice(channels, func(i, j int) bool { - return channelNumberLess(channels[i].Number, channels[j].Number) + return guide.ChannelNumberLess(channels[i].Number, channels[j].Number) }) return APIGuide{ @@ -154,18 +154,6 @@ func guideToJSON(g *guide.TVGuide) APIGuide { } } -// channelNumberLess compares channel numbers numerically where possible. -func channelNumberLess(a, b string) bool { - // Try to parse as float for numeric comparison (handles "5.1", "12", etc.) - var ai, bi float64 - _, errA := fmt.Sscanf(a, "%f", &ai) - _, errB := fmt.Sscanf(b, "%f", &bi) - if errA == nil && errB == nil { - return ai < bi - } - return a < b -} - // xmltvTimeToISO converts "20250225200000 +0000" → "2025-02-25T20:00:00Z" func xmltvTimeToISO(xmltvTime string) string { xmltvTime = strings.TrimSpace(xmltvTime) @@ -214,6 +202,7 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch endTime := midnight.Add(14 * 24 * time.Hour) channelMap := make(map[string]guide.Channel) + lineupMap := make(map[string]guide.LineupPosition) eventMap := make(map[string]bool) var programs []guide.Program @@ -238,6 +227,10 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch if _, exists := channelMap[ch.ChannelID]; !exists { channelMap[ch.ChannelID] = guide.ConvertChannel(ch) } + position := guide.ConvertLineupPosition(ch) + if _, exists := lineupMap[position.Key()]; !exists { + lineupMap[position.Key()] = position + } for _, ev := range ch.Events { dedupKey := ch.ChannelID + "|" + ev.StartTime + "|" + ev.EndTime @@ -261,6 +254,11 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch for _, ch := range channelMap { channels = append(channels, ch) } + lineup := make([]guide.LineupPosition, 0, len(lineupMap)) + for _, position := range lineupMap { + lineup = append(lineup, position) + } + guide.SortLineup(lineup) logoClient := tvlogo.NewClient(pref.Country, "tvlogo_cache.json") if logoClient != nil { @@ -268,6 +266,7 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch } report(scrapeProgressUpdate{Stage: "logos", Message: "Matching channel logos", Channels: len(channels), Programs: len(programs)}) enrichChannelIcons(logoClient, channels) + propagateChannelLogos(channels, lineup) enrichProgramThumbnails(tmdbClient, programs, func(completed, total int) { report(scrapeProgressUpdate{Stage: "tmdb", Message: fmt.Sprintf("Enriching program titles (%d of %d)", completed, total), Completed: completed, Total: total, Channels: len(channels), Programs: len(programs)}) }) @@ -275,28 +274,15 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch // Rewrite image URLs to go through the local proxy if baseURL != "" { - proxy := strings.TrimRight(baseURL, "/") + "/img?url=" - for i := range channels { - if channels[i].IconURL != "" { - channels[i].IconURL = proxy + neturl.QueryEscape(channels[i].IconURL) - } - } - for i := range programs { - if programs[i].IconSrc != "" { - programs[i].IconSrc = proxy + neturl.QueryEscape(programs[i].IconSrc) - } - for j := range programs[i].Images { - if programs[i].Images[j].URL != "" { - programs[i].Images[j].URL = proxy + neturl.QueryEscape(programs[i].Images[j].URL) - } - } - } + rewriteImageURLs(baseURL, channels, lineup, programs) log.Printf("Rewrote image URLs with base %s", baseURL) } tvGuide := &guide.TVGuide{ Channels: channels, Programs: programs, + Lineup: lineup, + Source: guide.SourceFromPreferences(pref, time.Now().UTC()), } if channelFilter != nil { @@ -974,9 +960,18 @@ func filterGuideChannels(g *guide.TVGuide, allowed map[string]bool) *guide.TVGui } } + var lineup []guide.LineupPosition + for _, position := range g.Lineup { + if allowed[position.ChannelNo] { + lineup = append(lineup, position) + } + } + return &guide.TVGuide{ Channels: channels, Programs: programs, + Lineup: lineup, + Source: g.Source, } } @@ -1289,6 +1284,43 @@ func fixDeadImageURLs(programs []guide.Program) { // resolves channel logos from the tv-logo/tv-logos repo, // replacing dead Gracenote icon URLs with verified GitHub-hosted PNGs. +// rewriteImageURLs routes every image URL through the local /img proxy. +func rewriteImageURLs(baseURL string, channels []guide.Channel, lineup []guide.LineupPosition, programs []guide.Program) { + proxy := strings.TrimRight(baseURL, "/") + "/img?url=" + rewrite := func(u string) string { + if u == "" { + return "" + } + return proxy + neturl.QueryEscape(u) + } + for i := range channels { + channels[i].IconURL = rewrite(channels[i].IconURL) + } + for i := range lineup { + lineup[i].LogoURL = rewrite(lineup[i].LogoURL) + } + for i := range programs { + programs[i].IconSrc = rewrite(programs[i].IconSrc) + for j := range programs[i].Images { + programs[i].Images[j].URL = rewrite(programs[i].Images[j].URL) + } + } +} + +// propagateChannelLogos copies each station's resolved icon onto every lineup +// position carrying that station, so logo resolution stays keyed by station. +func propagateChannelLogos(channels []guide.Channel, lineup []guide.LineupPosition) { + byStation := make(map[string]string, len(channels)) + for _, ch := range channels { + byStation[ch.ID] = ch.IconURL + } + for i := range lineup { + if logo, ok := byStation[lineup[i].StationID]; ok { + lineup[i].LogoURL = logo + } + } +} + func enrichChannelIcons(client *tvlogo.Client, channels []guide.Channel) { if client == nil { return