Keep raw Gracenote fields on channels and programs

.gitignore

@@ -2,6 +2,7 @@
 .env
 config.json
 *.xmltv
+!testdata/xmlguide_golden.xmltv
 tmdb_cache.json
 tvlogo_cache.json
 guide_cache.json

guide/guide.go

@@ -15,12 +15,15 @@ type TVGuide struct {
 }
 
 type Channel struct {
-	ID           string
-	DisplayNames []DisplayName
-	IconURL      string
-	CallSign     string // internal, not in template
-	Affiliate    string // internal, not in template
-	ChannelNo    string // internal, not in template
+	ID                string
+	DisplayNames      []DisplayName
+	IconURL           string
+	CallSign          string   // internal, not in template
+	Affiliate         string   // internal, not in template
+	ChannelNo         string   // internal, not in template
+	PlacementID       string   // internal, not in template; Gracenote row id, not a stable key
+	AffiliateCallSign string   // internal, not in template
+	Filters           []string // internal, not in template; Gracenote station filters, prefix stripped
 }
 
 type DisplayName struct {
@@ -45,6 +48,10 @@ type Program struct {
 	Country         string
 	EpisodeNumbers  []EpisodeNumber
 	Categories      []Category
+	Filters         []string // internal, not in template; raw Gracenote event filters, prefix stripped
+	TMSID           string   // internal, not in template
+	ReleaseYear     string   // internal, not in template
+	Generic         bool     // internal, not in template
 	New             bool
 	Premiere        bool
 	PreviouslyShown bool
@@ -119,13 +126,37 @@ func ConvertChannel(ch web.JSONChannel) Channel {
 			{Name: xmlEscape(ch.CallSign)},
 			{Name: xmlEscape(titleCase(ch.AffiliateName))},
 		},
-		IconURL:   iconURL,
-		CallSign:  ch.CallSign,
-		Affiliate: ch.AffiliateName,
-		ChannelNo: ch.ChannelNo,
+		IconURL:           iconURL,
+		CallSign:          ch.CallSign,
+		Affiliate:         ch.AffiliateName,
+		ChannelNo:         ch.ChannelNo,
+		PlacementID:       ch.ID,
+		AffiliateCallSign: normalizeNull(ch.AffiliateCallSign),
+		Filters:           stripFilterPrefixes(ch.StationFilters),
 	}
 }
 
+// normalizeNull maps Gracenote's literal "null" string to an empty value.
+func normalizeNull(s string) string {
+	if strings.EqualFold(strings.TrimSpace(s), "null") {
+		return ""
+	}
+	return s
+}
+
+// stripFilterPrefixes turns Gracenote filter tags such as "filter-sports" into
+// "sports". A nil input stays nil so callers can distinguish absent from empty.
+func stripFilterPrefixes(filters []string) []string {
+	if filters == nil {
+		return nil
+	}
+	out := make([]string, 0, len(filters))
+	for _, f := range filters {
+		out = append(out, strings.TrimPrefix(f, "filter-"))
+	}
+	return out
+}
+
 // converts a JSON event to a template Program struct.
 func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program {
 	season := 0
@@ -163,10 +194,13 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program {
 	// URL
 	programURL := "https://tvlistings.gracenote.com//overview.html?programSeriesId=" + ev.SeriesID + "&tmsId=" + ev.Program.ID
 
+	// Raw Gracenote filters, kept separately so consumers can tell them apart
+	// from the Series and Finale labels added below.
+	filters := stripFilterPrefixes(ev.Filter)
+
 	// Categories from filter array (strip "filter-" prefix)
 	var categories []Category
-	for _, f := range ev.Filter {
-		name := strings.TrimPrefix(f, "filter-")
+	for _, name := range filters {
 		categories = append(categories, Category{Name: name, Lang: lang})
 	}
 
@@ -266,6 +300,10 @@ func ConvertEvent(ev web.JSONEvent, channelID, lang, country string) Program {
 		Country:         country,
 		EpisodeNumbers:  episodeNumbers,
 		Categories:      categories,
+		Filters:         filters,
+		TMSID:           ev.Program.TmsID,
+		ReleaseYear:     string(ev.Program.ReleaseYear),
+		Generic:         bool(ev.Program.IsGeneric),
 		New:             isNew,
 		Premiere:        isPremiere,
 		PreviouslyShown: !isNew,

guide/guide_test.go

@@ -0,0 +1,115 @@
+package guide
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/daniel-widrick/GraceNoteScraper/web"
+)
+
+func sampleChannel() web.JSONChannel {
+	return web.JSONChannel{
+		ChannelID:         "53158",
+		ID:                "531580",
+		ChannelNo:         "2.1",
+		CallSign:          "WKTVDT",
+		AffiliateName:     "NATIONAL BROADCASTING COMPANY",
+		AffiliateCallSign: "null",
+		StationFilters:    []string{"filter-sports", "filter-news"},
+		Thumbnail:         "//images.example.invalid/station/53158.png?w=55",
+	}
+}
+
+func TestConvertChannelCarriesRawFields(t *testing.T) {
+	ch := ConvertChannel(sampleChannel())
+
+	if ch.ID != "53158" || ch.ChannelNo != "2.1" || ch.CallSign != "WKTVDT" {
+		t.Fatalf("basic fields wrong: %+v", ch)
+	}
+	if ch.PlacementID != "531580" {
+		t.Errorf("PlacementID = %q", ch.PlacementID)
+	}
+	if ch.AffiliateCallSign != "" {
+		t.Errorf(`"null" affiliate callsign should normalize to empty, got %q`, ch.AffiliateCallSign)
+	}
+	if !reflect.DeepEqual(ch.Filters, []string{"sports", "news"}) {
+		t.Errorf("Filters = %v", ch.Filters)
+	}
+	if ch.IconURL != "http://images.example.invalid/station/53158.png" {
+		t.Errorf("IconURL = %q", ch.IconURL)
+	}
+	// Existing XMLTV-facing fields must be untouched by the additions.
+	wantNames := []DisplayName{{"2.1 WKTVDT"}, {"2.1"}, {"WKTVDT"}, {"NATIONAL BROADCASTING COMPANY"}}
+	if !reflect.DeepEqual(ch.DisplayNames, wantNames) {
+		t.Errorf("DisplayNames = %v", ch.DisplayNames)
+	}
+}
+
+func TestConvertChannelKeepsRealAffiliateCallSign(t *testing.T) {
+	in := sampleChannel()
+	in.AffiliateCallSign = "NBC"
+	in.StationFilters = nil
+	ch := ConvertChannel(in)
+	if ch.AffiliateCallSign != "NBC" {
+		t.Errorf("AffiliateCallSign = %q", ch.AffiliateCallSign)
+	}
+	if ch.Filters != nil {
+		t.Errorf("nil station filters should stay nil, got %v", ch.Filters)
+	}
+}
+
+func sampleEvent() web.JSONEvent {
+	season, episode, title := "3", "7", "The One"
+	return web.JSONEvent{
+		StartTime: "2026-07-25T07:00:00Z",
+		EndTime:   "2026-07-25T08:00:00Z",
+		Duration:  "60",
+		SeriesID:  "SH06270099",
+		Flag:      []string{"New", "Finale"},
+		Filter:    []string{"filter-sports"},
+		Program: web.JSONProgram{
+			ID:           "EP062700990343",
+			TmsID:        "EP062700990343",
+			Title:        "Morning Business",
+			EpisodeTitle: &title,
+			Season:       &season,
+			Episode:      &episode,
+			ReleaseYear:  "2019",
+			IsGeneric:    true,
+		},
+	}
+}
+
+func TestConvertEventSeparatesRawFiltersFromCategories(t *testing.T) {
+	p := ConvertEvent(sampleEvent(), "53158", "en-us", "USA")
+
+	if !reflect.DeepEqual(p.Filters, []string{"sports"}) {
+		t.Errorf("Filters = %v", p.Filters)
+	}
+	// Categories keep the historical shape: raw filters, then Series (from an
+	// episode number), then Finale (from the flag).
+	want := []Category{{"sports", "en-us"}, {"Series", "en-us"}, {"Finale", "en-us"}}
+	if !reflect.DeepEqual(p.Categories, want) {
+		t.Errorf("Categories = %v, want %v", p.Categories, want)
+	}
+	if p.TMSID != "EP062700990343" || p.ReleaseYear != "2019" || !p.Generic {
+		t.Errorf("program metadata = tms %q year %q generic %v", p.TMSID, p.ReleaseYear, p.Generic)
+	}
+	if !p.New || p.PreviouslyShown {
+		t.Errorf("flag handling changed: new=%v previouslyShown=%v", p.New, p.PreviouslyShown)
+	}
+}
+
+func TestConvertEventWithoutFilters(t *testing.T) {
+	ev := sampleEvent()
+	ev.Filter = nil
+	ev.Flag = nil
+	ev.Program.Season, ev.Program.Episode = nil, nil
+	p := ConvertEvent(ev, "53158", "en", "USA")
+	if p.Filters != nil {
+		t.Errorf("Filters should be nil, got %v", p.Filters)
+	}
+	if len(p.Categories) != 0 {
+		t.Errorf("Categories should be empty, got %v", p.Categories)
+	}
+}

main.go

@@ -320,14 +320,20 @@ func runScrape(pref web.Preferences, tmdbClient *tmdb.Client, baseURL string, ch
 	return tvGuide, nil
 }
 
-func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error {
-	log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs))
-
-	// Parse embedded template
+// renderXMLTV writes the guide as XMLTV using the embedded template.
+func renderXMLTV(w io.Writer, tvGuide *guide.TVGuide) error {
 	tmpl, err := template.ParseFS(guideTmplFS, "guide.tmpl")
 	if err != nil {
 		return fmt.Errorf("failed to parse template: %w", err)
 	}
+	if err := tmpl.Execute(w, tvGuide); err != nil {
+		return fmt.Errorf("failed to execute template: %w", err)
+	}
+	return nil
+}
+
+func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error {
+	log.Printf("Rendering XMLTV: %d channels, %d programs", len(tvGuide.Channels), len(tvGuide.Programs))
 
 	// Atomic write: write to temp file, then rename
 	tmpFile, err := os.CreateTemp(".", "xmlguide-*.tmp")
@@ -336,10 +342,10 @@ func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error {
 	}
 	tmpName := tmpFile.Name()
 
-	if err := tmpl.Execute(tmpFile, tvGuide); err != nil {
+	if err := renderXMLTV(tmpFile, tvGuide); err != nil {
 		tmpFile.Close()
 		os.Remove(tmpName)
-		return fmt.Errorf("failed to execute template: %w", err)
+		return err
 	}
 	if err := tmpFile.Close(); err != nil {
 		os.Remove(tmpName)
@@ -363,15 +369,21 @@ func persistGuideFiles(tvGuide *guide.TVGuide, sourceFingerprint string) error {
 
 const guideCachePath = "guide_cache.json"
 
+// guideCacheVersion is bumped whenever the persisted guide shape gains fields
+// that a scrape must populate. An older cache is rebuilt rather than served
+// with empty fields.
+const guideCacheVersion = 2
+
 type guideCache struct {
+	Version           int           `json:"version"`
 	SavedAt           time.Time     `json:"saved_at"`
 	SourceFingerprint string        `json:"source_fingerprint"`
 	Guide             guide.TVGuide `json:"guide"`
 }
 
 // saveGuideCache persists the TVGuide to a JSON file.
 func saveGuideCache(g *guide.TVGuide, sourceFingerprint string) {
-	data, err := json.Marshal(guideCache{SavedAt: time.Now(), SourceFingerprint: sourceFingerprint, Guide: *g})
+	data, err := json.Marshal(guideCache{Version: guideCacheVersion, SavedAt: time.Now(), SourceFingerprint: sourceFingerprint, Guide: *g})
 	if err != nil {
 		log.Printf("guide cache: failed to marshal: %v", err)
 		return
@@ -395,6 +407,10 @@ func loadGuideCache(maxAge time.Duration, sourceFingerprint string) (*guide.TVGu
 		log.Printf("guide cache: corrupt, ignoring: %v", err)
 		return nil, 0, false
 	}
+	if c.Version != guideCacheVersion {
+		log.Printf("guide cache: schema version %d, want %d; rebuilding", c.Version, guideCacheVersion)
+		return nil, 0, false
+	}
 	if c.SourceFingerprint != sourceFingerprint {
 		log.Println("guide cache: source changed, ignoring cached guide")
 		return nil, 0, false

setup_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -226,3 +227,21 @@ func TestGuideCacheRequiresMatchingSource(t *testing.T) {
 		t.Fatal("cache did not load for matching source")
 	}
 }
+
+func TestGuideCacheRequiresCurrentSchemaVersion(t *testing.T) {
+	t.Chdir(t.TempDir())
+
+	// A cache written by an older build has no version field.
+	legacy := []byte(`{"saved_at":"` + time.Now().UTC().Format(time.RFC3339) + `","source_fingerprint":"src","guide":{"Channels":[],"Programs":[]}}`)
+	if err := os.WriteFile(guideCachePath, legacy, 0644); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, ok := loadGuideCache(time.Hour, "src"); ok {
+		t.Fatal("legacy cache without a version loaded")
+	}
+
+	saveGuideCache(&guide.TVGuide{}, "src")
+	if _, _, ok := loadGuideCache(time.Hour, "src"); !ok {
+		t.Fatal("current-version cache did not load")
+	}
+}