From aa0f4b3704516c0639e84ee84fb5d60e0b8d4cee Mon Sep 17 00:00:00 2001 From: Andreas Bielawski Date: Fri, 29 May 2026 13:42:40 +0200 Subject: [PATCH 01/49] FRB improvements According to https://rachelbythebay.com/frb/ --- .env.example | 7 + config/config.go | 51 +++ fetcher/fetcher.go | 304 ++++++++++++++++++ fetcher/fetcher_test.go | 189 +++++++++++ go.sum | 27 +- handler/feed_check.go | 423 ++++++++++++++++++++----- handler/feed_subscribe.go | 93 +++++- handler/handler.go | 8 +- main.go | 13 +- storage/abonnements.go | 145 ++++++--- storage/migrations/0003-efficiency.sql | 25 ++ 11 files changed, 1118 insertions(+), 167 deletions(-) create mode 100644 fetcher/fetcher.go create mode 100644 fetcher/fetcher_test.go create mode 100644 storage/migrations/0003-efficiency.sql diff --git a/.env.example b/.env.example index 87c066e..de4a3e8 100644 --- a/.env.example +++ b/.env.example @@ -6,3 +6,10 @@ MYSQL_USER=myuser MYSQL_PASSWORD=mypassword MYSQL_DB=mydb MYSQL_SOCKET=/run/mysqld/mysqld.sock + +# Polling behaviour (all optional) +POLL_INTERVAL=10m # base poll interval per feed; can be as low as 1m +POLL_INTERVAL_MAX=24h # upper cap when adaptive slow-down is on +POLL_ADAPTIVE=true # slow down feeds that rarely change (FRB023), set false to disable +POLL_CONCURRENCY=8 # max feeds fetched simultaneously +POLL_TICK=30s # scheduler granularity (how often due feeds are checked) diff --git a/config/config.go b/config/config.go index efd742f..8d18d57 100644 --- a/config/config.go +++ b/config/config.go @@ -2,11 +2,62 @@ package config import ( "os" + "strconv" "text/template" + "time" ) type Config struct { Template *template.Template + Poll PollConfig +} + +// PollConfig holds the feed-polling behaviour. There is intentionally no lower +// bound on PollInterval: polling as often as every minute is a supported +// use-case. Server hints (max-age, Retry-After, ttl) can only slow polling down. +type PollConfig struct { + Interval time.Duration // base poll interval (POLL_INTERVAL) + IntervalMax time.Duration // cap for adaptive slow-down (POLL_INTERVAL_MAX) + Adaptive bool // FRB023: slow feeds that rarely update (POLL_ADAPTIVE) + Concurrency int // max simultaneous fetches (POLL_CONCURRENCY) + Tick time.Duration // scheduler granularity (POLL_TICK) +} + +func GetPollConfig() PollConfig { + return PollConfig{ + Interval: durationEnv("POLL_INTERVAL", 10*time.Minute), + IntervalMax: durationEnv("POLL_INTERVAL_MAX", 24*time.Hour), + Adaptive: boolEnv("POLL_ADAPTIVE", true), + Concurrency: intEnv("POLL_CONCURRENCY", 8), + Tick: durationEnv("POLL_TICK", 30*time.Second), + } +} + +func durationEnv(key string, fallback time.Duration) time.Duration { + if v := os.Getenv(key); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + } + return fallback +} + +func intEnv(key string, fallback int) int { + if v := os.Getenv(key); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + } + return fallback +} + +func boolEnv(key string, fallback bool) bool { + if v := os.Getenv(key); v != "" { + if b, err := strconv.ParseBool(v); err == nil { + return b + } + } + return fallback } func fileExists(fileName string) bool { diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go new file mode 100644 index 0000000..f08f36e --- /dev/null +++ b/fetcher/fetcher.go @@ -0,0 +1,304 @@ +// Package fetcher performs well-behaved HTTP feed retrieval following the +// rachelbythebay Feed Reader Behavior project (FRB) and earth.org.uk +// RSS-efficiency recommendations: conditional GETs, identifying User-Agent, +// compression, server-hint parsing and manual redirect handling. +package fetcher + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "runtime/debug" + "strconv" + "strings" + "time" + + "github.com/mmcdole/gofeed" + ext "github.com/mmcdole/gofeed/extensions" + "github.com/mmcdole/gofeed/rss" +) + +const ( + defaultTimeout = 30 * time.Second + maxRedirects = 5 + maxBodyBytes = 16 << 20 // 16 MiB + bodySnippetLen = 500 + repoURL = "https://github.com/Brawl345/rssbot" +) + +type Fetcher struct { + client *http.Client + userAgent string +} + +// Result is the interpreted outcome of a single feed fetch. ETag and +// LastModified form an atomic set (FRB014) and are only populated from a +// status 200 response. +type Result struct { + Status int + NotModified bool // 304: nothing changed since the conditional request + Feed *gofeed.Feed // only set on 200 with a parseable feed + ETag string // exactly as received (FRB003) + LastModified string // exactly as received (FRB001) + MaxAge time.Duration // Cache-Control: max-age (FRB022), 0 if absent + RetryAfter time.Duration // Retry-After on 429/503 (FRB020/021), 0 if absent + FeedInterval time.Duration // ttl / sy:updatePeriod hint (FRB023/024), 0 if absent + SkipHours []int // RSS skipHours (FRB024) + SkipDays []string // RSS skipDays (FRB024) + PermanentURL string // set on a 301/308 chain (FRB130/131) + Body string // snippet of an error body for the user (FRB101/120) +} + +func New() *Fetcher { + return &Fetcher{ + client: &http.Client{ + Timeout: defaultTimeout, + // Handle redirects manually so permanent (301/308) moves can be + // persisted while temporary (302/307) ones are merely followed. + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + userAgent: buildUserAgent(), + } +} + +// UserAgent returns the identifying User-Agent string sent with every request. +func (f *Fetcher) UserAgent() string { return f.userAgent } + +// Fetch issues a single conditional GET for feedURL. etag and lastModified, if +// non-empty, are sent back verbatim as If-None-Match / If-Modified-Since. +func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string) (*Result, error) { + currentURL := feedURL + permanentPrefix := true + var permanentURL string + + for hop := 0; ; hop++ { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", f.userAgent) + // Conditional headers only apply to the originally requested resource; + // once we follow a redirect we no longer have cache values for it. + if hop == 0 { + if etag != "" { + req.Header.Set("If-None-Match", etag) + } + if lastModified != "" { + req.Header.Set("If-Modified-Since", lastModified) + } + } + + resp, err := f.client.Do(req) + if err != nil { + return nil, err + } + + if isRedirect(resp.StatusCode) { + location := resp.Header.Get("Location") + resp.Body.Close() + if location == "" { + return nil, fmt.Errorf("redirect status %d without Location header", resp.StatusCode) + } + if hop >= maxRedirects { + return nil, fmt.Errorf("too many redirects (>%d)", maxRedirects) + } + + next, err := resolveLocation(currentURL, location) + if err != nil { + return nil, err + } + if (resp.StatusCode == http.StatusMovedPermanently || + resp.StatusCode == http.StatusPermanentRedirect) && permanentPrefix { + permanentURL = next + } else { + permanentPrefix = false + } + currentURL = next + continue + } + + result := &Result{Status: resp.StatusCode} + if permanentURL != "" && permanentURL != feedURL { + result.PermanentURL = permanentURL + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) + resp.Body.Close() + if err != nil { + return nil, err + } + + switch resp.StatusCode { + case http.StatusNotModified: + result.NotModified = true + result.MaxAge = parseMaxAge(resp.Header.Get("Cache-Control")) + case http.StatusOK: + feed, err := gofeed.NewParser().Parse(bytes.NewReader(body)) + if err != nil { + result.Body = snippet(body) + return result, fmt.Errorf("not a valid feed: %w", err) + } + result.Feed = feed + result.ETag = resp.Header.Get("Etag") + result.LastModified = resp.Header.Get("Last-Modified") + result.MaxAge = parseMaxAge(resp.Header.Get("Cache-Control")) + applyFeedHints(result, body) + case http.StatusTooManyRequests, http.StatusServiceUnavailable: + result.RetryAfter = parseRetryAfter(resp.Header.Get("Retry-After")) + result.Body = snippet(body) + default: + result.Body = snippet(body) + } + + return result, nil + } +} + +func isRedirect(status int) bool { + switch status { + case http.StatusMovedPermanently, http.StatusFound, + http.StatusTemporaryRedirect, http.StatusPermanentRedirect: + return true + } + return false +} + +func resolveLocation(base, location string) (string, error) { + b, err := url.Parse(base) + if err != nil { + return "", err + } + l, err := url.Parse(location) + if err != nil { + return "", err + } + return b.ResolveReference(l).String(), nil +} + +func parseMaxAge(cacheControl string) time.Duration { + for _, part := range strings.Split(cacheControl, ",") { + part = strings.TrimSpace(part) + if v, ok := strings.CutPrefix(part, "max-age="); ok { + if secs, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && secs > 0 { + return time.Duration(secs) * time.Second + } + } + } + return 0 +} + +// parseRetryAfter accepts either delta-seconds or an HTTP-date (RFC 7231). +func parseRetryAfter(value string) time.Duration { + value = strings.TrimSpace(value) + if value == "" { + return 0 + } + if secs, err := strconv.Atoi(value); err == nil { + if secs < 0 { + return 0 + } + return time.Duration(secs) * time.Second + } + if t, err := http.ParseTime(value); err == nil { + if d := time.Until(t); d > 0 { + return d + } + } + return 0 +} + +// applyFeedHints extracts RSS polling hints (ttl, skipHours, skipDays, +// sy:updatePeriod/updateFrequency) from the raw body when it is an RSS feed. +func applyFeedHints(result *Result, body []byte) { + feedType := gofeed.DetectFeedType(bytes.NewReader(body)) + if feedType != gofeed.FeedTypeRSS { + return + } + rssFeed, err := (&rss.Parser{}).Parse(bytes.NewReader(body)) + if err != nil { + return + } + + for _, h := range rssFeed.SkipHours { + if n, err := strconv.Atoi(strings.TrimSpace(h)); err == nil && n >= 0 && n <= 23 { + result.SkipHours = append(result.SkipHours, n) + } + } + result.SkipDays = rssFeed.SkipDays + + if ttl := strings.TrimSpace(rssFeed.TTL); ttl != "" { + if mins, err := strconv.Atoi(ttl); err == nil && mins > 0 { + result.FeedInterval = time.Duration(mins) * time.Minute + } + } + + if sy := syInterval(rssFeed.Extensions); sy > result.FeedInterval { + result.FeedInterval = sy + } +} + +// syInterval derives a polling interval from the syndication module +// (sy:updatePeriod / sy:updateFrequency). Defaults follow the RSS 1.0 +// syndication spec: daily period, frequency of 1. +func syInterval(extensions ext.Extensions) time.Duration { + sy, ok := extensions["sy"] + if !ok { + return 0 + } + + period := 24 * time.Hour // "daily" default + if vals := sy["updatePeriod"]; len(vals) > 0 { + switch strings.ToLower(strings.TrimSpace(vals[0].Value)) { + case "hourly": + period = time.Hour + case "daily": + period = 24 * time.Hour + case "weekly": + period = 7 * 24 * time.Hour + case "monthly": + period = 30 * 24 * time.Hour + case "yearly": + period = 365 * 24 * time.Hour + } + } else { + return 0 // no syndication info present + } + + frequency := 1 + if vals := sy["updateFrequency"]; len(vals) > 0 { + if n, err := strconv.Atoi(strings.TrimSpace(vals[0].Value)); err == nil && n > 0 { + frequency = n + } + } + + return period / time.Duration(frequency) +} + +func snippet(body []byte) string { + s := strings.TrimSpace(string(body)) + if len(s) > bodySnippetLen { + return s[:bodySnippetLen] + "…" + } + return s +} + +func buildUserAgent() string { + version := "dev" + if info, ok := debug.ReadBuildInfo(); ok { + for _, setting := range info.Settings { + if setting.Key == "vcs.revision" && setting.Value != "" { + version = setting.Value + if len(version) > 7 { + version = version[:7] + } + break + } + } + } + return fmt.Sprintf("rssbot/%s (+%s)", version, repoURL) +} diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go new file mode 100644 index 0000000..82a315d --- /dev/null +++ b/fetcher/fetcher_test.go @@ -0,0 +1,189 @@ +package fetcher + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +const sampleRSS = ` + + + Test + https://example.org + 90 + 023 + Sunday + hourly + 2 + Item 1https://example.org/11 + +` + +func fetch(t *testing.T, f *Fetcher, url, etag, lm string) *Result { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + res, err := f.Fetch(ctx, url, etag, lm) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + return res +} + +func TestFetchSetsHeadersAndParses(t *testing.T) { + var gotMethod, gotUA, gotINM, gotIMS, gotAE string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotUA = r.Header.Get("User-Agent") + gotINM = r.Header.Get("If-None-Match") + gotIMS = r.Header.Get("If-Modified-Since") + gotAE = r.Header.Get("Accept-Encoding") + w.Header().Set("Etag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") + w.Header().Set("Cache-Control", "max-age=1800") + w.Write([]byte(sampleRSS)) + })) + defer srv.Close() + + f := New() + res := fetch(t, f, srv.URL, "", "") + + if gotMethod != http.MethodGet { + t.Errorf("method = %s, want GET (FRB050)", gotMethod) + } + if !strings.HasPrefix(gotUA, "rssbot/") || !strings.Contains(gotUA, "github.com/Brawl345/rssbot") { + t.Errorf("User-Agent = %q, want identifying rssbot UA (FRB080-090)", gotUA) + } + if gotINM != "" || gotIMS != "" { + t.Errorf("first request must be unconditional, got INM=%q IMS=%q (FRB012/013)", gotINM, gotIMS) + } + if !strings.Contains(gotAE, "gzip") { + t.Errorf("Accept-Encoding = %q, want gzip (FRB141)", gotAE) + } + if res.Status != 200 || res.Feed == nil || len(res.Feed.Items) != 1 { + t.Fatalf("unexpected result: status=%d feed=%v", res.Status, res.Feed) + } + if res.ETag != `"abc"` { + t.Errorf("ETag = %q, want %q stored verbatim (FRB003)", res.ETag, `"abc"`) + } + if res.LastModified != "Wed, 21 Oct 2015 07:28:00 GMT" { + t.Errorf("LastModified = %q, not stored verbatim (FRB001)", res.LastModified) + } + if res.MaxAge != 30*time.Minute { + t.Errorf("MaxAge = %s, want 30m (FRB022)", res.MaxAge) + } + // ttl=90m beats sy hourly/2 (=30m); both beat nothing. + if res.FeedInterval != 90*time.Minute { + t.Errorf("FeedInterval = %s, want 90m from ttl (FRB024)", res.FeedInterval) + } + if len(res.SkipHours) != 2 || res.SkipHours[0] != 0 || res.SkipHours[1] != 23 { + t.Errorf("SkipHours = %v, want [0 23] (FRB024)", res.SkipHours) + } + if len(res.SkipDays) != 1 || res.SkipDays[0] != "Sunday" { + t.Errorf("SkipDays = %v, want [Sunday] (FRB024)", res.SkipDays) + } +} + +func TestConditionalRequestEchoed(t *testing.T) { + var gotINM, gotIMS string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotINM = r.Header.Get("If-None-Match") + gotIMS = r.Header.Get("If-Modified-Since") + w.WriteHeader(http.StatusNotModified) + })) + defer srv.Close() + + f := New() + res := fetch(t, f, srv.URL, `"abc"`, "Wed, 21 Oct 2015 07:28:00 GMT") + + if gotINM != `"abc"` { + t.Errorf("If-None-Match = %q, want verbatim etag (FRB004/013)", gotINM) + } + if gotIMS != "Wed, 21 Oct 2015 07:28:00 GMT" { + t.Errorf("If-Modified-Since = %q, want verbatim last-modified (FRB002/012)", gotIMS) + } + if !res.NotModified || res.Status != http.StatusNotModified { + t.Errorf("expected 304 NotModified, got status=%d notmod=%v", res.Status, res.NotModified) + } +} + +func TestRetryAfterParsed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "120") + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte("slow down")) + })) + defer srv.Close() + + res := fetch(t, New(), srv.URL, "", "") + if res.Status != 429 { + t.Fatalf("status = %d, want 429", res.Status) + } + if res.RetryAfter != 2*time.Minute { + t.Errorf("RetryAfter = %s, want 2m (FRB020)", res.RetryAfter) + } + if res.Body != "slow down" { + t.Errorf("Body = %q, want error snippet (FRB120)", res.Body) + } +} + +func TestPermanentRedirectReported(t *testing.T) { + var target *httptest.Server + target = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(sampleRSS)) + })) + defer target.Close() + + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusMovedPermanently) + })) + defer src.Close() + + res := fetch(t, New(), src.URL, "", "") + if res.PermanentURL != target.URL { + t.Errorf("PermanentURL = %q, want %q (FRB130)", res.PermanentURL, target.URL) + } + if res.Status != 200 || res.Feed == nil { + t.Errorf("expected feed content after following 301, got status=%d", res.Status) + } +} + +func TestTemporaryRedirectNotPersisted(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(sampleRSS)) + })) + defer target.Close() + + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) // 302 + })) + defer src.Close() + + res := fetch(t, New(), src.URL, "", "") + if res.PermanentURL != "" { + t.Errorf("PermanentURL = %q, want empty for 302 (FRB133)", res.PermanentURL) + } + if res.Status != 200 || res.Feed == nil { + t.Errorf("expected feed content after following 302, got status=%d", res.Status) + } +} + +func TestNonFeedRejected(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("not a feed")) + })) + defer srv.Close() + + ctx := context.Background() + res, err := New().Fetch(ctx, srv.URL, "", "") + if err == nil { + t.Fatalf("expected error for non-feed body (FRB102)") + } + if res == nil || res.Body == "" { + t.Errorf("expected body snippet on parse failure (FRB101)") + } +} diff --git a/go.sum b/go.sum index 7435dea..5d12532 100644 --- a/go.sum +++ b/go.sum @@ -137,9 +137,6 @@ github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvSc github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= -github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -257,8 +254,6 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -290,9 +285,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -307,9 +300,7 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= @@ -321,14 +312,8 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s= -github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4= github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= -github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI= -github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= -github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= -github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= github.com/mmcdole/goxpp v1.1.1 h1:RGIX+D6iQRIunGHrKqnA2+700XMCnNv0bAOOv5MUhx8= github.com/mmcdole/goxpp v1.1.1/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -377,8 +362,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rubenv/sql-migrate v1.6.0 h1:IZpcTlAx/VKXphWEpwWJ7BaMq05tYtE80zYz+8a5Il8= -github.com/rubenv/sql-migrate v1.6.0/go.mod h1:m3ilnKP7sNb4eYkLsp6cGdPOl4OBcXM6rcbzU+Oqc5k= github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -530,8 +513,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -664,8 +645,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -911,8 +890,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/telebot.v3 v3.2.1 h1:3I4LohaAyJBiivGmkfB+CiVu7QFOWkuZ4+KHgO/G3rs= -gopkg.in/telebot.v3 v3.2.1/go.mod h1:GJKwwWqp9nSkIVN51eRKU78aB5f5OnQuWdwiIZfPbko= gopkg.in/telebot.v3 v3.3.8 h1:uVDGjak9l824FN9YARWUHMsiNZnlohAVwUycw21k6t8= gopkg.in/telebot.v3 v3.3.8/go.mod h1:1mlbqcLTVSfK9dx7fdp+Nb5HZsy4LLPtpZTKmwhwtzM= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handler/feed_check.go b/handler/feed_check.go index bcbcd8a..69477f7 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -2,7 +2,9 @@ package handler import ( "bytes" + "context" "errors" + "fmt" "html" "log" "net/url" @@ -11,10 +13,22 @@ import ( "sync" "time" + "github.com/Brawl345/rssbot/fetcher" "github.com/Brawl345/rssbot/storage" + "github.com/mmcdole/gofeed" "gopkg.in/telebot.v3" ) +const ( + // maxFeedErrors is how many consecutive failures a feed tolerates before it + // is retired (FRB110-119). + maxFeedErrors = 12 + // fetchTimeout bounds a single feed fetch. + fetchTimeout = 35 * time.Second +) + +var feedproxyRe = regexp.MustCompile("^https?://feedproxy.google.com/~r/(.+?)/.*") + type TemplateData struct { Title string FeedTitle string @@ -23,26 +37,37 @@ type TemplateData struct { PostDomain string } +// OnCheck is the scheduler tick. It polls only feeds that are currently due and +// reschedules itself after POLL_TICK. Because due-ness lives in the database, +// a process restart never causes a mass re-fetch (FRB037). func (h *Handler) OnCheck() { - var wg sync.WaitGroup - log.Println("===============================/") - abonnements, err := h.DB.Abonnements.GetAll() + defer time.AfterFunc(h.Config.Poll.Tick, h.OnCheck) + + abonnements, err := h.DB.Abonnements.GetDue() if err != nil { log.Println(err) - time.AfterFunc(1*time.Minute, h.OnCheck) + return + } + if len(abonnements) == 0 { return } replacements, err := h.DB.Replacements.List() if err != nil { log.Println(err) - time.AfterFunc(1*time.Minute, h.OnCheck) return } - if len(abonnements) == 0 { - log.Println("No feeds found, checkin again in 60 seconds") - } + log.Printf("Polling %d due feed(s)", len(abonnements)) + h.pollFeeds(abonnements, replacements) +} + +// pollFeeds fetches the due feeds through a bounded worker pool while +// serializing requests that target the same host (FRB033/034). +func (h *Handler) pollFeeds(abonnements []storage.Abonnement, replacements []storage.Replacement) { + sem := make(chan struct{}, h.Config.Poll.Concurrency) + hosts := &hostLocks{m: make(map[string]*sync.Mutex)} + var wg sync.WaitGroup for _, abonnement := range abonnements { abonnement := abonnement @@ -50,83 +75,325 @@ func (h *Handler) OnCheck() { go func() { defer wg.Done() - log.Printf("%s", abonnement.Feed.Url) + sem <- struct{}{} + defer func() { <-sem }() - var lastEntry *string - if abonnement.LastEntry.Valid { - lastEntry = &abonnement.LastEntry.String - } + lock := hosts.get(feedHost(abonnement.Feed.Url)) + lock.Lock() + defer lock.Unlock() - feed, err := abonnement.Feed.Check(lastEntry) - if err != nil { - log.Printf("%s: %s", abonnement.Feed.Url, err) - return - } + h.pollFeed(abonnement, replacements) + }() + } - for _, entry := range reverse(feed.Items) { - templateData := &TemplateData{} - if entry.Title != "" { - templateData.Title = html.EscapeString(entry.Title) - } else { - templateData.Title = "Kein Titel" - } - - templateData.FeedTitle = html.EscapeString(feed.Title) - - if entry.Content != "" { - templateData.Content = processContent(entry.Content, &replacements) - } else if entry.Description != "" { - templateData.Content = processContent(entry.Description, &replacements) - } - - if entry.Link != "" { - templateData.PostLink = entry.Link - } else { - templateData.PostLink = feed.Link - } - - re := regexp.MustCompile("^https?://feedproxy.google.com/~r/(.+?)/.*") - match := re.FindStringSubmatch(templateData.PostLink) - - if len(match) > 0 { - templateData.PostDomain = match[1] - } else { - parsedUrl, _ := url.Parse(templateData.PostLink) - templateData.PostDomain = parsedUrl.Host - } - - templateData.PostDomain = strings.Replace(templateData.PostDomain, "www.", "", 1) - - var tpl bytes.Buffer - err := h.Config.Template.Execute(&tpl, templateData) - if err != nil { - log.Printf("%s: %s", abonnement.Feed.Url, err) - return - } - - for _, chat := range abonnement.Chats { - err = h.sendText(chat.ID, tpl.String(), abonnement.Feed.Url) - if err != nil { - log.Printf("%s: %s", abonnement.Feed.Url, err) - } - } + wg.Wait() +} +func (h *Handler) pollFeed(abonnement storage.Abonnement, replacements []storage.Replacement) { + feed := abonnement.Feed + + var etag, lastModified string + if feed.ETag.Valid { + etag = feed.ETag.String + } + if feed.LastModified.Valid { + lastModified = feed.LastModified.String + } + + ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) + defer cancel() + + result, err := h.Fetcher.Fetch(ctx, feed.Url, etag, lastModified) + if err != nil { + h.handleSoftError(abonnement, feed, err.Error(), bodyOf(result)) + return + } + + // Permanent move: persist the new URL and stop polling the old one + // (FRB130/131/132). + if result.PermanentURL != "" { + if err := h.DB.Abonnements.SetFeedURL(feed.ID, result.PermanentURL); err != nil { + log.Printf("%s: could not update url to %s: %s", feed.Url, result.PermanentURL, err) + } else { + h.notify(abonnement, fmt.Sprintf("ℹ️ Feed wurde dauerhaft umgezogen:\n%s\n→ %s", + feed.Url, result.PermanentURL)) + feed.Url = result.PermanentURL + } + } + + switch { + case result.Status == 200 && result.Feed != nil: + h.handleOK(abonnement, feed, result, replacements) + case result.NotModified: + next := h.nextPoll(feed.UnchangedCount+1, result) + if err := h.DB.Abonnements.Reschedule(feed.ID, next, 0, feed.UnchangedCount+1); err != nil { + log.Printf("%s: reschedule failed: %s", feed.Url, err) + } + case result.Status == 429 || result.Status == 503: + h.handleRateLimit(abonnement, feed, result) + case result.Status == 410: + h.disable(abonnement, feed, "HTTP 410 Gone", result.Body) + default: + h.handleSoftError(abonnement, feed, fmt.Sprintf("HTTP %d", result.Status), result.Body) + } +} + +func (h *Handler) handleOK(abonnement storage.Abonnement, feed storage.Feed, result *fetcher.Result, replacements []storage.Replacement) { + gfeed := result.Feed + + var lastEntry *string + if feed.LastEntry.Valid { + lastEntry = &feed.LastEntry.String + } + newItems := filterNewItems(gfeed.Items, lastEntry) + + for _, entry := range reverse(newItems) { + templateData := &TemplateData{} + if entry.Title != "" { + templateData.Title = html.EscapeString(entry.Title) + } else { + templateData.Title = "Kein Titel" + } + + templateData.FeedTitle = html.EscapeString(gfeed.Title) + + if entry.Content != "" { + templateData.Content = processContent(entry.Content, &replacements) + } else if entry.Description != "" { + templateData.Content = processContent(entry.Description, &replacements) + } + + if entry.Link != "" { + templateData.PostLink = entry.Link + } else { + templateData.PostLink = gfeed.Link + } + + match := feedproxyRe.FindStringSubmatch(templateData.PostLink) + if len(match) > 0 { + templateData.PostDomain = match[1] + } else { + parsedUrl, _ := url.Parse(templateData.PostLink) + templateData.PostDomain = parsedUrl.Host + } + templateData.PostDomain = strings.Replace(templateData.PostDomain, "www.", "", 1) + + var tpl bytes.Buffer + if err := h.Config.Template.Execute(&tpl, templateData); err != nil { + log.Printf("%s: %s", feed.Url, err) + return + } + + for _, chat := range abonnement.Chats { + if err := h.sendText(chat.ID, tpl.String(), feed.Url); err != nil { + log.Printf("%s: %s", feed.Url, err) } - if len(feed.Items) > 0 { - var lastEntry *string - if feed.Items[0].GUID != "" { - lastEntry = &feed.Items[0].GUID - } else { - lastEntry = &feed.Items[0].Link - } - h.DB.Abonnements.SetLastEntry(abonnement.Feed.Url, lastEntry) - } - }() + } } - wg.Wait() - log.Println("/===============================") - time.AfterFunc(1*time.Minute, h.OnCheck) + newLastEntry := lastEntry + if len(gfeed.Items) > 0 { + if gfeed.Items[0].GUID != "" { + newLastEntry = &gfeed.Items[0].GUID + } else { + newLastEntry = &gfeed.Items[0].Link + } + } + + // FRB023: only reset the adaptive counter when genuinely new content arrived. + unchanged := feed.UnchangedCount + 1 + if len(newItems) > 0 { + unchanged = 0 + } + + next := h.nextPoll(unchanged, result) + err := h.DB.Abonnements.SetFeedState(feed.ID, newLastEntry, + nullableString(result.ETag), nullableString(result.LastModified), + next, 0, unchanged) + if err != nil { + log.Printf("%s: could not save state: %s", feed.Url, err) + } +} + +func (h *Handler) handleRateLimit(abonnement storage.Abonnement, feed storage.Feed, result *fetcher.Result) { + delay := result.RetryAfter + if delay <= 0 { + // FRB021: a 429/503 without a hint is still "slow down". + delay = h.Config.Poll.Interval * 4 + if delay > h.Config.Poll.IntervalMax { + delay = h.Config.Poll.IntervalMax + } + } + + // Keep the cached validators (FRB016) and do not count this toward retirement. + if err := h.DB.Abonnements.Reschedule(feed.ID, time.Now().Add(delay), feed.ErrorCount, feed.UnchangedCount); err != nil { + log.Printf("%s: reschedule failed: %s", feed.Url, err) + } + log.Printf("%s: HTTP %d, backing off %s", feed.Url, result.Status, delay.Round(time.Second)) + + if result.Status == 429 { + h.notify(abonnement, fmt.Sprintf( + "⚠️ Feed sendet \"429 Too Many Requests\":\n%s\nBackoff: %s. Eventuell ist das Poll-Intervall zu kurz.", + feed.Url, delay.Round(time.Second))) + } +} + +// handleSoftError applies exponential backoff and retires the feed once it has +// failed too many times in a row (FRB110-119). +func (h *Handler) handleSoftError(abonnement storage.Abonnement, feed storage.Feed, reason, body string) { + errorCount := feed.ErrorCount + 1 + if errorCount >= maxFeedErrors { + h.disable(abonnement, feed, reason, body) + return + } + + delay := h.errorBackoff(errorCount) + if err := h.DB.Abonnements.Reschedule(feed.ID, time.Now().Add(delay), errorCount, feed.UnchangedCount); err != nil { + log.Printf("%s: reschedule failed: %s", feed.Url, err) + } + log.Printf("%s: %s (error %d/%d), retrying in %s", feed.Url, reason, errorCount, maxFeedErrors, delay.Round(time.Second)) +} + +func (h *Handler) disable(abonnement storage.Abonnement, feed storage.Feed, reason, body string) { + if err := h.DB.Abonnements.DisableFeed(feed.ID, reason); err != nil { + log.Printf("%s: could not disable: %s", feed.Url, err) + return + } + log.Printf("%s: disabled (%s)", feed.Url, reason) + + msg := fmt.Sprintf("🚫 Feed wurde deaktiviert:\n%s\nGrund: %s", feed.Url, reason) + if body != "" { + msg += fmt.Sprintf("\n
%s
", html.EscapeString(body)) + } + h.notify(abonnement, msg) +} + +// nextPoll computes the next poll time: base interval, optionally stretched for +// feeds that rarely change (FRB023), never faster than server hints +// (max-age/ttl, FRB022/024), shifted out of skipHours/skipDays (FRB024). +func (h *Handler) nextPoll(unchangedCount int, result *fetcher.Result) time.Time { + interval := h.Config.Poll.Interval + + if h.Config.Poll.Adaptive && unchangedCount > 0 { + for i := 0; i < unchangedCount && interval < h.Config.Poll.IntervalMax; i++ { + interval *= 2 + } + if interval > h.Config.Poll.IntervalMax { + interval = h.Config.Poll.IntervalMax + } + } + + if result != nil { + if result.MaxAge > interval { + interval = result.MaxAge + } + if result.FeedInterval > interval { + interval = result.FeedInterval + } + } + + next := time.Now().Add(interval) + if result != nil { + next = applySkip(next, result.SkipHours, result.SkipDays) + } + return next +} + +func (h *Handler) errorBackoff(errorCount int) time.Duration { + interval := h.Config.Poll.Interval + for i := 1; i < errorCount && interval < h.Config.Poll.IntervalMax; i++ { + interval *= 2 + } + if interval > h.Config.Poll.IntervalMax { + interval = h.Config.Poll.IntervalMax + } + return interval +} + +func (h *Handler) notify(abonnement storage.Abonnement, text string) { + for _, chat := range abonnement.Chats { + if err := h.sendText(chat.ID, text, abonnement.Feed.Url); err != nil { + log.Printf("%s: notify failed: %s", abonnement.Feed.Url, err) + } + } +} + +// applySkip pushes t forward in whole hours until it is outside the feed's +// declared skipHours/skipDays (RSS uses UTC). Capped so an over-eager feed can +// never block polling forever. +func applySkip(t time.Time, skipHours []int, skipDays []string) time.Time { + if len(skipHours) == 0 && len(skipDays) == 0 { + return t + } + for i := 0; i < 48; i++ { + u := t.UTC() + if containsInt(skipHours, u.Hour()) || containsDay(skipDays, u.Weekday().String()) { + t = t.Add(time.Hour) + continue + } + break + } + return t +} + +func filterNewItems(items []*gofeed.Item, lastEntry *string) []*gofeed.Item { + if lastEntry == nil { + return items + } + for i, item := range items { + if item.GUID == *lastEntry || item.Link == *lastEntry { + return items[:i] + } + } + return items +} + +func feedHost(rawURL string) string { + if u, err := url.Parse(rawURL); err == nil && u.Host != "" { + return strings.ToLower(u.Host) + } + return rawURL +} + +func bodyOf(result *fetcher.Result) string { + if result == nil { + return "" + } + return result.Body +} + +func containsInt(s []int, v int) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} + +func containsDay(days []string, day string) bool { + for _, d := range days { + if strings.EqualFold(strings.TrimSpace(d), day) { + return true + } + } + return false +} + +type hostLocks struct { + mu sync.Mutex + m map[string]*sync.Mutex +} + +func (h *hostLocks) get(host string) *sync.Mutex { + h.mu.Lock() + defer h.mu.Unlock() + if lock, ok := h.m[host]; ok { + return lock + } + lock := &sync.Mutex{} + h.m[host] = lock + return lock } func processContent(content string, replacements *[]storage.Replacement) string { diff --git a/handler/feed_subscribe.go b/handler/feed_subscribe.go index 341c016..6d2c533 100644 --- a/handler/feed_subscribe.go +++ b/handler/feed_subscribe.go @@ -1,9 +1,14 @@ package handler import ( + "context" + "fmt" + "html" "log" + "net/url" + "strings" + "time" - "github.com/mmcdole/gofeed" "gopkg.in/telebot.v3" ) @@ -14,7 +19,12 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { return nil } - feedUrl := args[0] + feedUrl, err := sanitizeFeedURL(args[0]) + if err != nil { + // FRB060/061: report the problem, don't guess at a corrected URL. + return c.Send("❌ Ungültige URL: "+err.Error(), defaultSendOptions) + } + chatId := c.Chat().ID var chatTitle string if c.Message().Private() { @@ -43,19 +53,42 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { chatTitle = chatInfo.Title } - feed, err := gofeed.NewParser().ParseURL(feedUrl) - + // FRB036: a single request gathers everything needed to add the feed. + ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second) + defer cancel() + result, err := h.Fetcher.Fetch(ctx, feedUrl, "", "") if err != nil { - log.Println(err) - return c.Send("❌ Ungültiger Feed", defaultSendOptions) + log.Printf("subscribe %s: %s", feedUrl, err) + if result != nil && result.Body != "" { + // FRB101: surface the server's response so the user can act on it. + return c.Send(fmt.Sprintf("❌ Ungültiger Feed (HTTP %d):\n
%s
", + result.Status, html.EscapeString(result.Body)), defaultSendOptions) + } + return c.Send("❌ Feed konnte nicht abgerufen werden: "+html.EscapeString(err.Error()), defaultSendOptions) + } + + // FRB100: only add feeds that actually answer with 200. + if result.Status != 200 { + msg := fmt.Sprintf("❌ Der Server antwortete mit HTTP %d.", result.Status) + if result.Body != "" { + msg += fmt.Sprintf("\n
%s
", html.EscapeString(result.Body)) + } + return c.Send(msg, defaultSendOptions) + } + + // FRB102: must be a real feed. + feed := result.Feed + if feed == nil { + return c.Send("❌ Diese URL liefert keinen gültigen Feed.", defaultSendOptions) } if feed.FeedLink != "" { - feedUrl = feed.FeedLink + if normalized, err := sanitizeFeedURL(feed.FeedLink); err == nil { + feedUrl = normalized + } } exists, _ := h.DB.Abonnements.ExistsByFeedUrl(chatId, feedUrl) - if exists { return c.Send("✅ Du hast diesen Feed bereits abonniert.", defaultSendOptions) } @@ -69,7 +102,13 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { } } - err = h.DB.Abonnements.Create(chatId, chatTitle, feedUrl, lastEntry) + // Persist the cache validators from this fetch so the very first scheduled + // poll is already conditional. + etag := nullableString(result.ETag) + lastModified := nullableString(result.LastModified) + nextPollAt := time.Now().Add(h.Config.Poll.Interval) + + err = h.DB.Abonnements.Create(chatId, chatTitle, feedUrl, lastEntry, etag, lastModified, nextPollAt) if err != nil { log.Println(err) return c.Send("❌ Beim Abonnieren des Feeds ist ein Fehler aufgetreten.", defaultSendOptions) @@ -77,3 +116,39 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { return c.Send("✅ Der Feed wurde erfolgreich abonniert!", defaultSendOptions) } + +// sanitizeFeedURL trims surrounding whitespace and rejects URLs containing +// characters that should never appear in a real feed URL (FRB060/061). It does +// not rewrite or guess — it only validates. +func sanitizeFeedURL(raw string) (string, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "", fmt.Errorf("leer") + } + + for _, r := range trimmed { + if r <= ' ' || r == '"' || r == '\'' || r == '<' || r == '>' || r == '`' { + return "", fmt.Errorf("enthält ungültige Zeichen") + } + } + + parsed, err := url.Parse(trimmed) + if err != nil { + return "", fmt.Errorf("nicht parsebar") + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", fmt.Errorf("muss mit http:// oder https:// beginnen") + } + if parsed.Host == "" { + return "", fmt.Errorf("kein Host") + } + + return trimmed, nil +} + +func nullableString(s string) *string { + if s == "" { + return nil + } + return &s +} diff --git a/handler/handler.go b/handler/handler.go index 0741a8a..cd45459 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -2,12 +2,14 @@ package handler import ( "github.com/Brawl345/rssbot/config" + "github.com/Brawl345/rssbot/fetcher" "github.com/Brawl345/rssbot/storage" "gopkg.in/telebot.v3" ) type Handler struct { - Bot *telebot.Bot - Config *config.Config - DB *storage.DB + Bot *telebot.Bot + Config *config.Config + DB *storage.DB + Fetcher *fetcher.Fetcher } diff --git a/main.go b/main.go index 3611ee2..c7ad888 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "time" "github.com/Brawl345/rssbot/config" + "github.com/Brawl345/rssbot/fetcher" "github.com/Brawl345/rssbot/handler" _ "github.com/joho/godotenv/autoload" "gopkg.in/telebot.v3" @@ -26,6 +27,7 @@ func main() { cfg := &config.Config{ Template: tmpl, + Poll: config.GetPollConfig(), } db, err := storage.Connect() @@ -56,11 +58,14 @@ func main() { log.Printf("Logged in as @%s (%d)", bot.Me.Username, bot.Me.ID) h := handler.Handler{ - Bot: bot, - Config: cfg, - DB: db, + Bot: bot, + Config: cfg, + DB: db, + Fetcher: fetcher.New(), } + log.Printf("Feed fetcher User-Agent: %s", h.Fetcher.UserAgent()) + adminId, err := strconv.ParseInt(os.Getenv("ADMIN_ID"), 10, 64) if err != nil { @@ -93,7 +98,7 @@ func main() { time.AfterFunc(5*time.Second, h.OnCheck) - channel := make(chan os.Signal) + channel := make(chan os.Signal, 1) signal.Notify(channel, os.Interrupt, syscall.SIGTERM) signal.Notify(channel, os.Interrupt, syscall.SIGKILL) signal.Notify(channel, os.Interrupt, syscall.SIGINT) diff --git a/storage/abonnements.go b/storage/abonnements.go index 2b4d093..d1738f0 100644 --- a/storage/abonnements.go +++ b/storage/abonnements.go @@ -3,20 +3,24 @@ package storage import ( "context" "database/sql" - "github.com/jmoiron/sqlx" - "github.com/mmcdole/gofeed" "time" + + "github.com/jmoiron/sqlx" ) type ( AbonnementStorage interface { - Create(chatId int64, chatTitle string, feedUrl string, lastEntry *string) error + Create(chatId int64, chatTitle string, feedUrl string, lastEntry, etag, lastModified *string, nextPollAt time.Time) error Delete(chatId int64, feedId int64) error ExistsByFeedUrl(chatId int64, feedUrl string) (bool, error) ExistsById(chatId int64, feedId int64) (bool, error) GetByUser(chatId int64) ([]Feed, error) GetAll() ([]Abonnement, error) - SetLastEntry(feedUrl string, lastEntry *string) error + GetDue() ([]Abonnement, error) + SetFeedState(feedID int64, lastEntry, etag, lastModified *string, nextPollAt time.Time, errorCount, unchangedCount int) error + Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error + SetFeedURL(feedID int64, newURL string) error + DisableFeed(feedID int64, reason string) error } Abonnements struct { @@ -35,15 +39,23 @@ type ( } Feed struct { - ID int64 `db:"id"` - Url string `db:"url"` - LastEntry sql.NullString `db:"last_entry"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt sql.NullTime `db:"updated_at"` + ID int64 `db:"id"` + Url string `db:"url"` + LastEntry sql.NullString `db:"last_entry"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt sql.NullTime `db:"updated_at"` + ETag sql.NullString `db:"etag"` + LastModified sql.NullString `db:"last_modified"` + NextPollAt sql.NullTime `db:"next_poll_at"` + LastPollAt sql.NullTime `db:"last_poll_at"` + ErrorCount int `db:"error_count"` + UnchangedCount int `db:"unchanged_count"` + Disabled bool `db:"disabled"` + DisabledReason sql.NullString `db:"disabled_reason"` } ) -func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, lastEntry *string) error { +func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, lastEntry, etag, lastModified *string, nextPollAt time.Time) error { tx, err := db.BeginTxx(context.Background(), nil) if err != nil { return err @@ -57,8 +69,8 @@ func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, la if err != nil { // Feed does not exist yet, will be created - const insertFeedQuery = "INSERT INTO feeds (url, last_entry) VALUES (?, ?)" - result, err := tx.Exec(insertFeedQuery, feedUrl, lastEntry) + const insertFeedQuery = "INSERT INTO feeds (url, last_entry, etag, last_modified, next_poll_at) VALUES (?, ?, ?, ?, ?)" + result, err := tx.Exec(insertFeedQuery, feedUrl, lastEntry, etag, lastModified, nextPollAt) if err != nil { return err } @@ -167,67 +179,104 @@ WHERE chats.id = ?` return feeds, err } -func (db *Abonnements) GetAll() ([]Abonnement, error) { - const query = `SELECT chats.id AS "chat_id", chats.created_at AS "chat_created_at", chats.title, feeds.* +// abonnementSelect lists feed columns explicitly so the manual row scan does +// not depend on the physical column order of `feeds.*`. +const abonnementSelect = `SELECT chats.id, chats.created_at, chats.title, +feeds.id, feeds.url, feeds.last_entry, feeds.created_at, feeds.updated_at, +feeds.etag, feeds.last_modified, feeds.next_poll_at, feeds.last_poll_at, +feeds.error_count, feeds.unchanged_count, feeds.disabled, feeds.disabled_reason FROM abonnements JOIN chats ON abonnements.chat_id = chats.id JOIN feeds ON abonnements.feed_id = feeds.id` - rows, _ := db.Queryx(query) +func (db *Abonnements) GetAll() ([]Abonnement, error) { + rows, err := db.Queryx(abonnementSelect) + if err != nil { + return nil, err + } + return scanAbonnements(rows) +} + +// GetDue returns only feeds that are enabled and whose scheduled poll time has +// passed (or was never set). This is what keeps polling on a per-feed schedule +// and prevents a process restart from re-downloading everything (FRB037). +func (db *Abonnements) GetDue() ([]Abonnement, error) { + const where = ` WHERE feeds.disabled = 0 AND (feeds.next_poll_at IS NULL OR feeds.next_poll_at <= NOW())` + rows, err := db.Queryx(abonnementSelect + where) + if err != nil { + return nil, err + } + return scanAbonnements(rows) +} + +func scanAbonnements(rows *sqlx.Rows) ([]Abonnement, error) { defer rows.Close() - var abonnements []Abonnement - var feeds = make(map[int64]Feed) - var feedChats = make(map[int64][]Chat) + feeds := make(map[int64]Feed) + feedChats := make(map[int64][]Chat) + var order []int64 for rows.Next() { var chat Chat var feed Feed - rows.Scan(&chat.ID, &chat.CreatedAt, &chat.Title, - &feed.ID, &feed.Url, &feed.LastEntry, &feed.CreatedAt, &feed.UpdatedAt) - - feeds[feed.ID] = feed + err := rows.Scan(&chat.ID, &chat.CreatedAt, &chat.Title, + &feed.ID, &feed.Url, &feed.LastEntry, &feed.CreatedAt, &feed.UpdatedAt, + &feed.ETag, &feed.LastModified, &feed.NextPollAt, &feed.LastPollAt, + &feed.ErrorCount, &feed.UnchangedCount, &feed.Disabled, &feed.DisabledReason) + if err != nil { + return nil, err + } - if chats, ok := feedChats[feed.ID]; ok { - feedChats[feed.ID] = append(chats, chat) - } else { - feedChats[feed.ID] = []Chat{chat} + if _, seen := feeds[feed.ID]; !seen { + order = append(order, feed.ID) } + feeds[feed.ID] = feed + feedChats[feed.ID] = append(feedChats[feed.ID], chat) } - for feedId, feed := range feeds { + var abonnements []Abonnement + for _, feedId := range order { abonnements = append(abonnements, Abonnement{ - Feed: feed, + Feed: feeds[feedId], Chats: feedChats[feedId], }) } - return abonnements, nil + return abonnements, rows.Err() } -func (db *Abonnements) SetLastEntry(feedUrl string, lastEntry *string) error { +// SetFeedState writes the atomic cache set (etag + last_modified), the last seen +// entry and the next poll schedule after a successful 200 response. +func (db *Abonnements) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, nextPollAt time.Time, errorCount, unchangedCount int) error { const query = `UPDATE feeds -SET feeds.last_entry = ? -WHERE feeds.url = ?` - - _, err := db.Exec(query, lastEntry, feedUrl) +SET last_entry = ?, etag = ?, last_modified = ?, next_poll_at = ?, last_poll_at = NOW(), + error_count = ?, unchanged_count = ? +WHERE id = ?` + _, err := db.Exec(query, lastEntry, etag, lastModified, nextPollAt, errorCount, unchangedCount, feedID) return err } -func (feedToCheck Feed) Check(lastEntry *string) (*gofeed.Feed, error) { - feed, err := gofeed.NewParser().ParseURL(feedToCheck.Url) - if err != nil { - return nil, err - } +// Reschedule updates only the poll schedule and counters, preserving the cached +// etag/last_modified (FRB010-016) — used for 304, rate-limiting and transient +// errors. +func (db *Abonnements) Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error { + const query = `UPDATE feeds +SET next_poll_at = ?, last_poll_at = NOW(), error_count = ?, unchanged_count = ? +WHERE id = ?` + _, err := db.Exec(query, nextPollAt, errorCount, unchangedCount, feedID) + return err +} - if lastEntry != nil { - for i, item := range feed.Items { - if item.GUID == *lastEntry { - feed.Items = feed.Items[:i] - return feed, nil - } - } - } +// SetFeedURL persists a permanent redirect target (FRB130/131). +func (db *Abonnements) SetFeedURL(feedID int64, newURL string) error { + const query = `UPDATE feeds SET url = ? WHERE id = ?` + _, err := db.Exec(query, newURL, feedID) + return err +} - return feed, nil +// DisableFeed retires a feed that has gone away (FRB110-118). +func (db *Abonnements) DisableFeed(feedID int64, reason string) error { + const query = `UPDATE feeds SET disabled = 1, disabled_reason = ?, next_poll_at = NULL WHERE id = ?` + _, err := db.Exec(query, reason, feedID) + return err } diff --git a/storage/migrations/0003-efficiency.sql b/storage/migrations/0003-efficiency.sql new file mode 100644 index 0000000..c99dc8b --- /dev/null +++ b/storage/migrations/0003-efficiency.sql @@ -0,0 +1,25 @@ +-- +migrate Up + +ALTER TABLE `feeds` + ADD COLUMN `etag` VARCHAR(512) DEFAULT NULL, + ADD COLUMN `last_modified` VARCHAR(128) DEFAULT NULL, + ADD COLUMN `next_poll_at` DATETIME DEFAULT NULL, + ADD COLUMN `last_poll_at` DATETIME DEFAULT NULL, + ADD COLUMN `error_count` INT(11) NOT NULL DEFAULT 0, + ADD COLUMN `unchanged_count` INT(11) NOT NULL DEFAULT 0, + ADD COLUMN `disabled` TINYINT(1) NOT NULL DEFAULT 0, + ADD COLUMN `disabled_reason` VARCHAR(512) DEFAULT NULL, + ADD INDEX `idx_feeds_due` (`disabled`, `next_poll_at`) USING BTREE; + +-- +migrate Down + +ALTER TABLE `feeds` + DROP INDEX `idx_feeds_due`, + DROP COLUMN `etag`, + DROP COLUMN `last_modified`, + DROP COLUMN `next_poll_at`, + DROP COLUMN `last_poll_at`, + DROP COLUMN `error_count`, + DROP COLUMN `unchanged_count`, + DROP COLUMN `disabled`, + DROP COLUMN `disabled_reason`; From 656d581e758cea40e46d3cca8322e995a2c02407 Mon Sep 17 00:00:00 2001 From: Andreas Bielawski Date: Fri, 29 May 2026 14:00:21 +0200 Subject: [PATCH 02/49] send operational messages to admin only --- handler/feed_check.go | 12 ++++++++---- handler/handler.go | 1 + main.go | 5 +++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/handler/feed_check.go b/handler/feed_check.go index 69477f7..75dafc8 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -310,11 +310,15 @@ func (h *Handler) errorBackoff(errorCount int) time.Duration { return interval } +// notify sends operational messages (disable, redirect, rate-limit) to the +// admin only — never into the subscriber chats/channels, which are reserved for +// feed content. func (h *Handler) notify(abonnement storage.Abonnement, text string) { - for _, chat := range abonnement.Chats { - if err := h.sendText(chat.ID, text, abonnement.Feed.Url); err != nil { - log.Printf("%s: notify failed: %s", abonnement.Feed.Url, err) - } + if h.AdminID == 0 { + return + } + if err := h.sendText(h.AdminID, text, abonnement.Feed.Url); err != nil { + log.Printf("%s: notify failed: %s", abonnement.Feed.Url, err) } } diff --git a/handler/handler.go b/handler/handler.go index cd45459..1e74691 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -12,4 +12,5 @@ type Handler struct { Config *config.Config DB *storage.DB Fetcher *fetcher.Fetcher + AdminID int64 } diff --git a/main.go b/main.go index c7ad888..7d82627 100644 --- a/main.go +++ b/main.go @@ -57,17 +57,18 @@ func main() { log.Printf("Logged in as @%s (%d)", bot.Me.Username, bot.Me.ID) + adminId, err := strconv.ParseInt(os.Getenv("ADMIN_ID"), 10, 64) + h := handler.Handler{ Bot: bot, Config: cfg, DB: db, Fetcher: fetcher.New(), + AdminID: adminId, } log.Printf("Feed fetcher User-Agent: %s", h.Fetcher.UserAgent()) - adminId, err := strconv.ParseInt(os.Getenv("ADMIN_ID"), 10, 64) - if err != nil { // No admin = unsupported. // Else we would constantly have to check if the user is also From 16e28248c09f536c4ee95a752caf4c7dd23e1088 Mon Sep 17 00:00:00 2001 From: Andreas Bielawski Date: Thu, 23 Jul 2026 19:19:22 +0200 Subject: [PATCH 03/49] fix more issues & cleanup --- handler/feed_check.go | 122 ++++++++++++++++++++++++++--------------- main.go | 2 - storage/abonnements.go | 49 +++++++++++++++-- 3 files changed, 120 insertions(+), 53 deletions(-) diff --git a/handler/feed_check.go b/handler/feed_check.go index 75dafc8..84957e5 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -27,7 +27,34 @@ const ( fetchTimeout = 35 * time.Second ) -var feedproxyRe = regexp.MustCompile("^https?://feedproxy.google.com/~r/(.+?)/.*") +var ( + feedproxyRe = regexp.MustCompile("^https?://feedproxy.google.com/~r/(.+?)/.*") + blankLineRe = regexp.MustCompile("(?m)^\\s*$[\r\n]*") +) + +// compiledReplacement is a content filter with its regex compiled once per poll +// cycle instead of once per feed item. +type compiledReplacement struct { + re *regexp.Regexp // nil for a literal replacement + literal string +} + +func compileReplacements(replacements []storage.Replacement) []compiledReplacement { + compiled := make([]compiledReplacement, 0, len(replacements)) + for _, r := range replacements { + if r.IsRegex { + re, err := regexp.Compile(r.Value) + if err != nil { + log.Printf("skipping invalid replacement regex %q: %s", r.Value, err) + continue + } + compiled = append(compiled, compiledReplacement{re: re}) + } else { + compiled = append(compiled, compiledReplacement{literal: r.Value}) + } + } + return compiled +} type TemplateData struct { Title string @@ -59,37 +86,43 @@ func (h *Handler) OnCheck() { } log.Printf("Polling %d due feed(s)", len(abonnements)) - h.pollFeeds(abonnements, replacements) + h.pollFeeds(abonnements, compileReplacements(replacements)) } -// pollFeeds fetches the due feeds through a bounded worker pool while -// serializing requests that target the same host (FRB033/034). -func (h *Handler) pollFeeds(abonnements []storage.Abonnement, replacements []storage.Replacement) { - sem := make(chan struct{}, h.Config.Poll.Concurrency) +// pollFeeds fetches the due feeds through a fixed worker pool while serializing +// requests that target the same host (FRB033/034). +func (h *Handler) pollFeeds(abonnements []storage.Abonnement, replacements []compiledReplacement) { + workers := h.Config.Poll.Concurrency + if workers < 1 { + workers = 1 + } + hosts := &hostLocks{m: make(map[string]*sync.Mutex)} + jobs := make(chan storage.Abonnement) var wg sync.WaitGroup - for _, abonnement := range abonnements { - abonnement := abonnement + for i := 0; i < workers; i++ { wg.Add(1) go func() { defer wg.Done() - - sem <- struct{}{} - defer func() { <-sem }() - - lock := hosts.get(feedHost(abonnement.Feed.Url)) - lock.Lock() - defer lock.Unlock() - - h.pollFeed(abonnement, replacements) + for abonnement := range jobs { + lock := hosts.get(feedHost(abonnement.Feed.Url)) + lock.Lock() + h.pollFeed(abonnement, replacements) + lock.Unlock() + } }() } + for _, abonnement := range abonnements { + jobs <- abonnement + } + close(jobs) + wg.Wait() } -func (h *Handler) pollFeed(abonnement storage.Abonnement, replacements []storage.Replacement) { +func (h *Handler) pollFeed(abonnement storage.Abonnement, replacements []compiledReplacement) { feed := abonnement.Feed var etag, lastModified string @@ -112,11 +145,17 @@ func (h *Handler) pollFeed(abonnement storage.Abonnement, replacements []storage // Permanent move: persist the new URL and stop polling the old one // (FRB130/131/132). if result.PermanentURL != "" { - if err := h.DB.Abonnements.SetFeedURL(feed.ID, result.PermanentURL); err != nil { + merged, err := h.DB.Abonnements.MoveFeedURL(feed.ID, result.PermanentURL) + if err != nil { log.Printf("%s: could not update url to %s: %s", feed.Url, result.PermanentURL, err) } else { h.notify(abonnement, fmt.Sprintf("ℹ️ Feed wurde dauerhaft umgezogen:\n%s\n→ %s", feed.Url, result.PermanentURL)) + if merged { + // This feed row is gone; the existing feed at the target URL + // now owns these subscriptions and will deliver the content. + return + } feed.Url = result.PermanentURL } } @@ -138,7 +177,7 @@ func (h *Handler) pollFeed(abonnement storage.Abonnement, replacements []storage } } -func (h *Handler) handleOK(abonnement storage.Abonnement, feed storage.Feed, result *fetcher.Result, replacements []storage.Replacement) { +func (h *Handler) handleOK(abonnement storage.Abonnement, feed storage.Feed, result *fetcher.Result, replacements []compiledReplacement) { gfeed := result.Feed var lastEntry *string @@ -158,9 +197,9 @@ func (h *Handler) handleOK(abonnement storage.Abonnement, feed storage.Feed, res templateData.FeedTitle = html.EscapeString(gfeed.Title) if entry.Content != "" { - templateData.Content = processContent(entry.Content, &replacements) + templateData.Content = processContent(entry.Content, replacements) } else if entry.Description != "" { - templateData.Content = processContent(entry.Description, &replacements) + templateData.Content = processContent(entry.Description, replacements) } if entry.Link != "" { @@ -400,19 +439,18 @@ func (h *hostLocks) get(host string) *sync.Mutex { return lock } -func processContent(content string, replacements *[]storage.Replacement) string { +func processContent(content string, replacements []compiledReplacement) string { processed := html.UnescapeString(content) - for _, replacement := range *replacements { - if replacement.IsRegex { - re := regexp.MustCompile(replacement.Value) - processed = re.ReplaceAllString(processed, "") + for _, replacement := range replacements { + if replacement.re != nil { + processed = replacement.re.ReplaceAllString(processed, "") } else { - processed = strings.ReplaceAll(processed, replacement.Value, "") + processed = strings.ReplaceAll(processed, replacement.literal, "") } } - processed = regexp.MustCompile("(?m)^\\s*$[\r\n]*").ReplaceAllString(processed, "") + processed = blankLineRe.ReplaceAllString(processed, "") processed = strings.TrimSpace(processed) if len(processed) > 270 { @@ -424,22 +462,16 @@ func processContent(content string, replacements *[]storage.Replacement) string func (h *Handler) sendText(chatId int64, text string, url string) error { _, err := h.Bot.Send(telebot.ChatID(chatId), text, defaultSendOptions) + if err == nil { + return nil + } - var floodError *telebot.FloodError - - if err != nil { - if errors.As(err, &floodError) { - log.Printf("%s: Flood error, retrying after: %d seconds", url, - floodError.RetryAfter) - time.Sleep(time.Duration(err.(telebot.FloodError).RetryAfter) * time.Second) - err := h.sendText(chatId, text, url) - if err != nil { - return err - } - } else { - return err - } - + // telebot returns FloodError by value, so match the value type. + var floodError telebot.FloodError + if errors.As(err, &floodError) { + log.Printf("%s: Flood error, retrying after: %d seconds", url, floodError.RetryAfter) + time.Sleep(time.Duration(floodError.RetryAfter) * time.Second) + return h.sendText(chatId, text, url) } - return nil + return err } diff --git a/main.go b/main.go index 7d82627..36321c1 100644 --- a/main.go +++ b/main.go @@ -101,8 +101,6 @@ func main() { channel := make(chan os.Signal, 1) signal.Notify(channel, os.Interrupt, syscall.SIGTERM) - signal.Notify(channel, os.Interrupt, syscall.SIGKILL) - signal.Notify(channel, os.Interrupt, syscall.SIGINT) go func() { <-channel log.Println("Stopping...") diff --git a/storage/abonnements.go b/storage/abonnements.go index d1738f0..cd3aa49 100644 --- a/storage/abonnements.go +++ b/storage/abonnements.go @@ -3,6 +3,7 @@ package storage import ( "context" "database/sql" + "errors" "time" "github.com/jmoiron/sqlx" @@ -19,7 +20,7 @@ type ( GetDue() ([]Abonnement, error) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, nextPollAt time.Time, errorCount, unchangedCount int) error Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error - SetFeedURL(feedID int64, newURL string) error + MoveFeedURL(feedID int64, newURL string) (bool, error) DisableFeed(feedID int64, reason string) error } @@ -267,11 +268,47 @@ WHERE id = ?` return err } -// SetFeedURL persists a permanent redirect target (FRB130/131). -func (db *Abonnements) SetFeedURL(feedID int64, newURL string) error { - const query = `UPDATE feeds SET url = ? WHERE id = ?` - _, err := db.Exec(query, newURL, feedID) - return err +// MoveFeedURL persists a permanent redirect target (FRB130/131). If another feed +// already occupies newURL (feeds.url is UNIQUE), this feed's subscriptions are +// merged onto that existing feed instead and the old feed row is removed; the +// returned bool reports whether such a merge happened. +func (db *Abonnements) MoveFeedURL(feedID int64, newURL string) (bool, error) { + tx, err := db.BeginTxx(context.Background(), nil) + if err != nil { + return false, err + } + defer tx.Rollback() + + var targetID int64 + err = tx.Get(&targetID, "SELECT id FROM feeds WHERE url = ?", newURL) + switch { + case errors.Is(err, sql.ErrNoRows): + if _, err := tx.Exec("UPDATE feeds SET url = ? WHERE id = ?", newURL, feedID); err != nil { + return false, err + } + return false, tx.Commit() + case err != nil: + return false, err + case targetID == feedID: + return false, tx.Commit() + } + + // Repoint subscriptions onto the existing feed, dropping duplicates for + // chats already subscribed there, then delete the now-orphaned feed. + if _, err := tx.Exec("UPDATE IGNORE abonnements SET feed_id = ? WHERE feed_id = ?", targetID, feedID); err != nil { + return false, err + } + if _, err := tx.Exec("DELETE FROM abonnements WHERE feed_id = ?", feedID); err != nil { + return false, err + } + if _, err := tx.Exec("DELETE FROM feeds WHERE id = ?", feedID); err != nil { + return false, err + } + // Let the surviving feed pick up the merged subscribers on the next tick. + if _, err := tx.Exec("UPDATE feeds SET next_poll_at = NOW() WHERE id = ? AND disabled = 0", targetID); err != nil { + return false, err + } + return true, tx.Commit() } // DisableFeed retires a feed that has gone away (FRB110-118). From 5fbe70b08fff374cb8037a6c99d66e575503016c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:37:47 +0000 Subject: [PATCH 04/49] Include fetcher package in Docker build context Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 1d9ec25..672edb1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ * !config/ +!fetcher/ !handler/ !storage/ !go.mod From 9cd448db59e928ba744c6cdd002c3045b44710bb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:38:03 +0000 Subject: [PATCH 05/49] Compare poll times in Go time zone instead of NOW() Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- storage/abonnements.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/storage/abonnements.go b/storage/abonnements.go index cd3aa49..41b10e8 100644 --- a/storage/abonnements.go +++ b/storage/abonnements.go @@ -201,9 +201,11 @@ func (db *Abonnements) GetAll() ([]Abonnement, error) { // GetDue returns only feeds that are enabled and whose scheduled poll time has // passed (or was never set). This is what keeps polling on a per-feed schedule // and prevents a process restart from re-downloading everything (FRB037). +// Timestamps are always passed from Go instead of using NOW(), so they are +// written and compared in the driver's loc regardless of the MySQL time zone. func (db *Abonnements) GetDue() ([]Abonnement, error) { - const where = ` WHERE feeds.disabled = 0 AND (feeds.next_poll_at IS NULL OR feeds.next_poll_at <= NOW())` - rows, err := db.Queryx(abonnementSelect + where) + const where = ` WHERE feeds.disabled = 0 AND (feeds.next_poll_at IS NULL OR feeds.next_poll_at <= ?)` + rows, err := db.Queryx(abonnementSelect+where, time.Now()) if err != nil { return nil, err } @@ -250,10 +252,10 @@ func scanAbonnements(rows *sqlx.Rows) ([]Abonnement, error) { // entry and the next poll schedule after a successful 200 response. func (db *Abonnements) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, nextPollAt time.Time, errorCount, unchangedCount int) error { const query = `UPDATE feeds -SET last_entry = ?, etag = ?, last_modified = ?, next_poll_at = ?, last_poll_at = NOW(), +SET last_entry = ?, etag = ?, last_modified = ?, next_poll_at = ?, last_poll_at = ?, error_count = ?, unchanged_count = ? WHERE id = ?` - _, err := db.Exec(query, lastEntry, etag, lastModified, nextPollAt, errorCount, unchangedCount, feedID) + _, err := db.Exec(query, lastEntry, etag, lastModified, nextPollAt, time.Now(), errorCount, unchangedCount, feedID) return err } @@ -262,9 +264,9 @@ WHERE id = ?` // errors. func (db *Abonnements) Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error { const query = `UPDATE feeds -SET next_poll_at = ?, last_poll_at = NOW(), error_count = ?, unchanged_count = ? +SET next_poll_at = ?, last_poll_at = ?, error_count = ?, unchanged_count = ? WHERE id = ?` - _, err := db.Exec(query, nextPollAt, errorCount, unchangedCount, feedID) + _, err := db.Exec(query, nextPollAt, time.Now(), errorCount, unchangedCount, feedID) return err } @@ -305,7 +307,7 @@ func (db *Abonnements) MoveFeedURL(feedID int64, newURL string) (bool, error) { return false, err } // Let the surviving feed pick up the merged subscribers on the next tick. - if _, err := tx.Exec("UPDATE feeds SET next_poll_at = NOW() WHERE id = ? AND disabled = 0", targetID); err != nil { + if _, err := tx.Exec("UPDATE feeds SET next_poll_at = ? WHERE id = ? AND disabled = 0", time.Now(), targetID); err != nil { return false, err } return true, tx.Commit() From b1281e3de4e442c99f56e946034e6c6932406498 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:38:18 +0000 Subject: [PATCH 06/49] Send conditional headers on every redirect hop Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- fetcher/fetcher.go | 17 ++++++++--------- fetcher/fetcher_test.go | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index f08f36e..84f0821 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -82,15 +82,14 @@ func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string) return nil, err } req.Header.Set("User-Agent", f.userAgent) - // Conditional headers only apply to the originally requested resource; - // once we follow a redirect we no longer have cache values for it. - if hop == 0 { - if etag != "" { - req.Header.Set("If-None-Match", etag) - } - if lastModified != "" { - req.Header.Set("If-Modified-Since", lastModified) - } + // The stored validators belong to the final resource of the chain, so + // they are sent on every hop; otherwise feeds behind a temporary + // redirect could never be answered with 304. + if etag != "" { + req.Header.Set("If-None-Match", etag) + } + if lastModified != "" { + req.Header.Set("If-Modified-Since", lastModified) } resp, err := f.client.Do(req) diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index 82a315d..1b2eaab 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -111,6 +111,28 @@ func TestConditionalRequestEchoed(t *testing.T) { } } +func TestConditionalRequestThroughTemporaryRedirect(t *testing.T) { + var gotINM string + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotINM = r.Header.Get("If-None-Match") + w.WriteHeader(http.StatusNotModified) + })) + defer target.Close() + + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer src.Close() + + res := fetch(t, New(), src.URL, `"abc"`, "") + if gotINM != `"abc"` { + t.Errorf("If-None-Match after 302 = %q, want %q", gotINM, `"abc"`) + } + if !res.NotModified { + t.Errorf("expected 304 through temporary redirect, got status=%d", res.Status) + } +} + func TestRetryAfterParsed(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Retry-After", "120") From f7d53c964e53728df44c95d9b5279b5f449169e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:38:26 +0000 Subject: [PATCH 07/49] Group due feeds by host instead of per-host locks Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/feed_check.go | 47 ++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/handler/feed_check.go b/handler/feed_check.go index 84957e5..0f9bfd8 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -89,33 +89,42 @@ func (h *Handler) OnCheck() { h.pollFeeds(abonnements, compileReplacements(replacements)) } -// pollFeeds fetches the due feeds through a fixed worker pool while serializing -// requests that target the same host (FRB033/034). +// pollFeeds groups the due feeds by host and hands each group to a worker, so +// requests to the same host are serialized (FRB033/034) without idle workers +// blocking on a busy host. func (h *Handler) pollFeeds(abonnements []storage.Abonnement, replacements []compiledReplacement) { workers := h.Config.Poll.Concurrency if workers < 1 { workers = 1 } - hosts := &hostLocks{m: make(map[string]*sync.Mutex)} - jobs := make(chan storage.Abonnement) + groups := make(map[string][]storage.Abonnement) + var hosts []string + for _, abonnement := range abonnements { + host := feedHost(abonnement.Feed.Url) + if _, ok := groups[host]; !ok { + hosts = append(hosts, host) + } + groups[host] = append(groups[host], abonnement) + } + + jobs := make(chan []storage.Abonnement) var wg sync.WaitGroup for i := 0; i < workers; i++ { wg.Add(1) go func() { defer wg.Done() - for abonnement := range jobs { - lock := hosts.get(feedHost(abonnement.Feed.Url)) - lock.Lock() - h.pollFeed(abonnement, replacements) - lock.Unlock() + for group := range jobs { + for _, abonnement := range group { + h.pollFeed(abonnement, replacements) + } } }() } - for _, abonnement := range abonnements { - jobs <- abonnement + for _, host := range hosts { + jobs <- groups[host] } close(jobs) @@ -423,22 +432,6 @@ func containsDay(days []string, day string) bool { return false } -type hostLocks struct { - mu sync.Mutex - m map[string]*sync.Mutex -} - -func (h *hostLocks) get(host string) *sync.Mutex { - h.mu.Lock() - defer h.mu.Unlock() - if lock, ok := h.m[host]; ok { - return lock - } - lock := &sync.Mutex{} - h.m[host] = lock - return lock -} - func processContent(content string, replacements []compiledReplacement) string { processed := html.UnescapeString(content) From 2449abc5acdec59bb9d01b2ed8d26f95f67708c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:38:35 +0000 Subject: [PATCH 08/49] Cap server polling hints at POLL_INTERVAL_MAX Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/feed_check.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/handler/feed_check.go b/handler/feed_check.go index 0f9bfd8..eacb35a 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -268,10 +268,8 @@ func (h *Handler) handleRateLimit(abonnement storage.Abonnement, feed storage.Fe if delay <= 0 { // FRB021: a 429/503 without a hint is still "slow down". delay = h.Config.Poll.Interval * 4 - if delay > h.Config.Poll.IntervalMax { - delay = h.Config.Poll.IntervalMax - } } + delay = min(delay, h.Config.Poll.IntervalMax) // Keep the cached validators (FRB016) and do not count this toward retirement. if err := h.DB.Abonnements.Reschedule(feed.ID, time.Now().Add(delay), feed.ErrorCount, feed.UnchangedCount); err != nil { @@ -318,7 +316,8 @@ func (h *Handler) disable(abonnement storage.Abonnement, feed storage.Feed, reas // nextPoll computes the next poll time: base interval, optionally stretched for // feeds that rarely change (FRB023), never faster than server hints -// (max-age/ttl, FRB022/024), shifted out of skipHours/skipDays (FRB024). +// (max-age/ttl, FRB022/024, capped at POLL_INTERVAL_MAX), shifted out of +// skipHours/skipDays (FRB024). func (h *Handler) nextPoll(unchangedCount int, result *fetcher.Result) time.Time { interval := h.Config.Poll.Interval @@ -332,12 +331,8 @@ func (h *Handler) nextPoll(unchangedCount int, result *fetcher.Result) time.Time } if result != nil { - if result.MaxAge > interval { - interval = result.MaxAge - } - if result.FeedInterval > interval { - interval = result.FeedInterval - } + hint := min(max(result.MaxAge, result.FeedInterval), h.Config.Poll.IntervalMax) + interval = max(interval, hint) } next := time.Now().Add(interval) From 272cb919e47486f77eb1d7e59dea1f7f36e981b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:39:16 +0000 Subject: [PATCH 09/49] Persist feed polling hints so they apply after 304 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/feed_check.go | 3 + handler/feed_subscribe.go | 5 +- storage/abonnements.go | 77 ++++++++++++++++++++++---- storage/abonnements_test.go | 35 ++++++++++++ storage/migrations/0004-feed-hints.sql | 13 +++++ 5 files changed, 119 insertions(+), 14 deletions(-) create mode 100644 storage/abonnements_test.go create mode 100644 storage/migrations/0004-feed-hints.sql diff --git a/handler/feed_check.go b/handler/feed_check.go index eacb35a..be24de8 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -173,6 +173,8 @@ func (h *Handler) pollFeed(abonnement storage.Abonnement, replacements []compile case result.Status == 200 && result.Feed != nil: h.handleOK(abonnement, feed, result, replacements) case result.NotModified: + hints := feed.Hints() + result.FeedInterval, result.SkipHours, result.SkipDays = hints.Interval, hints.SkipHours, hints.SkipDays next := h.nextPoll(feed.UnchangedCount+1, result) if err := h.DB.Abonnements.Reschedule(feed.ID, next, 0, feed.UnchangedCount+1); err != nil { log.Printf("%s: reschedule failed: %s", feed.Url, err) @@ -257,6 +259,7 @@ func (h *Handler) handleOK(abonnement storage.Abonnement, feed storage.Feed, res next := h.nextPoll(unchanged, result) err := h.DB.Abonnements.SetFeedState(feed.ID, newLastEntry, nullableString(result.ETag), nullableString(result.LastModified), + storage.PollHints{Interval: result.FeedInterval, SkipHours: result.SkipHours, SkipDays: result.SkipDays}, next, 0, unchanged) if err != nil { log.Printf("%s: could not save state: %s", feed.Url, err) diff --git a/handler/feed_subscribe.go b/handler/feed_subscribe.go index 6d2c533..fa7ca9c 100644 --- a/handler/feed_subscribe.go +++ b/handler/feed_subscribe.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/Brawl345/rssbot/storage" "gopkg.in/telebot.v3" ) @@ -106,9 +107,9 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { // poll is already conditional. etag := nullableString(result.ETag) lastModified := nullableString(result.LastModified) - nextPollAt := time.Now().Add(h.Config.Poll.Interval) + hints := storage.PollHints{Interval: result.FeedInterval, SkipHours: result.SkipHours, SkipDays: result.SkipDays} - err = h.DB.Abonnements.Create(chatId, chatTitle, feedUrl, lastEntry, etag, lastModified, nextPollAt) + err = h.DB.Abonnements.Create(chatId, chatTitle, feedUrl, lastEntry, etag, lastModified, hints, h.nextPoll(0, result)) if err != nil { log.Println(err) return c.Send("❌ Beim Abonnieren des Feeds ist ein Fehler aufgetreten.", defaultSendOptions) diff --git a/storage/abonnements.go b/storage/abonnements.go index 41b10e8..b4e0fd5 100644 --- a/storage/abonnements.go +++ b/storage/abonnements.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "errors" + "strconv" + "strings" "time" "github.com/jmoiron/sqlx" @@ -11,14 +13,14 @@ import ( type ( AbonnementStorage interface { - Create(chatId int64, chatTitle string, feedUrl string, lastEntry, etag, lastModified *string, nextPollAt time.Time) error + Create(chatId int64, chatTitle string, feedUrl string, lastEntry, etag, lastModified *string, hints PollHints, nextPollAt time.Time) error Delete(chatId int64, feedId int64) error ExistsByFeedUrl(chatId int64, feedUrl string) (bool, error) ExistsById(chatId int64, feedId int64) (bool, error) GetByUser(chatId int64) ([]Feed, error) GetAll() ([]Abonnement, error) GetDue() ([]Abonnement, error) - SetFeedState(feedID int64, lastEntry, etag, lastModified *string, nextPollAt time.Time, errorCount, unchangedCount int) error + SetFeedState(feedID int64, lastEntry, etag, lastModified *string, hints PollHints, nextPollAt time.Time, errorCount, unchangedCount int) error Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error MoveFeedURL(feedID int64, newURL string) (bool, error) DisableFeed(feedID int64, reason string) error @@ -53,10 +55,54 @@ type ( UnchangedCount int `db:"unchanged_count"` Disabled bool `db:"disabled"` DisabledReason sql.NullString `db:"disabled_reason"` + FeedInterval int `db:"feed_interval"` + SkipHours sql.NullString `db:"skip_hours"` + SkipDays sql.NullString `db:"skip_days"` + } + + // PollHints are the polling hints a feed declares in its body (ttl, + // skipHours, skipDays). They are persisted so they still apply after a 304. + PollHints struct { + Interval time.Duration + SkipHours []int + SkipDays []string } ) -func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, lastEntry, etag, lastModified *string, nextPollAt time.Time) error { +// Hints decodes the persisted polling hints of a feed. +func (f Feed) Hints() PollHints { + hints := PollHints{Interval: time.Duration(f.FeedInterval) * time.Second} + if f.SkipHours.Valid { + for _, v := range strings.Split(f.SkipHours.String, ",") { + if n, err := strconv.Atoi(v); err == nil { + hints.SkipHours = append(hints.SkipHours, n) + } + } + } + if f.SkipDays.Valid && f.SkipDays.String != "" { + hints.SkipDays = strings.Split(f.SkipDays.String, ",") + } + return hints +} + +func (p PollHints) encode() (int, *string, *string) { + var skipHours, skipDays *string + if len(p.SkipHours) > 0 { + parts := make([]string, len(p.SkipHours)) + for i, h := range p.SkipHours { + parts[i] = strconv.Itoa(h) + } + joined := strings.Join(parts, ",") + skipHours = &joined + } + if len(p.SkipDays) > 0 { + joined := strings.Join(p.SkipDays, ",") + skipDays = &joined + } + return int(p.Interval / time.Second), skipHours, skipDays +} + +func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, lastEntry, etag, lastModified *string, hints PollHints, nextPollAt time.Time) error { tx, err := db.BeginTxx(context.Background(), nil) if err != nil { return err @@ -70,8 +116,10 @@ func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, la if err != nil { // Feed does not exist yet, will be created - const insertFeedQuery = "INSERT INTO feeds (url, last_entry, etag, last_modified, next_poll_at) VALUES (?, ?, ?, ?, ?)" - result, err := tx.Exec(insertFeedQuery, feedUrl, lastEntry, etag, lastModified, nextPollAt) + const insertFeedQuery = `INSERT INTO feeds (url, last_entry, etag, last_modified, feed_interval, skip_hours, skip_days, next_poll_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + interval, skipHours, skipDays := hints.encode() + result, err := tx.Exec(insertFeedQuery, feedUrl, lastEntry, etag, lastModified, interval, skipHours, skipDays, nextPollAt) if err != nil { return err } @@ -185,7 +233,8 @@ WHERE chats.id = ?` const abonnementSelect = `SELECT chats.id, chats.created_at, chats.title, feeds.id, feeds.url, feeds.last_entry, feeds.created_at, feeds.updated_at, feeds.etag, feeds.last_modified, feeds.next_poll_at, feeds.last_poll_at, -feeds.error_count, feeds.unchanged_count, feeds.disabled, feeds.disabled_reason +feeds.error_count, feeds.unchanged_count, feeds.disabled, feeds.disabled_reason, +feeds.feed_interval, feeds.skip_hours, feeds.skip_days FROM abonnements JOIN chats ON abonnements.chat_id = chats.id JOIN feeds ON abonnements.feed_id = feeds.id` @@ -225,7 +274,8 @@ func scanAbonnements(rows *sqlx.Rows) ([]Abonnement, error) { err := rows.Scan(&chat.ID, &chat.CreatedAt, &chat.Title, &feed.ID, &feed.Url, &feed.LastEntry, &feed.CreatedAt, &feed.UpdatedAt, &feed.ETag, &feed.LastModified, &feed.NextPollAt, &feed.LastPollAt, - &feed.ErrorCount, &feed.UnchangedCount, &feed.Disabled, &feed.DisabledReason) + &feed.ErrorCount, &feed.UnchangedCount, &feed.Disabled, &feed.DisabledReason, + &feed.FeedInterval, &feed.SkipHours, &feed.SkipDays) if err != nil { return nil, err } @@ -249,13 +299,16 @@ func scanAbonnements(rows *sqlx.Rows) ([]Abonnement, error) { } // SetFeedState writes the atomic cache set (etag + last_modified), the last seen -// entry and the next poll schedule after a successful 200 response. -func (db *Abonnements) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, nextPollAt time.Time, errorCount, unchangedCount int) error { +// entry, the feed's polling hints and the next poll schedule after a successful +// 200 response. +func (db *Abonnements) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, hints PollHints, nextPollAt time.Time, errorCount, unchangedCount int) error { const query = `UPDATE feeds -SET last_entry = ?, etag = ?, last_modified = ?, next_poll_at = ?, last_poll_at = ?, - error_count = ?, unchanged_count = ? +SET last_entry = ?, etag = ?, last_modified = ?, feed_interval = ?, skip_hours = ?, skip_days = ?, + next_poll_at = ?, last_poll_at = ?, error_count = ?, unchanged_count = ? WHERE id = ?` - _, err := db.Exec(query, lastEntry, etag, lastModified, nextPollAt, time.Now(), errorCount, unchangedCount, feedID) + interval, skipHours, skipDays := hints.encode() + _, err := db.Exec(query, lastEntry, etag, lastModified, interval, skipHours, skipDays, + nextPollAt, time.Now(), errorCount, unchangedCount, feedID) return err } diff --git a/storage/abonnements_test.go b/storage/abonnements_test.go new file mode 100644 index 0000000..3895d79 --- /dev/null +++ b/storage/abonnements_test.go @@ -0,0 +1,35 @@ +package storage + +import ( + "database/sql" + "reflect" + "testing" + "time" +) + +func TestPollHintsRoundTrip(t *testing.T) { + in := PollHints{Interval: 90 * time.Minute, SkipHours: []int{0, 23}, SkipDays: []string{"Saturday", "Sunday"}} + interval, skipHours, skipDays := in.encode() + + feed := Feed{FeedInterval: interval} + if skipHours != nil { + feed.SkipHours = sql.NullString{String: *skipHours, Valid: true} + } + if skipDays != nil { + feed.SkipDays = sql.NullString{String: *skipDays, Valid: true} + } + + if out := feed.Hints(); !reflect.DeepEqual(in, out) { + t.Errorf("round trip = %+v, want %+v", out, in) + } +} + +func TestPollHintsEmpty(t *testing.T) { + interval, skipHours, skipDays := PollHints{}.encode() + if interval != 0 || skipHours != nil || skipDays != nil { + t.Errorf("empty hints encoded to %d %v %v", interval, skipHours, skipDays) + } + if h := (Feed{}).Hints(); h.Interval != 0 || h.SkipHours != nil || h.SkipDays != nil { + t.Errorf("empty feed decoded to %+v", h) + } +} diff --git a/storage/migrations/0004-feed-hints.sql b/storage/migrations/0004-feed-hints.sql new file mode 100644 index 0000000..fef11f8 --- /dev/null +++ b/storage/migrations/0004-feed-hints.sql @@ -0,0 +1,13 @@ +-- +migrate Up + +ALTER TABLE `feeds` + ADD COLUMN `feed_interval` INT(11) NOT NULL DEFAULT 0, + ADD COLUMN `skip_hours` VARCHAR(100) DEFAULT NULL, + ADD COLUMN `skip_days` VARCHAR(100) DEFAULT NULL; + +-- +migrate Down + +ALTER TABLE `feeds` + DROP COLUMN `feed_interval`, + DROP COLUMN `skip_hours`, + DROP COLUMN `skip_days`; From b9a5bf24106771155e34eb3eb1b3a29fe7623441 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:39:30 +0000 Subject: [PATCH 10/49] Grow adaptive poll interval linearly, default cap 6h Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .env.example | 2 +- config/config.go | 2 +- handler/feed_check.go | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index de4a3e8..aac5ca2 100644 --- a/.env.example +++ b/.env.example @@ -9,7 +9,7 @@ MYSQL_SOCKET=/run/mysqld/mysqld.sock # Polling behaviour (all optional) POLL_INTERVAL=10m # base poll interval per feed; can be as low as 1m -POLL_INTERVAL_MAX=24h # upper cap when adaptive slow-down is on +POLL_INTERVAL_MAX=6h # upper cap for adaptive slow-down, error backoff and server hints POLL_ADAPTIVE=true # slow down feeds that rarely change (FRB023), set false to disable POLL_CONCURRENCY=8 # max feeds fetched simultaneously POLL_TICK=30s # scheduler granularity (how often due feeds are checked) diff --git a/config/config.go b/config/config.go index 8d18d57..f08b241 100644 --- a/config/config.go +++ b/config/config.go @@ -26,7 +26,7 @@ type PollConfig struct { func GetPollConfig() PollConfig { return PollConfig{ Interval: durationEnv("POLL_INTERVAL", 10*time.Minute), - IntervalMax: durationEnv("POLL_INTERVAL_MAX", 24*time.Hour), + IntervalMax: durationEnv("POLL_INTERVAL_MAX", 6*time.Hour), Adaptive: boolEnv("POLL_ADAPTIVE", true), Concurrency: intEnv("POLL_CONCURRENCY", 8), Tick: durationEnv("POLL_TICK", 30*time.Second), diff --git a/handler/feed_check.go b/handler/feed_check.go index be24de8..9e2634d 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -324,12 +324,13 @@ func (h *Handler) disable(abonnement storage.Abonnement, feed storage.Feed, reas func (h *Handler) nextPoll(unchangedCount int, result *fetcher.Result) time.Time { interval := h.Config.Poll.Interval + // Grow linearly so a feed that was quiet overnight is not delayed for hours + // once it becomes active again. if h.Config.Poll.Adaptive && unchangedCount > 0 { - for i := 0; i < unchangedCount && interval < h.Config.Poll.IntervalMax; i++ { - interval *= 2 - } - if interval > h.Config.Poll.IntervalMax { + if steps := time.Duration(unchangedCount); steps >= h.Config.Poll.IntervalMax/interval { interval = h.Config.Poll.IntervalMax + } else { + interval = min(interval*(1+steps), h.Config.Poll.IntervalMax) } } From 91646e3fe9cabc2d26e05e63b6ced4e4da24fb61 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:39:42 +0000 Subject: [PATCH 11/49] Ignore sy:updatePeriod as polling hint Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- fetcher/fetcher.go | 50 +++++------------------------------------ fetcher/fetcher_test.go | 12 +++++++++- 2 files changed, 16 insertions(+), 46 deletions(-) diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 84f0821..289293a 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -17,7 +17,6 @@ import ( "time" "github.com/mmcdole/gofeed" - ext "github.com/mmcdole/gofeed/extensions" "github.com/mmcdole/gofeed/rss" ) @@ -45,7 +44,7 @@ type Result struct { LastModified string // exactly as received (FRB001) MaxAge time.Duration // Cache-Control: max-age (FRB022), 0 if absent RetryAfter time.Duration // Retry-After on 429/503 (FRB020/021), 0 if absent - FeedInterval time.Duration // ttl / sy:updatePeriod hint (FRB023/024), 0 if absent + FeedInterval time.Duration // ttl hint (FRB023/024), 0 if absent SkipHours []int // RSS skipHours (FRB024) SkipDays []string // RSS skipDays (FRB024) PermanentURL string // set on a 301/308 chain (FRB130/131) @@ -211,8 +210,10 @@ func parseRetryAfter(value string) time.Duration { return 0 } -// applyFeedHints extracts RSS polling hints (ttl, skipHours, skipDays, -// sy:updatePeriod/updateFrequency) from the raw body when it is an RSS feed. +// applyFeedHints extracts RSS polling hints (ttl, skipHours, skipDays) from the +// raw body when it is an RSS feed. sy:updatePeriod is deliberately ignored: +// WordPress emits "hourly" by default for every feed, regardless of how often it +// actually changes. func applyFeedHints(result *Result, body []byte) { feedType := gofeed.DetectFeedType(bytes.NewReader(body)) if feedType != gofeed.FeedTypeRSS { @@ -235,47 +236,6 @@ func applyFeedHints(result *Result, body []byte) { result.FeedInterval = time.Duration(mins) * time.Minute } } - - if sy := syInterval(rssFeed.Extensions); sy > result.FeedInterval { - result.FeedInterval = sy - } -} - -// syInterval derives a polling interval from the syndication module -// (sy:updatePeriod / sy:updateFrequency). Defaults follow the RSS 1.0 -// syndication spec: daily period, frequency of 1. -func syInterval(extensions ext.Extensions) time.Duration { - sy, ok := extensions["sy"] - if !ok { - return 0 - } - - period := 24 * time.Hour // "daily" default - if vals := sy["updatePeriod"]; len(vals) > 0 { - switch strings.ToLower(strings.TrimSpace(vals[0].Value)) { - case "hourly": - period = time.Hour - case "daily": - period = 24 * time.Hour - case "weekly": - period = 7 * 24 * time.Hour - case "monthly": - period = 30 * 24 * time.Hour - case "yearly": - period = 365 * 24 * time.Hour - } - } else { - return 0 // no syndication info present - } - - frequency := 1 - if vals := sy["updateFrequency"]; len(vals) > 0 { - if n, err := strconv.Atoi(strings.TrimSpace(vals[0].Value)); err == nil && n > 0 { - frequency = n - } - } - - return period / time.Duration(frequency) } func snippet(body []byte) string { diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index 1b2eaab..cb09a7a 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -76,7 +76,6 @@ func TestFetchSetsHeadersAndParses(t *testing.T) { if res.MaxAge != 30*time.Minute { t.Errorf("MaxAge = %s, want 30m (FRB022)", res.MaxAge) } - // ttl=90m beats sy hourly/2 (=30m); both beat nothing. if res.FeedInterval != 90*time.Minute { t.Errorf("FeedInterval = %s, want 90m from ttl (FRB024)", res.FeedInterval) } @@ -88,6 +87,17 @@ func TestFetchSetsHeadersAndParses(t *testing.T) { } } +func TestSyndicationHintIgnored(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(strings.Replace(sampleRSS, "90", "", 1))) + })) + defer srv.Close() + + if res := fetch(t, New(), srv.URL, "", ""); res.FeedInterval != 0 { + t.Errorf("FeedInterval = %s, want 0 without ttl", res.FeedInterval) + } +} + func TestConditionalRequestEchoed(t *testing.T) { var gotINM, gotIMS string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From 0f2a3b523f34e0d26a4455b4a6833dbdf515f420 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:39:48 +0000 Subject: [PATCH 12/49] Skip item on template error instead of aborting feed update Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/feed_check.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler/feed_check.go b/handler/feed_check.go index 9e2634d..5bc9cff 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -231,7 +231,7 @@ func (h *Handler) handleOK(abonnement storage.Abonnement, feed storage.Feed, res var tpl bytes.Buffer if err := h.Config.Template.Execute(&tpl, templateData); err != nil { log.Printf("%s: %s", feed.Url, err) - return + continue } for _, chat := range abonnement.Chats { From be9b13dab6dfd23090a69480519df563783e5293 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:40:05 +0000 Subject: [PATCH 13/49] Reactivate disabled feeds on subscribe Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/feed_subscribe.go | 13 ++++++++++++- storage/abonnements.go | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/handler/feed_subscribe.go b/handler/feed_subscribe.go index fa7ca9c..25b85ad 100644 --- a/handler/feed_subscribe.go +++ b/handler/feed_subscribe.go @@ -89,8 +89,19 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { } } + nextPollAt := h.nextPoll(0, result) + + // The feed answered with a valid 200, so a previously retired feed works again. + reactivated, err := h.DB.Abonnements.ReactivateFeed(feedUrl, nextPollAt) + if err != nil { + log.Printf("subscribe %s: could not reactivate: %s", feedUrl, err) + } + exists, _ := h.DB.Abonnements.ExistsByFeedUrl(chatId, feedUrl) if exists { + if reactivated { + return c.Send("✅ Der deaktivierte Feed wurde wieder aktiviert.", defaultSendOptions) + } return c.Send("✅ Du hast diesen Feed bereits abonniert.", defaultSendOptions) } @@ -109,7 +120,7 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { lastModified := nullableString(result.LastModified) hints := storage.PollHints{Interval: result.FeedInterval, SkipHours: result.SkipHours, SkipDays: result.SkipDays} - err = h.DB.Abonnements.Create(chatId, chatTitle, feedUrl, lastEntry, etag, lastModified, hints, h.nextPoll(0, result)) + err = h.DB.Abonnements.Create(chatId, chatTitle, feedUrl, lastEntry, etag, lastModified, hints, nextPollAt) if err != nil { log.Println(err) return c.Send("❌ Beim Abonnieren des Feeds ist ein Fehler aufgetreten.", defaultSendOptions) diff --git a/storage/abonnements.go b/storage/abonnements.go index b4e0fd5..60387aa 100644 --- a/storage/abonnements.go +++ b/storage/abonnements.go @@ -24,6 +24,7 @@ type ( Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error MoveFeedURL(feedID int64, newURL string) (bool, error) DisableFeed(feedID int64, reason string) error + ReactivateFeed(feedUrl string, nextPollAt time.Time) (bool, error) } Abonnements struct { @@ -372,3 +373,16 @@ func (db *Abonnements) DisableFeed(feedID int64, reason string) error { _, err := db.Exec(query, reason, feedID) return err } + +// ReactivateFeed re-enables a retired feed, e.g. after it was successfully +// fetched again on subscribe. It reports whether the feed was disabled. +func (db *Abonnements) ReactivateFeed(feedUrl string, nextPollAt time.Time) (bool, error) { + const query = `UPDATE feeds SET disabled = 0, disabled_reason = NULL, error_count = 0, next_poll_at = ? +WHERE url = ? AND disabled = 1` + result, err := db.Exec(query, nextPollAt, feedUrl) + if err != nil { + return false, err + } + n, err := result.RowsAffected() + return n > 0, err +} From cc00ab87175940a25f44990eaa6cd51d3c47e316 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:40:19 +0000 Subject: [PATCH 14/49] Show disabled feeds in feed lists Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/feed_list.go | 2 +- handler/feed_list_all.go | 2 +- handler/start.go | 3 ++- handler/utils.go | 11 +++++++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/handler/feed_list.go b/handler/feed_list.go index e689c22..8f0bf6d 100644 --- a/handler/feed_list.go +++ b/handler/feed_list.go @@ -59,7 +59,7 @@ func (h *Handler) OnList(c telebot.Context) error { sb.WriteString(fmt.Sprintf("%s hat abonniert:\n", html.EscapeString(chatTitle))) for _, link := range links { - sb.WriteString(fmt.Sprintf("%d) %s\n", link.ID, html.EscapeString(link.Url))) + sb.WriteString(fmt.Sprintf("%d) %s%s\n", link.ID, html.EscapeString(link.Url), feedStatus(link))) } return c.Send(sb.String(), defaultSendOptions) diff --git a/handler/feed_list_all.go b/handler/feed_list_all.go index 95f7fa2..43c0509 100644 --- a/handler/feed_list_all.go +++ b/handler/feed_list_all.go @@ -27,7 +27,7 @@ func (h *Handler) OnListAll(c telebot.Context) error { sb := strings.Builder{} for _, abonnement := range abonnements { - sb.WriteString(fmt.Sprintf("%d) %s\n", abonnement.Feed.ID, abonnement.Feed.Url)) + sb.WriteString(fmt.Sprintf("%d) %s%s\n", abonnement.Feed.ID, abonnement.Feed.Url, feedStatus(abonnement.Feed))) for _, chat := range abonnement.Chats { sb.WriteString(fmt.Sprintf(" %d (%s)\n", chat.ID, diff --git a/handler/start.go b/handler/start.go index 2b5938a..6cab24e 100644 --- a/handler/start.go +++ b/handler/start.go @@ -16,7 +16,8 @@ func (h *Handler) OnStart(c telebot.Context) error { sb.WriteString("/repl_add String: Ersetzung hinzufügen\n") sb.WriteString("/repl_add_re RegEx: RegEx-Ersetzung hinzufügen\n") sb.WriteString("/repl_del Ersetzungs-ID: Ersetzung löschen\n\n") - sb.WriteString("[Chat] ist ein optionales Argument mit dem @Kanalnamen.") + sb.WriteString("[Chat] ist ein optionales Argument mit dem @Kanalnamen.\n") + sb.WriteString("Deaktivierte Feeds werden durch erneutes /sub wieder aktiviert.") return c.Send(sb.String(), defaultSendOptions) } diff --git a/handler/utils.go b/handler/utils.go index 093d844..51d7210 100644 --- a/handler/utils.go +++ b/handler/utils.go @@ -1,6 +1,10 @@ package handler import ( + "fmt" + "html" + + "github.com/Brawl345/rssbot/storage" "github.com/mmcdole/gofeed" "gopkg.in/telebot.v3" ) @@ -22,3 +26,10 @@ func reverse(s []*gofeed.Item) []*gofeed.Item { return a } + +func feedStatus(feed storage.Feed) string { + if !feed.Disabled { + return "" + } + return fmt.Sprintf(" 🚫 deaktiviert: %s", html.EscapeString(feed.DisabledReason.String)) +} From 9b2dddae167900c1e3b23d751360ae6c1d69e729 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:40:37 +0000 Subject: [PATCH 15/49] Retire feeds only after a week of continuous errors Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/feed_check.go | 16 +++++++++++----- storage/abonnements.go | 18 +++++++++++------- storage/migrations/0005-failing-since.sql | 9 +++++++++ 3 files changed, 31 insertions(+), 12 deletions(-) create mode 100644 storage/migrations/0005-failing-since.sql diff --git a/handler/feed_check.go b/handler/feed_check.go index 5bc9cff..d2d9561 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -20,9 +20,10 @@ import ( ) const ( - // maxFeedErrors is how many consecutive failures a feed tolerates before it - // is retired (FRB110-119). + // A feed is retired once it failed at least maxFeedErrors times in a row + // and has been failing for retireAfter (FRB110-119). maxFeedErrors = 12 + retireAfter = 7 * 24 * time.Hour // fetchTimeout bounds a single feed fetch. fetchTimeout = 35 * time.Second ) @@ -288,10 +289,14 @@ func (h *Handler) handleRateLimit(abonnement storage.Abonnement, feed storage.Fe } // handleSoftError applies exponential backoff and retires the feed once it has -// failed too many times in a row (FRB110-119). +// been failing for too long (FRB110-119). func (h *Handler) handleSoftError(abonnement storage.Abonnement, feed storage.Feed, reason, body string) { errorCount := feed.ErrorCount + 1 - if errorCount >= maxFeedErrors { + failingSince := time.Now() + if feed.FailingSince.Valid { + failingSince = feed.FailingSince.Time + } + if errorCount >= maxFeedErrors && time.Since(failingSince) >= retireAfter { h.disable(abonnement, feed, reason, body) return } @@ -300,7 +305,8 @@ func (h *Handler) handleSoftError(abonnement storage.Abonnement, feed storage.Fe if err := h.DB.Abonnements.Reschedule(feed.ID, time.Now().Add(delay), errorCount, feed.UnchangedCount); err != nil { log.Printf("%s: reschedule failed: %s", feed.Url, err) } - log.Printf("%s: %s (error %d/%d), retrying in %s", feed.Url, reason, errorCount, maxFeedErrors, delay.Round(time.Second)) + log.Printf("%s: %s (error %d, failing since %s), retrying in %s", + feed.Url, reason, errorCount, failingSince.Format(time.DateTime), delay.Round(time.Second)) } func (h *Handler) disable(abonnement storage.Abonnement, feed storage.Feed, reason, body string) { diff --git a/storage/abonnements.go b/storage/abonnements.go index 60387aa..5e91a0e 100644 --- a/storage/abonnements.go +++ b/storage/abonnements.go @@ -59,6 +59,7 @@ type ( FeedInterval int `db:"feed_interval"` SkipHours sql.NullString `db:"skip_hours"` SkipDays sql.NullString `db:"skip_days"` + FailingSince sql.NullTime `db:"failing_since"` } // PollHints are the polling hints a feed declares in its body (ttl, @@ -235,7 +236,7 @@ const abonnementSelect = `SELECT chats.id, chats.created_at, chats.title, feeds.id, feeds.url, feeds.last_entry, feeds.created_at, feeds.updated_at, feeds.etag, feeds.last_modified, feeds.next_poll_at, feeds.last_poll_at, feeds.error_count, feeds.unchanged_count, feeds.disabled, feeds.disabled_reason, -feeds.feed_interval, feeds.skip_hours, feeds.skip_days +feeds.feed_interval, feeds.skip_hours, feeds.skip_days, feeds.failing_since FROM abonnements JOIN chats ON abonnements.chat_id = chats.id JOIN feeds ON abonnements.feed_id = feeds.id` @@ -276,7 +277,7 @@ func scanAbonnements(rows *sqlx.Rows) ([]Abonnement, error) { &feed.ID, &feed.Url, &feed.LastEntry, &feed.CreatedAt, &feed.UpdatedAt, &feed.ETag, &feed.LastModified, &feed.NextPollAt, &feed.LastPollAt, &feed.ErrorCount, &feed.UnchangedCount, &feed.Disabled, &feed.DisabledReason, - &feed.FeedInterval, &feed.SkipHours, &feed.SkipDays) + &feed.FeedInterval, &feed.SkipHours, &feed.SkipDays, &feed.FailingSince) if err != nil { return nil, err } @@ -305,7 +306,7 @@ func scanAbonnements(rows *sqlx.Rows) ([]Abonnement, error) { func (db *Abonnements) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, hints PollHints, nextPollAt time.Time, errorCount, unchangedCount int) error { const query = `UPDATE feeds SET last_entry = ?, etag = ?, last_modified = ?, feed_interval = ?, skip_hours = ?, skip_days = ?, - next_poll_at = ?, last_poll_at = ?, error_count = ?, unchanged_count = ? + next_poll_at = ?, last_poll_at = ?, error_count = ?, unchanged_count = ?, failing_since = NULL WHERE id = ?` interval, skipHours, skipDays := hints.encode() _, err := db.Exec(query, lastEntry, etag, lastModified, interval, skipHours, skipDays, @@ -315,12 +316,15 @@ WHERE id = ?` // Reschedule updates only the poll schedule and counters, preserving the cached // etag/last_modified (FRB010-016) — used for 304, rate-limiting and transient -// errors. +// errors. failing_since marks the start of an error streak and is cleared once +// errorCount drops back to 0. func (db *Abonnements) Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error { const query = `UPDATE feeds -SET next_poll_at = ?, last_poll_at = ?, error_count = ?, unchanged_count = ? +SET next_poll_at = ?, last_poll_at = ?, error_count = ?, unchanged_count = ?, + failing_since = CASE WHEN ? > 0 THEN COALESCE(failing_since, ?) ELSE NULL END WHERE id = ?` - _, err := db.Exec(query, nextPollAt, time.Now(), errorCount, unchangedCount, feedID) + now := time.Now() + _, err := db.Exec(query, nextPollAt, now, errorCount, unchangedCount, errorCount, now, feedID) return err } @@ -377,7 +381,7 @@ func (db *Abonnements) DisableFeed(feedID int64, reason string) error { // ReactivateFeed re-enables a retired feed, e.g. after it was successfully // fetched again on subscribe. It reports whether the feed was disabled. func (db *Abonnements) ReactivateFeed(feedUrl string, nextPollAt time.Time) (bool, error) { - const query = `UPDATE feeds SET disabled = 0, disabled_reason = NULL, error_count = 0, next_poll_at = ? + const query = `UPDATE feeds SET disabled = 0, disabled_reason = NULL, error_count = 0, failing_since = NULL, next_poll_at = ? WHERE url = ? AND disabled = 1` result, err := db.Exec(query, nextPollAt, feedUrl) if err != nil { diff --git a/storage/migrations/0005-failing-since.sql b/storage/migrations/0005-failing-since.sql new file mode 100644 index 0000000..36f24da --- /dev/null +++ b/storage/migrations/0005-failing-since.sql @@ -0,0 +1,9 @@ +-- +migrate Up + +ALTER TABLE `feeds` + ADD COLUMN `failing_since` DATETIME DEFAULT NULL; + +-- +migrate Down + +ALTER TABLE `feeds` + DROP COLUMN `failing_since`; From 8dd76eff53c1fbda16ca3cb4c33e93d20720fb8a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:40:49 +0000 Subject: [PATCH 16/49] Persist permanent redirects only when target succeeds Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- fetcher/fetcher.go | 6 ++++-- fetcher/fetcher_test.go | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 289293a..0eb198d 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -47,7 +47,7 @@ type Result struct { FeedInterval time.Duration // ttl hint (FRB023/024), 0 if absent SkipHours []int // RSS skipHours (FRB024) SkipDays []string // RSS skipDays (FRB024) - PermanentURL string // set on a 301/308 chain (FRB130/131) + PermanentURL string // set on a 301/308 chain ending in 200/304 (FRB130/131) Body string // snippet of an error body for the user (FRB101/120) } @@ -121,7 +121,9 @@ func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string) } result := &Result{Status: resp.StatusCode} - if permanentURL != "" && permanentURL != feedURL { + // Only report a move when the new location actually serves the feed. + succeeded := resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotModified + if succeeded && permanentURL != "" && permanentURL != feedURL { result.PermanentURL = permanentURL } diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index cb09a7a..3b2a629 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -184,6 +184,24 @@ func TestPermanentRedirectReported(t *testing.T) { } } +func TestPermanentRedirectToErrorNotReported(t *testing.T) { + target := httptest.NewServer(http.NotFoundHandler()) + defer target.Close() + + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusMovedPermanently) + })) + defer src.Close() + + res := fetch(t, New(), src.URL, "", "") + if res.PermanentURL != "" { + t.Errorf("PermanentURL = %q, want empty when target fails", res.PermanentURL) + } + if res.Status != http.StatusNotFound { + t.Errorf("status = %d, want 404", res.Status) + } +} + func TestTemporaryRedirectNotPersisted(t *testing.T) { target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(sampleRSS)) From dbe271f6d01bfe8861641cb2f63d75bffc8e1c01 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:41:04 +0000 Subject: [PATCH 17/49] Truncate text on rune boundaries and drop invalid UTF-8 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- fetcher/fetcher.go | 7 ++++--- fetcher/snippet_test.go | 18 ++++++++++++++++++ handler/feed_check.go | 7 ++++--- handler/feed_check_test.go | 17 +++++++++++++++++ 4 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 fetcher/snippet_test.go create mode 100644 handler/feed_check_test.go diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 0eb198d..b151202 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" "github.com/mmcdole/gofeed" "github.com/mmcdole/gofeed/rss" @@ -241,9 +242,9 @@ func applyFeedHints(result *Result, body []byte) { } func snippet(body []byte) string { - s := strings.TrimSpace(string(body)) - if len(s) > bodySnippetLen { - return s[:bodySnippetLen] + "…" + s := strings.TrimSpace(strings.ToValidUTF8(string(body), "")) + if utf8.RuneCountInString(s) > bodySnippetLen { + return string([]rune(s)[:bodySnippetLen]) + "…" } return s } diff --git a/fetcher/snippet_test.go b/fetcher/snippet_test.go new file mode 100644 index 0000000..727ea9f --- /dev/null +++ b/fetcher/snippet_test.go @@ -0,0 +1,18 @@ +package fetcher + +import ( + "strings" + "testing" + "unicode/utf8" +) + +func TestSnippetKeepsValidUTF8(t *testing.T) { + body := []byte(strings.Repeat("ä", bodySnippetLen+10) + "\xff") + s := snippet(body) + if !utf8.ValidString(s) { + t.Fatalf("snippet is not valid UTF-8") + } + if n := utf8.RuneCountInString(s); n != bodySnippetLen+1 { + t.Errorf("snippet has %d runes, want %d", n, bodySnippetLen+1) + } +} diff --git a/handler/feed_check.go b/handler/feed_check.go index d2d9561..5a845b3 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -12,6 +12,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "github.com/Brawl345/rssbot/fetcher" "github.com/Brawl345/rssbot/storage" @@ -438,7 +439,7 @@ func containsDay(days []string, day string) bool { } func processContent(content string, replacements []compiledReplacement) string { - processed := html.UnescapeString(content) + processed := strings.ToValidUTF8(html.UnescapeString(content), "") for _, replacement := range replacements { if replacement.re != nil { @@ -451,8 +452,8 @@ func processContent(content string, replacements []compiledReplacement) string { processed = blankLineRe.ReplaceAllString(processed, "") processed = strings.TrimSpace(processed) - if len(processed) > 270 { - return processed[:270] + "..." + if utf8.RuneCountInString(processed) > 270 { + return string([]rune(processed)[:270]) + "..." } return processed diff --git a/handler/feed_check_test.go b/handler/feed_check_test.go new file mode 100644 index 0000000..6463754 --- /dev/null +++ b/handler/feed_check_test.go @@ -0,0 +1,17 @@ +package handler + +import ( + "strings" + "testing" + "unicode/utf8" +) + +func TestProcessContentKeepsValidUTF8(t *testing.T) { + out := processContent(strings.Repeat("ü", 300), nil) + if !utf8.ValidString(out) { + t.Fatalf("output is not valid UTF-8") + } + if !strings.HasSuffix(out, "...") || utf8.RuneCountInString(out) != 273 { + t.Errorf("unexpected truncation: %d runes", utf8.RuneCountInString(out)) + } +} From aa02095549c27a086cde80212f89df0f7b9956fb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:41:13 +0000 Subject: [PATCH 18/49] Escape URLs and reasons in HTML messages Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/feed_check.go | 6 +++--- handler/feed_list_all.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/handler/feed_check.go b/handler/feed_check.go index 5a845b3..ed35355 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -161,7 +161,7 @@ func (h *Handler) pollFeed(abonnement storage.Abonnement, replacements []compile log.Printf("%s: could not update url to %s: %s", feed.Url, result.PermanentURL, err) } else { h.notify(abonnement, fmt.Sprintf("ℹ️ Feed wurde dauerhaft umgezogen:\n%s\n→ %s", - feed.Url, result.PermanentURL)) + html.EscapeString(feed.Url), html.EscapeString(result.PermanentURL))) if merged { // This feed row is gone; the existing feed at the target URL // now owns these subscriptions and will deliver the content. @@ -285,7 +285,7 @@ func (h *Handler) handleRateLimit(abonnement storage.Abonnement, feed storage.Fe if result.Status == 429 { h.notify(abonnement, fmt.Sprintf( "⚠️ Feed sendet \"429 Too Many Requests\":\n%s\nBackoff: %s. Eventuell ist das Poll-Intervall zu kurz.", - feed.Url, delay.Round(time.Second))) + html.EscapeString(feed.Url), delay.Round(time.Second))) } } @@ -317,7 +317,7 @@ func (h *Handler) disable(abonnement storage.Abonnement, feed storage.Feed, reas } log.Printf("%s: disabled (%s)", feed.Url, reason) - msg := fmt.Sprintf("🚫 Feed wurde deaktiviert:\n%s\nGrund: %s", feed.Url, reason) + msg := fmt.Sprintf("🚫 Feed wurde deaktiviert:\n%s\nGrund: %s", html.EscapeString(feed.Url), html.EscapeString(reason)) if body != "" { msg += fmt.Sprintf("\n
%s
", html.EscapeString(body)) } diff --git a/handler/feed_list_all.go b/handler/feed_list_all.go index 43c0509..a4b41d2 100644 --- a/handler/feed_list_all.go +++ b/handler/feed_list_all.go @@ -27,7 +27,7 @@ func (h *Handler) OnListAll(c telebot.Context) error { sb := strings.Builder{} for _, abonnement := range abonnements { - sb.WriteString(fmt.Sprintf("%d) %s%s\n", abonnement.Feed.ID, abonnement.Feed.Url, feedStatus(abonnement.Feed))) + sb.WriteString(fmt.Sprintf("%d) %s%s\n", abonnement.Feed.ID, html.EscapeString(abonnement.Feed.Url), feedStatus(abonnement.Feed))) for _, chat := range abonnement.Chats { sb.WriteString(fmt.Sprintf(" %d (%s)\n", chat.ID, From 5009dd70504ffecf2a25c58737848fdaf2cf153f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:41:37 +0000 Subject: [PATCH 19/49] Refuse redirects from public feeds to private addresses Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- fetcher/fetcher.go | 59 ++++++++++++++++++++++++++++++++++++++--- fetcher/fetcher_test.go | 27 +++++++++++++++++++ 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index b151202..486c9f9 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "io" + "net" "net/http" "net/url" "runtime/debug" @@ -30,8 +31,9 @@ const ( ) type Fetcher struct { - client *http.Client - userAgent string + client *http.Client + userAgent string + privateHost func(ctx context.Context, host string) bool } // Result is the interpreted outcome of a single feed fetch. ETag and @@ -62,7 +64,8 @@ func New() *Fetcher { return http.ErrUseLastResponse }, }, - userAgent: buildUserAgent(), + userAgent: buildUserAgent(), + privateHost: isPrivateHost, } } @@ -75,6 +78,10 @@ func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string) currentURL := feedURL permanentPrefix := true var permanentURL string + // A feed the admin subscribed to on an internal address may redirect + // internally, but a public feed must not bounce the bot into the local + // network (SSRF). Only resolved once a redirect is actually followed. + var originPrivate *bool for hop := 0; ; hop++ { req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentURL, nil) @@ -111,6 +118,13 @@ func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string) if err != nil { return nil, err } + if originPrivate == nil { + private := f.privateHost(ctx, hostOf(feedURL)) + originPrivate = &private + } + if !*originPrivate && f.privateHost(ctx, hostOf(next)) { + return nil, fmt.Errorf("redirect to private address %s refused", next) + } if (resp.StatusCode == http.StatusMovedPermanently || resp.StatusCode == http.StatusPermanentRedirect) && permanentPrefix { permanentURL = next @@ -178,7 +192,44 @@ func resolveLocation(base, location string) (string, error) { if err != nil { return "", err } - return b.ResolveReference(l).String(), nil + next := b.ResolveReference(l) + if next.Scheme != "http" && next.Scheme != "https" { + return "", fmt.Errorf("redirect to unsupported scheme %q", next.Scheme) + } + return next.String(), nil +} + +func hostOf(rawURL string) string { + if u, err := url.Parse(rawURL); err == nil { + return u.Hostname() + } + return "" +} + +// isPrivateHost reports whether host is or resolves to a loopback, private, +// link-local or unspecified address. +func isPrivateHost(ctx context.Context, host string) bool { + if host == "" { + return false + } + if ip := net.ParseIP(host); ip != nil { + return isPrivateIP(ip) + } + addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil { + return false + } + for _, addr := range addrs { + if isPrivateIP(addr.IP) { + return true + } + } + return false +} + +func isPrivateIP(ip net.IP) bool { + return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || + ip.IsLinkLocalMulticast() || ip.IsUnspecified() } func parseMaxAge(cacheControl string) time.Duration { diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index 3b2a629..79c01fa 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -222,6 +222,33 @@ func TestTemporaryRedirectNotPersisted(t *testing.T) { } } +func TestRedirectFromPublicToPrivateRefused(t *testing.T) { + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://internal.test/feed", http.StatusFound) + })) + defer src.Close() + + f := New() + f.privateHost = func(_ context.Context, host string) bool { return host == "internal.test" } + + _, err := f.Fetch(context.Background(), src.URL, "", "") + if err == nil || !strings.Contains(err.Error(), "private") { + t.Fatalf("err = %v, want refused private redirect", err) + } +} + +func TestRedirectToUnsupportedSchemeRefused(t *testing.T) { + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "file:///etc/passwd", http.StatusFound) + })) + defer src.Close() + + _, err := New().Fetch(context.Background(), src.URL, "", "") + if err == nil || !strings.Contains(err.Error(), "scheme") { + t.Fatalf("err = %v, want refused scheme", err) + } +} + func TestNonFeedRejected(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("not a feed")) From db8579c15094e2587fa1d824e3fb55095bcc41c6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:41:49 +0000 Subject: [PATCH 20/49] Warn about invalid poll settings Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- config/config.go | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/config/config.go b/config/config.go index f08b241..17c4071 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,7 @@ package config import ( + "log" "os" "strconv" "text/template" @@ -24,39 +25,53 @@ type PollConfig struct { } func GetPollConfig() PollConfig { - return PollConfig{ + cfg := PollConfig{ Interval: durationEnv("POLL_INTERVAL", 10*time.Minute), IntervalMax: durationEnv("POLL_INTERVAL_MAX", 6*time.Hour), Adaptive: boolEnv("POLL_ADAPTIVE", true), Concurrency: intEnv("POLL_CONCURRENCY", 8), Tick: durationEnv("POLL_TICK", 30*time.Second), } + if cfg.IntervalMax < cfg.Interval { + log.Printf("POLL_INTERVAL_MAX (%s) is below POLL_INTERVAL (%s), using %s", cfg.IntervalMax, cfg.Interval, cfg.Interval) + cfg.IntervalMax = cfg.Interval + } + return cfg } func durationEnv(key string, fallback time.Duration) time.Duration { - if v := os.Getenv(key); v != "" { - if d, err := time.ParseDuration(v); err == nil && d > 0 { - return d - } + v := os.Getenv(key) + if v == "" { + return fallback + } + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d } + log.Printf("Invalid %s=%q, using default %s", key, v, fallback) return fallback } func intEnv(key string, fallback int) int { - if v := os.Getenv(key); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - return n - } + v := os.Getenv(key) + if v == "" { + return fallback } + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + log.Printf("Invalid %s=%q, using default %d", key, v, fallback) return fallback } func boolEnv(key string, fallback bool) bool { - if v := os.Getenv(key); v != "" { - if b, err := strconv.ParseBool(v); err == nil { - return b - } + v := os.Getenv(key) + if v == "" { + return fallback + } + if b, err := strconv.ParseBool(v); err == nil { + return b } + log.Printf("Invalid %s=%q, using default %t", key, v, fallback) return fallback } From cad149171d828a462b8837c35f8153c61a43e402 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:42:03 +0000 Subject: [PATCH 21/49] Set User-Agent version via ldflags in Docker and Nix builds Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .github/workflows/build.yml | 2 +- Dockerfile | 3 ++- fetcher/fetcher.go | 8 ++++++++ flake.nix | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3686459..bd0edad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,7 @@ jobs: - uses: actions/checkout@v4 - name: Build image - run: docker build . --file Dockerfile --tag $NAME --label "runnumber=${GITHUB_RUN_ID}" + run: docker build . --file Dockerfile --tag $NAME --build-arg VERSION="${GITHUB_SHA::7}" --label "runnumber=${GITHUB_RUN_ID}" - name: Log in to registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin diff --git a/Dockerfile b/Dockerfile index 6a461b0..7f931e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,8 @@ WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . ./ -RUN CGO_ENABLED=0 GOOS=linux go build -o /rssbot +ARG VERSION=dev +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X github.com/Brawl345/rssbot/fetcher.Version=${VERSION}" -o /rssbot FROM gcr.io/distroless/static-debian12 AS release-stage WORKDIR /app diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 486c9f9..28aa0fd 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -300,7 +300,15 @@ func snippet(body []byte) string { return s } +// Version can be set at build time via +// -ldflags "-X github.com/Brawl345/rssbot/fetcher.Version=...". Otherwise the +// VCS revision embedded by the Go toolchain is used. +var Version string + func buildUserAgent() string { + if Version != "" { + return fmt.Sprintf("rssbot/%s (+%s)", Version, repoURL) + } version := "dev" if info, ok := debug.ReadBuildInfo(); ok { for _, setting := range info.Settings { diff --git a/flake.nix b/flake.nix index 01c2a93..7e85e54 100644 --- a/flake.nix +++ b/flake.nix @@ -52,6 +52,7 @@ ldflags = [ "-s" "-w" + "-X github.com/Brawl345/rssbot/fetcher.Version=${version}" ]; meta = { From f4d878ebe8751670a1a6b8c827ea525ee71284d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:46:44 +0000 Subject: [PATCH 22/49] Harden build workflow per zizmor Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .github/workflows/build.yml | 54 ++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd0edad..173ac3c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,12 @@ on: - push - pull_request +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + env: NAME: rssbot @@ -10,6 +16,8 @@ jobs: build: name: build runs-on: ubuntu-latest + permissions: + contents: read # checkout strategy: matrix: GOOS: [windows, linux, darwin] @@ -25,6 +33,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v4 + with: + persist-credentials: false - name: Setup Go uses: actions/setup-go@v5 @@ -35,16 +45,16 @@ jobs: env: GOOS: ${{ matrix.GOOS }} GOARCH: ${{ matrix.GOARCH }} - run: go build -ldflags="-s -w" -o dist/$NAME-$GOOS-$GOARCH + run: go build -ldflags="-s -w" -o "dist/${NAME}-${GOOS}-${GOARCH}" - name: Rename binaries (Windows) if: matrix.GOOS == 'windows' - run: for x in dist/$NAME-windows-*; do mv $x $x.exe; done + run: for x in dist/"${NAME}"-windows-*; do mv "$x" "$x.exe"; done - name: Upload binary uses: actions/upload-artifact@v4 with: - name: ${{env.NAME}}-${{ matrix.GOOS }}-${{ matrix.GOARCH }}-${{github.sha}} + name: ${{ env.NAME }}-${{ matrix.GOOS }}-${{ matrix.GOARCH }}-${{ github.sha }} path: dist/* retention-days: 90 @@ -52,31 +62,31 @@ jobs: name: docker runs-on: ubuntu-latest permissions: - packages: write - contents: read + contents: read # checkout + packages: write # push image to ghcr.io steps: - - uses: actions/checkout@v4 + - name: Checkout repo + uses: actions/checkout@v4 + with: + persist-credentials: false - name: Build image - run: docker build . --file Dockerfile --tag $NAME --build-arg VERSION="${GITHUB_SHA::7}" --label "runnumber=${GITHUB_RUN_ID}" + run: docker build . --file Dockerfile --tag "$NAME" --build-arg VERSION="${GITHUB_SHA::7}" --label "runnumber=${GITHUB_RUN_ID}" - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin - name: Push image run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$NAME - - # This changes all uppercase characters to lowercase. - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') - # This strips the git ref prefix from the version. - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - # This strips the "v" prefix from the tag name. - [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') - # This uses the Docker `latest` tag convention. - [ "$VERSION" == "master" ] && VERSION=latest - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - docker tag $NAME $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION + image_id="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/${NAME}" + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + version="${GITHUB_REF_NAME#v}" + else + version=latest + fi + echo "Pushing ${image_id}:${version}" + docker tag "$NAME" "${image_id}:${version}" + docker push "${image_id}:${version}" From fb9c2ea31ec22a0702ac02433f424f60afa78209 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:46:45 +0000 Subject: [PATCH 23/49] Push Docker images only from master and tags Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 173ac3c..03a6c0d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,11 +75,13 @@ jobs: run: docker build . --file Dockerfile --tag "$NAME" --build-arg VERSION="${GITHUB_SHA::7}" --label "runnumber=${GITHUB_RUN_ID}" - name: Log in to registry + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin - name: Push image + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) run: | image_id="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/${NAME}" if [[ "$GITHUB_REF" == refs/tags/* ]]; then From 1055ade18f5d4dc015bb0d163e1e33dac0d9428b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:47:25 +0000 Subject: [PATCH 24/49] Pin GitHub Actions to commit SHAs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03a6c0d..fc23676 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,12 +32,12 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: go-version-file: 'go.mod' @@ -52,7 +52,7 @@ jobs: run: for x in dist/"${NAME}"-windows-*; do mv "$x" "$x.exe"; done - name: Upload binary - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ env.NAME }}-${{ matrix.GOOS }}-${{ matrix.GOARCH }}-${{ github.sha }} path: dist/* @@ -67,7 +67,7 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: persist-credentials: false From 74eb537717412992af86ff2e2714d8878c93c843 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:54:53 +0000 Subject: [PATCH 25/49] Update flake inputs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index a87f67d..c1c295a 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1725534445, - "narHash": "sha256-Yd0FK9SkWy+ZPuNqUgmVPXokxDgMJoGuNpMEtkfcf84=", + "lastModified": 1790178388, + "narHash": "sha256-kK3t7gwoz4Nx8RF46cs1Xz/skKNcK7y3KYP+gGqX6W8=", "owner": "nixos", "repo": "nixpkgs", - "rev": "9bb1e7571aadf31ddb4af77fc64b2d59580f9a39", + "rev": "00455b0a3690d3f5dc61e9aef4277dc86235b73f", "type": "github" }, "original": { From 79bafb643be7fb0aa122f609c9486db8890b8388 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:56:01 +0000 Subject: [PATCH 26/49] Bump Go to 1.26 and update dependencies Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- flake.nix | 2 +- go.mod | 21 ++++++++------------ go.sum | 58 +++++++++++++++---------------------------------------- 3 files changed, 25 insertions(+), 56 deletions(-) diff --git a/flake.nix b/flake.nix index 7e85e54..6f7ea9a 100644 --- a/flake.nix +++ b/flake.nix @@ -47,7 +47,7 @@ # Update the hash if go dependencies change! # vendorHash = pkgs.lib.fakeHash; - vendorHash = "sha256-mo30V7ISVFY8Rl3yXChP6pbehV9hTPH3UlBLDb1dzNE="; + vendorHash = "sha256-EQOlQCllo/nwJTv/ZhlR/ORKj1Kt6+mFHPIbEFE3QvI="; ldflags = [ "-s" diff --git a/go.mod b/go.mod index 95cef4d..9bad6dd 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,20 @@ module github.com/Brawl345/rssbot -go 1.22 +go 1.26.0 require ( - github.com/go-sql-driver/mysql v1.8.1 + github.com/go-sql-driver/mysql v1.10.1 github.com/jmoiron/sqlx v1.4.0 github.com/joho/godotenv v1.5.1 - github.com/mmcdole/gofeed v1.3.0 - github.com/rubenv/sql-migrate v1.7.0 + github.com/mmcdole/gofeed v1.4.2 + github.com/rubenv/sql-migrate v1.8.1 gopkg.in/telebot.v3 v3.3.8 ) require ( - filippo.io/edwards25519 v1.1.0 // indirect - github.com/PuerkitoBio/goquery v1.8.1 // indirect - github.com/andybalholm/cascadia v1.3.2 // indirect + filippo.io/edwards25519 v1.2.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mmcdole/goxpp v1.1.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/text v0.18.0 // indirect + github.com/mmcdole/goxpp/v2 v2.0.0 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/text v0.42.0 // indirect ) diff --git a/go.sum b/go.sum index 5d12532..d97688d 100644 --- a/go.sum +++ b/go.sum @@ -54,22 +54,18 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= -github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= -github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -137,8 +133,9 @@ github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvSc github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -263,7 +260,6 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -312,16 +308,14 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= -github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= -github.com/mmcdole/goxpp v1.1.1 h1:RGIX+D6iQRIunGHrKqnA2+700XMCnNv0bAOOv5MUhx8= -github.com/mmcdole/goxpp v1.1.1/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/mmcdole/gofeed v1.4.2 h1:XFFOtsNNZg+zudjtMXb8BI0J/YdnSIGfjEpPgPnZib0= +github.com/mmcdole/gofeed v1.4.2/go.mod h1:X5x1PyeibJi152VEya0AsV+PW4daYmCD4LJaJbeFkcs= +github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ= +github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -362,8 +356,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= -github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= +github.com/rubenv/sql-migrate v1.8.1 h1:EPNwCvjAowHI3TnZ+4fQu3a915OpnQoPAjTXCGOy2U0= +github.com/rubenv/sql-migrate v1.8.1/go.mod h1:BTIKBORjzyxZDS6dzoiw6eAFYJ1iNlGAtjn4LGeVjS8= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -388,8 +382,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -397,7 +391,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= @@ -421,7 +414,6 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -459,8 +451,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -501,7 +491,6 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -509,12 +498,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -547,8 +532,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -627,13 +610,8 @@ golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -643,10 +621,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -705,8 +681,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From d581b533a351e50385a5896eff57174db2e1ad1b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:56:33 +0000 Subject: [PATCH 27/49] Use Go 1.27 and Debian 13 base images pinned by digest Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7f931e0..2fc222f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22 AS build-stage +FROM golang:1.27.1@sha256:3680233e3204827fbdc66088528ae6d4b3d034f51d03a99d454f6de034888244 AS build-stage WORKDIR /app COPY go.mod go.sum ./ RUN go mod download @@ -6,7 +6,7 @@ COPY . ./ ARG VERSION=dev RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X github.com/Brawl345/rssbot/fetcher.Version=${VERSION}" -o /rssbot -FROM gcr.io/distroless/static-debian12 AS release-stage +FROM gcr.io/distroless/static-debian13:nonroot@sha256:e2e927ec666bae08560abb3c55d0659eceabb657f56b6782ab500a9fc7f555e3 AS release-stage WORKDIR /app COPY --from=build-stage /rssbot /app/rssbot USER nonroot:nonroot From e7275c8828dd7dd55bd990543fefa9716514e448 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:57:07 +0000 Subject: [PATCH 28/49] Update GitHub Actions to latest versions Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc23676..9d37c9f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,12 +32,12 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Setup Go - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' @@ -52,7 +52,7 @@ jobs: run: for x in dist/"${NAME}"-windows-*; do mv "$x" "$x.exe"; done - name: Upload binary - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ env.NAME }}-${{ matrix.GOOS }}-${{ matrix.GOARCH }}-${{ github.sha }} path: dist/* @@ -67,7 +67,7 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false From d8a584bf29ed590f3901c38683627d9df4e98b8f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 11:57:18 +0000 Subject: [PATCH 29/49] Build release binaries with latest stable Go Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d37c9f..8849cc9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,7 +39,7 @@ jobs: - name: Setup Go uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version-file: 'go.mod' + go-version: stable - name: Build env: From f0b5201bb3babef778930323c48e3e6fc328a914 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:00:37 +0000 Subject: [PATCH 30/49] Handle ignored errors and use EXISTS for existence checks Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- fetcher/fetcher.go | 6 ++++-- fetcher/fetcher_test.go | 12 ++++++------ handler/feed_subscribe.go | 6 +++++- handler/feed_unsubscribe.go | 6 +++++- storage/abonnements.go | 31 +++++++++++++++++-------------- 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go index 28aa0fd..6b043d2 100644 --- a/fetcher/fetcher.go +++ b/fetcher/fetcher.go @@ -106,7 +106,9 @@ func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string) if isRedirect(resp.StatusCode) { location := resp.Header.Get("Location") - resp.Body.Close() + // Drain a little so the connection can be reused for the next hop. + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) + _ = resp.Body.Close() if location == "" { return nil, fmt.Errorf("redirect status %d without Location header", resp.StatusCode) } @@ -143,7 +145,7 @@ func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string) } body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) - resp.Body.Close() + _ = resp.Body.Close() if err != nil { return nil, err } diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index 79c01fa..542e08b 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -45,7 +45,7 @@ func TestFetchSetsHeadersAndParses(t *testing.T) { w.Header().Set("Etag", `"abc"`) w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") w.Header().Set("Cache-Control", "max-age=1800") - w.Write([]byte(sampleRSS)) + _, _ = w.Write([]byte(sampleRSS)) })) defer srv.Close() @@ -89,7 +89,7 @@ func TestFetchSetsHeadersAndParses(t *testing.T) { func TestSyndicationHintIgnored(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(strings.Replace(sampleRSS, "90", "", 1))) + _, _ = w.Write([]byte(strings.Replace(sampleRSS, "90", "", 1))) })) defer srv.Close() @@ -147,7 +147,7 @@ func TestRetryAfterParsed(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Retry-After", "120") w.WriteHeader(http.StatusTooManyRequests) - w.Write([]byte("slow down")) + _, _ = w.Write([]byte("slow down")) })) defer srv.Close() @@ -166,7 +166,7 @@ func TestRetryAfterParsed(t *testing.T) { func TestPermanentRedirectReported(t *testing.T) { var target *httptest.Server target = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(sampleRSS)) + _, _ = w.Write([]byte(sampleRSS)) })) defer target.Close() @@ -204,7 +204,7 @@ func TestPermanentRedirectToErrorNotReported(t *testing.T) { func TestTemporaryRedirectNotPersisted(t *testing.T) { target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(sampleRSS)) + _, _ = w.Write([]byte(sampleRSS)) })) defer target.Close() @@ -251,7 +251,7 @@ func TestRedirectToUnsupportedSchemeRefused(t *testing.T) { func TestNonFeedRejected(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("not a feed")) + _, _ = w.Write([]byte("not a feed")) })) defer srv.Close() diff --git a/handler/feed_subscribe.go b/handler/feed_subscribe.go index 25b85ad..69a9cd4 100644 --- a/handler/feed_subscribe.go +++ b/handler/feed_subscribe.go @@ -97,7 +97,11 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { log.Printf("subscribe %s: could not reactivate: %s", feedUrl, err) } - exists, _ := h.DB.Abonnements.ExistsByFeedUrl(chatId, feedUrl) + exists, err := h.DB.Abonnements.ExistsByFeedUrl(chatId, feedUrl) + if err != nil { + log.Println(err) + return c.Send("❌ Beim Abonnieren des Feeds ist ein Fehler aufgetreten.", defaultSendOptions) + } if exists { if reactivated { return c.Send("✅ Der deaktivierte Feed wurde wieder aktiviert.", defaultSendOptions) diff --git a/handler/feed_unsubscribe.go b/handler/feed_unsubscribe.go index 3b99c02..e89027d 100644 --- a/handler/feed_unsubscribe.go +++ b/handler/feed_unsubscribe.go @@ -39,7 +39,11 @@ func (h *Handler) OnUnsubscribe(c telebot.Context) error { chatId = chatInfo.ID } - exists, _ := h.DB.Abonnements.ExistsById(chatId, feedId) + exists, err := h.DB.Abonnements.ExistsById(chatId, feedId) + if err != nil { + log.Println(err) + return c.Send("❌ Beim Deabonnieren ist ein Fehler aufgetreten.", defaultSendOptions) + } if !exists { return c.Send("❌ Dieser Feed wurde nicht abonniert.", defaultSendOptions) diff --git a/storage/abonnements.go b/storage/abonnements.go index 5e91a0e..6cf0b76 100644 --- a/storage/abonnements.go +++ b/storage/abonnements.go @@ -110,7 +110,7 @@ func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, la return err } - defer tx.Rollback() + defer func() { _ = tx.Rollback() }() const feedQuery = "SELECT id FROM feeds WHERE url = ?" var feedId int64 @@ -156,7 +156,7 @@ func (db *Abonnements) Delete(chatId int64, feedId int64) error { return err } - defer tx.Rollback() + defer func() { _ = tx.Rollback() }() const deleteAbonnementQuery = "DELETE FROM abonnements WHERE abonnements.chat_id = ? AND abonnements.feed_id = ?" _, err = tx.Exec(deleteAbonnementQuery, chatId, feedId) @@ -165,9 +165,11 @@ func (db *Abonnements) Delete(chatId int64, feedId int64) error { } // Check if user has other abonnements - const hasOtherAbonnementsQuery = "SELECT 1 FROM abonnements WHERE abonnements.chat_id = ?" + const hasOtherAbonnementsQuery = "SELECT EXISTS(SELECT 1 FROM abonnements WHERE abonnements.chat_id = ?)" var hasOtherAbonnements bool - tx.Get(&hasOtherAbonnements, hasOtherAbonnementsQuery, chatId) + if err = tx.Get(&hasOtherAbonnements, hasOtherAbonnementsQuery, chatId); err != nil { + return err + } if !hasOtherAbonnements { const deleteChatQuery = "DELETE FROM chats WHERE chats.id = ?" @@ -178,9 +180,11 @@ func (db *Abonnements) Delete(chatId int64, feedId int64) error { } // Check if feed has abonnement from other users - const hasOtherUsersQuery = "SELECT 1 FROM abonnements WHERE abonnements.feed_id = ?" + const hasOtherUsersQuery = "SELECT EXISTS(SELECT 1 FROM abonnements WHERE abonnements.feed_id = ?)" var hasOtherUsers bool - tx.Get(&hasOtherUsers, hasOtherUsersQuery, feedId) + if err = tx.Get(&hasOtherUsers, hasOtherUsersQuery, feedId); err != nil { + return err + } if !hasOtherUsers { const deleteFeedQuery = "DELETE FROM feeds WHERE feeds.id = ?" @@ -198,11 +202,10 @@ func (db *Abonnements) Delete(chatId int64, feedId int64) error { } func (db *Abonnements) ExistsByFeedUrl(chatId int64, feedUrl string) (bool, error) { - const query = `SELECT 1 FROM abonnements -JOIN chats ON abonnements.chat_id = chats.id + const query = `SELECT EXISTS(SELECT 1 FROM abonnements JOIN feeds ON abonnements.feed_id = feeds.id -WHERE chats.id = ? -AND feeds.url = ?` +WHERE abonnements.chat_id = ? +AND feeds.url = ?)` var exists bool err := db.Get(&exists, query, chatId, feedUrl) @@ -210,9 +213,9 @@ AND feeds.url = ?` } func (db *Abonnements) ExistsById(chatId int64, feedId int64) (bool, error) { - const query = `SELECT 1 FROM abonnements + const query = `SELECT EXISTS(SELECT 1 FROM abonnements WHERE abonnements.chat_id = ? -AND abonnements.feed_id = ?` +AND abonnements.feed_id = ?)` var exists bool err := db.Get(&exists, query, chatId, feedId) @@ -264,7 +267,7 @@ func (db *Abonnements) GetDue() ([]Abonnement, error) { } func scanAbonnements(rows *sqlx.Rows) ([]Abonnement, error) { - defer rows.Close() + defer func() { _ = rows.Close() }() feeds := make(map[int64]Feed) feedChats := make(map[int64][]Chat) @@ -337,7 +340,7 @@ func (db *Abonnements) MoveFeedURL(feedID int64, newURL string) (bool, error) { if err != nil { return false, err } - defer tx.Rollback() + defer func() { _ = tx.Rollback() }() var targetID int64 err = tx.Get(&targetID, "SELECT id FROM feeds WHERE url = ?", newURL) From 822b0486f5139c2dc706c37df16e3cc0bdd21d5f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:00:52 +0000 Subject: [PATCH 31/49] Fix staticcheck findings Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- fetcher/fetcher_test.go | 3 +-- handler/feed_check.go | 6 +++--- handler/feed_list.go | 4 ++-- handler/feed_list_all.go | 5 ++--- handler/replacement_list.go | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go index 542e08b..3c32746 100644 --- a/fetcher/fetcher_test.go +++ b/fetcher/fetcher_test.go @@ -164,8 +164,7 @@ func TestRetryAfterParsed(t *testing.T) { } func TestPermanentRedirectReported(t *testing.T) { - var target *httptest.Server - target = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(sampleRSS)) })) defer target.Close() diff --git a/handler/feed_check.go b/handler/feed_check.go index ed35355..570190f 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -103,7 +103,7 @@ func (h *Handler) pollFeeds(abonnements []storage.Abonnement, replacements []com groups := make(map[string][]storage.Abonnement) var hosts []string for _, abonnement := range abonnements { - host := feedHost(abonnement.Feed.Url) + host := feedHost(abonnement.Url) if _, ok := groups[host]; !ok { hosts = append(hosts, host) } @@ -371,8 +371,8 @@ func (h *Handler) notify(abonnement storage.Abonnement, text string) { if h.AdminID == 0 { return } - if err := h.sendText(h.AdminID, text, abonnement.Feed.Url); err != nil { - log.Printf("%s: notify failed: %s", abonnement.Feed.Url, err) + if err := h.sendText(h.AdminID, text, abonnement.Url); err != nil { + log.Printf("%s: notify failed: %s", abonnement.Url, err) } } diff --git a/handler/feed_list.go b/handler/feed_list.go index 8f0bf6d..454ad1b 100644 --- a/handler/feed_list.go +++ b/handler/feed_list.go @@ -56,10 +56,10 @@ func (h *Handler) OnList(c telebot.Context) error { } var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s hat abonniert:\n", html.EscapeString(chatTitle))) + fmt.Fprintf(&sb, "%s hat abonniert:\n", html.EscapeString(chatTitle)) for _, link := range links { - sb.WriteString(fmt.Sprintf("%d) %s%s\n", link.ID, html.EscapeString(link.Url), feedStatus(link))) + fmt.Fprintf(&sb, "%d) %s%s\n", link.ID, html.EscapeString(link.Url), feedStatus(link)) } return c.Send(sb.String(), defaultSendOptions) diff --git a/handler/feed_list_all.go b/handler/feed_list_all.go index a4b41d2..63a5628 100644 --- a/handler/feed_list_all.go +++ b/handler/feed_list_all.go @@ -27,11 +27,10 @@ func (h *Handler) OnListAll(c telebot.Context) error { sb := strings.Builder{} for _, abonnement := range abonnements { - sb.WriteString(fmt.Sprintf("%d) %s%s\n", abonnement.Feed.ID, html.EscapeString(abonnement.Feed.Url), feedStatus(abonnement.Feed))) + fmt.Fprintf(&sb, "%d) %s%s\n", abonnement.ID, html.EscapeString(abonnement.Url), feedStatus(abonnement.Feed)) for _, chat := range abonnement.Chats { - sb.WriteString(fmt.Sprintf(" %d (%s)\n", chat.ID, - html.EscapeString(chat.Title))) + fmt.Fprintf(&sb, " %d (%s)\n", chat.ID, html.EscapeString(chat.Title)) } sb.WriteString("\n") diff --git a/handler/replacement_list.go b/handler/replacement_list.go index fb29702..3da0b1c 100644 --- a/handler/replacement_list.go +++ b/handler/replacement_list.go @@ -29,7 +29,7 @@ func (h *Handler) OnListReplacements(c telebot.Context) error { sb := strings.Builder{} for _, replacement := range replacements { - sb.WriteString(fmt.Sprintf("%d) %s", replacement.ID, html.EscapeString(replacement.Value))) + fmt.Fprintf(&sb, "%d) %s", replacement.ID, html.EscapeString(replacement.Value)) if replacement.IsRegex { sb.WriteString(" (RegEx)") } From 571858691179f055031589704fc4bec6a49fe414 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:03:04 +0000 Subject: [PATCH 32/49] Add handler unit and polling integration tests Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/feed_check_test.go | 250 +++++++++++++++++++- handler/poll_test.go | 459 +++++++++++++++++++++++++++++++++++++ 2 files changed, 708 insertions(+), 1 deletion(-) create mode 100644 handler/poll_test.go diff --git a/handler/feed_check_test.go b/handler/feed_check_test.go index 6463754..9088c50 100644 --- a/handler/feed_check_test.go +++ b/handler/feed_check_test.go @@ -1,13 +1,251 @@ package handler import ( + "database/sql" "strings" "testing" + "time" "unicode/utf8" + + "github.com/Brawl345/rssbot/config" + "github.com/Brawl345/rssbot/fetcher" + "github.com/Brawl345/rssbot/storage" + "github.com/mmcdole/gofeed" ) +func testPollConfig() config.PollConfig { + return config.PollConfig{ + Interval: 10 * time.Minute, + IntervalMax: 6 * time.Hour, + Adaptive: true, + Concurrency: 4, + Tick: time.Second, + } +} + +func handlerWithPoll(poll config.PollConfig) *Handler { + return &Handler{Config: &config.Config{Poll: poll}} +} + +func assertDelay(t *testing.T, next time.Time, want time.Duration) { + t.Helper() + got := time.Until(next) + if got < want-time.Second || got > want+time.Second { + t.Errorf("next poll in %s, want %s", got.Round(time.Second), want) + } +} + +func TestNextPollAdaptive(t *testing.T) { + h := handlerWithPoll(testPollConfig()) + + tests := []struct { + unchanged int + want time.Duration + }{ + {0, 10 * time.Minute}, + {1, 20 * time.Minute}, + {2, 30 * time.Minute}, + {35, 6 * time.Hour}, + {36, 6 * time.Hour}, + {1 << 30, 6 * time.Hour}, + } + for _, tt := range tests { + assertDelay(t, h.nextPoll(tt.unchanged, nil), tt.want) + } +} + +func TestNextPollNotAdaptive(t *testing.T) { + poll := testPollConfig() + poll.Adaptive = false + assertDelay(t, handlerWithPoll(poll).nextPoll(50, nil), 10*time.Minute) +} + +func TestNextPollHints(t *testing.T) { + h := handlerWithPoll(testPollConfig()) + + tests := []struct { + name string + result fetcher.Result + want time.Duration + }{ + {"max-age slows down", fetcher.Result{MaxAge: time.Hour}, time.Hour}, + {"ttl slows down", fetcher.Result{FeedInterval: 90 * time.Minute}, 90 * time.Minute}, + {"larger hint wins", fetcher.Result{MaxAge: time.Hour, FeedInterval: 2 * time.Hour}, 2 * time.Hour}, + {"hint never speeds up", fetcher.Result{MaxAge: time.Minute}, 10 * time.Minute}, + {"hint capped", fetcher.Result{MaxAge: 365 * 24 * time.Hour}, 6 * time.Hour}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertDelay(t, h.nextPoll(0, &tt.result), tt.want) + }) + } +} + +func TestErrorBackoff(t *testing.T) { + h := handlerWithPoll(testPollConfig()) + + tests := []struct { + errors int + want time.Duration + }{ + {1, 10 * time.Minute}, + {2, 20 * time.Minute}, + {3, 40 * time.Minute}, + {6, 320 * time.Minute}, + {7, 6 * time.Hour}, + {100, 6 * time.Hour}, + } + for _, tt := range tests { + if got := h.errorBackoff(tt.errors); got != tt.want { + t.Errorf("errorBackoff(%d) = %s, want %s", tt.errors, got, tt.want) + } + } +} + +func TestApplySkip(t *testing.T) { + // 2026-01-04 is a Sunday. + at := func(day, hour int) time.Time { return time.Date(2026, 1, day, hour, 30, 0, 0, time.UTC) } + + tests := []struct { + name string + t time.Time + hours []int + days []string + want time.Time + }{ + {"no hints", at(4, 23), nil, nil, at(4, 23)}, + {"outside skipHours", at(4, 22), []int{23, 0}, nil, at(4, 22)}, + {"inside skipHours", at(4, 23), []int{23, 0}, nil, at(5, 1)}, + {"skipDays", at(4, 10), nil, []string{"Sunday"}, time.Date(2026, 1, 5, 0, 30, 0, 0, time.UTC)}, + {"skipDays is case-insensitive", at(4, 10), nil, []string{" sunday "}, time.Date(2026, 1, 5, 0, 30, 0, 0, time.UTC)}, + {"everything skipped is capped", at(4, 10), allHours(), nil, at(6, 10)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := applySkip(tt.t, tt.hours, tt.days); !got.Equal(tt.want) { + t.Errorf("applySkip = %s, want %s", got, tt.want) + } + }) + } +} + +func allHours() []int { + hours := make([]int, 24) + for i := range hours { + hours[i] = i + } + return hours +} + +func TestFilterNewItems(t *testing.T) { + items := []*gofeed.Item{ + {GUID: "3", Link: "https://example.org/3"}, + {GUID: "", Link: "https://example.org/2"}, + {GUID: "1", Link: "https://example.org/1"}, + } + ptr := func(s string) *string { return &s } + + tests := []struct { + name string + lastEntry *string + want int + }{ + {"first poll", nil, 3}, + {"match by GUID", ptr("1"), 2}, + {"match by link", ptr("https://example.org/2"), 1}, + {"newest already seen", ptr("3"), 0}, + {"unknown entry returns all", ptr("gone"), 3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := filterNewItems(items, tt.lastEntry); len(got) != tt.want { + t.Errorf("got %d items, want %d", len(got), tt.want) + } + }) + } +} + +func TestReverse(t *testing.T) { + items := []*gofeed.Item{{GUID: "a"}, {GUID: "b"}, {GUID: "c"}} + got := reverse(items) + if got[0].GUID != "c" || got[2].GUID != "a" { + t.Errorf("reverse = %v", []string{got[0].GUID, got[1].GUID, got[2].GUID}) + } + if items[0].GUID != "a" { + t.Errorf("reverse modified its input") + } +} + +func TestFeedHost(t *testing.T) { + tests := map[string]string{ + "https://Example.org/feed.xml": "example.org", + "http://example.org:8080/rss": "example.org:8080", + "not a url": "not a url", + "https://www.example.org/a?b=c": "www.example.org", + } + for in, want := range tests { + if got := feedHost(in); got != want { + t.Errorf("feedHost(%q) = %q, want %q", in, got, want) + } + } +} + +func TestSanitizeFeedURL(t *testing.T) { + valid := map[string]string{ + "https://example.org/feed": "https://example.org/feed", + " http://example.org/rss.xml ": "http://example.org/rss.xml", + "https://example.org/?a=1&b=2": "https://example.org/?a=1&b=2", + } + for in, want := range valid { + got, err := sanitizeFeedURL(in) + if err != nil || got != want { + t.Errorf("sanitizeFeedURL(%q) = %q, %v; want %q", in, got, err, want) + } + } + + invalid := []string{ + "", + " ", + "ftp://example.org/feed", + "example.org/feed", + "https:///feed", + "https://example.org/a b", + `https://example.org/"feed"`, + "https://example.org/", + "javascript:alert(1)", + } + for _, in := range invalid { + if got, err := sanitizeFeedURL(in); err == nil { + t.Errorf("sanitizeFeedURL(%q) = %q, want error", in, got) + } + } +} + +func TestCompileReplacementsSkipsInvalidRegex(t *testing.T) { + compiled := compileReplacements([]storage.Replacement{ + {Value: "[mehr]"}, + {Value: "([", IsRegex: true}, + {Value: "<.*?>", IsRegex: true}, + }) + if len(compiled) != 2 { + t.Fatalf("got %d replacements, want 2", len(compiled)) + } +} + +func TestProcessContent(t *testing.T) { + replacements := compileReplacements([]storage.Replacement{ + {Value: "[mehr]"}, + {Value: "<.*?>", IsRegex: true}, + }) + + got := processContent("

Hallo & Tschüss [mehr]

\n\n \n

Zeile 2

", replacements) + if want := "Hallo & Tschüss \nZeile 2"; got != want { + t.Errorf("processContent = %q, want %q", got, want) + } +} + func TestProcessContentKeepsValidUTF8(t *testing.T) { - out := processContent(strings.Repeat("ü", 300), nil) + out := processContent(strings.Repeat("ü", 300)+"\xff", nil) if !utf8.ValidString(out) { t.Fatalf("output is not valid UTF-8") } @@ -15,3 +253,13 @@ func TestProcessContentKeepsValidUTF8(t *testing.T) { t.Errorf("unexpected truncation: %d runes", utf8.RuneCountInString(out)) } } + +func TestFeedStatus(t *testing.T) { + if got := feedStatus(storage.Feed{}); got != "" { + t.Errorf("active feed status = %q, want empty", got) + } + disabled := storage.Feed{Disabled: true, DisabledReason: sql.NullString{String: "HTTP <410>", Valid: true}} + if got := feedStatus(disabled); !strings.Contains(got, "HTTP <410>") { + t.Errorf("disabled feed status = %q, want escaped reason", got) + } +} diff --git a/handler/poll_test.go b/handler/poll_test.go new file mode 100644 index 0000000..c6e04a9 --- /dev/null +++ b/handler/poll_test.go @@ -0,0 +1,459 @@ +package handler + +import ( + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "text/template" + "time" + + "github.com/Brawl345/rssbot/config" + "github.com/Brawl345/rssbot/fetcher" + "github.com/Brawl345/rssbot/storage" + "gopkg.in/telebot.v3" +) + +const adminID = 999 + +type sentMessage struct { + ChatID int64 + Text string +} + +// fakeTelegram answers sendMessage calls like the Bot API and records them. +type fakeTelegram struct { + mu sync.Mutex + sent []sentMessage +} + +func (f *fakeTelegram) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var params struct { + ChatID string `json:"chat_id"` + Text string `json:"text"` + } + _ = json.NewDecoder(r.Body).Decode(¶ms) + var chatID int64 + _, _ = fmt.Sscan(params.ChatID, &chatID) + + f.mu.Lock() + f.sent = append(f.sent, sentMessage{ChatID: chatID, Text: params.Text}) + f.mu.Unlock() + + _, _ = fmt.Fprintf(w, `{"ok":true,"result":{"message_id":1,"date":0,"chat":{"id":%d,"type":"private"}}}`, chatID) +} + +func (f *fakeTelegram) messages() []sentMessage { + f.mu.Lock() + defer f.mu.Unlock() + return append([]sentMessage(nil), f.sent...) +} + +func (f *fakeTelegram) messagesTo(chatID int64) []string { + var texts []string + for _, m := range f.messages() { + if m.ChatID == chatID { + texts = append(texts, m.Text) + } + } + return texts +} + +type stateCall struct { + FeedID int64 + LastEntry *string + ETag *string + LastModified *string + Hints storage.PollHints + NextPollAt time.Time + ErrorCount int + UnchangedCount int +} + +type rescheduleCall struct { + FeedID int64 + NextPollAt time.Time + ErrorCount int + UnchangedCount int +} + +// fakeStore records the scheduling writes of the poller. Methods the poller +// does not use are left to the embedded nil interface and panic if called. +type fakeStore struct { + storage.AbonnementStorage + + mu sync.Mutex + states []stateCall + reschedules []rescheduleCall + disabled map[int64]string + moved map[int64]string + mergeOnMove bool +} + +func newFakeStore() *fakeStore { + return &fakeStore{disabled: map[int64]string{}, moved: map[int64]string{}} +} + +func (s *fakeStore) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, hints storage.PollHints, nextPollAt time.Time, errorCount, unchangedCount int) error { + s.mu.Lock() + defer s.mu.Unlock() + s.states = append(s.states, stateCall{feedID, lastEntry, etag, lastModified, hints, nextPollAt, errorCount, unchangedCount}) + return nil +} + +func (s *fakeStore) Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error { + s.mu.Lock() + defer s.mu.Unlock() + s.reschedules = append(s.reschedules, rescheduleCall{feedID, nextPollAt, errorCount, unchangedCount}) + return nil +} + +func (s *fakeStore) DisableFeed(feedID int64, reason string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.disabled[feedID] = reason + return nil +} + +func (s *fakeStore) MoveFeedURL(feedID int64, newURL string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.moved[feedID] = newURL + return s.mergeOnMove, nil +} + +type pollEnv struct { + h *Handler + store *fakeStore + tg *fakeTelegram +} + +func newPollEnv(t *testing.T) *pollEnv { + t.Helper() + + tg := &fakeTelegram{} + tgServer := httptest.NewServer(tg) + t.Cleanup(tgServer.Close) + + bot, err := telebot.NewBot(telebot.Settings{Token: "test", URL: tgServer.URL, Offline: true}) + if err != nil { + t.Fatal(err) + } + + tmpl, err := config.GetTemplate("does-not-exist.gohtml") + if err != nil { + t.Fatal(err) + } + + store := newFakeStore() + return &pollEnv{ + h: &Handler{ + Bot: bot, + Config: &config.Config{Template: tmpl, Poll: testPollConfig()}, + DB: &storage.DB{Abonnements: store}, + Fetcher: fetcher.New(), + AdminID: adminID, + }, + store: store, + tg: tg, + } +} + +func feedServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return srv +} + +func abonnement(url string, chats ...int64) storage.Abonnement { + ab := storage.Abonnement{Feed: storage.Feed{ID: 1, Url: url}} + for _, id := range chats { + ab.Chats = append(ab.Chats, storage.Chat{ID: id}) + } + return ab +} + +func rss(ttl string, guids ...string) string { + var sb strings.Builder + sb.WriteString(`Test & Feedhttps://example.org`) + sb.WriteString(ttl) + for _, guid := range guids { + fmt.Fprintf(&sb, `Item %shttps://www.example.org/%s%sText %s`, guid, guid, guid, guid) + } + sb.WriteString(``) + return sb.String() +} + +func TestPollSendsNewItemsOldestFirst(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Etag", `"v2"`) + _, _ = w.Write([]byte(rss("120", "3", "2", "1"))) + }) + + ab := abonnement(srv.URL, 10, 11) + ab.LastEntry = sql.NullString{String: "1", Valid: true} + env.h.pollFeed(ab, nil) + + for _, chat := range []int64{10, 11} { + msgs := env.tg.messagesTo(chat) + if len(msgs) != 2 { + t.Fatalf("chat %d got %d messages, want 2", chat, len(msgs)) + } + if !strings.Contains(msgs[0], "Item 2") || !strings.Contains(msgs[1], "Item 3") { + t.Errorf("chat %d messages not oldest first: %q", chat, msgs) + } + if !strings.Contains(msgs[0], "Test & Feed") || !strings.Contains(msgs[0], "Weiterlesen auf example.org") { + t.Errorf("unexpected message rendering: %q", msgs[0]) + } + } + if msgs := env.tg.messagesTo(adminID); len(msgs) != 0 { + t.Errorf("admin got unexpected messages: %q", msgs) + } + + if len(env.store.states) != 1 { + t.Fatalf("SetFeedState called %d times, want 1", len(env.store.states)) + } + state := env.store.states[0] + if state.LastEntry == nil || *state.LastEntry != "3" { + t.Errorf("last entry = %v, want 3", state.LastEntry) + } + if state.ETag == nil || *state.ETag != `"v2"` { + t.Errorf("etag = %v, want \"v2\"", state.ETag) + } + if state.UnchangedCount != 0 || state.ErrorCount != 0 { + t.Errorf("counters = %d/%d, want 0/0", state.ErrorCount, state.UnchangedCount) + } + if state.Hints.Interval != 2*time.Hour { + t.Errorf("stored ttl hint = %s, want 2h", state.Hints.Interval) + } + assertDelay(t, state.NextPollAt, 2*time.Hour) +} + +func TestPollUnchangedFeedIncreasesInterval(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(rss("", "1"))) + }) + + ab := abonnement(srv.URL, 10) + ab.LastEntry = sql.NullString{String: "1", Valid: true} + ab.UnchangedCount = 2 + env.h.pollFeed(ab, nil) + + if msgs := env.tg.messages(); len(msgs) != 0 { + t.Errorf("unexpected messages: %v", msgs) + } + state := env.store.states[0] + if state.UnchangedCount != 3 { + t.Errorf("unchanged = %d, want 3", state.UnchangedCount) + } + assertDelay(t, state.NextPollAt, 40*time.Minute) +} + +func TestPollNotModifiedUsesStoredHints(t *testing.T) { + env := newPollEnv(t) + var gotINM string + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + gotINM = r.Header.Get("If-None-Match") + w.WriteHeader(http.StatusNotModified) + }) + + ab := abonnement(srv.URL, 10) + ab.ETag = sql.NullString{String: `"v1"`, Valid: true} + ab.FeedInterval = int((3 * time.Hour).Seconds()) + env.h.pollFeed(ab, nil) + + if gotINM != `"v1"` { + t.Errorf("If-None-Match = %q, want stored etag", gotINM) + } + if len(env.store.states) != 0 { + t.Errorf("304 must not overwrite the cached state") + } + if len(env.store.reschedules) != 1 { + t.Fatalf("Reschedule called %d times, want 1", len(env.store.reschedules)) + } + r := env.store.reschedules[0] + if r.ErrorCount != 0 || r.UnchangedCount != 1 { + t.Errorf("counters = %d/%d, want 0/1", r.ErrorCount, r.UnchangedCount) + } + assertDelay(t, r.NextPollAt, 3*time.Hour) +} + +func TestPollGoneDisablesAndNotifiesAdmin(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusGone) + _, _ = w.Write([]byte("gone")) + }) + + env.h.pollFeed(abonnement(srv.URL+"/?a=1&b=2", 10), nil) + + if reason := env.store.disabled[1]; reason != "HTTP 410 Gone" { + t.Errorf("disabled reason = %q", reason) + } + if msgs := env.tg.messagesTo(10); len(msgs) != 0 { + t.Errorf("subscriber must not get operational messages: %q", msgs) + } + admin := env.tg.messagesTo(adminID) + if len(admin) != 1 { + t.Fatalf("admin got %d messages, want 1", len(admin)) + } + if !strings.Contains(admin[0], "a=1&b=2") || !strings.Contains(admin[0], "<b>gone</b>") { + t.Errorf("admin message not escaped: %q", admin[0]) + } +} + +func TestPollServerErrorBacksOff(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + + ab := abonnement(srv.URL, 10) + ab.ErrorCount = 2 + env.h.pollFeed(ab, nil) + + if len(env.store.disabled) != 0 { + t.Errorf("feed must not be disabled yet") + } + r := env.store.reschedules[0] + if r.ErrorCount != 3 { + t.Errorf("error count = %d, want 3", r.ErrorCount) + } + assertDelay(t, r.NextPollAt, 40*time.Minute) +} + +func TestPollRetiresFeedAfterLongFailure(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + recent := abonnement(srv.URL, 10) + recent.ErrorCount = 50 + recent.FailingSince = sql.NullTime{Time: time.Now().Add(-24 * time.Hour), Valid: true} + env.h.pollFeed(recent, nil) + if len(env.store.disabled) != 0 { + t.Fatalf("feed failing for one day must not be retired") + } + + old := abonnement(srv.URL, 10) + old.ErrorCount = maxFeedErrors - 1 + old.FailingSince = sql.NullTime{Time: time.Now().Add(-8 * 24 * time.Hour), Valid: true} + env.h.pollFeed(old, nil) + if reason := env.store.disabled[1]; reason != "HTTP 404" { + t.Errorf("disabled reason = %q, want HTTP 404", reason) + } +} + +func TestPollRateLimitKeepsErrorCount(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "3600") + w.WriteHeader(http.StatusTooManyRequests) + }) + + ab := abonnement(srv.URL, 10) + ab.ErrorCount = 1 + env.h.pollFeed(ab, nil) + + r := env.store.reschedules[0] + if r.ErrorCount != 1 { + t.Errorf("error count = %d, want unchanged 1", r.ErrorCount) + } + assertDelay(t, r.NextPollAt, time.Hour) + if len(env.tg.messagesTo(adminID)) != 1 { + t.Errorf("admin should be notified about 429") + } +} + +func TestPollPermanentRedirect(t *testing.T) { + target := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(rss("", "1"))) + }) + src := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusMovedPermanently) + }) + + t.Run("renamed", func(t *testing.T) { + env := newPollEnv(t) + env.h.pollFeed(abonnement(src.URL, 10), nil) + + if env.store.moved[1] != target.URL { + t.Errorf("moved to %q, want %q", env.store.moved[1], target.URL) + } + if len(env.store.states) != 1 { + t.Errorf("state of the renamed feed must be saved") + } + if len(env.tg.messagesTo(adminID)) != 1 { + t.Errorf("admin should be notified about the move") + } + }) + + t.Run("merged", func(t *testing.T) { + env := newPollEnv(t) + env.store.mergeOnMove = true + env.h.pollFeed(abonnement(src.URL, 10), nil) + + if len(env.store.states) != 0 || len(env.tg.messagesTo(10)) != 0 { + t.Errorf("merged feed must be left to the surviving feed") + } + }) +} + +func TestPollTemplateErrorStillSavesState(t *testing.T) { + env := newPollEnv(t) + env.h.Config.Template = template.Must(template.New("post").Parse("{{.Missing}}")) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(rss("", "2", "1"))) + }) + + env.h.pollFeed(abonnement(srv.URL, 10), nil) + + if len(env.tg.messages()) != 0 { + t.Errorf("no message can be rendered") + } + if len(env.store.states) != 1 { + t.Fatalf("state must be saved even if rendering fails") + } +} + +func TestPollFeedsSerializesSameHost(t *testing.T) { + env := newPollEnv(t) + var inFlight, maxInFlight, requests atomic.Int32 + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + n := inFlight.Add(1) + defer inFlight.Add(-1) + for { + m := maxInFlight.Load() + if n <= m || maxInFlight.CompareAndSwap(m, n) { + break + } + } + requests.Add(1) + time.Sleep(20 * time.Millisecond) + w.WriteHeader(http.StatusNotModified) + }) + + var abonnements []storage.Abonnement + for i := 1; i <= 5; i++ { + ab := abonnement(fmt.Sprintf("%s/feed%d", srv.URL, i), 10) + ab.ID = int64(i) + abonnements = append(abonnements, ab) + } + env.h.pollFeeds(abonnements, nil) + + if requests.Load() != 5 { + t.Errorf("got %d requests, want 5", requests.Load()) + } + if maxInFlight.Load() != 1 { + t.Errorf("max concurrent requests to one host = %d, want 1", maxInFlight.Load()) + } +} From edbc4bd93a833c68a4e194d07dab14dd5de97426 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:03:33 +0000 Subject: [PATCH 33/49] Add config tests Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- config/config_test.go | 75 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 config/config_test.go diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..909fd9c --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,75 @@ +package config + +import ( + "testing" + "time" +) + +func TestGetPollConfigDefaults(t *testing.T) { + for _, key := range []string{"POLL_INTERVAL", "POLL_INTERVAL_MAX", "POLL_ADAPTIVE", "POLL_CONCURRENCY", "POLL_TICK"} { + t.Setenv(key, "") + } + + want := PollConfig{ + Interval: 10 * time.Minute, + IntervalMax: 6 * time.Hour, + Adaptive: true, + Concurrency: 8, + Tick: 30 * time.Second, + } + if got := GetPollConfig(); got != want { + t.Errorf("GetPollConfig() = %+v, want %+v", got, want) + } +} + +func TestGetPollConfigFromEnv(t *testing.T) { + t.Setenv("POLL_INTERVAL", "1m") + t.Setenv("POLL_INTERVAL_MAX", "2h") + t.Setenv("POLL_ADAPTIVE", "false") + t.Setenv("POLL_CONCURRENCY", "3") + t.Setenv("POLL_TICK", "5s") + + want := PollConfig{ + Interval: time.Minute, + IntervalMax: 2 * time.Hour, + Adaptive: false, + Concurrency: 3, + Tick: 5 * time.Second, + } + if got := GetPollConfig(); got != want { + t.Errorf("GetPollConfig() = %+v, want %+v", got, want) + } +} + +func TestGetPollConfigInvalidValuesFallBack(t *testing.T) { + t.Setenv("POLL_INTERVAL", "10m # comment") + t.Setenv("POLL_INTERVAL_MAX", "-1h") + t.Setenv("POLL_ADAPTIVE", "maybe") + t.Setenv("POLL_CONCURRENCY", "0") + t.Setenv("POLL_TICK", "fast") + + got := GetPollConfig() + if got.Interval != 10*time.Minute || got.IntervalMax != 6*time.Hour || !got.Adaptive || + got.Concurrency != 8 || got.Tick != 30*time.Second { + t.Errorf("invalid values did not fall back to defaults: %+v", got) + } +} + +func TestGetPollConfigRaisesMaxToInterval(t *testing.T) { + t.Setenv("POLL_INTERVAL", "12h") + t.Setenv("POLL_INTERVAL_MAX", "1h") + + if got := GetPollConfig(); got.IntervalMax != 12*time.Hour { + t.Errorf("IntervalMax = %s, want 12h", got.IntervalMax) + } +} + +func TestGetTemplateDefault(t *testing.T) { + tmpl, err := GetTemplate("does-not-exist.gohtml") + if err != nil { + t.Fatal(err) + } + if tmpl.Name() != "post" { + t.Errorf("template name = %q, want post", tmpl.Name()) + } +} From 028951ff6a6810a3bdfedb31e61ecb318da1ecb5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:03:34 +0000 Subject: [PATCH 34/49] Extend fetcher tests Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- fetcher/parse_test.go | 163 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 fetcher/parse_test.go diff --git a/fetcher/parse_test.go b/fetcher/parse_test.go new file mode 100644 index 0000000..c1f8b9d --- /dev/null +++ b/fetcher/parse_test.go @@ -0,0 +1,163 @@ +package fetcher + +import ( + "compress/gzip" + "context" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestParseMaxAge(t *testing.T) { + tests := map[string]time.Duration{ + "": 0, + "max-age=600": 10 * time.Minute, + "public, max-age=3600, s-maxage=60": time.Hour, + "no-cache": 0, + "max-age=0": 0, + "max-age=-5": 0, + "max-age=abc": 0, + } + for in, want := range tests { + if got := parseMaxAge(in); got != want { + t.Errorf("parseMaxAge(%q) = %s, want %s", in, got, want) + } + } +} + +func TestParseRetryAfter(t *testing.T) { + tests := map[string]time.Duration{ + "": 0, + "120": 2 * time.Minute, + " 30 ": 30 * time.Second, + "-1": 0, + "garbage": 0, + "Wed, 21 Oct 2015 07:28:00 GMT": 0, + } + for in, want := range tests { + if got := parseRetryAfter(in); got != want { + t.Errorf("parseRetryAfter(%q) = %s, want %s", in, got, want) + } + } + + future := time.Now().Add(time.Hour).UTC().Format(http.TimeFormat) + if got := parseRetryAfter(future); got < 59*time.Minute || got > time.Hour { + t.Errorf("parseRetryAfter(%q) = %s, want ~1h", future, got) + } +} + +func TestIsPrivateIP(t *testing.T) { + tests := map[string]bool{ + "127.0.0.1": true, + "10.1.2.3": true, + "192.168.0.1": true, + "172.16.0.1": true, + "169.254.169.254": true, + "0.0.0.0": true, + "::1": true, + "fe80::1": true, + "fd00::1": true, + "1.1.1.1": false, + "2606:4700::1111": false, + } + for in, want := range tests { + if got := isPrivateIP(net.ParseIP(in)); got != want { + t.Errorf("isPrivateIP(%s) = %v, want %v", in, got, want) + } + } +} + +func TestIsPrivateHostLiteral(t *testing.T) { + ctx := context.Background() + if !isPrivateHost(ctx, "127.0.0.1") || isPrivateHost(ctx, "8.8.8.8") || isPrivateHost(ctx, "") { + t.Errorf("isPrivateHost misclassified an IP literal") + } +} + +func TestTooManyRedirects(t *testing.T) { + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, srv.URL+r.URL.Path+"x", http.StatusFound) + })) + defer srv.Close() + + _, err := New().Fetch(context.Background(), srv.URL+"/", "", "") + if err == nil || !strings.Contains(err.Error(), "too many redirects") { + t.Fatalf("err = %v, want too many redirects", err) + } +} + +func TestRedirectWithoutLocation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusMovedPermanently) + })) + defer srv.Close() + + if _, err := New().Fetch(context.Background(), srv.URL, "", ""); err == nil { + t.Fatal("expected error for redirect without Location") + } +} + +func TestPermanentChainBrokenByTemporaryRedirect(t *testing.T) { + final := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(sampleRSS)) + })) + defer final.Close() + temp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, final.URL, http.StatusTemporaryRedirect) + })) + defer temp.Close() + perm := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, temp.URL, http.StatusPermanentRedirect) + })) + defer perm.Close() + + res := fetch(t, New(), perm.URL, "", "") + if res.PermanentURL != temp.URL { + t.Errorf("PermanentURL = %q, want the last permanent target %q", res.PermanentURL, temp.URL) + } +} + +func TestGzipResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + http.Error(w, "gzip expected", http.StatusBadRequest) + return + } + w.Header().Set("Content-Encoding", "gzip") + gz := gzip.NewWriter(w) + _, _ = gz.Write([]byte(sampleRSS)) + _ = gz.Close() + })) + defer srv.Close() + + res := fetch(t, New(), srv.URL, "", "") + if res.Feed == nil || len(res.Feed.Items) != 1 { + t.Fatalf("gzip feed not decoded: status=%d", res.Status) + } +} + +func TestErrorStatusKeepsBodySnippet(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "maintenance", http.StatusInternalServerError) + })) + defer srv.Close() + + res := fetch(t, New(), srv.URL, "", "") + if res.Status != http.StatusInternalServerError || res.Body != "maintenance" || res.Feed != nil { + t.Errorf("unexpected result: status=%d body=%q", res.Status, res.Body) + } +} + +func TestUserAgentVersion(t *testing.T) { + old := Version + t.Cleanup(func() { Version = old }) + + Version = "abc1234" + if ua := buildUserAgent(); ua != "rssbot/abc1234 (+https://github.com/Brawl345/rssbot)" { + t.Errorf("User-Agent = %q", ua) + } +} From 2a4f209cc7ca5dcae5122f910ce753418cdf5d52 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:05:37 +0000 Subject: [PATCH 35/49] Add down migrations for initial schema Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- storage/migrations/0001-init.sql | 7 +++++++ storage/migrations/0002-replacements.sql | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/storage/migrations/0001-init.sql b/storage/migrations/0001-init.sql index 51cc6ae..18f56d2 100644 --- a/storage/migrations/0001-init.sql +++ b/storage/migrations/0001-init.sql @@ -31,3 +31,10 @@ CREATE TABLE IF NOT EXISTS `abonnements` CONSTRAINT `FK_abonnements_feeds` FOREIGN KEY (`feed_id`) REFERENCES `feeds` (`id`) ON UPDATE CASCADE ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; + + +-- +migrate Down + +DROP TABLE `abonnements`; +DROP TABLE `feeds`; +DROP TABLE `chats`; diff --git a/storage/migrations/0002-replacements.sql b/storage/migrations/0002-replacements.sql index 196afa9..97a878b 100644 --- a/storage/migrations/0002-replacements.sql +++ b/storage/migrations/0002-replacements.sql @@ -52,4 +52,8 @@ VALUES ('[←]', 0), ('Der Beitrag.*erschien zuerst auf .+.', 1), ('The post.*appeared first on .+.', 1), ('http://www.serienjunkies.de/.*.html', 1), - ('<.*?>', 1); \ No newline at end of file + ('<.*?>', 1); + +-- +migrate Down + +DROP TABLE `replacements`; From e5dc54bb086fdf3c74b5592e7b870117f7d4a848 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:05:38 +0000 Subject: [PATCH 36/49] Add storage integration tests Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- storage/integration_test.go | 384 ++++++++++++++++++++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 storage/integration_test.go diff --git a/storage/integration_test.go b/storage/integration_test.go new file mode 100644 index 0000000..2a281e7 --- /dev/null +++ b/storage/integration_test.go @@ -0,0 +1,384 @@ +package storage + +import ( + "errors" + "os" + "sort" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" + migrate "github.com/rubenv/sql-migrate" +) + +// Integration tests run against a real MySQL/MariaDB when RSSBOT_TEST_DSN is +// set, e.g. "user:pass@tcp(127.0.0.1:3306)/rssbot_test?parseTime=True&loc=Local". +// The database is wiped before every test. +func newTestDB(t *testing.T) *DB { + t.Helper() + dsn := os.Getenv("RSSBOT_TEST_DSN") + if dsn == "" { + t.Skip("RSSBOT_TEST_DSN not set") + } + + conn, err := sqlx.Connect("mysql", dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + + for _, table := range []string{"abonnements", "chats", "feeds", "replacements", "gorp_migrations"} { + if _, err := conn.Exec("DROP TABLE IF EXISTS " + table); err != nil { + t.Fatal(err) + } + } + + db := &DB{DB: conn, Abonnements: &Abonnements{DB: conn}, Replacements: &Replacements{DB: conn}} + if _, err := db.Migrate(); err != nil { + t.Fatal(err) + } + return db +} + +func mustCreate(t *testing.T, db *DB, chatID int64, url string, nextPollAt time.Time) { + t.Helper() + if err := db.Abonnements.Create(chatID, "chat", url, nil, nil, nil, PollHints{}, nextPollAt); err != nil { + t.Fatal(err) + } +} + +func feedByURL(t *testing.T, db *DB, url string) (Abonnement, bool) { + t.Helper() + all, err := db.Abonnements.GetAll() + if err != nil { + t.Fatal(err) + } + for _, ab := range all { + if ab.Url == url { + return ab, true + } + } + return Abonnement{}, false +} + +func dueURLs(t *testing.T, db *DB) []string { + t.Helper() + due, err := db.Abonnements.GetDue() + if err != nil { + t.Fatal(err) + } + var urls []string + for _, ab := range due { + urls = append(urls, ab.Url) + } + sort.Strings(urls) + return urls +} + +func chatIDs(ab Abonnement) []int64 { + var ids []int64 + for _, c := range ab.Chats { + ids = append(ids, c.ID) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + return ids +} + +func TestMigrationsRoundTrip(t *testing.T) { + db := newTestDB(t) + source := &migrate.EmbedFileSystemMigrationSource{FileSystem: embeddedMigrations, Root: "migrations"} + + down, err := migrate.Exec(db.DB.DB, "mysql", source, migrate.Down) + if err != nil { + t.Fatalf("migrate down: %v", err) + } + up, err := migrate.Exec(db.DB.DB, "mysql", source, migrate.Up) + if err != nil { + t.Fatalf("migrate up: %v", err) + } + if down != up || up == 0 { + t.Errorf("down=%d up=%d, want equal and non-zero", down, up) + } +} + +func TestGetDueRespectsSchedule(t *testing.T) { + db := newTestDB(t) + now := time.Now() + + mustCreate(t, db, 1, "https://example.org/past", now.Add(-time.Minute)) + mustCreate(t, db, 1, "https://example.org/soon", now.Add(time.Hour)) + mustCreate(t, db, 1, "https://example.org/later", now.Add(10*time.Hour)) + + got := dueURLs(t, db) + if len(got) != 1 || got[0] != "https://example.org/past" { + t.Errorf("due feeds = %v, want only the past one (time zone mismatch?)", got) + } +} + +func TestCreateSharesFeedBetweenChats(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now()) + mustCreate(t, db, 2, url, time.Now()) + + all, err := db.Abonnements.GetAll() + if err != nil { + t.Fatal(err) + } + if len(all) != 1 { + t.Fatalf("got %d feeds, want 1", len(all)) + } + if ids := chatIDs(all[0]); len(ids) != 2 || ids[0] != 1 || ids[1] != 2 { + t.Errorf("chats = %v, want [1 2]", ids) + } + + feeds, err := db.Abonnements.GetByUser(2) + if err != nil { + t.Fatalf("GetByUser must map every feeds column: %v", err) + } + if len(feeds) != 1 || feeds[0].Url != url { + t.Errorf("GetByUser = %+v", feeds) + } +} + +func TestExists(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now()) + ab, _ := feedByURL(t, db, url) + + checks := []struct { + name string + fn func() (bool, error) + want bool + }{ + {"by url", func() (bool, error) { return db.Abonnements.ExistsByFeedUrl(1, url) }, true}, + {"by url other chat", func() (bool, error) { return db.Abonnements.ExistsByFeedUrl(2, url) }, false}, + {"by url unknown", func() (bool, error) { return db.Abonnements.ExistsByFeedUrl(1, "https://nope") }, false}, + {"by id", func() (bool, error) { return db.Abonnements.ExistsById(1, ab.ID) }, true}, + {"by id unknown", func() (bool, error) { return db.Abonnements.ExistsById(1, ab.ID+1) }, false}, + } + for _, c := range checks { + got, err := c.fn() + if err != nil || got != c.want { + t.Errorf("%s = %v, %v; want %v, nil", c.name, got, err, c.want) + } + } +} + +func TestSetFeedStatePersistsCacheAndHints(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now()) + ab, _ := feedByURL(t, db, url) + + if err := db.Abonnements.Reschedule(ab.ID, time.Now(), 3, 0); err != nil { + t.Fatal(err) + } + + entry, etag, lm := "guid-1", `"v1"`, "Wed, 21 Oct 2015 07:28:00 GMT" + hints := PollHints{Interval: 2 * time.Hour, SkipHours: []int{1, 2}, SkipDays: []string{"Sunday"}} + next := time.Now().Add(time.Hour).Truncate(time.Second) + if err := db.Abonnements.SetFeedState(ab.ID, &entry, &etag, &lm, hints, next, 0, 4); err != nil { + t.Fatal(err) + } + + got, _ := feedByURL(t, db, url) + if got.LastEntry.String != entry || got.ETag.String != etag || got.LastModified.String != lm { + t.Errorf("cache state not persisted: %+v", got.Feed) + } + if got.ErrorCount != 0 || got.UnchangedCount != 4 || got.FailingSince.Valid { + t.Errorf("counters not reset: errors=%d unchanged=%d failingSince=%v", got.ErrorCount, got.UnchangedCount, got.FailingSince) + } + if !got.NextPollAt.Time.Equal(next) { + t.Errorf("next_poll_at = %s, want %s", got.NextPollAt.Time, next) + } + if h := got.Hints(); h.Interval != hints.Interval || len(h.SkipHours) != 2 || len(h.SkipDays) != 1 { + t.Errorf("hints = %+v, want %+v", h, hints) + } +} + +func TestRescheduleTracksFailingSince(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now()) + ab, _ := feedByURL(t, db, url) + + if err := db.Abonnements.Reschedule(ab.ID, time.Now(), 1, 0); err != nil { + t.Fatal(err) + } + first, _ := feedByURL(t, db, url) + if !first.FailingSince.Valid { + t.Fatal("failing_since not set on first error") + } + + time.Sleep(1100 * time.Millisecond) + if err := db.Abonnements.Reschedule(ab.ID, time.Now(), 2, 0); err != nil { + t.Fatal(err) + } + second, _ := feedByURL(t, db, url) + if !second.FailingSince.Time.Equal(first.FailingSince.Time) { + t.Errorf("failing_since moved from %s to %s", first.FailingSince.Time, second.FailingSince.Time) + } + + if err := db.Abonnements.Reschedule(ab.ID, time.Now(), 0, 1); err != nil { + t.Fatal(err) + } + recovered, _ := feedByURL(t, db, url) + if recovered.FailingSince.Valid || recovered.ErrorCount != 0 { + t.Errorf("error streak not cleared: %+v", recovered.Feed) + } +} + +func TestDisableAndReactivate(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now().Add(-time.Minute)) + ab, _ := feedByURL(t, db, url) + + if err := db.Abonnements.Reschedule(ab.ID, time.Now().Add(-time.Minute), 5, 0); err != nil { + t.Fatal(err) + } + if err := db.Abonnements.DisableFeed(ab.ID, "HTTP 410 Gone"); err != nil { + t.Fatal(err) + } + if due := dueURLs(t, db); len(due) != 0 { + t.Errorf("disabled feed is due: %v", due) + } + disabled, _ := feedByURL(t, db, url) + if !disabled.Disabled || disabled.DisabledReason.String != "HTTP 410 Gone" { + t.Errorf("feed not disabled: %+v", disabled.Feed) + } + + ok, err := db.Abonnements.ReactivateFeed(url, time.Now().Add(-time.Second)) + if err != nil || !ok { + t.Fatalf("ReactivateFeed = %v, %v; want true", ok, err) + } + reactivated, _ := feedByURL(t, db, url) + if reactivated.Disabled || reactivated.DisabledReason.Valid || reactivated.ErrorCount != 0 || reactivated.FailingSince.Valid { + t.Errorf("feed not fully reactivated: %+v", reactivated.Feed) + } + if due := dueURLs(t, db); len(due) != 1 { + t.Errorf("reactivated feed is not due") + } + + if ok, err := db.Abonnements.ReactivateFeed(url, time.Now()); err != nil || ok { + t.Errorf("ReactivateFeed on active feed = %v, %v; want false", ok, err) + } +} + +func TestMoveFeedURLRenames(t *testing.T) { + db := newTestDB(t) + mustCreate(t, db, 1, "http://example.org/feed", time.Now()) + ab, _ := feedByURL(t, db, "http://example.org/feed") + + merged, err := db.Abonnements.MoveFeedURL(ab.ID, "https://example.org/feed") + if err != nil || merged { + t.Fatalf("MoveFeedURL = %v, %v; want false, nil", merged, err) + } + if _, ok := feedByURL(t, db, "https://example.org/feed"); !ok { + t.Error("feed not renamed") + } +} + +func TestMoveFeedURLMergesIntoExistingFeed(t *testing.T) { + db := newTestDB(t) + oldURL, newURL := "http://example.org/old", "https://example.org/new" + later := time.Now().Add(time.Hour) + mustCreate(t, db, 1, oldURL, later) + mustCreate(t, db, 2, oldURL, later) + mustCreate(t, db, 2, newURL, later) + mustCreate(t, db, 3, newURL, later) + old, _ := feedByURL(t, db, oldURL) + + merged, err := db.Abonnements.MoveFeedURL(old.ID, newURL) + if err != nil || !merged { + t.Fatalf("MoveFeedURL = %v, %v; want true, nil", merged, err) + } + if _, ok := feedByURL(t, db, oldURL); ok { + t.Error("old feed still exists") + } + target, _ := feedByURL(t, db, newURL) + if ids := chatIDs(target); len(ids) != 3 { + t.Errorf("chats after merge = %v, want [1 2 3]", ids) + } + if due := dueURLs(t, db); len(due) != 1 || due[0] != newURL { + t.Errorf("merged feed should be due immediately, due = %v", due) + } +} + +func TestDeleteRemovesOrphans(t *testing.T) { + db := newTestDB(t) + shared, single := "https://example.org/shared", "https://example.org/single" + mustCreate(t, db, 1, shared, time.Now()) + mustCreate(t, db, 2, shared, time.Now()) + mustCreate(t, db, 1, single, time.Now()) + sharedFeed, _ := feedByURL(t, db, shared) + singleFeed, _ := feedByURL(t, db, single) + + if err := db.Abonnements.Delete(1, singleFeed.ID); err != nil { + t.Fatal(err) + } + if _, ok := feedByURL(t, db, single); ok { + t.Error("orphaned feed not deleted") + } + + if err := db.Abonnements.Delete(2, sharedFeed.ID); err != nil { + t.Fatal(err) + } + remaining, ok := feedByURL(t, db, shared) + if !ok || len(remaining.Chats) != 1 || remaining.Chats[0].ID != 1 { + t.Errorf("shared feed should remain for chat 1: %+v", remaining) + } + + var chats int + if err := db.Get(&chats, "SELECT COUNT(*) FROM chats"); err != nil { + t.Fatal(err) + } + if chats != 1 { + t.Errorf("got %d chats, want 1 (chat 2 has no abonnements left)", chats) + } +} + +func TestReplacements(t *testing.T) { + db := newTestDB(t) + + seeded, err := db.Replacements.List() + if err != nil { + t.Fatal(err) + } + if len(seeded) == 0 { + t.Fatal("default replacements missing") + } + + if err := db.Replacements.Create("[werbung]", false); err != nil { + t.Fatal(err) + } + err = db.Replacements.Create("[werbung]", false) + var mysqlErr *mysql.MySQLError + if !errors.As(err, &mysqlErr) || mysqlErr.Number != 1062 { + t.Errorf("duplicate replacement: err = %v, want MySQL error 1062", err) + } + + list, err := db.Replacements.List() + if err != nil { + t.Fatal(err) + } + if len(list) != len(seeded)+1 { + t.Fatalf("got %d replacements, want %d", len(list), len(seeded)+1) + } + + var id int64 + for _, r := range list { + if r.Value == "[werbung]" { + id = r.ID + } + } + if err := db.Replacements.Delete(id); err != nil { + t.Errorf("Delete: %v", err) + } + if err := db.Replacements.Delete(id); err == nil { + t.Error("deleting a missing replacement should fail") + } +} From 7cf2bc5905b4c63d409835602b86c538437ff2bb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:07:07 +0000 Subject: [PATCH 37/49] Add command handler tests Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- handler/commands_test.go | 233 +++++++++++++++++++++++++++++++++++++++ handler/poll_test.go | 12 +- 2 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 handler/commands_test.go diff --git a/handler/commands_test.go b/handler/commands_test.go new file mode 100644 index 0000000..186e21c --- /dev/null +++ b/handler/commands_test.go @@ -0,0 +1,233 @@ +package handler + +import ( + "database/sql" + "net/http" + "strings" + "testing" + "time" + + "github.com/Brawl345/rssbot/storage" + "gopkg.in/telebot.v3" +) + +const userID = 10 + +type createCall struct { + ChatID int64 + FeedURL string + LastEntry *string + ETag *string + LastModified *string + Hints storage.PollHints + NextPollAt time.Time +} + +func (s *fakeStore) Create(chatID int64, chatTitle string, feedURL string, lastEntry, etag, lastModified *string, hints storage.PollHints, nextPollAt time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + s.created = append(s.created, createCall{chatID, feedURL, lastEntry, etag, lastModified, hints, nextPollAt}) + s.subscriptions[chatID] = append(s.subscriptions[chatID], storage.Feed{Url: feedURL}) + return nil +} + +func (s *fakeStore) Delete(chatID int64, feedID int64) error { + s.mu.Lock() + defer s.mu.Unlock() + s.deleted = append(s.deleted, [2]int64{chatID, feedID}) + return nil +} + +func (s *fakeStore) ExistsByFeedUrl(chatID int64, feedURL string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, f := range s.subscriptions[chatID] { + if f.Url == feedURL { + return true, nil + } + } + return false, nil +} + +func (s *fakeStore) ExistsById(chatID int64, feedID int64) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, f := range s.subscriptions[chatID] { + if f.ID == feedID { + return true, nil + } + } + return false, nil +} + +func (s *fakeStore) GetByUser(chatID int64) ([]storage.Feed, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.subscriptions[chatID], nil +} + +func (s *fakeStore) ReactivateFeed(feedURL string, nextPollAt time.Time) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + was := s.disabledURLs[feedURL] + delete(s.disabledURLs, feedURL) + return was, nil +} + +func (env *pollEnv) command(t *testing.T, handler telebot.HandlerFunc, payload string) string { + t.Helper() + before := len(env.tg.messagesTo(userID)) + ctx := env.h.Bot.NewContext(telebot.Update{Message: &telebot.Message{ + Payload: payload, + Chat: &telebot.Chat{ID: userID, Type: telebot.ChatPrivate, FirstName: "Tester"}, + }}) + if err := handler(ctx); err != nil { + t.Fatalf("handler returned %v", err) + } + msgs := env.tg.messagesTo(userID) + if len(msgs) != before+1 { + t.Fatalf("got %d replies, want 1", len(msgs)-before) + } + return msgs[len(msgs)-1] +} + +func TestSubscribeRejectsInvalidInput(t *testing.T) { + html := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("no feed")) + }) + missing := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "nope", http.StatusNotFound) + }) + + tests := []struct { + name string + payload string + want string + }{ + {"bad scheme", "ftp://example.org/feed", "Ungültige URL"}, + {"bad characters", "https://example.org/", "Ungültige URL"}, + {"not found", missing.URL, "HTTP 404"}, + {"not a feed", html.URL, "Ungültiger Feed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env := newPollEnv(t) + reply := env.command(t, env.h.OnSubscribe, tt.payload) + if !strings.Contains(reply, tt.want) { + t.Errorf("reply = %q, want %q", reply, tt.want) + } + if len(env.store.created) != 0 { + t.Errorf("feed must not be created") + } + }) + } +} + +func TestSubscribeStoresValidatorsAndHints(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Etag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") + _, _ = w.Write([]byte(rss("120", "2", "1"))) + }) + + reply := env.command(t, env.h.OnSubscribe, srv.URL) + if !strings.Contains(reply, "erfolgreich abonniert") { + t.Fatalf("reply = %q", reply) + } + if len(env.store.created) != 1 { + t.Fatalf("Create called %d times, want 1", len(env.store.created)) + } + c := env.store.created[0] + if c.ChatID != userID || c.FeedURL != srv.URL { + t.Errorf("created %d/%q", c.ChatID, c.FeedURL) + } + if c.LastEntry == nil || *c.LastEntry != "2" { + t.Errorf("last entry = %v, want newest item", c.LastEntry) + } + if c.ETag == nil || *c.ETag != `"abc"` || c.LastModified == nil { + t.Errorf("validators not stored: etag=%v lm=%v", c.ETag, c.LastModified) + } + if c.Hints.Interval != 2*time.Hour { + t.Errorf("ttl hint = %s, want 2h", c.Hints.Interval) + } + assertDelay(t, c.NextPollAt, 2*time.Hour) +} + +func TestSubscribeUsesSelfLink(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(` +Thttps://example.org + +Aa`)) + }) + + env.command(t, env.h.OnSubscribe, srv.URL) + if got := env.store.created[0].FeedURL; got != "https://example.org/canonical.xml" { + t.Errorf("subscribed to %q, want the self link", got) + } +} + +func TestSubscribeExisting(t *testing.T) { + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(rss("", "1"))) + }) + + t.Run("already subscribed", func(t *testing.T) { + env := newPollEnv(t) + env.store.subscriptions[userID] = []storage.Feed{{Url: srv.URL}} + if reply := env.command(t, env.h.OnSubscribe, srv.URL); !strings.Contains(reply, "bereits abonniert") { + t.Errorf("reply = %q", reply) + } + if len(env.store.created) != 0 { + t.Errorf("duplicate subscription created") + } + }) + + t.Run("disabled feed is reactivated", func(t *testing.T) { + env := newPollEnv(t) + env.store.subscriptions[userID] = []storage.Feed{{Url: srv.URL}} + env.store.disabledURLs[srv.URL] = true + if reply := env.command(t, env.h.OnSubscribe, srv.URL); !strings.Contains(reply, "wieder aktiviert") { + t.Errorf("reply = %q", reply) + } + if env.store.disabledURLs[srv.URL] { + t.Errorf("feed still disabled") + } + }) +} + +func TestUnsubscribe(t *testing.T) { + env := newPollEnv(t) + env.store.subscriptions[userID] = []storage.Feed{{ID: 7, Url: "https://example.org/feed"}} + + if reply := env.command(t, env.h.OnUnsubscribe, "abc"); !strings.Contains(reply, "Feed-ID") { + t.Errorf("reply = %q", reply) + } + if reply := env.command(t, env.h.OnUnsubscribe, "8"); !strings.Contains(reply, "nicht abonniert") { + t.Errorf("reply = %q", reply) + } + if reply := env.command(t, env.h.OnUnsubscribe, "7"); !strings.Contains(reply, "deabonniert") { + t.Errorf("reply = %q", reply) + } + if len(env.store.deleted) != 1 || env.store.deleted[0] != [2]int64{userID, 7} { + t.Errorf("deleted = %v, want [[%d 7]]", env.store.deleted, userID) + } +} + +func TestListShowsDisabledFeeds(t *testing.T) { + env := newPollEnv(t) + env.store.subscriptions[userID] = []storage.Feed{ + {ID: 1, Url: "https://example.org/a?x=1&y=2"}, + {ID: 2, Url: "https://example.org/b", Disabled: true, DisabledReason: sql.NullString{String: "HTTP 410 Gone", Valid: true}}, + } + + reply := env.command(t, env.h.OnList, "") + if !strings.Contains(reply, "a?x=1&y=2") { + t.Errorf("URL not escaped: %q", reply) + } + if strings.Count(reply, "🚫") != 1 || !strings.Contains(reply, "HTTP 410 Gone") { + t.Errorf("disabled marker missing: %q", reply) + } +} diff --git a/handler/poll_test.go b/handler/poll_test.go index c6e04a9..74c9bac 100644 --- a/handler/poll_test.go +++ b/handler/poll_test.go @@ -93,10 +93,20 @@ type fakeStore struct { disabled map[int64]string moved map[int64]string mergeOnMove bool + + created []createCall + deleted [][2]int64 + subscriptions map[int64][]storage.Feed + disabledURLs map[string]bool } func newFakeStore() *fakeStore { - return &fakeStore{disabled: map[int64]string{}, moved: map[int64]string{}} + return &fakeStore{ + disabled: map[int64]string{}, + moved: map[int64]string{}, + subscriptions: map[int64][]storage.Feed{}, + disabledURLs: map[string]bool{}, + } } func (s *fakeStore) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, hints storage.PollHints, nextPollAt time.Time, errorCount, unchangedCount int) error { From 98491db4d116f2cc6f61b0ea28daa47eb6b24f26 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:07:07 +0000 Subject: [PATCH 38/49] Run vet and tests with MariaDB in CI Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .github/workflows/build.yml | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8849cc9..b444f56 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,6 +13,48 @@ env: NAME: rssbot jobs: + test: + name: test + runs-on: ubuntu-latest + permissions: + contents: read # checkout + services: + mariadb: + image: mariadb:12.3@sha256:805c8e104bd563d5bfa24fadd3f31cd419ea859cb5277f32b5dbf2db714f9ed1 + env: + MARIADB_DATABASE: rssbot_test + MARIADB_USER: rssbot + MARIADB_PASSWORD: rssbot + MARIADB_RANDOM_ROOT_PASSWORD: '1' + # Differs from the runner's UTC to catch time zone bugs. + TZ: Asia/Tokyo + ports: + - 3306:3306 + options: >- + --health-cmd "healthcheck.sh --connect --innodb_initialized" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + + steps: + - name: Checkout repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + + - name: Vet + run: go vet ./... + + - name: Test + env: + RSSBOT_TEST_DSN: rssbot:rssbot@tcp(127.0.0.1:3306)/rssbot_test?charset=utf8mb4&parseTime=True&loc=Local + run: go test -race ./... + build: name: build runs-on: ubuntu-latest @@ -60,6 +102,7 @@ jobs: docker: name: docker + needs: test runs-on: ubuntu-latest permissions: contents: read # checkout From 7e6e254977abdba2614fb66ec7f07e7ba876775f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:11:30 +0000 Subject: [PATCH 39/49] Use binary collation for replacements and feed URLs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- storage/integration_test.go | 14 ++++++++++++++ storage/migrations/0002-replacements.sql | 2 +- storage/migrations/0006-binary-collation.sql | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 storage/migrations/0006-binary-collation.sql diff --git a/storage/integration_test.go b/storage/integration_test.go index 2a281e7..9afcb31 100644 --- a/storage/integration_test.go +++ b/storage/integration_test.go @@ -382,3 +382,17 @@ func TestReplacements(t *testing.T) { t.Error("deleting a missing replacement should fail") } } + +func TestFeedURLsAreCaseSensitive(t *testing.T) { + db := newTestDB(t) + mustCreate(t, db, 1, "https://example.org/Feed", time.Now()) + mustCreate(t, db, 1, "https://example.org/feed", time.Now()) + + all, err := db.Abonnements.GetAll() + if err != nil { + t.Fatal(err) + } + if len(all) != 2 { + t.Errorf("got %d feeds, want 2 distinct URLs", len(all)) + } +} diff --git a/storage/migrations/0002-replacements.sql b/storage/migrations/0002-replacements.sql index 97a878b..0071e48 100644 --- a/storage/migrations/0002-replacements.sql +++ b/storage/migrations/0002-replacements.sql @@ -3,7 +3,7 @@ CREATE TABLE `replacements` ( `id` INT(11) NOT NULL AUTO_INCREMENT, - `value` VARCHAR(255) NOT NULL, + `value` VARCHAR(255) NOT NULL COLLATE utf8mb4_bin, `is_regex` TINYINT(1) NOT NULL DEFAULT '0', `created_at` DATETIME NOT NULL DEFAULT current_timestamp(), PRIMARY KEY (`id`) USING BTREE, diff --git a/storage/migrations/0006-binary-collation.sql b/storage/migrations/0006-binary-collation.sql new file mode 100644 index 0000000..1bb8814 --- /dev/null +++ b/storage/migrations/0006-binary-collation.sql @@ -0,0 +1,17 @@ +-- +migrate Up + +-- Newer MariaDB defaults to utf8mb4_uca1400_ai_ci, which treats e.g. "…" and +-- "..." or differently cased URL paths as equal. Compare byte-exact instead. +ALTER TABLE `replacements` + MODIFY `value` VARCHAR(255) NOT NULL COLLATE utf8mb4_bin; + +ALTER TABLE `feeds` + MODIFY `url` VARCHAR(512) NOT NULL COLLATE utf8mb4_bin; + +-- +migrate Down + +ALTER TABLE `feeds` + MODIFY `url` VARCHAR(512) NOT NULL COLLATE utf8mb4_general_ci; + +ALTER TABLE `replacements` + MODIFY `value` VARCHAR(255) NOT NULL COLLATE utf8mb4_general_ci; From 964315bc7cde28e6a5cdecb5f2fc6ba924845cee Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:11:31 +0000 Subject: [PATCH 40/49] Test against MariaDB 11.4, 12.3 and 13.0 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .github/workflows/build.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b444f56..04b2ed2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,13 +14,23 @@ env: jobs: test: - name: test + name: test (MariaDB ${{ matrix.mariadb.version }}) runs-on: ubuntu-latest permissions: contents: read # checkout + strategy: + fail-fast: false + matrix: + mariadb: + - version: '11.4' # nixpkgs default + image: mariadb:11.4@sha256:70cc072b29b4a89ae07abb2d4da2c64678a7f2dfe092751bb51c87d67dc1338b + - version: '12.3' # LTS + image: mariadb:12.3@sha256:805c8e104bd563d5bfa24fadd3f31cd419ea859cb5277f32b5dbf2db714f9ed1 + - version: '13.0' # latest, used by docker-compose.yml + image: mariadb:13.0@sha256:d4fdec0510ad498e4f3127da30a99df3745bd6d5e611ae6ac5f76403d9284a8d services: mariadb: - image: mariadb:12.3@sha256:805c8e104bd563d5bfa24fadd3f31cd419ea859cb5277f32b5dbf2db714f9ed1 + image: ${{ matrix.mariadb.image }} env: MARIADB_DATABASE: rssbot_test MARIADB_USER: rssbot From 8dd5dd9a1a2b4d382785d30a5b4fc6555201f0bd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:14:50 +0000 Subject: [PATCH 41/49] Document polling behaviour in README Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- README.md | 46 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c9e41ae..8dbafff 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,9 @@ Only one user (the "admin") can manage the bot, but it's possible to let the bot The bot's language is German, but it should be self-explanatory. ## Features -* Checks feeds every minute (after all checks are finished) -* Concurrent checks +* Every feed has its own schedule, see [How polling works](#how-polling-works) +* Polite fetching: conditional requests, compression, backoff on errors and rate limits +* Slows down feeds that rarely change (optional) * Can post private, in channels or groups * Custom post format with a `post.gohtml` file * Supports "replacements" where specific words will be removed (limited Regex is also supported). This is useful for spam like "Read more on XYZ" and stuff @@ -20,7 +21,46 @@ The bot's language is German, but it should be self-explanatory. 3. (Optional) Create a `post.gohtml` with a custom Go HTML template that will be used for posts (see below) 4. Run and done! Database migrations are applied automatically. -Feeds are checked every minute after the latest check finished (it waits for five seconds the first time after the bot starts). +## How polling works + +The bot does not fetch all feeds at once. Every feed has its own "next poll" time stored in the database. Every `POLL_TICK` (default: 30 seconds) the bot fetches the feeds that are due and then calculates their next poll time. Because the schedule lives in the database, restarting the bot does not trigger a re-download of all feeds. + +Feeds are fetched in parallel (up to `POLL_CONCURRENCY`), but feeds on the same host are fetched one after another so a single server is never hit with several requests at once. + +Each request sends the `ETag` and `Last-Modified` values from the previous response. If nothing changed, the server can answer with a tiny `304 Not Modified` instead of the whole feed. Responses are compressed when the server supports it. + +### When is a feed polled next? + +1. **Base interval:** `POLL_INTERVAL` (default: 10 minutes). +2. **Adaptive slow-down** (`POLL_ADAPTIVE`, on by default): every poll without a new entry adds one base interval to the wait time, up to `POLL_INTERVAL_MAX` (default: 6 hours). As soon as a new entry shows up, the feed is back to the base interval. + + | Polls without new entries | Wait until next poll (defaults) | + |---------------------------|---------------------------------| + | 0 | 10 min | + | 1 | 20 min | + | 2 | 30 min | + | 9 (≈ 8 hours of silence) | 1 h 40 min | + | 35 (≈ 4 days of silence) | 6 h (maximum) | + + So a feed that was quiet over night may take up to ~1 h 40 min to deliver its first new post in the morning, while active feeds stay at 10 minutes. + + **With `POLL_ADAPTIVE=false`** every feed is polled every `POLL_INTERVAL`, no matter how often it changes. New posts arrive faster, but the bot sends more requests. Thanks to the conditional requests, most of them are cheap `304` answers. +3. **Server hints:** if the server asks for a longer interval (`Cache-Control: max-age` header or `` in an RSS feed), the bot waits at least that long, but never longer than `POLL_INTERVAL_MAX`. Hints can only slow polling down, never speed it up. +4. **Quiet hours:** RSS `` and `` (in UTC) are respected by moving the next poll out of these times. + +The actual time can be up to `POLL_TICK` later than calculated. + +### Errors, rate limits and moved feeds + +* **Temporary errors** (timeouts, HTTP 404/500, invalid feed, …): the wait time doubles with every consecutive failure (10 min, 20 min, 40 min, …, up to `POLL_INTERVAL_MAX`). A feed is **disabled** after it failed at least 12 times in a row *and* has been failing for 7 days. +* **HTTP 410 Gone:** the feed is disabled immediately. +* **HTTP 429/503:** the bot waits as long as the `Retry-After` header says (or 4 × `POLL_INTERVAL` without it, capped at `POLL_INTERVAL_MAX`). This does not count as an error. +* **Permanent redirects (301/308):** the new URL is saved automatically. If another subscription already uses the new URL, both are merged. Temporary redirects (302/307) are only followed. +* Redirects from a public feed to a private/local network address are refused. + +The admin gets a private message when a feed is disabled, moved or rate limited. Subscribed channels and groups only ever receive feed entries. + +Disabled feeds are marked with 🚫 in `/rss`. To enable one again, simply subscribe to it again with `/sub`. ### Use your own template From 8ee8f054ef6504f37b7d0b90c0f31060d0d76dc1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:14:50 +0000 Subject: [PATCH 42/49] Clarify .env.example comments Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .env.example | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index aac5ca2..5c72d44 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,34 @@ +# Your Telegram user ID. Only this user can control the bot. ADMIN_ID=1337 +# Token from @BotFather BOT_TOKEN=123456789:abcdefghijklmnopqrstuvxwyz -MYSQL_HOST=127.0.0.1 # Use 'db' for Docker + +# Database connection (MySQL or MariaDB) +# Use 'db' as host with the included docker-compose.yml +MYSQL_HOST=127.0.0.1 MYSQL_PORT=3306 MYSQL_USER=myuser MYSQL_PASSWORD=mypassword MYSQL_DB=mydb -MYSQL_SOCKET=/run/mysqld/mysqld.sock +# Connect via Unix socket instead. If set, MYSQL_HOST, MYSQL_PORT and +# MYSQL_PASSWORD are ignored (socket authentication). +#MYSQL_SOCKET=/run/mysqld/mysqld.sock +# Encrypt the TCP connection: false (default), true, skip-verify or preferred +#MYSQL_TLS=false + +# Polling (all optional, the values below are the defaults) +# Durations are written like 30s, 10m, 1h30m or 6h. +# See "How polling works" in the README for details. -# Polling behaviour (all optional) -POLL_INTERVAL=10m # base poll interval per feed; can be as low as 1m -POLL_INTERVAL_MAX=6h # upper cap for adaptive slow-down, error backoff and server hints -POLL_ADAPTIVE=true # slow down feeds that rarely change (FRB023), set false to disable -POLL_CONCURRENCY=8 # max feeds fetched simultaneously -POLL_TICK=30s # scheduler granularity (how often due feeds are checked) +# How often each feed is checked. Can be as low as 1m. +#POLL_INTERVAL=10m +# The longest a feed ever waits between two checks. Limits the adaptive +# slow-down, the waiting time after errors and intervals requested by servers. +#POLL_INTERVAL_MAX=6h +# true: check feeds that rarely get new entries less often (up to POLL_INTERVAL_MAX) +# false: check every feed every POLL_INTERVAL +#POLL_ADAPTIVE=true +# How many feeds are downloaded at the same time +#POLL_CONCURRENCY=8 +# How often the bot looks for feeds that are due. Rarely needs changing. +#POLL_TICK=30s From 498c6977383bdec7f8e1f744217dc399a74fb895 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:18:44 +0000 Subject: [PATCH 43/49] Load post template from POST_TEMPLATE Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- .env.example | 4 ++++ README.md | 4 ++-- config/config.go | 9 +++++++++ config/config_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ main.go | 2 +- 5 files changed, 59 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 5c72d44..db0889c 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,10 @@ ADMIN_ID=1337 # Token from @BotFather BOT_TOKEN=123456789:abcdefghijklmnopqrstuvxwyz +# Custom post template (optional). Without it, post.gohtml from the working +# directory or the built-in template is used. See README. +#POST_TEMPLATE=/path/to/post.gohtml + # Database connection (MySQL or MariaDB) # Use 'db' as host with the included docker-compose.yml MYSQL_HOST=127.0.0.1 diff --git a/README.md b/README.md index 8dbafff..dbfe33a 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The bot's language is German, but it should be self-explanatory. 1. Download binary for your system from Releases or build it yourself 2. Copy ".env.example" to ".env" and fill it in -3. (Optional) Create a `post.gohtml` with a custom Go HTML template that will be used for posts (see below) +3. (Optional) Create a `post.gohtml` with a custom Go template that will be used for posts (see below) 4. Run and done! Database migrations are applied automatically. ## How polling works @@ -64,7 +64,7 @@ Disabled feeds are marked with 🚫 in `/rss`. To enable one again, simply subsc ### Use your own template -The bot reads the `post.gohtml` from the same directory and uses it as a [Go template](https://pkg.go.dev/text/template) where it inserts the data. Take a look inside the [handler/feed_check.go](handler/feed_check.go) file (the `TemplateData` struct) to see all available fields. You can find the default template inside the [config/config.go](config/config.go) file. [Limited HTML](https://core.telegram.org/bots/api#html-style) is supported and all fields are sanitized with HTML tags removed and "replacements" applied. +The bot reads the file set in `POST_TEMPLATE` (or `post.gohtml` from the working directory, e.g. `/app/post.gohtml` in Docker) and uses it as a [Go template](https://pkg.go.dev/text/template) where it inserts the data. Take a look inside the [handler/feed_check.go](handler/feed_check.go) file (the `TemplateData` struct) to see all available fields. You can find the default template inside the [config/config.go](config/config.go) file. [Limited HTML](https://core.telegram.org/bots/api#html-style) is supported and all fields are sanitized with HTML tags removed and "replacements" applied. Example: diff --git a/config/config.go b/config/config.go index 17c4071..1107365 100644 --- a/config/config.go +++ b/config/config.go @@ -82,6 +82,15 @@ func fileExists(fileName string) bool { return false } +// LoadTemplate loads the post template from POST_TEMPLATE. Without it, +// post.gohtml in the working directory or the built-in default is used. +func LoadTemplate() (*template.Template, error) { + if path := os.Getenv("POST_TEMPLATE"); path != "" { + return template.ParseFiles(path) + } + return GetTemplate("post.gohtml") +} + func GetTemplate(path string) (*template.Template, error) { if fileExists(path) { return template.ParseFiles(path) diff --git a/config/config_test.go b/config/config_test.go index 909fd9c..5103346 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,6 +1,9 @@ package config import ( + "bytes" + "os" + "path/filepath" "testing" "time" ) @@ -73,3 +76,43 @@ func TestGetTemplateDefault(t *testing.T) { t.Errorf("template name = %q, want post", tmpl.Name()) } } + +func TestLoadTemplateFromEnv(t *testing.T) { + path := filepath.Join(t.TempDir(), "custom.gohtml") + if err := os.WriteFile(path, []byte("[#RSS] {{.Title}}"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("POST_TEMPLATE", path) + + tmpl, err := LoadTemplate() + if err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := tmpl.Execute(&out, struct{ Title string }{"Hallo"}); err != nil { + t.Fatal(err) + } + if out.String() != "[#RSS] Hallo" { + t.Errorf("rendered %q", out.String()) + } +} + +func TestLoadTemplateMissingFileFails(t *testing.T) { + t.Setenv("POST_TEMPLATE", filepath.Join(t.TempDir(), "missing.gohtml")) + if _, err := LoadTemplate(); err == nil { + t.Error("an explicitly configured but missing template must be an error") + } +} + +func TestLoadTemplateDefault(t *testing.T) { + t.Setenv("POST_TEMPLATE", "") + t.Chdir(t.TempDir()) + + tmpl, err := LoadTemplate() + if err != nil { + t.Fatal(err) + } + if tmpl.Name() != "post" { + t.Errorf("template name = %q, want built-in post", tmpl.Name()) + } +} diff --git a/main.go b/main.go index 36321c1..d3a653e 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,7 @@ import ( ) func main() { - tmpl, err := config.GetTemplate("post.gohtml") + tmpl, err := config.LoadTemplate() if err != nil { log.Fatal("Invalid template: ", err) } From 33f4bcd4a087d614c618c019a465fde5f370b99c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:19:37 +0000 Subject: [PATCH 44/49] Fix NixOS module service dependencies and group Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- module.nix | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/module.nix b/module.nix index df80edb..49de1b7 100644 --- a/module.nix +++ b/module.nix @@ -31,6 +31,12 @@ in description = "User under which RSS Bot runs."; }; + group = mkOption { + type = types.str; + default = defaultUser; + description = "Group under which RSS Bot runs."; + }; + adminId = mkOption { type = types.int; description = "Admin ID"; @@ -125,8 +131,9 @@ in systemd.services.rssbot = { description = "RSS Bot for Telegram"; - after = [ "network-online.target" "mysql.service" ]; - requires = [ "network-online.target" "mysql.service" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ] ++ optional cfg.database.createLocally "mysql.service"; + requires = optional cfg.database.createLocally "mysql.service"; wantedBy = [ "multi-user.target" ]; script = '' @@ -145,7 +152,7 @@ in Restart = "always"; User = cfg.user; - Group = defaultUser; + Group = cfg.group; }; environment = { @@ -158,14 +165,16 @@ in }; }; - users = optionalAttrs (cfg.user == defaultUser) { - users.${defaultUser} = { + users.users = optionalAttrs (cfg.user == defaultUser) { + ${defaultUser} = { isSystemUser = true; - group = defaultUser; + inherit (cfg) group; description = "RSS Bot user"; }; + }; - groups.${defaultUser} = { }; + users.groups = optionalAttrs (cfg.group == defaultUser) { + ${defaultUser} = { }; }; }; From ece29a230738da0b046d3f0715bd64218e2af17e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:19:37 +0000 Subject: [PATCH 45/49] Use flake package as NixOS module default Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- flake.nix | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 6f7ea9a..c401a6d 100644 --- a/flake.nix +++ b/flake.nix @@ -23,11 +23,16 @@ { nixosModules = { - default = ./module.nix; + default = + { lib, pkgs, ... }: + { + imports = [ ./module.nix ]; + services.rssbot.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.default; + }; }; overlays.default = final: prev: { - rssbot = self.packages.${prev.system}.default; + rssbot = self.packages.${final.stdenv.hostPlatform.system}.default; }; devShells = forAllSystems (pkgs: { @@ -63,7 +68,7 @@ }; }; - default = self.packages.${pkgs.system}.rssbot; + default = self.packages.${pkgs.stdenv.hostPlatform.system}.rssbot; }); }; } From 8259cbd4376dcfc6355524f7679d46f75d6c8f56 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:20:25 +0000 Subject: [PATCH 46/49] Add poll and template options to NixOS module Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- module.nix | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/module.nix b/module.nix index 49de1b7..4b66d70 100644 --- a/module.nix +++ b/module.nix @@ -17,6 +17,8 @@ let optional optionalAttrs optionalString + mapNullable + boolToString ; in { @@ -47,12 +49,70 @@ in description = "File containing Telegram Bot Token"; }; - # TODO: Find a way to load a custom post.gohtml - # template = mkOption { - # type = types.nullOr types.lines; - # default = null; - # description = "Custom post.gohtml template content"; - # }; + template = mkOption { + type = types.nullOr types.lines; + default = null; + example = '' + [#RSS] {{.Title}} + {{.FeedTitle}} + {{- if ne .Content "" }} + {{.Content}} + {{- end }} + {{.PostDomain}} + ''; + description = "Custom post template (Go template). Uses the built-in template if null."; + }; + + templateFile = mkOption { + type = types.nullOr types.path; + default = null; + example = "./post.gohtml"; + description = "Path to a custom post template file. Alternative to `template`."; + }; + + poll = { + interval = mkOption { + type = types.nullOr types.str; + default = null; + example = "5m"; + description = "How often each feed is checked. Uses the bot's default (10m) if null."; + }; + + intervalMax = mkOption { + type = types.nullOr types.str; + default = null; + example = "2h"; + description = '' + The longest a feed ever waits between two checks. Limits the adaptive slow-down, + the waiting time after errors and intervals requested by servers. + Uses the bot's default (6h) if null. + ''; + }; + + adaptive = mkOption { + type = types.nullOr types.bool; + default = null; + example = false; + description = '' + Check feeds that rarely get new entries less often (up to `intervalMax`). + If false, every feed is checked every `interval`. Uses the bot's default (true) if null. + ''; + }; + + concurrency = mkOption { + type = types.nullOr types.ints.positive; + default = null; + example = 4; + description = "How many feeds are downloaded at the same time. Uses the bot's default (8) if null."; + }; + + tick = mkOption { + type = types.nullOr types.str; + default = null; + example = "1m"; + description = "How often the bot looks for feeds that are due. Uses the bot's default (30s) if null."; + }; + }; database = { host = lib.mkOption { @@ -113,6 +173,10 @@ in assertion = cfg.database.socket != null || cfg.database.passwordFile != null; message = "Either services.rssbot.database.socket or services.rssbot.database.passwordFile must be set."; } + { + assertion = !(cfg.template != null && cfg.templateFile != null); + message = "Only one of services.rssbot.template or services.rssbot.templateFile can be set."; + } ]; services.mysql = lib.mkIf cfg.database.createLocally { @@ -162,6 +226,18 @@ in MYSQL_USER = cfg.database.user; MYSQL_DB = cfg.database.name; MYSQL_SOCKET = cfg.database.socket; + + POST_TEMPLATE = + if cfg.template != null then + pkgs.writeText "post.gohtml" cfg.template + else + mapNullable (file: "${file}") cfg.templateFile; + + POLL_INTERVAL = cfg.poll.interval; + POLL_INTERVAL_MAX = cfg.poll.intervalMax; + POLL_ADAPTIVE = mapNullable boolToString cfg.poll.adaptive; + POLL_CONCURRENCY = mapNullable toString cfg.poll.concurrency; + POLL_TICK = cfg.poll.tick; }; }; From 8907aec4e76070d93c9beaa137d82a484367bd8b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:22:08 +0000 Subject: [PATCH 47/49] Harden rssbot systemd service Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- module.nix | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/module.nix b/module.nix index 4b66d70..bd70dde 100644 --- a/module.nix +++ b/module.nix @@ -212,11 +212,46 @@ in serviceConfig = { LoadCredential = [ "BOT_TOKEN:${cfg.botTokenFile}" - ] ++ optional (cfg.database.passwordFile != null) "MYSQL_PASSWORD:${cfg.database.passwordFile}"; + ] + ++ optional (cfg.database.passwordFile != null) "MYSQL_PASSWORD:${cfg.database.passwordFile}"; Restart = "always"; User = cfg.user; Group = cfg.group; + + # Hardening + CapabilityBoundingSet = ""; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateTmp = true; + ProcSubset = "pid"; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + UMask = "0077"; }; environment = { From ae1b116296132af46c5824d1c3f9bb97c6bc671c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:22:08 +0000 Subject: [PATCH 48/49] Add NixOS VM test for the module Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- flake.nix | 10 ++++++++++ nixos-test.nix | 31 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 nixos-test.nix diff --git a/flake.nix b/flake.nix index c401a6d..c730765 100644 --- a/flake.nix +++ b/flake.nix @@ -44,6 +44,16 @@ }; }); + checks = + nixpkgs.lib.genAttrs + [ + "x86_64-linux" + "aarch64-linux" + ] + (system: { + nixos-module = nixpkgs.legacyPackages.${system}.testers.runNixOSTest (import ./nixos-test.nix self); + }); + packages = forAllSystems (pkgs: { rssbot = pkgs.buildGoModule { pname = "rssbot"; diff --git a/nixos-test.nix b/nixos-test.nix new file mode 100644 index 0000000..b65d5fb --- /dev/null +++ b/nixos-test.nix @@ -0,0 +1,31 @@ +# Starts the bot with the NixOS module and a local MariaDB. Without network the +# bot cannot reach Telegram, but it connects to the database and applies the +# migrations first, which covers the module, the hardening and socket auth. +self: +{ pkgs, ... }: +{ + name = "rssbot"; + + nodes.machine = { + imports = [ self.nixosModules.default ]; + + services.rssbot = { + enable = true; + adminId = 1337; + botTokenFile = pkgs.writeText "rssbot-token" "123456789:test"; + template = "{{.Title}}"; + poll = { + interval = "5m"; + adaptive = false; + }; + }; + }; + + testScript = '' + machine.wait_for_unit("mysql.service") + machine.wait_until_succeeds("journalctl -u rssbot.service | grep -q 'Applied 6 migration'") + machine.succeed("mysql -N rssbot -e 'SELECT COUNT(*) FROM replacements' | grep -qx 41") + machine.succeed("systemctl show rssbot.service -p Environment | grep -q POLL_INTERVAL=5m") + print(machine.succeed("systemd-analyze security rssbot.service | tail -1")) + ''; +} From ce9932d9ef998c2489c9f57947465adfc5d939cd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 12:22:08 +0000 Subject: [PATCH 49/49] Document NixOS module in README Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01W6fMYGug2BYBwi3P9Coyie --- README.md | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dbfe33a..79e3705 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,40 @@ The bot's language is German, but it should be self-explanatory. 3. (Optional) Create a `post.gohtml` with a custom Go template that will be used for posts (see below) 4. Run and done! Database migrations are applied automatically. +### NixOS + +The flake provides a NixOS module. By default it creates a local MariaDB database and connects via Unix socket. + +```nix +{ + inputs.rssbot.url = "github:Brawl345/rssbot"; + + outputs = { nixpkgs, rssbot, ... }: { + nixosConfigurations.myhost = nixpkgs.lib.nixosSystem { + modules = [ + rssbot.nixosModules.default + { + services.rssbot = { + enable = true; + adminId = 1337; + botTokenFile = "/run/secrets/rssbot-token"; + + # Optional + templateFile = ./post.gohtml; # or: template = "{{.Title}} ..."; + poll = { + interval = "5m"; + adaptive = false; + }; + }; + } + ]; + }; + }; +} +``` + +To use an existing database instead, set `database.createLocally = false` together with `database.host` and `database.passwordFile`. `nix flake check` runs a VM test of the module. + ## How polling works The bot does not fetch all feeds at once. Every feed has its own "next poll" time stored in the database. Every `POLL_TICK` (default: 30 seconds) the bot fetches the feeds that are due and then calculates their next poll time. Because the schedule lives in the database, restarting the bot does not trigger a re-download of all feeds. @@ -62,7 +96,7 @@ The admin gets a private message when a feed is disabled, moved or rate limited. Disabled feeds are marked with 🚫 in `/rss`. To enable one again, simply subscribe to it again with `/sub`. -### Use your own template +## Use your own template The bot reads the file set in `POST_TEMPLATE` (or `post.gohtml` from the working directory, e.g. `/app/post.gohtml` in Docker) and uses it as a [Go template](https://pkg.go.dev/text/template) where it inserts the data. Take a look inside the [handler/feed_check.go](handler/feed_check.go) file (the `TemplateData` struct) to see all available fields. You can find the default template inside the [config/config.go](config/config.go) file. [Limited HTML](https://core.telegram.org/bots/api#html-style) is supported and all fields are sanitized with HTML tags removed and "replacements" applied.