diff --git a/.dockerignore b/.dockerignore index 1d9ec25..672edb1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ * !config/ +!fetcher/ !handler/ !storage/ !go.mod diff --git a/.env.example b/.env.example index 87c066e..db0889c 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,38 @@ +# Your Telegram user ID. Only this user can control the bot. ADMIN_ID=1337 +# Token from @BotFather BOT_TOKEN=123456789:abcdefghijklmnopqrstuvxwyz -MYSQL_HOST=127.0.0.1 # Use 'db' for Docker + +# Custom post template (optional). Without it, post.gohtml from the working +# directory or the built-in template is used. See README. +#POST_TEMPLATE=/path/to/post.gohtml + +# Database connection (MySQL or MariaDB) +# Use 'db' as host with the included docker-compose.yml +MYSQL_HOST=127.0.0.1 MYSQL_PORT=3306 MYSQL_USER=myuser MYSQL_PASSWORD=mypassword MYSQL_DB=mydb -MYSQL_SOCKET=/run/mysqld/mysqld.sock +# Connect via Unix socket instead. If set, MYSQL_HOST, MYSQL_PORT and +# MYSQL_PASSWORD are ignored (socket authentication). +#MYSQL_SOCKET=/run/mysqld/mysqld.sock +# Encrypt the TCP connection: false (default), true, skip-verify or preferred +#MYSQL_TLS=false + +# Polling (all optional, the values below are the defaults) +# Durations are written like 30s, 10m, 1h30m or 6h. +# See "How polling works" in the README for details. + +# How often each feed is checked. Can be as low as 1m. +#POLL_INTERVAL=10m +# The longest a feed ever waits between two checks. Limits the adaptive +# slow-down, the waiting time after errors and intervals requested by servers. +#POLL_INTERVAL_MAX=6h +# true: check feeds that rarely get new entries less often (up to POLL_INTERVAL_MAX) +# false: check every feed every POLL_INTERVAL +#POLL_ADAPTIVE=true +# How many feeds are downloaded at the same time +#POLL_CONCURRENCY=8 +# How often the bot looks for feeds that are due. Rarely needs changing. +#POLL_TICK=30s diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3686459..04b2ed2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,13 +3,73 @@ on: - push - pull_request +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} + env: NAME: rssbot jobs: + test: + name: test (MariaDB ${{ matrix.mariadb.version }}) + runs-on: ubuntu-latest + permissions: + contents: read # checkout + strategy: + fail-fast: false + matrix: + mariadb: + - version: '11.4' # nixpkgs default + image: mariadb:11.4@sha256:70cc072b29b4a89ae07abb2d4da2c64678a7f2dfe092751bb51c87d67dc1338b + - version: '12.3' # LTS + image: mariadb:12.3@sha256:805c8e104bd563d5bfa24fadd3f31cd419ea859cb5277f32b5dbf2db714f9ed1 + - version: '13.0' # latest, used by docker-compose.yml + image: mariadb:13.0@sha256:d4fdec0510ad498e4f3127da30a99df3745bd6d5e611ae6ac5f76403d9284a8d + services: + mariadb: + image: ${{ matrix.mariadb.image }} + env: + MARIADB_DATABASE: rssbot_test + MARIADB_USER: rssbot + MARIADB_PASSWORD: rssbot + MARIADB_RANDOM_ROOT_PASSWORD: '1' + # Differs from the runner's UTC to catch time zone bugs. + TZ: Asia/Tokyo + ports: + - 3306:3306 + options: >- + --health-cmd "healthcheck.sh --connect --innodb_initialized" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + + steps: + - name: Checkout repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + + - name: Vet + run: go vet ./... + + - name: Test + env: + RSSBOT_TEST_DSN: rssbot:rssbot@tcp(127.0.0.1:3306)/rssbot_test?charset=utf8mb4&parseTime=True&loc=Local + run: go test -race ./... + build: name: build runs-on: ubuntu-latest + permissions: + contents: read # checkout strategy: matrix: GOOS: [windows, linux, darwin] @@ -24,59 +84,64 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version-file: 'go.mod' + go-version: stable - name: Build env: GOOS: ${{ matrix.GOOS }} GOARCH: ${{ matrix.GOARCH }} - run: go build -ldflags="-s -w" -o dist/$NAME-$GOOS-$GOARCH + run: go build -ldflags="-s -w" -o "dist/${NAME}-${GOOS}-${GOARCH}" - name: Rename binaries (Windows) if: matrix.GOOS == 'windows' - run: for x in dist/$NAME-windows-*; do mv $x $x.exe; done + run: for x in dist/"${NAME}"-windows-*; do mv "$x" "$x.exe"; done - name: Upload binary - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{env.NAME}}-${{ matrix.GOOS }}-${{ matrix.GOARCH }}-${{github.sha}} + name: ${{ env.NAME }}-${{ matrix.GOOS }}-${{ matrix.GOARCH }}-${{ github.sha }} path: dist/* retention-days: 90 docker: name: docker + needs: test runs-on: ubuntu-latest permissions: - packages: write - contents: read + contents: read # checkout + packages: write # push image to ghcr.io steps: - - uses: actions/checkout@v4 + - name: Checkout repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Build image - run: docker build . --file Dockerfile --tag $NAME --label "runnumber=${GITHUB_RUN_ID}" + run: docker build . --file Dockerfile --tag "$NAME" --build-arg VERSION="${GITHUB_SHA::7}" --label "runnumber=${GITHUB_RUN_ID}" - name: Log in to registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin - name: Push image + if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$NAME - - # This changes all uppercase characters to lowercase. - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') - # This strips the git ref prefix from the version. - VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - # This strips the "v" prefix from the tag name. - [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') - # This uses the Docker `latest` tag convention. - [ "$VERSION" == "master" ] && VERSION=latest - echo IMAGE_ID=$IMAGE_ID - echo VERSION=$VERSION - docker tag $NAME $IMAGE_ID:$VERSION - docker push $IMAGE_ID:$VERSION + image_id="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/${NAME}" + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + version="${GITHUB_REF_NAME#v}" + else + version=latest + fi + echo "Pushing ${image_id}:${version}" + docker tag "$NAME" "${image_id}:${version}" + docker push "${image_id}:${version}" diff --git a/Dockerfile b/Dockerfile index 6a461b0..2fc222f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,12 @@ -FROM golang:1.22 AS build-stage +FROM golang:1.27.1@sha256:3680233e3204827fbdc66088528ae6d4b3d034f51d03a99d454f6de034888244 AS build-stage WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . ./ -RUN CGO_ENABLED=0 GOOS=linux go build -o /rssbot +ARG VERSION=dev +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X github.com/Brawl345/rssbot/fetcher.Version=${VERSION}" -o /rssbot -FROM gcr.io/distroless/static-debian12 AS release-stage +FROM gcr.io/distroless/static-debian13:nonroot@sha256:e2e927ec666bae08560abb3c55d0659eceabb657f56b6782ab500a9fc7f555e3 AS release-stage WORKDIR /app COPY --from=build-stage /rssbot /app/rssbot USER nonroot:nonroot diff --git a/README.md b/README.md index c9e41ae..79e3705 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,9 @@ Only one user (the "admin") can manage the bot, but it's possible to let the bot The bot's language is German, but it should be self-explanatory. ## Features -* Checks feeds every minute (after all checks are finished) -* Concurrent checks +* Every feed has its own schedule, see [How polling works](#how-polling-works) +* Polite fetching: conditional requests, compression, backoff on errors and rate limits +* Slows down feeds that rarely change (optional) * Can post private, in channels or groups * Custom post format with a `post.gohtml` file * Supports "replacements" where specific words will be removed (limited Regex is also supported). This is useful for spam like "Read more on XYZ" and stuff @@ -17,14 +18,87 @@ The bot's language is German, but it should be self-explanatory. 1. Download binary for your system from Releases or build it yourself 2. Copy ".env.example" to ".env" and fill it in -3. (Optional) Create a `post.gohtml` with a custom Go HTML template that will be used for posts (see below) +3. (Optional) Create a `post.gohtml` with a custom Go template that will be used for posts (see below) 4. Run and done! Database migrations are applied automatically. -Feeds are checked every minute after the latest check finished (it waits for five seconds the first time after the bot starts). +### NixOS -### Use your own template +The flake provides a NixOS module. By default it creates a local MariaDB database and connects via Unix socket. -The bot reads the `post.gohtml` from the same directory and uses it as a [Go template](https://pkg.go.dev/text/template) where it inserts the data. Take a look inside the [handler/feed_check.go](handler/feed_check.go) file (the `TemplateData` struct) to see all available fields. You can find the default template inside the [config/config.go](config/config.go) file. [Limited HTML](https://core.telegram.org/bots/api#html-style) is supported and all fields are sanitized with HTML tags removed and "replacements" applied. +```nix +{ + inputs.rssbot.url = "github:Brawl345/rssbot"; + + outputs = { nixpkgs, rssbot, ... }: { + nixosConfigurations.myhost = nixpkgs.lib.nixosSystem { + modules = [ + rssbot.nixosModules.default + { + services.rssbot = { + enable = true; + adminId = 1337; + botTokenFile = "/run/secrets/rssbot-token"; + + # Optional + templateFile = ./post.gohtml; # or: template = "{{.Title}} ..."; + poll = { + interval = "5m"; + adaptive = false; + }; + }; + } + ]; + }; + }; +} +``` + +To use an existing database instead, set `database.createLocally = false` together with `database.host` and `database.passwordFile`. `nix flake check` runs a VM test of the module. + +## How polling works + +The bot does not fetch all feeds at once. Every feed has its own "next poll" time stored in the database. Every `POLL_TICK` (default: 30 seconds) the bot fetches the feeds that are due and then calculates their next poll time. Because the schedule lives in the database, restarting the bot does not trigger a re-download of all feeds. + +Feeds are fetched in parallel (up to `POLL_CONCURRENCY`), but feeds on the same host are fetched one after another so a single server is never hit with several requests at once. + +Each request sends the `ETag` and `Last-Modified` values from the previous response. If nothing changed, the server can answer with a tiny `304 Not Modified` instead of the whole feed. Responses are compressed when the server supports it. + +### When is a feed polled next? + +1. **Base interval:** `POLL_INTERVAL` (default: 10 minutes). +2. **Adaptive slow-down** (`POLL_ADAPTIVE`, on by default): every poll without a new entry adds one base interval to the wait time, up to `POLL_INTERVAL_MAX` (default: 6 hours). As soon as a new entry shows up, the feed is back to the base interval. + + | Polls without new entries | Wait until next poll (defaults) | + |---------------------------|---------------------------------| + | 0 | 10 min | + | 1 | 20 min | + | 2 | 30 min | + | 9 (≈ 8 hours of silence) | 1 h 40 min | + | 35 (≈ 4 days of silence) | 6 h (maximum) | + + So a feed that was quiet over night may take up to ~1 h 40 min to deliver its first new post in the morning, while active feeds stay at 10 minutes. + + **With `POLL_ADAPTIVE=false`** every feed is polled every `POLL_INTERVAL`, no matter how often it changes. New posts arrive faster, but the bot sends more requests. Thanks to the conditional requests, most of them are cheap `304` answers. +3. **Server hints:** if the server asks for a longer interval (`Cache-Control: max-age` header or `` in an RSS feed), the bot waits at least that long, but never longer than `POLL_INTERVAL_MAX`. Hints can only slow polling down, never speed it up. +4. **Quiet hours:** RSS `` and `` (in UTC) are respected by moving the next poll out of these times. + +The actual time can be up to `POLL_TICK` later than calculated. + +### Errors, rate limits and moved feeds + +* **Temporary errors** (timeouts, HTTP 404/500, invalid feed, …): the wait time doubles with every consecutive failure (10 min, 20 min, 40 min, …, up to `POLL_INTERVAL_MAX`). A feed is **disabled** after it failed at least 12 times in a row *and* has been failing for 7 days. +* **HTTP 410 Gone:** the feed is disabled immediately. +* **HTTP 429/503:** the bot waits as long as the `Retry-After` header says (or 4 × `POLL_INTERVAL` without it, capped at `POLL_INTERVAL_MAX`). This does not count as an error. +* **Permanent redirects (301/308):** the new URL is saved automatically. If another subscription already uses the new URL, both are merged. Temporary redirects (302/307) are only followed. +* Redirects from a public feed to a private/local network address are refused. + +The admin gets a private message when a feed is disabled, moved or rate limited. Subscribed channels and groups only ever receive feed entries. + +Disabled feeds are marked with 🚫 in `/rss`. To enable one again, simply subscribe to it again with `/sub`. + +## Use your own template + +The bot reads the file set in `POST_TEMPLATE` (or `post.gohtml` from the working directory, e.g. `/app/post.gohtml` in Docker) and uses it as a [Go template](https://pkg.go.dev/text/template) where it inserts the data. Take a look inside the [handler/feed_check.go](handler/feed_check.go) file (the `TemplateData` struct) to see all available fields. You can find the default template inside the [config/config.go](config/config.go) file. [Limited HTML](https://core.telegram.org/bots/api#html-style) is supported and all fields are sanitized with HTML tags removed and "replacements" applied. Example: diff --git a/config/config.go b/config/config.go index efd742f..1107365 100644 --- a/config/config.go +++ b/config/config.go @@ -1,12 +1,78 @@ package config import ( + "log" "os" + "strconv" "text/template" + "time" ) type Config struct { Template *template.Template + Poll PollConfig +} + +// PollConfig holds the feed-polling behaviour. There is intentionally no lower +// bound on PollInterval: polling as often as every minute is a supported +// use-case. Server hints (max-age, Retry-After, ttl) can only slow polling down. +type PollConfig struct { + Interval time.Duration // base poll interval (POLL_INTERVAL) + IntervalMax time.Duration // cap for adaptive slow-down (POLL_INTERVAL_MAX) + Adaptive bool // FRB023: slow feeds that rarely update (POLL_ADAPTIVE) + Concurrency int // max simultaneous fetches (POLL_CONCURRENCY) + Tick time.Duration // scheduler granularity (POLL_TICK) +} + +func GetPollConfig() PollConfig { + cfg := PollConfig{ + Interval: durationEnv("POLL_INTERVAL", 10*time.Minute), + IntervalMax: durationEnv("POLL_INTERVAL_MAX", 6*time.Hour), + Adaptive: boolEnv("POLL_ADAPTIVE", true), + Concurrency: intEnv("POLL_CONCURRENCY", 8), + Tick: durationEnv("POLL_TICK", 30*time.Second), + } + if cfg.IntervalMax < cfg.Interval { + log.Printf("POLL_INTERVAL_MAX (%s) is below POLL_INTERVAL (%s), using %s", cfg.IntervalMax, cfg.Interval, cfg.Interval) + cfg.IntervalMax = cfg.Interval + } + return cfg +} + +func durationEnv(key string, fallback time.Duration) time.Duration { + v := os.Getenv(key) + if v == "" { + return fallback + } + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + log.Printf("Invalid %s=%q, using default %s", key, v, fallback) + return fallback +} + +func intEnv(key string, fallback int) int { + v := os.Getenv(key) + if v == "" { + return fallback + } + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + log.Printf("Invalid %s=%q, using default %d", key, v, fallback) + return fallback +} + +func boolEnv(key string, fallback bool) bool { + v := os.Getenv(key) + if v == "" { + return fallback + } + if b, err := strconv.ParseBool(v); err == nil { + return b + } + log.Printf("Invalid %s=%q, using default %t", key, v, fallback) + return fallback } func fileExists(fileName string) bool { @@ -16,6 +82,15 @@ func fileExists(fileName string) bool { return false } +// LoadTemplate loads the post template from POST_TEMPLATE. Without it, +// post.gohtml in the working directory or the built-in default is used. +func LoadTemplate() (*template.Template, error) { + if path := os.Getenv("POST_TEMPLATE"); path != "" { + return template.ParseFiles(path) + } + return GetTemplate("post.gohtml") +} + func GetTemplate(path string) (*template.Template, error) { if fileExists(path) { return template.ParseFiles(path) diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..5103346 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,118 @@ +package config + +import ( + "bytes" + "os" + "path/filepath" + "testing" + "time" +) + +func TestGetPollConfigDefaults(t *testing.T) { + for _, key := range []string{"POLL_INTERVAL", "POLL_INTERVAL_MAX", "POLL_ADAPTIVE", "POLL_CONCURRENCY", "POLL_TICK"} { + t.Setenv(key, "") + } + + want := PollConfig{ + Interval: 10 * time.Minute, + IntervalMax: 6 * time.Hour, + Adaptive: true, + Concurrency: 8, + Tick: 30 * time.Second, + } + if got := GetPollConfig(); got != want { + t.Errorf("GetPollConfig() = %+v, want %+v", got, want) + } +} + +func TestGetPollConfigFromEnv(t *testing.T) { + t.Setenv("POLL_INTERVAL", "1m") + t.Setenv("POLL_INTERVAL_MAX", "2h") + t.Setenv("POLL_ADAPTIVE", "false") + t.Setenv("POLL_CONCURRENCY", "3") + t.Setenv("POLL_TICK", "5s") + + want := PollConfig{ + Interval: time.Minute, + IntervalMax: 2 * time.Hour, + Adaptive: false, + Concurrency: 3, + Tick: 5 * time.Second, + } + if got := GetPollConfig(); got != want { + t.Errorf("GetPollConfig() = %+v, want %+v", got, want) + } +} + +func TestGetPollConfigInvalidValuesFallBack(t *testing.T) { + t.Setenv("POLL_INTERVAL", "10m # comment") + t.Setenv("POLL_INTERVAL_MAX", "-1h") + t.Setenv("POLL_ADAPTIVE", "maybe") + t.Setenv("POLL_CONCURRENCY", "0") + t.Setenv("POLL_TICK", "fast") + + got := GetPollConfig() + if got.Interval != 10*time.Minute || got.IntervalMax != 6*time.Hour || !got.Adaptive || + got.Concurrency != 8 || got.Tick != 30*time.Second { + t.Errorf("invalid values did not fall back to defaults: %+v", got) + } +} + +func TestGetPollConfigRaisesMaxToInterval(t *testing.T) { + t.Setenv("POLL_INTERVAL", "12h") + t.Setenv("POLL_INTERVAL_MAX", "1h") + + if got := GetPollConfig(); got.IntervalMax != 12*time.Hour { + t.Errorf("IntervalMax = %s, want 12h", got.IntervalMax) + } +} + +func TestGetTemplateDefault(t *testing.T) { + tmpl, err := GetTemplate("does-not-exist.gohtml") + if err != nil { + t.Fatal(err) + } + if tmpl.Name() != "post" { + t.Errorf("template name = %q, want post", tmpl.Name()) + } +} + +func TestLoadTemplateFromEnv(t *testing.T) { + path := filepath.Join(t.TempDir(), "custom.gohtml") + if err := os.WriteFile(path, []byte("[#RSS] {{.Title}}"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("POST_TEMPLATE", path) + + tmpl, err := LoadTemplate() + if err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := tmpl.Execute(&out, struct{ Title string }{"Hallo"}); err != nil { + t.Fatal(err) + } + if out.String() != "[#RSS] Hallo" { + t.Errorf("rendered %q", out.String()) + } +} + +func TestLoadTemplateMissingFileFails(t *testing.T) { + t.Setenv("POST_TEMPLATE", filepath.Join(t.TempDir(), "missing.gohtml")) + if _, err := LoadTemplate(); err == nil { + t.Error("an explicitly configured but missing template must be an error") + } +} + +func TestLoadTemplateDefault(t *testing.T) { + t.Setenv("POST_TEMPLATE", "") + t.Chdir(t.TempDir()) + + tmpl, err := LoadTemplate() + if err != nil { + t.Fatal(err) + } + if tmpl.Name() != "post" { + t.Errorf("template name = %q, want built-in post", tmpl.Name()) + } +} diff --git a/fetcher/fetcher.go b/fetcher/fetcher.go new file mode 100644 index 0000000..6b043d2 --- /dev/null +++ b/fetcher/fetcher.go @@ -0,0 +1,327 @@ +// Package fetcher performs well-behaved HTTP feed retrieval following the +// rachelbythebay Feed Reader Behavior project (FRB) and earth.org.uk +// RSS-efficiency recommendations: conditional GETs, identifying User-Agent, +// compression, server-hint parsing and manual redirect handling. +package fetcher + +import ( + "bytes" + "context" + "fmt" + "io" + "net" + "net/http" + "net/url" + "runtime/debug" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/mmcdole/gofeed" + "github.com/mmcdole/gofeed/rss" +) + +const ( + defaultTimeout = 30 * time.Second + maxRedirects = 5 + maxBodyBytes = 16 << 20 // 16 MiB + bodySnippetLen = 500 + repoURL = "https://github.com/Brawl345/rssbot" +) + +type Fetcher struct { + client *http.Client + userAgent string + privateHost func(ctx context.Context, host string) bool +} + +// Result is the interpreted outcome of a single feed fetch. ETag and +// LastModified form an atomic set (FRB014) and are only populated from a +// status 200 response. +type Result struct { + Status int + NotModified bool // 304: nothing changed since the conditional request + Feed *gofeed.Feed // only set on 200 with a parseable feed + ETag string // exactly as received (FRB003) + LastModified string // exactly as received (FRB001) + MaxAge time.Duration // Cache-Control: max-age (FRB022), 0 if absent + RetryAfter time.Duration // Retry-After on 429/503 (FRB020/021), 0 if absent + FeedInterval time.Duration // ttl hint (FRB023/024), 0 if absent + SkipHours []int // RSS skipHours (FRB024) + SkipDays []string // RSS skipDays (FRB024) + PermanentURL string // set on a 301/308 chain ending in 200/304 (FRB130/131) + Body string // snippet of an error body for the user (FRB101/120) +} + +func New() *Fetcher { + return &Fetcher{ + client: &http.Client{ + Timeout: defaultTimeout, + // Handle redirects manually so permanent (301/308) moves can be + // persisted while temporary (302/307) ones are merely followed. + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + userAgent: buildUserAgent(), + privateHost: isPrivateHost, + } +} + +// UserAgent returns the identifying User-Agent string sent with every request. +func (f *Fetcher) UserAgent() string { return f.userAgent } + +// Fetch issues a single conditional GET for feedURL. etag and lastModified, if +// non-empty, are sent back verbatim as If-None-Match / If-Modified-Since. +func (f *Fetcher) Fetch(ctx context.Context, feedURL, etag, lastModified string) (*Result, error) { + currentURL := feedURL + permanentPrefix := true + var permanentURL string + // A feed the admin subscribed to on an internal address may redirect + // internally, but a public feed must not bounce the bot into the local + // network (SSRF). Only resolved once a redirect is actually followed. + var originPrivate *bool + + for hop := 0; ; hop++ { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, currentURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", f.userAgent) + // The stored validators belong to the final resource of the chain, so + // they are sent on every hop; otherwise feeds behind a temporary + // redirect could never be answered with 304. + if etag != "" { + req.Header.Set("If-None-Match", etag) + } + if lastModified != "" { + req.Header.Set("If-Modified-Since", lastModified) + } + + resp, err := f.client.Do(req) + if err != nil { + return nil, err + } + + if isRedirect(resp.StatusCode) { + location := resp.Header.Get("Location") + // Drain a little so the connection can be reused for the next hop. + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4<<10)) + _ = resp.Body.Close() + if location == "" { + return nil, fmt.Errorf("redirect status %d without Location header", resp.StatusCode) + } + if hop >= maxRedirects { + return nil, fmt.Errorf("too many redirects (>%d)", maxRedirects) + } + + next, err := resolveLocation(currentURL, location) + if err != nil { + return nil, err + } + if originPrivate == nil { + private := f.privateHost(ctx, hostOf(feedURL)) + originPrivate = &private + } + if !*originPrivate && f.privateHost(ctx, hostOf(next)) { + return nil, fmt.Errorf("redirect to private address %s refused", next) + } + if (resp.StatusCode == http.StatusMovedPermanently || + resp.StatusCode == http.StatusPermanentRedirect) && permanentPrefix { + permanentURL = next + } else { + permanentPrefix = false + } + currentURL = next + continue + } + + result := &Result{Status: resp.StatusCode} + // Only report a move when the new location actually serves the feed. + succeeded := resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotModified + if succeeded && permanentURL != "" && permanentURL != feedURL { + result.PermanentURL = permanentURL + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) + _ = resp.Body.Close() + if err != nil { + return nil, err + } + + switch resp.StatusCode { + case http.StatusNotModified: + result.NotModified = true + result.MaxAge = parseMaxAge(resp.Header.Get("Cache-Control")) + case http.StatusOK: + feed, err := gofeed.NewParser().Parse(bytes.NewReader(body)) + if err != nil { + result.Body = snippet(body) + return result, fmt.Errorf("not a valid feed: %w", err) + } + result.Feed = feed + result.ETag = resp.Header.Get("Etag") + result.LastModified = resp.Header.Get("Last-Modified") + result.MaxAge = parseMaxAge(resp.Header.Get("Cache-Control")) + applyFeedHints(result, body) + case http.StatusTooManyRequests, http.StatusServiceUnavailable: + result.RetryAfter = parseRetryAfter(resp.Header.Get("Retry-After")) + result.Body = snippet(body) + default: + result.Body = snippet(body) + } + + return result, nil + } +} + +func isRedirect(status int) bool { + switch status { + case http.StatusMovedPermanently, http.StatusFound, + http.StatusTemporaryRedirect, http.StatusPermanentRedirect: + return true + } + return false +} + +func resolveLocation(base, location string) (string, error) { + b, err := url.Parse(base) + if err != nil { + return "", err + } + l, err := url.Parse(location) + if err != nil { + return "", err + } + next := b.ResolveReference(l) + if next.Scheme != "http" && next.Scheme != "https" { + return "", fmt.Errorf("redirect to unsupported scheme %q", next.Scheme) + } + return next.String(), nil +} + +func hostOf(rawURL string) string { + if u, err := url.Parse(rawURL); err == nil { + return u.Hostname() + } + return "" +} + +// isPrivateHost reports whether host is or resolves to a loopback, private, +// link-local or unspecified address. +func isPrivateHost(ctx context.Context, host string) bool { + if host == "" { + return false + } + if ip := net.ParseIP(host); ip != nil { + return isPrivateIP(ip) + } + addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil { + return false + } + for _, addr := range addrs { + if isPrivateIP(addr.IP) { + return true + } + } + return false +} + +func isPrivateIP(ip net.IP) bool { + return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || + ip.IsLinkLocalMulticast() || ip.IsUnspecified() +} + +func parseMaxAge(cacheControl string) time.Duration { + for _, part := range strings.Split(cacheControl, ",") { + part = strings.TrimSpace(part) + if v, ok := strings.CutPrefix(part, "max-age="); ok { + if secs, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && secs > 0 { + return time.Duration(secs) * time.Second + } + } + } + return 0 +} + +// parseRetryAfter accepts either delta-seconds or an HTTP-date (RFC 7231). +func parseRetryAfter(value string) time.Duration { + value = strings.TrimSpace(value) + if value == "" { + return 0 + } + if secs, err := strconv.Atoi(value); err == nil { + if secs < 0 { + return 0 + } + return time.Duration(secs) * time.Second + } + if t, err := http.ParseTime(value); err == nil { + if d := time.Until(t); d > 0 { + return d + } + } + return 0 +} + +// applyFeedHints extracts RSS polling hints (ttl, skipHours, skipDays) from the +// raw body when it is an RSS feed. sy:updatePeriod is deliberately ignored: +// WordPress emits "hourly" by default for every feed, regardless of how often it +// actually changes. +func applyFeedHints(result *Result, body []byte) { + feedType := gofeed.DetectFeedType(bytes.NewReader(body)) + if feedType != gofeed.FeedTypeRSS { + return + } + rssFeed, err := (&rss.Parser{}).Parse(bytes.NewReader(body)) + if err != nil { + return + } + + for _, h := range rssFeed.SkipHours { + if n, err := strconv.Atoi(strings.TrimSpace(h)); err == nil && n >= 0 && n <= 23 { + result.SkipHours = append(result.SkipHours, n) + } + } + result.SkipDays = rssFeed.SkipDays + + if ttl := strings.TrimSpace(rssFeed.TTL); ttl != "" { + if mins, err := strconv.Atoi(ttl); err == nil && mins > 0 { + result.FeedInterval = time.Duration(mins) * time.Minute + } + } +} + +func snippet(body []byte) string { + s := strings.TrimSpace(strings.ToValidUTF8(string(body), "")) + if utf8.RuneCountInString(s) > bodySnippetLen { + return string([]rune(s)[:bodySnippetLen]) + "…" + } + return s +} + +// Version can be set at build time via +// -ldflags "-X github.com/Brawl345/rssbot/fetcher.Version=...". Otherwise the +// VCS revision embedded by the Go toolchain is used. +var Version string + +func buildUserAgent() string { + if Version != "" { + return fmt.Sprintf("rssbot/%s (+%s)", Version, repoURL) + } + version := "dev" + if info, ok := debug.ReadBuildInfo(); ok { + for _, setting := range info.Settings { + if setting.Key == "vcs.revision" && setting.Value != "" { + version = setting.Value + if len(version) > 7 { + version = version[:7] + } + break + } + } + } + return fmt.Sprintf("rssbot/%s (+%s)", version, repoURL) +} diff --git a/fetcher/fetcher_test.go b/fetcher/fetcher_test.go new file mode 100644 index 0000000..3c32746 --- /dev/null +++ b/fetcher/fetcher_test.go @@ -0,0 +1,265 @@ +package fetcher + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +const sampleRSS = ` + + + Test + https://example.org + 90 + 023 + Sunday + hourly + 2 + Item 1https://example.org/11 + +` + +func fetch(t *testing.T, f *Fetcher, url, etag, lm string) *Result { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + res, err := f.Fetch(ctx, url, etag, lm) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + return res +} + +func TestFetchSetsHeadersAndParses(t *testing.T) { + var gotMethod, gotUA, gotINM, gotIMS, gotAE string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotMethod = r.Method + gotUA = r.Header.Get("User-Agent") + gotINM = r.Header.Get("If-None-Match") + gotIMS = r.Header.Get("If-Modified-Since") + gotAE = r.Header.Get("Accept-Encoding") + w.Header().Set("Etag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") + w.Header().Set("Cache-Control", "max-age=1800") + _, _ = w.Write([]byte(sampleRSS)) + })) + defer srv.Close() + + f := New() + res := fetch(t, f, srv.URL, "", "") + + if gotMethod != http.MethodGet { + t.Errorf("method = %s, want GET (FRB050)", gotMethod) + } + if !strings.HasPrefix(gotUA, "rssbot/") || !strings.Contains(gotUA, "github.com/Brawl345/rssbot") { + t.Errorf("User-Agent = %q, want identifying rssbot UA (FRB080-090)", gotUA) + } + if gotINM != "" || gotIMS != "" { + t.Errorf("first request must be unconditional, got INM=%q IMS=%q (FRB012/013)", gotINM, gotIMS) + } + if !strings.Contains(gotAE, "gzip") { + t.Errorf("Accept-Encoding = %q, want gzip (FRB141)", gotAE) + } + if res.Status != 200 || res.Feed == nil || len(res.Feed.Items) != 1 { + t.Fatalf("unexpected result: status=%d feed=%v", res.Status, res.Feed) + } + if res.ETag != `"abc"` { + t.Errorf("ETag = %q, want %q stored verbatim (FRB003)", res.ETag, `"abc"`) + } + if res.LastModified != "Wed, 21 Oct 2015 07:28:00 GMT" { + t.Errorf("LastModified = %q, not stored verbatim (FRB001)", res.LastModified) + } + if res.MaxAge != 30*time.Minute { + t.Errorf("MaxAge = %s, want 30m (FRB022)", res.MaxAge) + } + if res.FeedInterval != 90*time.Minute { + t.Errorf("FeedInterval = %s, want 90m from ttl (FRB024)", res.FeedInterval) + } + if len(res.SkipHours) != 2 || res.SkipHours[0] != 0 || res.SkipHours[1] != 23 { + t.Errorf("SkipHours = %v, want [0 23] (FRB024)", res.SkipHours) + } + if len(res.SkipDays) != 1 || res.SkipDays[0] != "Sunday" { + t.Errorf("SkipDays = %v, want [Sunday] (FRB024)", res.SkipDays) + } +} + +func TestSyndicationHintIgnored(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(strings.Replace(sampleRSS, "90", "", 1))) + })) + defer srv.Close() + + if res := fetch(t, New(), srv.URL, "", ""); res.FeedInterval != 0 { + t.Errorf("FeedInterval = %s, want 0 without ttl", res.FeedInterval) + } +} + +func TestConditionalRequestEchoed(t *testing.T) { + var gotINM, gotIMS string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotINM = r.Header.Get("If-None-Match") + gotIMS = r.Header.Get("If-Modified-Since") + w.WriteHeader(http.StatusNotModified) + })) + defer srv.Close() + + f := New() + res := fetch(t, f, srv.URL, `"abc"`, "Wed, 21 Oct 2015 07:28:00 GMT") + + if gotINM != `"abc"` { + t.Errorf("If-None-Match = %q, want verbatim etag (FRB004/013)", gotINM) + } + if gotIMS != "Wed, 21 Oct 2015 07:28:00 GMT" { + t.Errorf("If-Modified-Since = %q, want verbatim last-modified (FRB002/012)", gotIMS) + } + if !res.NotModified || res.Status != http.StatusNotModified { + t.Errorf("expected 304 NotModified, got status=%d notmod=%v", res.Status, res.NotModified) + } +} + +func TestConditionalRequestThroughTemporaryRedirect(t *testing.T) { + var gotINM string + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotINM = r.Header.Get("If-None-Match") + w.WriteHeader(http.StatusNotModified) + })) + defer target.Close() + + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer src.Close() + + res := fetch(t, New(), src.URL, `"abc"`, "") + if gotINM != `"abc"` { + t.Errorf("If-None-Match after 302 = %q, want %q", gotINM, `"abc"`) + } + if !res.NotModified { + t.Errorf("expected 304 through temporary redirect, got status=%d", res.Status) + } +} + +func TestRetryAfterParsed(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "120") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte("slow down")) + })) + defer srv.Close() + + res := fetch(t, New(), srv.URL, "", "") + if res.Status != 429 { + t.Fatalf("status = %d, want 429", res.Status) + } + if res.RetryAfter != 2*time.Minute { + t.Errorf("RetryAfter = %s, want 2m (FRB020)", res.RetryAfter) + } + if res.Body != "slow down" { + t.Errorf("Body = %q, want error snippet (FRB120)", res.Body) + } +} + +func TestPermanentRedirectReported(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(sampleRSS)) + })) + defer target.Close() + + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusMovedPermanently) + })) + defer src.Close() + + res := fetch(t, New(), src.URL, "", "") + if res.PermanentURL != target.URL { + t.Errorf("PermanentURL = %q, want %q (FRB130)", res.PermanentURL, target.URL) + } + if res.Status != 200 || res.Feed == nil { + t.Errorf("expected feed content after following 301, got status=%d", res.Status) + } +} + +func TestPermanentRedirectToErrorNotReported(t *testing.T) { + target := httptest.NewServer(http.NotFoundHandler()) + defer target.Close() + + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusMovedPermanently) + })) + defer src.Close() + + res := fetch(t, New(), src.URL, "", "") + if res.PermanentURL != "" { + t.Errorf("PermanentURL = %q, want empty when target fails", res.PermanentURL) + } + if res.Status != http.StatusNotFound { + t.Errorf("status = %d, want 404", res.Status) + } +} + +func TestTemporaryRedirectNotPersisted(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(sampleRSS)) + })) + defer target.Close() + + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) // 302 + })) + defer src.Close() + + res := fetch(t, New(), src.URL, "", "") + if res.PermanentURL != "" { + t.Errorf("PermanentURL = %q, want empty for 302 (FRB133)", res.PermanentURL) + } + if res.Status != 200 || res.Feed == nil { + t.Errorf("expected feed content after following 302, got status=%d", res.Status) + } +} + +func TestRedirectFromPublicToPrivateRefused(t *testing.T) { + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://internal.test/feed", http.StatusFound) + })) + defer src.Close() + + f := New() + f.privateHost = func(_ context.Context, host string) bool { return host == "internal.test" } + + _, err := f.Fetch(context.Background(), src.URL, "", "") + if err == nil || !strings.Contains(err.Error(), "private") { + t.Fatalf("err = %v, want refused private redirect", err) + } +} + +func TestRedirectToUnsupportedSchemeRefused(t *testing.T) { + src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "file:///etc/passwd", http.StatusFound) + })) + defer src.Close() + + _, err := New().Fetch(context.Background(), src.URL, "", "") + if err == nil || !strings.Contains(err.Error(), "scheme") { + t.Fatalf("err = %v, want refused scheme", err) + } +} + +func TestNonFeedRejected(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("not a feed")) + })) + defer srv.Close() + + ctx := context.Background() + res, err := New().Fetch(ctx, srv.URL, "", "") + if err == nil { + t.Fatalf("expected error for non-feed body (FRB102)") + } + if res == nil || res.Body == "" { + t.Errorf("expected body snippet on parse failure (FRB101)") + } +} diff --git a/fetcher/parse_test.go b/fetcher/parse_test.go new file mode 100644 index 0000000..c1f8b9d --- /dev/null +++ b/fetcher/parse_test.go @@ -0,0 +1,163 @@ +package fetcher + +import ( + "compress/gzip" + "context" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestParseMaxAge(t *testing.T) { + tests := map[string]time.Duration{ + "": 0, + "max-age=600": 10 * time.Minute, + "public, max-age=3600, s-maxage=60": time.Hour, + "no-cache": 0, + "max-age=0": 0, + "max-age=-5": 0, + "max-age=abc": 0, + } + for in, want := range tests { + if got := parseMaxAge(in); got != want { + t.Errorf("parseMaxAge(%q) = %s, want %s", in, got, want) + } + } +} + +func TestParseRetryAfter(t *testing.T) { + tests := map[string]time.Duration{ + "": 0, + "120": 2 * time.Minute, + " 30 ": 30 * time.Second, + "-1": 0, + "garbage": 0, + "Wed, 21 Oct 2015 07:28:00 GMT": 0, + } + for in, want := range tests { + if got := parseRetryAfter(in); got != want { + t.Errorf("parseRetryAfter(%q) = %s, want %s", in, got, want) + } + } + + future := time.Now().Add(time.Hour).UTC().Format(http.TimeFormat) + if got := parseRetryAfter(future); got < 59*time.Minute || got > time.Hour { + t.Errorf("parseRetryAfter(%q) = %s, want ~1h", future, got) + } +} + +func TestIsPrivateIP(t *testing.T) { + tests := map[string]bool{ + "127.0.0.1": true, + "10.1.2.3": true, + "192.168.0.1": true, + "172.16.0.1": true, + "169.254.169.254": true, + "0.0.0.0": true, + "::1": true, + "fe80::1": true, + "fd00::1": true, + "1.1.1.1": false, + "2606:4700::1111": false, + } + for in, want := range tests { + if got := isPrivateIP(net.ParseIP(in)); got != want { + t.Errorf("isPrivateIP(%s) = %v, want %v", in, got, want) + } + } +} + +func TestIsPrivateHostLiteral(t *testing.T) { + ctx := context.Background() + if !isPrivateHost(ctx, "127.0.0.1") || isPrivateHost(ctx, "8.8.8.8") || isPrivateHost(ctx, "") { + t.Errorf("isPrivateHost misclassified an IP literal") + } +} + +func TestTooManyRedirects(t *testing.T) { + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, srv.URL+r.URL.Path+"x", http.StatusFound) + })) + defer srv.Close() + + _, err := New().Fetch(context.Background(), srv.URL+"/", "", "") + if err == nil || !strings.Contains(err.Error(), "too many redirects") { + t.Fatalf("err = %v, want too many redirects", err) + } +} + +func TestRedirectWithoutLocation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusMovedPermanently) + })) + defer srv.Close() + + if _, err := New().Fetch(context.Background(), srv.URL, "", ""); err == nil { + t.Fatal("expected error for redirect without Location") + } +} + +func TestPermanentChainBrokenByTemporaryRedirect(t *testing.T) { + final := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(sampleRSS)) + })) + defer final.Close() + temp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, final.URL, http.StatusTemporaryRedirect) + })) + defer temp.Close() + perm := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, temp.URL, http.StatusPermanentRedirect) + })) + defer perm.Close() + + res := fetch(t, New(), perm.URL, "", "") + if res.PermanentURL != temp.URL { + t.Errorf("PermanentURL = %q, want the last permanent target %q", res.PermanentURL, temp.URL) + } +} + +func TestGzipResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + http.Error(w, "gzip expected", http.StatusBadRequest) + return + } + w.Header().Set("Content-Encoding", "gzip") + gz := gzip.NewWriter(w) + _, _ = gz.Write([]byte(sampleRSS)) + _ = gz.Close() + })) + defer srv.Close() + + res := fetch(t, New(), srv.URL, "", "") + if res.Feed == nil || len(res.Feed.Items) != 1 { + t.Fatalf("gzip feed not decoded: status=%d", res.Status) + } +} + +func TestErrorStatusKeepsBodySnippet(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "maintenance", http.StatusInternalServerError) + })) + defer srv.Close() + + res := fetch(t, New(), srv.URL, "", "") + if res.Status != http.StatusInternalServerError || res.Body != "maintenance" || res.Feed != nil { + t.Errorf("unexpected result: status=%d body=%q", res.Status, res.Body) + } +} + +func TestUserAgentVersion(t *testing.T) { + old := Version + t.Cleanup(func() { Version = old }) + + Version = "abc1234" + if ua := buildUserAgent(); ua != "rssbot/abc1234 (+https://github.com/Brawl345/rssbot)" { + t.Errorf("User-Agent = %q", ua) + } +} diff --git a/fetcher/snippet_test.go b/fetcher/snippet_test.go new file mode 100644 index 0000000..727ea9f --- /dev/null +++ b/fetcher/snippet_test.go @@ -0,0 +1,18 @@ +package fetcher + +import ( + "strings" + "testing" + "unicode/utf8" +) + +func TestSnippetKeepsValidUTF8(t *testing.T) { + body := []byte(strings.Repeat("ä", bodySnippetLen+10) + "\xff") + s := snippet(body) + if !utf8.ValidString(s) { + t.Fatalf("snippet is not valid UTF-8") + } + if n := utf8.RuneCountInString(s); n != bodySnippetLen+1 { + t.Errorf("snippet has %d runes, want %d", n, bodySnippetLen+1) + } +} diff --git a/flake.lock b/flake.lock index a87f67d..c1c295a 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1725534445, - "narHash": "sha256-Yd0FK9SkWy+ZPuNqUgmVPXokxDgMJoGuNpMEtkfcf84=", + "lastModified": 1790178388, + "narHash": "sha256-kK3t7gwoz4Nx8RF46cs1Xz/skKNcK7y3KYP+gGqX6W8=", "owner": "nixos", "repo": "nixpkgs", - "rev": "9bb1e7571aadf31ddb4af77fc64b2d59580f9a39", + "rev": "00455b0a3690d3f5dc61e9aef4277dc86235b73f", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 01c2a93..c730765 100644 --- a/flake.nix +++ b/flake.nix @@ -23,11 +23,16 @@ { nixosModules = { - default = ./module.nix; + default = + { lib, pkgs, ... }: + { + imports = [ ./module.nix ]; + services.rssbot.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.default; + }; }; overlays.default = final: prev: { - rssbot = self.packages.${prev.system}.default; + rssbot = self.packages.${final.stdenv.hostPlatform.system}.default; }; devShells = forAllSystems (pkgs: { @@ -39,6 +44,16 @@ }; }); + checks = + nixpkgs.lib.genAttrs + [ + "x86_64-linux" + "aarch64-linux" + ] + (system: { + nixos-module = nixpkgs.legacyPackages.${system}.testers.runNixOSTest (import ./nixos-test.nix self); + }); + packages = forAllSystems (pkgs: { rssbot = pkgs.buildGoModule { pname = "rssbot"; @@ -47,11 +62,12 @@ # Update the hash if go dependencies change! # vendorHash = pkgs.lib.fakeHash; - vendorHash = "sha256-mo30V7ISVFY8Rl3yXChP6pbehV9hTPH3UlBLDb1dzNE="; + vendorHash = "sha256-EQOlQCllo/nwJTv/ZhlR/ORKj1Kt6+mFHPIbEFE3QvI="; ldflags = [ "-s" "-w" + "-X github.com/Brawl345/rssbot/fetcher.Version=${version}" ]; meta = { @@ -62,7 +78,7 @@ }; }; - default = self.packages.${pkgs.system}.rssbot; + default = self.packages.${pkgs.stdenv.hostPlatform.system}.rssbot; }); }; } diff --git a/go.mod b/go.mod index 95cef4d..9bad6dd 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,20 @@ module github.com/Brawl345/rssbot -go 1.22 +go 1.26.0 require ( - github.com/go-sql-driver/mysql v1.8.1 + github.com/go-sql-driver/mysql v1.10.1 github.com/jmoiron/sqlx v1.4.0 github.com/joho/godotenv v1.5.1 - github.com/mmcdole/gofeed v1.3.0 - github.com/rubenv/sql-migrate v1.7.0 + github.com/mmcdole/gofeed v1.4.2 + github.com/rubenv/sql-migrate v1.8.1 gopkg.in/telebot.v3 v3.3.8 ) require ( - filippo.io/edwards25519 v1.1.0 // indirect - github.com/PuerkitoBio/goquery v1.8.1 // indirect - github.com/andybalholm/cascadia v1.3.2 // indirect + filippo.io/edwards25519 v1.2.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mmcdole/goxpp v1.1.1 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/text v0.18.0 // indirect + github.com/mmcdole/goxpp/v2 v2.0.0 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/text v0.42.0 // indirect ) diff --git a/go.sum b/go.sum index 7435dea..d97688d 100644 --- a/go.sum +++ b/go.sum @@ -54,22 +54,18 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= -github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= -github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -137,11 +133,9 @@ github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvSc github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= -github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME= +github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -257,8 +251,6 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= -github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -268,7 +260,6 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -290,9 +281,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -307,9 +296,7 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= @@ -321,22 +308,14 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s= -github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4= -github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= -github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= -github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI= -github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= -github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= -github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= -github.com/mmcdole/goxpp v1.1.1 h1:RGIX+D6iQRIunGHrKqnA2+700XMCnNv0bAOOv5MUhx8= -github.com/mmcdole/goxpp v1.1.1/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/mmcdole/gofeed v1.4.2 h1:XFFOtsNNZg+zudjtMXb8BI0J/YdnSIGfjEpPgPnZib0= +github.com/mmcdole/gofeed v1.4.2/go.mod h1:X5x1PyeibJi152VEya0AsV+PW4daYmCD4LJaJbeFkcs= +github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ= +github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -377,10 +356,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rubenv/sql-migrate v1.6.0 h1:IZpcTlAx/VKXphWEpwWJ7BaMq05tYtE80zYz+8a5Il8= -github.com/rubenv/sql-migrate v1.6.0/go.mod h1:m3ilnKP7sNb4eYkLsp6cGdPOl4OBcXM6rcbzU+Oqc5k= -github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= -github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= +github.com/rubenv/sql-migrate v1.8.1 h1:EPNwCvjAowHI3TnZ+4fQu3a915OpnQoPAjTXCGOy2U0= +github.com/rubenv/sql-migrate v1.8.1/go.mod h1:BTIKBORjzyxZDS6dzoiw6eAFYJ1iNlGAtjn4LGeVjS8= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -405,8 +382,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -414,7 +391,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU= @@ -438,7 +414,6 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -476,8 +451,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -518,7 +491,6 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -526,14 +498,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -566,8 +532,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -646,13 +610,8 @@ golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -662,12 +621,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -726,8 +681,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -911,8 +864,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/telebot.v3 v3.2.1 h1:3I4LohaAyJBiivGmkfB+CiVu7QFOWkuZ4+KHgO/G3rs= -gopkg.in/telebot.v3 v3.2.1/go.mod h1:GJKwwWqp9nSkIVN51eRKU78aB5f5OnQuWdwiIZfPbko= gopkg.in/telebot.v3 v3.3.8 h1:uVDGjak9l824FN9YARWUHMsiNZnlohAVwUycw21k6t8= gopkg.in/telebot.v3 v3.3.8/go.mod h1:1mlbqcLTVSfK9dx7fdp+Nb5HZsy4LLPtpZTKmwhwtzM= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handler/commands_test.go b/handler/commands_test.go new file mode 100644 index 0000000..186e21c --- /dev/null +++ b/handler/commands_test.go @@ -0,0 +1,233 @@ +package handler + +import ( + "database/sql" + "net/http" + "strings" + "testing" + "time" + + "github.com/Brawl345/rssbot/storage" + "gopkg.in/telebot.v3" +) + +const userID = 10 + +type createCall struct { + ChatID int64 + FeedURL string + LastEntry *string + ETag *string + LastModified *string + Hints storage.PollHints + NextPollAt time.Time +} + +func (s *fakeStore) Create(chatID int64, chatTitle string, feedURL string, lastEntry, etag, lastModified *string, hints storage.PollHints, nextPollAt time.Time) error { + s.mu.Lock() + defer s.mu.Unlock() + s.created = append(s.created, createCall{chatID, feedURL, lastEntry, etag, lastModified, hints, nextPollAt}) + s.subscriptions[chatID] = append(s.subscriptions[chatID], storage.Feed{Url: feedURL}) + return nil +} + +func (s *fakeStore) Delete(chatID int64, feedID int64) error { + s.mu.Lock() + defer s.mu.Unlock() + s.deleted = append(s.deleted, [2]int64{chatID, feedID}) + return nil +} + +func (s *fakeStore) ExistsByFeedUrl(chatID int64, feedURL string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, f := range s.subscriptions[chatID] { + if f.Url == feedURL { + return true, nil + } + } + return false, nil +} + +func (s *fakeStore) ExistsById(chatID int64, feedID int64) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, f := range s.subscriptions[chatID] { + if f.ID == feedID { + return true, nil + } + } + return false, nil +} + +func (s *fakeStore) GetByUser(chatID int64) ([]storage.Feed, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.subscriptions[chatID], nil +} + +func (s *fakeStore) ReactivateFeed(feedURL string, nextPollAt time.Time) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + was := s.disabledURLs[feedURL] + delete(s.disabledURLs, feedURL) + return was, nil +} + +func (env *pollEnv) command(t *testing.T, handler telebot.HandlerFunc, payload string) string { + t.Helper() + before := len(env.tg.messagesTo(userID)) + ctx := env.h.Bot.NewContext(telebot.Update{Message: &telebot.Message{ + Payload: payload, + Chat: &telebot.Chat{ID: userID, Type: telebot.ChatPrivate, FirstName: "Tester"}, + }}) + if err := handler(ctx); err != nil { + t.Fatalf("handler returned %v", err) + } + msgs := env.tg.messagesTo(userID) + if len(msgs) != before+1 { + t.Fatalf("got %d replies, want 1", len(msgs)-before) + } + return msgs[len(msgs)-1] +} + +func TestSubscribeRejectsInvalidInput(t *testing.T) { + html := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("no feed")) + }) + missing := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "nope", http.StatusNotFound) + }) + + tests := []struct { + name string + payload string + want string + }{ + {"bad scheme", "ftp://example.org/feed", "Ungültige URL"}, + {"bad characters", "https://example.org/", "Ungültige URL"}, + {"not found", missing.URL, "HTTP 404"}, + {"not a feed", html.URL, "Ungültiger Feed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env := newPollEnv(t) + reply := env.command(t, env.h.OnSubscribe, tt.payload) + if !strings.Contains(reply, tt.want) { + t.Errorf("reply = %q, want %q", reply, tt.want) + } + if len(env.store.created) != 0 { + t.Errorf("feed must not be created") + } + }) + } +} + +func TestSubscribeStoresValidatorsAndHints(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Etag", `"abc"`) + w.Header().Set("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT") + _, _ = w.Write([]byte(rss("120", "2", "1"))) + }) + + reply := env.command(t, env.h.OnSubscribe, srv.URL) + if !strings.Contains(reply, "erfolgreich abonniert") { + t.Fatalf("reply = %q", reply) + } + if len(env.store.created) != 1 { + t.Fatalf("Create called %d times, want 1", len(env.store.created)) + } + c := env.store.created[0] + if c.ChatID != userID || c.FeedURL != srv.URL { + t.Errorf("created %d/%q", c.ChatID, c.FeedURL) + } + if c.LastEntry == nil || *c.LastEntry != "2" { + t.Errorf("last entry = %v, want newest item", c.LastEntry) + } + if c.ETag == nil || *c.ETag != `"abc"` || c.LastModified == nil { + t.Errorf("validators not stored: etag=%v lm=%v", c.ETag, c.LastModified) + } + if c.Hints.Interval != 2*time.Hour { + t.Errorf("ttl hint = %s, want 2h", c.Hints.Interval) + } + assertDelay(t, c.NextPollAt, 2*time.Hour) +} + +func TestSubscribeUsesSelfLink(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(` +Thttps://example.org + +Aa`)) + }) + + env.command(t, env.h.OnSubscribe, srv.URL) + if got := env.store.created[0].FeedURL; got != "https://example.org/canonical.xml" { + t.Errorf("subscribed to %q, want the self link", got) + } +} + +func TestSubscribeExisting(t *testing.T) { + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(rss("", "1"))) + }) + + t.Run("already subscribed", func(t *testing.T) { + env := newPollEnv(t) + env.store.subscriptions[userID] = []storage.Feed{{Url: srv.URL}} + if reply := env.command(t, env.h.OnSubscribe, srv.URL); !strings.Contains(reply, "bereits abonniert") { + t.Errorf("reply = %q", reply) + } + if len(env.store.created) != 0 { + t.Errorf("duplicate subscription created") + } + }) + + t.Run("disabled feed is reactivated", func(t *testing.T) { + env := newPollEnv(t) + env.store.subscriptions[userID] = []storage.Feed{{Url: srv.URL}} + env.store.disabledURLs[srv.URL] = true + if reply := env.command(t, env.h.OnSubscribe, srv.URL); !strings.Contains(reply, "wieder aktiviert") { + t.Errorf("reply = %q", reply) + } + if env.store.disabledURLs[srv.URL] { + t.Errorf("feed still disabled") + } + }) +} + +func TestUnsubscribe(t *testing.T) { + env := newPollEnv(t) + env.store.subscriptions[userID] = []storage.Feed{{ID: 7, Url: "https://example.org/feed"}} + + if reply := env.command(t, env.h.OnUnsubscribe, "abc"); !strings.Contains(reply, "Feed-ID") { + t.Errorf("reply = %q", reply) + } + if reply := env.command(t, env.h.OnUnsubscribe, "8"); !strings.Contains(reply, "nicht abonniert") { + t.Errorf("reply = %q", reply) + } + if reply := env.command(t, env.h.OnUnsubscribe, "7"); !strings.Contains(reply, "deabonniert") { + t.Errorf("reply = %q", reply) + } + if len(env.store.deleted) != 1 || env.store.deleted[0] != [2]int64{userID, 7} { + t.Errorf("deleted = %v, want [[%d 7]]", env.store.deleted, userID) + } +} + +func TestListShowsDisabledFeeds(t *testing.T) { + env := newPollEnv(t) + env.store.subscriptions[userID] = []storage.Feed{ + {ID: 1, Url: "https://example.org/a?x=1&y=2"}, + {ID: 2, Url: "https://example.org/b", Disabled: true, DisabledReason: sql.NullString{String: "HTTP 410 Gone", Valid: true}}, + } + + reply := env.command(t, env.h.OnList, "") + if !strings.Contains(reply, "a?x=1&y=2") { + t.Errorf("URL not escaped: %q", reply) + } + if strings.Count(reply, "🚫") != 1 || !strings.Contains(reply, "HTTP 410 Gone") { + t.Errorf("disabled marker missing: %q", reply) + } +} diff --git a/handler/feed_check.go b/handler/feed_check.go index bcbcd8a..570190f 100644 --- a/handler/feed_check.go +++ b/handler/feed_check.go @@ -2,7 +2,9 @@ package handler import ( "bytes" + "context" "errors" + "fmt" "html" "log" "net/url" @@ -10,11 +12,52 @@ import ( "strings" "sync" "time" + "unicode/utf8" + "github.com/Brawl345/rssbot/fetcher" "github.com/Brawl345/rssbot/storage" + "github.com/mmcdole/gofeed" "gopkg.in/telebot.v3" ) +const ( + // A feed is retired once it failed at least maxFeedErrors times in a row + // and has been failing for retireAfter (FRB110-119). + maxFeedErrors = 12 + retireAfter = 7 * 24 * time.Hour + // fetchTimeout bounds a single feed fetch. + fetchTimeout = 35 * time.Second +) + +var ( + feedproxyRe = regexp.MustCompile("^https?://feedproxy.google.com/~r/(.+?)/.*") + blankLineRe = regexp.MustCompile("(?m)^\\s*$[\r\n]*") +) + +// compiledReplacement is a content filter with its regex compiled once per poll +// cycle instead of once per feed item. +type compiledReplacement struct { + re *regexp.Regexp // nil for a literal replacement + literal string +} + +func compileReplacements(replacements []storage.Replacement) []compiledReplacement { + compiled := make([]compiledReplacement, 0, len(replacements)) + for _, r := range replacements { + if r.IsRegex { + re, err := regexp.Compile(r.Value) + if err != nil { + log.Printf("skipping invalid replacement regex %q: %s", r.Value, err) + continue + } + compiled = append(compiled, compiledReplacement{re: re}) + } else { + compiled = append(compiled, compiledReplacement{literal: r.Value}) + } + } + return compiled +} + type TemplateData struct { Title string FeedTitle string @@ -23,152 +66,411 @@ type TemplateData struct { PostDomain string } +// OnCheck is the scheduler tick. It polls only feeds that are currently due and +// reschedules itself after POLL_TICK. Because due-ness lives in the database, +// a process restart never causes a mass re-fetch (FRB037). func (h *Handler) OnCheck() { - var wg sync.WaitGroup - log.Println("===============================/") - abonnements, err := h.DB.Abonnements.GetAll() + defer time.AfterFunc(h.Config.Poll.Tick, h.OnCheck) + + abonnements, err := h.DB.Abonnements.GetDue() if err != nil { log.Println(err) - time.AfterFunc(1*time.Minute, h.OnCheck) + return + } + if len(abonnements) == 0 { return } replacements, err := h.DB.Replacements.List() if err != nil { log.Println(err) - time.AfterFunc(1*time.Minute, h.OnCheck) return } - if len(abonnements) == 0 { - log.Println("No feeds found, checkin again in 60 seconds") + log.Printf("Polling %d due feed(s)", len(abonnements)) + h.pollFeeds(abonnements, compileReplacements(replacements)) +} + +// pollFeeds groups the due feeds by host and hands each group to a worker, so +// requests to the same host are serialized (FRB033/034) without idle workers +// blocking on a busy host. +func (h *Handler) pollFeeds(abonnements []storage.Abonnement, replacements []compiledReplacement) { + workers := h.Config.Poll.Concurrency + if workers < 1 { + workers = 1 } + groups := make(map[string][]storage.Abonnement) + var hosts []string for _, abonnement := range abonnements { - abonnement := abonnement + host := feedHost(abonnement.Url) + if _, ok := groups[host]; !ok { + hosts = append(hosts, host) + } + groups[host] = append(groups[host], abonnement) + } + + jobs := make(chan []storage.Abonnement) + var wg sync.WaitGroup + + for i := 0; i < workers; i++ { wg.Add(1) go func() { defer wg.Done() + for group := range jobs { + for _, abonnement := range group { + h.pollFeed(abonnement, replacements) + } + } + }() + } - log.Printf("%s", abonnement.Feed.Url) + for _, host := range hosts { + jobs <- groups[host] + } + close(jobs) - var lastEntry *string - if abonnement.LastEntry.Valid { - lastEntry = &abonnement.LastEntry.String - } + wg.Wait() +} - feed, err := abonnement.Feed.Check(lastEntry) - if err != nil { - log.Printf("%s: %s", abonnement.Feed.Url, err) +func (h *Handler) pollFeed(abonnement storage.Abonnement, replacements []compiledReplacement) { + feed := abonnement.Feed + + var etag, lastModified string + if feed.ETag.Valid { + etag = feed.ETag.String + } + if feed.LastModified.Valid { + lastModified = feed.LastModified.String + } + + ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout) + defer cancel() + + result, err := h.Fetcher.Fetch(ctx, feed.Url, etag, lastModified) + if err != nil { + h.handleSoftError(abonnement, feed, err.Error(), bodyOf(result)) + return + } + + // Permanent move: persist the new URL and stop polling the old one + // (FRB130/131/132). + if result.PermanentURL != "" { + merged, err := h.DB.Abonnements.MoveFeedURL(feed.ID, result.PermanentURL) + if err != nil { + log.Printf("%s: could not update url to %s: %s", feed.Url, result.PermanentURL, err) + } else { + h.notify(abonnement, fmt.Sprintf("ℹ️ Feed wurde dauerhaft umgezogen:\n%s\n→ %s", + html.EscapeString(feed.Url), html.EscapeString(result.PermanentURL))) + if merged { + // This feed row is gone; the existing feed at the target URL + // now owns these subscriptions and will deliver the content. return } + feed.Url = result.PermanentURL + } + } - for _, entry := range reverse(feed.Items) { - templateData := &TemplateData{} - if entry.Title != "" { - templateData.Title = html.EscapeString(entry.Title) - } else { - templateData.Title = "Kein Titel" - } + switch { + case result.Status == 200 && result.Feed != nil: + h.handleOK(abonnement, feed, result, replacements) + case result.NotModified: + hints := feed.Hints() + result.FeedInterval, result.SkipHours, result.SkipDays = hints.Interval, hints.SkipHours, hints.SkipDays + next := h.nextPoll(feed.UnchangedCount+1, result) + if err := h.DB.Abonnements.Reschedule(feed.ID, next, 0, feed.UnchangedCount+1); err != nil { + log.Printf("%s: reschedule failed: %s", feed.Url, err) + } + case result.Status == 429 || result.Status == 503: + h.handleRateLimit(abonnement, feed, result) + case result.Status == 410: + h.disable(abonnement, feed, "HTTP 410 Gone", result.Body) + default: + h.handleSoftError(abonnement, feed, fmt.Sprintf("HTTP %d", result.Status), result.Body) + } +} - templateData.FeedTitle = html.EscapeString(feed.Title) +func (h *Handler) handleOK(abonnement storage.Abonnement, feed storage.Feed, result *fetcher.Result, replacements []compiledReplacement) { + gfeed := result.Feed - if entry.Content != "" { - templateData.Content = processContent(entry.Content, &replacements) - } else if entry.Description != "" { - templateData.Content = processContent(entry.Description, &replacements) - } + var lastEntry *string + if feed.LastEntry.Valid { + lastEntry = &feed.LastEntry.String + } + newItems := filterNewItems(gfeed.Items, lastEntry) - if entry.Link != "" { - templateData.PostLink = entry.Link - } else { - templateData.PostLink = feed.Link - } + for _, entry := range reverse(newItems) { + templateData := &TemplateData{} + if entry.Title != "" { + templateData.Title = html.EscapeString(entry.Title) + } else { + templateData.Title = "Kein Titel" + } - re := regexp.MustCompile("^https?://feedproxy.google.com/~r/(.+?)/.*") - match := re.FindStringSubmatch(templateData.PostLink) + templateData.FeedTitle = html.EscapeString(gfeed.Title) - if len(match) > 0 { - templateData.PostDomain = match[1] - } else { - parsedUrl, _ := url.Parse(templateData.PostLink) - templateData.PostDomain = parsedUrl.Host - } + if entry.Content != "" { + templateData.Content = processContent(entry.Content, replacements) + } else if entry.Description != "" { + templateData.Content = processContent(entry.Description, replacements) + } - templateData.PostDomain = strings.Replace(templateData.PostDomain, "www.", "", 1) + if entry.Link != "" { + templateData.PostLink = entry.Link + } else { + templateData.PostLink = gfeed.Link + } - var tpl bytes.Buffer - err := h.Config.Template.Execute(&tpl, templateData) - if err != nil { - log.Printf("%s: %s", abonnement.Feed.Url, err) - return - } + match := feedproxyRe.FindStringSubmatch(templateData.PostLink) + if len(match) > 0 { + templateData.PostDomain = match[1] + } else { + parsedUrl, _ := url.Parse(templateData.PostLink) + templateData.PostDomain = parsedUrl.Host + } + templateData.PostDomain = strings.Replace(templateData.PostDomain, "www.", "", 1) - for _, chat := range abonnement.Chats { - err = h.sendText(chat.ID, tpl.String(), abonnement.Feed.Url) - if err != nil { - log.Printf("%s: %s", abonnement.Feed.Url, err) - } - } + var tpl bytes.Buffer + if err := h.Config.Template.Execute(&tpl, templateData); err != nil { + log.Printf("%s: %s", feed.Url, err) + continue + } + for _, chat := range abonnement.Chats { + if err := h.sendText(chat.ID, tpl.String(), feed.Url); err != nil { + log.Printf("%s: %s", feed.Url, err) } - if len(feed.Items) > 0 { - var lastEntry *string - if feed.Items[0].GUID != "" { - lastEntry = &feed.Items[0].GUID - } else { - lastEntry = &feed.Items[0].Link - } - h.DB.Abonnements.SetLastEntry(abonnement.Feed.Url, lastEntry) - } - }() + } } - wg.Wait() - log.Println("/===============================") - time.AfterFunc(1*time.Minute, h.OnCheck) + newLastEntry := lastEntry + if len(gfeed.Items) > 0 { + if gfeed.Items[0].GUID != "" { + newLastEntry = &gfeed.Items[0].GUID + } else { + newLastEntry = &gfeed.Items[0].Link + } + } + + // FRB023: only reset the adaptive counter when genuinely new content arrived. + unchanged := feed.UnchangedCount + 1 + if len(newItems) > 0 { + unchanged = 0 + } + + next := h.nextPoll(unchanged, result) + err := h.DB.Abonnements.SetFeedState(feed.ID, newLastEntry, + nullableString(result.ETag), nullableString(result.LastModified), + storage.PollHints{Interval: result.FeedInterval, SkipHours: result.SkipHours, SkipDays: result.SkipDays}, + next, 0, unchanged) + if err != nil { + log.Printf("%s: could not save state: %s", feed.Url, err) + } +} + +func (h *Handler) handleRateLimit(abonnement storage.Abonnement, feed storage.Feed, result *fetcher.Result) { + delay := result.RetryAfter + if delay <= 0 { + // FRB021: a 429/503 without a hint is still "slow down". + delay = h.Config.Poll.Interval * 4 + } + delay = min(delay, h.Config.Poll.IntervalMax) + + // Keep the cached validators (FRB016) and do not count this toward retirement. + if err := h.DB.Abonnements.Reschedule(feed.ID, time.Now().Add(delay), feed.ErrorCount, feed.UnchangedCount); err != nil { + log.Printf("%s: reschedule failed: %s", feed.Url, err) + } + log.Printf("%s: HTTP %d, backing off %s", feed.Url, result.Status, delay.Round(time.Second)) + + if result.Status == 429 { + h.notify(abonnement, fmt.Sprintf( + "⚠️ Feed sendet \"429 Too Many Requests\":\n%s\nBackoff: %s. Eventuell ist das Poll-Intervall zu kurz.", + html.EscapeString(feed.Url), delay.Round(time.Second))) + } } -func processContent(content string, replacements *[]storage.Replacement) string { - processed := html.UnescapeString(content) +// handleSoftError applies exponential backoff and retires the feed once it has +// been failing for too long (FRB110-119). +func (h *Handler) handleSoftError(abonnement storage.Abonnement, feed storage.Feed, reason, body string) { + errorCount := feed.ErrorCount + 1 + failingSince := time.Now() + if feed.FailingSince.Valid { + failingSince = feed.FailingSince.Time + } + if errorCount >= maxFeedErrors && time.Since(failingSince) >= retireAfter { + h.disable(abonnement, feed, reason, body) + return + } + + delay := h.errorBackoff(errorCount) + if err := h.DB.Abonnements.Reschedule(feed.ID, time.Now().Add(delay), errorCount, feed.UnchangedCount); err != nil { + log.Printf("%s: reschedule failed: %s", feed.Url, err) + } + log.Printf("%s: %s (error %d, failing since %s), retrying in %s", + feed.Url, reason, errorCount, failingSince.Format(time.DateTime), delay.Round(time.Second)) +} + +func (h *Handler) disable(abonnement storage.Abonnement, feed storage.Feed, reason, body string) { + if err := h.DB.Abonnements.DisableFeed(feed.ID, reason); err != nil { + log.Printf("%s: could not disable: %s", feed.Url, err) + return + } + log.Printf("%s: disabled (%s)", feed.Url, reason) - for _, replacement := range *replacements { - if replacement.IsRegex { - re := regexp.MustCompile(replacement.Value) - processed = re.ReplaceAllString(processed, "") + msg := fmt.Sprintf("🚫 Feed wurde deaktiviert:\n%s\nGrund: %s", html.EscapeString(feed.Url), html.EscapeString(reason)) + if body != "" { + msg += fmt.Sprintf("\n
%s
", html.EscapeString(body)) + } + h.notify(abonnement, msg) +} + +// nextPoll computes the next poll time: base interval, optionally stretched for +// feeds that rarely change (FRB023), never faster than server hints +// (max-age/ttl, FRB022/024, capped at POLL_INTERVAL_MAX), shifted out of +// skipHours/skipDays (FRB024). +func (h *Handler) nextPoll(unchangedCount int, result *fetcher.Result) time.Time { + interval := h.Config.Poll.Interval + + // Grow linearly so a feed that was quiet overnight is not delayed for hours + // once it becomes active again. + if h.Config.Poll.Adaptive && unchangedCount > 0 { + if steps := time.Duration(unchangedCount); steps >= h.Config.Poll.IntervalMax/interval { + interval = h.Config.Poll.IntervalMax } else { - processed = strings.ReplaceAll(processed, replacement.Value, "") + interval = min(interval*(1+steps), h.Config.Poll.IntervalMax) } } - processed = regexp.MustCompile("(?m)^\\s*$[\r\n]*").ReplaceAllString(processed, "") - processed = strings.TrimSpace(processed) + if result != nil { + hint := min(max(result.MaxAge, result.FeedInterval), h.Config.Poll.IntervalMax) + interval = max(interval, hint) + } - if len(processed) > 270 { - return processed[:270] + "..." + next := time.Now().Add(interval) + if result != nil { + next = applySkip(next, result.SkipHours, result.SkipDays) } + return next +} - return processed +func (h *Handler) errorBackoff(errorCount int) time.Duration { + interval := h.Config.Poll.Interval + for i := 1; i < errorCount && interval < h.Config.Poll.IntervalMax; i++ { + interval *= 2 + } + if interval > h.Config.Poll.IntervalMax { + interval = h.Config.Poll.IntervalMax + } + return interval } -func (h *Handler) sendText(chatId int64, text string, url string) error { - _, err := h.Bot.Send(telebot.ChatID(chatId), text, defaultSendOptions) +// notify sends operational messages (disable, redirect, rate-limit) to the +// admin only — never into the subscriber chats/channels, which are reserved for +// feed content. +func (h *Handler) notify(abonnement storage.Abonnement, text string) { + if h.AdminID == 0 { + return + } + if err := h.sendText(h.AdminID, text, abonnement.Url); err != nil { + log.Printf("%s: notify failed: %s", abonnement.Url, err) + } +} + +// applySkip pushes t forward in whole hours until it is outside the feed's +// declared skipHours/skipDays (RSS uses UTC). Capped so an over-eager feed can +// never block polling forever. +func applySkip(t time.Time, skipHours []int, skipDays []string) time.Time { + if len(skipHours) == 0 && len(skipDays) == 0 { + return t + } + for i := 0; i < 48; i++ { + u := t.UTC() + if containsInt(skipHours, u.Hour()) || containsDay(skipDays, u.Weekday().String()) { + t = t.Add(time.Hour) + continue + } + break + } + return t +} - var floodError *telebot.FloodError +func filterNewItems(items []*gofeed.Item, lastEntry *string) []*gofeed.Item { + if lastEntry == nil { + return items + } + for i, item := range items { + if item.GUID == *lastEntry || item.Link == *lastEntry { + return items[:i] + } + } + return items +} - if err != nil { - if errors.As(err, &floodError) { - log.Printf("%s: Flood error, retrying after: %d seconds", url, - floodError.RetryAfter) - time.Sleep(time.Duration(err.(telebot.FloodError).RetryAfter) * time.Second) - err := h.sendText(chatId, text, url) - if err != nil { - return err - } +func feedHost(rawURL string) string { + if u, err := url.Parse(rawURL); err == nil && u.Host != "" { + return strings.ToLower(u.Host) + } + return rawURL +} + +func bodyOf(result *fetcher.Result) string { + if result == nil { + return "" + } + return result.Body +} + +func containsInt(s []int, v int) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} + +func containsDay(days []string, day string) bool { + for _, d := range days { + if strings.EqualFold(strings.TrimSpace(d), day) { + return true + } + } + return false +} + +func processContent(content string, replacements []compiledReplacement) string { + processed := strings.ToValidUTF8(html.UnescapeString(content), "") + + for _, replacement := range replacements { + if replacement.re != nil { + processed = replacement.re.ReplaceAllString(processed, "") } else { - return err + processed = strings.ReplaceAll(processed, replacement.literal, "") } + } + + processed = blankLineRe.ReplaceAllString(processed, "") + processed = strings.TrimSpace(processed) + + if utf8.RuneCountInString(processed) > 270 { + return string([]rune(processed)[:270]) + "..." + } + + return processed +} + +func (h *Handler) sendText(chatId int64, text string, url string) error { + _, err := h.Bot.Send(telebot.ChatID(chatId), text, defaultSendOptions) + if err == nil { + return nil + } + // telebot returns FloodError by value, so match the value type. + var floodError telebot.FloodError + if errors.As(err, &floodError) { + log.Printf("%s: Flood error, retrying after: %d seconds", url, floodError.RetryAfter) + time.Sleep(time.Duration(floodError.RetryAfter) * time.Second) + return h.sendText(chatId, text, url) } - return nil + return err } diff --git a/handler/feed_check_test.go b/handler/feed_check_test.go new file mode 100644 index 0000000..9088c50 --- /dev/null +++ b/handler/feed_check_test.go @@ -0,0 +1,265 @@ +package handler + +import ( + "database/sql" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/Brawl345/rssbot/config" + "github.com/Brawl345/rssbot/fetcher" + "github.com/Brawl345/rssbot/storage" + "github.com/mmcdole/gofeed" +) + +func testPollConfig() config.PollConfig { + return config.PollConfig{ + Interval: 10 * time.Minute, + IntervalMax: 6 * time.Hour, + Adaptive: true, + Concurrency: 4, + Tick: time.Second, + } +} + +func handlerWithPoll(poll config.PollConfig) *Handler { + return &Handler{Config: &config.Config{Poll: poll}} +} + +func assertDelay(t *testing.T, next time.Time, want time.Duration) { + t.Helper() + got := time.Until(next) + if got < want-time.Second || got > want+time.Second { + t.Errorf("next poll in %s, want %s", got.Round(time.Second), want) + } +} + +func TestNextPollAdaptive(t *testing.T) { + h := handlerWithPoll(testPollConfig()) + + tests := []struct { + unchanged int + want time.Duration + }{ + {0, 10 * time.Minute}, + {1, 20 * time.Minute}, + {2, 30 * time.Minute}, + {35, 6 * time.Hour}, + {36, 6 * time.Hour}, + {1 << 30, 6 * time.Hour}, + } + for _, tt := range tests { + assertDelay(t, h.nextPoll(tt.unchanged, nil), tt.want) + } +} + +func TestNextPollNotAdaptive(t *testing.T) { + poll := testPollConfig() + poll.Adaptive = false + assertDelay(t, handlerWithPoll(poll).nextPoll(50, nil), 10*time.Minute) +} + +func TestNextPollHints(t *testing.T) { + h := handlerWithPoll(testPollConfig()) + + tests := []struct { + name string + result fetcher.Result + want time.Duration + }{ + {"max-age slows down", fetcher.Result{MaxAge: time.Hour}, time.Hour}, + {"ttl slows down", fetcher.Result{FeedInterval: 90 * time.Minute}, 90 * time.Minute}, + {"larger hint wins", fetcher.Result{MaxAge: time.Hour, FeedInterval: 2 * time.Hour}, 2 * time.Hour}, + {"hint never speeds up", fetcher.Result{MaxAge: time.Minute}, 10 * time.Minute}, + {"hint capped", fetcher.Result{MaxAge: 365 * 24 * time.Hour}, 6 * time.Hour}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertDelay(t, h.nextPoll(0, &tt.result), tt.want) + }) + } +} + +func TestErrorBackoff(t *testing.T) { + h := handlerWithPoll(testPollConfig()) + + tests := []struct { + errors int + want time.Duration + }{ + {1, 10 * time.Minute}, + {2, 20 * time.Minute}, + {3, 40 * time.Minute}, + {6, 320 * time.Minute}, + {7, 6 * time.Hour}, + {100, 6 * time.Hour}, + } + for _, tt := range tests { + if got := h.errorBackoff(tt.errors); got != tt.want { + t.Errorf("errorBackoff(%d) = %s, want %s", tt.errors, got, tt.want) + } + } +} + +func TestApplySkip(t *testing.T) { + // 2026-01-04 is a Sunday. + at := func(day, hour int) time.Time { return time.Date(2026, 1, day, hour, 30, 0, 0, time.UTC) } + + tests := []struct { + name string + t time.Time + hours []int + days []string + want time.Time + }{ + {"no hints", at(4, 23), nil, nil, at(4, 23)}, + {"outside skipHours", at(4, 22), []int{23, 0}, nil, at(4, 22)}, + {"inside skipHours", at(4, 23), []int{23, 0}, nil, at(5, 1)}, + {"skipDays", at(4, 10), nil, []string{"Sunday"}, time.Date(2026, 1, 5, 0, 30, 0, 0, time.UTC)}, + {"skipDays is case-insensitive", at(4, 10), nil, []string{" sunday "}, time.Date(2026, 1, 5, 0, 30, 0, 0, time.UTC)}, + {"everything skipped is capped", at(4, 10), allHours(), nil, at(6, 10)}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := applySkip(tt.t, tt.hours, tt.days); !got.Equal(tt.want) { + t.Errorf("applySkip = %s, want %s", got, tt.want) + } + }) + } +} + +func allHours() []int { + hours := make([]int, 24) + for i := range hours { + hours[i] = i + } + return hours +} + +func TestFilterNewItems(t *testing.T) { + items := []*gofeed.Item{ + {GUID: "3", Link: "https://example.org/3"}, + {GUID: "", Link: "https://example.org/2"}, + {GUID: "1", Link: "https://example.org/1"}, + } + ptr := func(s string) *string { return &s } + + tests := []struct { + name string + lastEntry *string + want int + }{ + {"first poll", nil, 3}, + {"match by GUID", ptr("1"), 2}, + {"match by link", ptr("https://example.org/2"), 1}, + {"newest already seen", ptr("3"), 0}, + {"unknown entry returns all", ptr("gone"), 3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := filterNewItems(items, tt.lastEntry); len(got) != tt.want { + t.Errorf("got %d items, want %d", len(got), tt.want) + } + }) + } +} + +func TestReverse(t *testing.T) { + items := []*gofeed.Item{{GUID: "a"}, {GUID: "b"}, {GUID: "c"}} + got := reverse(items) + if got[0].GUID != "c" || got[2].GUID != "a" { + t.Errorf("reverse = %v", []string{got[0].GUID, got[1].GUID, got[2].GUID}) + } + if items[0].GUID != "a" { + t.Errorf("reverse modified its input") + } +} + +func TestFeedHost(t *testing.T) { + tests := map[string]string{ + "https://Example.org/feed.xml": "example.org", + "http://example.org:8080/rss": "example.org:8080", + "not a url": "not a url", + "https://www.example.org/a?b=c": "www.example.org", + } + for in, want := range tests { + if got := feedHost(in); got != want { + t.Errorf("feedHost(%q) = %q, want %q", in, got, want) + } + } +} + +func TestSanitizeFeedURL(t *testing.T) { + valid := map[string]string{ + "https://example.org/feed": "https://example.org/feed", + " http://example.org/rss.xml ": "http://example.org/rss.xml", + "https://example.org/?a=1&b=2": "https://example.org/?a=1&b=2", + } + for in, want := range valid { + got, err := sanitizeFeedURL(in) + if err != nil || got != want { + t.Errorf("sanitizeFeedURL(%q) = %q, %v; want %q", in, got, err, want) + } + } + + invalid := []string{ + "", + " ", + "ftp://example.org/feed", + "example.org/feed", + "https:///feed", + "https://example.org/a b", + `https://example.org/"feed"`, + "https://example.org/", + "javascript:alert(1)", + } + for _, in := range invalid { + if got, err := sanitizeFeedURL(in); err == nil { + t.Errorf("sanitizeFeedURL(%q) = %q, want error", in, got) + } + } +} + +func TestCompileReplacementsSkipsInvalidRegex(t *testing.T) { + compiled := compileReplacements([]storage.Replacement{ + {Value: "[mehr]"}, + {Value: "([", IsRegex: true}, + {Value: "<.*?>", IsRegex: true}, + }) + if len(compiled) != 2 { + t.Fatalf("got %d replacements, want 2", len(compiled)) + } +} + +func TestProcessContent(t *testing.T) { + replacements := compileReplacements([]storage.Replacement{ + {Value: "[mehr]"}, + {Value: "<.*?>", IsRegex: true}, + }) + + got := processContent("

Hallo & Tschüss [mehr]

\n\n \n

Zeile 2

", replacements) + if want := "Hallo & Tschüss \nZeile 2"; got != want { + t.Errorf("processContent = %q, want %q", got, want) + } +} + +func TestProcessContentKeepsValidUTF8(t *testing.T) { + out := processContent(strings.Repeat("ü", 300)+"\xff", nil) + if !utf8.ValidString(out) { + t.Fatalf("output is not valid UTF-8") + } + if !strings.HasSuffix(out, "...") || utf8.RuneCountInString(out) != 273 { + t.Errorf("unexpected truncation: %d runes", utf8.RuneCountInString(out)) + } +} + +func TestFeedStatus(t *testing.T) { + if got := feedStatus(storage.Feed{}); got != "" { + t.Errorf("active feed status = %q, want empty", got) + } + disabled := storage.Feed{Disabled: true, DisabledReason: sql.NullString{String: "HTTP <410>", Valid: true}} + if got := feedStatus(disabled); !strings.Contains(got, "HTTP <410>") { + t.Errorf("disabled feed status = %q, want escaped reason", got) + } +} diff --git a/handler/feed_list.go b/handler/feed_list.go index e689c22..454ad1b 100644 --- a/handler/feed_list.go +++ b/handler/feed_list.go @@ -56,10 +56,10 @@ func (h *Handler) OnList(c telebot.Context) error { } var sb strings.Builder - sb.WriteString(fmt.Sprintf("%s hat abonniert:\n", html.EscapeString(chatTitle))) + fmt.Fprintf(&sb, "%s hat abonniert:\n", html.EscapeString(chatTitle)) for _, link := range links { - sb.WriteString(fmt.Sprintf("%d) %s\n", link.ID, html.EscapeString(link.Url))) + fmt.Fprintf(&sb, "%d) %s%s\n", link.ID, html.EscapeString(link.Url), feedStatus(link)) } return c.Send(sb.String(), defaultSendOptions) diff --git a/handler/feed_list_all.go b/handler/feed_list_all.go index 95f7fa2..63a5628 100644 --- a/handler/feed_list_all.go +++ b/handler/feed_list_all.go @@ -27,11 +27,10 @@ func (h *Handler) OnListAll(c telebot.Context) error { sb := strings.Builder{} for _, abonnement := range abonnements { - sb.WriteString(fmt.Sprintf("%d) %s\n", abonnement.Feed.ID, abonnement.Feed.Url)) + fmt.Fprintf(&sb, "%d) %s%s\n", abonnement.ID, html.EscapeString(abonnement.Url), feedStatus(abonnement.Feed)) for _, chat := range abonnement.Chats { - sb.WriteString(fmt.Sprintf(" %d (%s)\n", chat.ID, - html.EscapeString(chat.Title))) + fmt.Fprintf(&sb, " %d (%s)\n", chat.ID, html.EscapeString(chat.Title)) } sb.WriteString("\n") diff --git a/handler/feed_subscribe.go b/handler/feed_subscribe.go index 341c016..69a9cd4 100644 --- a/handler/feed_subscribe.go +++ b/handler/feed_subscribe.go @@ -1,9 +1,15 @@ package handler import ( + "context" + "fmt" + "html" "log" + "net/url" + "strings" + "time" - "github.com/mmcdole/gofeed" + "github.com/Brawl345/rssbot/storage" "gopkg.in/telebot.v3" ) @@ -14,7 +20,12 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { return nil } - feedUrl := args[0] + feedUrl, err := sanitizeFeedURL(args[0]) + if err != nil { + // FRB060/061: report the problem, don't guess at a corrected URL. + return c.Send("❌ Ungültige URL: "+err.Error(), defaultSendOptions) + } + chatId := c.Chat().ID var chatTitle string if c.Message().Private() { @@ -43,20 +54,58 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { chatTitle = chatInfo.Title } - feed, err := gofeed.NewParser().ParseURL(feedUrl) - + // FRB036: a single request gathers everything needed to add the feed. + ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second) + defer cancel() + result, err := h.Fetcher.Fetch(ctx, feedUrl, "", "") if err != nil { - log.Println(err) - return c.Send("❌ Ungültiger Feed", defaultSendOptions) + log.Printf("subscribe %s: %s", feedUrl, err) + if result != nil && result.Body != "" { + // FRB101: surface the server's response so the user can act on it. + return c.Send(fmt.Sprintf("❌ Ungültiger Feed (HTTP %d):\n
%s
", + result.Status, html.EscapeString(result.Body)), defaultSendOptions) + } + return c.Send("❌ Feed konnte nicht abgerufen werden: "+html.EscapeString(err.Error()), defaultSendOptions) + } + + // FRB100: only add feeds that actually answer with 200. + if result.Status != 200 { + msg := fmt.Sprintf("❌ Der Server antwortete mit HTTP %d.", result.Status) + if result.Body != "" { + msg += fmt.Sprintf("\n
%s
", html.EscapeString(result.Body)) + } + return c.Send(msg, defaultSendOptions) + } + + // FRB102: must be a real feed. + feed := result.Feed + if feed == nil { + return c.Send("❌ Diese URL liefert keinen gültigen Feed.", defaultSendOptions) } if feed.FeedLink != "" { - feedUrl = feed.FeedLink + if normalized, err := sanitizeFeedURL(feed.FeedLink); err == nil { + feedUrl = normalized + } } - exists, _ := h.DB.Abonnements.ExistsByFeedUrl(chatId, feedUrl) + nextPollAt := h.nextPoll(0, result) + + // The feed answered with a valid 200, so a previously retired feed works again. + reactivated, err := h.DB.Abonnements.ReactivateFeed(feedUrl, nextPollAt) + if err != nil { + log.Printf("subscribe %s: could not reactivate: %s", feedUrl, err) + } + exists, err := h.DB.Abonnements.ExistsByFeedUrl(chatId, feedUrl) + if err != nil { + log.Println(err) + return c.Send("❌ Beim Abonnieren des Feeds ist ein Fehler aufgetreten.", defaultSendOptions) + } if exists { + if reactivated { + return c.Send("✅ Der deaktivierte Feed wurde wieder aktiviert.", defaultSendOptions) + } return c.Send("✅ Du hast diesen Feed bereits abonniert.", defaultSendOptions) } @@ -69,7 +118,13 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { } } - err = h.DB.Abonnements.Create(chatId, chatTitle, feedUrl, lastEntry) + // Persist the cache validators from this fetch so the very first scheduled + // poll is already conditional. + etag := nullableString(result.ETag) + lastModified := nullableString(result.LastModified) + hints := storage.PollHints{Interval: result.FeedInterval, SkipHours: result.SkipHours, SkipDays: result.SkipDays} + + err = h.DB.Abonnements.Create(chatId, chatTitle, feedUrl, lastEntry, etag, lastModified, hints, nextPollAt) if err != nil { log.Println(err) return c.Send("❌ Beim Abonnieren des Feeds ist ein Fehler aufgetreten.", defaultSendOptions) @@ -77,3 +132,39 @@ func (h *Handler) OnSubscribe(c telebot.Context) error { return c.Send("✅ Der Feed wurde erfolgreich abonniert!", defaultSendOptions) } + +// sanitizeFeedURL trims surrounding whitespace and rejects URLs containing +// characters that should never appear in a real feed URL (FRB060/061). It does +// not rewrite or guess — it only validates. +func sanitizeFeedURL(raw string) (string, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "", fmt.Errorf("leer") + } + + for _, r := range trimmed { + if r <= ' ' || r == '"' || r == '\'' || r == '<' || r == '>' || r == '`' { + return "", fmt.Errorf("enthält ungültige Zeichen") + } + } + + parsed, err := url.Parse(trimmed) + if err != nil { + return "", fmt.Errorf("nicht parsebar") + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", fmt.Errorf("muss mit http:// oder https:// beginnen") + } + if parsed.Host == "" { + return "", fmt.Errorf("kein Host") + } + + return trimmed, nil +} + +func nullableString(s string) *string { + if s == "" { + return nil + } + return &s +} diff --git a/handler/feed_unsubscribe.go b/handler/feed_unsubscribe.go index 3b99c02..e89027d 100644 --- a/handler/feed_unsubscribe.go +++ b/handler/feed_unsubscribe.go @@ -39,7 +39,11 @@ func (h *Handler) OnUnsubscribe(c telebot.Context) error { chatId = chatInfo.ID } - exists, _ := h.DB.Abonnements.ExistsById(chatId, feedId) + exists, err := h.DB.Abonnements.ExistsById(chatId, feedId) + if err != nil { + log.Println(err) + return c.Send("❌ Beim Deabonnieren ist ein Fehler aufgetreten.", defaultSendOptions) + } if !exists { return c.Send("❌ Dieser Feed wurde nicht abonniert.", defaultSendOptions) diff --git a/handler/handler.go b/handler/handler.go index 0741a8a..1e74691 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -2,12 +2,15 @@ package handler import ( "github.com/Brawl345/rssbot/config" + "github.com/Brawl345/rssbot/fetcher" "github.com/Brawl345/rssbot/storage" "gopkg.in/telebot.v3" ) type Handler struct { - Bot *telebot.Bot - Config *config.Config - DB *storage.DB + Bot *telebot.Bot + Config *config.Config + DB *storage.DB + Fetcher *fetcher.Fetcher + AdminID int64 } diff --git a/handler/poll_test.go b/handler/poll_test.go new file mode 100644 index 0000000..74c9bac --- /dev/null +++ b/handler/poll_test.go @@ -0,0 +1,469 @@ +package handler + +import ( + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "text/template" + "time" + + "github.com/Brawl345/rssbot/config" + "github.com/Brawl345/rssbot/fetcher" + "github.com/Brawl345/rssbot/storage" + "gopkg.in/telebot.v3" +) + +const adminID = 999 + +type sentMessage struct { + ChatID int64 + Text string +} + +// fakeTelegram answers sendMessage calls like the Bot API and records them. +type fakeTelegram struct { + mu sync.Mutex + sent []sentMessage +} + +func (f *fakeTelegram) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var params struct { + ChatID string `json:"chat_id"` + Text string `json:"text"` + } + _ = json.NewDecoder(r.Body).Decode(¶ms) + var chatID int64 + _, _ = fmt.Sscan(params.ChatID, &chatID) + + f.mu.Lock() + f.sent = append(f.sent, sentMessage{ChatID: chatID, Text: params.Text}) + f.mu.Unlock() + + _, _ = fmt.Fprintf(w, `{"ok":true,"result":{"message_id":1,"date":0,"chat":{"id":%d,"type":"private"}}}`, chatID) +} + +func (f *fakeTelegram) messages() []sentMessage { + f.mu.Lock() + defer f.mu.Unlock() + return append([]sentMessage(nil), f.sent...) +} + +func (f *fakeTelegram) messagesTo(chatID int64) []string { + var texts []string + for _, m := range f.messages() { + if m.ChatID == chatID { + texts = append(texts, m.Text) + } + } + return texts +} + +type stateCall struct { + FeedID int64 + LastEntry *string + ETag *string + LastModified *string + Hints storage.PollHints + NextPollAt time.Time + ErrorCount int + UnchangedCount int +} + +type rescheduleCall struct { + FeedID int64 + NextPollAt time.Time + ErrorCount int + UnchangedCount int +} + +// fakeStore records the scheduling writes of the poller. Methods the poller +// does not use are left to the embedded nil interface and panic if called. +type fakeStore struct { + storage.AbonnementStorage + + mu sync.Mutex + states []stateCall + reschedules []rescheduleCall + disabled map[int64]string + moved map[int64]string + mergeOnMove bool + + created []createCall + deleted [][2]int64 + subscriptions map[int64][]storage.Feed + disabledURLs map[string]bool +} + +func newFakeStore() *fakeStore { + return &fakeStore{ + disabled: map[int64]string{}, + moved: map[int64]string{}, + subscriptions: map[int64][]storage.Feed{}, + disabledURLs: map[string]bool{}, + } +} + +func (s *fakeStore) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, hints storage.PollHints, nextPollAt time.Time, errorCount, unchangedCount int) error { + s.mu.Lock() + defer s.mu.Unlock() + s.states = append(s.states, stateCall{feedID, lastEntry, etag, lastModified, hints, nextPollAt, errorCount, unchangedCount}) + return nil +} + +func (s *fakeStore) Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error { + s.mu.Lock() + defer s.mu.Unlock() + s.reschedules = append(s.reschedules, rescheduleCall{feedID, nextPollAt, errorCount, unchangedCount}) + return nil +} + +func (s *fakeStore) DisableFeed(feedID int64, reason string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.disabled[feedID] = reason + return nil +} + +func (s *fakeStore) MoveFeedURL(feedID int64, newURL string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.moved[feedID] = newURL + return s.mergeOnMove, nil +} + +type pollEnv struct { + h *Handler + store *fakeStore + tg *fakeTelegram +} + +func newPollEnv(t *testing.T) *pollEnv { + t.Helper() + + tg := &fakeTelegram{} + tgServer := httptest.NewServer(tg) + t.Cleanup(tgServer.Close) + + bot, err := telebot.NewBot(telebot.Settings{Token: "test", URL: tgServer.URL, Offline: true}) + if err != nil { + t.Fatal(err) + } + + tmpl, err := config.GetTemplate("does-not-exist.gohtml") + if err != nil { + t.Fatal(err) + } + + store := newFakeStore() + return &pollEnv{ + h: &Handler{ + Bot: bot, + Config: &config.Config{Template: tmpl, Poll: testPollConfig()}, + DB: &storage.DB{Abonnements: store}, + Fetcher: fetcher.New(), + AdminID: adminID, + }, + store: store, + tg: tg, + } +} + +func feedServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return srv +} + +func abonnement(url string, chats ...int64) storage.Abonnement { + ab := storage.Abonnement{Feed: storage.Feed{ID: 1, Url: url}} + for _, id := range chats { + ab.Chats = append(ab.Chats, storage.Chat{ID: id}) + } + return ab +} + +func rss(ttl string, guids ...string) string { + var sb strings.Builder + sb.WriteString(`Test & Feedhttps://example.org`) + sb.WriteString(ttl) + for _, guid := range guids { + fmt.Fprintf(&sb, `Item %shttps://www.example.org/%s%sText %s`, guid, guid, guid, guid) + } + sb.WriteString(``) + return sb.String() +} + +func TestPollSendsNewItemsOldestFirst(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Etag", `"v2"`) + _, _ = w.Write([]byte(rss("120", "3", "2", "1"))) + }) + + ab := abonnement(srv.URL, 10, 11) + ab.LastEntry = sql.NullString{String: "1", Valid: true} + env.h.pollFeed(ab, nil) + + for _, chat := range []int64{10, 11} { + msgs := env.tg.messagesTo(chat) + if len(msgs) != 2 { + t.Fatalf("chat %d got %d messages, want 2", chat, len(msgs)) + } + if !strings.Contains(msgs[0], "Item 2") || !strings.Contains(msgs[1], "Item 3") { + t.Errorf("chat %d messages not oldest first: %q", chat, msgs) + } + if !strings.Contains(msgs[0], "Test & Feed") || !strings.Contains(msgs[0], "Weiterlesen auf example.org") { + t.Errorf("unexpected message rendering: %q", msgs[0]) + } + } + if msgs := env.tg.messagesTo(adminID); len(msgs) != 0 { + t.Errorf("admin got unexpected messages: %q", msgs) + } + + if len(env.store.states) != 1 { + t.Fatalf("SetFeedState called %d times, want 1", len(env.store.states)) + } + state := env.store.states[0] + if state.LastEntry == nil || *state.LastEntry != "3" { + t.Errorf("last entry = %v, want 3", state.LastEntry) + } + if state.ETag == nil || *state.ETag != `"v2"` { + t.Errorf("etag = %v, want \"v2\"", state.ETag) + } + if state.UnchangedCount != 0 || state.ErrorCount != 0 { + t.Errorf("counters = %d/%d, want 0/0", state.ErrorCount, state.UnchangedCount) + } + if state.Hints.Interval != 2*time.Hour { + t.Errorf("stored ttl hint = %s, want 2h", state.Hints.Interval) + } + assertDelay(t, state.NextPollAt, 2*time.Hour) +} + +func TestPollUnchangedFeedIncreasesInterval(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(rss("", "1"))) + }) + + ab := abonnement(srv.URL, 10) + ab.LastEntry = sql.NullString{String: "1", Valid: true} + ab.UnchangedCount = 2 + env.h.pollFeed(ab, nil) + + if msgs := env.tg.messages(); len(msgs) != 0 { + t.Errorf("unexpected messages: %v", msgs) + } + state := env.store.states[0] + if state.UnchangedCount != 3 { + t.Errorf("unchanged = %d, want 3", state.UnchangedCount) + } + assertDelay(t, state.NextPollAt, 40*time.Minute) +} + +func TestPollNotModifiedUsesStoredHints(t *testing.T) { + env := newPollEnv(t) + var gotINM string + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + gotINM = r.Header.Get("If-None-Match") + w.WriteHeader(http.StatusNotModified) + }) + + ab := abonnement(srv.URL, 10) + ab.ETag = sql.NullString{String: `"v1"`, Valid: true} + ab.FeedInterval = int((3 * time.Hour).Seconds()) + env.h.pollFeed(ab, nil) + + if gotINM != `"v1"` { + t.Errorf("If-None-Match = %q, want stored etag", gotINM) + } + if len(env.store.states) != 0 { + t.Errorf("304 must not overwrite the cached state") + } + if len(env.store.reschedules) != 1 { + t.Fatalf("Reschedule called %d times, want 1", len(env.store.reschedules)) + } + r := env.store.reschedules[0] + if r.ErrorCount != 0 || r.UnchangedCount != 1 { + t.Errorf("counters = %d/%d, want 0/1", r.ErrorCount, r.UnchangedCount) + } + assertDelay(t, r.NextPollAt, 3*time.Hour) +} + +func TestPollGoneDisablesAndNotifiesAdmin(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusGone) + _, _ = w.Write([]byte("gone")) + }) + + env.h.pollFeed(abonnement(srv.URL+"/?a=1&b=2", 10), nil) + + if reason := env.store.disabled[1]; reason != "HTTP 410 Gone" { + t.Errorf("disabled reason = %q", reason) + } + if msgs := env.tg.messagesTo(10); len(msgs) != 0 { + t.Errorf("subscriber must not get operational messages: %q", msgs) + } + admin := env.tg.messagesTo(adminID) + if len(admin) != 1 { + t.Fatalf("admin got %d messages, want 1", len(admin)) + } + if !strings.Contains(admin[0], "a=1&b=2") || !strings.Contains(admin[0], "<b>gone</b>") { + t.Errorf("admin message not escaped: %q", admin[0]) + } +} + +func TestPollServerErrorBacksOff(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + + ab := abonnement(srv.URL, 10) + ab.ErrorCount = 2 + env.h.pollFeed(ab, nil) + + if len(env.store.disabled) != 0 { + t.Errorf("feed must not be disabled yet") + } + r := env.store.reschedules[0] + if r.ErrorCount != 3 { + t.Errorf("error count = %d, want 3", r.ErrorCount) + } + assertDelay(t, r.NextPollAt, 40*time.Minute) +} + +func TestPollRetiresFeedAfterLongFailure(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + recent := abonnement(srv.URL, 10) + recent.ErrorCount = 50 + recent.FailingSince = sql.NullTime{Time: time.Now().Add(-24 * time.Hour), Valid: true} + env.h.pollFeed(recent, nil) + if len(env.store.disabled) != 0 { + t.Fatalf("feed failing for one day must not be retired") + } + + old := abonnement(srv.URL, 10) + old.ErrorCount = maxFeedErrors - 1 + old.FailingSince = sql.NullTime{Time: time.Now().Add(-8 * 24 * time.Hour), Valid: true} + env.h.pollFeed(old, nil) + if reason := env.store.disabled[1]; reason != "HTTP 404" { + t.Errorf("disabled reason = %q, want HTTP 404", reason) + } +} + +func TestPollRateLimitKeepsErrorCount(t *testing.T) { + env := newPollEnv(t) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Retry-After", "3600") + w.WriteHeader(http.StatusTooManyRequests) + }) + + ab := abonnement(srv.URL, 10) + ab.ErrorCount = 1 + env.h.pollFeed(ab, nil) + + r := env.store.reschedules[0] + if r.ErrorCount != 1 { + t.Errorf("error count = %d, want unchanged 1", r.ErrorCount) + } + assertDelay(t, r.NextPollAt, time.Hour) + if len(env.tg.messagesTo(adminID)) != 1 { + t.Errorf("admin should be notified about 429") + } +} + +func TestPollPermanentRedirect(t *testing.T) { + target := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(rss("", "1"))) + }) + src := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusMovedPermanently) + }) + + t.Run("renamed", func(t *testing.T) { + env := newPollEnv(t) + env.h.pollFeed(abonnement(src.URL, 10), nil) + + if env.store.moved[1] != target.URL { + t.Errorf("moved to %q, want %q", env.store.moved[1], target.URL) + } + if len(env.store.states) != 1 { + t.Errorf("state of the renamed feed must be saved") + } + if len(env.tg.messagesTo(adminID)) != 1 { + t.Errorf("admin should be notified about the move") + } + }) + + t.Run("merged", func(t *testing.T) { + env := newPollEnv(t) + env.store.mergeOnMove = true + env.h.pollFeed(abonnement(src.URL, 10), nil) + + if len(env.store.states) != 0 || len(env.tg.messagesTo(10)) != 0 { + t.Errorf("merged feed must be left to the surviving feed") + } + }) +} + +func TestPollTemplateErrorStillSavesState(t *testing.T) { + env := newPollEnv(t) + env.h.Config.Template = template.Must(template.New("post").Parse("{{.Missing}}")) + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(rss("", "2", "1"))) + }) + + env.h.pollFeed(abonnement(srv.URL, 10), nil) + + if len(env.tg.messages()) != 0 { + t.Errorf("no message can be rendered") + } + if len(env.store.states) != 1 { + t.Fatalf("state must be saved even if rendering fails") + } +} + +func TestPollFeedsSerializesSameHost(t *testing.T) { + env := newPollEnv(t) + var inFlight, maxInFlight, requests atomic.Int32 + srv := feedServer(t, func(w http.ResponseWriter, r *http.Request) { + n := inFlight.Add(1) + defer inFlight.Add(-1) + for { + m := maxInFlight.Load() + if n <= m || maxInFlight.CompareAndSwap(m, n) { + break + } + } + requests.Add(1) + time.Sleep(20 * time.Millisecond) + w.WriteHeader(http.StatusNotModified) + }) + + var abonnements []storage.Abonnement + for i := 1; i <= 5; i++ { + ab := abonnement(fmt.Sprintf("%s/feed%d", srv.URL, i), 10) + ab.ID = int64(i) + abonnements = append(abonnements, ab) + } + env.h.pollFeeds(abonnements, nil) + + if requests.Load() != 5 { + t.Errorf("got %d requests, want 5", requests.Load()) + } + if maxInFlight.Load() != 1 { + t.Errorf("max concurrent requests to one host = %d, want 1", maxInFlight.Load()) + } +} diff --git a/handler/replacement_list.go b/handler/replacement_list.go index fb29702..3da0b1c 100644 --- a/handler/replacement_list.go +++ b/handler/replacement_list.go @@ -29,7 +29,7 @@ func (h *Handler) OnListReplacements(c telebot.Context) error { sb := strings.Builder{} for _, replacement := range replacements { - sb.WriteString(fmt.Sprintf("%d) %s", replacement.ID, html.EscapeString(replacement.Value))) + fmt.Fprintf(&sb, "%d) %s", replacement.ID, html.EscapeString(replacement.Value)) if replacement.IsRegex { sb.WriteString(" (RegEx)") } diff --git a/handler/start.go b/handler/start.go index 2b5938a..6cab24e 100644 --- a/handler/start.go +++ b/handler/start.go @@ -16,7 +16,8 @@ func (h *Handler) OnStart(c telebot.Context) error { sb.WriteString("/repl_add String: Ersetzung hinzufügen\n") sb.WriteString("/repl_add_re RegEx: RegEx-Ersetzung hinzufügen\n") sb.WriteString("/repl_del Ersetzungs-ID: Ersetzung löschen\n\n") - sb.WriteString("[Chat] ist ein optionales Argument mit dem @Kanalnamen.") + sb.WriteString("[Chat] ist ein optionales Argument mit dem @Kanalnamen.\n") + sb.WriteString("Deaktivierte Feeds werden durch erneutes /sub wieder aktiviert.") return c.Send(sb.String(), defaultSendOptions) } diff --git a/handler/utils.go b/handler/utils.go index 093d844..51d7210 100644 --- a/handler/utils.go +++ b/handler/utils.go @@ -1,6 +1,10 @@ package handler import ( + "fmt" + "html" + + "github.com/Brawl345/rssbot/storage" "github.com/mmcdole/gofeed" "gopkg.in/telebot.v3" ) @@ -22,3 +26,10 @@ func reverse(s []*gofeed.Item) []*gofeed.Item { return a } + +func feedStatus(feed storage.Feed) string { + if !feed.Disabled { + return "" + } + return fmt.Sprintf(" 🚫 deaktiviert: %s", html.EscapeString(feed.DisabledReason.String)) +} diff --git a/main.go b/main.go index 3611ee2..d3a653e 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "time" "github.com/Brawl345/rssbot/config" + "github.com/Brawl345/rssbot/fetcher" "github.com/Brawl345/rssbot/handler" _ "github.com/joho/godotenv/autoload" "gopkg.in/telebot.v3" @@ -19,13 +20,14 @@ import ( ) func main() { - tmpl, err := config.GetTemplate("post.gohtml") + tmpl, err := config.LoadTemplate() if err != nil { log.Fatal("Invalid template: ", err) } cfg := &config.Config{ Template: tmpl, + Poll: config.GetPollConfig(), } db, err := storage.Connect() @@ -55,13 +57,17 @@ func main() { log.Printf("Logged in as @%s (%d)", bot.Me.Username, bot.Me.ID) + adminId, err := strconv.ParseInt(os.Getenv("ADMIN_ID"), 10, 64) + h := handler.Handler{ - Bot: bot, - Config: cfg, - DB: db, + Bot: bot, + Config: cfg, + DB: db, + Fetcher: fetcher.New(), + AdminID: adminId, } - adminId, err := strconv.ParseInt(os.Getenv("ADMIN_ID"), 10, 64) + log.Printf("Feed fetcher User-Agent: %s", h.Fetcher.UserAgent()) if err != nil { // No admin = unsupported. @@ -93,10 +99,8 @@ func main() { time.AfterFunc(5*time.Second, h.OnCheck) - channel := make(chan os.Signal) + channel := make(chan os.Signal, 1) signal.Notify(channel, os.Interrupt, syscall.SIGTERM) - signal.Notify(channel, os.Interrupt, syscall.SIGKILL) - signal.Notify(channel, os.Interrupt, syscall.SIGINT) go func() { <-channel log.Println("Stopping...") diff --git a/module.nix b/module.nix index df80edb..bd70dde 100644 --- a/module.nix +++ b/module.nix @@ -17,6 +17,8 @@ let optional optionalAttrs optionalString + mapNullable + boolToString ; in { @@ -31,6 +33,12 @@ in description = "User under which RSS Bot runs."; }; + group = mkOption { + type = types.str; + default = defaultUser; + description = "Group under which RSS Bot runs."; + }; + adminId = mkOption { type = types.int; description = "Admin ID"; @@ -41,12 +49,70 @@ in description = "File containing Telegram Bot Token"; }; - # TODO: Find a way to load a custom post.gohtml - # template = mkOption { - # type = types.nullOr types.lines; - # default = null; - # description = "Custom post.gohtml template content"; - # }; + template = mkOption { + type = types.nullOr types.lines; + default = null; + example = '' + [#RSS] {{.Title}} + {{.FeedTitle}} + {{- if ne .Content "" }} + {{.Content}} + {{- end }} + {{.PostDomain}} + ''; + description = "Custom post template (Go template). Uses the built-in template if null."; + }; + + templateFile = mkOption { + type = types.nullOr types.path; + default = null; + example = "./post.gohtml"; + description = "Path to a custom post template file. Alternative to `template`."; + }; + + poll = { + interval = mkOption { + type = types.nullOr types.str; + default = null; + example = "5m"; + description = "How often each feed is checked. Uses the bot's default (10m) if null."; + }; + + intervalMax = mkOption { + type = types.nullOr types.str; + default = null; + example = "2h"; + description = '' + The longest a feed ever waits between two checks. Limits the adaptive slow-down, + the waiting time after errors and intervals requested by servers. + Uses the bot's default (6h) if null. + ''; + }; + + adaptive = mkOption { + type = types.nullOr types.bool; + default = null; + example = false; + description = '' + Check feeds that rarely get new entries less often (up to `intervalMax`). + If false, every feed is checked every `interval`. Uses the bot's default (true) if null. + ''; + }; + + concurrency = mkOption { + type = types.nullOr types.ints.positive; + default = null; + example = 4; + description = "How many feeds are downloaded at the same time. Uses the bot's default (8) if null."; + }; + + tick = mkOption { + type = types.nullOr types.str; + default = null; + example = "1m"; + description = "How often the bot looks for feeds that are due. Uses the bot's default (30s) if null."; + }; + }; database = { host = lib.mkOption { @@ -107,6 +173,10 @@ in assertion = cfg.database.socket != null || cfg.database.passwordFile != null; message = "Either services.rssbot.database.socket or services.rssbot.database.passwordFile must be set."; } + { + assertion = !(cfg.template != null && cfg.templateFile != null); + message = "Only one of services.rssbot.template or services.rssbot.templateFile can be set."; + } ]; services.mysql = lib.mkIf cfg.database.createLocally { @@ -125,8 +195,9 @@ in systemd.services.rssbot = { description = "RSS Bot for Telegram"; - after = [ "network-online.target" "mysql.service" ]; - requires = [ "network-online.target" "mysql.service" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ] ++ optional cfg.database.createLocally "mysql.service"; + requires = optional cfg.database.createLocally "mysql.service"; wantedBy = [ "multi-user.target" ]; script = '' @@ -141,11 +212,46 @@ in serviceConfig = { LoadCredential = [ "BOT_TOKEN:${cfg.botTokenFile}" - ] ++ optional (cfg.database.passwordFile != null) "MYSQL_PASSWORD:${cfg.database.passwordFile}"; + ] + ++ optional (cfg.database.passwordFile != null) "MYSQL_PASSWORD:${cfg.database.passwordFile}"; Restart = "always"; User = cfg.user; - Group = defaultUser; + Group = cfg.group; + + # Hardening + CapabilityBoundingSet = ""; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateTmp = true; + ProcSubset = "pid"; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + "AF_UNIX" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged" + "~@resources" + ]; + UMask = "0077"; }; environment = { @@ -155,17 +261,31 @@ in MYSQL_USER = cfg.database.user; MYSQL_DB = cfg.database.name; MYSQL_SOCKET = cfg.database.socket; + + POST_TEMPLATE = + if cfg.template != null then + pkgs.writeText "post.gohtml" cfg.template + else + mapNullable (file: "${file}") cfg.templateFile; + + POLL_INTERVAL = cfg.poll.interval; + POLL_INTERVAL_MAX = cfg.poll.intervalMax; + POLL_ADAPTIVE = mapNullable boolToString cfg.poll.adaptive; + POLL_CONCURRENCY = mapNullable toString cfg.poll.concurrency; + POLL_TICK = cfg.poll.tick; }; }; - users = optionalAttrs (cfg.user == defaultUser) { - users.${defaultUser} = { + users.users = optionalAttrs (cfg.user == defaultUser) { + ${defaultUser} = { isSystemUser = true; - group = defaultUser; + inherit (cfg) group; description = "RSS Bot user"; }; + }; - groups.${defaultUser} = { }; + users.groups = optionalAttrs (cfg.group == defaultUser) { + ${defaultUser} = { }; }; }; diff --git a/nixos-test.nix b/nixos-test.nix new file mode 100644 index 0000000..b65d5fb --- /dev/null +++ b/nixos-test.nix @@ -0,0 +1,31 @@ +# Starts the bot with the NixOS module and a local MariaDB. Without network the +# bot cannot reach Telegram, but it connects to the database and applies the +# migrations first, which covers the module, the hardening and socket auth. +self: +{ pkgs, ... }: +{ + name = "rssbot"; + + nodes.machine = { + imports = [ self.nixosModules.default ]; + + services.rssbot = { + enable = true; + adminId = 1337; + botTokenFile = pkgs.writeText "rssbot-token" "123456789:test"; + template = "{{.Title}}"; + poll = { + interval = "5m"; + adaptive = false; + }; + }; + }; + + testScript = '' + machine.wait_for_unit("mysql.service") + machine.wait_until_succeeds("journalctl -u rssbot.service | grep -q 'Applied 6 migration'") + machine.succeed("mysql -N rssbot -e 'SELECT COUNT(*) FROM replacements' | grep -qx 41") + machine.succeed("systemctl show rssbot.service -p Environment | grep -q POLL_INTERVAL=5m") + print(machine.succeed("systemd-analyze security rssbot.service | tail -1")) + ''; +} diff --git a/storage/abonnements.go b/storage/abonnements.go index 2b4d093..6cf0b76 100644 --- a/storage/abonnements.go +++ b/storage/abonnements.go @@ -3,20 +3,28 @@ package storage import ( "context" "database/sql" - "github.com/jmoiron/sqlx" - "github.com/mmcdole/gofeed" + "errors" + "strconv" + "strings" "time" + + "github.com/jmoiron/sqlx" ) type ( AbonnementStorage interface { - Create(chatId int64, chatTitle string, feedUrl string, lastEntry *string) error + Create(chatId int64, chatTitle string, feedUrl string, lastEntry, etag, lastModified *string, hints PollHints, nextPollAt time.Time) error Delete(chatId int64, feedId int64) error ExistsByFeedUrl(chatId int64, feedUrl string) (bool, error) ExistsById(chatId int64, feedId int64) (bool, error) GetByUser(chatId int64) ([]Feed, error) GetAll() ([]Abonnement, error) - SetLastEntry(feedUrl string, lastEntry *string) error + GetDue() ([]Abonnement, error) + SetFeedState(feedID int64, lastEntry, etag, lastModified *string, hints PollHints, nextPollAt time.Time, errorCount, unchangedCount int) error + Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error + MoveFeedURL(feedID int64, newURL string) (bool, error) + DisableFeed(feedID int64, reason string) error + ReactivateFeed(feedUrl string, nextPollAt time.Time) (bool, error) } Abonnements struct { @@ -35,21 +43,74 @@ type ( } Feed struct { - ID int64 `db:"id"` - Url string `db:"url"` - LastEntry sql.NullString `db:"last_entry"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt sql.NullTime `db:"updated_at"` + ID int64 `db:"id"` + Url string `db:"url"` + LastEntry sql.NullString `db:"last_entry"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt sql.NullTime `db:"updated_at"` + ETag sql.NullString `db:"etag"` + LastModified sql.NullString `db:"last_modified"` + NextPollAt sql.NullTime `db:"next_poll_at"` + LastPollAt sql.NullTime `db:"last_poll_at"` + ErrorCount int `db:"error_count"` + UnchangedCount int `db:"unchanged_count"` + Disabled bool `db:"disabled"` + DisabledReason sql.NullString `db:"disabled_reason"` + FeedInterval int `db:"feed_interval"` + SkipHours sql.NullString `db:"skip_hours"` + SkipDays sql.NullString `db:"skip_days"` + FailingSince sql.NullTime `db:"failing_since"` + } + + // PollHints are the polling hints a feed declares in its body (ttl, + // skipHours, skipDays). They are persisted so they still apply after a 304. + PollHints struct { + Interval time.Duration + SkipHours []int + SkipDays []string } ) -func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, lastEntry *string) error { +// Hints decodes the persisted polling hints of a feed. +func (f Feed) Hints() PollHints { + hints := PollHints{Interval: time.Duration(f.FeedInterval) * time.Second} + if f.SkipHours.Valid { + for _, v := range strings.Split(f.SkipHours.String, ",") { + if n, err := strconv.Atoi(v); err == nil { + hints.SkipHours = append(hints.SkipHours, n) + } + } + } + if f.SkipDays.Valid && f.SkipDays.String != "" { + hints.SkipDays = strings.Split(f.SkipDays.String, ",") + } + return hints +} + +func (p PollHints) encode() (int, *string, *string) { + var skipHours, skipDays *string + if len(p.SkipHours) > 0 { + parts := make([]string, len(p.SkipHours)) + for i, h := range p.SkipHours { + parts[i] = strconv.Itoa(h) + } + joined := strings.Join(parts, ",") + skipHours = &joined + } + if len(p.SkipDays) > 0 { + joined := strings.Join(p.SkipDays, ",") + skipDays = &joined + } + return int(p.Interval / time.Second), skipHours, skipDays +} + +func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, lastEntry, etag, lastModified *string, hints PollHints, nextPollAt time.Time) error { tx, err := db.BeginTxx(context.Background(), nil) if err != nil { return err } - defer tx.Rollback() + defer func() { _ = tx.Rollback() }() const feedQuery = "SELECT id FROM feeds WHERE url = ?" var feedId int64 @@ -57,8 +118,10 @@ func (db *Abonnements) Create(chatId int64, chatTitle string, feedUrl string, la if err != nil { // Feed does not exist yet, will be created - const insertFeedQuery = "INSERT INTO feeds (url, last_entry) VALUES (?, ?)" - result, err := tx.Exec(insertFeedQuery, feedUrl, lastEntry) + const insertFeedQuery = `INSERT INTO feeds (url, last_entry, etag, last_modified, feed_interval, skip_hours, skip_days, next_poll_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + interval, skipHours, skipDays := hints.encode() + result, err := tx.Exec(insertFeedQuery, feedUrl, lastEntry, etag, lastModified, interval, skipHours, skipDays, nextPollAt) if err != nil { return err } @@ -93,7 +156,7 @@ func (db *Abonnements) Delete(chatId int64, feedId int64) error { return err } - defer tx.Rollback() + defer func() { _ = tx.Rollback() }() const deleteAbonnementQuery = "DELETE FROM abonnements WHERE abonnements.chat_id = ? AND abonnements.feed_id = ?" _, err = tx.Exec(deleteAbonnementQuery, chatId, feedId) @@ -102,9 +165,11 @@ func (db *Abonnements) Delete(chatId int64, feedId int64) error { } // Check if user has other abonnements - const hasOtherAbonnementsQuery = "SELECT 1 FROM abonnements WHERE abonnements.chat_id = ?" + const hasOtherAbonnementsQuery = "SELECT EXISTS(SELECT 1 FROM abonnements WHERE abonnements.chat_id = ?)" var hasOtherAbonnements bool - tx.Get(&hasOtherAbonnements, hasOtherAbonnementsQuery, chatId) + if err = tx.Get(&hasOtherAbonnements, hasOtherAbonnementsQuery, chatId); err != nil { + return err + } if !hasOtherAbonnements { const deleteChatQuery = "DELETE FROM chats WHERE chats.id = ?" @@ -115,9 +180,11 @@ func (db *Abonnements) Delete(chatId int64, feedId int64) error { } // Check if feed has abonnement from other users - const hasOtherUsersQuery = "SELECT 1 FROM abonnements WHERE abonnements.feed_id = ?" + const hasOtherUsersQuery = "SELECT EXISTS(SELECT 1 FROM abonnements WHERE abonnements.feed_id = ?)" var hasOtherUsers bool - tx.Get(&hasOtherUsers, hasOtherUsersQuery, feedId) + if err = tx.Get(&hasOtherUsers, hasOtherUsersQuery, feedId); err != nil { + return err + } if !hasOtherUsers { const deleteFeedQuery = "DELETE FROM feeds WHERE feeds.id = ?" @@ -135,11 +202,10 @@ func (db *Abonnements) Delete(chatId int64, feedId int64) error { } func (db *Abonnements) ExistsByFeedUrl(chatId int64, feedUrl string) (bool, error) { - const query = `SELECT 1 FROM abonnements -JOIN chats ON abonnements.chat_id = chats.id + const query = `SELECT EXISTS(SELECT 1 FROM abonnements JOIN feeds ON abonnements.feed_id = feeds.id -WHERE chats.id = ? -AND feeds.url = ?` +WHERE abonnements.chat_id = ? +AND feeds.url = ?)` var exists bool err := db.Get(&exists, query, chatId, feedUrl) @@ -147,9 +213,9 @@ AND feeds.url = ?` } func (db *Abonnements) ExistsById(chatId int64, feedId int64) (bool, error) { - const query = `SELECT 1 FROM abonnements + const query = `SELECT EXISTS(SELECT 1 FROM abonnements WHERE abonnements.chat_id = ? -AND abonnements.feed_id = ?` +AND abonnements.feed_id = ?)` var exists bool err := db.Get(&exists, query, chatId, feedId) @@ -167,67 +233,163 @@ WHERE chats.id = ?` return feeds, err } -func (db *Abonnements) GetAll() ([]Abonnement, error) { - const query = `SELECT chats.id AS "chat_id", chats.created_at AS "chat_created_at", chats.title, feeds.* +// abonnementSelect lists feed columns explicitly so the manual row scan does +// not depend on the physical column order of `feeds.*`. +const abonnementSelect = `SELECT chats.id, chats.created_at, chats.title, +feeds.id, feeds.url, feeds.last_entry, feeds.created_at, feeds.updated_at, +feeds.etag, feeds.last_modified, feeds.next_poll_at, feeds.last_poll_at, +feeds.error_count, feeds.unchanged_count, feeds.disabled, feeds.disabled_reason, +feeds.feed_interval, feeds.skip_hours, feeds.skip_days, feeds.failing_since FROM abonnements JOIN chats ON abonnements.chat_id = chats.id JOIN feeds ON abonnements.feed_id = feeds.id` - rows, _ := db.Queryx(query) - defer rows.Close() +func (db *Abonnements) GetAll() ([]Abonnement, error) { + rows, err := db.Queryx(abonnementSelect) + if err != nil { + return nil, err + } + return scanAbonnements(rows) +} - var abonnements []Abonnement - var feeds = make(map[int64]Feed) - var feedChats = make(map[int64][]Chat) +// GetDue returns only feeds that are enabled and whose scheduled poll time has +// passed (or was never set). This is what keeps polling on a per-feed schedule +// and prevents a process restart from re-downloading everything (FRB037). +// Timestamps are always passed from Go instead of using NOW(), so they are +// written and compared in the driver's loc regardless of the MySQL time zone. +func (db *Abonnements) GetDue() ([]Abonnement, error) { + const where = ` WHERE feeds.disabled = 0 AND (feeds.next_poll_at IS NULL OR feeds.next_poll_at <= ?)` + rows, err := db.Queryx(abonnementSelect+where, time.Now()) + if err != nil { + return nil, err + } + return scanAbonnements(rows) +} + +func scanAbonnements(rows *sqlx.Rows) ([]Abonnement, error) { + defer func() { _ = rows.Close() }() + + feeds := make(map[int64]Feed) + feedChats := make(map[int64][]Chat) + var order []int64 for rows.Next() { var chat Chat var feed Feed - rows.Scan(&chat.ID, &chat.CreatedAt, &chat.Title, - &feed.ID, &feed.Url, &feed.LastEntry, &feed.CreatedAt, &feed.UpdatedAt) - - feeds[feed.ID] = feed + err := rows.Scan(&chat.ID, &chat.CreatedAt, &chat.Title, + &feed.ID, &feed.Url, &feed.LastEntry, &feed.CreatedAt, &feed.UpdatedAt, + &feed.ETag, &feed.LastModified, &feed.NextPollAt, &feed.LastPollAt, + &feed.ErrorCount, &feed.UnchangedCount, &feed.Disabled, &feed.DisabledReason, + &feed.FeedInterval, &feed.SkipHours, &feed.SkipDays, &feed.FailingSince) + if err != nil { + return nil, err + } - if chats, ok := feedChats[feed.ID]; ok { - feedChats[feed.ID] = append(chats, chat) - } else { - feedChats[feed.ID] = []Chat{chat} + if _, seen := feeds[feed.ID]; !seen { + order = append(order, feed.ID) } + feeds[feed.ID] = feed + feedChats[feed.ID] = append(feedChats[feed.ID], chat) } - for feedId, feed := range feeds { + var abonnements []Abonnement + for _, feedId := range order { abonnements = append(abonnements, Abonnement{ - Feed: feed, + Feed: feeds[feedId], Chats: feedChats[feedId], }) } - return abonnements, nil + return abonnements, rows.Err() } -func (db *Abonnements) SetLastEntry(feedUrl string, lastEntry *string) error { +// SetFeedState writes the atomic cache set (etag + last_modified), the last seen +// entry, the feed's polling hints and the next poll schedule after a successful +// 200 response. +func (db *Abonnements) SetFeedState(feedID int64, lastEntry, etag, lastModified *string, hints PollHints, nextPollAt time.Time, errorCount, unchangedCount int) error { const query = `UPDATE feeds -SET feeds.last_entry = ? -WHERE feeds.url = ?` +SET last_entry = ?, etag = ?, last_modified = ?, feed_interval = ?, skip_hours = ?, skip_days = ?, + next_poll_at = ?, last_poll_at = ?, error_count = ?, unchanged_count = ?, failing_since = NULL +WHERE id = ?` + interval, skipHours, skipDays := hints.encode() + _, err := db.Exec(query, lastEntry, etag, lastModified, interval, skipHours, skipDays, + nextPollAt, time.Now(), errorCount, unchangedCount, feedID) + return err +} - _, err := db.Exec(query, lastEntry, feedUrl) +// Reschedule updates only the poll schedule and counters, preserving the cached +// etag/last_modified (FRB010-016) — used for 304, rate-limiting and transient +// errors. failing_since marks the start of an error streak and is cleared once +// errorCount drops back to 0. +func (db *Abonnements) Reschedule(feedID int64, nextPollAt time.Time, errorCount, unchangedCount int) error { + const query = `UPDATE feeds +SET next_poll_at = ?, last_poll_at = ?, error_count = ?, unchanged_count = ?, + failing_since = CASE WHEN ? > 0 THEN COALESCE(failing_since, ?) ELSE NULL END +WHERE id = ?` + now := time.Now() + _, err := db.Exec(query, nextPollAt, now, errorCount, unchangedCount, errorCount, now, feedID) return err } -func (feedToCheck Feed) Check(lastEntry *string) (*gofeed.Feed, error) { - feed, err := gofeed.NewParser().ParseURL(feedToCheck.Url) +// MoveFeedURL persists a permanent redirect target (FRB130/131). If another feed +// already occupies newURL (feeds.url is UNIQUE), this feed's subscriptions are +// merged onto that existing feed instead and the old feed row is removed; the +// returned bool reports whether such a merge happened. +func (db *Abonnements) MoveFeedURL(feedID int64, newURL string) (bool, error) { + tx, err := db.BeginTxx(context.Background(), nil) if err != nil { - return nil, err + return false, err } - - if lastEntry != nil { - for i, item := range feed.Items { - if item.GUID == *lastEntry { - feed.Items = feed.Items[:i] - return feed, nil - } + defer func() { _ = tx.Rollback() }() + + var targetID int64 + err = tx.Get(&targetID, "SELECT id FROM feeds WHERE url = ?", newURL) + switch { + case errors.Is(err, sql.ErrNoRows): + if _, err := tx.Exec("UPDATE feeds SET url = ? WHERE id = ?", newURL, feedID); err != nil { + return false, err } + return false, tx.Commit() + case err != nil: + return false, err + case targetID == feedID: + return false, tx.Commit() + } + + // Repoint subscriptions onto the existing feed, dropping duplicates for + // chats already subscribed there, then delete the now-orphaned feed. + if _, err := tx.Exec("UPDATE IGNORE abonnements SET feed_id = ? WHERE feed_id = ?", targetID, feedID); err != nil { + return false, err + } + if _, err := tx.Exec("DELETE FROM abonnements WHERE feed_id = ?", feedID); err != nil { + return false, err + } + if _, err := tx.Exec("DELETE FROM feeds WHERE id = ?", feedID); err != nil { + return false, err } + // Let the surviving feed pick up the merged subscribers on the next tick. + if _, err := tx.Exec("UPDATE feeds SET next_poll_at = ? WHERE id = ? AND disabled = 0", time.Now(), targetID); err != nil { + return false, err + } + return true, tx.Commit() +} - return feed, nil +// DisableFeed retires a feed that has gone away (FRB110-118). +func (db *Abonnements) DisableFeed(feedID int64, reason string) error { + const query = `UPDATE feeds SET disabled = 1, disabled_reason = ?, next_poll_at = NULL WHERE id = ?` + _, err := db.Exec(query, reason, feedID) + return err +} + +// ReactivateFeed re-enables a retired feed, e.g. after it was successfully +// fetched again on subscribe. It reports whether the feed was disabled. +func (db *Abonnements) ReactivateFeed(feedUrl string, nextPollAt time.Time) (bool, error) { + const query = `UPDATE feeds SET disabled = 0, disabled_reason = NULL, error_count = 0, failing_since = NULL, next_poll_at = ? +WHERE url = ? AND disabled = 1` + result, err := db.Exec(query, nextPollAt, feedUrl) + if err != nil { + return false, err + } + n, err := result.RowsAffected() + return n > 0, err } diff --git a/storage/abonnements_test.go b/storage/abonnements_test.go new file mode 100644 index 0000000..3895d79 --- /dev/null +++ b/storage/abonnements_test.go @@ -0,0 +1,35 @@ +package storage + +import ( + "database/sql" + "reflect" + "testing" + "time" +) + +func TestPollHintsRoundTrip(t *testing.T) { + in := PollHints{Interval: 90 * time.Minute, SkipHours: []int{0, 23}, SkipDays: []string{"Saturday", "Sunday"}} + interval, skipHours, skipDays := in.encode() + + feed := Feed{FeedInterval: interval} + if skipHours != nil { + feed.SkipHours = sql.NullString{String: *skipHours, Valid: true} + } + if skipDays != nil { + feed.SkipDays = sql.NullString{String: *skipDays, Valid: true} + } + + if out := feed.Hints(); !reflect.DeepEqual(in, out) { + t.Errorf("round trip = %+v, want %+v", out, in) + } +} + +func TestPollHintsEmpty(t *testing.T) { + interval, skipHours, skipDays := PollHints{}.encode() + if interval != 0 || skipHours != nil || skipDays != nil { + t.Errorf("empty hints encoded to %d %v %v", interval, skipHours, skipDays) + } + if h := (Feed{}).Hints(); h.Interval != 0 || h.SkipHours != nil || h.SkipDays != nil { + t.Errorf("empty feed decoded to %+v", h) + } +} diff --git a/storage/integration_test.go b/storage/integration_test.go new file mode 100644 index 0000000..9afcb31 --- /dev/null +++ b/storage/integration_test.go @@ -0,0 +1,398 @@ +package storage + +import ( + "errors" + "os" + "sort" + "testing" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" + migrate "github.com/rubenv/sql-migrate" +) + +// Integration tests run against a real MySQL/MariaDB when RSSBOT_TEST_DSN is +// set, e.g. "user:pass@tcp(127.0.0.1:3306)/rssbot_test?parseTime=True&loc=Local". +// The database is wiped before every test. +func newTestDB(t *testing.T) *DB { + t.Helper() + dsn := os.Getenv("RSSBOT_TEST_DSN") + if dsn == "" { + t.Skip("RSSBOT_TEST_DSN not set") + } + + conn, err := sqlx.Connect("mysql", dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + + for _, table := range []string{"abonnements", "chats", "feeds", "replacements", "gorp_migrations"} { + if _, err := conn.Exec("DROP TABLE IF EXISTS " + table); err != nil { + t.Fatal(err) + } + } + + db := &DB{DB: conn, Abonnements: &Abonnements{DB: conn}, Replacements: &Replacements{DB: conn}} + if _, err := db.Migrate(); err != nil { + t.Fatal(err) + } + return db +} + +func mustCreate(t *testing.T, db *DB, chatID int64, url string, nextPollAt time.Time) { + t.Helper() + if err := db.Abonnements.Create(chatID, "chat", url, nil, nil, nil, PollHints{}, nextPollAt); err != nil { + t.Fatal(err) + } +} + +func feedByURL(t *testing.T, db *DB, url string) (Abonnement, bool) { + t.Helper() + all, err := db.Abonnements.GetAll() + if err != nil { + t.Fatal(err) + } + for _, ab := range all { + if ab.Url == url { + return ab, true + } + } + return Abonnement{}, false +} + +func dueURLs(t *testing.T, db *DB) []string { + t.Helper() + due, err := db.Abonnements.GetDue() + if err != nil { + t.Fatal(err) + } + var urls []string + for _, ab := range due { + urls = append(urls, ab.Url) + } + sort.Strings(urls) + return urls +} + +func chatIDs(ab Abonnement) []int64 { + var ids []int64 + for _, c := range ab.Chats { + ids = append(ids, c.ID) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + return ids +} + +func TestMigrationsRoundTrip(t *testing.T) { + db := newTestDB(t) + source := &migrate.EmbedFileSystemMigrationSource{FileSystem: embeddedMigrations, Root: "migrations"} + + down, err := migrate.Exec(db.DB.DB, "mysql", source, migrate.Down) + if err != nil { + t.Fatalf("migrate down: %v", err) + } + up, err := migrate.Exec(db.DB.DB, "mysql", source, migrate.Up) + if err != nil { + t.Fatalf("migrate up: %v", err) + } + if down != up || up == 0 { + t.Errorf("down=%d up=%d, want equal and non-zero", down, up) + } +} + +func TestGetDueRespectsSchedule(t *testing.T) { + db := newTestDB(t) + now := time.Now() + + mustCreate(t, db, 1, "https://example.org/past", now.Add(-time.Minute)) + mustCreate(t, db, 1, "https://example.org/soon", now.Add(time.Hour)) + mustCreate(t, db, 1, "https://example.org/later", now.Add(10*time.Hour)) + + got := dueURLs(t, db) + if len(got) != 1 || got[0] != "https://example.org/past" { + t.Errorf("due feeds = %v, want only the past one (time zone mismatch?)", got) + } +} + +func TestCreateSharesFeedBetweenChats(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now()) + mustCreate(t, db, 2, url, time.Now()) + + all, err := db.Abonnements.GetAll() + if err != nil { + t.Fatal(err) + } + if len(all) != 1 { + t.Fatalf("got %d feeds, want 1", len(all)) + } + if ids := chatIDs(all[0]); len(ids) != 2 || ids[0] != 1 || ids[1] != 2 { + t.Errorf("chats = %v, want [1 2]", ids) + } + + feeds, err := db.Abonnements.GetByUser(2) + if err != nil { + t.Fatalf("GetByUser must map every feeds column: %v", err) + } + if len(feeds) != 1 || feeds[0].Url != url { + t.Errorf("GetByUser = %+v", feeds) + } +} + +func TestExists(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now()) + ab, _ := feedByURL(t, db, url) + + checks := []struct { + name string + fn func() (bool, error) + want bool + }{ + {"by url", func() (bool, error) { return db.Abonnements.ExistsByFeedUrl(1, url) }, true}, + {"by url other chat", func() (bool, error) { return db.Abonnements.ExistsByFeedUrl(2, url) }, false}, + {"by url unknown", func() (bool, error) { return db.Abonnements.ExistsByFeedUrl(1, "https://nope") }, false}, + {"by id", func() (bool, error) { return db.Abonnements.ExistsById(1, ab.ID) }, true}, + {"by id unknown", func() (bool, error) { return db.Abonnements.ExistsById(1, ab.ID+1) }, false}, + } + for _, c := range checks { + got, err := c.fn() + if err != nil || got != c.want { + t.Errorf("%s = %v, %v; want %v, nil", c.name, got, err, c.want) + } + } +} + +func TestSetFeedStatePersistsCacheAndHints(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now()) + ab, _ := feedByURL(t, db, url) + + if err := db.Abonnements.Reschedule(ab.ID, time.Now(), 3, 0); err != nil { + t.Fatal(err) + } + + entry, etag, lm := "guid-1", `"v1"`, "Wed, 21 Oct 2015 07:28:00 GMT" + hints := PollHints{Interval: 2 * time.Hour, SkipHours: []int{1, 2}, SkipDays: []string{"Sunday"}} + next := time.Now().Add(time.Hour).Truncate(time.Second) + if err := db.Abonnements.SetFeedState(ab.ID, &entry, &etag, &lm, hints, next, 0, 4); err != nil { + t.Fatal(err) + } + + got, _ := feedByURL(t, db, url) + if got.LastEntry.String != entry || got.ETag.String != etag || got.LastModified.String != lm { + t.Errorf("cache state not persisted: %+v", got.Feed) + } + if got.ErrorCount != 0 || got.UnchangedCount != 4 || got.FailingSince.Valid { + t.Errorf("counters not reset: errors=%d unchanged=%d failingSince=%v", got.ErrorCount, got.UnchangedCount, got.FailingSince) + } + if !got.NextPollAt.Time.Equal(next) { + t.Errorf("next_poll_at = %s, want %s", got.NextPollAt.Time, next) + } + if h := got.Hints(); h.Interval != hints.Interval || len(h.SkipHours) != 2 || len(h.SkipDays) != 1 { + t.Errorf("hints = %+v, want %+v", h, hints) + } +} + +func TestRescheduleTracksFailingSince(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now()) + ab, _ := feedByURL(t, db, url) + + if err := db.Abonnements.Reschedule(ab.ID, time.Now(), 1, 0); err != nil { + t.Fatal(err) + } + first, _ := feedByURL(t, db, url) + if !first.FailingSince.Valid { + t.Fatal("failing_since not set on first error") + } + + time.Sleep(1100 * time.Millisecond) + if err := db.Abonnements.Reschedule(ab.ID, time.Now(), 2, 0); err != nil { + t.Fatal(err) + } + second, _ := feedByURL(t, db, url) + if !second.FailingSince.Time.Equal(first.FailingSince.Time) { + t.Errorf("failing_since moved from %s to %s", first.FailingSince.Time, second.FailingSince.Time) + } + + if err := db.Abonnements.Reschedule(ab.ID, time.Now(), 0, 1); err != nil { + t.Fatal(err) + } + recovered, _ := feedByURL(t, db, url) + if recovered.FailingSince.Valid || recovered.ErrorCount != 0 { + t.Errorf("error streak not cleared: %+v", recovered.Feed) + } +} + +func TestDisableAndReactivate(t *testing.T) { + db := newTestDB(t) + url := "https://example.org/feed" + mustCreate(t, db, 1, url, time.Now().Add(-time.Minute)) + ab, _ := feedByURL(t, db, url) + + if err := db.Abonnements.Reschedule(ab.ID, time.Now().Add(-time.Minute), 5, 0); err != nil { + t.Fatal(err) + } + if err := db.Abonnements.DisableFeed(ab.ID, "HTTP 410 Gone"); err != nil { + t.Fatal(err) + } + if due := dueURLs(t, db); len(due) != 0 { + t.Errorf("disabled feed is due: %v", due) + } + disabled, _ := feedByURL(t, db, url) + if !disabled.Disabled || disabled.DisabledReason.String != "HTTP 410 Gone" { + t.Errorf("feed not disabled: %+v", disabled.Feed) + } + + ok, err := db.Abonnements.ReactivateFeed(url, time.Now().Add(-time.Second)) + if err != nil || !ok { + t.Fatalf("ReactivateFeed = %v, %v; want true", ok, err) + } + reactivated, _ := feedByURL(t, db, url) + if reactivated.Disabled || reactivated.DisabledReason.Valid || reactivated.ErrorCount != 0 || reactivated.FailingSince.Valid { + t.Errorf("feed not fully reactivated: %+v", reactivated.Feed) + } + if due := dueURLs(t, db); len(due) != 1 { + t.Errorf("reactivated feed is not due") + } + + if ok, err := db.Abonnements.ReactivateFeed(url, time.Now()); err != nil || ok { + t.Errorf("ReactivateFeed on active feed = %v, %v; want false", ok, err) + } +} + +func TestMoveFeedURLRenames(t *testing.T) { + db := newTestDB(t) + mustCreate(t, db, 1, "http://example.org/feed", time.Now()) + ab, _ := feedByURL(t, db, "http://example.org/feed") + + merged, err := db.Abonnements.MoveFeedURL(ab.ID, "https://example.org/feed") + if err != nil || merged { + t.Fatalf("MoveFeedURL = %v, %v; want false, nil", merged, err) + } + if _, ok := feedByURL(t, db, "https://example.org/feed"); !ok { + t.Error("feed not renamed") + } +} + +func TestMoveFeedURLMergesIntoExistingFeed(t *testing.T) { + db := newTestDB(t) + oldURL, newURL := "http://example.org/old", "https://example.org/new" + later := time.Now().Add(time.Hour) + mustCreate(t, db, 1, oldURL, later) + mustCreate(t, db, 2, oldURL, later) + mustCreate(t, db, 2, newURL, later) + mustCreate(t, db, 3, newURL, later) + old, _ := feedByURL(t, db, oldURL) + + merged, err := db.Abonnements.MoveFeedURL(old.ID, newURL) + if err != nil || !merged { + t.Fatalf("MoveFeedURL = %v, %v; want true, nil", merged, err) + } + if _, ok := feedByURL(t, db, oldURL); ok { + t.Error("old feed still exists") + } + target, _ := feedByURL(t, db, newURL) + if ids := chatIDs(target); len(ids) != 3 { + t.Errorf("chats after merge = %v, want [1 2 3]", ids) + } + if due := dueURLs(t, db); len(due) != 1 || due[0] != newURL { + t.Errorf("merged feed should be due immediately, due = %v", due) + } +} + +func TestDeleteRemovesOrphans(t *testing.T) { + db := newTestDB(t) + shared, single := "https://example.org/shared", "https://example.org/single" + mustCreate(t, db, 1, shared, time.Now()) + mustCreate(t, db, 2, shared, time.Now()) + mustCreate(t, db, 1, single, time.Now()) + sharedFeed, _ := feedByURL(t, db, shared) + singleFeed, _ := feedByURL(t, db, single) + + if err := db.Abonnements.Delete(1, singleFeed.ID); err != nil { + t.Fatal(err) + } + if _, ok := feedByURL(t, db, single); ok { + t.Error("orphaned feed not deleted") + } + + if err := db.Abonnements.Delete(2, sharedFeed.ID); err != nil { + t.Fatal(err) + } + remaining, ok := feedByURL(t, db, shared) + if !ok || len(remaining.Chats) != 1 || remaining.Chats[0].ID != 1 { + t.Errorf("shared feed should remain for chat 1: %+v", remaining) + } + + var chats int + if err := db.Get(&chats, "SELECT COUNT(*) FROM chats"); err != nil { + t.Fatal(err) + } + if chats != 1 { + t.Errorf("got %d chats, want 1 (chat 2 has no abonnements left)", chats) + } +} + +func TestReplacements(t *testing.T) { + db := newTestDB(t) + + seeded, err := db.Replacements.List() + if err != nil { + t.Fatal(err) + } + if len(seeded) == 0 { + t.Fatal("default replacements missing") + } + + if err := db.Replacements.Create("[werbung]", false); err != nil { + t.Fatal(err) + } + err = db.Replacements.Create("[werbung]", false) + var mysqlErr *mysql.MySQLError + if !errors.As(err, &mysqlErr) || mysqlErr.Number != 1062 { + t.Errorf("duplicate replacement: err = %v, want MySQL error 1062", err) + } + + list, err := db.Replacements.List() + if err != nil { + t.Fatal(err) + } + if len(list) != len(seeded)+1 { + t.Fatalf("got %d replacements, want %d", len(list), len(seeded)+1) + } + + var id int64 + for _, r := range list { + if r.Value == "[werbung]" { + id = r.ID + } + } + if err := db.Replacements.Delete(id); err != nil { + t.Errorf("Delete: %v", err) + } + if err := db.Replacements.Delete(id); err == nil { + t.Error("deleting a missing replacement should fail") + } +} + +func TestFeedURLsAreCaseSensitive(t *testing.T) { + db := newTestDB(t) + mustCreate(t, db, 1, "https://example.org/Feed", time.Now()) + mustCreate(t, db, 1, "https://example.org/feed", time.Now()) + + all, err := db.Abonnements.GetAll() + if err != nil { + t.Fatal(err) + } + if len(all) != 2 { + t.Errorf("got %d feeds, want 2 distinct URLs", len(all)) + } +} diff --git a/storage/migrations/0001-init.sql b/storage/migrations/0001-init.sql index 51cc6ae..18f56d2 100644 --- a/storage/migrations/0001-init.sql +++ b/storage/migrations/0001-init.sql @@ -31,3 +31,10 @@ CREATE TABLE IF NOT EXISTS `abonnements` CONSTRAINT `FK_abonnements_feeds` FOREIGN KEY (`feed_id`) REFERENCES `feeds` (`id`) ON UPDATE CASCADE ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; + + +-- +migrate Down + +DROP TABLE `abonnements`; +DROP TABLE `feeds`; +DROP TABLE `chats`; diff --git a/storage/migrations/0002-replacements.sql b/storage/migrations/0002-replacements.sql index 196afa9..0071e48 100644 --- a/storage/migrations/0002-replacements.sql +++ b/storage/migrations/0002-replacements.sql @@ -3,7 +3,7 @@ CREATE TABLE `replacements` ( `id` INT(11) NOT NULL AUTO_INCREMENT, - `value` VARCHAR(255) NOT NULL, + `value` VARCHAR(255) NOT NULL COLLATE utf8mb4_bin, `is_regex` TINYINT(1) NOT NULL DEFAULT '0', `created_at` DATETIME NOT NULL DEFAULT current_timestamp(), PRIMARY KEY (`id`) USING BTREE, @@ -52,4 +52,8 @@ VALUES ('[←]', 0), ('Der Beitrag.*erschien zuerst auf .+.', 1), ('The post.*appeared first on .+.', 1), ('http://www.serienjunkies.de/.*.html', 1), - ('<.*?>', 1); \ No newline at end of file + ('<.*?>', 1); + +-- +migrate Down + +DROP TABLE `replacements`; diff --git a/storage/migrations/0003-efficiency.sql b/storage/migrations/0003-efficiency.sql new file mode 100644 index 0000000..c99dc8b --- /dev/null +++ b/storage/migrations/0003-efficiency.sql @@ -0,0 +1,25 @@ +-- +migrate Up + +ALTER TABLE `feeds` + ADD COLUMN `etag` VARCHAR(512) DEFAULT NULL, + ADD COLUMN `last_modified` VARCHAR(128) DEFAULT NULL, + ADD COLUMN `next_poll_at` DATETIME DEFAULT NULL, + ADD COLUMN `last_poll_at` DATETIME DEFAULT NULL, + ADD COLUMN `error_count` INT(11) NOT NULL DEFAULT 0, + ADD COLUMN `unchanged_count` INT(11) NOT NULL DEFAULT 0, + ADD COLUMN `disabled` TINYINT(1) NOT NULL DEFAULT 0, + ADD COLUMN `disabled_reason` VARCHAR(512) DEFAULT NULL, + ADD INDEX `idx_feeds_due` (`disabled`, `next_poll_at`) USING BTREE; + +-- +migrate Down + +ALTER TABLE `feeds` + DROP INDEX `idx_feeds_due`, + DROP COLUMN `etag`, + DROP COLUMN `last_modified`, + DROP COLUMN `next_poll_at`, + DROP COLUMN `last_poll_at`, + DROP COLUMN `error_count`, + DROP COLUMN `unchanged_count`, + DROP COLUMN `disabled`, + DROP COLUMN `disabled_reason`; diff --git a/storage/migrations/0004-feed-hints.sql b/storage/migrations/0004-feed-hints.sql new file mode 100644 index 0000000..fef11f8 --- /dev/null +++ b/storage/migrations/0004-feed-hints.sql @@ -0,0 +1,13 @@ +-- +migrate Up + +ALTER TABLE `feeds` + ADD COLUMN `feed_interval` INT(11) NOT NULL DEFAULT 0, + ADD COLUMN `skip_hours` VARCHAR(100) DEFAULT NULL, + ADD COLUMN `skip_days` VARCHAR(100) DEFAULT NULL; + +-- +migrate Down + +ALTER TABLE `feeds` + DROP COLUMN `feed_interval`, + DROP COLUMN `skip_hours`, + DROP COLUMN `skip_days`; diff --git a/storage/migrations/0005-failing-since.sql b/storage/migrations/0005-failing-since.sql new file mode 100644 index 0000000..36f24da --- /dev/null +++ b/storage/migrations/0005-failing-since.sql @@ -0,0 +1,9 @@ +-- +migrate Up + +ALTER TABLE `feeds` + ADD COLUMN `failing_since` DATETIME DEFAULT NULL; + +-- +migrate Down + +ALTER TABLE `feeds` + DROP COLUMN `failing_since`; diff --git a/storage/migrations/0006-binary-collation.sql b/storage/migrations/0006-binary-collation.sql new file mode 100644 index 0000000..1bb8814 --- /dev/null +++ b/storage/migrations/0006-binary-collation.sql @@ -0,0 +1,17 @@ +-- +migrate Up + +-- Newer MariaDB defaults to utf8mb4_uca1400_ai_ci, which treats e.g. "…" and +-- "..." or differently cased URL paths as equal. Compare byte-exact instead. +ALTER TABLE `replacements` + MODIFY `value` VARCHAR(255) NOT NULL COLLATE utf8mb4_bin; + +ALTER TABLE `feeds` + MODIFY `url` VARCHAR(512) NOT NULL COLLATE utf8mb4_bin; + +-- +migrate Down + +ALTER TABLE `feeds` + MODIFY `url` VARCHAR(512) NOT NULL COLLATE utf8mb4_general_ci; + +ALTER TABLE `replacements` + MODIFY `value` VARCHAR(255) NOT NULL COLLATE utf8mb4_general_ci;