Smarter feed polling, bug fixes, tests and NixOS module update

.dockerignore

@@ -1,5 +1,6 @@
 *
 !config/
+!fetcher/
 !handler/
 !storage/
 !go.mod

.env.example

@@ -1,8 +1,38 @@
+# Your Telegram user ID. Only this user can control the bot.
 ADMIN_ID=1337
+# Token from @BotFather
 BOT_TOKEN=123456789:abcdefghijklmnopqrstuvxwyz
-MYSQL_HOST=127.0.0.1 # Use 'db' for Docker
+
+# Custom post template (optional). Without it, post.gohtml from the working
+# directory or the built-in template is used. See README.
+#POST_TEMPLATE=/path/to/post.gohtml
+
+# Database connection (MySQL or MariaDB)
+# Use 'db' as host with the included docker-compose.yml
+MYSQL_HOST=127.0.0.1
 MYSQL_PORT=3306
 MYSQL_USER=myuser
 MYSQL_PASSWORD=mypassword
 MYSQL_DB=mydb
-MYSQL_SOCKET=/run/mysqld/mysqld.sock
+# Connect via Unix socket instead. If set, MYSQL_HOST, MYSQL_PORT and
+# MYSQL_PASSWORD are ignored (socket authentication).
+#MYSQL_SOCKET=/run/mysqld/mysqld.sock
+# Encrypt the TCP connection: false (default), true, skip-verify or preferred
+#MYSQL_TLS=false
+
+# Polling (all optional, the values below are the defaults)
+# Durations are written like 30s, 10m, 1h30m or 6h.
+# See "How polling works" in the README for details.
+
+# How often each feed is checked. Can be as low as 1m.
+#POLL_INTERVAL=10m
+# The longest a feed ever waits between two checks. Limits the adaptive
+# slow-down, the waiting time after errors and intervals requested by servers.
+#POLL_INTERVAL_MAX=6h
+# true: check feeds that rarely get new entries less often (up to POLL_INTERVAL_MAX)
+# false: check every feed every POLL_INTERVAL
+#POLL_ADAPTIVE=true
+# How many feeds are downloaded at the same time
+#POLL_CONCURRENCY=8
+# How often the bot looks for feeds that are due. Rarely needs changing.
+#POLL_TICK=30s

.github/workflows/build.yml

@@ -3,13 +3,73 @@ on:
   - push
   - pull_request
 
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
 env:
   NAME: rssbot
 
 jobs:
+  test:
+    name: test (MariaDB ${{ matrix.mariadb.version }})
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read # checkout
+    strategy:
+      fail-fast: false
+      matrix:
+        mariadb:
+          - version: '11.4' # nixpkgs default
+            image: mariadb:11.4@sha256:70cc072b29b4a89ae07abb2d4da2c64678a7f2dfe092751bb51c87d67dc1338b
+          - version: '12.3' # LTS
+            image: mariadb:12.3@sha256:805c8e104bd563d5bfa24fadd3f31cd419ea859cb5277f32b5dbf2db714f9ed1
+          - version: '13.0' # latest, used by docker-compose.yml
+            image: mariadb:13.0@sha256:d4fdec0510ad498e4f3127da30a99df3745bd6d5e611ae6ac5f76403d9284a8d
+    services:
+      mariadb:
+        image: ${{ matrix.mariadb.image }}
+        env:
+          MARIADB_DATABASE: rssbot_test
+          MARIADB_USER: rssbot
+          MARIADB_PASSWORD: rssbot
+          MARIADB_RANDOM_ROOT_PASSWORD: '1'
+          # Differs from the runner's UTC to catch time zone bugs.
+          TZ: Asia/Tokyo
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd "healthcheck.sh --connect --innodb_initialized"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Vet
+        run: go vet ./...
+
+      - name: Test
+        env:
+          RSSBOT_TEST_DSN: rssbot:rssbot@tcp(127.0.0.1:3306)/rssbot_test?charset=utf8mb4&parseTime=True&loc=Local
+        run: go test -race ./...
+
   build:
     name: build
     runs-on: ubuntu-latest
+    permissions:
+      contents: read # checkout
     strategy:
       matrix:
         GOOS: [windows, linux, darwin]
@@ -24,59 +84,64 @@ jobs:
 
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
-          go-version-file: 'go.mod'
+          go-version: stable
 
       - name: Build
         env:
           GOOS: ${{ matrix.GOOS }}
           GOARCH: ${{ matrix.GOARCH }}
-        run: go build -ldflags="-s -w" -o dist/$NAME-$GOOS-$GOARCH
+        run: go build -ldflags="-s -w" -o "dist/${NAME}-${GOOS}-${GOARCH}"
 
       - name: Rename binaries (Windows)
         if: matrix.GOOS == 'windows'
-        run: for x in dist/$NAME-windows-*; do mv $x $x.exe; done
+        run: for x in dist/"${NAME}"-windows-*; do mv "$x" "$x.exe"; done
 
       - name: Upload binary
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: ${{env.NAME}}-${{ matrix.GOOS }}-${{ matrix.GOARCH }}-${{github.sha}}
+          name: ${{ env.NAME }}-${{ matrix.GOOS }}-${{ matrix.GOARCH }}-${{ github.sha }}
           path: dist/*
           retention-days: 90
 
   docker:
     name: docker
+    needs: test
     runs-on: ubuntu-latest
     permissions:
-      packages: write
-      contents: read
+      contents: read # checkout
+      packages: write # push image to ghcr.io
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout repo
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
 
       - name: Build image
-        run: docker build . --file Dockerfile --tag $NAME --label "runnumber=${GITHUB_RUN_ID}"
+        run: docker build . --file Dockerfile --tag "$NAME" --build-arg VERSION="${GITHUB_SHA::7}" --label "runnumber=${GITHUB_RUN_ID}"
 
       - name: Log in to registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/'))
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: echo "$GITHUB_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
 
       - name: Push image
+        if: github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/'))
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$NAME
-
-          # This changes all uppercase characters to lowercase.
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
-          # This strips the git ref prefix from the version.
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-          # This strips the "v" prefix from the tag name.
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-          # This uses the Docker `latest` tag convention.
-          [ "$VERSION" == "master" ] && VERSION=latest
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
-          docker tag $NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+          image_id="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/${NAME}"
+          if [[ "$GITHUB_REF" == refs/tags/* ]]; then
+            version="${GITHUB_REF_NAME#v}"
+          else
+            version=latest
+          fi
+          echo "Pushing ${image_id}:${version}"
+          docker tag "$NAME" "${image_id}:${version}"
+          docker push "${image_id}:${version}"

Dockerfile

@@ -1,11 +1,12 @@
-FROM golang:1.22 AS build-stage
+FROM golang:1.27.1@sha256:3680233e3204827fbdc66088528ae6d4b3d034f51d03a99d454f6de034888244 AS build-stage
 WORKDIR /app
 COPY go.mod go.sum ./
 RUN go mod download
 COPY . ./
-RUN CGO_ENABLED=0 GOOS=linux go build -o /rssbot
+ARG VERSION=dev
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X github.com/Brawl345/rssbot/fetcher.Version=${VERSION}" -o /rssbot
 
-FROM gcr.io/distroless/static-debian12 AS release-stage
+FROM gcr.io/distroless/static-debian13:nonroot@sha256:e2e927ec666bae08560abb3c55d0659eceabb657f56b6782ab500a9fc7f555e3 AS release-stage
 WORKDIR /app
 COPY --from=build-stage /rssbot /app/rssbot
 USER nonroot:nonroot

README.md

@@ -7,8 +7,9 @@ Only one user (the "admin") can manage the bot, but it's possible to let the bot
 The bot's language is German, but it should be self-explanatory.
 
 ## Features
-* Checks feeds every minute (after all checks are finished)
-* Concurrent checks
+* Every feed has its own schedule, see [How polling works](#how-polling-works)
+* Polite fetching: conditional requests, compression, backoff on errors and rate limits
+* Slows down feeds that rarely change (optional)
 * Can post private, in channels or groups
 * Custom post format with a `post.gohtml` file
 * Supports "replacements" where specific words will be removed (limited Regex is also supported). This is useful for spam like "Read more on XYZ" and stuff
@@ -17,14 +18,87 @@ The bot's language is German, but it should be self-explanatory.
 
 1. Download binary for your system from Releases or build it yourself
 2. Copy ".env.example" to ".env" and fill it in
-3. (Optional) Create a `post.gohtml` with a custom Go HTML template that will be used for posts (see below)
+3. (Optional) Create a `post.gohtml` with a custom Go template that will be used for posts (see below)
 4. Run and done! Database migrations are applied automatically.
 
-Feeds are checked every minute after the latest check finished (it waits for five seconds the first time after the bot starts).
+### NixOS
 
-### Use your own template
+The flake provides a NixOS module. By default it creates a local MariaDB database and connects via Unix socket.
 
-The bot reads the `post.gohtml` from the same directory and uses it as a [Go template](https://pkg.go.dev/text/template) where it inserts the data. Take a look inside the [handler/feed_check.go](handler/feed_check.go) file (the `TemplateData` struct) to see all available fields. You can find the default template inside the [config/config.go](config/config.go) file. [Limited HTML](https://core.telegram.org/bots/api#html-style) is supported and all fields are sanitized with HTML tags removed and "replacements" applied. 
+```nix
+{
+  inputs.rssbot.url = "github:Brawl345/rssbot";
+
+  outputs = { nixpkgs, rssbot, ... }: {
+    nixosConfigurations.myhost = nixpkgs.lib.nixosSystem {
+      modules = [
+        rssbot.nixosModules.default
+        {
+          services.rssbot = {
+            enable = true;
+            adminId = 1337;
+            botTokenFile = "/run/secrets/rssbot-token";
+
+            # Optional
+            templateFile = ./post.gohtml; # or: template = "<b>{{.Title}}</b> ...";
+            poll = {
+              interval = "5m";
+              adaptive = false;
+            };
+          };
+        }
+      ];
+    };
+  };
+}
+```
+
+To use an existing database instead, set `database.createLocally = false` together with `database.host` and `database.passwordFile`. `nix flake check` runs a VM test of the module.
+
+## How polling works
+
+The bot does not fetch all feeds at once. Every feed has its own "next poll" time stored in the database. Every `POLL_TICK` (default: 30 seconds) the bot fetches the feeds that are due and then calculates their next poll time. Because the schedule lives in the database, restarting the bot does not trigger a re-download of all feeds.
+
+Feeds are fetched in parallel (up to `POLL_CONCURRENCY`), but feeds on the same host are fetched one after another so a single server is never hit with several requests at once.
+
+Each request sends the `ETag` and `Last-Modified` values from the previous response. If nothing changed, the server can answer with a tiny `304 Not Modified` instead of the whole feed. Responses are compressed when the server supports it.
+
+### When is a feed polled next?
+
+1. **Base interval:** `POLL_INTERVAL` (default: 10 minutes).
+2. **Adaptive slow-down** (`POLL_ADAPTIVE`, on by default): every poll without a new entry adds one base interval to the wait time, up to `POLL_INTERVAL_MAX` (default: 6 hours). As soon as a new entry shows up, the feed is back to the base interval.
+
+   | Polls without new entries | Wait until next poll (defaults) |
+   |---------------------------|---------------------------------|
+   | 0                         | 10 min                          |
+   | 1                         | 20 min                          |
+   | 2                         | 30 min                          |
+   | 9 (≈ 8 hours of silence)  | 1 h 40 min                      |
+   | 35 (≈ 4 days of silence)  | 6 h (maximum)                   |
+
+   So a feed that was quiet over night may take up to ~1 h 40 min to deliver its first new post in the morning, while active feeds stay at 10 minutes.
+
+   **With `POLL_ADAPTIVE=false`** every feed is polled every `POLL_INTERVAL`, no matter how often it changes. New posts arrive faster, but the bot sends more requests. Thanks to the conditional requests, most of them are cheap `304` answers.
+3. **Server hints:** if the server asks for a longer interval (`Cache-Control: max-age` header or `<ttl>` in an RSS feed), the bot waits at least that long, but never longer than `POLL_INTERVAL_MAX`. Hints can only slow polling down, never speed it up.
+4. **Quiet hours:** RSS `<skipHours>` and `<skipDays>` (in UTC) are respected by moving the next poll out of these times.
+
+The actual time can be up to `POLL_TICK` later than calculated.
+
+### Errors, rate limits and moved feeds
+
+* **Temporary errors** (timeouts, HTTP 404/500, invalid feed, …): the wait time doubles with every consecutive failure (10 min, 20 min, 40 min, …, up to `POLL_INTERVAL_MAX`). A feed is **disabled** after it failed at least 12 times in a row *and* has been failing for 7 days.
+* **HTTP 410 Gone:** the feed is disabled immediately.
+* **HTTP 429/503:** the bot waits as long as the `Retry-After` header says (or 4 × `POLL_INTERVAL` without it, capped at `POLL_INTERVAL_MAX`). This does not count as an error.
+* **Permanent redirects (301/308):** the new URL is saved automatically. If another subscription already uses the new URL, both are merged. Temporary redirects (302/307) are only followed.
+* Redirects from a public feed to a private/local network address are refused.
+
+The admin gets a private message when a feed is disabled, moved or rate limited. Subscribed channels and groups only ever receive feed entries.
+
+Disabled feeds are marked with 🚫 in `/rss`. To enable one again, simply subscribe to it again with `/sub`.
+
+## Use your own template
+
+The bot reads the file set in `POST_TEMPLATE` (or `post.gohtml` from the working directory, e.g. `/app/post.gohtml` in Docker) and uses it as a [Go template](https://pkg.go.dev/text/template) where it inserts the data. Take a look inside the [handler/feed_check.go](handler/feed_check.go) file (the `TemplateData` struct) to see all available fields. You can find the default template inside the [config/config.go](config/config.go) file. [Limited HTML](https://core.telegram.org/bots/api#html-style) is supported and all fields are sanitized with HTML tags removed and "replacements" applied. 
 
 Example: