Add optional Robotty missed-message recovery

Cargo.toml

@@ -12,6 +12,7 @@ aide = { version = "0.15.0", features = [
 ] }
 anyhow = "1.0.75"
 axum = { version = "0.8.4", features = ["tokio"] }
+blake3 = "1.8.2"
 chrono = { version = "0.4.27", features = ["serde"] }
 clap = { version = "4.4.1", features = ["derive"] }
 clickhouse = { version = "0.13.0", default-features = false, features = [
@@ -27,12 +28,14 @@ indexmap = "2.2.6"
 lazy_static = "1.4.0"
 mimalloc = { version = "0.1.38", default-features = false }
 mime_guess = "2.0.4"
-prometheus = "0.13.3"
+prometheus = "0.14"
 rand = "0.9.0"
 rayon = "1.7.0"
 reqwest = { version = "0.12.4", features = [
+    "json",
     "rustls-tls",
 ], default-features = false }
+rustls = { version = "0.23", default-features = false, features = ["ring"] }
 rust-embed = { version = "8.0.0", features = ["interpolate-folder-path"] }
 schemars = { version = "0.9", features = ["chrono04"] }
 serde = { version = "1.0.188", features = ["derive"] }
@@ -49,7 +52,7 @@ tower-http = { version = "0.6.1", features = [
 ] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
-twitch-irc = { version = "5.0.1", default-features = false, features = [
+twitch-irc = { version = "6.1.2", default-features = false, features = [
     "metrics-collection",
     "transport-tcp-rustls-webpki-roots",
 ] }
@@ -61,7 +64,8 @@ twitch_api = { version = "0.7.0", features = [
 ] }
 tmi = { version = "0.7.0", default-features = false, features = ["simd"] }
 axum-prometheus = "0.8.0"
-metrics-prometheus = "0.8.0"
+metrics-prometheus = "0.11.2"
+moka = { version = "0.12.14", features = ["sync"] }
 axum-extra = { version = "0.10.0", features = ["typed-header"] }
 bitflags = { version = "2.5.0", features = ["serde"] }
 

config.dist.json

@@ -4,6 +4,9 @@
   "clickhouseUsername": "user",
   "clickhousePassword": "SuperSecretPassword",
   "listenAddress": "0.0.0.0:8025",
+  "recentMessagesEnabled": false,
+  "recentMessagesUrl": "https://recent-messages.robotty.de/api/v2/recent-messages",
+  "recentMessagesLimit": 800,
   "channels": ["12345"],
   "clientID": "id",
   "clientSecret": "secret",

docs/CONFIG.md

@@ -9,13 +9,20 @@ Available options:
 - `clickhousePassword` (string): Clickhouse password.
 - `clickhouseFlushInterval` (number): Interval (in seconds) of how often messages should be flushed to the database. A lower value means that logs are available sooner at the expensive of higher database load. Defaults to 10.
 - `listenAddress` (string): Listening address for the web server. Defaults to `0.0.0.0:8025`.
+- `recentMessagesEnabled` (boolean): Enables best-effort Robotty recent-message recovery. Defaults to `false`.
+- `recentMessagesUrl` (string): Robotty-compatible recent-messages base URL. Defaults to `https://recent-messages.robotty.de/api/v2/recent-messages`. Only HTTP and HTTPS URLs are accepted.
+- `recentMessagesLimit` (number): Maximum number of recent messages requested per channel. Defaults to `800`.
 - `channels` (array of strings): List of channel ids to be logged.
 - `clientId` (string): Twitch client id.
 - `clientSecret` (string): Twitch client secret.
 - `admins` (array of strings): List of usernames who are allowed to use administration commands.
 - `optOut` (object of strings: booleans): List of user ids who opted out from being logged.
 - `adminAPIKey` (string): API key for admin requests
 
+When enabled, rustlog requests recent messages after it successfully joins a channel at startup, after reconnecting, and after a runtime channel addition. Backfill is best-effort: request, response, and individual IRC parsing failures are logged without stopping live Twitch ingest.
+
+Replayed messages never execute chat commands. A live command still executes if recovery has already stored its message. Removing a channel stops further recovery requests and prevents pending responses from being queued for storage.
+
 Example config:
 ```json
 {
@@ -24,6 +31,9 @@ Example config:
   "clickhouseUsername": "user",
   "clickhousePassword": "SuperSecretPassword",
   "listenAddress": "0.0.0.0:8025",
+  "recentMessagesEnabled": false,
+  "recentMessagesUrl": "https://recent-messages.robotty.de/api/v2/recent-messages",
+  "recentMessagesLimit": 800,
   "channels": ["12345"],
   "clientID": "id",
   "clientSecret": "secret",