feat: port upstream reconciliation efficiency, failure backoff, multicluster naming and image cleanup

.github/workflows/tests.yaml

@@ -17,6 +17,19 @@ jobs:
       - name: Run Rust unit tests
         run: cargo test --all-targets --locked
 
+  docker-smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build locked release image
+        run: docker build --progress=plain -t robotlb:pr .
+      - name: Verify runtime dependencies and CLI startup
+        run: |
+          docker run --rm --entrypoint /bin/sh robotlb:pr -c '
+            ldd /usr/local/bin/robotlb
+            /usr/local/bin/robotlb --help > /dev/null
+          '
+
   helm-unittest:
     runs-on: ubuntu-latest
     steps:

Dockerfile

@@ -1,18 +1,20 @@
 FROM rust:1.93.1-bookworm AS builder
 
-RUN apt update && apt-get install -y pkg-config libjemalloc-dev libssl-dev && apt-get clean
+RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
 COPY . .
 
 ENV RUST_BACKTRACE=1
 ENV JEMALLOC_SYS_WITH_MALLOC_CONF="background_thread:true,tcache:false,dirty_decay_ms:100,muzzy_decay_ms:100,abort_conf:true"
-RUN cargo build --release
+RUN cargo build --release --locked
 
 FROM debian:bookworm AS base
 
-RUN apt-get update && apt install -y libssl-dev ca-certificates libjemalloc-dev && apt-get clean
+RUN apt-get update \
+    && apt-get install --yes --no-install-recommends libssl3 ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /app/target/release/robotlb /usr/local/bin/
 ENV PATH=/usr/local/bin:$PATH

README.md

@@ -231,3 +231,43 @@ spec:
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=forkline/robotlb&type=Date)](https://star-history.com/#forkline/robotlb&Date)
+
+## Reconciliation and multi-cluster operation
+
+robotlb handles `LoadBalancer` Services whose `spec.loadBalancerClass` is unset
+or set to `robotlb`. When other LoadBalancer controllers run in the cluster,
+set `spec.loadBalancerClass: robotlb` **when creating the Service** to prevent
+competing controllers from repeatedly overwriting `status.loadBalancer`.
+The class is effectively immutable after creation; recreating the Service can
+replace the LB and change its IP.
+
+Node changes that affect targets (readiness, labels, cordon state or addresses)
+trigger reconciliation immediately; kubelet heartbeats do not. For `Local`
+Services, EndpointSlice updates reconcile only when relevant target nodes change:
+Pod-selector Services ignore endpoint readiness/address churn, while selectorless
+Services track ready endpoint nodes. EndpointSlice deletions and ownership-label
+changes also trigger reconciliation. On otherwise unchanged Services, the operator
+compares the desired Service spec, `robotlb/` annotations, target IPs and listener
+ports with the last successful reconcile and skips Hetzner requests, but repairs
+unexpected ingress status changes from the last successful status snapshot.
+Every `ROBOTLB_RESYNC_INTERVAL` seconds (default **300**) it performs a full
+check anyway to recover out-of-band changes made in Hetzner.
+
+A failing Service retries independently with exponential backoff from 5 seconds
+up to 5 minutes, without suppressing reconciliation triggered by Kubernetes
+watch events. Project-wide Hetzner API rate limits use the separate shared gate
+without resetting the per-Service failure backoff. After Hetzner creates an LB,
+the operator retries every 15 seconds until an ingress address is available.
+After 180 seconds without an ingress address, it reports a reconciliation error
+and emits throttled Kubernetes Warning Events while continuing the same 15-second
+polling interval. Incomplete provisioning is never memoized. Other errors use
+the normal failure backoff.
+
+When several clusters share one Hetzner Cloud project, set
+`ROBOTLB_CLUSTER_NAME` (for example `prod`) to a DNS-label prefix
+of 1–63 lowercase letters, digits and hyphens. Default LB names then become
+`<cluster>.<service>.<namespace>`, with a stable hash suffix when exceeding
+Hetzner's 128-character limit. Explicit `robotlb/balancer` names are not
+prefixed. Existing owned balancers continue to be selected by immutable Service
+UID labels, so changing the prefix does **not** rename or adopt them.
+Unlabelled legacy balancers are not automatically adopted.

helm/values.yaml

@@ -16,6 +16,10 @@ replicaCount: 1
 
 envs:
   ROBOTLB_LOG_LEVEL: "INFO"
+  # Optional cluster prefix when multiple clusters share one Hetzner project.
+  # ROBOTLB_CLUSTER_NAME: "prod"
+  # Full drift resync interval; Node and EndpointSlice changes trigger earlier.
+  ROBOTLB_RESYNC_INTERVAL: "300"
 
 existingSecrets: []
 

src/backoff.rs

@@ -0,0 +1,128 @@
+use std::{
+    collections::HashMap,
+    sync::Mutex,
+    time::{Duration, Instant},
+};
+
+const FIRST_DELAY: Duration = Duration::from_secs(5);
+const MAX_DELAY: Duration = Duration::from_secs(300);
+/// A service deleted before it got the robotlb finalizer is not reconciled again to be
+/// forgotten on success, so an entry that has not failed for this long is dropped. It is
+/// well above `MAX_DELAY` plus one rate limit pause; a service held at the rate limit gate
+/// for longer starts short again.
+const FORGET_AFTER: Duration = Duration::from_secs(3600);
+
+/// Retry delay of each service whose reconciliation keeps failing.
+///
+/// It doubles from `FIRST_DELAY` up to `MAX_DELAY` like the upstream cloud-provider
+/// service controller:
+/// <https://github.com/kubernetes/cloud-provider/blob/master/controllers/service/controller.go>
+#[derive(Debug, Default)]
+pub struct ErrorBackoff {
+    failures: Mutex<HashMap<String, Failures>>,
+}
+
+#[derive(Debug)]
+struct Failures {
+    count: u32,
+    last: Instant,
+}
+
+impl ErrorBackoff {
+    /// Record a failed reconciliation of `service` and return how long to wait before the next one.
+    pub fn on_failure(&self, service: &str, now: Instant) -> Duration {
+        let mut failures = self
+            .failures
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner);
+        failures.retain(|_, failure| now.saturating_duration_since(failure.last) <= FORGET_AFTER);
+        let failure = failures.entry(service.to_string()).or_insert(Failures {
+            count: 0,
+            last: now,
+        });
+        let delay = FIRST_DELAY
+            .saturating_mul(2_u32.saturating_pow(failure.count))
+            .min(MAX_DELAY);
+        failure.count = failure.count.saturating_add(1);
+        failure.last = now;
+        drop(failures);
+        delay
+    }
+
+    /// Start the delay of `service` short again.
+    pub fn forget(&self, service: &str) {
+        self.failures
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .remove(service);
+    }
+
+    #[cfg(test)]
+    fn tracked(&self) -> usize {
+        self.failures
+            .lock()
+            .unwrap_or_else(std::sync::PoisonError::into_inner)
+            .len()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{ErrorBackoff, FORGET_AFTER};
+    use std::time::{Duration, Instant};
+
+    #[test]
+    fn repeated_failures_double_the_delay_up_to_the_cap() {
+        let backoff = ErrorBackoff::default();
+        let mut now = Instant::now();
+        let mut delays = Vec::new();
+        for _ in 0..9 {
+            let delay = backoff.on_failure("shop/web", now);
+            delays.push(delay.as_secs());
+            now += delay;
+        }
+        assert_eq!(delays, vec![5, 10, 20, 40, 80, 160, 300, 300, 300]);
+    }
+
+    #[test]
+    fn services_back_off_independently() {
+        let backoff = ErrorBackoff::default();
+        let now = Instant::now();
+        backoff.on_failure("shop/web", now);
+        backoff.on_failure("shop/web", now);
+        assert_eq!(backoff.on_failure("shop/api", now), Duration::from_secs(5));
+        assert_eq!(backoff.on_failure("shop/web", now), Duration::from_secs(20));
+    }
+
+    #[test]
+    fn a_forgotten_service_starts_short_again() {
+        let backoff = ErrorBackoff::default();
+        let now = Instant::now();
+        backoff.on_failure("shop/web", now);
+        backoff.on_failure("shop/web", now);
+        backoff.forget("shop/web");
+        assert_eq!(backoff.on_failure("shop/web", now), Duration::from_secs(5));
+    }
+
+    #[test]
+    fn a_long_failing_service_is_not_forgotten() {
+        let backoff = ErrorBackoff::default();
+        let mut now = Instant::now();
+        for _ in 0..20 {
+            now += backoff.on_failure("shop/web", now);
+        }
+        assert_eq!(
+            backoff.on_failure("shop/web", now),
+            Duration::from_secs(300)
+        );
+    }
+
+    #[test]
+    fn services_that_stopped_failing_are_dropped() {
+        let backoff = ErrorBackoff::default();
+        let now = Instant::now();
+        backoff.on_failure("shop/deleted", now);
+        backoff.on_failure("shop/web", now + FORGET_AFTER + Duration::from_secs(1));
+        assert_eq!(backoff.tracked(), 1);
+    }
+}

src/config.rs

@@ -13,6 +13,10 @@ pub struct OperatorConfig {
     #[arg(short = 't', long, env = "ROBOTLB_HCLOUD_TOKEN")]
     pub hcloud_token: String,
 
+    /// Optional cluster prefix for default LB names when sharing a Hetzner project.
+    #[arg(long, env = "ROBOTLB_CLUSTER_NAME", value_parser = crate::lb::parse_cluster_name)]
+    pub cluster_name: Option<String>,
+
     /// Default network to use for load balancers.
     /// If not set, then only network from the service annotation will be used.
     #[arg(long, env = "ROBOTLB_DEFAULT_NETWORK", default_value = None)]
@@ -23,6 +27,15 @@ pub struct OperatorConfig {
     #[arg(long, env = "ROBOTLB_DYNAMIC_NODE_SELECTOR", default_value = "true")]
     pub dynamic_node_selector: bool,
 
+    /// Maximum seconds between full Hetzner drift checks, even with no changes.
+    #[arg(
+        long,
+        env = "ROBOTLB_RESYNC_INTERVAL",
+        default_value = "300",
+        value_parser = clap::value_parser!(u64).range(1..=31_536_000)
+    )]
+    pub resync_interval: u64,
+
     /// Default load balancer healthcheck retries cound.
     #[arg(long, env = "ROBOTLB_DEFAULT_LB_RETRIES", default_value = "3")]
     pub default_lb_retries: i32,
@@ -140,8 +153,10 @@ mod tests {
             .expect("config should parse");
 
         assert_eq!(config.hcloud_token, "token");
+        assert_eq!(config.cluster_name, None);
         assert_eq!(config.default_network, None);
         assert!(config.dynamic_node_selector);
+        assert_eq!(config.resync_interval, 300);
         assert_eq!(config.default_lb_retries, 3);
         assert_eq!(config.default_lb_timeout, 10);
         assert_eq!(config.default_lb_interval, 15);
@@ -167,6 +182,10 @@ mod tests {
             "robotlb",
             "--hcloud-token",
             "token",
+            "--cluster-name",
+            "prod",
+            "--resync-interval",
+            "120",
             "--default-network",
             "prod-net",
             "--default-lb-retries",
@@ -204,6 +223,8 @@ mod tests {
         .expect("config should parse");
 
         assert_eq!(config.default_network.as_deref(), Some("prod-net"));
+        assert_eq!(config.resync_interval, 120);
+        assert_eq!(config.cluster_name.as_deref(), Some("prod"));
         assert!(config.dynamic_node_selector);
         assert_eq!(config.default_lb_retries, 5);
         assert_eq!(config.default_lb_timeout, 7);