diff --git a/api/middleware/circuitbreaker.txt b/api/middleware/circuitbreaker.txt index 15ffdd399..e23f80bd7 100644 --- a/api/middleware/circuitbreaker.txt +++ b/api/middleware/circuitbreaker.txt @@ -16,6 +16,7 @@ type Breaker struct, not comparable type Breaker struct, unexported fields type Config struct type Config struct, CooldownPeriod time.Duration +type Config struct, CountUnmatched bool type Config struct, ErrorHandler func(*github.com/goceleris/celeris.Context, error) error type Config struct, HalfOpenMax int type Config struct, IsError func(error, int) bool @@ -25,7 +26,7 @@ type Config struct, Skip func(*github.com/goceleris/celeris.Context) bool type Config struct, SkipPaths []string type Config struct, Threshold float64 type Config struct, WindowSize time.Duration -type Config struct, field order Skip, SkipPaths, Threshold, MinRequests, WindowSize, CooldownPeriod, HalfOpenMax, IsError, OnStateChange, ErrorHandler +type Config struct, field order Skip, SkipPaths, Threshold, MinRequests, WindowSize, CooldownPeriod, HalfOpenMax, IsError, OnStateChange, ErrorHandler, CountUnmatched type Config struct, not comparable type State int var ErrServiceUnavailable *github.com/goceleris/celeris.HTTPError diff --git a/middleware/circuitbreaker/circuitbreaker.go b/middleware/circuitbreaker/circuitbreaker.go index 79d0e4c26..2b1d63a2d 100644 --- a/middleware/circuitbreaker/circuitbreaker.go +++ b/middleware/circuitbreaker/circuitbreaker.go @@ -73,6 +73,20 @@ func (b *Breaker) Reset() { } } +// unmatched reports whether no route matched the request: the router answers +// it (a 404, a 405, or the automatic OPTIONS answer) and runs the middleware +// mounted with Server.Use on the way, with the sentinel FullPath documented on +// [celeris.Context.FullPath]. Such a request reaches no downstream, so it is +// neither refused nor counted: a 404 for /favicon.ico is no sign that the +// downstream recovered, and must not close a HalfOpen breaker (celeris#888). +func unmatched(c *celeris.Context) bool { + switch c.FullPath() { + case "", "", "": + return true + } + return false +} + // New creates a circuit breaker middleware with the given config. func New(config ...Config) celeris.HandlerFunc { mw, _ := NewWithBreaker(config...) @@ -103,9 +117,10 @@ func NewWithBreaker(config ...Config) (celeris.HandlerFunc, *Breaker) { } errHandler := cfg.ErrorHandler + countUnmatched := cfg.CountUnmatched handler := func(c *celeris.Context) error { - if skip.ShouldSkip(c) { + if skip.ShouldSkip(c) || (!countUnmatched && unmatched(c)) { return c.Next() } diff --git a/middleware/circuitbreaker/config.go b/middleware/circuitbreaker/config.go index c1bfc2303..aef6236b9 100644 --- a/middleware/circuitbreaker/config.go +++ b/middleware/circuitbreaker/config.go @@ -73,6 +73,15 @@ type Config struct { // ErrorHandler is called to produce a response when the breaker is open. // Default: returns ErrServiceUnavailable (503). ErrorHandler func(c *celeris.Context, err error) error + + // CountUnmatched treats a request no route matched (FullPath + // "", "" or "") like any other: + // refused while Open, counted, and taken for the HalfOpen probe. Set it + // when a [celeris.Server.NotFound] or [celeris.Server.MethodNotAllowed] + // handler, or a middleware mounted with Server.Use that serves its own + // paths, forwards to the downstream the breaker protects. Default false: + // such a request is passed on untouched (celeris#888). + CountUnmatched bool } // ErrServiceUnavailable aliases [celeris.ErrServiceUnavailable] so diff --git a/middleware/circuitbreaker/doc.go b/middleware/circuitbreaker/doc.go index b74bc24d8..af2ed7253 100644 --- a/middleware/circuitbreaker/doc.go +++ b/middleware/circuitbreaker/doc.go @@ -9,6 +9,15 @@ // half-open probe that panics reopens the breaker); the panic continues to // the recovery middleware. // +// A request no route matched (the router answers it with a 404, a 405 or the +// automatic OPTIONS answer, and runs the middleware mounted with Server.Use +// on the way) reaches no downstream: it is passed on untouched, neither +// refused while the breaker is Open nor counted, so a stray request for +// /favicon.ico cannot close a HalfOpen breaker. When such a request does reach +// the protected downstream (a Server.NotFound handler that forwards to a +// legacy backend, a Use-mounted middleware serving its own paths), set +// [Config.CountUnmatched] to refuse and count it like any other. +// // Use [New] to create middleware with default settings (50% threshold, // minimum 10 requests, 10 s window, 30 s cooldown). Pass a [Config] to // tune [Config.Threshold], [Config.MinRequests], [Config.WindowSize], diff --git a/middleware/circuitbreaker/unmatched_888_test.go b/middleware/circuitbreaker/unmatched_888_test.go new file mode 100644 index 000000000..90caba3ab --- /dev/null +++ b/middleware/circuitbreaker/unmatched_888_test.go @@ -0,0 +1,157 @@ +package circuitbreaker + +import ( + "context" + "io" + "net" + "net/http" + "testing" + "time" + + "github.com/goceleris/celeris" +) + +// TestUnmatchedRequestsDoNotCountAgainstTheBreaker888: a breaker mounted with +// Server.Use also sees the requests no route matched (a crawler asking for +// /favicon.ico, a method a path does not take, the automatic OPTIONS answer). +// They reach no downstream, and a 404 is not a failure under the default +// IsError, so while the breaker was HalfOpen (or Open past its cooldown) the +// first of them was taken for the probe, counted a success, and closed a +// breaker whose downstream had not been asked. +func TestUnmatchedRequestsDoNotCountAgainstTheBreaker888(t *testing.T) { + mw, brk := NewWithBreaker(Config{ + Threshold: 0.5, + MinRequests: 2, + WindowSize: time.Minute, + CooldownPeriod: 50 * time.Millisecond, + HalfOpenMax: 1, + }) + s := celeris.New(celeris.Config{Engine: celeris.Std}) + s.Use(mw) + s.GET("/boom", func(*celeris.Context) error { return celeris.NewHTTPError(500, "down") }) + s.GET("/ok", func(c *celeris.Context) error { return c.String(200, "ok") }) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- s.StartWithListenerAndContext(ctx, ln) }() + defer func() { + cancel() + select { + case err := <-done: + if err != nil { + t.Logf("server stopped: %v", err) + } + case <-time.After(10 * time.Second): + t.Error("server did not stop within 10s") + } + }() + base := "http://" + ln.Addr().String() + cl := &http.Client{Timeout: 5 * time.Second} + do := func(method, path string) int { + t.Helper() + var resp *http.Response + var err error + for deadline := time.Now().Add(5 * time.Second); ; { + req, _ := http.NewRequest(method, base+path, nil) + resp, err = cl.Do(req) + if err == nil || time.Now().After(deadline) { + break + } + time.Sleep(20 * time.Millisecond) + } + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + return resp.StatusCode + } + + if st := do("GET", "/ok"); st != 200 { + t.Fatalf("GET /ok: %d", st) + } + do("GET", "/boom") + do("GET", "/boom") + if brk.State() != Open { + t.Fatalf("after two failures: %s, want open", brk.State()) + } + time.Sleep(80 * time.Millisecond) // past the cooldown + + for _, rq := range []struct { + method, path string + want int + }{ + {"GET", "/favicon.ico", 404}, // + {"DELETE", "/ok", 405}, // + {"OPTIONS", "/ok", 200}, // + } { + if st := do(rq.method, rq.path); st != rq.want { + t.Errorf("%s %s: %d, want %d", rq.method, rq.path, st, rq.want) + } + if got := brk.State(); got != Open { + t.Fatalf("after %s %s: breaker %s, want it still open (the request reached no downstream)", rq.method, rq.path, got) + } + } + // A request the downstream answers is the probe. + if st := do("GET", "/ok"); st != 200 { + t.Errorf("probe GET /ok: %d, want 200", st) + } + // The std engine flushes a c.String reply to the client from inside the + // handler, before the breaker middleware regains control after Next and + // records the probe's result, so the client can see the 200 and read the + // state a moment before the HalfOpen -> Closed transition. Wait for it. + // (A failed probe is not racy this way: the error is returned up the chain + // and answered after the breaker has settled.) The checks inside the loop + // above stay immediate: an unmatched request is never counted, so it + // neither starts nor settles a probe, and the Open state only changes when + // a counted request arrives. + deadline := time.Now().Add(5 * time.Second) + for brk.State() != Closed && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + if got := brk.State(); got != Closed { + t.Errorf("after a successful probe: %s, want closed", got) + } +} + +// TestCountUnmatchedRefusesFallback888: with CountUnmatched a NotFound handler +// that forwards to the protected downstream is refused while Open. +func TestCountUnmatchedRefusesFallback888(t *testing.T) { + mw, brk := NewWithBreaker(Config{CountUnmatched: true, MinRequests: 1, Threshold: 0.5, CooldownPeriod: time.Hour}) + s := celeris.New(celeris.Config{Engine: celeris.Std}) + s.Use(mw) + s.NotFound(func(c *celeris.Context) error { return celeris.NewHTTPError(502, "upstream down") }) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = s.StartWithListenerAndContext(ctx, ln) }() + get := func() int { + for deadline := time.Now().Add(5 * time.Second); ; { + resp, err := http.Get("http://" + ln.Addr().String() + "/legacy") + if err == nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + return resp.StatusCode + } + if time.Now().After(deadline) { + t.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + } + } + if st := get(); st != 502 { + t.Fatalf("first: %d", st) + } + if brk.State() != Open { + t.Fatalf("state %s, want open", brk.State()) + } + if st := get(); st != 503 { + t.Errorf("while open: %d, want 503", st) + } +}