From 6c95fa8b4dc37f18c85bd41f7589e760df545bde Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 31 Oct 2026 20:57:29 +0100 Subject: [PATCH 1/3] epoll: tests that an IdleTimeout, ReadTimeout and WriteTimeout of 1 s close on time with ReadHeaderTimeout off, and that the sweep timer ticks do not restart the idle backoff (Refs #1194) RED on main: with ReadHeaderTimeout -1 the engine's 25 ms timerfd does not exist, and a 1 s IdleTimeout closes an idle keep-alive connection at 3.2 to 5.8 s (docker arm64, 2 CPUs), a stalled upload at ReadTimeout + 4 s, a non-reading peer at 4 to 5 s instead of 2 s. --- .../epoll/fine_sweep_1194_linux_test.go | 185 ++++++++++++++++++ .../epoll/sweep_tick_idle_1194_linux_test.go | 51 +++++ 2 files changed, 236 insertions(+) create mode 100644 internal/engine/epoll/fine_sweep_1194_linux_test.go create mode 100644 internal/engine/epoll/sweep_tick_idle_1194_linux_test.go diff --git a/internal/engine/epoll/fine_sweep_1194_linux_test.go b/internal/engine/epoll/fine_sweep_1194_linux_test.go new file mode 100644 index 000000000..05da6c08f --- /dev/null +++ b/internal/engine/epoll/fine_sweep_1194_linux_test.go @@ -0,0 +1,185 @@ +//go:build linux + +package epoll + +import ( + "bufio" + "context" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "strconv" + "testing" + "time" + + "github.com/goceleris/celeris/internal/protocol/h2/stream" + "github.com/goceleris/celeris/internal/resource" +) + +// celeris#1194. The sweep (checkTimeouts) is what closes a connection that ran +// past its IdleTimeout, ReadTimeout or WriteTimeout, and its 25 ms kernel timer +// (the timerfd) used to exist only with ReadHeaderTimeout set. With +// ReadHeaderTimeout -1 an idle loop swept once per 1024 returns of epoll_wait, +// which with the listener open settles at a 4 ms wait: a sweep every 4 to 5 s +// (measured: a 1 s IdleTimeout closed the connection at 3.2 to 5.8 s; with no +// listener the wait is 500 ms and a sweep comes every 8 minutes). The tests pin +// the bound the documentation promises with ReadHeaderTimeout off: a timeout of +// 1 s closes at 1 s plus one sweep period (25 ms), and these windows carry the +// scheduling jitter of a loaded runner on top. + +// A connection is closed no sooner than slack1194 before its due time (the +// client's clock starts after the server's stamp), and no later than lateness1194 +// after it. The sweep is 25 ms apart; the lateness measured on a container of 2 +// CPUs is 0 to 30 ms (see the pull request), and the window is the 100 ms the +// bound of the documentation allows plus margin for a loaded runner. +const ( + slack1194 = 50 * time.Millisecond + lateness1194 = 200 * time.Millisecond +) + +// handler1194 answers every request with "ok", and /big with a body that no +// socket buffer of a loopback connection holds. +type handler1194 struct{ big []byte } + +func (h handler1194) HandleStream(_ context.Context, s *stream.Stream) error { + if s.ResponseWriter == nil { + return nil + } + body := []byte("ok") + if s.Path == "/big" { + body = h.big + } + return s.ResponseWriter.WriteResponse(s, 200, + [][2]string{{"content-type", "application/octet-stream"}, {"content-length", strconv.Itoa(len(body))}}, body) +} + +// start1194 starts an engine with ReadHeaderTimeout off, so that no timeout +// but the one a test sets is enabled (the others are at their defaults, 60 s +// and 600 s), and returns its address and a channel that receives the time of +// every connection it closes. +func start1194(t *testing.T, mut func(*resource.Config)) (*Engine, string, <-chan time.Time) { + t.Helper() + closed := make(chan time.Time, 8) + eng, addr := startE2E885(t, false, handler1194{big: make([]byte, 32<<20)}, func(c *resource.Config) { + c.ReadHeaderTimeout = -1 + c.Logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + c.Resources.SocketSend = 16 << 10 + c.OnDisconnect = func(string) { + select { + case closed <- time.Now(): + default: + } + } + mut(c) + }) + return eng, addr, closed +} + +// closeWithin1194 waits for the engine to close the connection and asserts it +// did so within [earliest-slack, latest+lateness] of t0. +func closeWithin1194(t *testing.T, what string, closed <-chan time.Time, t0 time.Time, earliest, latest time.Duration) { + t.Helper() + select { + case at := <-closed: + got := at.Sub(t0) + t.Logf("%s: closed after %v", what, got.Round(time.Millisecond)) + if got < earliest-slack1194 { + t.Errorf("%s: closed after %v, want no sooner than %v (a timeout that fired early)", what, got.Round(time.Millisecond), earliest-slack1194) + } + if got > latest+lateness1194 { + t.Errorf("%s: closed after %v, want within %v of %v (a sweep that waits for the next event or the 500 ms wait)", + what, got.Round(time.Millisecond), lateness1194, latest) + } + case <-time.After(latest + 15*time.Second): + t.Fatalf("%s: not closed %v after the start, due by %v", what, time.Since(t0).Round(time.Millisecond), latest) + } +} + +// An idle keep-alive connection is closed at IdleTimeout (1 s). +func TestIdleTimeoutIsOnTimeWithoutReadHeaderTimeout1194(t *testing.T) { + _, addr, closed := start1194(t, func(c *resource.Config) { c.IdleTimeout = time.Second }) + c, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + _ = c.SetDeadline(time.Now().Add(20 * time.Second)) + if _, err := fmt.Fprint(c, "GET / HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil { + t.Fatal(err) + } + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + if err != nil { + t.Fatalf("first response: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + closeWithin1194(t, "idle keep-alive conn (IdleTimeout 1 s)", closed, time.Now(), time.Second, time.Second) +} + +// A request body that stops arriving is cut at ReadTimeout (1 s) from the last +// byte. +func TestReadTimeoutIsOnTimeWithoutReadHeaderTimeout1194(t *testing.T) { + _, addr, closed := start1194(t, func(c *resource.Config) { c.ReadTimeout = time.Second }) + c, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + _ = c.SetDeadline(time.Now().Add(20 * time.Second)) + if _, err := fmt.Fprint(c, "POST /up HTTP/1.1\r\nHost: x\r\nContent-Length: 1000\r\n\r\n"+string(make([]byte, 100))); err != nil { + t.Fatal(err) + } + closeWithin1194(t, "stalled upload (ReadTimeout 1 s)", closed, time.Now(), time.Second, time.Second) +} + +// A peer that takes nothing of a response it asked for is closed by the stall +// rule of WriteTimeout (1 s): the first sweep that finds the response parked +// takes the baseline, and the window ends at the first sweep a WriteTimeout +// later. The bytes the peer's kernel acknowledges in the first window (more +// than the rule's floor of 64 KiB on a loopback connection) pass it, and the +// close comes at the end of the second: 2.00 to 2.07 s measured with the sweep +// every 25 ms (4.0 to 4.4 s without), 2.06 to 2.17 s under -race, where the +// baseline sweep comes later, and 1.24 s once, with the first window failed +// (the peer's kernel acknowledged less than the floor). The window runs from +// one WriteTimeout to two and 300 ms more than the other tests allow, the +// stall rule's windows being measured from a sweep, not from the request. +func TestWriteTimeoutIsOnTimeWithoutReadHeaderTimeout1194(t *testing.T) { + _, addr, closed := start1194(t, func(c *resource.Config) { c.WriteTimeout = time.Second }) + c, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + _ = c.SetDeadline(time.Now().Add(20 * time.Second)) + t0 := time.Now() + if _, err := fmt.Fprint(c, "GET /big HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil { + t.Fatal(err) + } + closeWithin1194(t, "non-reading peer (WriteTimeout 1 s)", closed, t0, time.Second, 2*time.Second+300*time.Millisecond) +} + +// Every timeout rides the sweep, so the sweep has its kernel timer whatever +// the timeouts are, ReadHeaderTimeout included: a loop with all four off still +// has the WebSocket idle deadline of the middleware, the HTTP/2 deadlines and +// the closing drain to reap. +func TestEveryLoopHasASweepTimer1194(t *testing.T) { + for _, tc := range []struct { + name string + mut func(*resource.Config) + }{ + {"readheader-off", func(*resource.Config) {}}, + {"all-off", func(c *resource.Config) { c.ReadTimeout, c.WriteTimeout, c.IdleTimeout = -1, -1, -1 }}, + {"readheader-default", func(c *resource.Config) { c.ReadHeaderTimeout = 0 }}, + } { + t.Run(tc.name, func(t *testing.T) { + eng, _, _ := start1194(t, tc.mut) + for i := 0; i < eng.NumWorkers(); i++ { + if l := eng.WorkerLoop(i).(*Loop); l.timerFD < 0 { + t.Errorf("loop %d has no sweep timerfd", i) + } + } + }) + } +} diff --git a/internal/engine/epoll/sweep_tick_idle_1194_linux_test.go b/internal/engine/epoll/sweep_tick_idle_1194_linux_test.go new file mode 100644 index 000000000..f540a16ce --- /dev/null +++ b/internal/engine/epoll/sweep_tick_idle_1194_linux_test.go @@ -0,0 +1,51 @@ +//go:build linux + +package epoll + +import ( + "sync" + "testing" + "time" + + "github.com/goceleris/celeris/internal/resource" +) + +// celeris#1194. The sweep timer ticks every 25 ms on every loop, and a tick is +// an event of the epoll_wait it ends. Counted as activity it reset the adaptive +// backoff of the wait (adaptiveTimeoutMs) each time, so an idle loop never got +// past a few waits of 2 ms: 675 wakeups a second, against 215 for a loop with +// no timer. A round whose only event is the tick is an idle round now. +// +// The sweep runs on the loop's thread right at the tick, before the round is +// counted, so the counter it sees is the idle rounds since the last activity: +// a few (under 20) when each tick starts the backoff over, hundreds when the +// loop settles into its longest wait (above 100 rounds). +func TestSweepTickAloneIsAnIdleRound1194(t *testing.T) { + var ( + mu sync.Mutex + most = map[*Loop]uint32{} + ) + hook := func(l *Loop) { + mu.Lock() + if l.consecutiveEmpty > most[l] { + most[l] = l.consecutiveEmpty + } + mu.Unlock() + } + testHookCheckTimeouts.Store(&hook) + t.Cleanup(func() { testHookCheckTimeouts.Store(nil) }) + + eng, _, _ := start1194(t, func(c *resource.Config) { c.ReadHeaderTimeout = 0 }) + time.Sleep(1500 * time.Millisecond) + + for i := 0; i < eng.NumWorkers(); i++ { + l := eng.WorkerLoop(i).(*Loop) + mu.Lock() + got := most[l] + mu.Unlock() + t.Logf("loop %d: at most %d idle rounds in a row at a tick of the sweep timer in 1.5 s", i, got) + if got <= 100 { + t.Errorf("loop %d: %d idle rounds in a row at a tick, want more than 100 (the longest wait): the tick of the sweep timer starts the backoff of an idle loop over", i, got) + } + } +} From a6f4ca23e171566284b70d6b30ac2ae2989583dd Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 31 Oct 2026 20:58:20 +0100 Subject: [PATCH 2/3] epoll: the 25 ms sweep timer is armed on every loop, whatever the timeouts are; its tick does not restart the idle backoff (Refs #1194) The timerfd existed only with ReadHeaderTimeout set. With it off, every deadline that rides the sweep (IdleTimeout, ReadTimeout, WriteTimeout, the closing drain, the HTTP/2 deadlines, a detached connection's idle deadline) was seen once per 1024 returns of epoll_wait, 4 to 5 s on an idle loop with its listener open. The timerfd, the sweep gate (every 32 iterations) and the epoll_wait cap (the sweep period) no longer depend on the configuration: a loop sweeps like the default configuration's. A tick of the timerfd as the only event of a round is an idle round for the adaptive backoff of the wait. Counted as activity it restarted the backoff every 25 ms and kept an idle loop at 675 wakeups a second (215 without the timer); with this it settles at the 4 ms wait it has without one. IdleTimeout, ReadTimeout and WriteTimeout document the bound; the tests that used ReadHeaderTimeout -1 as 'no timerfd' disarm the timerfd by hand. --- config.go | 21 +++- .../checktimeouts_floor_885_linux_test.go | 46 +++++-- .../consecutive_empty_queue_885_linux_test.go | 12 +- internal/engine/epoll/engine.go | 16 +-- internal/engine/epoll/loop.go | 112 ++++++++++-------- .../engine/epoll/pause_wake_886_linux_test.go | 24 ++-- .../epoll/resume_wake_886_linux_test.go | 51 +++++--- .../epoll/timeout_sentinel_linux_test.go | 40 ++----- .../sse/sse_nonreading_888_linux_test.go | 12 +- 9 files changed, 197 insertions(+), 137 deletions(-) diff --git a/config.go b/config.go index 11ac70efd..c9108fa96 100644 --- a/config.go +++ b/config.go @@ -236,7 +236,9 @@ type Config struct { WorkloadHint WorkloadHint // ReadTimeout is the max duration for reading the entire request. - // Zero uses the default (60s). Set to -1 for no timeout. + // Zero uses the default (60s). Set to -1 for no timeout. The native engines + // enforce it in a sweep of their connections, about 30 ms after its time on + // epoll (see IdleTimeout). // // On the native engines (epoll, io_uring, adaptive) it is measured from the // connection's last read (on io_uring also from the start and the completion @@ -357,10 +359,9 @@ type Config struct { // WriteTimeout with that in mind. WriteTimeout -1 is no write timeout at // any rate. What the client sends is not progress: a pipelined request or // an HTTP/2 PING does not hold a parked response. The times above hold - // with ReadHeaderTimeout set (the default): the engine's 25 ms sweep timer - // is armed only then, and without it a stalled response can outlast them by - // seconds (measured: WriteTimeout 1s, ReadHeaderTimeout -1, closed at 5 s, - // on main too). + // with the epoll sweep at its 25 ms period, which every worker has whatever + // ReadHeaderTimeout and the other timeouts are; io_uring sweeps more coarsely, + // see IdleTimeout. // // While the rule has a baseline for a parked response (from the first sweep // that finds it parked), it is the only bound on the connection: @@ -516,6 +517,16 @@ type Config struct { // failed getsockopt) the period is always extended. A frame for a stream the server reset moves the period only for a // request that expired at ReadTimeout, for the 2 x ReadTimeout its late DATA may // take. On epoll, for a response parked on the socket, see WriteTimeout. + // + // The native engines look at IdleTimeout, ReadTimeout and WriteTimeout in a + // sweep of their connections, so one is enforced a little after its time, not + // at it. On epoll the sweep runs every 25 ms, from a kernel timer each worker + // arms whatever ReadHeaderTimeout and the timeouts are, so a timeout is + // enforced within about 30 ms of its time (a 1 s IdleTimeout closed an idle + // connection at 1.00 to 1.03 s on docker arm64 with 2 CPUs, with + // ReadHeaderTimeout off as well as set). On io_uring the sweep of an idle + // worker is about a second apart, so a timeout of a few seconds or less is + // approximate by that much there. IdleTimeout time.Duration // ShutdownTimeout is the deadline of the graceful shutdown that cancelling // the context of StartWithContext or StartWithListenerAndContext starts diff --git a/internal/engine/epoll/checktimeouts_floor_885_linux_test.go b/internal/engine/epoll/checktimeouts_floor_885_linux_test.go index abb7536cf..359aafdfe 100644 --- a/internal/engine/epoll/checktimeouts_floor_885_linux_test.go +++ b/internal/engine/epoll/checktimeouts_floor_885_linux_test.go @@ -14,6 +14,8 @@ import ( "testing" "time" + "golang.org/x/sys/unix" + "github.com/goceleris/celeris/internal/conn" "github.com/goceleris/celeris/internal/engine" "github.com/goceleris/celeris/internal/mono" @@ -22,8 +24,8 @@ import ( ) // celeris#885 item 29: checkTimeouts is driven by the loop's iterations (every -// 32 epoll_wait returns when ReadHeaderTimeout is set, which is the default, or -// a conn is detached), so a busy loop swept thousands of times a second where +// 32 epoll_wait returns, since celeris#1194; before, when ReadHeaderTimeout was +// set, which is the default, or a conn was detached), so a busy loop swept thousands of times a second where // the 25 ms timerfd promises 40 (measured: 5,400 to 6,200 a second in all on // two loops with sixteen busy clients, 12.9% of a core on 8192 idle conns). // The sweep now has a floor between two of them (sweepFloor), outside @@ -32,8 +34,8 @@ import ( // Two tests. The first counts the sweeps one busy loop makes in a window, under // sixteen busy keep-alive clients and the default configuration; it fails on a // loop whose gate has no floor. The second is the price of the floor and the -// guard against a floor set too high: with the timerfd off (ReadHeaderTimeout -// disabled) the gate is the only source of sweeps on a busy loop, and a +// guard against a floor set too high: with the timerfd disarmed +// (every loop has one since celeris#1194) the gate is the only source of sweeps on a busy loop, and a // detached conn's idle deadline (a WebSocket's, an SSE stream's) must still be // reaped about when it expires. @@ -41,6 +43,12 @@ import ( type sweepProbe885 struct { addr string + // disarm, when set, disarms the timerfd of every loop at its first sweep, + // so that the gate is the only source of sweeps left (every loop has a + // timerfd since celeris#1194, whatever its ReadHeaderTimeout). + disarm bool + disarmed map[*Loop]bool // guarded by mu + mu sync.Mutex at map[*Loop][]int64 // UnixNano of each sweep, by loop live map[*Loop]int // the conns a loop had at its last sweep @@ -59,7 +67,14 @@ func (p *sweepProbe885) hook(l *Loop) { p.at[l] = append(p.at[l], mono.Now()) p.live[l] = len(l.liveConns) p.timerFD, p.sawLoop = l.timerFD, true + first := p.disarm && !p.disarmed[l] + if first { + p.disarmed[l] = true + } p.mu.Unlock() + if first && l.timerFD >= 0 { + _ = unix.TimerfdSettime(l.timerFD, 0, &unix.ItimerSpec{}, nil) + } for _, cs := range l.liveConns { if h1 := cs.h1State; h1 != nil && h1.Detached.Load() && !p.seen[cs] { p.seen[cs] = true @@ -136,6 +151,12 @@ var _ stream.AsyncRouteResolver = floorHandler885{} // startFloorEngine885 serves cfg on two loops (the least the engine accepts) // and returns the address and the probe. The engine is stopped, and Listen waited for, at cleanup. func startFloorEngine885(t *testing.T, cfg resource.Config) (string, *sweepProbe885) { + return startFloorEngine885x(t, cfg, false) +} + +// startFloorEngine885x is startFloorEngine885 with the timerfds disarmed when +// disarm is set (see sweepProbe885). +func startFloorEngine885x(t *testing.T, cfg resource.Config, disarm bool) (string, *sweepProbe885) { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -150,7 +171,7 @@ func startFloorEngine885(t *testing.T, cfg resource.Config) (string, *sweepProbe if err != nil { t.Fatalf("epoll engine: %v", err) // not a skip: a skip would take the witness out of CI silently } - p := &sweepProbe885{addr: addr, at: map[*Loop][]int64{}, live: map[*Loop]int{}, + p := &sweepProbe885{addr: addr, disarm: disarm, disarmed: map[*Loop]bool{}, at: map[*Loop][]int64{}, live: map[*Loop]int{}, detached: make(chan *conn.H1State, 16), seen: map[*connState]bool{}} hook := p.hook testHookCheckTimeouts.Store(&hook) @@ -292,16 +313,16 @@ func TestBusyLoopSweepsAtMostOnceAFloorPeriod885(t *testing.T) { } // TestDetachedIdleDeadlineIsReapedPromptlyUnderLoad885 is the price of the -// floor. ReadHeaderTimeout is disabled, so there is no timerfd and the gate -// (every 32 iterations while a conn is detached) is the only sweep on a busy -// loop. Sixteen clients keep the loop busy; a detached conn (a WebSocket's, +// floor. The timerfd is disarmed (every loop has one since celeris#1194, +// whatever its ReadHeaderTimeout), so the gate (every 32 iterations) is the only +// sweep on a busy loop. Sixteen clients keep the loop busy; a detached conn (a WebSocket's, // SSE's) gets an idle deadline 200 ms ahead and must be reaped (within 400 ms // of it, a sanity bound). The median gap between two sweeps is the structural // bound, and does not depend on where in a gap the deadline falls: a floor of // 250 ms moves every gap past it (the control of that mutant), where a floor // of the timerfd's 25 ms leaves the gaps at the time of 32 iterations. func TestDetachedIdleDeadlineIsReapedPromptlyUnderLoad885(t *testing.T) { - addr, p := startFloorEngine885(t, resource.Config{ReadHeaderTimeout: -1, AsyncHandlers: true}) + addr, p := startFloorEngine885x(t, resource.Config{ReadHeaderTimeout: -1, AsyncHandlers: true}, true) dc, err := net.DialTimeout("tcp", addr, 3*time.Second) if err != nil { t.Fatal(err) @@ -373,8 +394,11 @@ func TestDetachedIdleDeadlineIsReapedPromptlyUnderLoad885(t *testing.T) { t.Logf("celeris885 item 29: detached conn reaped %v after its deadline; %d sweeps of its loop in %v, gaps median %v max %v, %d requests in all; timerFD=%d", late.Round(time.Millisecond), len(w), to.Sub(from).Round(time.Millisecond), medGap.Round(time.Millisecond), maxGap.Round(time.Millisecond), reqs, tfd) - if !saw || tfd >= 0 { - t.Fatalf("PREMISE: the loop has a timerfd (%d, saw %v): ReadHeaderTimeout -1 disables it, and without that the gate is not the only sweep", tfd, saw) + p.mu.Lock() + off := p.disarmed[detLoop] + p.mu.Unlock() + if !saw || !off { + t.Fatalf("PREMISE: the timerfd of the detached conn's loop was not disarmed (timerFD %d, saw %v, disarmed %v): without that the gate is not the only sweep", tfd, saw, off) } if reqs < 1000 { t.Fatalf("PREMISE: the loop was not busy: %d requests in %v", reqs, to.Sub(from)) diff --git a/internal/engine/epoll/consecutive_empty_queue_885_linux_test.go b/internal/engine/epoll/consecutive_empty_queue_885_linux_test.go index dd413f734..4666575c8 100644 --- a/internal/engine/epoll/consecutive_empty_queue_885_linux_test.go +++ b/internal/engine/epoll/consecutive_empty_queue_885_linux_test.go @@ -30,15 +30,19 @@ const sndbuf885x42 = 16 << 20 // gives a few hundred KiB), so the loop reads it over several rounds: a turn is // driverReadBudget reads of driverReadBufSize (256 KiB), and every round after // the first is an epoll_wait(0) that collects nothing. The engine has -// ReadHeaderTimeout -1, so no 25 ms timerfd can end the flood's counting between -// two rounds. On the last read the count is 0 (round 1 had an event), and with +// ReadHeaderTimeout -1 and its 25 ms timerfd disarmed, so that nothing can end +// the flood's counting between two rounds. On the last read the count is 0 (round 1 had an event), and with // more than eleven turns the wait the loop would make next (adaptiveTimeoutMs) // is the base, 1 ms: the rounds a queue forced to zero are not idle rounds. func TestQueuedZeroTimeoutRoundsDoNotStretchTheIdleWait885(t *testing.T) { eng, _ := startE2E885(t, false, newPathHandler885(1), func(c *resource.Config) { c.ReadHeaderTimeout = -1 }) l := eng.WorkerLoop(0).(*Loop) - if l.timerFD >= 0 { - t.Fatalf("apparatus: the loop has a sweep timerfd, which can fire between two rounds of the flood") + if l.timerFD < 0 { + t.Fatalf("apparatus: the loop has no sweep timerfd to disarm (every loop has one, celeris#1194)") + } + // The timerfd can fire between two rounds of the flood: disarm it. + if err := unix.TimerfdSettime(l.timerFD, 0, &unix.ItimerSpec{}, nil); err != nil { + t.Fatalf("apparatus: disarm the sweep timerfd: %v", err) } a, peer := socketpairNonblocking(t) t.Cleanup(func() { _ = unix.Close(a); _ = unix.Close(peer) }) diff --git a/internal/engine/epoll/engine.go b/internal/engine/epoll/engine.go index a00f6a261..ae03afa1b 100644 --- a/internal/engine/epoll/engine.go +++ b/internal/engine/epoll/engine.go @@ -447,14 +447,14 @@ func (e *Engine) ResumeAccept() error { l.listenFDClosed.Store(false) l.wakeIfSuspended() // A loop that has not parked closed its listener at the pause and - // went back to epoll_wait, which is 500 ms long with no listener, - // and a loop without a header deadline has no 25 ms timerfd to end - // it sooner. That is a loop that holds a connection, and also an - // idle one just after the pause (it parks at the iteration after - // the close). wakeIfSuspended does not reach it: the wake eventfd, - // which is in the epoll set, does (celeris#886). A leaf lock and one - // write(2); a loop that is running just reads the counter at its - // next wakeup. + // went back to epoll_wait, which is 500 ms long with no listener; + // its 25 ms timerfd (every loop has one since celeris#1194) ends the + // wait sooner, but only at the next tick. That is a loop that holds a + // connection, and also an idle one just after the pause (it parks at + // the iteration after the close). wakeIfSuspended does not reach it: + // the wake eventfd, which is in the epoll set, does (celeris#886). A + // leaf lock and one write(2); a loop that is running just reads the + // counter at its next wakeup. l.wakeFD.Signal() } return nil diff --git a/internal/engine/epoll/loop.go b/internal/engine/epoll/loop.go index 7ae96a724..de9744fe3 100644 --- a/internal/engine/epoll/loop.go +++ b/internal/engine/epoll/loop.go @@ -301,7 +301,7 @@ type Loop struct { detachQMu sync.Mutex detachQSpare []*connState // reuse slice to avoid alloc in drainDetachQueue detachQPending atomic.Int32 // 1 when detachQueue has entries; gates the hot-path drain - detachedCount int // number of currently-detached conns; gates idle-deadline sweep + detachedCount int // number of currently-detached conns (accounting; the sweep cadence no longer reads it, celeris#1194) // adoptQueue holds io_uring→epoll transplant hand-offs (#383 reverse // direction): real connected fds to adopt onto this loop. Cross-thread: @@ -533,26 +533,35 @@ func (l *Loop) run(ctx context.Context) { // slowloris walker's 2s slack budget for slow refapps. Mirrors // the kernel-precision that iouring gets from per-conn // IORING_OP_TIMEOUT SQEs. - if l.cfg.ReadHeaderTimeout > 0 { - tfd, tfdErr := unix.TimerfdCreate(unix.CLOCK_MONOTONIC, unix.TFD_NONBLOCK|unix.TFD_CLOEXEC) - if tfdErr == nil { - // sweepFloor periodic (25 ms); first fire at the same. - spec := unix.ItimerSpec{ - Interval: unix.Timespec{Sec: 0, Nsec: int64(sweepFloor)}, - Value: unix.Timespec{Sec: 0, Nsec: int64(sweepFloor)}, - } - if err := unix.TimerfdSettime(tfd, 0, &spec, nil); err != nil { - _ = unix.Close(tfd) - tfd = -1 - } else if err := unix.EpollCtl(epollFD, unix.EPOLL_CTL_ADD, tfd, &unix.EpollEvent{ - Events: unix.EPOLLIN, - Fd: int32(tfd), - }); err != nil { - _ = unix.Close(tfd) - tfd = -1 - } - l.timerFD = tfd + // + // Armed whatever the timeouts are (celeris#1194). It was armed only with + // ReadHeaderTimeout set, and every other deadline that rides the sweep + // (IdleTimeout, ReadTimeout, WriteTimeout, the closing drain, the HTTP/2 + // deadlines, a detached connection's idle deadline) was then seen once per + // 1024 returns of epoll_wait: an idle loop with its listener open settles + // at a 4 ms wait, so a sweep every 4 to 5 s (a 1 s IdleTimeout closed at 3.2 + // to 5.8 s), and with no listener (a pause, a drain) at a 500 ms wait, so a + // sweep every 8 minutes. The default configuration (ReadHeaderTimeout 10 s) + // has always armed it, so this changes the loop only for a server that + // turned ReadHeaderTimeout off; see sweepFloor for what it costs. + tfd, tfdErr := unix.TimerfdCreate(unix.CLOCK_MONOTONIC, unix.TFD_NONBLOCK|unix.TFD_CLOEXEC) + if tfdErr == nil { + // sweepFloor periodic (25 ms); first fire at the same. + spec := unix.ItimerSpec{ + Interval: unix.Timespec{Sec: 0, Nsec: int64(sweepFloor)}, + Value: unix.Timespec{Sec: 0, Nsec: int64(sweepFloor)}, + } + if err := unix.TimerfdSettime(tfd, 0, &spec, nil); err != nil { + _ = unix.Close(tfd) + tfd = -1 + } else if err := unix.EpollCtl(epollFD, unix.EPOLL_CTL_ADD, tfd, &unix.EpollEvent{ + Events: unix.EPOLLIN, + Fd: int32(tfd), + }); err != nil { + _ = unix.Close(tfd) + tfd = -1 } + l.timerFD = tfd } // celeris#639: listenFD is certainly this loop's socket here; after ready @@ -700,6 +709,7 @@ func (l *Loop) run(ctx context.Context) { now = l.cachedNow l.loopSlot.Enter(now) // the batch below runs handlers: a stuck one shows (celeris#1170) } + timerOnly := false // the batch is the sweep timer's tick and nothing else (celeris#1194) for i := range n { ev := &l.events[i] fd := int(ev.Fd) @@ -721,6 +731,7 @@ func (l *Loop) run(ctx context.Context) { var buf [8]byte _, _ = unix.Read(l.timerFD, buf[:]) l.checkTimeouts() + timerOnly = n == 1 continue } @@ -840,8 +851,13 @@ func (l *Loop) run(ctx context.Context) { // Adaptive timeout tracking. A round that collected nothing counts as // an idle one only if the loop was willing to wait: a queued driver // conn forced the timeout to 0, which says nothing about idleness - // (celeris#885 item 42). - if n > 0 { + // (celeris#885 item 42). So does a round whose only event is the tick + // of the sweep timer (celeris#1194): that is the loop waking itself, + // every 25 ms, and counted as activity it started the backoff over + // each time (a wait of 1 ms ten times, then 2 ms, where an idle loop + // settles at 4 ms), which is 675 wakeups a second on an idle loop + // where 215 are enough (measured, docker arm64). + if n > 0 && !timerOnly { l.consecutiveEmpty = 0 } else if owedDriverReads == 0 { l.consecutiveEmpty++ @@ -920,16 +936,16 @@ func (l *Loop) run(ctx context.Context) { // to sweepFloor plus the iterations of one gate stride after it // expires, not 32 iterations after; the same 25 ms the timerfd gives // a header deadline. A clock that stepped back (d < 0) sweeps, or - // the sweep would wait for the clock to catch up. When - // ReadHeaderTimeout is enabled (the v1.4.11 slowloris defence), - // the gate ALSO tightens to 0x1F — the in-process synthetic - // reproducer (test/integration/slowloris_synthetic_test.go) showed - // epoll's HeaderDeadline firing 2s late without this. + // the sweep would wait for the clock to catch up. The gate is every + // 32 iterations whatever the timeouts are (celeris#1194; the v1.4.11 + // slowloris defence tightened it for ReadHeaderTimeout alone, and the + // in-process synthetic reproducer, + // test/integration/slowloris_synthetic_test.go, showed epoll's + // HeaderDeadline firing 2 s late without it): it is a clock read per + // 32 iterations, and the other deadlines that ride the sweep need it + // as much. l.tickCounter++ - gate := uint32(0x3FF) - if l.detachedCount > 0 || l.cfg.ReadHeaderTimeout > 0 { - gate = 0x1F - } + const gate = uint32(0x1F) if l.tickCounter&gate == 0 { // The clock is read at a gate hit only, one iteration in 32. if d := mono.Now() - l.lastSweep; d >= int64(sweepFloor) || d < 0 { @@ -4499,18 +4515,14 @@ func (l *Loop) adaptiveTimeoutMs(base int) int { if len(l.h2Conns) > 0 && l.wakeFD.FD() < 0 { return 1 } - // Cap the timeout when detached conns exist so checkTimeouts can fire - // the WS-supplied IdleDeadline before its expiry. - maxMs := 500 - if l.detachedCount > 0 { - maxMs = 50 - } - // Slowloris defence: cap epoll_wait when ReadHeaderTimeout is enabled - // so checkTimeouts fires the HeaderDeadline reliably under low traffic. - // 25ms × 0x1F gate = 800ms worst-case sweep latency. - if l.cfg.ReadHeaderTimeout > 0 && maxMs > 25 { - maxMs = 25 - } + // The wait never exceeds the sweep period, so that checkTimeouts fires the + // deadlines that ride it (the header deadline, idle, read and write + // timeouts, a detached conn's idle deadline) reliably under low traffic, + // whatever the timeouts are (celeris#1194; the cap was 25 ms with + // ReadHeaderTimeout set, 50 ms with a conn detached, 500 ms otherwise). The + // timerfd ends the wait on its own; the cap is what keeps the bound if the + // timerfd could not be created. + maxMs := int(sweepFloor / time.Millisecond) switch { case l.consecutiveEmpty == 0: return base @@ -4634,12 +4646,14 @@ var testHookCheckTimeouts atomic.Pointer[func(*Loop)] // checkTimeouts scans active connections and closes any that have exceeded // their configured timeout. // -// Cadence: the run loop calls it from the gate every 32 epoll_wait iterations -// when ReadHeaderTimeout is set (the default is 10 s) or a conn is detached, -// otherwise every 1024, but only when sweepFloor (25 ms) has passed since the -// last sweep (lastSweep, stamped here, so the timerfd's sweeps count too); the -// 25 ms timerfd (armed only when ReadHeaderTimeout is set) guarantees one on an -// idle loop. A busy loop therefore sweeps about 40 times a second, where the +// Cadence: the run loop calls it from the gate every 32 epoll_wait iterations, +// but only when sweepFloor (25 ms) has passed since the last sweep (lastSweep, +// stamped here, so the timerfd's sweeps count too); the 25 ms timerfd, which +// every loop arms whatever the timeouts are (celeris#1194; it was armed only with +// ReadHeaderTimeout set, and the gate was every 1024 iterations without one), +// guarantees one on an idle loop. So every deadline that rides the sweep is +// seen within one sweepFloor of its time, plus the scheduling of the loop's +// thread. A busy loop therefore sweeps about 40 times a second, where the // gate alone swept as often as its iterations allowed: measured on 2 loops with // 4096 idle keep-alive conns each, before the floor (celeris PR 969): about 120 // sweeps a second in all with no traffic (the timerfd alone gives 80), and diff --git a/internal/engine/epoll/pause_wake_886_linux_test.go b/internal/engine/epoll/pause_wake_886_linux_test.go index cfa7837a4..856aec885 100644 --- a/internal/engine/epoll/pause_wake_886_linux_test.go +++ b/internal/engine/epoll/pause_wake_886_linux_test.go @@ -49,26 +49,28 @@ func pauseLag886(t *testing.T, e *Engine, port, want int, gaveup time.Duration) // TestBeginPauseAcceptReachesAnIdleLoop886: a pause must close every loop's // listener within lagBound, whatever the loop is waiting for. The default arm -// is the guard that the 25 ms timerfd keeps the lag short; the arm without a -// header deadline has no timerfd and sits in epoll_wait for up to 500 ms, which -// is the one the eventfd signal is for. +// is the guard that the 25 ms timerfd keeps the lag short; the arm with the +// timerfd disarmed (every loop has one since celeris#1194, whatever its +// ReadHeaderTimeout) sits in epoll_wait for up to 500 ms, which is the one the +// eventfd signal is for. func TestBeginPauseAcceptReachesAnIdleLoop886(t *testing.T) { const ( cycles = 8 lagBound = 100 * time.Millisecond ) for _, tc := range []struct { - name string - rht time.Duration - conns int + name string + rht time.Duration + disarm bool + conns int }{ - {"default-read-header-timeout/live-connections", 0, 16}, - {"no-read-header-timeout/live-connections", -1, 16}, - {"default-read-header-timeout/no-connections", 0, 0}, - {"no-read-header-timeout/no-connections", -1, 0}, + {"default-read-header-timeout/live-connections", 0, false, 16}, + {"no-timerfd/live-connections", -1, true, 16}, + {"default-read-header-timeout/no-connections", 0, false, 0}, + {"no-timerfd/no-connections", -1, true, 0}, } { t.Run(tc.name, func(t *testing.T) { - r := start886(t, 2, tc.rht) + r := start886(t, 2, tc.rht, tc.disarm) r.e.mu.Lock() want := len(r.e.loops) r.e.mu.Unlock() diff --git a/internal/engine/epoll/resume_wake_886_linux_test.go b/internal/engine/epoll/resume_wake_886_linux_test.go index 765dc60cf..8422d2d3d 100644 --- a/internal/engine/epoll/resume_wake_886_linux_test.go +++ b/internal/engine/epoll/resume_wake_886_linux_test.go @@ -13,6 +13,8 @@ import ( "testing" "time" + "golang.org/x/sys/unix" + celerisengine "github.com/goceleris/celeris/internal/engine" "github.com/goceleris/celeris/internal/resource" ) @@ -26,10 +28,12 @@ import ( // the iteration after the close). The wake eventfd is armed in every mode, so // ResumeAccept signals it. // -// A loop whose Config has a ReadHeaderTimeout (the default, 10 s) also owns a -// 25 ms timerfd, which ends that wait on its own: the lag is at most 25 ms -// there (measured 18 to 24 ms on origin/main), and only a loop without a header -// deadline (ReadHeaderTimeout < 0) sits out the whole 500 ms. +// Every loop owns a 25 ms timerfd, which ends that wait on its own: the lag is +// at most 25 ms there (measured 18 to 24 ms on origin/main), and only a loop +// whose timerfd is disarmed sits out the whole 500 ms. Until celeris#1194 that +// was a loop without a header deadline (ReadHeaderTimeout < 0), which had no +// timerfd; the tests disarm it by hand now, so that the eventfd signal is still +// what they test. // live886 opens n keep-alive connections to addr, each with one answered // request, so that they are registered with a loop: a loop that holds a @@ -87,9 +91,10 @@ func resumeLag886(t *testing.T, e *Engine, port, want int, gaveup time.Duration) return gaveup } -// start886 is start895 with a ReadHeaderTimeout: a negative one disables the -// header deadline, and with it the loop's 25 ms timerfd. -func start886(t *testing.T, loops int, readHeaderTimeout time.Duration) *run895 { +// start886 is start895 with a ReadHeaderTimeout, and with the loops' 25 ms +// timerfd disarmed when disarmTimer is set (celeris#1194: every loop has one, +// whatever the ReadHeaderTimeout). +func start886(t *testing.T, loops int, readHeaderTimeout time.Duration, disarmTimer bool) *run895 { t.Helper() e, err := New(resource.Config{ Addr: "127.0.0.1:0", @@ -110,6 +115,21 @@ func start886(t *testing.T, loops int, readHeaderTimeout time.Duration) *run895 for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl); { if e.Addr() != nil { t.Cleanup(func() { cancel(); <-done }) + if disarmTimer { + // Every loop's timerfd was created before its ready signal, and + // Addr is stored after every ready: reading timerFD here is + // ordered after it. + e.mu.Lock() + defer e.mu.Unlock() + for _, l := range e.loops { + if l.timerFD < 0 { + t.Fatalf("celeris886 PREMISE: loop %d has no timerfd to disarm", l.id) + } + if err := unix.TimerfdSettime(l.timerFD, 0, &unix.ItimerSpec{}, nil); err != nil { + t.Fatalf("celeris886 PREMISE: disarm timerfd: %v", err) + } + } + } return r } select { @@ -136,17 +156,18 @@ func TestResumeAcceptReachesALoopThatCannotPark886(t *testing.T) { lagBound = 200 * time.Millisecond ) for _, tc := range []struct { - name string - rht time.Duration - conns int + name string + rht time.Duration + disarm bool + conns int }{ - {"default-read-header-timeout/live-connections", 0, 16}, - {"no-read-header-timeout/live-connections", -1, 16}, - {"default-read-header-timeout/no-connections", 0, 0}, - {"no-read-header-timeout/no-connections", -1, 0}, + {"default-read-header-timeout/live-connections", 0, false, 16}, + {"no-timerfd/live-connections", -1, true, 16}, + {"default-read-header-timeout/no-connections", 0, false, 0}, + {"no-timerfd/no-connections", -1, true, 0}, } { t.Run(tc.name, func(t *testing.T) { - r := start886(t, 2, tc.rht) + r := start886(t, 2, tc.rht, tc.disarm) r.e.mu.Lock() want := len(r.e.loops) r.e.mu.Unlock() diff --git a/internal/engine/epoll/timeout_sentinel_linux_test.go b/internal/engine/epoll/timeout_sentinel_linux_test.go index ad26c658b..9667f303b 100644 --- a/internal/engine/epoll/timeout_sentinel_linux_test.go +++ b/internal/engine/epoll/timeout_sentinel_linux_test.go @@ -16,21 +16,23 @@ import ( // documented -1 "no timeout" sentinel must survive the double normalisation // (Server.doPrepare applies WithDefaults, then New applies it again) and reach // the loop config as disabled (0). The loop reads these fields with `> 0` -// guards (loop.go checkTimeouts / the epoll_wait cap), so 0 is "off" and a +// guards (loop.go checkTimeouts and the header deadline), so 0 is "off" and a // reinstated 10s default is an enforced timeout the caller asked to disable. +// (The epoll_wait cap and the sweep gate used to follow ReadHeaderTimeout, and +// this test read the sentinel through them; since celeris#1194 they do not, and +// the fields are read directly.) func TestTimeoutSentinelReachesEngine(t *testing.T) { const explicit = 3 * time.Second cases := []struct { - name string - in time.Duration - wantRead, wantHeader, wantWrite time.Duration - wantIdle time.Duration - wantHeaderDeadlineSweepGateEnabled bool + name string + in time.Duration + wantRead, wantHeader, wantWrite time.Duration + wantIdle time.Duration }{ - {"disabled", -1, 0, 0, 0, 0, false}, - {"unset", 0, 60 * time.Second, 10 * time.Second, 60 * time.Second, 600 * time.Second, true}, - {"explicit", explicit, explicit, explicit, explicit, explicit, true}, + {"disabled", -1, 0, 0, 0, 0}, + {"unset", 0, 60 * time.Second, 10 * time.Second, 60 * time.Second, 600 * time.Second}, + {"explicit", explicit, explicit, explicit, explicit, explicit}, } for _, tc := range cases { @@ -68,26 +70,6 @@ func TestTimeoutSentinelReachesEngine(t *testing.T) { if e.cfg.IdleTimeout != tc.wantIdle { t.Errorf("cfg.IdleTimeout = %v, want %v", e.cfg.IdleTimeout, tc.wantIdle) } - // Consequence (celeris#584 on the epoll side): the - // epoll_wait cap and the checkTimeouts sweep gate are both - // `cfg.ReadHeaderTimeout > 0`. With the sentinel honoured - // the cap is driven by detachedCount alone; with the - // default reinstated it is pinned at 25ms forever. - l := &Loop{cfg: e.cfg, consecutiveEmpty: 200} - idle := l.adaptiveTimeoutMs(100) - l.detachedCount = 1 - detached := l.adaptiveTimeoutMs(100) - if tc.wantHeaderDeadlineSweepGateEnabled { - if idle != 25 || detached != 25 { - t.Errorf("adaptiveTimeoutMs with ReadHeaderTimeout=%v: idle=%dms detached=%dms, want 25/25", - e.cfg.ReadHeaderTimeout, idle, detached) - } - } else { - if idle != 400 || detached != 50 { - t.Errorf("adaptiveTimeoutMs with ReadHeaderTimeout disabled: idle=%dms detached=%dms, want 400/50", - idle, detached) - } - } }) } } diff --git a/middleware/sse/sse_nonreading_888_linux_test.go b/middleware/sse/sse_nonreading_888_linux_test.go index 05f71eb7d..35b770f17 100644 --- a/middleware/sse/sse_nonreading_888_linux_test.go +++ b/middleware/sse/sse_nonreading_888_linux_test.go @@ -108,9 +108,10 @@ type producer888 struct { flood int // writeTimeout is the server's WriteTimeout (nonReadWriteTimeout888 when - // zero). noHeaderTimer sets ReadHeaderTimeout -1, which takes away the - // epoll engine's 25 ms timerfd: with a detached conn the sweep then - // depends on the loop's own iterations. + // zero). noHeaderTimer sets ReadHeaderTimeout -1, which used to take away + // the epoll engine's 25 ms timerfd, so that the sweep depended on the + // loop's own iterations (celeris#1194: every epoll loop has the timerfd + // now; the cell keeps the bound for a server without a header deadline). writeTimeout time.Duration noHeaderTimer bool @@ -320,8 +321,9 @@ func TestNonReadingClientEndsStream888(t *testing.T) { // WriteTimeout 2 s puts the close after the fast phase. {"small-events", producer888{fastBytes: 4 << 10, fastEvery: 500 * time.Microsecond, fastFor: 1200 * time.Millisecond, settle: 100 * time.Millisecond, trickleBytes: 1000, trickleEvery: 300 * time.Millisecond, writeTimeout: 2 * time.Second, dropEarly: true}}, - // The same without the epoll timerfd (ReadHeaderTimeout -1), so nothing - // but the loop's own iterations reaches the sweep. + // The same with ReadHeaderTimeout -1, which used to take the epoll + // timerfd away, so that nothing but the loop's own iterations reached + // the sweep (celeris#1194). {"small-events-no-header-timer", producer888{fastBytes: 4 << 10, fastEvery: 500 * time.Microsecond, fastFor: 1200 * time.Millisecond, settle: 100 * time.Millisecond, trickleBytes: 1000, trickleEvery: 300 * time.Millisecond, writeTimeout: 2 * time.Second, noHeaderTimer: true, dropEarly: true}}, } From 49e12b30183aa00a1ce177ea38e044a744f57eba Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sun, 1 Nov 2026 01:02:09 +0100 Subject: [PATCH 3/3] epoll: a sweep timer that cannot be set up is logged, comments and docs say what bounds the sweep; the write test's window is 3 s (Refs #1194) Review of the PR: the ceiling of the epoll_wait wait does not keep the bound without the timerfd (the backoff tops out at 4 ms, so it never binds); the loop then sweeps from the gate alone, about 130 ms apart with its listener open. The failure of the timerfd's creation, settime or epoll_ctl is logged once per loop, with a test through a seam. config.go: one paragraph on the sweep (ReadTimeout's, merged with the one of the IdleTimeout doc), one io_uring figure, the detached-stream rule says 25 ms on epoll, WriteTimeout is excluded from 'about 30 ms'. loop.go: the stale 'every 1024 iterations' comment. --- config.go | 41 +++++------- .../epoll/fine_sweep_1194_linux_test.go | 66 ++++++++++++++++++- internal/engine/epoll/loop.go | 39 +++++++---- 3 files changed, 105 insertions(+), 41 deletions(-) diff --git a/config.go b/config.go index c63f051aa..837f815bb 100644 --- a/config.go +++ b/config.go @@ -236,9 +236,7 @@ type Config struct { WorkloadHint WorkloadHint // ReadTimeout is the max duration for reading the entire request. - // Zero uses the default (60s). Set to -1 for no timeout. The native engines - // enforce it in a sweep of their connections, about 30 ms after its time on - // epoll (see IdleTimeout). + // Zero uses the default (60s). Set to -1 for no timeout. // // On the native engines (epoll, io_uring, adaptive) it is measured from the // connection's last read (on io_uring also from the start and the completion @@ -276,16 +274,19 @@ type Config struct { // take, from a client that keeps sending frames, is not bounded by ReadTimeout. // // The native engines look at ReadTimeout, IdleTimeout and WriteTimeout in a - // sweep of their connections, so one is enforced up to a sweep period after - // its time: 25 ms on epoll when ReadHeaderTimeout is set (the default), and on - // io_uring about 0.8 s (0.83 s measured on an idle worker; about 1 s with - // ReadHeaderTimeout off). With ReadHeaderTimeout off epoll has no such timer and - // looks much more rarely (a 1 s IdleTimeout closed an idle connection at 3 to 6 s - // in measurement). A timeout of a few seconds or less is approximate by that - // much on io_uring. The HTTP/1 header deadline, and the budget of a connection - // whose protocol is not decided yet, have a timer of their own on io_uring and - // are not; the HTTP/2 header-block bound of ReadHeaderTimeout is checked by the - // sweep and is late by up to a sweep period. + // sweep of their connections, so ReadTimeout and IdleTimeout are enforced up to + // a sweep period after their time. On epoll the sweep runs every 25 ms, from a + // kernel timer each worker arms whatever ReadHeaderTimeout and the timeouts + // are, so within about 30 ms (a 1 s IdleTimeout closed an idle connection at + // 1.00 to 1.03 s on docker arm64 with 2 CPUs, with ReadHeaderTimeout off as well + // as set). On io_uring the sweep of an idle worker is about 0.8 s apart (0.83 s + // measured; about 1 s with ReadHeaderTimeout off), so a timeout of a few + // seconds or less is approximate by that much there. WriteTimeout is judged in + // windows that end at a sweep too: see WriteTimeout. The HTTP/1 header + // deadline, and the budget of a connection whose protocol is not decided yet, + // have a timer of their own on io_uring and are not late; the HTTP/2 + // header-block bound of ReadHeaderTimeout is checked by the sweep and is late + // by up to a sweep period. ReadTimeout time.Duration // ReadHeaderTimeout caps the read of just the request line + headers // (separately from the request body, which ReadTimeout covers). This @@ -340,7 +341,7 @@ type Config struct { // is bounded by the minimum rate described below); -1 sets no such bound. The close comes between about 3/4 // and 5/4 of WriteTimeout after the peer last took a byte (its progress // is sampled every quarter of WriteTimeout; a SEND completion restarts the - // clock at once), plus the engine's sweep interval (up to about 0.1 s on + // clock at once), plus the engine's sweep interval (25 ms on // epoll, up to 2 s on io_uring; a WriteTimeout shorter than that can // double). An h2c stream is not covered yet (celeris#884). // On the std engine net/http arms WriteTimeout as one deadline for the @@ -384,7 +385,7 @@ type Config struct { // an HTTP/2 PING does not hold a parked response. The times above hold // with the epoll sweep at its 25 ms period, which every worker has whatever // ReadHeaderTimeout and the other timeouts are; io_uring sweeps more coarsely, - // see IdleTimeout. + // see ReadTimeout. // // While the rule has a baseline for a parked response (from the first sweep // that finds it parked), it is the only bound on the connection: @@ -541,15 +542,7 @@ type Config struct { // request that expired at ReadTimeout, for the 2 x ReadTimeout its late DATA may // take. On epoll, for a response parked on the socket, see WriteTimeout. // - // The native engines look at IdleTimeout, ReadTimeout and WriteTimeout in a - // sweep of their connections, so one is enforced a little after its time, not - // at it. On epoll the sweep runs every 25 ms, from a kernel timer each worker - // arms whatever ReadHeaderTimeout and the timeouts are, so a timeout is - // enforced within about 30 ms of its time (a 1 s IdleTimeout closed an idle - // connection at 1.00 to 1.03 s on docker arm64 with 2 CPUs, with - // ReadHeaderTimeout off as well as set). On io_uring the sweep of an idle - // worker is about a second apart, so a timeout of a few seconds or less is - // approximate by that much there. + // How closely IdleTimeout is enforced is described under ReadTimeout. IdleTimeout time.Duration // ShutdownTimeout is the deadline of the graceful shutdown that cancelling // the context of StartWithContext or StartWithListenerAndContext starts diff --git a/internal/engine/epoll/fine_sweep_1194_linux_test.go b/internal/engine/epoll/fine_sweep_1194_linux_test.go index 05da6c08f..aeb6fcafe 100644 --- a/internal/engine/epoll/fine_sweep_1194_linux_test.go +++ b/internal/engine/epoll/fine_sweep_1194_linux_test.go @@ -11,9 +11,13 @@ import ( "net" "net/http" "strconv" + "strings" + "sync" "testing" "time" + "golang.org/x/sys/unix" + "github.com/goceleris/celeris/internal/protocol/h2/stream" "github.com/goceleris/celeris/internal/resource" ) @@ -143,8 +147,9 @@ func TestReadTimeoutIsOnTimeWithoutReadHeaderTimeout1194(t *testing.T) { // every 25 ms (4.0 to 4.4 s without), 2.06 to 2.17 s under -race, where the // baseline sweep comes later, and 1.24 s once, with the first window failed // (the peer's kernel acknowledged less than the floor). The window runs from -// one WriteTimeout to two and 300 ms more than the other tests allow, the -// stall rule's windows being measured from a sweep, not from the request. +// one WriteTimeout to two and 800 ms more than the other tests allow (3.0 s +// with the lateness; main closes at 4.2 s and later), the stall rule's windows +// being measured from a sweep, not from the request. func TestWriteTimeoutIsOnTimeWithoutReadHeaderTimeout1194(t *testing.T) { _, addr, closed := start1194(t, func(c *resource.Config) { c.WriteTimeout = time.Second }) c, err := net.Dial("tcp", addr) @@ -157,7 +162,7 @@ func TestWriteTimeoutIsOnTimeWithoutReadHeaderTimeout1194(t *testing.T) { if _, err := fmt.Fprint(c, "GET /big HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil { t.Fatal(err) } - closeWithin1194(t, "non-reading peer (WriteTimeout 1 s)", closed, t0, time.Second, 2*time.Second+300*time.Millisecond) + closeWithin1194(t, "non-reading peer (WriteTimeout 1 s)", closed, t0, time.Second, 2*time.Second+800*time.Millisecond) } // Every timeout rides the sweep, so the sweep has its kernel timer whatever @@ -183,3 +188,58 @@ func TestEveryLoopHasASweepTimer1194(t *testing.T) { }) } } + +// logBuf1194 is a log sink a test reads while the loops write to it. +type logBuf1194 struct { + mu sync.Mutex + sb strings.Builder +} + +func (b *logBuf1194) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.Write(p) +} + +func (b *logBuf1194) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.String() +} + +// A loop whose sweep timer cannot be set up says so, once per loop: without it +// the deadlines that ride the sweep are late (about 130 ms with the listener +// open, 16 s without one) and the documentation's 25 ms does not hold. +func TestSweepTimerFailureIsLogged1194(t *testing.T) { + orig := timerfdCreate + timerfdCreate = func(int, int) (int, error) { return -1, unix.EMFILE } + t.Cleanup(func() { timerfdCreate = orig }) // registered first: runs after the engine has stopped + logs := &logBuf1194{} + eng, addr, _ := start1194(t, func(c *resource.Config) { + c.Logger = slog.New(slog.NewTextHandler(logs, nil)) + }) + for i := 0; i < eng.NumWorkers(); i++ { + if l := eng.WorkerLoop(i).(*Loop); l.timerFD >= 0 { + t.Errorf("loop %d has a timerfd (%d) though its creation failed", i, l.timerFD) + } + } + if got, want := strings.Count(logs.String(), "the sweep timer could not be set up"), eng.NumWorkers(); got != want { + t.Errorf("%d log lines about the sweep timer, want %d (one per loop):\n%s", got, want, logs.String()) + } + if !strings.Contains(logs.String(), "EMFILE") && !strings.Contains(logs.String(), "too many open files") { + t.Errorf("the log line does not carry the error:\n%s", logs.String()) + } + // The loop still serves. + c, err := net.Dial("tcp", addr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + _ = c.SetDeadline(time.Now().Add(10 * time.Second)) + if _, err := fmt.Fprint(c, "GET / HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil { + t.Fatal(err) + } + if _, err := http.ReadResponse(bufio.NewReader(c), nil); err != nil { + t.Fatalf("a loop without a sweep timer does not serve: %v", err) + } +} diff --git a/internal/engine/epoll/loop.go b/internal/engine/epoll/loop.go index de9744fe3..33946090b 100644 --- a/internal/engine/epoll/loop.go +++ b/internal/engine/epoll/loop.go @@ -544,7 +544,7 @@ func (l *Loop) run(ctx context.Context) { // sweep every 8 minutes. The default configuration (ReadHeaderTimeout 10 s) // has always armed it, so this changes the loop only for a server that // turned ReadHeaderTimeout off; see sweepFloor for what it costs. - tfd, tfdErr := unix.TimerfdCreate(unix.CLOCK_MONOTONIC, unix.TFD_NONBLOCK|unix.TFD_CLOEXEC) + tfd, tfdErr := timerfdCreate(unix.CLOCK_MONOTONIC, unix.TFD_NONBLOCK|unix.TFD_CLOEXEC) if tfdErr == nil { // sweepFloor periodic (25 ms); first fire at the same. spec := unix.ItimerSpec{ @@ -553,16 +553,24 @@ func (l *Loop) run(ctx context.Context) { } if err := unix.TimerfdSettime(tfd, 0, &spec, nil); err != nil { _ = unix.Close(tfd) - tfd = -1 + tfd, tfdErr = -1, err } else if err := unix.EpollCtl(epollFD, unix.EPOLL_CTL_ADD, tfd, &unix.EpollEvent{ Events: unix.EPOLLIN, Fd: int32(tfd), }); err != nil { _ = unix.Close(tfd) - tfd = -1 + tfd, tfdErr = -1, err } l.timerFD = tfd } + if tfdErr != nil && l.logger != nil { + // Not silent (celeris#1194): without the timer the deadlines that ride + // the sweep are seen from the loop's gate alone, about 130 ms apart + // with the listener open and 16 s apart while it has none, where the + // documentation promises 25 ms. + l.logger.Warn("epoll: the sweep timer could not be set up; timeouts are enforced late on this loop", + "loop", l.id, "err", tfdErr) + } // celeris#639: listenFD is certainly this loop's socket here; after ready // a cancelled context closes it in shutdown. @@ -922,10 +930,9 @@ func (l *Loop) run(ctx context.Context) { l.bytesWrittenBatch = 0 } - // Check connection timeouts. Default cadence is every 1024 iterations; - // when detached conns exist with idle deadlines the gate tightens to - // every 32 iterations so the WS idle-close fires within its - // configured budget. An iteration is one epoll_wait return, so the + // Check connection timeouts. The gate is every 32 iterations (it was + // 1024 without ReadHeaderTimeout or a detached conn, celeris#1194). An + // iteration is one epoll_wait return, so the // wall time between gate hits depends on the load: on a busy loop it // is well under a millisecond, and the timerfd (25 ms) is only the // floor on an idle one. So a gate hit sweeps only when sweepFloor has @@ -4515,13 +4522,13 @@ func (l *Loop) adaptiveTimeoutMs(base int) int { if len(l.h2Conns) > 0 && l.wakeFD.FD() < 0 { return 1 } - // The wait never exceeds the sweep period, so that checkTimeouts fires the - // deadlines that ride it (the header deadline, idle, read and write - // timeouts, a detached conn's idle deadline) reliably under low traffic, - // whatever the timeouts are (celeris#1194; the cap was 25 ms with - // ReadHeaderTimeout set, 50 ms with a conn detached, 500 ms otherwise). The - // timerfd ends the wait on its own; the cap is what keeps the bound if the - // timerfd could not be created. + // A ceiling for the wait at the sweep period (celeris#1194; it was 25 ms + // with ReadHeaderTimeout set, 50 ms with a conn detached, 500 ms otherwise). + // With base 1 ms the backoff below tops out at 4 ms, so the ceiling does not + // bind today, and it is not what bounds the sweep: the timerfd is. A loop + // whose timerfd could not be created (it logs that once) sweeps from the gate + // alone, every 32 iterations, about 130 ms apart with its listener open, + // and about 16 s apart while it has no listener and waits 500 ms (see run). maxMs := int(sweepFloor / time.Millisecond) switch { case l.consecutiveEmpty == 0: @@ -4628,6 +4635,10 @@ func snapshotH1Deadlines(cs *connState) (snap h1DeadlineSnapshot, ok bool) { return snap, true } +// timerfdCreate is unix.TimerfdCreate; a test replaces it to fail the creation +// (celeris#1194). Set before an engine is created and restored after it stops. +var timerfdCreate = unix.TimerfdCreate + // sweepFloor is the least time between two checkTimeouts sweeps that the run // loop's iteration gate asks for, and the period of the timerfd that // guarantees one on an idle loop. A busy loop's iterations are well under a